From 80b0ca70c702aa811c7420589bf0ab4519654e70 Mon Sep 17 00:00:00 2001 From: Vladimir Sitnikov Date: Sun, 2 Nov 2025 19:03:08 +0300 Subject: [PATCH 001/562] chore: add gradle/actions/dependency-submission workflow --- .../workflows/gradle-dependency-submit.yaml | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 .github/workflows/gradle-dependency-submit.yaml diff --git a/.github/workflows/gradle-dependency-submit.yaml b/.github/workflows/gradle-dependency-submit.yaml new file mode 100644 index 000000000000..701872b7419a --- /dev/null +++ b/.github/workflows/gradle-dependency-submit.yaml @@ -0,0 +1,31 @@ +name: Dependency Submission + +# See https://github.com/gradle/actions/blob/768a17f3488dc3fe0155ff431553e1f53d57e22e/dependency-submission/README.md#the-dependency-submission-action +# The action allows GitHub to alert about reported vulnerabilities in the project +on: + push: + branches: + - main + +# Declare default permissions as read-only. +permissions: read-all + +jobs: + dependency-submission: + name: Submit dependencies + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Checkout sources + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5 + with: + persist-credentials: false + - name: Set up JDK 21 + uses: actions/setup-java@dded0888837ed1f317902acf8a20df0ad188d165 # v5 + with: + distribution: zulu + java-version: 21 + server-id: central + - name: Generate and submit dependency graph + uses: gradle/actions/dependency-submission@4d9f0ba0025fe599b4ebab900eb7f3a1d93ef4c2 # v5 From 219998efcc3cf9d37540ee07b26036bad65fd51a Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Sat, 1 Nov 2025 17:52:36 +0800 Subject: [PATCH 002/562] [CALCITE-7258] RelBuilder.filter should throw if the condition is not BOOLEAN --- .../java/org/apache/calcite/rel/core/Filter.java | 14 ++++++++++++++ .../org/apache/calcite/test/RelBuilderTest.java | 14 ++++++++++++++ .../java/org/apache/calcite/piglet/Handler.java | 6 +++++- .../org/apache/calcite/piglet/PigRelOpVisitor.java | 7 ++++++- .../java/org/apache/calcite/test/PigRelExTest.java | 11 ++++++----- .../java/org/apache/calcite/test/PigletTest.java | 2 +- 6 files changed, 46 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rel/core/Filter.java b/core/src/main/java/org/apache/calcite/rel/core/Filter.java index 14b4b2fcc27f..27aafd375285 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/Filter.java +++ b/core/src/main/java/org/apache/calcite/rel/core/Filter.java @@ -28,12 +28,14 @@ import org.apache.calcite.rel.hint.RelHint; import org.apache.calcite.rel.metadata.RelMdUtil; import org.apache.calcite.rel.metadata.RelMetadataQuery; +import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexChecker; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexOver; import org.apache.calcite.rex.RexProgram; import org.apache.calcite.rex.RexShuttle; import org.apache.calcite.rex.RexUtil; +import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.util.Litmus; import com.google.common.collect.ImmutableList; @@ -146,6 +148,18 @@ public final boolean containsOver() { if (RexUtil.isNullabilityCast(getCluster().getTypeFactory(), condition)) { return litmus.fail("Cast for just nullability not allowed"); } + + final RelDataType conditionType = condition.getType(); + if (!conditionType.isNullable() && conditionType.getSqlTypeName() != SqlTypeName.BOOLEAN) { + return litmus.fail("Filter condition must have type BOOLEAN, got " + conditionType); + } + if (conditionType.isNullable() + && conditionType.getSqlTypeName() != SqlTypeName.BOOLEAN + && conditionType.getSqlTypeName() != SqlTypeName.NULL) { + return litmus.fail("Filter condition must have type BOOLEAN or NULL, got " + + conditionType); + } + final RexChecker checker = new RexChecker(getInput().getRowType(), context, litmus); condition.accept(checker); diff --git a/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java b/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java index f1dc0baac96b..b54e08e26d4f 100644 --- a/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java @@ -5969,6 +5969,20 @@ private static RelNode buildCorrelateWithJoin(JoinRelType type, RelBuilder build } } + /** Test case for + * [CALCITE-7258] + * RelBuilder.filter should throw if the condition is not BOOLEAN. */ + @Test void testFilterWithNonBooleanLiteralCondition() { + final RelBuilder builder = RelBuilder.create(config().build()); + try { + builder.scan("EMP") + .filter(builder.literal("foo")) + .build(); + } catch (Error e) { + assertTrue(e.getMessage().contains("Filter condition must have type BOOLEAN")); + } + } + /** Operand to a user-defined function. */ private interface Arg { String name(); diff --git a/piglet/src/main/java/org/apache/calcite/piglet/Handler.java b/piglet/src/main/java/org/apache/calcite/piglet/Handler.java index 04646b5661ae..d8acf8432a79 100644 --- a/piglet/src/main/java/org/apache/calcite/piglet/Handler.java +++ b/piglet/src/main/java/org/apache/calcite/piglet/Handler.java @@ -107,7 +107,11 @@ public Handler handle(Ast.Node node) { builder.clear(); input = map.get(filter.source.value); builder.push(input); - final RexNode rexNode = toRex(filter.condition); + RexNode rexNode = toRex(filter.condition); + if (rexNode.getType().getSqlTypeName() != SqlTypeName.BOOLEAN) { + RelDataType boolType = builder.getTypeFactory().createSqlType(SqlTypeName.BOOLEAN); + rexNode = builder.getRexBuilder().makeCast(boolType, rexNode); + } builder.filter(rexNode); register(filter.target.value); return this; diff --git a/piglet/src/main/java/org/apache/calcite/piglet/PigRelOpVisitor.java b/piglet/src/main/java/org/apache/calcite/piglet/PigRelOpVisitor.java index 62bf4f0beaa5..e1c90e25cc2e 100644 --- a/piglet/src/main/java/org/apache/calcite/piglet/PigRelOpVisitor.java +++ b/piglet/src/main/java/org/apache/calcite/piglet/PigRelOpVisitor.java @@ -29,6 +29,7 @@ import org.apache.calcite.rex.RexWindowBounds; import org.apache.calcite.sql.SqlAggFunction; import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.tools.RelBuilder; import org.apache.calcite.util.ImmutableBitSet; import org.apache.calcite.util.Pair; @@ -169,7 +170,11 @@ List translate() throws FrontendException { } @Override public void visit(LOFilter filter) throws FrontendException { - final RexNode relExFilter = PigRelExVisitor.translatePigEx(builder, filter.getFilterPlan()); + RexNode relExFilter = PigRelExVisitor.translatePigEx(builder, filter.getFilterPlan()); + if (relExFilter.getType().getSqlTypeName() != SqlTypeName.BOOLEAN) { + RelDataType boolType = builder.getTypeFactory().createSqlType(SqlTypeName.BOOLEAN); + relExFilter = builder.getRexBuilder().makeCast(boolType, relExFilter); + } builder.filter(relExFilter); builder.register(filter); } diff --git a/piglet/src/test/java/org/apache/calcite/test/PigRelExTest.java b/piglet/src/test/java/org/apache/calcite/test/PigRelExTest.java index 5d56694c04cb..aff0759796e8 100644 --- a/piglet/src/test/java/org/apache/calcite/test/PigRelExTest.java +++ b/piglet/src/test/java/org/apache/calcite/test/PigRelExTest.java @@ -173,15 +173,16 @@ public void testMatch() { } @Test void testTupleDereference() { - checkTranslation("k2.k21", inTree("[$11.k21]")); - checkTranslation("k2.(k21, k22)", inTree("[ROW($11.k21, $11.k22)]")); + checkTranslation("k2.k21", inTree("[<>($11.k21, 0)]")); + checkTranslation("k2.(k21, k22)", inTree("[CAST(ROW($11.k21, $11.k22)):BOOLEAN NOT NULL]")); checkTranslation("k2.k22.(k221,k222)", - inTree("[ROW($11.k22.k221, $11.k22.k222)]")); + inTree("[CAST(ROW($11.k22.k221, $11.k22.k222)):BOOLEAN NOT NULL]")); } @Test void testBagDereference() { - checkTranslation("l2.l22", inTree("[MULTISET_PROJECTION($13, 1)]")); - checkTranslation("l2.(l21, l22)", inTree("[MULTISET_PROJECTION($13, 0, 1)]")); + checkTranslation("l2.l22", inTree("[CAST(MULTISET_PROJECTION($13, 1)):BOOLEAN NOT NULL]")); + checkTranslation("l2.(l21, l22)", + inTree("[CAST(MULTISET_PROJECTION($13, 0, 1)):BOOLEAN NOT NULL]")); } @Test void testMapLookup() { diff --git a/piglet/src/test/java/org/apache/calcite/test/PigletTest.java b/piglet/src/test/java/org/apache/calcite/test/PigletTest.java index 5586b7fb27ee..3eddf31c8a66 100644 --- a/piglet/src/test/java/org/apache/calcite/test/PigletTest.java +++ b/piglet/src/test/java/org/apache/calcite/test/PigletTest.java @@ -172,7 +172,7 @@ private static Fluent pig(String pig) { @Test void testFilter() throws ParseException { final String s = "A = LOAD 'DEPT';\n" + "B = FILTER A BY DEPTNO;"; - final String expected = "LogicalFilter(condition=[$0])\n" + final String expected = "LogicalFilter(condition=[<>($0, 0)])\n" + " LogicalTableScan(table=[[scott, DEPT]])\n"; pig(s).explainContains(expected); } From da9af5d508b4283601c7e575895df466fcc80b60 Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Mon, 3 Nov 2025 22:55:28 +0800 Subject: [PATCH 003/562] [CALCITE-5223] AdjustProjectForCountAggregateRule throws ArrayIndexOutOfBoundsException --- core/src/test/resources/sql/sub-query.iq | 39 ++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/core/src/test/resources/sql/sub-query.iq b/core/src/test/resources/sql/sub-query.iq index 182f3f4e3108..4a9a30fa5a9c 100644 --- a/core/src/test/resources/sql/sub-query.iq +++ b/core/src/test/resources/sql/sub-query.iq @@ -5388,4 +5388,43 @@ and exists( !ok + +!use scott +# [CALCITE-5223] AdjustProjectForCountAggregateRule throws ArrayIndexOutOfBoundsException +SELECT deptno, ename, job, sal, + sal / (SELECT SUM(sal) FROM Emp WHERE deptno = e.deptno) AS pct_dept +FROM Emp AS e +WHERE job = 'CLERK' +ORDER BY deptno; ++--------+--------+-------+---------+---------------------+ +| DEPTNO | ENAME | JOB | SAL | PCT_DEPT | ++--------+--------+-------+---------+---------------------+ +| 10 | MILLER | CLERK | 1300.00 | 0.1485714285714286 | +| 20 | SMITH | CLERK | 800.00 | 0.07356321839080460 | +| 20 | ADAMS | CLERK | 1100.00 | 0.1011494252873563 | +| 30 | JAMES | CLERK | 950.00 | 0.1010638297872340 | ++--------+--------+-------+---------+---------------------+ +(4 rows) + +!ok + +# [CALCITE-5223] AdjustProjectForCountAggregateRule throws ArrayIndexOutOfBoundsException +SELECT deptno, ename, job, sal, + sal / (SELECT SUM(sal) FROM Emp WHERE deptno = e.deptno) AS pct_dept, + sal / (SELECT SUM(sal) FROM Emp) AS pct_total +FROM Emp AS e +WHERE job = 'CLERK' +ORDER BY deptno; ++--------+--------+-------+---------+---------------------+---------------------+ +| DEPTNO | ENAME | JOB | SAL | PCT_DEPT | PCT_TOTAL | ++--------+--------+-------+---------+---------------------+---------------------+ +| 10 | MILLER | CLERK | 1300.00 | 0.1485714285714286 | 0.04478897502153316 | +| 20 | SMITH | CLERK | 800.00 | 0.07356321839080460 | 0.02756244616709733 | +| 20 | ADAMS | CLERK | 1100.00 | 0.1011494252873563 | 0.03789836347975883 | +| 30 | JAMES | CLERK | 950.00 | 0.1010638297872340 | 0.03273040482342808 | ++--------+--------+-------+---------+---------------------+---------------------+ +(4 rows) + +!ok + # End sub-query.iq From f0cca921891c2efe6e30184f7296e1c45b66cf69 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Mon, 27 Oct 2025 18:19:33 -0700 Subject: [PATCH 004/562] [CALCITE-7251] SEARCH and WINDOW operations should carry source position information Signed-off-by: Mihai Budiu --- .../org/apache/calcite/rel/core/Window.java | 26 ++++++- .../calcite/rel/externalize/RelJson.java | 16 +++- .../calcite/rel/logical/LogicalWindow.java | 1 + .../rel/rules/ProjectWindowTransposeRule.java | 2 +- .../rel/rules/ReduceExpressionsRule.java | 3 +- .../org/apache/calcite/rex/RexBuilder.java | 35 +++++++-- .../java/org/apache/calcite/rex/RexCall.java | 2 +- .../org/apache/calcite/rex/RexCopier.java | 2 +- .../java/org/apache/calcite/rex/RexOver.java | 34 +++++++- .../org/apache/calcite/rex/RexShuttle.java | 1 + .../org/apache/calcite/rex/RexSimplify.java | 43 ++++++---- .../java/org/apache/calcite/rex/RexUtil.java | 78 ++++++++++++++----- .../calcite/sql2rel/ConvertToChecked.java | 2 +- .../calcite/sql2rel/SqlToRelConverter.java | 3 +- .../org/apache/calcite/tools/RelBuilder.java | 49 ++++++++---- .../calcite/rel/externalize/RelJsonTest.java | 45 +++++++++++ 16 files changed, 276 insertions(+), 66 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rel/core/Window.java b/core/src/main/java/org/apache/calcite/rel/core/Window.java index fd4770db6ec4..2adf54c4f379 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/Window.java +++ b/core/src/main/java/org/apache/calcite/rel/core/Window.java @@ -41,6 +41,7 @@ import org.apache.calcite.rex.RexWindowBound; import org.apache.calcite.rex.RexWindowExclusion; import org.apache.calcite.sql.SqlAggFunction; +import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.util.ImmutableBitSet; import org.apache.calcite.util.ImmutableIntList; import org.apache.calcite.util.Litmus; @@ -425,7 +426,7 @@ public RexWinAggCall( List operands, int ordinal, boolean distinct) { - this(aggFun, type, operands, ordinal, distinct, false); + this(SqlParserPos.ZERO, aggFun, type, operands, ordinal, distinct, false); } /** @@ -436,6 +437,7 @@ public RexWinAggCall( * @param operands Operands to call * @param ordinal Ordinal within its partition * @param distinct Eliminate duplicates before applying aggregate function + * @deprecated Use {@link RexWinAggCall#RexWinAggCall(SqlParserPos, SqlAggFunction, RelDataType, List, int, boolean, boolean)} */ public RexWinAggCall( SqlAggFunction aggFun, @@ -444,7 +446,27 @@ public RexWinAggCall( int ordinal, boolean distinct, boolean ignoreNulls) { - super(type, aggFun, operands); + this(SqlParserPos.ZERO, aggFun, type, operands, ordinal, distinct, ignoreNulls); + } + + /** + * Creates a RexWinAggCall. + * + * @param aggFun Aggregate function + * @param type Result type + * @param operands Operands to call + * @param ordinal Ordinal within its partition + * @param distinct Eliminate duplicates before applying aggregate function + */ + public RexWinAggCall( + SqlParserPos pos, + SqlAggFunction aggFun, + RelDataType type, + List operands, + int ordinal, + boolean distinct, + boolean ignoreNulls) { + super(pos, type, aggFun, operands); this.ordinal = ordinal; this.distinct = distinct; this.ignoreNulls = ignoreNulls; diff --git a/core/src/main/java/org/apache/calcite/rel/externalize/RelJson.java b/core/src/main/java/org/apache/calcite/rel/externalize/RelJson.java index 17a34411eb8e..2409ac6090e2 100644 --- a/core/src/main/java/org/apache/calcite/rel/externalize/RelJson.java +++ b/core/src/main/java/org/apache/calcite/rel/externalize/RelJson.java @@ -298,6 +298,15 @@ private static RexNode translateInput(RelJson relJson, int input, throw new RuntimeException("input field " + input + " is out of range"); } + public Object toJson(SqlParserPos pos) { + final Map map = jsonBuilder().map(); + map.put("line", pos.getLineNum()); + map.put("column", pos.getColumnNum()); + map.put("end_line", pos.getEndLineNum()); + map.put("end_column", pos.getEndColumnNum()); + return map; + } + public Object toJson(RelCollationImpl node) { final List list = new ArrayList<>(); for (RelFieldCollation fieldCollation : node.getFieldCollations()) { @@ -455,6 +464,8 @@ public Object toJson(AggregateCall node) { || value instanceof String || value instanceof Boolean) { return value; + } else if (value instanceof SqlParserPos) { + return toJson((SqlParserPos) value); } else if (value instanceof RexNode) { return toJson((RexNode) value); } else if (value instanceof RexWindow) { @@ -641,6 +652,9 @@ public Object toJson(RexNode node) { if (node instanceof RexCall) { final RexCall call = (RexCall) node; map = jsonBuilder().map(); + if (call.getParserPosition() != SqlParserPos.ZERO) { + map.put("pos", toJson(call.getParserPosition())); + } map.put("op", toJson(call.getOperator())); final List<@Nullable Object> list = jsonBuilder().list(); for (RexNode operand : call.getOperands()) { @@ -797,7 +811,7 @@ public RexNode toRex(RelOptCluster cluster, Object o) { exclude = RexWindowExclusion.EXCLUDE_NO_OTHER; } final boolean distinct = get((Map) map, "distinct"); - return rexBuilder.makeOver(type, operator, rexOperands, partitionKeys, + return rexBuilder.makeOver(SqlParserPos.ZERO, type, operator, rexOperands, partitionKeys, ImmutableList.copyOf(orderKeys), requireNonNull(lowerBound, "lowerBound"), requireNonNull(upperBound, "upperBound"), diff --git a/core/src/main/java/org/apache/calcite/rel/logical/LogicalWindow.java b/core/src/main/java/org/apache/calcite/rel/logical/LogicalWindow.java index 1dcff97b1833..d17b8aef859e 100644 --- a/core/src/main/java/org/apache/calcite/rel/logical/LogicalWindow.java +++ b/core/src/main/java/org/apache/calcite/rel/logical/LogicalWindow.java @@ -182,6 +182,7 @@ public static RelNode create(RelOptCluster cluster, for (RexOver over : entry.getValue()) { final RexWinAggCall aggCall = new RexWinAggCall( + over.getParserPosition(), over.getAggOperator(), over.getType(), toInputRefs(over.operands), diff --git a/core/src/main/java/org/apache/calcite/rel/rules/ProjectWindowTransposeRule.java b/core/src/main/java/org/apache/calcite/rel/rules/ProjectWindowTransposeRule.java index b2c366f14d55..fac0f459accb 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/ProjectWindowTransposeRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/ProjectWindowTransposeRule.java @@ -121,7 +121,7 @@ public ProjectWindowTransposeRule(RelBuilderFactory relBuilderFactory) { boolean[] update = {false}; final List clonedOperands = visitList(call.operands, update); if (update[0]) { - return new Window.RexWinAggCall( + return new Window.RexWinAggCall(call.getParserPosition(), (SqlAggFunction) call.getOperator(), call.getType(), clonedOperands, aggCall.ordinal, aggCall.distinct, aggCall.ignoreNulls); diff --git a/core/src/main/java/org/apache/calcite/rel/rules/ReduceExpressionsRule.java b/core/src/main/java/org/apache/calcite/rel/rules/ReduceExpressionsRule.java index 605f212a7659..8a2e5b09b41d 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/ReduceExpressionsRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/ReduceExpressionsRule.java @@ -579,7 +579,8 @@ public WindowReduceExpressionsRule(Class windowClass, final List expList = new ArrayList<>(aggCall.getOperands()); if (reduceExpressions(window, expList, predicates)) { aggCall = - new Window.RexWinAggCall((SqlAggFunction) aggCall.getOperator(), + new Window.RexWinAggCall(aggCall.getParserPosition(), + (SqlAggFunction) aggCall.getOperator(), aggCall.type, expList, aggCall.ordinal, aggCall.distinct, aggCall.ignoreNulls); reduced = true; diff --git a/core/src/main/java/org/apache/calcite/rex/RexBuilder.java b/core/src/main/java/org/apache/calcite/rex/RexBuilder.java index d5492f06347f..b77fe0283095 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexBuilder.java +++ b/core/src/main/java/org/apache/calcite/rex/RexBuilder.java @@ -459,8 +459,30 @@ public RexNode makeOver( boolean nullWhenCountZero, boolean distinct, boolean ignoreNulls) { - return makeOver(type, operator, exprs, partitionKeys, orderKeys, lowerBound, upperBound, - RexWindowExclusion.EXCLUDE_NO_OTHER, rows, allowPartial, nullWhenCountZero, distinct, + return makeOver(SqlParserPos.ZERO, type, operator, exprs, partitionKeys, orderKeys, lowerBound, + upperBound, RexWindowExclusion.EXCLUDE_NO_OTHER, rows, allowPartial, nullWhenCountZero, + distinct, ignoreNulls); + } + + /** + * Creates a call to a windowed agg. + */ + public RexNode makeOver( + RelDataType type, + SqlAggFunction operator, + List exprs, + List partitionKeys, + ImmutableList orderKeys, + RexWindowBound lowerBound, + RexWindowBound upperBound, + RexWindowExclusion exclude, + boolean rows, + boolean allowPartial, + boolean nullWhenCountZero, + boolean distinct, + boolean ignoreNulls) { + return makeOver(SqlParserPos.ZERO, type, operator, exprs, partitionKeys, orderKeys, + lowerBound, upperBound, exclude, rows, allowPartial, nullWhenCountZero, distinct, ignoreNulls); } @@ -468,6 +490,7 @@ public RexNode makeOver( * Creates a call to a windowed agg. */ public RexNode makeOver( + SqlParserPos pos, RelDataType type, SqlAggFunction operator, List exprs, @@ -490,7 +513,7 @@ public RexNode makeOver( rows, exclude); RexNode result = - new RexOver(type, operator, exprs, window, distinct, ignoreNulls); + new RexOver(pos, type, operator, exprs, window, distinct, ignoreNulls); // This should be correct but need time to go over test results. // Also want to look at combing with section below. @@ -500,12 +523,12 @@ public RexNode makeOver( result = makeCall(SqlStdOperatorTable.CASE, makeCall(SqlStdOperatorTable.GREATER_THAN, - new RexOver(bigintType, SqlStdOperatorTable.COUNT, exprs, + new RexOver(pos, bigintType, SqlStdOperatorTable.COUNT, exprs, window, distinct, ignoreNulls), makeLiteral(BigDecimal.ZERO, bigintType, SqlTypeName.DECIMAL)), ensureType(type, // SUM0 is non-nullable, thus need a cast - new RexOver(typeFactory.createTypeWithNullability(type, false), + new RexOver(pos, typeFactory.createTypeWithNullability(type, false), operator, exprs, window, distinct, ignoreNulls), false), makeNullLiteral(type)); @@ -520,7 +543,7 @@ public RexNode makeOver( SqlStdOperatorTable.CASE, makeCall( SqlStdOperatorTable.GREATER_THAN_OR_EQUAL, - new RexOver( + new RexOver(pos, bigintType, SqlStdOperatorTable.COUNT, ImmutableList.of(), diff --git a/core/src/main/java/org/apache/calcite/rex/RexCall.java b/core/src/main/java/org/apache/calcite/rex/RexCall.java index ecd3cccb2c78..5be1fead88e7 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexCall.java +++ b/core/src/main/java/org/apache/calcite/rex/RexCall.java @@ -61,7 +61,7 @@ public class RexCall extends RexNode { * the source position, so the backend can produce runtime error messages * pointing to the original source position. * For calls that are can never generate runtime failures, this field may - * be ZERO. Note that some optimizations may "lost" position information. */ + * be ZERO. Note that some optimizations may "lose" position information. */ public final SqlParserPos pos; public final SqlOperator op; public final ImmutableList operands; diff --git a/core/src/main/java/org/apache/calcite/rex/RexCopier.java b/core/src/main/java/org/apache/calcite/rex/RexCopier.java index 1f5700e48208..499d81b7b96e 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexCopier.java +++ b/core/src/main/java/org/apache/calcite/rex/RexCopier.java @@ -50,7 +50,7 @@ private RelDataType copy(RelDataType type) { @Override public RexNode visitOver(RexOver over) { final boolean[] update = null; - return new RexOver(copy(over.getType()), over.getAggOperator(), + return new RexOver(over.getParserPosition(), copy(over.getType()), over.getAggOperator(), visitList(over.getOperands(), update), visitWindow(over.getWindow()), over.isDistinct(), over.ignoreNulls()); } diff --git a/core/src/main/java/org/apache/calcite/rex/RexOver.java b/core/src/main/java/org/apache/calcite/rex/RexOver.java index 63a263e6b703..0b01838b385a 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexOver.java +++ b/core/src/main/java/org/apache/calcite/rex/RexOver.java @@ -19,6 +19,7 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.sql.SqlAggFunction; import org.apache.calcite.sql.SqlWindow; +import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.util.ControlFlowException; import org.apache.calcite.util.Util; @@ -58,6 +59,7 @@ public class RexOver extends RexCall { *
  • window = {@link SqlWindow}(ROWS 3 PRECEDING) * * + * @param pos Parser position * @param type Result type * @param op Aggregate operator * @param operands Operands list @@ -65,19 +67,49 @@ public class RexOver extends RexCall { * @param distinct Aggregate operator is applied on distinct elements */ RexOver( + SqlParserPos pos, RelDataType type, SqlAggFunction op, List operands, RexWindow window, boolean distinct, boolean ignoreNulls) { - super(type, op, operands); + super(pos, type, op, operands); checkArgument(op.isAggregator()); this.window = requireNonNull(window, "window"); this.distinct = distinct; this.ignoreNulls = ignoreNulls; } + /** + * Creates a RexOver. + * + *

    For example, "SUM(DISTINCT x) OVER (ROWS 3 PRECEDING)" is represented + * as: + * + *

      + *
    • type = Integer, + *
    • op = {@link org.apache.calcite.sql.fun.SqlStdOperatorTable#SUM}, + *
    • operands = { {@link RexFieldAccess}("x") } + *
    • window = {@link SqlWindow}(ROWS 3 PRECEDING) + *
    + * + * @param type Result type + * @param op Aggregate operator + * @param operands Operands list + * @param window Window specification + * @param distinct Aggregate operator is applied on distinct elements + */ + RexOver( + RelDataType type, + SqlAggFunction op, + List operands, + RexWindow window, + boolean distinct, + boolean ignoreNulls) { + this(SqlParserPos.ZERO, type, op, operands, window, distinct, ignoreNulls); + } + //~ Methods ---------------------------------------------------------------- /** diff --git a/core/src/main/java/org/apache/calcite/rex/RexShuttle.java b/core/src/main/java/org/apache/calcite/rex/RexShuttle.java index 2f6b4ec576f3..a3cf2b9d5e5f 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexShuttle.java +++ b/core/src/main/java/org/apache/calcite/rex/RexShuttle.java @@ -50,6 +50,7 @@ public class RexShuttle implements RexVisitor { // watch out for special operators like CAST and NEW where // the type is embedded in the original call. return new RexOver( + over.getParserPosition(), over.getType(), overAggregator, clonedOperands, diff --git a/core/src/main/java/org/apache/calcite/rex/RexSimplify.java b/core/src/main/java/org/apache/calcite/rex/RexSimplify.java index 9bacc060a1c2..4a9677c27de9 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexSimplify.java +++ b/core/src/main/java/org/apache/calcite/rex/RexSimplify.java @@ -28,6 +28,7 @@ import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.sql.type.SqlTypeCoercionRule; import org.apache.calcite.sql.type.SqlTypeFamily; import org.apache.calcite.sql.type.SqlTypeName; @@ -2380,7 +2381,7 @@ private RexNode simplifySearch(RexCall call, RexUnknownAs unknownAs) { RexLiteral literal = (RexLiteral) call.getOperands().get(1); final Sarg sarg = castNonNull(literal.getValueAs(Sarg.class)); if (sarg.isAll() || sarg.isNone()) { - RexNode rexNode = RexUtil.simpleSarg(rexBuilder, searchOperand, sarg, unknownAs); + RexNode rexNode = RexUtil.simpleSarg(call.pos, rexBuilder, searchOperand, sarg, unknownAs); return simplify(rexNode, unknownAs); } // Remove null from sarg if the left-hand side is never null @@ -2689,7 +2690,7 @@ private RexNode flattenAggregate(RexNode e) { rexBuilder.makeWindow(ImmutableList.of(), ImmutableList.of(), RexWindowBounds.CURRENT_ROW, RexWindowBounds.CURRENT_ROW, true); - return new RexOver(call.type, (SqlAggFunction) call.op, call.operands, + return new RexOver(call.pos, call.type, (SqlAggFunction) call.op, call.operands, w, false, false); } return super.visitCall(call); @@ -3208,12 +3209,12 @@ private boolean accept_(RexNode e, List newTerms) { case SEARCH: case IS_NOT_DISTINCT_FROM: case IS_DISTINCT_FROM: - return accept2(((RexCall) e).operands.get(0), + return accept2(((RexCall) e).getParserPosition(), ((RexCall) e).operands.get(0), ((RexCall) e).operands.get(1), e.getKind(), newTerms); case IS_NULL: case IS_NOT_NULL: final RexNode arg = ((RexCall) e).operands.get(0); - return accept1(arg, e.getKind(), newTerms); + return accept1(((RexCall) e).getParserPosition(), arg, e.getKind(), newTerms); default: return false; } @@ -3231,13 +3232,13 @@ private boolean accept_(RexNode e, List newTerms) { * @param newTerms the list to which the Sarg will be added if accepted * @return true if the operands can be converted to a Sarg, false otherwise */ - private boolean accept2(RexNode left, RexNode right, SqlKind kind, + private boolean accept2(SqlParserPos pos, RexNode left, RexNode right, SqlKind kind, List newTerms) { if (right.isA(SqlKind.LITERAL) && RexUtil.isDeterministic(left)) { - return accept2b(left, kind, (RexLiteral) right, newTerms); + return accept2b(pos, left, kind, (RexLiteral) right, newTerms); } if (left.isA(SqlKind.LITERAL) && RexUtil.isDeterministic(right)) { - return accept2b(right, kind.reverse(), (RexLiteral) left, newTerms); + return accept2b(pos, right, kind.reverse(), (RexLiteral) left, newTerms); } return false; } @@ -3255,10 +3256,10 @@ private static E addFluent(List list, E e) { * @param newTerms the list to which the Sarg is added * @return true since the operand is always converted to a Sarg */ - private boolean accept1(RexNode e, SqlKind kind, List newTerms) { + private boolean accept1(SqlParserPos pos, RexNode e, SqlKind kind, List newTerms) { final RexSargBuilder b = map.computeIfAbsent(e, e2 -> - addFluent(newTerms, new RexSargBuilder(e2, rexBuilder, negate))); + addFluent(newTerms, new RexSargBuilder(pos, e2, rexBuilder, negate))); switch (negate ? kind.negate() : kind) { case IS_NULL: b.nullAs = b.nullAs.or(TRUE); @@ -3283,7 +3284,7 @@ private boolean accept1(RexNode e, SqlKind kind, List newTerms) { * @param newTerms the list to which the Sarg is added if accepted * @return false if the literal operand is null, true otherwise */ - private boolean accept2b(RexNode e, SqlKind kind, + private boolean accept2b(SqlParserPos pos, RexNode e, SqlKind kind, RexLiteral literal, List newTerms) { if (literal.getValue() == null) { // Cannot include expressions 'x > NULL' in a Sarg. Comparing to a NULL @@ -3293,7 +3294,8 @@ private boolean accept2b(RexNode e, SqlKind kind, } final RexSargBuilder b = map.computeIfAbsent(e, e2 -> - addFluent(newTerms, new RexSargBuilder(e2, rexBuilder, negate))); + addFluent(newTerms, new RexSargBuilder(pos, e2, rexBuilder, negate))); + b.addPosition(pos); if (negate) { kind = kind.negateNullSafe(); } @@ -3374,10 +3376,10 @@ static RexNode fix(RexBuilder rexBuilder, RexNode term, if (isSmall && simpleSarg(sarg)) { // Expand small sargs into comparisons in order to avoid plan changes // and better readability. - return RexUtil.sargRef(rexBuilder, sargBuilder.ref, sarg, + return RexUtil.sargRef(sargBuilder.pos, rexBuilder, sargBuilder.ref, sarg, term.getType(), unknownAs); } - return rexBuilder.makeCall(SqlStdOperatorTable.SEARCH, sargBuilder.ref, + return rexBuilder.makeCall(sargBuilder.pos, SqlStdOperatorTable.SEARCH, sargBuilder.ref, rexBuilder.makeSearchArgumentLiteral(sarg, term.getType())); } return term; @@ -3406,6 +3408,10 @@ static RexNode fix(RexBuilder rexBuilder, RexNode term, * {@code UNKNOWN OR FALSE OR UNKNOWN} returns {@code UNKNOWN}; * {@code FALSE OR FALSE} returns {@code FALSE}. */ private static class RexSargBuilder extends RexNode { + // The position is MUTABLE: it contains the SUM of the positions of + // all expressions that compose the search. This is not ideal, but it's better + // than having no source position information at all. + SqlParserPos pos; final RexNode ref; final RexBuilder rexBuilder; final boolean negate; @@ -3415,7 +3421,8 @@ private static class RexSargBuilder extends RexNode { boolean mergedSarg; RexUnknownAs nullAs = FALSE; - RexSargBuilder(RexNode ref, RexBuilder rexBuilder, boolean negate) { + RexSargBuilder(SqlParserPos pos, RexNode ref, RexBuilder rexBuilder, boolean negate) { + this.pos = pos; this.ref = requireNonNull(ref, "ref"); this.rexBuilder = requireNonNull(rexBuilder, "rexBuilder"); this.negate = negate; @@ -3463,6 +3470,10 @@ > Sarg build(boolean negate) { throw new UnsupportedOperationException(); } + public SqlParserPos getPos() { + return pos; + } + @Override public int hashCode() { throw new UnsupportedOperationException(); } @@ -3471,6 +3482,10 @@ void addAll() { rangeSet.add(Range.all()); } + void addPosition(SqlParserPos pos) { + this.pos = SqlParserPos.sum(ImmutableList.of(this.pos, pos)); + } + void addRange(Range range, RelDataType type) { addRange(range, type, UNKNOWN); } diff --git a/core/src/main/java/org/apache/calcite/rex/RexUtil.java b/core/src/main/java/org/apache/calcite/rex/RexUtil.java index 2894b2fc7c71..5904bba9229f 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexUtil.java +++ b/core/src/main/java/org/apache/calcite/rex/RexUtil.java @@ -39,6 +39,7 @@ import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.sql.type.SqlTypeFamily; import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.sql.type.SqlTypeUtil; @@ -624,51 +625,70 @@ public static RexShuttle searchShuttle(RexBuilder rexBuilder, return new SearchExpandingShuttle(program, rexBuilder, maxComplexity); } - public static > RexNode sargRef(RexBuilder rexBuilder, + public static > RexNode sargRef(SqlParserPos pos, RexBuilder rexBuilder, RexNode ref, Sarg sarg, RelDataType type, RexUnknownAs unknownAs) { if (sarg.isAll() || sarg.isNone()) { - return simpleSarg(rexBuilder, ref, sarg, unknownAs); + return simpleSarg(pos, rexBuilder, ref, sarg, unknownAs); } final List orList = new ArrayList<>(); if (sarg.nullAs == RexUnknownAs.TRUE && unknownAs == RexUnknownAs.UNKNOWN) { - orList.add(rexBuilder.makeCall(SqlStdOperatorTable.IS_NULL, ref)); + orList.add(rexBuilder.makeCall(pos, SqlStdOperatorTable.IS_NULL, ref)); } if (sarg.isPoints()) { // Generate 'ref = value1 OR ... OR ref = valueN' sarg.rangeSet.asRanges().forEach(range -> orList.add( - rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, ref, + rexBuilder.makeCall(pos, SqlStdOperatorTable.EQUALS, ref, rexBuilder.makeLiteral(range.lowerEndpoint(), type, true, true)))); } else if (sarg.isComplementedPoints()) { // Generate 'ref <> value1 AND ... AND ref <> valueN' final List list = sarg.rangeSet.complement().asRanges().stream() .map(range -> - rexBuilder.makeCall(SqlStdOperatorTable.NOT_EQUALS, ref, + rexBuilder.makeCall(pos, SqlStdOperatorTable.NOT_EQUALS, ref, rexBuilder.makeLiteral(range.lowerEndpoint(), type, true, true))) .collect(toImmutableList()); orList.add(composeConjunction(rexBuilder, list)); } else { final RangeSets.Consumer consumer = - new RangeToRex<>(ref, orList, rexBuilder, type); + new RangeToRex<>(pos, ref, orList, rexBuilder, type); RangeSets.forEach(sarg.rangeSet, consumer); } RexNode node = composeDisjunction(rexBuilder, orList); if (sarg.nullAs == RexUnknownAs.FALSE && unknownAs == RexUnknownAs.UNKNOWN) { node = - rexBuilder.makeCall(SqlStdOperatorTable.AND, - rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL, ref), + rexBuilder.makeCall(pos, SqlStdOperatorTable.AND, + rexBuilder.makeCall(pos, SqlStdOperatorTable.IS_NOT_NULL, ref), node); } return node; } - /** Expands an 'all' or 'none' sarg. */ + /** + * Create a sargRef object. + * + * @deprecated Use + * {@link RexUtil#sargRef(SqlParserPos, RexBuilder, RexNode, Sarg, RelDataType, RexUnknownAs)}. */ + public static > RexNode sargRef(RexBuilder rexBuilder, + RexNode ref, Sarg sarg, RelDataType type, RexUnknownAs unknownAs) { + return sargRef(SqlParserPos.ZERO, rexBuilder, ref, sarg, type, unknownAs); + } + + /** Expands an 'all' or 'none' sarg. + * + * @deprecated Use + * {@link RexUtil#simpleSarg(SqlParserPos, RexBuilder, RexNode, Sarg, RexUnknownAs)} */ public static > RexNode simpleSarg(RexBuilder rexBuilder, RexNode ref, Sarg sarg, RexUnknownAs unknownAs) { + return simpleSarg(SqlParserPos.ZERO, rexBuilder, ref, sarg, unknownAs); + } + + /** Expands an 'all' or 'none' sarg. */ + public static > RexNode simpleSarg(SqlParserPos pos, + RexBuilder rexBuilder, RexNode ref, Sarg sarg, RexUnknownAs unknownAs) { assert sarg.isAll() || sarg.isNone(); final RexUnknownAs nullAs = sarg.nullAs == RexUnknownAs.UNKNOWN ? unknownAs @@ -678,11 +698,11 @@ public static > RexNode simpleSarg(RexBuilder rexBuilder case TRUE: return rexBuilder.makeLiteral(true); case FALSE: - return rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL, ref); + return rexBuilder.makeCall(pos, SqlStdOperatorTable.IS_NOT_NULL, ref); case UNKNOWN: // "x IS NOT NULL OR UNKNOWN" - return rexBuilder.makeCall(SqlStdOperatorTable.OR, - rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL, ref), + return rexBuilder.makeCall(pos, SqlStdOperatorTable.OR, + rexBuilder.makeCall(pos, SqlStdOperatorTable.IS_NOT_NULL, ref), rexBuilder.makeNullLiteral( rexBuilder.typeFactory.createSqlType(SqlTypeName.BOOLEAN))); } @@ -690,12 +710,12 @@ public static > RexNode simpleSarg(RexBuilder rexBuilder if (sarg.isNone()) { switch (nullAs) { case TRUE: - return rexBuilder.makeCall(SqlStdOperatorTable.IS_NULL, ref); + return rexBuilder.makeCall(pos, SqlStdOperatorTable.IS_NULL, ref); case FALSE: return rexBuilder.makeLiteral(false); case UNKNOWN: // "CASE WHEN x IS NULL THEN UNKNOWN ELSE FALSE END", or "x <> x" - return rexBuilder.makeCall(SqlStdOperatorTable.NOT_EQUALS, ref, ref); + return rexBuilder.makeCall(pos, SqlStdOperatorTable.NOT_EQUALS, ref, ref); } } throw new AssertionError(); @@ -1288,6 +1308,20 @@ public static RexNode composeConjunction(RexBuilder rexBuilder, return requireNonNull(e, "e"); } + /** Summarize the position of all the nodes as the sum of all the positions. */ + static SqlParserPos summarizePosition(Iterable nodes) { + List validPositions = new ArrayList<>(); + for (RexNode node : nodes) { + if (node instanceof RexCall) { + SqlParserPos position = ((RexCall) node).getParserPosition(); + if (!position.equals(SqlParserPos.ZERO)) { + validPositions.add(position); + } + } + } + return SqlParserPos.sum(validPositions); + } + /** * Converts a collection of expressions into an AND. * If there are zero expressions, returns TRUE. @@ -1310,7 +1344,8 @@ public static RexNode composeConjunction(RexBuilder rexBuilder, if (containsFalse(list)) { return rexBuilder.makeLiteral(false); } - return rexBuilder.makeCall(SqlStdOperatorTable.AND, list); + final SqlParserPos pos = summarizePosition(nodes); + return rexBuilder.makeCall(pos, SqlStdOperatorTable.AND, list); } } @@ -1380,7 +1415,8 @@ public static RexNode composeDisjunction(RexBuilder rexBuilder, if (containsTrue(list)) { return rexBuilder.makeLiteral(true); } - return rexBuilder.makeCall(SqlStdOperatorTable.OR, list); + final SqlParserPos pos = summarizePosition(nodes); + return rexBuilder.makeCall(pos, SqlStdOperatorTable.OR, list); } } @@ -3376,13 +3412,15 @@ public boolean anyContain(Iterable nodes) { * @param Value type */ private static class RangeToRex> implements RangeSets.Consumer { + private final SqlParserPos pos; private final List list; private final RexBuilder rexBuilder; private final RelDataType type; private final RexNode ref; - RangeToRex(RexNode ref, List list, RexBuilder rexBuilder, + RangeToRex(SqlParserPos pos, RexNode ref, List list, RexBuilder rexBuilder, RelDataType type) { + this.pos = requireNonNull(pos, "pos"); this.ref = requireNonNull(ref, "ref"); this.list = requireNonNull(list, "list"); this.rexBuilder = requireNonNull(rexBuilder, "rexBuilder"); @@ -3390,11 +3428,11 @@ private static class RangeToRex> } private void addAnd(RexNode... nodes) { - list.add(rexBuilder.makeCall(SqlStdOperatorTable.AND, nodes)); + list.add(rexBuilder.makeCall(pos, SqlStdOperatorTable.AND, nodes)); } private RexNode op(SqlOperator op, C value) { - return rexBuilder.makeCall(op, ref, + return rexBuilder.makeCall(pos, op, ref, rexBuilder.makeLiteral(value, type, true, true)); } @@ -3485,7 +3523,7 @@ private static class SearchExpandingShuttle extends RexShuttle { (RexLiteral) deref(program, call.operands.get(1)); final Sarg sarg = requireNonNull(literal.getValueAs(Sarg.class), "Sarg"); if (maxComplexity < 0 || sarg.complexity() < maxComplexity) { - return sargRef(rexBuilder, ref, sarg, literal.getType(), + return sargRef(call.pos, rexBuilder, ref, sarg, literal.getType(), RexUnknownAs.UNKNOWN); } // Sarg is complex (therefore useful); fall through diff --git a/core/src/main/java/org/apache/calcite/sql2rel/ConvertToChecked.java b/core/src/main/java/org/apache/calcite/sql2rel/ConvertToChecked.java index f2a44602d109..a5e658e54a83 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/ConvertToChecked.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/ConvertToChecked.java @@ -101,7 +101,7 @@ class ConvertRexToChecked extends RexShuttle { } else { result = call; } - return builder.makeCast(call.getType(), result); + return builder.makeCast(call.getParserPosition(), call.getType(), result); } else if (!SqlTypeName.EXACT_TYPES.contains(resultType)) { // Do not rewrite operator if the type is e.g., DOUBLE or DATE operator = call.getOperator(); diff --git a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java index 66d0ed8c723d..7b1fc64bbbfc 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java @@ -2574,7 +2574,7 @@ private void convertUnnest(Blackboard bb, SqlCall call, @Nullable List f Ord.forEach(nodes, (node, i) -> { final RexNode e = bb.convertExpression(node); final String alias = SqlValidatorUtil.alias(node, i); - exprs.add(relBuilder.alias(e, alias)); + exprs.add(relBuilder.alias(node.getParserPosition(), e, alias)); }); RelNode child = (null != bb.root) ? bb.root : LogicalValues.createOneRow(cluster); @@ -5780,6 +5780,7 @@ && isConvertedSubq(rex)) { && kind == SqlKind.EXISTS) { fieldAccess = rexBuilder.makeCall( + expr.getParserPosition(), SqlStdOperatorTable.IS_NOT_NULL, fieldAccess); } diff --git a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java index 2aca3fc58ad9..44a799972720 100644 --- a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java +++ b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java @@ -741,6 +741,11 @@ public RexNode call(SqlOperator operator, RexNode... operands) { return call(operator, ImmutableList.copyOf(operands)); } + /** Creates a call to a scalar operator. */ + public RexNode call(SqlParserPos pos, SqlOperator operator, RexNode... operands) { + return call(pos, operator, ImmutableList.copyOf(operands)); + } + /** Creates a call to a scalar operator. */ private RexCall call(SqlParserPos pos, SqlOperator operator, List operandList) { switch (operator.getKind()) { @@ -1219,7 +1224,7 @@ public RexNode cast(SqlParserPos pos, RexNode expr, SqlTypeName typeName, int pr * * @see #project */ - public RexNode alias(RexNode expr, String alias) { + public RexNode alias(SqlParserPos pos, RexNode expr, String alias) { final RexNode aliasLiteral = literal(alias); switch (expr.getKind()) { case AS: @@ -1231,10 +1236,14 @@ public RexNode alias(RexNode expr, String alias) { expr = call.operands.get(0); // strip current (incorrect) alias, and fall through default: - return call(SqlStdOperatorTable.AS, expr, aliasLiteral); + return call(pos, SqlStdOperatorTable.AS, expr, aliasLiteral); } } + public RexNode alias(RexNode expr, String alias) { + return alias(SqlParserPos.ZERO, expr, alias); + } + private RexNode aliasMaybe(RexNode node, @Nullable String name) { return name == null ? node : alias(node, name); } @@ -4628,7 +4637,7 @@ private class AggCallImpl implements AggCallPlus { } @Override public OverCall over() { - return new OverCallImpl(aggFunction, distinct, operands, ignoreNulls, + return new OverCallImpl(pos, aggFunction, distinct, operands, ignoreNulls, alias); } @@ -4702,7 +4711,7 @@ private class AggCallImpl2 implements AggCallPlus { } @Override public OverCall over() { - return new OverCallImpl(aggregateCall.getAggregation(), + return new OverCallImpl(aggregateCall.getParserPosition(), aggregateCall.getAggregation(), aggregateCall.isDistinct(), operands, aggregateCall.ignoreNulls(), aggregateCall.name); } @@ -4803,6 +4812,8 @@ private class AggCallImpl2 implements AggCallPlus { * does the same but also assigns an column alias. */ public interface OverCall { + SqlParserPos getPosition(); + /** Performs an action on this OverCall. */ default R let(Function consumer) { return consumer.apply(this); @@ -4894,6 +4905,7 @@ default OverCall rangeTo(RexWindowBound upper) { /** Implementation of {@link OverCall}. */ private class OverCallImpl implements OverCall { + private final SqlParserPos pos; private final ImmutableList operands; private final boolean ignoreNulls; private final @Nullable String alias; @@ -4908,12 +4920,13 @@ private class OverCallImpl implements OverCall { private final SqlAggFunction op; private final boolean distinct; - private OverCallImpl(SqlAggFunction op, boolean distinct, + private OverCallImpl(SqlParserPos pos, SqlAggFunction op, boolean distinct, ImmutableList operands, boolean ignoreNulls, @Nullable String alias, ImmutableList partitionKeys, ImmutableList sortKeys, boolean rows, RexWindowBound lowerBound, RexWindowBound upperBound, boolean nullWhenCountZero, boolean allowPartial, RexWindowExclusion exclude) { + this.pos = pos; this.op = op; this.distinct = distinct; this.operands = operands; @@ -4930,14 +4943,18 @@ private OverCallImpl(SqlAggFunction op, boolean distinct, } /** Creates an OverCallImpl with default settings. */ - OverCallImpl(SqlAggFunction op, boolean distinct, + OverCallImpl(SqlParserPos pos, SqlAggFunction op, boolean distinct, ImmutableList operands, boolean ignoreNulls, @Nullable String alias) { - this(op, distinct, operands, ignoreNulls, alias, ImmutableList.of(), + this(pos, op, distinct, operands, ignoreNulls, alias, ImmutableList.of(), ImmutableList.of(), true, RexWindowBounds.UNBOUNDED_PRECEDING, RexWindowBounds.UNBOUNDED_FOLLOWING, false, true, RexWindowExclusion.EXCLUDE_NO_OTHER); } + @Override public SqlParserPos getPosition() { + return pos; + } + @Override public OverCall partitionBy( Iterable expressions) { return partitionBy_(ImmutableList.copyOf(expressions)); @@ -4948,13 +4965,13 @@ private OverCallImpl(SqlAggFunction op, boolean distinct, } private OverCall partitionBy_(ImmutableList partitionKeys) { - return new OverCallImpl(op, distinct, operands, ignoreNulls, alias, + return new OverCallImpl(pos, op, distinct, operands, ignoreNulls, alias, partitionKeys, sortKeys, rows, lowerBound, upperBound, nullWhenCountZero, allowPartial, exclude); } private OverCall orderBy_(ImmutableList sortKeys) { - return new OverCallImpl(op, distinct, operands, ignoreNulls, alias, + return new OverCallImpl(pos, op, distinct, operands, ignoreNulls, alias, partitionKeys, sortKeys, rows, lowerBound, upperBound, nullWhenCountZero, allowPartial, exclude); } @@ -4975,38 +4992,38 @@ private OverCall orderBy_(ImmutableList sortKeys) { @Override public OverCall rowsBetween(RexWindowBound lowerBound, RexWindowBound upperBound) { - return new OverCallImpl(op, distinct, operands, ignoreNulls, alias, + return new OverCallImpl(pos, op, distinct, operands, ignoreNulls, alias, partitionKeys, sortKeys, true, lowerBound, upperBound, nullWhenCountZero, allowPartial, exclude); } @Override public OverCall rangeBetween(RexWindowBound lowerBound, RexWindowBound upperBound) { - return new OverCallImpl(op, distinct, operands, ignoreNulls, alias, + return new OverCallImpl(pos, op, distinct, operands, ignoreNulls, alias, partitionKeys, sortKeys, false, lowerBound, upperBound, nullWhenCountZero, allowPartial, exclude); } @Override public OverCall exclude(RexWindowExclusion exclude) { - return new OverCallImpl(op, distinct, operands, ignoreNulls, alias, + return new OverCallImpl(pos, op, distinct, operands, ignoreNulls, alias, partitionKeys, sortKeys, rows, lowerBound, upperBound, nullWhenCountZero, allowPartial, exclude); } @Override public OverCall allowPartial(boolean allowPartial) { - return new OverCallImpl(op, distinct, operands, ignoreNulls, alias, + return new OverCallImpl(pos, op, distinct, operands, ignoreNulls, alias, partitionKeys, sortKeys, rows, lowerBound, upperBound, nullWhenCountZero, allowPartial, exclude); } @Override public OverCall nullWhenCountZero(boolean nullWhenCountZero) { - return new OverCallImpl(op, distinct, operands, ignoreNulls, alias, + return new OverCallImpl(pos, op, distinct, operands, ignoreNulls, alias, partitionKeys, sortKeys, rows, lowerBound, upperBound, nullWhenCountZero, allowPartial, exclude); } @Override public RexNode as(String alias) { - return new OverCallImpl(op, distinct, operands, ignoreNulls, alias, + return new OverCallImpl(pos, op, distinct, operands, ignoreNulls, alias, partitionKeys, sortKeys, rows, lowerBound, upperBound, nullWhenCountZero, allowPartial, exclude).toRex(); } @@ -5021,7 +5038,7 @@ private OverCall orderBy_(ImmutableList sortKeys) { }; final RelDataType type = op.inferReturnType(bind); final RexNode over = getRexBuilder() - .makeOver(type, op, operands, partitionKeys, sortKeys, + .makeOver(pos, type, op, operands, partitionKeys, sortKeys, lowerBound, upperBound, exclude, rows, allowPartial, nullWhenCountZero, distinct, ignoreNulls); return aliasMaybe(over, alias); diff --git a/core/src/test/java/org/apache/calcite/rel/externalize/RelJsonTest.java b/core/src/test/java/org/apache/calcite/rel/externalize/RelJsonTest.java index 9bc8040b9044..034365aede5c 100644 --- a/core/src/test/java/org/apache/calcite/rel/externalize/RelJsonTest.java +++ b/core/src/test/java/org/apache/calcite/rel/externalize/RelJsonTest.java @@ -104,6 +104,12 @@ plan, containsString("{\n" + " }\n" + " ],\n" + " \"expression\": {\n" + + " \"pos\": {\n" + + " \"line\": 1,\n" + + " \"column\": 32,\n" + + " \"end_line\": 1,\n" + + " \"end_column\": 36\n" + + " },\n" + " \"op\": {\n" + " \"name\": \">\",\n" + " \"kind\": \"GREATER_THAN\",\n" @@ -129,4 +135,43 @@ plan, containsString("{\n" + " }\n" + " }")); } + + /** Test case for [CALCITE-7251] + * SEARCH and WINDOW operations should carry source position information. */ + @Test void testSearchPosition() + throws SqlParseException, ValidationException, RelConversionException { + final String query = "SELECT val IN (1, 2, 3, 4)\n" + + "FROM (\n" + + " VALUES (10), (30), (20), (40)\n" + + ") AS t(val)"; + SqlOperatorTable opTab = SqlLibraryOperatorTableFactory.INSTANCE + .getOperatorTable(EnumSet.of(SqlLibrary.STANDARD, SqlLibrary.SPARK)); + final SchemaPlus rootSchema = Frameworks.createRootSchema(true); + final FrameworkConfig config = Frameworks.newConfigBuilder() + .parserConfig(SqlParser.Config.DEFAULT) + .operatorTable(opTab) + .defaultSchema(rootSchema) + .build(); + Planner planner = Frameworks.getPlanner(config); + SqlNode n = planner.parse(query); + n = planner.validate(n); + RelNode root = planner.rel(n).project(); + String plan = + RelOptUtil.dumpPlan("-- Plan", root, + SqlExplainFormat.JSON, SqlExplainLevel.DIGEST_ATTRIBUTES); + assertThat( + plan, containsString("\"exprs\": [\n" + + " {\n" + + " \"pos\": {\n" + + " \"line\": 1,\n" + + " \"column\": 8,\n" + + " \"end_line\": 1,\n" + + " \"end_column\": 25\n" + + " },\n" + + " \"op\": {\n" + + " \"name\": \"SEARCH\",\n" + + " \"kind\": \"SEARCH\",\n" + + " \"syntax\": \"INTERNAL\"\n" + + " },")); + } } From ab81314b69912e7cc7b6c624a0265c597c3110f4 Mon Sep 17 00:00:00 2001 From: dssysolyatin Date: Tue, 4 Nov 2025 14:56:53 +0200 Subject: [PATCH 005/562] [CALCITE-7268] SqlToRelConverter throws exception if lambda contains IN --- .../apache/calcite/sql2rel/SqlToRelConverter.java | 4 ++++ .../apache/calcite/test/SqlToRelConverterTest.java | 13 +++++++++++++ .../apache/calcite/test/SqlToRelConverterTest.xml | 11 +++++++++++ 3 files changed, 28 insertions(+) diff --git a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java index 7b1fc64bbbfc..e621ead19872 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java @@ -2100,6 +2100,9 @@ private void findSubQueries( } final SqlKind kind = node.getKind(); switch (kind) { + // A lambda has its own scope, which is not part of the blackboard. + case LAMBDA: + return; case EXISTS: case UNIQUE: case SELECT: @@ -2282,6 +2285,7 @@ private RexNode convertLambda(Blackboard bb, SqlNode node) { final Blackboard lambdaBb = createBlackboard(scope, nameToNodeMap, false); lambdaBb.setRoot(castNonNull(bb.inputs)); + replaceSubQueries(lambdaBb, call.getExpression(), RelOptUtil.Logic.TRUE_FALSE_UNKNOWN); final RexNode expr = lambdaBb.convertExpression(call.getExpression()); return rexBuilder.makeLambdaCall(expr, parameters); } diff --git a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java index d5648f9776fc..6a3fe210d979 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java @@ -143,6 +143,19 @@ public static void checkActualAndReferenceFiles() { .ok(); } + /** Test case for + * [CALCITE-7268] + * SqlToRelConverter throws exception if lambda contains IN. */ + @Test void testLambdaExpressionContainsIn() { + final String sql = "select \"EXISTS\"(ARRAY[1,2,3,4], (n) -> n IN (1,3))"; + fixture() + .withFactory(c -> + c.withOperatorTable(t -> SqlValidatorTest.operatorTableFor(SqlLibrary.SPARK))) + .withSql(sql) + .ok(); + + } + /** Test case for * [CALCITE-3679] * Allow lambda expressions in SQL queries. */ diff --git a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml index a530308abfc3..88a93e3d1bd2 100644 --- a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml @@ -4694,6 +4694,17 @@ LogicalProject(EXPR$0=[HIGHER_ORDER_FUNCTION2(1, () -> -1)]) +(DEPTNO, 1))]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +]]> + + + + + n IN (1,3))]]> + + + OR(=(N, 1), =(N, 3)))]) + LogicalValues(tuples=[[{ 0 }]]) ]]> From 974c1e136a45814786b969efb07ae2756e279d77 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Wed, 5 Nov 2025 17:52:54 -0800 Subject: [PATCH 006/562] [CALCITE-7273] CoreRules.JOIN_REDUCE_EXPRESSIONS throws when applied to an ASOF JOIN Signed-off-by: Mihai Budiu --- .../rel/rules/ReduceExpressionsRule.java | 7 ++++++ .../apache/calcite/test/RelOptRulesTest.java | 13 ++++++++++ .../apache/calcite/test/RelOptRulesTest.xml | 24 +++++++++++++++++++ 3 files changed, 44 insertions(+) diff --git a/core/src/main/java/org/apache/calcite/rel/rules/ReduceExpressionsRule.java b/core/src/main/java/org/apache/calcite/rel/rules/ReduceExpressionsRule.java index 8a2e5b09b41d..fa6864ac42c9 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/ReduceExpressionsRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/ReduceExpressionsRule.java @@ -30,6 +30,7 @@ import org.apache.calcite.rel.core.Join; import org.apache.calcite.rel.core.Project; import org.apache.calcite.rel.core.Window; +import org.apache.calcite.rel.logical.LogicalAsofJoin; import org.apache.calcite.rel.logical.LogicalCalc; import org.apache.calcite.rel.logical.LogicalFilter; import org.apache.calcite.rel.logical.LogicalProject; @@ -373,6 +374,12 @@ public JoinReduceExpressionsRule(Class joinClass, @Override public void onMatch(RelOptRuleCall call) { final Join join = call.rel(0); + if (join instanceof LogicalAsofJoin) { + // Currently ASOF JOINs are restricted by the validator to use very specific + // conditions, so there isn't much to simplify about them. + // Moreover, calling join.copy() below for an ASOF JOIN will throw. + return; + } final List expList = Lists.newArrayList(join.getCondition()); final int fieldCount = join.getLeft().getRowType().getFieldCount(); final RelMetadataQuery mq = call.getMetadataQuery(); diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index 08d7f52c2978..a44486e9d9b8 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -3391,6 +3391,19 @@ private void checkPushJoinThroughUnionOnRightDoesNotMatchSemiOrAntiJoin(JoinRelT .checkUnchanged(); } + /** Test case for [CALCITE-7273] + * CoreRules.JOIN_REDUCE_EXPRESSIONS throws when applied to an ASOF JOIN. */ + @Test void testAsofOptJoin() { + // Had to use ROW values to cause the optimization rule to match + final String sql = "SELECT *\n" + + "FROM (VALUES (NULL, ROW(0, 1)), (1, ROW(0, 2))) AS t1(k, t)\n" + + "ASOF JOIN (VALUES (2, ROW(0, 3))) AS t2(k, t)\n" + + "MATCH_CONDITION t2.t < t1.t\n" + + "ON t1.k = t2.k\n"; + sql(sql).withRule(CoreRules.JOIN_REDUCE_EXPRESSIONS) + .checkUnchanged(); + } + /** Tests to see if the final branch of union is missed. */ @Test void testUnionMergeRule() { final String sql = "select * from (\n" diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index 1c3ab3e6682e..3c86cc1b2f94 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -1586,6 +1586,30 @@ LogicalProject(K=[$0], T=[$1], K0=[$2], T0=[$3]) LogicalAsofJoin(condition=[=($0, $2)], joinType=[asof], matchCondition=[<($3, $1)]) LogicalValues(tuples=[[{ null, 0 }, { 1, null }, { 1, 0 }, { 1, 1 }, { 1, 2 }, { 1, 3 }, { 1, 4 }, { 2, 3 }, { 3, 4 }]]) LogicalValues(tuples=[[{ 1, null }, { 1, 2 }, { 1, 3 }, { 2, 10 }, { 2, 0 }]]) +]]> + + + + + + + + From 3b1d929b8bf9cfe69e86dff774c378ea46fccecb Mon Sep 17 00:00:00 2001 From: Sergey Nuyanzin Date: Sun, 2 Nov 2025 15:27:55 +0100 Subject: [PATCH 007/562] [CALCITE-7261] `DiffRepository` generation xml does not respect alphabetical order --- .../apache/calcite/test/DiffRepository.java | 8 +- .../test/DiffRepositoryGeneratedFileTest.java | 78 +++++++++++++++++++ .../test/DiffRepositoryGeneratedFileTest.xml | 18 +++++ 3 files changed, 100 insertions(+), 4 deletions(-) create mode 100644 testkit/src/test/java/org/apache/calcite/test/DiffRepositoryGeneratedFileTest.java create mode 100644 testkit/src/test/resources/org/apache/calcite/test/DiffRepositoryGeneratedFileTest.xml diff --git a/testkit/src/main/java/org/apache/calcite/test/DiffRepository.java b/testkit/src/main/java/org/apache/calcite/test/DiffRepository.java index 735a1247f488..db60757f4bd9 100644 --- a/testkit/src/main/java/org/apache/calcite/test/DiffRepository.java +++ b/testkit/src/main/java/org/apache/calcite/test/DiffRepository.java @@ -586,7 +586,7 @@ private synchronized void update( int i = 0; final List names = Pair.left(map); for (String s : names) { - if (s.compareToIgnoreCase(testCaseName) <= 0) { + if (s.compareTo(testCaseName) <= 0) { ++i; } } @@ -598,13 +598,13 @@ private synchronized void update( // will end up in exactly the right position, and if the list is not sorted, // the new item will end up in approximately the right position. while (i < map.size() - && names.get(i).compareToIgnoreCase(testCaseName) < 0) { + && names.get(i).compareTo(testCaseName) < 0) { ++i; } - if (i >= map.size() - 1) { + if (i > map.size() - 1) { return null; } - while (i >= 0 && names.get(i).compareToIgnoreCase(testCaseName) > 0) { + while (i >= 0 && names.get(i).compareTo(testCaseName) > 0) { --i; } return map.get(i + 1).right; diff --git a/testkit/src/test/java/org/apache/calcite/test/DiffRepositoryGeneratedFileTest.java b/testkit/src/test/java/org/apache/calcite/test/DiffRepositoryGeneratedFileTest.java new file mode 100644 index 000000000000..a14f4e1b894a --- /dev/null +++ b/testkit/src/test/java/org/apache/calcite/test/DiffRepositoryGeneratedFileTest.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.test; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.endsWith; + +/** + * Tests checking generated actual XML version. + * + *

    Test case for + * [CALCITE-7261] + * DiffRepository generation xml doesn't respect alphabetical order. + */ +public class DiffRepositoryGeneratedFileTest { + private static final DiffRepository REPO = + DiffRepository.lookup(DiffRepositoryGeneratedFileTest.class); + + @AfterAll + static void tearDown() throws IOException { + final String fileContent = + String.join("\n", Files.readAllLines(Paths.get(REPO.logFilePath()))); + assertThat( + fileContent, endsWith( + "\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + "")); + } + + @Test void testDiff() { + REPO.set("resource", "diff"); + } + + // Here method name was intentionally written in lower case to highlight + // discrepancy between checking logic and generated logic + @Test void testmulti() { + REPO.set("resource", "multi"); + } + + @Test void testNull() { + REPO.set("resource", "null"); + } +} diff --git a/testkit/src/test/resources/org/apache/calcite/test/DiffRepositoryGeneratedFileTest.xml b/testkit/src/test/resources/org/apache/calcite/test/DiffRepositoryGeneratedFileTest.xml new file mode 100644 index 000000000000..ef64ba1e38fa --- /dev/null +++ b/testkit/src/test/resources/org/apache/calcite/test/DiffRepositoryGeneratedFileTest.xml @@ -0,0 +1,18 @@ + + + From 32d421ee740e9fcdf541584af96b1a853800eb19 Mon Sep 17 00:00:00 2001 From: khanhkhanhlele Date: Fri, 7 Nov 2025 10:21:53 +0700 Subject: [PATCH 008/562] Fix typos in some files --- .../main/java/org/apache/calcite/materialize/MutableNode.java | 2 +- site/_posts/2021-10-19-release-1.28.0.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/materialize/MutableNode.java b/core/src/main/java/org/apache/calcite/materialize/MutableNode.java index eb3e78862137..278c946966d0 100644 --- a/core/src/main/java/org/apache/calcite/materialize/MutableNode.java +++ b/core/src/main/java/org/apache/calcite/materialize/MutableNode.java @@ -88,7 +88,7 @@ void flatten(List flatNodes) { } } - /** Returns whether this node is cylic, in an undirected sense; that is, + /** Returns whether this node is cyclic, in an undirected sense; that is, * whether the same descendant can be reached by more than one route. */ boolean isCyclic() { final Set descendants = new HashSet<>(); diff --git a/site/_posts/2021-10-19-release-1.28.0.md b/site/_posts/2021-10-19-release-1.28.0.md index ca62f2fdc094..e691004a0ccd 100644 --- a/site/_posts/2021-10-19-release-1.28.0.md +++ b/site/_posts/2021-10-19-release-1.28.0.md @@ -78,7 +78,7 @@ changes needed to be made: [`@Value.Immutable`](https://immutables.github.io/immutable.html#value) annotation. * Where `RelRule.Config` subclasses were nested 2+ classes deep, the - interfaces have been marked deprecated and are superceded by new, + interfaces have been marked deprecated and are superseded by new, uniquely named interfaces. The original Configs extend the new uniquely named interfaces. Subclassing these work as before and the existing rule signatures accept any previously implemented Config From 0abcd6a944ecabf832d406d82daf66bc9820910c Mon Sep 17 00:00:00 2001 From: dssysolyatin Date: Fri, 7 Nov 2025 11:17:36 +0200 Subject: [PATCH 009/562] [CALCITE-7276] SqlToRelConverter throws exception for UPDATE if identifier expansion disabled --- .../calcite/sql2rel/SqlToRelConverter.java | 14 +++++++------- .../calcite/test/SqlToRelConverterTest.java | 17 +++++++++++++++++ .../calcite/test/SqlToRelConverterTest.xml | 12 ++++++++++++ 3 files changed, 36 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java index e621ead19872..392fcdda1042 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java @@ -4470,17 +4470,17 @@ private RelNode convertUpdate(SqlUpdate call) { targetColumnNameList.add(field.getName()); } - // `sourceSelect` should contain target columns values plus source expressions - if (sourceSelect.getSelectList().size() + RelNode sourceRel = convertSelect(sourceSelect, false); + bb.setRoot(sourceRel, false); + + // `sourceRel` should contain target columns values plus source expressions + if (sourceRel.getRowType().getFieldCount() != targetTable.getRowType().getFieldCount() + call.getSourceExpressionList().size()) { throw new AssertionError( - "Unexpected select list size. Select list should contain both target table columns and " - + "set expressions"); + "Unexpected source select row type. Select row type should contain both target table " + + "columns and set expressions"); } - RelNode sourceRel = convertSelect(sourceSelect, false); - bb.setRoot(sourceRel, false); - // sourceRel already contains all source expressions. Only create references to those fields. List rexExpressionList = Util.transform( diff --git a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java index 6a3fe210d979..e0477c650121 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java @@ -3404,6 +3404,23 @@ void checkCorrelatedMapSubQuery(boolean expand) { sql(sql).ok(); } + /** + * Test case for + * [CALCITE-7276] + * SqlToRelConverter throws exception for UPDATE if identifier expansion disabled. + */ + @Test void testUpdateWithIdentifierExpansionDisabled() { + final String sql = "update emp set empno = empno + 1"; + sql(sql) + .withFactory(f -> + f.withValidator((opTab, catalogReader, typeFactory, config) + -> SqlValidatorUtil.newValidator(opTab, catalogReader, + typeFactory, config.withIdentifierExpansion(false)))) + .withTrim(false) + .ok(); + } + + @Test void testUpdateSubQuery() { final String sql = "update emp\n" + "set empno = (\n" diff --git a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml index 88a93e3d1bd2..b640d7dddeb2 100644 --- a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml @@ -9725,6 +9725,18 @@ LogicalTableModify(table=[[CATALOG, SALES, EMP]], operation=[UPDATE], updateColu LogicalTableModify(table=[[CATALOG, STRUCT, T]], operation=[UPDATE], updateColumnList=[["F0"."C0"]], sourceExpressionList=[[$9]], flattened=[true]) LogicalProject("K0"=[$0], "C1"=[$1], "F1"."A0"=[$2], "F2"."A0"=[$3], "F0"."C0"=[$4], "F1"."C0"=[$5], "F0"."C1"=[$6], "F1"."C2"=[$7], "F2"."C3"=[$8], EXPR$0=[+($4, 1)]) LogicalTableScan(table=[[CATALOG, STRUCT, T]]) +]]> + + + + + + + + From 44894167230c7d95e81407000f3d750f95ff1683 Mon Sep 17 00:00:00 2001 From: zhuyufeng0809 <1547107965@qq.com> Date: Thu, 30 Oct 2025 12:03:51 +0800 Subject: [PATCH 010/562] [CALCITE-7256] Make the fields of SqlTableRef public --- .../main/java/org/apache/calcite/sql/SqlTableRef.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql/SqlTableRef.java b/core/src/main/java/org/apache/calcite/sql/SqlTableRef.java index 8d164b77a12a..e4665c116c82 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlTableRef.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlTableRef.java @@ -36,8 +36,15 @@ public class SqlTableRef extends SqlCall { //~ Instance fields -------------------------------------------------------- - private final SqlIdentifier tableName; - private final SqlNodeList hints; + /** + * Table name identifier. + */ + public final SqlIdentifier tableName; + + /** + * List of SQL hints associated with the table. + */ + public final SqlNodeList hints; //~ Static fields/initializers --------------------------------------------- From 54f0add10da33150767d7a286feb81649ccd8e0c Mon Sep 17 00:00:00 2001 From: Ruben Quesada Lopez Date: Tue, 4 Nov 2025 10:02:56 +0000 Subject: [PATCH 011/562] [CALCITE-7266] Optimize the "well-known count bug" correction --- .../calcite/sql2rel/RelDecorrelator.java | 126 ++++++++++-------- .../calcite/sql2rel/RelDecorrelatorTest.java | 98 +++++++++++--- .../calcite/test/SqlToRelConverterTest.xml | 26 ++-- core/src/test/resources/sql/sub-query.iq | 16 +++ 4 files changed, 171 insertions(+), 95 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java b/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java index 42e3a84f963c..e9d7f498f949 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java @@ -221,14 +221,21 @@ public static RelNode decorrelateQuery(RelNode rootRel, return decorrelateQuery(rootRel, relBuilder, null); } + public static RelNode decorrelateQuery(RelNode rootRel, + RelBuilder relBuilder, @Nullable RuleSet decorrelationRules) { + return decorrelateQuery(rootRel, relBuilder, decorrelationRules, null); + } + /** * Decorrelates a query specifying a set of rules to be used in the * "remove correlation via rules" pre-processing. * * @param rootRel Root node of the query * @param relBuilder Builder for relational expressions - * @param decorrelationRules Rules to be used in the decorrelation, if null - * a default rule set will be used + * @param decorrelationRules Rules to attempt some initial rule-based-decorrelation conversions, + * if null a default rule set will be used + * @param preDecorrelateRules Pre-process rules to be used before the main decorrelation + * procedure, if null a default rule set will be used * * @return Equivalent query with all * {@link org.apache.calcite.rel.core.Correlate} instances removed @@ -236,7 +243,8 @@ public static RelNode decorrelateQuery(RelNode rootRel, * @see #removeCorrelationViaRule(RelNode, RuleSet) */ public static RelNode decorrelateQuery(RelNode rootRel, - RelBuilder relBuilder, @Nullable RuleSet decorrelationRules) { + RelBuilder relBuilder, @Nullable RuleSet decorrelationRules, + @Nullable RuleSet preDecorrelateRules) { final CorelMap corelMap = new CorelMapBuilder().build(rootRel); if (!corelMap.hasCorrelation()) { return rootRel; @@ -258,7 +266,7 @@ public static RelNode decorrelateQuery(RelNode rootRel, } if (!decorrelator.cm.mapCorToCorRel.isEmpty()) { - newRootRel = decorrelator.decorrelate(newRootRel); + newRootRel = decorrelator.decorrelate(newRootRel, preDecorrelateRules); } Litmus.THROW.check( rootRel.getRowType().equalsSansFieldNames(newRootRel.getRowType()), @@ -282,49 +290,56 @@ protected RelBuilderFactory relBuilderFactory() { } protected RelNode decorrelate(RelNode root) { - // first adjust count() expression if any - final RelBuilderFactory f = relBuilderFactory(); - HepProgram program = HepProgram.builder() - .addRuleInstance( - AdjustProjectForCountAggregateRule.DEFAULT_WITHOUT_FAVLOR - .withRelBuilderFactory(f).toRule()) - .addRuleInstance( - AdjustProjectForCountAggregateRule.DEFAULT_WITH_FAVLOR - .withRelBuilderFactory(f).toRule()) - .addRuleInstance( - FilterJoinRule.FilterIntoJoinRule.FilterIntoJoinRuleConfig.DEFAULT - .withRelBuilderFactory(f) - .withOperandSupplier(b0 -> - b0.operand(Filter.class).oneInput(b1 -> - b1.operand(Join.class).anyInputs())) - .withDescription("FilterJoinRule:filter") - .as(FilterJoinRule.FilterIntoJoinRule.FilterIntoJoinRuleConfig.class) - .withSmart(true) - .withPredicate((join, joinType, exp) -> true) - .as(FilterJoinRule.FilterIntoJoinRule.FilterIntoJoinRuleConfig.class) - .toRule()) - .addRuleInstance( - CoreRules.FILTER_PROJECT_TRANSPOSE.config - .withRelBuilderFactory(f) - .as(FilterProjectTransposeRule.Config.class) - .withOperandFor(Filter.class, filter -> - !RexUtil.containsCorrelation(filter.getCondition()), - Project.class, project -> true) - .withCopyFilter(true) - .withCopyProject(true) - .toRule()) - .addRuleInstance(FilterCorrelateRule.Config.DEFAULT - .withRelBuilderFactory(f) - .toRule()) - .addRuleInstance(FilterFlattenCorrelatedConditionRule.Config.DEFAULT - .withRelBuilderFactory(f) - .toRule()) - .build(); + return decorrelate(root, null); + } - HepPlanner planner = createPlanner(program); + protected RelNode decorrelate(RelNode root, @Nullable RuleSet preDecorrelateRules) { + final RelBuilderFactory f = relBuilderFactory(); + final HepProgram program; + if (preDecorrelateRules != null) { + program = ruleSetToHepProgram(preDecorrelateRules); + } else { + // Use a default set of pre-decorrelate rules: + // adjust count() expression if any, and do some filter-related transformations + program = HepProgram.builder() + .addRuleInstance( + AdjustProjectForCountAggregateRule.DEFAULT_WITHOUT_FAVLOR + .withRelBuilderFactory(f).toRule()) + .addRuleInstance( + AdjustProjectForCountAggregateRule.DEFAULT_WITH_FAVLOR + .withRelBuilderFactory(f).toRule()) + .addRuleInstance( + FilterJoinRule.FilterIntoJoinRule.FilterIntoJoinRuleConfig.DEFAULT + .withRelBuilderFactory(f) + .withOperandSupplier(b0 -> + b0.operand(Filter.class).oneInput(b1 -> + b1.operand(Join.class).anyInputs())) + .withDescription("FilterJoinRule:filter") + .as(FilterJoinRule.FilterIntoJoinRule.FilterIntoJoinRuleConfig.class) + .withSmart(true) + .withPredicate((join, joinType, exp) -> true) + .as(FilterJoinRule.FilterIntoJoinRule.FilterIntoJoinRuleConfig.class) + .toRule()) + .addRuleInstance( + CoreRules.FILTER_PROJECT_TRANSPOSE.config + .withRelBuilderFactory(f) + .as(FilterProjectTransposeRule.Config.class) + .withOperandFor(Filter.class, filter -> + !RexUtil.containsCorrelation(filter.getCondition()), + Project.class, project -> true) + .withCopyFilter(true) + .withCopyProject(true) + .toRule()) + .addRuleInstance(FilterCorrelateRule.Config.DEFAULT + .withRelBuilderFactory(f) + .toRule()) + .addRuleInstance(FilterFlattenCorrelatedConditionRule.Config.DEFAULT + .withRelBuilderFactory(f) + .toRule()) + .build(); + } - planner.setRoot(root); - root = planner.findBestExp(); + root = applyHepProgram(root, program); if (SQL2REL_LOGGER.isDebugEnabled()) { SQL2REL_LOGGER.debug("Plan before extracting correlated computations:\n" + RelOptUtil.toString(root)); @@ -374,11 +389,7 @@ protected RelNode decorrelate(RelNode root) { builder.addRuleCollection(getPostDecorrelateRules()); } final HepProgram program2 = builder.build(); - - final HepPlanner planner2 = createPlanner(program2); - final RelNode newRoot = result; - planner2.setRoot(newRoot); - return planner2.findBestExp(); + return applyHepProgram(result, program2); } return root; @@ -434,7 +445,7 @@ public RelNode removeCorrelationViaRule(RelNode root) { .addRuleInstance( RemoveCorrelationForScalarAggregateRule.DEFAULT.withRelBuilderFactory(f).toRule()) .build(); - return removeCorrelationViaRule(root, program); + return applyHepProgram(root, program); } /** @@ -443,6 +454,10 @@ public RelNode removeCorrelationViaRule(RelNode root) { * {@link org.apache.calcite.rel.core.Correlate}s might be removable in such way). */ public RelNode removeCorrelationViaRule(RelNode root, RuleSet ruleSet) { + return applyHepProgram(root, ruleSetToHepProgram(ruleSet)); + } + + private HepProgram ruleSetToHepProgram(RuleSet ruleSet) { final RelBuilderFactory f = relBuilderFactory(); final HepProgramBuilder builder = HepProgram.builder(); for (RelOptRule rule : ruleSet) { @@ -451,11 +466,10 @@ public RelNode removeCorrelationViaRule(RelNode root, RuleSet ruleSet) { } builder.addRuleInstance(rule); } - final HepProgram program = builder.build(); - return removeCorrelationViaRule(root, program); + return builder.build(); } - private RelNode removeCorrelationViaRule(RelNode root, HepProgram program) { + private RelNode applyHepProgram(RelNode root, HepProgram program) { HepPlanner planner = createPlanner(program); planner.setRoot(root); return planner.findBestExp(); @@ -1637,7 +1651,9 @@ private static boolean isWidening(RelDataType type, RelDataType type1) { } frameStack.push(Pair.of(rel.getCorrelationId(), leftFrame)); - final Frame rightFrame = getInvoke(oldRight, true, rel, parentPropagatesNullValues); + final Frame rightFrame = + getInvoke(oldRight, true, rel, + rel.getJoinType() == JoinRelType.LEFT || parentPropagatesNullValues); frameStack.pop(); if (rightFrame == null || rightFrame.corDefOutputs.isEmpty()) { diff --git a/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java b/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java index 40b621277818..c079af7740c9 100644 --- a/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java +++ b/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java @@ -208,6 +208,70 @@ public static Frameworks.ConfigBuilder config() { assertThat(after, hasTree(planAfter)); } + @Test void testDecorrelateCountBug() { + final FrameworkConfig frameworkConfig = config().build(); + final RelBuilder builder = RelBuilder.create(frameworkConfig); + final RelOptCluster cluster = builder.getCluster(); + final Planner planner = Frameworks.getPlanner(frameworkConfig); + final String sql = "SELECT deptno, " + + "(SELECT CASE WHEN SUM(sal) > 10 then 'VIP' else 'Regular' END expr " + + " FROM emp e WHERE d.deptno = e.deptno) a " + + "FROM dept d"; + final RelNode originalRel; + try { + final SqlNode parse = planner.parse(sql); + final SqlNode validate = planner.validate(parse); + originalRel = planner.rel(validate).rel; + } catch (Exception e) { + throw TestUtil.rethrow(e); + } + + final HepProgram hepProgram = HepProgram.builder() + .addRuleCollection( + ImmutableList.of( + // SubQuery program rules + CoreRules.FILTER_SUB_QUERY_TO_CORRELATE, + CoreRules.PROJECT_SUB_QUERY_TO_CORRELATE, + CoreRules.JOIN_SUB_QUERY_TO_CORRELATE)) + .build(); + final Program program = + Programs.of(hepProgram, true, + requireNonNull(cluster.getMetadataProvider())); + final RelNode before = + program.run(cluster.getPlanner(), originalRel, cluster.traitSet(), + Collections.emptyList(), Collections.emptyList()); + final String planBefore = "" + + "LogicalProject(DEPTNO=[$0], A=[$3])\n" + + " LogicalCorrelate(correlation=[$cor0], joinType=[left], requiredColumns=[{0}])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n" + + " LogicalProject(EXPR=[CASE(>($0, 10.00), 'VIP ', 'Regular')])\n" + + " LogicalAggregate(group=[{}], agg#0=[SUM($0)])\n" + + " LogicalProject(SAL=[$5])\n" + + " LogicalFilter(condition=[=($cor0.DEPTNO, $7)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + assertThat(before, hasTree(planBefore)); + + // Decorrelate without any rules, just "purely" decorrelation algorithm on RelDecorrelator + final RelNode after = + RelDecorrelator.decorrelateQuery(before, builder, RuleSets.ofList(Collections.emptyList()), + RuleSets.ofList(Collections.emptyList())); + + // Verify plan + final String planAfter = "" + + "LogicalProject(DEPTNO=[$0], A=[$3])\n" + + " LogicalJoin(condition=[=($0, $4)], joinType=[left])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n" + + " LogicalProject(EXPR=[CASE(>($2, 10.00), 'VIP ', 'Regular')], DEPTNO=[$0])\n" + + " LogicalJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left])\n" + + " LogicalProject(DEPTNO=[$0])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n" + + " LogicalAggregate(group=[{0}], agg#0=[SUM($1)])\n" + + " LogicalProject(DEPTNO=[$7], SAL=[$5])\n" + + " LogicalFilter(condition=[IS NOT NULL($7)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + assertThat(after, hasTree(planAfter)); + } + /** * Test case for * [CALCITE-6468] RelDecorrelator @@ -269,23 +333,17 @@ public static Frameworks.ConfigBuilder config() { // Verify plan final String planAfter = "" + "LogicalProject(EXPR$0=[1])\n" - + " LogicalJoin(condition=[AND(IS NOT DISTINCT FROM($0, $2), >($1, $3))], joinType=[inner])\n" + + " LogicalJoin(condition=[AND(=($0, $2), >($1, $3))], joinType=[inner])\n" + " LogicalAggregate(group=[{0}], TOTAL=[SUM($1)])\n" + " LogicalProject(DEPTNO=[$7], SAL=[$5])\n" + " LogicalTableScan(table=[[scott, EMP]])\n" - + " LogicalProject(DEPTNO=[$0], EXPR$0=[$2])\n" - + " LogicalJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left])\n" - + " LogicalProject(DEPTNO=[$0])\n" - + " LogicalAggregate(group=[{0}], TOTAL=[SUM($1)])\n" - + " LogicalProject(DEPTNO=[$7], SAL=[$5])\n" - + " LogicalTableScan(table=[[scott, EMP]])\n" - + " LogicalAggregate(group=[{0}], EXPR$0=[AVG($1)])\n" - + " LogicalProject(DEPTNO=[$0], TOTAL=[$1])\n" - + " LogicalAggregate(group=[{0}], TOTAL=[SUM($1)])\n" - + " LogicalProject(DEPTNO=[$0], SAL=[$1])\n" - + " LogicalFilter(condition=[IS NOT NULL($0)])\n" - + " LogicalProject(DEPTNO=[$7], SAL=[$5])\n" - + " LogicalTableScan(table=[[scott, EMP]])\n"; + + " LogicalAggregate(group=[{0}], EXPR$0=[AVG($1)])\n" + + " LogicalProject(DEPTNO=[$0], TOTAL=[$1])\n" + + " LogicalAggregate(group=[{0}], TOTAL=[SUM($1)])\n" + + " LogicalProject(DEPTNO=[$0], SAL=[$1])\n" + + " LogicalFilter(condition=[IS NOT NULL($0)])\n" + + " LogicalProject(DEPTNO=[$7], SAL=[$5])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; assertThat(after, hasTree(planAfter)); } @@ -366,15 +424,11 @@ public static Frameworks.ConfigBuilder config() { RelDecorrelator.decorrelateQuery(original, builder, noRules); final String planDecorrelatedNoRules = "" + "LogicalProject(EXPR$0=[ROW($9, $1)])\n" - + " LogicalJoin(condition=[IS NOT DISTINCT FROM($7, $8)], joinType=[left])\n" + + " LogicalJoin(condition=[=($7, $8)], joinType=[left])\n" + " LogicalTableScan(table=[[scott, EMP]])\n" - + " LogicalProject(DEPTNO1=[$0], $f1=[$2])\n" - + " LogicalJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left])\n" - + " LogicalAggregate(group=[{7}])\n" - + " LogicalTableScan(table=[[scott, EMP]])\n" - + " LogicalAggregate(group=[{0}], agg#0=[SINGLE_VALUE($1)])\n" - + " LogicalProject(DEPTNO1=[$0], DEPTNO=[$0])\n" - + " LogicalTableScan(table=[[scott, DEPT]])\n"; + + " LogicalAggregate(group=[{0}], agg#0=[SINGLE_VALUE($1)])\n" + + " LogicalProject(DEPTNO1=[$0], DEPTNO=[$0])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n"; assertThat(decorrelatedNoRules, hasTree(planDecorrelatedNoRules)); } } diff --git a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml index b640d7dddeb2..45351cfb6fdf 100644 --- a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml @@ -5523,16 +5523,11 @@ LogicalProject(D2=[$0], D3=[$1]) LogicalJoin(condition=[=($0, $1)], joinType=[left]) LogicalProject(D1=[+($0, 1)]) LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) - LogicalProject(D4=[$0], D6=[$2], $f2=[$3]) - LogicalJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left]) - LogicalAggregate(group=[{0}]) - LogicalProject(D1=[+($0, 1)]) + LogicalAggregate(group=[{0, 1}], agg#0=[MIN($2)]) + LogicalProject(D4=[$0], D6=[$2], $f0=[true]) + LogicalFilter(condition=[=($1, $0)]) + LogicalProject(D4=[+($0, 4)], D5=[+($0, 5)], D6=[+($0, 6)]) LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) - LogicalAggregate(group=[{0, 1}], agg#0=[MIN($2)]) - LogicalProject(D4=[$0], D6=[$2], $f0=[true]) - LogicalFilter(condition=[=($1, $0)]) - LogicalProject(D4=[+($0, 4)], D5=[+($0, 5)], D6=[+($0, 6)]) - LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) ]]> @@ -5558,16 +5553,11 @@ LogicalProject(D2=[$0], D3=[$1]) LogicalJoin(condition=[=($0, $1)], joinType=[left]) LogicalProject(D1=[+($0, 1)]) LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) - LogicalProject(D4=[$0], D6=[$2], $f2=[$3]) - LogicalJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left]) - LogicalAggregate(group=[{0}]) - LogicalProject(D1=[+($0, 1)]) + LogicalAggregate(group=[{0, 1}], agg#0=[MIN($2)]) + LogicalProject(D4=[$0], D6=[$2], $f0=[true]) + LogicalFilter(condition=[=($1, $0)]) + LogicalProject(D4=[+($0, 4)], D5=[+($0, 5)], D6=[+($0, 6)]) LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) - LogicalAggregate(group=[{0, 1}], agg#0=[MIN($2)]) - LogicalProject(D4=[$0], D6=[$2], $f0=[true]) - LogicalFilter(condition=[=($1, $0)]) - LogicalProject(D4=[+($0, 4)], D5=[+($0, 5)], D6=[+($0, 6)]) - LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) ]]> diff --git a/core/src/test/resources/sql/sub-query.iq b/core/src/test/resources/sql/sub-query.iq index 4a9a30fa5a9c..015a94f69ee9 100644 --- a/core/src/test/resources/sql/sub-query.iq +++ b/core/src/test/resources/sql/sub-query.iq @@ -4396,6 +4396,22 @@ WHERE 'Regular' IN ( !ok +SELECT deptno, (SELECT CASE WHEN SUM(sal) > 10 then 'VIP' else 'Regular' END expr + FROM emp e + WHERE d.deptno = e.deptno) a +FROM dept d; ++--------+---------+ +| DEPTNO | A | ++--------+---------+ +| 10 | VIP | +| 20 | VIP | +| 30 | VIP | +| 40 | Regular | ++--------+---------+ +(4 rows) + +!ok + # Test case for [CALCITE-5789] select deptno from dept d1 where exists ( select 1 from dept d2 where d2.deptno = d1.deptno and exists ( From 43c6a1f88a52b60da484b5bc86680249c683d0d9 Mon Sep 17 00:00:00 2001 From: Stamatis Zampetakis Date: Fri, 7 Nov 2025 16:10:35 +0100 Subject: [PATCH 012/562] [CALCITE-7281] Deprecate NullPolicy.ANY in favor of NullPolicy.SEMI_STRICT --- .../adapter/enumerable/NullPolicy.java | 4 ++- .../adapter/enumerable/RexImpTable.java | 35 +++++++++---------- .../advise/SqlAdvisorGetHintsFunction.java | 2 +- .../advise/SqlAdvisorGetHintsFunction2.java | 2 +- 4 files changed, 22 insertions(+), 21 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/NullPolicy.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/NullPolicy.java index 1d426ea43b1c..4000f97620b8 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/NullPolicy.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/NullPolicy.java @@ -32,7 +32,9 @@ public enum NullPolicy { STRICT, /** Returns null if one of the arguments is null, and possibly other times. */ SEMI_STRICT, - /** If any of the arguments are null, return null. */ + /** If any of the arguments are null, return null. + * @deprecated {@link #SEMI_STRICT} has identical semantics so use this instead. */ + @Deprecated ANY, /** If the first argument is null, return null. */ ARG0, diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java index 521cc9f7e4f7..ca8f93feb9fe 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java @@ -716,8 +716,8 @@ void populate1() { defineMethod(SHA512, BuiltInMethod.SHA512.method, NullPolicy.STRICT); defineMethod(SUBSTRING, BuiltInMethod.SUBSTRING.method, NullPolicy.STRICT); defineMethod(FORMAT_NUMBER, BuiltInMethod.FORMAT_NUMBER.method, NullPolicy.STRICT); - defineMethod(LEFT, BuiltInMethod.LEFT.method, NullPolicy.ANY); - defineMethod(RIGHT, BuiltInMethod.RIGHT.method, NullPolicy.ANY); + defineMethod(LEFT, BuiltInMethod.LEFT.method, NullPolicy.SEMI_STRICT); + defineMethod(RIGHT, BuiltInMethod.RIGHT.method, NullPolicy.SEMI_STRICT); defineMethod(LPAD, BuiltInMethod.LPAD.method, NullPolicy.STRICT); defineMethod(RPAD, BuiltInMethod.RPAD.method, NullPolicy.STRICT); defineMethod(STARTS_WITH, BuiltInMethod.STARTS_WITH.method, NullPolicy.STRICT); @@ -803,7 +803,7 @@ void populate1() { defineReflective(REGEXP_INSTR, BuiltInMethod.REGEXP_INSTR2.method, BuiltInMethod.REGEXP_INSTR3.method, BuiltInMethod.REGEXP_INSTR4.method, BuiltInMethod.REGEXP_INSTR5.method); - defineMethod(FIND_IN_SET, BuiltInMethod.FIND_IN_SET.method, NullPolicy.ANY); + defineMethod(FIND_IN_SET, BuiltInMethod.FIND_IN_SET.method, NullPolicy.SEMI_STRICT); define(TRIM, new TrimImplementor()); @@ -1073,23 +1073,23 @@ void populate2() { NullPolicy.STRICT); defineMethod(SLICE, BuiltInMethod.SLICE.method, NullPolicy.STRICT); defineMethod(ELEMENT, BuiltInMethod.ELEMENT.method, NullPolicy.STRICT); - defineMethod(STRUCT_ACCESS, BuiltInMethod.STRUCT_ACCESS.method, NullPolicy.ANY); + defineMethod(STRUCT_ACCESS, BuiltInMethod.STRUCT_ACCESS.method, NullPolicy.SEMI_STRICT); defineMethod(MEMBER_OF, BuiltInMethod.MEMBER_OF.method, NullPolicy.NONE); defineMethod(ARRAY_APPEND, BuiltInMethod.ARRAY_APPEND.method, NullPolicy.ARG0); defineMethod(ARRAY_COMPACT, BuiltInMethod.ARRAY_COMPACT.method, NullPolicy.STRICT); - defineMethod(ARRAY_CONTAINS, BuiltInMethod.LIST_CONTAINS.method, NullPolicy.ANY); + defineMethod(ARRAY_CONTAINS, BuiltInMethod.LIST_CONTAINS.method, NullPolicy.SEMI_STRICT); defineMethod(ARRAY_DISTINCT, BuiltInMethod.ARRAY_DISTINCT.method, NullPolicy.STRICT); - defineMethod(ARRAY_EXCEPT, BuiltInMethod.ARRAY_EXCEPT.method, NullPolicy.ANY); + defineMethod(ARRAY_EXCEPT, BuiltInMethod.ARRAY_EXCEPT.method, NullPolicy.SEMI_STRICT); defineMethod(ARRAY_JOIN, BuiltInMethod.ARRAY_TO_STRING.method, NullPolicy.STRICT); defineMethod(ARRAY_INSERT, BuiltInMethod.ARRAY_INSERT.method, NullPolicy.ARG0); - defineMethod(ARRAY_INTERSECT, BuiltInMethod.ARRAY_INTERSECT.method, NullPolicy.ANY); + defineMethod(ARRAY_INTERSECT, BuiltInMethod.ARRAY_INTERSECT.method, NullPolicy.SEMI_STRICT); defineMethod(ARRAY_LENGTH, BuiltInMethod.COLLECTION_SIZE.method, NullPolicy.STRICT); defineMethod(ARRAY_MAX, BuiltInMethod.ARRAY_MAX.method, NullPolicy.STRICT); defineMethod(ARRAY_MIN, BuiltInMethod.ARRAY_MIN.method, NullPolicy.STRICT); defineMethod(ARRAY_PREPEND, BuiltInMethod.ARRAY_PREPEND.method, NullPolicy.ARG0); - defineMethod(ARRAY_POSITION, BuiltInMethod.ARRAY_POSITION.method, NullPolicy.ANY); - defineMethod(ARRAY_REMOVE, BuiltInMethod.ARRAY_REMOVE.method, NullPolicy.ANY); + defineMethod(ARRAY_POSITION, BuiltInMethod.ARRAY_POSITION.method, NullPolicy.SEMI_STRICT); + defineMethod(ARRAY_REMOVE, BuiltInMethod.ARRAY_REMOVE.method, NullPolicy.SEMI_STRICT); defineMethod(ARRAY_REPEAT, BuiltInMethod.ARRAY_REPEAT.method, NullPolicy.NONE); defineMethod(ARRAY_REVERSE, BuiltInMethod.ARRAY_REVERSE.method, NullPolicy.STRICT); defineMethod(ARRAY_SIZE, BuiltInMethod.COLLECTION_SIZE.method, NullPolicy.STRICT); @@ -1097,16 +1097,16 @@ void populate2() { defineMethod(ARRAY_TO_STRING, BuiltInMethod.ARRAY_TO_STRING.method, NullPolicy.STRICT); defineMethod(STRING_TO_ARRAY, BuiltInMethod.STRING_TO_ARRAY.method, NullPolicy.ARG0); - defineMethod(ARRAY_UNION, BuiltInMethod.ARRAY_UNION.method, NullPolicy.ANY); - defineMethod(ARRAYS_OVERLAP, BuiltInMethod.ARRAYS_OVERLAP.method, NullPolicy.ANY); - defineMethod(ARRAYS_ZIP, BuiltInMethod.ARRAYS_ZIP.method, NullPolicy.ANY); - defineMethod(EXISTS, BuiltInMethod.EXISTS.method, NullPolicy.ANY); - defineMethod(MAP_CONCAT, BuiltInMethod.MAP_CONCAT.method, NullPolicy.ANY); - defineMethod(MAP_CONTAINS_KEY, BuiltInMethod.MAP_CONTAINS_KEY.method, NullPolicy.ANY); + defineMethod(ARRAY_UNION, BuiltInMethod.ARRAY_UNION.method, NullPolicy.SEMI_STRICT); + defineMethod(ARRAYS_OVERLAP, BuiltInMethod.ARRAYS_OVERLAP.method, NullPolicy.SEMI_STRICT); + defineMethod(ARRAYS_ZIP, BuiltInMethod.ARRAYS_ZIP.method, NullPolicy.SEMI_STRICT); + defineMethod(EXISTS, BuiltInMethod.EXISTS.method, NullPolicy.SEMI_STRICT); + defineMethod(MAP_CONCAT, BuiltInMethod.MAP_CONCAT.method, NullPolicy.SEMI_STRICT); + defineMethod(MAP_CONTAINS_KEY, BuiltInMethod.MAP_CONTAINS_KEY.method, NullPolicy.SEMI_STRICT); defineMethod(MAP_ENTRIES, BuiltInMethod.MAP_ENTRIES.method, NullPolicy.STRICT); defineMethod(MAP_KEYS, BuiltInMethod.MAP_KEYS.method, NullPolicy.STRICT); defineMethod(MAP_VALUES, BuiltInMethod.MAP_VALUES.method, NullPolicy.STRICT); - defineMethod(MAP_FROM_ARRAYS, BuiltInMethod.MAP_FROM_ARRAYS.method, NullPolicy.ANY); + defineMethod(MAP_FROM_ARRAYS, BuiltInMethod.MAP_FROM_ARRAYS.method, NullPolicy.SEMI_STRICT); defineMethod(MAP_FROM_ENTRIES, BuiltInMethod.MAP_FROM_ENTRIES.method, NullPolicy.STRICT); define(STR_TO_MAP, new StringToMapImplementor()); defineMethod(SUBSTRING_INDEX, BuiltInMethod.SUBSTRING_INDEX.method, NullPolicy.STRICT); @@ -4391,7 +4391,6 @@ private static List harmonize(final List argValueList, private List unboxIfNecessary(final List argValueList) { switch (nullPolicy) { case STRICT: - case ANY: case SEMI_STRICT: return Util.transform(argValueList, AbstractRexCallImplementor::unboxExpression); @@ -4943,7 +4942,7 @@ private static class QuantifyCollectionImplementor extends AbstractRexCallImplem QuantifyCollectionImplementor(SqlBinaryOperator binaryOperator, RexCallImplementor binaryImplementor) { - super("quantify", NullPolicy.ANY, false); + super("quantify", NullPolicy.SEMI_STRICT, false); this.binaryOperator = binaryOperator; this.binaryImplementor = binaryImplementor; } diff --git a/core/src/main/java/org/apache/calcite/sql/advise/SqlAdvisorGetHintsFunction.java b/core/src/main/java/org/apache/calcite/sql/advise/SqlAdvisorGetHintsFunction.java index d49478ad6bcc..d1b05764e893 100644 --- a/core/src/main/java/org/apache/calcite/sql/advise/SqlAdvisorGetHintsFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/advise/SqlAdvisorGetHintsFunction.java @@ -65,7 +65,7 @@ public class SqlAdvisorGetHintsFunction (translator, call, operands) -> Expressions.call(GET_COMPLETION_HINTS, Iterables.concat(Collections.singleton(ADVISOR), operands)), - NullPolicy.ANY, false); + NullPolicy.SEMI_STRICT, false); private static final List PARAMETERS = ReflectiveFunctionBase.builder() diff --git a/core/src/main/java/org/apache/calcite/sql/advise/SqlAdvisorGetHintsFunction2.java b/core/src/main/java/org/apache/calcite/sql/advise/SqlAdvisorGetHintsFunction2.java index 1f5fe1a36073..0ec33fda81b3 100644 --- a/core/src/main/java/org/apache/calcite/sql/advise/SqlAdvisorGetHintsFunction2.java +++ b/core/src/main/java/org/apache/calcite/sql/advise/SqlAdvisorGetHintsFunction2.java @@ -67,7 +67,7 @@ public class SqlAdvisorGetHintsFunction2 (translator, call, operands) -> Expressions.call(GET_COMPLETION_HINTS, Iterables.concat(Collections.singleton(ADVISOR), operands)), - NullPolicy.ANY, false); + NullPolicy.SEMI_STRICT, false); private static final List PARAMETERS = ReflectiveFunctionBase.builder() From 064834c904633e814ddf29ed99428d0ebf72bf8e Mon Sep 17 00:00:00 2001 From: lawlie8 Date: Thu, 13 Nov 2025 16:39:13 +0530 Subject: [PATCH 013/562] [CALCITE-7290] Update json-path to 2.10.0 --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index d5776ab2f212..7d021d2f344a 100644 --- a/gradle.properties +++ b/gradle.properties @@ -133,7 +133,7 @@ jna.version=5.14.0 jna-platform.version=5.14.0 joda-time.version=2.8.1 joou.version=0.9.4 -json-path.version=2.9.0 +json-path.version=2.10.0 json-smart.version=2.6.0 jsr305.version=3.0.2 jsoup.version=1.11.3 From 47fb60bfa0589cd0ce0b9e7815e657a6af8742c8 Mon Sep 17 00:00:00 2001 From: Thomas Rebele Date: Thu, 13 Nov 2025 18:59:05 +0100 Subject: [PATCH 014/562] [CALCITE-7291] Verify that the same exception is thrown for the original and simplified expression in RexSimplify#verify --- .../org/apache/calcite/rex/RexSimplify.java | 46 +++++++++++++++---- .../apache/calcite/rex/RexProgramTest.java | 4 -- 2 files changed, 37 insertions(+), 13 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rex/RexSimplify.java b/core/src/main/java/org/apache/calcite/rex/RexSimplify.java index 4a9677c27de9..3f7860749445 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexSimplify.java +++ b/core/src/main/java/org/apache/calcite/rex/RexSimplify.java @@ -440,7 +440,7 @@ private RexNode simplifyArithmetic(RexCall e) { assert e.getOperands().size() == 2; switch (e.getKind()) { - // These simplifications are safe for both checked and unchecked arithemtic. + // These simplifications are safe for both checked and unchecked arithmetic. case PLUS: case CHECKED_PLUS: return simplifyPlus(e); @@ -454,7 +454,7 @@ private RexNode simplifyArithmetic(RexCall e) { case CHECKED_DIVIDE: return simplifyDivide(e); default: - throw new IllegalArgumentException("Unsupported arithmeitc operation " + e.getKind()); + throw new IllegalArgumentException("Unsupported arithmetic operation " + e.getKind()); } } @@ -2309,6 +2309,20 @@ private RexNode simplifyOrs(List terms, RexUnknownAs unknownAs) { return RexUtil.composeDisjunction(rexBuilder, terms); } + private Pair evaluate(RexNode e, Map map) { + Comparable c = null; + RuntimeException ex = null; + try { + c = RexInterpreter.evaluate(e, map); + } catch (RuntimeException exception) { + ex = exception; + } + if (c == null && ex == null) { + throw new AssertionError("interpreter returned null for " + e); + } + return Pair.of(c, ex); + } + private void verify(RexNode before, RexNode simplified, RexUnknownAs unknownAs) { if (simplified.isAlwaysFalse() && before.isAlwaysTrue()) { @@ -2339,14 +2353,28 @@ private void verify(RexNode before, RexNode simplified, RexUnknownAs unknownAs) continue assignment_loop; } } - Comparable v0 = RexInterpreter.evaluate(foo0.e, map); - if (v0 == null) { - throw new AssertionError("interpreter returned null for " + foo0.e); - } - Comparable v1 = RexInterpreter.evaluate(foo1.e, map); - if (v1 == null) { - throw new AssertionError("interpreter returned null for " + foo1.e); + Pair p0 = evaluate(foo0.e, map); + Pair p1 = evaluate(foo1.e, map); + if (p0.right != null || p1.right != null) { + if (p0.right == null || p1.right == null) { + throw Util.first(p0.right, p1.right); + } + if (!p0.right.getClass().equals(p1.right.getClass())) { + AssertionError error = new AssertionError("exception class does not match"); + error.addSuppressed(p0.right); + error.addSuppressed(p1.right); + throw error; + } + if (!java.util.Objects.equals(p0.right.getMessage(), p1.right.getMessage())) { + AssertionError error = new AssertionError("exception message does not match"); + error.addSuppressed(p0.right); + error.addSuppressed(p1.right); + throw error; + } + continue; } + Comparable v0 = p0.left; + Comparable v1 = p1.left; if (before.getType().getSqlTypeName() == SqlTypeName.BOOLEAN) { switch (unknownAs) { case FALSE: diff --git a/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java b/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java index 558cedcd42ac..22d84ed4f0ec 100644 --- a/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java +++ b/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java @@ -2673,7 +2673,6 @@ trueLiteral, literal(1), @Test void testSimplifyCaseCompactionDiv() { // FIXME: RexInterpreter currently evaluates children beforehand. - simplify = simplify.withParanoid(false); RexNode caseNode = case_(vBool(0), vInt(0), eq(div(literal(3), vIntNotNull()), literal(11)), vInt(0), @@ -2685,7 +2684,6 @@ trueLiteral, literal(1), /** Tests a CASE value branch that contains division. */ @Test void testSimplifyCaseDiv1() { // FIXME: RexInterpreter currently evaluates children beforehand. - simplify = simplify.withParanoid(false); RexNode caseNode = case_(ne(vIntNotNull(), literal(0)), eq(div(literal(3), vIntNotNull()), literal(11)), @@ -2696,7 +2694,6 @@ trueLiteral, literal(1), /** Tests a CASE condition that contains division. */ @Test void testSimplifyCaseDiv2() { // FIXME: RexInterpreter currently evaluates children beforehand. - simplify = simplify.withParanoid(false); RexNode caseNode = case_(eq(vIntNotNull(), literal(0)), trueLiteral, gt(div(literal(3), vIntNotNull()), literal(1)), trueLiteral, @@ -2718,7 +2715,6 @@ trueLiteral, literal(1), // null + (a/0)/4 // ==> // null + (a/0)/4 - simplify = simplify.withParanoid(false); RexNode divideNode0 = plus(nullInt, div(div(vIntNotNull(), literal(0)), literal(4))); checkSimplifyUnchanged(divideNode0); // null + a/4 From a32b58fa3d06182354c394884d1be31de002b676 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Sat, 15 Nov 2025 15:50:59 -0800 Subject: [PATCH 015/562] [CALCITE-7293] MAP constructor cannot handle VARIANT values that need casts Signed-off-by: Mihai Budiu --- .../main/java/org/apache/calcite/sql/type/SqlTypeUtil.java | 3 ++- .../test/java/org/apache/calcite/test/SqlValidatorTest.java | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/calcite/sql/type/SqlTypeUtil.java b/core/src/main/java/org/apache/calcite/sql/type/SqlTypeUtil.java index 122708a41b14..2808c89514cf 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/SqlTypeUtil.java +++ b/core/src/main/java/org/apache/calcite/sql/type/SqlTypeUtil.java @@ -2045,7 +2045,8 @@ public static boolean isAtomic(RelDataType type) { || SqlTypeUtil.isNumeric(type) || SqlTypeUtil.isString(type) || SqlTypeUtil.isBoolean(type) - || typeName == SqlTypeName.UUID; + || typeName == SqlTypeName.UUID + || typeName == SqlTypeName.VARIANT; } /** Returns a DECIMAL type with the maximum precision for the current diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index a4b1dc79ad05..129261c9e9e7 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -1516,6 +1516,10 @@ void testLikeAndSimilarFails() { .columnType("VARIANT NOT NULL ARRAY NOT NULL"); expr("cast(MAP['a','b','c','d'] AS MAP)") .columnType("(VARCHAR NOT NULL, VARIANT) MAP NOT NULL"); + // Test case for [CALCITE-7293] https://issues.apache.org/jira/browse/CALCITE-7293 + // MAP constructor cannot handle VARIANT values that need casts + expr("MAP['a', CAST('x' AS VARIANT), 'b', CAST(NULL AS VARIANT)]") + .columnType("(CHAR(1) NOT NULL, VARIANT) MAP NOT NULL"); } @Test void testAccessVariant() { From 63ee430674b5427aafa5804c202167391c1217b6 Mon Sep 17 00:00:00 2001 From: Thomas Rebele Date: Thu, 13 Nov 2025 21:14:17 +0100 Subject: [PATCH 016/562] [CALCITE-7292] Replace "case when true then deptno else null end" with a non-simplifiable expression --- .../apache/calcite/test/RelOptRulesTest.java | 4 +- .../calcite/test/SqlToRelConverterTest.java | 14 +++--- .../apache/calcite/test/RelOptRulesTest.xml | 24 +++++----- .../calcite/test/SqlToRelConverterTest.xml | 32 ++++++------- core/src/test/resources/sql/sub-query.iq | 46 +++++++++---------- 5 files changed, 60 insertions(+), 60 deletions(-) diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index a44486e9d9b8..fe7b1bfcf9d9 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -8634,7 +8634,7 @@ private void checkSemiJoinRuleOnAntiJoin(RelOptRule rule) { @Test void testExpandProjectInNullable() { final String sql = "with e2 as (\n" - + " select empno, case when true then deptno else null end as deptno\n" + + " select empno, case when deptno > 0 then deptno else null end as deptno\n" + " from sales.emp)\n" + "select empno,\n" + " deptno in (select deptno from e2 where empno < 20) as d\n" @@ -8756,7 +8756,7 @@ private void checkSemiJoinRuleOnAntiJoin(RelOptRule rule) { final String sql = "select empno\n" + "from sales.emp\n" + "where empno\n" - + " < case deptno in (select case when true then deptno else null end\n" + + " < case deptno in (select case when deptno > 0 then deptno else null end\n" + " from sales.emp where empno < 20)\n" + " when true then 10\n" + " when false then 20\n" diff --git a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java index e0477c650121..b332be680e1b 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java @@ -2210,7 +2210,7 @@ void checkCorrelatedMapSubQuery(boolean expand) { // -- it's not forced into 2-valued by the "... IS TRUE" wrapper as in the // WHERE clause -- so the translation is more complicated. final String sql = "select name, deptno in (\n" - + " select case when true then deptno else null end from emp)\n" + + " select case when deptno > 0 then deptno else null end from emp)\n" + "from dept"; sql(sql).ok(); } @@ -2220,7 +2220,7 @@ void checkCorrelatedMapSubQuery(boolean expand) { // -- it's not forced into 2-valued by the "... IS TRUE" wrapper as in the // WHERE clause -- so the translation is more complicated. final String sql = "select name, deptno in (\n" - + " select case when true then deptno else null end from emp)\n" + + " select case when deptno > 0 then deptno else null end from emp)\n" + "from dept"; sql(sql).withExpand(false).ok(); } @@ -2231,14 +2231,14 @@ void checkCorrelatedMapSubQuery(boolean expand) { + "group by deptno\n" + "having count(*) > 2\n" + "and deptno in (\n" - + " select case when true then deptno else null end from emp)"; + + " select case when deptno > 0 then deptno else null end from emp)"; sql(sql).withExpand(false).ok(); } @Test void testUncorrelatedScalarSubQueryInOrderRex() { final String sql = "select ename\n" + "from emp\n" - + "order by (select case when true then deptno else null end from emp) desc,\n" + + "order by (select case when deptno > 0 then deptno else null end from emp) desc,\n" + " ename"; sql(sql).withExpand(false).ok(); } @@ -2247,7 +2247,7 @@ void checkCorrelatedMapSubQuery(boolean expand) { final String sql = "select sum(sal) as s\n" + "from emp\n" + "group by deptno\n" - + "order by (select case when true then deptno else null end from emp) desc,\n" + + "order by (select case when deptno > 0 then deptno else null end from emp) desc,\n" + " count(*)"; sql(sql).withExpand(false).ok(); } @@ -2263,14 +2263,14 @@ void checkCorrelatedMapSubQuery(boolean expand) { * an extra NOT. Both queries require 3-valued logic. */ @Test void testNotInUncorrelatedSubQueryInSelect() { final String sql = "select empno, deptno not in (\n" - + " select case when true then deptno else null end from dept)\n" + + " select case when deptno > 0 then deptno else null end from dept)\n" + "from emp"; sql(sql).ok(); } @Test void testNotInUncorrelatedSubQueryInSelectRex() { final String sql = "select empno, deptno not in (\n" - + " select case when true then deptno else null end from dept)\n" + + " select case when deptno > 0 then deptno else null end from dept)\n" + "from emp"; sql(sql).withExpand(false).ok(); } diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index 3c86cc1b2f94..7268a959a501 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -4632,7 +4632,7 @@ LogicalProject(EMPNO=[$0]) 0 then deptno else null end from sales.emp where empno < 20) when true then 10 when false then 20 @@ -4643,11 +4643,11 @@ where empno ($7, 0), CAST($7):INTEGER, null:INTEGER)]) LogicalFilter(condition=[<($0, 20)]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) }), true), 10, =(IN($7, { -LogicalProject(EXPR$0=[CASE(true, CAST($7):INTEGER, null:INTEGER)]) +LogicalProject(EXPR$0=[CASE(>($7, 0), CAST($7):INTEGER, null:INTEGER)]) LogicalFilter(condition=[<($0, 20)]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) }), false), 20, 30))]) @@ -4663,11 +4663,11 @@ LogicalProject(EMPNO=[$0]) LogicalJoin(condition=[true], joinType=[inner]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) LogicalAggregate(group=[{}], c=[COUNT()], ck=[COUNT($0)]) - LogicalProject(EXPR$0=[CASE(true, CAST($7):INTEGER, null:INTEGER)]) + LogicalProject(EXPR$0=[CASE(>($7, 0), CAST($7):INTEGER, null:INTEGER)]) LogicalFilter(condition=[<($0, 20)]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) LogicalAggregate(group=[{0}], i=[LITERAL_AGG(true)]) - LogicalProject(EXPR$0=[CASE(true, CAST($7):INTEGER, null:INTEGER)]) + LogicalProject(EXPR$0=[CASE(>($7, 0), CAST($7):INTEGER, null:INTEGER)]) LogicalFilter(condition=[<($0, 20)]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) ]]> @@ -5138,7 +5138,7 @@ LogicalProject(EMPNO=[$0], D=[CASE(IS NOT NULL($11), true, false)]) 0 then deptno else null end as deptno from sales.emp) select empno, deptno in (select deptno from e2 where empno < 20) as d @@ -5146,10 +5146,10 @@ from e2]]> ($7, 0), CAST($7):INTEGER, null:INTEGER), { LogicalProject(DEPTNO=[$1]) LogicalFilter(condition=[<($0, 20)]) - LogicalProject(EMPNO=[$0], DEPTNO=[CASE(true, CAST($7):INTEGER, null:INTEGER)]) + LogicalProject(EMPNO=[$0], DEPTNO=[CASE(>($7, 0), CAST($7):INTEGER, null:INTEGER)]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) })]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) @@ -5157,19 +5157,19 @@ LogicalProject(DEPTNO=[$1]) ($7, 0), CAST($7):INTEGER, null:INTEGER)), null:BOOLEAN, IS NOT NULL($12), true, <($10, $9), null:BOOLEAN, false)]) + LogicalJoin(condition=[=(CASE(>($7, 0), CAST($7):INTEGER, null:INTEGER), $11)], joinType=[left]) LogicalJoin(condition=[true], joinType=[inner]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) LogicalAggregate(group=[{}], c=[COUNT()], ck=[COUNT($0)]) LogicalProject(DEPTNO=[$1]) LogicalFilter(condition=[<($0, 20)]) - LogicalProject(EMPNO=[$0], DEPTNO=[CASE(true, CAST($7):INTEGER, null:INTEGER)]) + LogicalProject(EMPNO=[$0], DEPTNO=[CASE(>($7, 0), CAST($7):INTEGER, null:INTEGER)]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) LogicalAggregate(group=[{0}], i=[LITERAL_AGG(true)]) LogicalProject(DEPTNO=[$1]) LogicalFilter(condition=[<($0, 20)]) - LogicalProject(EMPNO=[$0], DEPTNO=[CASE(true, CAST($7):INTEGER, null:INTEGER)]) + LogicalProject(EMPNO=[$0], DEPTNO=[CASE(>($7, 0), CAST($7):INTEGER, null:INTEGER)]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) ]]> diff --git a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml index 45351cfb6fdf..382e39292f7a 100644 --- a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml @@ -3242,13 +3242,13 @@ from emp group by deptno having count(*) > 2 and deptno in ( - select case when true then deptno else null end from emp)]]> + select case when deptno > 0 then deptno else null end from emp)]]> ($2, 2), IN($0, { -LogicalProject(EXPR$0=[CAST($7):INTEGER]) +LogicalProject(EXPR$0=[CASE(>($7, 0), $7, null:INTEGER)]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) }))]) LogicalAggregate(group=[{0}], S=[SUM($1)], agg#1=[COUNT()]) @@ -3260,7 +3260,7 @@ LogicalProject(EXPR$0=[CAST($7):INTEGER]) 0 then deptno else null end from emp) from dept]]> @@ -3271,10 +3271,10 @@ LogicalProject(NAME=[$1], EXPR$1=[OR(AND(IS NOT NULL($6), <>($2, 0)), AND(<($3, LogicalJoin(condition=[true], joinType=[inner]) LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) LogicalAggregate(group=[{}], agg#0=[COUNT()], agg#1=[COUNT($0)]) - LogicalProject(EXPR$0=[CAST($7):INTEGER], $f1=[true]) + LogicalProject(EXPR$0=[CASE(>($7, 0), $7, null:INTEGER)], $f1=[true]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) LogicalAggregate(group=[{0}], agg#0=[MIN($1)]) - LogicalProject(EXPR$0=[CAST($7):INTEGER], $f1=[true]) + LogicalProject(EXPR$0=[CASE(>($7, 0), $7, null:INTEGER)], $f1=[true]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) ]]> @@ -3282,13 +3282,13 @@ LogicalProject(NAME=[$1], EXPR$1=[OR(AND(IS NOT NULL($6), <>($2, 0)), AND(<($3, 0 then deptno else null end from emp) from dept]]> ($7, 0), $7, null:INTEGER)]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) })]) LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) @@ -5709,7 +5709,7 @@ LogicalProject(EMPNO=[$0]) 0 then deptno else null end from dept) from emp]]> @@ -5720,10 +5720,10 @@ LogicalProject(EMPNO=[$0], EXPR$1=[OR(=($9, 0), AND(<($10, $9), null, IS NULL($1 LogicalJoin(condition=[true], joinType=[inner]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) LogicalAggregate(group=[{}], agg#0=[COUNT()], agg#1=[COUNT($0)]) - LogicalProject(EXPR$0=[CAST($0):INTEGER], $f1=[true]) + LogicalProject(EXPR$0=[CASE(>($0, 0), $0, null:INTEGER)], $f1=[true]) LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) LogicalAggregate(group=[{0}], agg#0=[MIN($1)]) - LogicalProject(EXPR$0=[CAST($0):INTEGER], $f1=[true]) + LogicalProject(EXPR$0=[CASE(>($0, 0), $0, null:INTEGER)], $f1=[true]) LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) ]]> @@ -5849,13 +5849,13 @@ LogicalProject(DEPTNO=[$0]) 0 then deptno else null end from dept) from emp]]> ($0, 0), $0, null:INTEGER)]) LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) }))]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) @@ -9054,7 +9054,7 @@ LogicalAggregate(group=[{}], EXPR$0=[MIN($0)]) 0 then deptno else null end from emp) desc, count(*)]]> @@ -9062,7 +9062,7 @@ order by (select case when true then deptno else null end from emp) desc, LogicalProject(S=[$0]) LogicalSort(sort0=[$1], sort1=[$2], dir0=[DESC], dir1=[ASC]) LogicalProject(S=[$1], EXPR$1=[$SCALAR_QUERY({ -LogicalProject(EXPR$0=[CAST($7):INTEGER]) +LogicalProject(EXPR$0=[CASE(>($7, 0), $7, null:INTEGER)]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) })], EXPR$2=[$2]) LogicalAggregate(group=[{0}], S=[SUM($1)], agg#1=[COUNT()]) @@ -9075,7 +9075,7 @@ LogicalProject(EXPR$0=[CAST($7):INTEGER]) 0 then deptno else null end from emp) desc, ename]]> @@ -9083,7 +9083,7 @@ order by (select case when true then deptno else null end from emp) desc, LogicalProject(ENAME=[$0]) LogicalSort(sort0=[$1], sort1=[$0], dir0=[DESC], dir1=[ASC]) LogicalProject(ENAME=[$1], EXPR$1=[$SCALAR_QUERY({ -LogicalProject(EXPR$0=[CAST($7):INTEGER]) +LogicalProject(EXPR$0=[CASE(>($7, 0), $7, null:INTEGER)]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) })]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) diff --git a/core/src/test/resources/sql/sub-query.iq b/core/src/test/resources/sql/sub-query.iq index 015a94f69ee9..1dd2edf4df00 100644 --- a/core/src/test/resources/sql/sub-query.iq +++ b/core/src/test/resources/sql/sub-query.iq @@ -1472,7 +1472,7 @@ EnumerableCalc(expr#0..3=[{inputs}], expr#4=[null:BOOLEAN], expr#5=[IS NOT NULL( # Test project null IN nullable select sal, cast(null as int) IN ( - select case when true then deptno else null end + select case when deptno > 0 then deptno else null end from "scott".dept) from "scott".emp; SAL | EXPR$1 @@ -1501,7 +1501,7 @@ EnumerableCalc(expr#0..3=[{inputs}], expr#4=[null:BOOLEAN], expr#5=[IS NOT NULL( EnumerableLimit(fetch=[1]) EnumerableSort(sort0=[$0], dir0=[DESC]) EnumerableAggregate(group=[{0}], c=[COUNT()]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[true], cs=[$t3]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[CAST($t0):INTEGER NOT NULL], expr#4=[0], expr#5=[>($t3, $t4)], expr#6=[null:TINYINT], expr#7=[CASE($t5, $t0, $t6)], expr#8=[IS NOT NULL($t7)], cs=[$t8]) EnumerableTableScan(table=[[scott, DEPT]]) !plan @@ -1542,7 +1542,7 @@ EnumerableCalc(expr#0..2=[{inputs}], expr#3=[IS NOT NULL($t2)], SAL=[$t1], EXPR$ # Test project literal IN nullable select sal, 10 IN ( - select case when true then deptno else null end + select case when deptno > 0 then deptno else null end from "scott".dept) from "scott".emp; SAL | EXPR$1 @@ -1571,7 +1571,7 @@ EnumerableCalc(expr#0..3=[{inputs}], expr#4=[IS FALSE($t2)], expr#5=[null:BOOLEA EnumerableLimit(fetch=[1]) EnumerableSort(sort0=[$0], dir0=[DESC]) EnumerableAggregate(group=[{0}], c=[COUNT()]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[IS NOT NULL($t0)], expr#4=[CAST($t0):INTEGER], expr#5=[10], expr#6=[=($t4, $t5)], cs=[$t3], $condition=[$t6]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[CAST($t0):INTEGER NOT NULL], expr#4=[0], expr#5=[>($t3, $t4)], expr#6=[null:TINYINT], expr#7=[CASE($t5, $t0, $t6)], expr#8=[IS NOT NULL($t7)], expr#9=[CAST($t7):INTEGER], expr#10=[Sarg[10; NULL AS TRUE]], expr#11=[SEARCH($t9, $t10)], cs=[$t8], $condition=[$t11]) EnumerableTableScan(table=[[scott, DEPT]]) !plan @@ -1722,7 +1722,7 @@ EnumerableCalc(expr#0..3=[{inputs}], expr#4=[IS NULL($t3)], expr#5=[null:BOOLEAN # Test project null NOT IN nullable select sal, cast(null as int) NOT IN ( - select case when true then deptno else null end + select case when deptno > 0 then deptno else null end from "scott".dept) from "scott".emp; SAL | EXPR$1 @@ -1751,7 +1751,7 @@ EnumerableCalc(expr#0..3=[{inputs}], expr#4=[IS NULL($t3)], expr#5=[null:BOOLEAN EnumerableLimit(fetch=[1]) EnumerableSort(sort0=[$0], dir0=[DESC]) EnumerableAggregate(group=[{0}], c=[COUNT()]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[true], cs=[$t3]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[CAST($t0):INTEGER NOT NULL], expr#4=[0], expr#5=[>($t3, $t4)], expr#6=[null:TINYINT], expr#7=[CASE($t5, $t0, $t6)], expr#8=[IS NOT NULL($t7)], cs=[$t8]) EnumerableTableScan(table=[[scott, DEPT]]) !plan @@ -1792,7 +1792,7 @@ EnumerableCalc(expr#0..2=[{inputs}], expr#3=[IS NULL($t2)], SAL=[$t1], EXPR$1=[$ # Test project literal NOT IN nullable select sal, 10 NOT IN ( - select case when true then deptno else null end + select case when deptno > 0 then deptno else null end from "scott".dept) from "scott".emp; SAL | EXPR$1 @@ -1821,7 +1821,7 @@ EnumerableCalc(expr#0..3=[{inputs}], expr#4=[IS NULL($t3)], expr#5=[IS FALSE($t2 EnumerableLimit(fetch=[1]) EnumerableSort(sort0=[$0], dir0=[DESC]) EnumerableAggregate(group=[{0}], c=[COUNT()]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[IS NOT NULL($t0)], expr#4=[CAST($t0):INTEGER], expr#5=[10], expr#6=[=($t4, $t5)], cs=[$t3], $condition=[$t6]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[CAST($t0):INTEGER NOT NULL], expr#4=[0], expr#5=[>($t3, $t4)], expr#6=[null:TINYINT], expr#7=[CASE($t5, $t0, $t6)], expr#8=[IS NOT NULL($t7)], expr#9=[CAST($t7):INTEGER], expr#10=[Sarg[10; NULL AS TRUE]], expr#11=[SEARCH($t9, $t10)], cs=[$t8], $condition=[$t11]) EnumerableTableScan(table=[[scott, DEPT]]) !plan @@ -1916,7 +1916,7 @@ EnumerableValues(tuples=[[]]) # Test filter null IN nullable select sal from "scott".emp where cast(null as int) IN ( - select case when true then deptno else null end + select case when deptno > 0 then deptno else null end from "scott".dept); SAL ----- @@ -1962,7 +1962,7 @@ EnumerableCalc(expr#0..2=[{inputs}], SAL=[$t1]) # Test filter literal IN nullable select sal from "scott".emp where 10 IN ( - select case when true then deptno else null end + select case when deptno > 0 then deptno else null end from "scott".dept); SAL --------- @@ -1988,7 +1988,7 @@ EnumerableCalc(expr#0..2=[{inputs}], SAL=[$t1]) EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], SAL=[$t5]) EnumerableTableScan(table=[[scott, EMP]]) EnumerableAggregate(group=[{0}]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[true], expr#4=[10], expr#5=[CAST($t0):INTEGER], expr#6=[=($t4, $t5)], cs=[$t3], $condition=[$t6]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[true], expr#4=[10], expr#5=[CAST($t0):INTEGER NOT NULL], expr#6=[0], expr#7=[>($t5, $t6)], expr#8=[null:TINYINT], expr#9=[CASE($t7, $t0, $t8)], expr#10=[CAST($t9):INTEGER], expr#11=[=($t4, $t10)], cs=[$t3], $condition=[$t11]) EnumerableTableScan(table=[[scott, DEPT]]) !plan @@ -2079,7 +2079,7 @@ EnumerableCalc(expr#0..3=[{inputs}], expr#4=[IS NULL($t3)], SAL=[$t1], $conditio # Test filter null NOT IN nullable select sal from "scott".emp where cast(null as int) NOT IN ( - select case when true then deptno else null end + select case when deptno > 0 then deptno else null end from "scott".dept); SAL ----- @@ -2093,7 +2093,7 @@ EnumerableCalc(expr#0..3=[{inputs}], expr#4=[IS NULL($t3)], SAL=[$t1], $conditio EnumerableLimit(fetch=[1]) EnumerableSort(sort0=[$0], dir0=[DESC]) EnumerableAggregate(group=[{0}], c=[COUNT()]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[true], cs=[$t3]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[CAST($t0):INTEGER NOT NULL], expr#4=[0], expr#5=[>($t3, $t4)], expr#6=[null:TINYINT], expr#7=[CASE($t5, $t0, $t6)], expr#8=[IS NOT NULL($t7)], cs=[$t8]) EnumerableTableScan(table=[[scott, DEPT]]) !plan @@ -2121,7 +2121,7 @@ EnumerableCalc(expr#0..3=[{inputs}], expr#4=[IS NULL($t3)], expr#5=[NOT($t2)], e # Test filter literal NOT IN nullable select sal from "scott".emp where 10 NOT IN ( - select case when true then deptno else null end + select case when deptno > 0 then deptno else null end from "scott".dept); SAL ----- @@ -2135,7 +2135,7 @@ EnumerableCalc(expr#0..3=[{inputs}], expr#4=[IS NULL($t3)], expr#5=[NOT($t2)], e EnumerableLimit(fetch=[1]) EnumerableSort(sort0=[$0], dir0=[DESC]) EnumerableAggregate(group=[{0}], c=[COUNT()]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[IS NOT NULL($t0)], expr#4=[CAST($t0):INTEGER], expr#5=[10], expr#6=[=($t4, $t5)], cs=[$t3], $condition=[$t6]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[CAST($t0):INTEGER NOT NULL], expr#4=[0], expr#5=[>($t3, $t4)], expr#6=[null:TINYINT], expr#7=[CASE($t5, $t0, $t6)], expr#8=[IS NOT NULL($t7)], expr#9=[CAST($t7):INTEGER], expr#10=[Sarg[10; NULL AS TRUE]], expr#11=[SEARCH($t9, $t10)], cs=[$t8], $condition=[$t11]) EnumerableTableScan(table=[[scott, DEPT]]) !plan @@ -2253,7 +2253,7 @@ EnumerableCalc(expr#0..4=[{inputs}], expr#5=[RAND()], expr#6=[CAST($t5):INTEGER # Test filter null IN nullable correlated select sal from "scott".emp e where cast(null as int) IN ( - select case when true then deptno else null end + select case when deptno > 0 then deptno else null end from "scott".dept d where e.deptno=d.deptno); SAL ----- @@ -2287,7 +2287,7 @@ EnumerableCalc(expr#0..2=[{inputs}], SAL=[$t1]) # Test filter literal IN nullable correlated select sal from "scott".emp e where 10 IN ( - select case when true then deptno else null end + select case when deptno > 0 then deptno else null end from "scott".dept d where e.deptno=d.deptno); SAL --------- @@ -2301,7 +2301,7 @@ EnumerableCalc(expr#0..2=[{inputs}], SAL=[$t1]) EnumerableHashJoin(condition=[=($2, $3)], joinType=[semi]) EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], SAL=[$t5], DEPTNO=[$t7]) EnumerableTableScan(table=[[scott, EMP]]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[10], expr#4=[CAST($t0):INTEGER], expr#5=[=($t3, $t4)], DEPTNO=[$t0], $condition=[$t5]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[10], expr#4=[CAST($t0):INTEGER NOT NULL], expr#5=[0], expr#6=[>($t4, $t5)], expr#7=[null:TINYINT], expr#8=[CASE($t6, $t0, $t7)], expr#9=[CAST($t8):INTEGER], expr#10=[=($t3, $t9)], DEPTNO=[$t0], $condition=[$t10]) EnumerableTableScan(table=[[scott, DEPT]]) !plan @@ -2369,7 +2369,7 @@ EnumerableValues(tuples=[[]]) # Test filter null NOT IN nullable correlated select sal from "scott".emp e where cast(null as int) NOT IN ( - select case when true then deptno else null end + select case when deptno > 0 then deptno else null end from "scott".dept d where e.deptno=d.deptno); SAL ----- @@ -2415,7 +2415,7 @@ EnumerableCalc(expr#0..4=[{inputs}], expr#5=[NOT($t3)], expr#6=[IS NOT NULL($t3) # Test filter literal NOT IN nullable correlated select sal from "scott".emp e where 10 NOT IN ( - select case when true then deptno else null end + select case when deptno > 0 then deptno else null end from "scott".dept d where e.deptno=d.deptno); SAL --------- @@ -2439,9 +2439,9 @@ EnumerableCalc(expr#0..4=[{inputs}], expr#5=[NOT($t3)], expr#6=[IS NOT NULL($t3) EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], SAL=[$t5], DEPTNO=[$t7]) EnumerableTableScan(table=[[scott, EMP]]) EnumerableSort(sort0=[$1], dir0=[ASC]) - EnumerableCalc(expr#0..1=[{inputs}], cs=[$t1], DEPTNO=[$t0]) - EnumerableWindow(window#0=[window(partition {0} aggs [FIRST_VALUE($1)])], constants=[[true]]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[CAST($t0):INTEGER], expr#4=[10], expr#5=[=($t3, $t4)], DEPTNO=[$t0], $condition=[$t5]) + EnumerableCalc(expr#0..2=[{inputs}], cs=[$t2], DEPTNO=[$t0]) + EnumerableWindow(window#0=[window(partition {0} order by [1 DESC] range between UNBOUNDED PRECEDING and UNBOUNDED FOLLOWING aggs [FIRST_VALUE($1)])]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[CAST($t0):INTEGER NOT NULL], expr#4=[0], expr#5=[>($t3, $t4)], expr#6=[null:TINYINT], expr#7=[CASE($t5, $t0, $t6)], expr#8=[IS NOT NULL($t7)], expr#9=[CAST($t7):INTEGER], expr#10=[Sarg[10; NULL AS TRUE]], expr#11=[SEARCH($t9, $t10)], DEPTNO=[$t0], $1=[$t8], $condition=[$t11]) EnumerableTableScan(table=[[scott, DEPT]]) !plan From dfc727c591e17b7ce63e51ff44fc7430d549145b Mon Sep 17 00:00:00 2001 From: Thomas Rebele Date: Fri, 17 Dec 2021 17:47:42 +0100 Subject: [PATCH 017/562] [CALCITE-4947] Checkstyle fails on classes generated by Intellij when using option "build and run [tests] using Intellij IDEA" --- build.gradle.kts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index 86be76ef0ab8..4be15b5a7a6b 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -319,7 +319,9 @@ val javaccGeneratedPatterns = arrayOf( "**/parser/**/Token.*", "**/parser/**/TokenMgrError.*", "**/org/apache/calcite/runtime/Resources.java", - "**/parser/**/*ParserTokenManager.*" + "**/parser/**/*ParserTokenManager.*", + "generated_tests/org/apache/calcite/**/*", + "generated/org/apache/calcite/**/*" ) fun PatternFilterable.excludeJavaCcGenerated() { From 54c758a3567a96fd85ca5127c0ad5930b1bf063c Mon Sep 17 00:00:00 2001 From: Thomas Rebele Date: Fri, 17 Dec 2021 17:47:42 +0100 Subject: [PATCH 018/562] [CALCITE-4947] Checkstyle fails on classes generated by Intellij when using option "build and run [tests] using Intellij IDEA" Addendum to the previous commit: ignore files generated by Intellij during the compilation. --- build.gradle.kts | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 4be15b5a7a6b..33ff8905c808 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -319,15 +319,24 @@ val javaccGeneratedPatterns = arrayOf( "**/parser/**/Token.*", "**/parser/**/TokenMgrError.*", "**/org/apache/calcite/runtime/Resources.java", - "**/parser/**/*ParserTokenManager.*", - "generated_tests/org/apache/calcite/**/*", - "generated/org/apache/calcite/**/*" + "**/parser/**/*ParserTokenManager.*" ) fun PatternFilterable.excludeJavaCcGenerated() { exclude(*javaccGeneratedPatterns) } +val intellijGeneratedPatterns = arrayOf( + "generated_tests/org/apache/calcite/**/*", + "generated/org/apache/calcite/**/*" +) + +// Ignore files generated by Intellij's compiler +// when using the option to build and run tests using Intellij IDEA instead of Gradle +fun PatternFilterable.excludeIntellijGenerated() { + exclude(*intellijGeneratedPatterns) +} + fun com.github.autostyle.gradle.BaseFormatExtension.license() { licenseHeader(rootProject.ide.licenseHeader) { copyrightStyle("bat", com.github.autostyle.generic.DefaultCopyrightStyle.PAAMAYIM_NEKUDOTAYIM) @@ -466,6 +475,7 @@ allprojects { // On the other hand, supporessions.xml still analyzes the file, and // then it recognizes it should suppress all the output. excludeJavaCcGenerated() + excludeIntellijGenerated() // Workaround for https://github.com/gradle/gradle/issues/13927 // Absolute paths must not be used as they defeat Gradle build cache // Unfortunately, Gradle passes only config_loc variable by default, so we make @@ -505,6 +515,7 @@ allprojects { tasks { configureEach { excludeJavaCcGenerated() + excludeIntellijGenerated() (options as StandardJavadocDocletOptions).apply { // Please refrain from using non-ASCII chars below since the options are passed as // javadoc.options file which is parsed with "default encoding" @@ -812,6 +823,7 @@ allprojects { configureEach { excludeJavaCcGenerated() + excludeIntellijGenerated() exclude( "**/org/apache/calcite/adapter/os/Processes${'$'}ProcessFactory.class", "**/org/apache/calcite/adapter/os/OsAdapterTest.class", @@ -836,6 +848,7 @@ allprojects { if (enableCheckerframework) { options.forkOptions.memoryMaximumSize = "2g" } + excludeIntellijGenerated() } configureEach { outputs.cacheIf("test results depend on the database configuration, so we shouldn't cache it") { From fc1fa4998c4568d46af9a879eeeeaf89f747c52e Mon Sep 17 00:00:00 2001 From: Thomas Rebele Date: Mon, 25 Aug 2025 18:24:38 +0200 Subject: [PATCH 019/562] [CALCITE-7145] RexSimplify should not simplify IS NULL(10/0) --- .../org/apache/calcite/rex/RexSimplify.java | 12 +++- .../apache/calcite/rex/RexProgramTest.java | 56 +++++++++++++++++-- .../apache/calcite/test/RelOptRulesTest.java | 13 +++-- .../apache/calcite/test/RelOptRulesTest.xml | 2 +- .../apache/calcite/test/DruidAdapter2IT.java | 2 + .../apache/calcite/test/DruidAdapterIT.java | 2 + 6 files changed, 75 insertions(+), 12 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rex/RexSimplify.java b/core/src/main/java/org/apache/calcite/rex/RexSimplify.java index 3f7860749445..2ad299675fd8 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexSimplify.java +++ b/core/src/main/java/org/apache/calcite/rex/RexSimplify.java @@ -1143,7 +1143,8 @@ private RexNode simplifyIs(RexCall call, RexUnknownAs unknownAs) { // "(CASE WHEN FALSE THEN 1 ELSE 2) IS NOT NULL" we first simplify the // argument to "2", and only then we can simplify "2 IS NOT NULL" to "TRUE". a = simplify(a, UNKNOWN); - if (!a.getType().isNullable() && isSafeExpression(a)) { + boolean isSafe = isSafeExpression(a); + if (!a.getType().isNullable() && isSafe) { return rexBuilder.makeLiteral(true); } if (RexUtil.isLosslessCast(a)) { @@ -1158,6 +1159,9 @@ private RexNode simplifyIs(RexCall call, RexUnknownAs unknownAs) { if (hasCustomNullabilityRules(a.getKind())) { return null; } + if (!isSafe) { + return rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL, a); + } switch (Strong.policy(a)) { case NOT_NULL: return rexBuilder.makeLiteral(true); @@ -1198,7 +1202,8 @@ private RexNode simplifyIs(RexCall call, RexUnknownAs unknownAs) { // "(CASE WHEN FALSE THEN 1 ELSE 2) IS NULL" we first simplify the // argument to "2", and only then we can simplify "2 IS NULL" to "FALSE". a = simplify(a, UNKNOWN); - if (!a.getType().isNullable() && isSafeExpression(a)) { + boolean isSafe = isSafeExpression(a); + if (!a.getType().isNullable() && isSafe) { return rexBuilder.makeLiteral(false); } if (RexUtil.isLosslessCast(a)) { @@ -1213,6 +1218,9 @@ private RexNode simplifyIs(RexCall call, RexUnknownAs unknownAs) { if (hasCustomNullabilityRules(a.getKind())) { return null; } + if (!isSafe) { + return rexBuilder.makeCall(SqlStdOperatorTable.IS_NULL, a); + } switch (Strong.policy(a)) { case NOT_NULL: return rexBuilder.makeLiteral(false); diff --git a/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java b/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java index 22d84ed4f0ec..91f55624e8b5 100644 --- a/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java +++ b/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java @@ -2715,6 +2715,7 @@ trueLiteral, literal(1), // null + (a/0)/4 // ==> // null + (a/0)/4 + // a/0 throws an exception so it may not be simplified RexNode divideNode0 = plus(nullInt, div(div(vIntNotNull(), literal(0)), literal(4))); checkSimplifyUnchanged(divideNode0); // null + a/4 @@ -2730,13 +2731,56 @@ trueLiteral, literal(1), // null + null/0 // ==> // null + null/0 - RexNode divideNode3 = plus(nullInt, div(vIntNotNull(), literal(0))); + RexNode divideNode3 = plus(nullInt, div(nullInt, literal(0))); checkSimplifyUnchanged(divideNode3); + // null + a/0 + // ==> + // null + a/0 + RexNode divideNode4 = plus(nullInt, div(vIntNotNull(), literal(0))); + checkSimplifyUnchanged(divideNode4); // null + a/b // ==> // null + a/b - RexNode divideNode4 = plus(nullInt, div(vIntNotNull(), vIntNotNull())); - checkSimplifyUnchanged(divideNode4); + // b might be 0 and throw an exception, so a/b cannot be simplified. + // E.g., PostgreSQL throws a division-by-zero error for the following query: + // SELECT NULL + (a/b) FROM (select 1 as a, 0 as b) t + RexNode divideNode5 = plus(nullInt, div(vIntNotNull(), vIntNotNull())); + checkSimplifyUnchanged(divideNode5); + // null/(1/0) + // ==> + // null/(1/0) + RexNode divideNode7 = div(nullInt, div(literal(1), literal(0))); + checkSimplifyUnchanged(divideNode7); + // (1/0)/null + // ==> + // (1/0)/null + RexNode divideNode8 = div(div(literal(1), literal(0)), nullInt); + checkSimplifyUnchanged(divideNode8); + } + + /** Test cases for IS NULL(x/y). + * See [CALCITE-7145] + * RexSimplify should not simplify IS NULL(10/0). */ + @Test void testSimplifyIsNullDivide() { + RelDataType intType = + typeFactory.createTypeWithNullability( + typeFactory.createSqlType(SqlTypeName.INTEGER), false); + + checkSimplifyUnchanged(isNull(div(vIntNotNull(), literal(0)))); + checkSimplifyUnchanged(isNull(div(vIntNotNull(), cast(literal(0), intType)))); + + checkSimplifyUnchanged(isNull(div(cast(literal(2), intType), vIntNotNull()))); + checkSimplifyUnchanged(isNull(div(vIntNotNull(), vIntNotNull()))); + + checkSimplifyUnchanged(isNotNull(div(vIntNotNull(), literal(0)))); + checkSimplifyUnchanged(isNotNull(div(vIntNotNull(), cast(literal(0), intType)))); + + checkSimplifyUnchanged(isNotNull(div(cast(literal(2), intType), vIntNotNull()))); + checkSimplifyUnchanged(isNotNull(div(vIntNotNull(), vIntNotNull()))); + + checkSimplifyUnchanged(isNull(div(vDecimalNotNull(), literal(0)))); + + checkSimplifyUnchanged(isNotNull(div(vDecimalNotNull(), literal(0)))); } @Test void testPushNotIntoCase() { @@ -2744,9 +2788,9 @@ trueLiteral, literal(1), not( case_( isTrue(vBool()), vBool(1), - gt(div(vIntNotNull(), literal(2)), literal(1)), vBool(2), + gt(div(vIntNotNull(), vInt(1)), literal(1)), vBool(2), vBool(3))), - "CASE(?0.bool0, NOT(?0.bool1), >(/(?0.notNullInt0, 2), 1), NOT(?0.bool2), NOT(?0.bool3))"); + "CASE(?0.bool0, NOT(?0.bool1), >(/(?0.notNullInt0, ?0.int1), 1), NOT(?0.bool2), NOT(?0.bool3))"); } @Test void testNotRecursion() { @@ -4244,7 +4288,9 @@ private SqlSpecialOperatorWithPolicy(String name, SqlKind kind, int prec, boolea checkSimplify(mul(nullInt, a), "null:INTEGER"); checkSimplify(div(a, one), "?0.notNullInt1"); + checkSimplify(div(nullInt, one), "null:INTEGER"); checkSimplify(div(a, nullInt), "null:INTEGER"); + checkSimplify(div(zero, nullInt), "null:INTEGER"); checkSimplify(add(b, half), "?0.notNullDecimal2"); diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index fe7b1bfcf9d9..6b2d1533c6f3 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -5720,6 +5720,8 @@ private void checkEmptyJoin(RelOptFixture f) { final Function relFn = b -> { final RexBuilder rexBuilder = b.getRexBuilder(); final RelDataType type = rexBuilder.getTypeFactory().createSqlType(SqlTypeName.BIGINT); + final RelDataType nullableType = rexBuilder.getTypeFactory() + .createTypeWithNullability(type, true); RelNode left = b .values(new String[]{"x", "y"}, 1, 2, 2, 1).build(); @@ -5727,19 +5729,22 @@ private void checkEmptyJoin(RelOptFixture f) { RexLiteral literal1 = rexBuilder.makeLiteral(1, type); RexLiteral literal2 = rexBuilder.makeLiteral(2, type); RexLiteral literal3 = rexBuilder.makeLiteral(3, type); + // the MOD (%) operation needs to be unsafe to reproduce the scenario + RexNode param0 = rexBuilder.makeDynamicParam(nullableType, 0); + RexNode param1 = rexBuilder.makeDynamicParam(nullableType, 1); - // CASE WHEN x % 2 = 1 THEN x < 2 - // WHEN x % 3 = 2 THEN x < 1 + // CASE WHEN x % param0 = 1 THEN x < 2 + // WHEN x % param1 = 2 THEN x < 1 // ELSE x < 3 final RexNode caseRexNode = rexBuilder.makeCall( SqlStdOperatorTable.CASE, rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, - rexBuilder.makeCall(SqlStdOperatorTable.MOD, ref, literal2), + rexBuilder.makeCall(SqlStdOperatorTable.MOD, ref, param0), literal1), rexBuilder.makeCall(SqlStdOperatorTable.LESS_THAN, ref, literal2), rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, - rexBuilder.makeCall(SqlStdOperatorTable.MOD, ref, literal3), + rexBuilder.makeCall(SqlStdOperatorTable.MOD, ref, param1), literal2), rexBuilder.makeCall(SqlStdOperatorTable.LESS_THAN, ref, literal1), rexBuilder.makeCall(SqlStdOperatorTable.LESS_THAN, ref, literal3)); diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index 7268a959a501..651b6ad52074 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -15311,7 +15311,7 @@ LogicalProject(QX=[CAST(CASE(=($0, 1), 1, 2)):INTEGER]) diff --git a/druid/src/test/java/org/apache/calcite/test/DruidAdapter2IT.java b/druid/src/test/java/org/apache/calcite/test/DruidAdapter2IT.java index d9b403a59690..eec575e331d3 100644 --- a/druid/src/test/java/org/apache/calcite/test/DruidAdapter2IT.java +++ b/druid/src/test/java/org/apache/calcite/test/DruidAdapter2IT.java @@ -33,6 +33,7 @@ import org.junit.jupiter.api.Assumptions; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import java.net.URL; @@ -2902,6 +2903,7 @@ private void testCountWithApproxDistinct(boolean approx, String sql, .returnsUnordered("EXPR$0=86829"); } + @Disabled("CALCITE-7271") @Test void testComplexExpressionsIsNull() { final String sql = "SELECT COUNT(*) FROM \"foodmart\" where ( cast(null as INTEGER) + cast" + "(\"city\" as INTEGER)) IS NULL"; diff --git a/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java b/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java index f7bf8294dd0d..87f2c65f4cc4 100644 --- a/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java +++ b/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java @@ -33,6 +33,7 @@ import org.junit.jupiter.api.Assumptions; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import java.net.URL; @@ -3464,6 +3465,7 @@ private void testCountWithApproxDistinct(boolean approx, String sql, String expe .returnsUnordered("EXPR$0=86829"); } + @Disabled("CALCITE-7271") @Test void testComplexExpressionsIsNull() { final String sql = "SELECT COUNT(*) FROM \"foodmart\" where ( cast(null as INTEGER) + cast" + "(\"city\" as INTEGER)) IS NULL"; From e4b818c6052b46f322cd5b969622284132c6111b Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Mon, 17 Nov 2025 23:07:20 +0800 Subject: [PATCH 020/562] [CALCITE-6176] JOIN_SUB_QUERY_TO_CORRELATE rule incorrectly handles EXISTS in LEFT JOIN ON clause --- .../apache/calcite/test/RelOptRulesTest.java | 60 ++++++++ .../apache/calcite/test/RelOptRulesTest.xml | 136 ++++++++++++++++++ core/src/test/resources/sql/join.iq | 62 ++++++++ 3 files changed, 258 insertions(+) diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index 6b2d1533c6f3..92e27614e57e 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -10038,6 +10038,66 @@ public interface Config extends RelRule.Config { .checkUnchanged(); } + /** Test case for + * [CALCITE-6176] + * JOIN_SUB_QUERY_TO_CORRELATE rule incorrectly handles EXISTS in LEFT JOIN ON clause. */ + @Test void testJoinSubQueryRemoveRuleWithNotExists() { + final String sql = "select *\n" + + "from (select 1 id) t1\n" + + "left join (select 2 id) t2\n" + + "on not exists(select *\n" + + " from (select 3 id) p\n" + + " where p.id = t2.id)"; + sql(sql) + .withRule(CoreRules.JOIN_SUB_QUERY_TO_CORRELATE) + .check(); + } + + /** Test case for + * [CALCITE-6176] + * JOIN_SUB_QUERY_TO_CORRELATE rule incorrectly handles EXISTS in LEFT JOIN ON clause. */ + @Test void testJoinSubQueryRemoveRuleWithOrExists() { + final String sql = "select *\n" + + "from (select 1 id) t1\n" + + "left join (select 2 id) t2\n" + + "on t1.id = t2.id or exists(select *\n" + + " from (select 3 id) p\n" + + " where p.id = t2.id)"; + sql(sql) + .withRule(CoreRules.JOIN_SUB_QUERY_TO_CORRELATE) + .check(); + } + + /** Test case for + * [CALCITE-6176] + * JOIN_SUB_QUERY_TO_CORRELATE rule incorrectly handles EXISTS in LEFT JOIN ON clause. */ + @Test void testJoinSubQueryRemoveRuleWithOrNotExists() { + final String sql = "select *\n" + + "from (select 1 id) t1\n" + + "left join (select 2 id) t2\n" + + "on t1.id = t2.id or not exists(select *\n" + + " from (select 3 id) p\n" + + " where p.id = t2.id)"; + sql(sql) + .withRule(CoreRules.JOIN_SUB_QUERY_TO_CORRELATE) + .check(); + } + + /** Test case for + * [CALCITE-6176] + * JOIN_SUB_QUERY_TO_CORRELATE rule incorrectly handles EXISTS in LEFT JOIN ON clause. */ + @Test void testJoinSubQueryRemoveRuleWithAndNotExists() { + final String sql = "select *\n" + + "from (select 1 id union all select 2) t1\n" + + "left join (select 2 id) t2\n" + + "on t1.id = t2.id and not exists(select *\n" + + " from (select 3 id) p\n" + + " where p.id = t1.id)"; + sql(sql) + .withRule(CoreRules.JOIN_SUB_QUERY_TO_CORRELATE) + .check(); + } + /** Test case for * [CALCITE-2295] * Correlated SubQuery with Project will generate error plan. */ diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index 651b6ad52074..0fe14272f3be 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -8293,6 +8293,74 @@ LogicalProject(SAL=[$5]) LogicalProject(SAL=[$5], $f9=[=($5, 4)]) LogicalFilter(condition=[AND(=($7, 20), >($5, 1000))]) LogicalTableScan(table=[[CATALOG, SALES, EMPNULLABLES]]) +]]> + + + + + + + + + + + + + + + + + + + + + + @@ -8325,6 +8393,74 @@ LogicalProject(EMPNO=[$0]) LogicalProject(DEPTNO=[$0], i=[true]) LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) +]]> + + + + + + + + + + + + + + + + + + + + + + diff --git a/core/src/test/resources/sql/join.iq b/core/src/test/resources/sql/join.iq index 1e1f5135e2e9..0c7573baee97 100644 --- a/core/src/test/resources/sql/join.iq +++ b/core/src/test/resources/sql/join.iq @@ -1030,4 +1030,66 @@ natural join "scott".dept order by empno; !ok +# [CALCITE-6176] JOIN_SUB_QUERY_TO_CORRELATE rule incorrectly handles EXISTS in LEFT JOIN ON clause +select * +from (select 1 id) t1 +left join (select 2 id) t2 +on not exists(select * + from (select 3 id) p + where p.id = t2.id); ++----+-----+ +| ID | ID0 | ++----+-----+ +| 1 | 2 | ++----+-----+ +(1 row) + +!ok + +select * +from (select 1 id union all select 2) t1 +left join (select 2 id) t2 +on t1.id = t2.id and not exists(select * + from (select 3 id) p + where p.id = t1.id); ++----+-----+ +| ID | ID0 | ++----+-----+ +| 1 | | +| 2 | 2 | ++----+-----+ +(2 rows) + +!ok + +select * +from (select 1 id) t1 +left join (select 2 id) t2 +on t1.id = t2.id or exists(select * + from (select 3 id) p + where p.id = t2.id); ++----+-----+ +| ID | ID0 | ++----+-----+ +| 1 | | ++----+-----+ +(1 row) + +!ok + +select * +from (select 1 id) t1 +left join (select 2 id) t2 +on t1.id = t2.id or not exists(select * + from (select 3 id) p + where p.id = t2.id); ++----+-----+ +| ID | ID0 | ++----+-----+ +| 1 | 2 | ++----+-----+ +(1 row) + +!ok + # End join.iq From 2102614e2f1794c472a3536b63b0a9d1a9fe3b90 Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Wed, 19 Nov 2025 23:31:22 +0800 Subject: [PATCH 021/562] Replace `replace` with `toLinux` --- .../apache/calcite/test/RelMetadataTest.java | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java b/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java index 12f85ad5c9e1..9213bc23454b 100644 --- a/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java @@ -310,7 +310,7 @@ final RelMetadataFixture sql(String sql) { // Plan is: assertThat( - RelOptUtil.toString(relNode).replace("\r\n", "\n"), + Util.toLinux(RelOptUtil.toString(relNode)), is("" + "LogicalProject(EMPNO=[$0], DEPTNO=[$7], DEPTNOPLUS1=[+($7, 1)], RAND=[RAND()])\n" + " LogicalFilter(condition=[<($7, 20)])\n" @@ -349,7 +349,7 @@ final RelMetadataFixture sql(String sql) { final RelNode relNode = sql(sql).toRel(); assertThat( - RelOptUtil.toString(relNode).replace("\r\n", "\n"), + Util.toLinux(RelOptUtil.toString(relNode)), is("" + "LogicalAggregate(group=[{0}], count=[COUNT()], SUMSAL=[SUM($1)])\n" + " LogicalProject(DEPTNO=[$7], SAL=[$5])\n" @@ -380,7 +380,7 @@ final RelMetadataFixture sql(String sql) { final RelNode relNode = sql(sql).toRel(); assertThat( - RelOptUtil.toString(relNode).replace("\r\n", "\n"), + Util.toLinux(RelOptUtil.toString(relNode)), is("" + "LogicalProject(DEPTNO=[$7], DEPTNO2=[$7], DEPTNO3=[+($7, 1)])\n" + " LogicalTableScan(table=[[CATALOG, SALES, EMP]])\n")); @@ -407,7 +407,7 @@ final RelMetadataFixture sql(String sql) { final RelNode relNode = sql(sql).toRel(); assertThat( - RelOptUtil.toString(relNode).replace("\r\n", "\n"), + Util.toLinux(RelOptUtil.toString(relNode)), is("" + "LogicalProject(EMPNO=[$0], ENAME=[$1], DEPTNO=[$7], NAME=[$10])\n" + " LogicalJoin(condition=[=($7, $9)], joinType=[inner])\n" @@ -445,7 +445,7 @@ final RelMetadataFixture sql(String sql) { final RelNode relNode = sql(sql).toRel(); assertThat( - RelOptUtil.toString(relNode).replace("\r\n", "\n"), + Util.toLinux(RelOptUtil.toString(relNode)), is("" + "LogicalProject(EMPNO=[$0], ENAME=[$1], SAL=[$5])\n" + " LogicalFilter(condition=[>($5, 1000)])\n" @@ -474,7 +474,7 @@ final RelMetadataFixture sql(String sql) { final RelNode relNode = sql(sql).toRel(); assertThat( - RelOptUtil.toString(relNode).replace("\r\n", "\n"), + Util.toLinux(RelOptUtil.toString(relNode)), is("" + "LogicalProject(EMPNO=[$0], SAL=[$5], RAISED_SAL=[*($5, 1.1:DECIMAL(2, 1))]," + " SAL_CATEGORY=[CASE(>($5, 2000), 'high', 'low ')])\n" @@ -505,7 +505,7 @@ final RelMetadataFixture sql(String sql) { final RelNode relNode = sql(sql).toRel(); assertThat( - RelOptUtil.toString(relNode).replace("\r\n", "\n"), + Util.toLinux(RelOptUtil.toString(relNode)), is("" + "LogicalUnion(all=[true])\n" + " LogicalProject(EMPNO=[$0], DEPTNO=[$7])\n" @@ -534,7 +534,7 @@ final RelMetadataFixture sql(String sql) { final RelNode relNode = sql(sql).toRel(); assertThat( - RelOptUtil.toString(relNode).replace("\r\n", "\n"), + Util.toLinux(RelOptUtil.toString(relNode)), is("" + "LogicalProject(EMPNO=[$0], DEPTNO=[$7], COMPOSITE=[+(*($0, 100), $7)])\n" + " LogicalTableScan(table=[[CATALOG, SALES, EMP]])\n")); @@ -559,7 +559,7 @@ final RelMetadataFixture sql(String sql) { final RelNode relNode = sql(sql).toRel(); assertThat( - RelOptUtil.toString(relNode).replace("\r\n", "\n"), + Util.toLinux(RelOptUtil.toString(relNode)), is("" + "LogicalProject(EMPNO=[$0], CONSTANT_COL=[100], DEPTNO=[$7])\n" + " LogicalTableScan(table=[[CATALOG, SALES, EMP]])\n")); @@ -586,7 +586,7 @@ final RelMetadataFixture sql(String sql) { final RelNode relNode = sql(sql).toRel(); assertThat( - RelOptUtil.toString(relNode).replace("\r\n", "\n"), + Util.toLinux(RelOptUtil.toString(relNode)), is("" + "LogicalProject(EMPNO=[$0], RANDOM1=[RAND()], RANDOM2=[RAND()])\n" + " LogicalTableScan(table=[[CATALOG, SALES, EMP]])\n")); @@ -614,7 +614,7 @@ final RelMetadataFixture sql(String sql) { final RelNode relNode = sql(sql).toRel(); assertThat( - RelOptUtil.toString(relNode).replace("\r\n", "\n"), + Util.toLinux(RelOptUtil.toString(relNode)), is("" + "LogicalProject(EMPNO=[$0], DEPTNO=[$7], NAME=[$10])\n" + " LogicalJoin(condition=[=($7, $9)], joinType=[inner])\n" @@ -639,7 +639,7 @@ final RelMetadataFixture sql(String sql) { final RelNode relNode = sql(sql).toRel(); assertThat( - RelOptUtil.toString(relNode).replace("\r\n", "\n"), + Util.toLinux(RelOptUtil.toString(relNode)), is("" + "LogicalProject(EMPNO=[$0], DEPTNO=[$7], NAME=[$10])\n" + " LogicalJoin(condition=[=($7, $9)], joinType=[left])\n" @@ -664,7 +664,7 @@ final RelMetadataFixture sql(String sql) { final RelNode relNode = sql(sql).toRel(); assertThat( - RelOptUtil.toString(relNode).replace("\r\n", "\n"), + Util.toLinux(RelOptUtil.toString(relNode)), is("" + "LogicalProject(EMPNO=[$2], DEPTNO=[$9], NAME=[$1])\n" + " LogicalJoin(condition=[=($9, $0)], joinType=[right])\n" @@ -689,7 +689,7 @@ final RelMetadataFixture sql(String sql) { final RelNode relNode = sql(sql).toRel(); assertThat( - RelOptUtil.toString(relNode).replace("\r\n", "\n"), + Util.toLinux(RelOptUtil.toString(relNode)), is("" + "LogicalProject(EMPNO=[$0], DEPTNO=[$7], NAME=[$10])\n" + " LogicalJoin(condition=[=($7, $9)], joinType=[full])\n" @@ -716,7 +716,7 @@ final RelMetadataFixture sql(String sql) { final RelNode relNode = sql(sql).toRel(); assertThat( - RelOptUtil.toString(relNode).replace("\r\n", "\n"), + Util.toLinux(RelOptUtil.toString(relNode)), is("" + "LogicalSort(sort0=[$0], sort1=[$2], dir0=[ASC], dir1=[ASC])\n" + " LogicalProject(EMPNO=[$0], ENAME=[$1], DEPTNO=[$7], SAL=[$5])\n" @@ -755,7 +755,7 @@ final RelMetadataFixture sql(String sql) { final RelNode relNode = sql(sql).toRel(); assertThat( - RelOptUtil.toString(relNode).replace("\r\n", "\n"), + Util.toLinux(RelOptUtil.toString(relNode)), is("" + "LogicalSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC])\n" + " LogicalProject(D1=[$0], D2=[$1], EMPNO=[$2], ENAME=[$3], JOB=[$4], MGR=[$5]," @@ -796,7 +796,7 @@ final RelMetadataFixture sql(String sql) { final RelNode relNode = sql(sql).toRel(); assertThat( - RelOptUtil.toString(relNode).replace("\r\n", "\n"), + Util.toLinux(RelOptUtil.toString(relNode)), is("" + "LogicalProject(EMPNO=[$0], SAL=[$5], MGR=[$12], DEPTNO=[$16])\n" + " LogicalFilter(condition=[=($0, $5)])\n" @@ -826,7 +826,7 @@ final RelMetadataFixture sql(String sql) { final RelNode relNode = sql(sql).toRel(); assertThat( - RelOptUtil.toString(relNode).replace("\r\n", "\n"), + Util.toLinux(RelOptUtil.toString(relNode)), is("" + "LogicalAggregate(group=[{0, 1}], Z=[SUM($2)])\n" + " LogicalProject(ID1=[$0], ID2=[$1], AGE=[$3])\n" @@ -869,7 +869,7 @@ final RelMetadataFixture sql(String sql) { planner.setRoot(relNode); final RelNode plannedNode = planner.findBestExp(); assertThat( - RelOptUtil.toString(plannedNode).replace("\r\n", "\n"), + Util.toLinux(RelOptUtil.toString(plannedNode)), is("" + "LogicalSort(sort0=[$1], sort1=[$3], sort2=[$4], dir0=[ASC], dir1=[ASC]," + " dir2=[ASC])\n" From 28f245da95745de77854b4bbe5f556ddfc6a61bb Mon Sep 17 00:00:00 2001 From: TJ Banghart Date: Tue, 18 Nov 2025 21:24:44 -0800 Subject: [PATCH 022/562] [CALCITE-7265] Allow RelOptFixture.relFn to be used with VolcanoPlanner --- .../org/apache/calcite/tools/RelBuilder.java | 7 +++++++ .../apache/calcite/test/RelOptRulesTest.java | 19 +++++++++++++++++++ .../apache/calcite/test/RelOptRulesTest.xml | 14 ++++++++++++++ .../org/apache/calcite/test/RelSupplier.java | 6 ++++++ 4 files changed, 46 insertions(+) diff --git a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java index 44a799972720..b8500177c0a6 100644 --- a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java +++ b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java @@ -255,6 +255,13 @@ public static RelBuilder create(FrameworkConfig config) { new RelBuilder(config.getContext(), cluster, relOptSchema)); } + /** Creates a RelBuilder with a given RelOptCluster. */ + public static RelBuilder create(FrameworkConfig config, RelOptCluster existingCluster) { + return Frameworks.withPrepare(config, + (cluster, relOptSchema, rootSchema, statement) -> + new RelBuilder(config.getContext(), existingCluster, relOptSchema)); + } + /** Creates a copy of this RelBuilder, with the same state as this, applying * a transform to the config. */ public RelBuilder transform(UnaryOperator transform) { diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index 92e27614e57e..53e4b5feefa8 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -11491,4 +11491,23 @@ private void checkLoptOptimizeJoinRule(LoptOptimizeJoinRule rule) { }) .check(); } + + /** Test case of + * [CALCITE-7265] + * Verify that relFn works with VolcanoPlanner. */ + @Test void testRelFnWithVolcanoPlanner() { + final Function relFn = b -> + b.scan("EMP") + .filter( + b.call(SqlStdOperatorTable.GREATER_THAN, + b.field("SAL"), + b.literal(2000))) + .build(); + + relFn(relFn) + .withVolcanoPlanner(false, p -> { + RelOptUtil.registerDefaultRules(p, false, false); + }) + .check(); + } } diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index 0fe14272f3be..a9b4bc81a0f6 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -16510,6 +16510,20 @@ LogicalProject(ENAME=[$0], EMPNO=[$1], EMPNO_R=[$3], TYPE=[1]) LogicalProject(ENAME=[$1], EMPNO=[$0], __SOURCE__TYPE__=['bounded']) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) LogicalTableFunctionScan(invocation=[RAMP($cor0.EMPNO)], rowType=[RecordType(INTEGER I)]) +]]> + + + + + ($5, 2000)]) + LogicalTableScan(table=[[scott, EMP]]) +]]> + + + ($5, 2000)]) + EnumerableTableScan(table=[[scott, EMP]]) ]]> diff --git a/testkit/src/main/java/org/apache/calcite/test/RelSupplier.java b/testkit/src/main/java/org/apache/calcite/test/RelSupplier.java index 44a82f54df96..ffcc7bd45f4d 100644 --- a/testkit/src/main/java/org/apache/calcite/test/RelSupplier.java +++ b/testkit/src/main/java/org/apache/calcite/test/RelSupplier.java @@ -16,7 +16,9 @@ */ package org.apache.calcite.test; +import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.plan.RelTraitDef; +import org.apache.calcite.plan.volcano.VolcanoPlanner; import org.apache.calcite.rel.RelNode; import org.apache.calcite.sql.parser.SqlParser; import org.apache.calcite.tools.FrameworkConfig; @@ -128,6 +130,10 @@ private FnRelSupplier(Function relFn) { } @Override public RelNode apply(RelOptFixture fixture) { + if (fixture.planner instanceof VolcanoPlanner) { + RelOptCluster existingCluster = fixture.factory.createSqlToRelConverter().getCluster(); + return relFn.apply(RelBuilder.create(FRAMEWORK_CONFIG, existingCluster)); + } return relFn.apply(RelBuilder.create(FRAMEWORK_CONFIG)); } From b3f2a84fb4ac93c91426eeb59b6fbeb6ad9bf63e Mon Sep 17 00:00:00 2001 From: Thomas Rebele Date: Mon, 17 Nov 2025 19:07:46 +0100 Subject: [PATCH 023/562] [CALCITE-7295] RexSimplify should simplify a division with a NULL argument --- .../org/apache/calcite/rex/RexSimplify.java | 34 ++++++--- .../apache/calcite/rex/RexProgramTest.java | 74 +++++++++++-------- 2 files changed, 66 insertions(+), 42 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rex/RexSimplify.java b/core/src/main/java/org/apache/calcite/rex/RexSimplify.java index 2ad299675fd8..07047311374c 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexSimplify.java +++ b/core/src/main/java/org/apache/calcite/rex/RexSimplify.java @@ -418,17 +418,22 @@ private RexNode simplifyGenericNode(RexCall e) { */ private static int findLiteralIndex(List operands, BigDecimal value) { for (int i = 0; i < operands.size(); i++) { - if (operands.get(i).isA(SqlKind.LITERAL)) { - Comparable comparable = ((RexLiteral) operands.get(i)).getValue(); - if (comparable instanceof BigDecimal - && value.compareTo((BigDecimal) comparable) == 0) { - return i; - } + if (checkLiteralValue(operands.get(i), value)) { + return i; } } return -1; } + /** Check whether the operand is a BigDecimal literal of the specified value. */ + private static boolean checkLiteralValue(RexNode operand, BigDecimal value) { + if (!operand.isA(SqlKind.LITERAL)) { + return false; + } + Comparable comparable = ((RexLiteral) operand).getValue(); + return comparable instanceof BigDecimal && value.compareTo((BigDecimal) comparable) == 0; + } + private RexNode simplifyArithmetic(RexCall e) { if (e.getType().getSqlTypeName().getFamily() != SqlTypeFamily.NUMERIC || e.getOperands().stream().anyMatch( @@ -491,8 +496,8 @@ private RexNode simplifyMultiply(RexCall e) { } private RexNode simplifyDivide(RexCall e) { - final int oneIndex = findLiteralIndex(e.operands, BigDecimal.ONE); - if (oneIndex == 1) { + RexNode rightOperand = e.getOperands().get(1); + if (checkLiteralValue(rightOperand, BigDecimal.ONE)) { RexNode leftOperand = e.getOperands().get(0); return leftOperand.getType().equals(e.getType()) ? leftOperand : rexBuilder.makeCast(e.getParserPosition(), e.getType(), leftOperand); @@ -1561,14 +1566,19 @@ enum SafeRexVisitor implements RexVisitor { case DIVIDE: case MOD: List operands = call.getOperands(); - boolean isSafe = RexVisitorImpl.visitArrayAnd(this, ImmutableList.of(operands.get(0))); - if (!isSafe) { + boolean areOperandsSafe = RexVisitorImpl.visitArrayAnd(this, call.operands); + if (!areOperandsSafe) { return false; } + boolean hasNullOperand = RexUtil.isNullLiteral(operands.get(0), true) + || RexUtil.isNullLiteral(operands.get(1), true); + if (hasNullOperand) { + return true; + } if (operands.get(1) instanceof RexLiteral) { - RexLiteral literal = (RexLiteral) operands.get(1); - return RexUtil.isNullLiteral(literal, true); + return !checkLiteralValue(operands.get(1), BigDecimal.ZERO); } + // the safety of division could not be deduced, so assume it is unsafe return false; default: break; diff --git a/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java b/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java index 91f55624e8b5..fc769f966cfe 100644 --- a/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java +++ b/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java @@ -2712,48 +2712,37 @@ trueLiteral, literal(1), * [CALCITE-7032] * Simplify 'NULL > ALL (ARRAY[1,2,NULL])' to 'NULL'. */ @Test void testSimplifyDivideSafe() { - // null + (a/0)/4 - // ==> - // null + (a/0)/4 + // null + (a/0)/4 ==> null + (a/0)/4 // a/0 throws an exception so it may not be simplified RexNode divideNode0 = plus(nullInt, div(div(vIntNotNull(), literal(0)), literal(4))); checkSimplifyUnchanged(divideNode0); - // null + a/4 - // ==> - // null + a/4 + // null + a/4 ==> null RexNode divideNode1 = plus(nullInt, div(vIntNotNull(), literal(4))); - checkSimplifyUnchanged(divideNode1); - // null + a/null - // ==> - // null + checkSimplify(divideNode1, "null:INTEGER"); + // null + a/null ==> null RexNode divideNode2 = plus(nullInt, div(vIntNotNull(), nullInt)); checkSimplify(divideNode2, "null:INTEGER"); - // null + null/0 - // ==> - // null + null/0 + // null + null/0 ==> null + // The SQL standard gives NULL a higher priority than division by 0. + // This is the same behavior as PostgreSQL, Oracle, SQLite, MariaDB, and MySQL. RexNode divideNode3 = plus(nullInt, div(nullInt, literal(0))); - checkSimplifyUnchanged(divideNode3); - // null + a/0 - // ==> - // null + a/0 + checkSimplify(divideNode3, "null:INTEGER"); + // null + a/0 ==> null + a/0 RexNode divideNode4 = plus(nullInt, div(vIntNotNull(), literal(0))); checkSimplifyUnchanged(divideNode4); - // null + a/b - // ==> - // null + a/b + // null + a/b ==> null + a/b // b might be 0 and throw an exception, so a/b cannot be simplified. // E.g., PostgreSQL throws a division-by-zero error for the following query: // SELECT NULL + (a/b) FROM (select 1 as a, 0 as b) t RexNode divideNode5 = plus(nullInt, div(vIntNotNull(), vIntNotNull())); checkSimplifyUnchanged(divideNode5); - // null/(1/0) - // ==> - // null/(1/0) + // (1/0) + (null/0) ==> (1/0) + null + RexNode divideNode6 = plus(div(literal(1), literal(0)), div(nullInt, literal(0))); + checkSimplify(divideNode6, "+(/(1, 0), null)"); + // null/(1/0) ==> null/(1/0) RexNode divideNode7 = div(nullInt, div(literal(1), literal(0))); checkSimplifyUnchanged(divideNode7); - // (1/0)/null - // ==> - // (1/0)/null + // (1/0)/null ==> (1/0)/null RexNode divideNode8 = div(div(literal(1), literal(0)), nullInt); checkSimplifyUnchanged(divideNode8); } @@ -2762,25 +2751,50 @@ trueLiteral, literal(1), * See [CALCITE-7145] * RexSimplify should not simplify IS NULL(10/0). */ @Test void testSimplifyIsNullDivide() { - RelDataType intType = - typeFactory.createTypeWithNullability( - typeFactory.createSqlType(SqlTypeName.INTEGER), false); + RelDataType intType = tInt(false); checkSimplifyUnchanged(isNull(div(vIntNotNull(), literal(0)))); checkSimplifyUnchanged(isNull(div(vIntNotNull(), cast(literal(0), intType)))); + checkSimplify(isNull(div(vIntNotNull(), cast(literal(2), intType))), "false"); checkSimplifyUnchanged(isNull(div(cast(literal(2), intType), vIntNotNull()))); checkSimplifyUnchanged(isNull(div(vIntNotNull(), vIntNotNull()))); checkSimplifyUnchanged(isNotNull(div(vIntNotNull(), literal(0)))); checkSimplifyUnchanged(isNotNull(div(vIntNotNull(), cast(literal(0), intType)))); + checkSimplify(isNotNull(div(vIntNotNull(), cast(literal(2), intType))), "true"); checkSimplifyUnchanged(isNotNull(div(cast(literal(2), intType), vIntNotNull()))); checkSimplifyUnchanged(isNotNull(div(vIntNotNull(), vIntNotNull()))); checkSimplifyUnchanged(isNull(div(vDecimalNotNull(), literal(0)))); + checkSimplify( + isNull(div(vDecimalNotNull(), cast(literal(BigDecimal.valueOf(2.5)), intType))), + "false"); checkSimplifyUnchanged(isNotNull(div(vDecimalNotNull(), literal(0)))); + checkSimplify( + isNotNull(div(vDecimalNotNull(), cast(literal(BigDecimal.valueOf(2.5)), intType))), + "true"); + } + + /** + * Test cases for [CALCITE-7295] + * RexSimplify should simplify a division with a NULL argument. + */ + @Test void testSimplifyIsNullDivideWithNullArgument() { + checkSimplify(isNull(div(nullInt, literal(0))), "true"); + checkSimplify(isNull(div(literal(0), nullInt)), "true"); + + checkSimplify(isNotNull(div(nullInt, literal(0))), "false"); + checkSimplify(isNotNull(div(literal(0), nullInt)), "false"); + + checkSimplify(isNull(div(nullDecimal, literal(BigDecimal.ZERO))), "true"); + checkSimplify(isNotNull(div(nullDecimal, literal(BigDecimal.ZERO))), "false"); + + // do not simplify if one of the operands may throw an exception + checkSimplifyUnchanged(div(nullInt, cast(vVarchar(), tInt(false)))); + checkSimplifyUnchanged(div(cast(vVarchar(), tInt(false)), nullInt)); } @Test void testPushNotIntoCase() { @@ -2996,7 +3010,6 @@ private SqlOperator getNoDeterministicOperator() { checkSimplifyUnchanged(isNotNull(cast(vVarchar(), tVarbinary(true)))); } - @Test void checkSimplifyDynamicParam() { checkSimplify(isNotNull(lt(vInt(0), vInt(1))), "AND(IS NOT NULL(?0.int0), IS NOT NULL(?0.int1))"); @@ -4291,6 +4304,7 @@ private SqlSpecialOperatorWithPolicy(String name, SqlKind kind, int prec, boolea checkSimplify(div(nullInt, one), "null:INTEGER"); checkSimplify(div(a, nullInt), "null:INTEGER"); checkSimplify(div(zero, nullInt), "null:INTEGER"); + checkSimplify(div(nullInt, zero), "null:INTEGER"); checkSimplify(add(b, half), "?0.notNullDecimal2"); From 894d56408b78d94819312be926442567996144e8 Mon Sep 17 00:00:00 2001 From: xuzifu666 Date: Wed, 12 Nov 2025 20:22:30 +0800 Subject: [PATCH 024/562] [CALCITE-7287] In simplifyLike, the makeLiteral call does not preserve the RelDataType --- core/src/main/java/org/apache/calcite/rex/RexSimplify.java | 7 ++++--- .../java/org/apache/calcite/rex/RexProgramBuilderBase.java | 7 +++++++ .../test/java/org/apache/calcite/rex/RexProgramTest.java | 7 +++++++ 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rex/RexSimplify.java b/core/src/main/java/org/apache/calcite/rex/RexSimplify.java index 07047311374c..3b0b04043b51 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexSimplify.java +++ b/core/src/main/java/org/apache/calcite/rex/RexSimplify.java @@ -522,15 +522,16 @@ private RexNode simplifyLike(RexCall e, RexUnknownAs unknownAs) { if (e.operands.size() == 2) { e = (RexCall) rexBuilder .makeCall(e.getParserPosition(), e.getOperator(), e.operands.get(0), - rexBuilder.makeLiteral(simplifyLikeString(likeStr, '\\', '%'))); + rexBuilder.makeLiteral(simplifyLikeString(likeStr, '\\', '%'), + e.operands.get(1).getType(), true, true)); } if (e.operands.size() == 3 && e.operands.get(2) instanceof RexLiteral) { final RexLiteral escapeLiteral = (RexLiteral) e.operands.get(2); Character escape = requireNonNull(escapeLiteral.getValueAs(Character.class)); e = (RexCall) rexBuilder .makeCall(e.getParserPosition(), e.getOperator(), e.operands.get(0), - rexBuilder.makeLiteral(simplifyLikeString(likeStr, escape, '%')), - escapeLiteral); + rexBuilder.makeLiteral(simplifyLikeString(likeStr, escape, '%'), + e.operands.get(1).getType(), true, true), escapeLiteral); } } return simplifyGenericNode(e); diff --git a/core/src/test/java/org/apache/calcite/rex/RexProgramBuilderBase.java b/core/src/test/java/org/apache/calcite/rex/RexProgramBuilderBase.java index b0286bd73a2a..4b873fce5c82 100644 --- a/core/src/test/java/org/apache/calcite/rex/RexProgramBuilderBase.java +++ b/core/src/test/java/org/apache/calcite/rex/RexProgramBuilderBase.java @@ -539,6 +539,13 @@ protected RexLiteral literal(String value) { return rexBuilder.makeLiteral(value, nonNullableVarchar); } + protected RexLiteral literalVarchar(String value) { + if (value == null) { + return rexBuilder.makeNullLiteral(nullableVarchar); + } + return (RexLiteral) rexBuilder.makeLiteral(value, nonNullableVarchar, true, true); + } + protected RexLiteral literal(double value) { return rexBuilder.makeApproxLiteral(value, nonNullableDouble); } diff --git a/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java b/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java index fc769f966cfe..5e727d300521 100644 --- a/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java +++ b/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java @@ -4092,6 +4092,13 @@ private void checkSarg(String message, Sarg sarg, checkSimplify(or(isNull(ref), like(ref, literal("%"), literal("#"))), "true"); checkSimplifyUnchanged(like(ref, literal("%A"))); + + RexCall simplifyLike = (RexCall) simplify.simplify(like(ref, literalVarchar("%%A"))); + assertThat(simplifyLike.getOperands().get(1).getType().getSqlTypeName().name(), is("VARCHAR")); + + simplifyLike = (RexCall) simplify.simplify(like(ref, literal("%%A"))); + assertThat(simplifyLike.getOperands().get(1).getType().getSqlTypeName().name(), is("CHAR")); + checkSimplify(like(ref, literal("%%A")), "LIKE($0, '%A')"); checkSimplify(like(ref, literal("%%%_A%%B%%")), "LIKE($0, '_%A%B%')"); checkSimplify(like(ref, literal("%%A%%%")), "LIKE($0, '%A%')"); From 75bddbae9d67dc580ba2ff8723228853b6bb4a5e Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Sat, 22 Nov 2025 10:19:13 +0800 Subject: [PATCH 025/562] [CALCITE-5465] Rule of AGGREGATE_EXPAND_DISTINCT_AGGREGATES produces an incorrect plan when sql has distinct agg-call with rollup --- ...AggregateExpandDistinctAggregatesRule.java | 375 +++++++++++++++--- .../apache/calcite/test/RelOptRulesTest.java | 39 ++ .../apache/calcite/test/RelOptRulesTest.xml | 164 ++++++-- core/src/test/resources/sql/agg.iq | 135 ++++++- core/src/test/resources/sql/sub-query.iq | 8 +- .../apache/calcite/test/SparkAdapterTest.java | 8 +- 6 files changed, 612 insertions(+), 117 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rel/rules/AggregateExpandDistinctAggregatesRule.java b/core/src/main/java/org/apache/calcite/rel/rules/AggregateExpandDistinctAggregatesRule.java index 0c17d87d719c..b25eceb38e3c 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/AggregateExpandDistinctAggregatesRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/AggregateExpandDistinctAggregatesRule.java @@ -31,7 +31,6 @@ import org.apache.calcite.rex.RexInputRef; import org.apache.calcite.rex.RexNode; import org.apache.calcite.runtime.PairList; -import org.apache.calcite.sql.SqlAggFunction; import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.fun.SqlSumEmptyIsZeroAggFunction; @@ -52,6 +51,7 @@ import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; @@ -61,6 +61,7 @@ import java.util.NavigableSet; import java.util.Set; import java.util.TreeSet; +import java.util.function.BiConsumer; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -425,39 +426,116 @@ private static RelBuilder convertSingletonDistinct(RelBuilder relBuilder, return relBuilder; } + /** + * Rewrite aggregates that use GROUPING SETS. The following SQL/plan example + * serves as the concrete blueprint, starting from the original statement and + * plan-before outputs and then rebuilding the plan-after tree from the bottom + * (line 7) back to the top (line 1): + * + *

    Original SQL: + * 

    {@code
+   * SELECT deptno, COUNT(DISTINCT sal)
+   * FROM emp
+   * GROUP BY ROLLUP(deptno)
+   * }
    + * + *

    Plan before rewrite: + * 

    {@code
+   * LogicalAggregate(group=[{0}], groups=[[{0}, {}]], EXPR$1=[COUNT(DISTINCT $1)])
+   *   LogicalProject(DEPTNO=[$7], SAL=[$5])
+   *     LogicalTableScan(table=[[CATALOG, SALES, EMP]])
+   * }
    + * + *

    Plan after rewrite (lines referenced below): + * 

    {@code
+   * 1 LogicalProject(DEPTNO=[$0],
+   *     EXPR$1=[CAST(CASE(=($5, 0), $1, =($5, 1), $2, null:BIGINT)):BIGINT NOT NULL])
+   * 2  LogicalFilter(condition=[OR(AND(=($5, 0), >($3, 0)), =($5, 1))])
+   * 3    LogicalAggregate(group=[{0}], groups=[[{0}, {}]],
+   *          EXPR$1_g0=[COUNT($1) FILTER $2],
+   *          EXPR$1_g1=[COUNT($1) FILTER $4],
+   *          $g_present_0=[COUNT() FILTER $3],
+   *          $g_present_1=[COUNT() FILTER $5],
+   *          $g_final=[GROUPING($0)])
+   * 4      LogicalProject(DEPTNO=[$0], SAL=[$1],
+   *            $g_0=[=($2, 0)], $g_1=[=($2, 1)],
+   *            $g_2=[=($2, 2)], $g_3=[=($2, 3)])
+   * 5        LogicalAggregate(group=[{0, 1}],
+   *              groups=[[{0, 1}, {0}, {1}, {}]], $g=[GROUPING($0, $1)])
+   * 6          LogicalProject(DEPTNO=[$7], SAL=[$5])
+   * 7            LogicalTableScan(table=[[CATALOG, SALES, EMP]])
+   * }
    + * + *

    The method performs the following actions: + *

      + *
    • Reuse the incoming scan and projection (lines 7 and 6) by pushing the + * original aggregate input onto the builder.
      • + *
    • Enumerate all grouping-set combinations and run the "bottom" aggregate + * over {@code fullGroupSet} to materialize line 5, including the internal + * {@code GROUPING()} value.
      • + *
    • Project the boolean selector columns that compare {@code GROUPING()} + * outputs to the required combinations, which surfaces line 4.
      • + *
    • Build the "upper" grouping-set aggregates with per-set FILTER clauses, + * reproducing line 3 and retaining presence counters / grouping ids.
      • + *
    • Assemble {@code keepConditions} so we can emit the filter of line 2 that + * drops internal-only rows.
      • + *
    • Produce the final projection (line 1) that routes each aggregate result + * to the user-visible columns.
      • + *
    + */ private static void rewriteUsingGroupingSets(RelOptRuleCall call, Aggregate aggregate) { + final ImmutableBitSet aggregateGroupSet = aggregate.getGroupSet(); + final ImmutableList aggregateGroupingSets = aggregate.getGroupSets(); + final Set groupSetTreeSet = new TreeSet<>(ImmutableBitSet.ORDERING); - // GroupSet to distinct filter arg map, - // filterArg will be -1 for non-distinct agg call. - // Using `Set` here because it's possible that two agg calls - // have different filterArgs but same groupSet. + // Map from a set of group keys -> which filter args (if any) contributed + // to that combination. Used to generate boolean marker columns later which + // indicate whether a bottom-row should be considered for a particular + // (grouping-set, filter) combination. final Map> distinctFilterArgMap = new HashMap<>(); + + // Enumerating every required grouping-set combination, including distinct + // args or filter columns relied on by the downstream projection(line 4). + BiConsumer addGroupSet = (groupSet, filterArg) -> { + groupSetTreeSet.add(groupSet); + distinctFilterArgMap.computeIfAbsent(groupSet, g -> new HashSet<>()).add(filterArg); + }; + + // Always include the base group set and each declared grouping set. -1 means "no filter". + addGroupSet.accept(aggregateGroupSet, -1); + for (ImmutableBitSet groupingSet : aggregateGroupingSets) { + addGroupSet.accept(groupingSet, -1); + } + + // For each DISTINCT aggregate, include grouping-set combinations: + // (distinct args) ∪ (grouping set). This ensures we compute bottom rows + // at a granularity sufficient to evaluate DISTINCT per grouping set. If + // the DISTINCT agg has a FILTER, include that filter column in the + // grouping so that the downstream boolean selector can distinguish + // filtered rows. For example, with COUNT(DISTINCT sal) and grouping sets + // (deptno) and (), we add {deptno, sal} and {sal}. for (AggregateCall aggCall : aggregate.getAggCallList()) { - ImmutableBitSet groupSet; - int filterArg; if (!aggCall.isDistinct()) { - filterArg = -1; - groupSet = aggregate.getGroupSet(); - groupSetTreeSet.add(aggregate.getGroupSet()); - } else { - filterArg = aggCall.filterArg; - groupSet = - ImmutableBitSet.of(aggCall.getArgList()) - .setIf(filterArg, filterArg >= 0) - .union(aggregate.getGroupSet()); - groupSetTreeSet.add(groupSet); + continue; + } + final ImmutableBitSet args = ImmutableBitSet.of(aggCall.getArgList()); + for (ImmutableBitSet groupingSet : aggregateGroupingSets) { + ImmutableBitSet groupSet = args.union(groupingSet); + if (aggCall.filterArg >= 0) { + groupSet = groupSet.set(aggCall.filterArg); + } + addGroupSet.accept(groupSet, aggCall.filterArg); } - Set filterList = distinctFilterArgMap - .computeIfAbsent(groupSet, g -> new HashSet<>()); - filterList.add(filterArg); } final ImmutableList groupSets = ImmutableList.copyOf(groupSetTreeSet); + // fullGroupSet is the union of all bits that appear in any grouping set. final ImmutableBitSet fullGroupSet = ImmutableBitSet.union(groupSets); + // Whether the bottom aggregate must account for an "empty" grouping set. final boolean bottomHasEmptyGroup = groupSets.contains(ImmutableBitSet.of()); final List distinctAggCalls = new ArrayList<>(); @@ -472,12 +550,19 @@ private static void rewriteUsingGroupingSets(RelOptRuleCall call, } final RelBuilder relBuilder = call.builder(); + final RexBuilder rexBuilder = aggregate.getCluster().getRexBuilder(); + // Lines 7 & 6: reuse the existing scan+projection feeding the original aggregate. relBuilder.push(aggregate.getInput()); - final int groupCount = fullGroupSet.cardinality(); - - // Get the base ordinal of filter args for different groupSets. + final int bottomGroupCount = fullGroupSet.cardinality(); + + // Map each (groupSet, filterArg) pair to an output field index in the + // bottom projection. These fields become boolean/marker columns used to + // implement FILTER(...) and to detect whether a grouping set had rows. + // The numbering starts after the bottom group fields and the distinct + // aggregate columns; 'z' is the running output field index for these + // selector markers. final Map, Integer> filters = new LinkedHashMap<>(); - int z = groupCount + distinctAggCalls.size(); + int z = bottomGroupCount + distinctAggCalls.size(); for (ImmutableBitSet groupSet : groupSets) { Set filterArgList = distinctFilterArgMap.get(groupSet); for (Integer filterArg : requireNonNull(filterArgList, "filterArgList")) { @@ -492,12 +577,14 @@ private static void rewriteUsingGroupingSets(RelOptRuleCall call, null, RelCollations.EMPTY, bottomHasEmptyGroup, relBuilder.peek(), null, "$g")); + // Line 5: bottom aggregate materializes every grouping-set combination and + // produces the GROUPING() value needed by later steps. relBuilder.aggregate( relBuilder.groupKey(fullGroupSet, groupSets), distinctAggCalls); - // GROUPING returns an integer (0 or 1). Add a project to convert those - // values to BOOLEAN. + // Line 4: convert GROUPING() into named selector columns ($g_*) that pick + // rows for each grouping-set/filter combination. if (!filters.isEmpty()) { final List nodes = new ArrayList<>(relBuilder.fields()); final RexNode nodeZ = nodes.remove(nodes.size() - 1); @@ -520,43 +607,208 @@ private static void rewriteUsingGroupingSets(RelOptRuleCall call, relBuilder.project(nodes); } - int x = groupCount; - final ImmutableBitSet groupSet = aggregate.getGroupSet(); - final List newCalls = new ArrayList<>(); - for (AggregateCall aggCall : aggregate.getAggCallList()) { - final int newFilterArg; - final List newArgList; - final SqlAggFunction aggregation; + // Compute the remapped top-group key and grouping sets. The top-group key + // selects which fields of the bottom result correspond to the original + // aggregate's group-by columns. Upper aggregates(line 3) will group by this key. + final ImmutableBitSet topGroupKey = remap(fullGroupSet, aggregateGroupSet); + final ImmutableList topGroupingSets = + remap(fullGroupSet, aggregate.getGroupSets()); + final int topGroupCount = topGroupKey.cardinality(); + final boolean needsGroupingIndicators = aggregate.getGroupType() != Group.SIMPLE; + final List groupingIndicatorOrdinals; + if (needsGroupingIndicators) { + groupingIndicatorOrdinals = + new ArrayList<>(Collections.nCopies(aggregateGroupingSets.size(), -1)); + } else { + groupingIndicatorOrdinals = ImmutableList.of(); + } + + int valueIndex = bottomGroupCount; + // line 3 will be built from this list + final List upperAggCalls = new ArrayList<>(); + final List> aggCallOrdinals = new ArrayList<>(); + final List aggCalls = aggregate.getAggCallList(); + + // The first part of line 3: Build upper aggregates per declared grouping set. + // For each original aggCall we create one upper agg per declared grouping set. + // The upper aggregate groups by {@code topGroupKey} and uses the boolean marker + // columns (placed at known ordinals) as the FILTER argument for the + // corresponding per-group aggregation. The list {@code aggCallOrdinals} + // records, for each original aggCall, the output field ordinals of the + // corresponding upper-aggregate results (one per grouping set). + for (AggregateCall aggCall : aggCalls) { + final List ordinals = new ArrayList<>(); if (!aggCall.isDistinct()) { - aggregation = SqlStdOperatorTable.MIN; - newArgList = ImmutableIntList.of(x++); - newFilterArg = - requireNonNull(filters.get(Pair.of(groupSet, -1)), - "filters.get(Pair.of(groupSet, -1))"); + final int inputIndex = valueIndex++; + final List args = ImmutableIntList.of(inputIndex); + for (int g = 0; g < aggregateGroupingSets.size(); g++) { + final ImmutableBitSet groupingSet = aggregateGroupingSets.get(g); + final int newFilterArg = + requireNonNull(filters.get(Pair.of(groupingSet, -1)), + () -> "filters.get(" + groupingSet + ", -1)"); + final String upperAggName = upperAggCallName(aggCall, g); + // Each filtered grouping set emits exactly one row per group, + // so MIN just passes that value through without re-aggregation + final AggregateCall newCall = + AggregateCall.create(aggCall.getParserPosition(), + SqlStdOperatorTable.MIN, false, aggCall.isApproximate(), + aggCall.ignoreNulls(), aggCall.rexList, args, newFilterArg, + aggCall.distinctKeys, aggCall.collation, aggregate.hasEmptyGroup(), + relBuilder.peek(), null, upperAggName); + upperAggCalls.add(newCall); + ordinals.add(topGroupCount + upperAggCalls.size() - 1); + } } else { - aggregation = aggCall.getAggregation(); - newArgList = remap(fullGroupSet, aggCall.getArgList()); - final ImmutableBitSet newGroupSet = ImmutableBitSet.of(aggCall.getArgList()) - .setIf(aggCall.filterArg, aggCall.filterArg >= 0) - .union(groupSet); - newFilterArg = - requireNonNull(filters.get(Pair.of(newGroupSet, aggCall.filterArg)), - "filters.get(of(newGroupSet, aggCall.filterArg))"); + final List newArgList = remap(fullGroupSet, aggCall.getArgList()); + for (int g = 0; g < aggregateGroupingSets.size(); g++) { + final ImmutableBitSet groupingSet = aggregateGroupingSets.get(g); + final ImmutableBitSet newGroupSet = ImmutableBitSet.of(aggCall.getArgList()) + .setIf(aggCall.filterArg, aggCall.filterArg >= 0) + .union(groupingSet); + final int newFilterArg = + requireNonNull(filters.get(Pair.of(newGroupSet, aggCall.filterArg)), + () -> "filters.get(" + newGroupSet + ", " + aggCall.filterArg + ")"); + final String upperAggName = upperAggCallName(aggCall, g); + final AggregateCall newCall = + AggregateCall.create(aggCall.getParserPosition(), aggCall.getAggregation(), false, + aggCall.isApproximate(), aggCall.ignoreNulls(), + aggCall.rexList, newArgList, newFilterArg, + aggCall.distinctKeys, aggCall.collation, + aggregate.hasEmptyGroup(), relBuilder.peek(), null, upperAggName); + upperAggCalls.add(newCall); + ordinals.add(topGroupCount + upperAggCalls.size() - 1); + } } - final AggregateCall newCall = - AggregateCall.create(aggCall.getParserPosition(), aggregation, false, - aggCall.isApproximate(), aggCall.ignoreNulls(), - aggCall.rexList, newArgList, newFilterArg, - aggCall.distinctKeys, aggCall.collation, - aggregate.hasEmptyGroup(), relBuilder.peek(), null, aggCall.name); - newCalls.add(newCall); + aggCallOrdinals.add(ordinals); } + // The second part of line 3: If grouping indicators are needed + // (ROLLUP/CUBE/GROUPING SETS with more than one grouping set), add + // COUNT(...) presence calls which are later used to determine whether + // a grouping set produced any rows. These calls implement the + // semantics where empty grouping sets must still produce a result. + if (needsGroupingIndicators) { + for (int g = 0; g < aggregateGroupingSets.size(); g++) { + final ImmutableBitSet groupingSet = aggregateGroupingSets.get(g); + final Integer filterField = filters.get(Pair.of(groupingSet, -1)); + if (filterField == null) { + continue; + } + final AggregateCall presenceCall = + AggregateCall.create(SqlStdOperatorTable.COUNT, false, false, false, + ImmutableList.of(), ImmutableIntList.of(), filterField, null, + RelCollations.EMPTY, aggregate.hasEmptyGroup(), relBuilder.peek(), null, + "$g_present_" + g); + upperAggCalls.add(presenceCall); + groupingIndicatorOrdinals.set(g, topGroupCount + upperAggCalls.size() - 1); + } + } + + // The third part of line 3: If there are multiple declared grouping sets, + // then we need a GROUPING() value in the upper aggregate so we can later + // route results to the correct output using CASE expressions. Compute and + // append that grouping-call if required. + final boolean needsGroupingId = aggregateGroupingSets.size() > 1; + final int groupingIdOrdinal; + if (needsGroupingId) { + final ImmutableBitSet remappedGroupSet = remap(fullGroupSet, aggregateGroupSet); + final AggregateCall groupingCall = + AggregateCall.create(SqlStdOperatorTable.GROUPING, false, false, false, + ImmutableList.of(), ImmutableIntList.copyOf(remappedGroupSet.asList()), -1, null, + RelCollations.EMPTY, aggregate.hasEmptyGroup(), relBuilder.peek(), null, "$g_final"); + upperAggCalls.add(groupingCall); + groupingIdOrdinal = topGroupCount + upperAggCalls.size() - 1; + } else { + groupingIdOrdinal = -1; + } + + // The final part of line 3: build the upper aggregate layer, grouping by the + // original keys and applying FILTERs (and presence/grouping columns) per + // declared set. relBuilder.aggregate( - relBuilder.groupKey( - remap(fullGroupSet, groupSet), - remap(fullGroupSet, aggregate.getGroupSets())), - newCalls); + relBuilder.groupKey(topGroupKey, topGroupingSets), + upperAggCalls); + + final ImmutableList groupingIdColumns = + ImmutableList.copyOf(Util.range(topGroupCount)); + final RexNode groupingIdRef = needsGroupingId ? relBuilder.field(groupingIdOrdinal) : null; + + if (needsGroupingIndicators) { + final List keepConditions = new ArrayList<>(); + for (int g = 0; g < aggregateGroupingSets.size(); g++) { + final int indicatorOrdinal = groupingIndicatorOrdinals.get(g); + if (indicatorOrdinal < 0) { + continue; + } + final ImmutableBitSet groupingSet = aggregateGroupingSets.get(g); + final RexNode requiredRows; + if (groupingSet.isEmpty()) { + // Empty grouping sets must still produce a row even if the input is + // empty, so do not require any contributing tuples. + requiredRows = relBuilder.literal(true); + } else { + requiredRows = + relBuilder.greaterThan(relBuilder.field(indicatorOrdinal), + relBuilder.literal(0)); + } + + final RexNode groupingMatches; + if (needsGroupingId) { + final long groupingValue = + groupValue(groupingIdColumns, remap(aggregateGroupSet, groupingSet)); + groupingMatches = + relBuilder.equals(requireNonNull(groupingIdRef, "groupingIdRef"), + relBuilder.literal(groupingValue)); + } else { + groupingMatches = relBuilder.literal(true); + } + keepConditions.add(relBuilder.and(groupingMatches, requiredRows)); + } + + // Line 2: filter away rows produced solely for internal combinations. + if (!keepConditions.isEmpty()) { + RexNode condition = keepConditions.get(0); + for (int i = 1; i < keepConditions.size(); i++) { + condition = relBuilder.or(condition, keepConditions.get(i)); + } + relBuilder.filter(condition); + } + } + + // Assemble the projections for line 1 here + final List projects = new ArrayList<>(); + final List finalFieldNames = aggregate.getRowType().getFieldNames(); + for (int i = 0; i < topGroupCount; i++) { + projects.add(relBuilder.field(i)); + } + + for (int i = 0; i < aggCalls.size(); i++) { + final AggregateCall aggCall = aggCalls.get(i); + final List ordinals = aggCallOrdinals.get(i); + if (!needsGroupingId || ordinals.size() == 1) { + projects.add(relBuilder.field(ordinals.get(0))); + continue; + } + + final List caseOperands = new ArrayList<>(); + for (int g = 0; g < aggregateGroupingSets.size(); g++) { + final ImmutableBitSet groupingSet = aggregateGroupingSets.get(g); + final long groupingValue = + groupValue(groupingIdColumns, remap(aggregateGroupSet, groupingSet)); + caseOperands.add( + relBuilder.equals(requireNonNull(groupingIdRef, "groupingIdRef"), + relBuilder.literal(groupingValue))); + caseOperands.add(relBuilder.field(ordinals.get(g))); + } + caseOperands.add(rexBuilder.makeNullLiteral(aggCall.getType())); + projects.add( + relBuilder.call(SqlStdOperatorTable.CASE, + caseOperands.toArray(new RexNode[0]))); + } + + // Line 1: final projection routes per-set aggregates back into the original + // output schema (including CASE routing when needed). + relBuilder.project(projects, finalFieldNames); relBuilder.convert(aggregate.getRowType(), true); call.transformTo(relBuilder.build()); } @@ -612,6 +864,15 @@ private static int remap(ImmutableBitSet groupSet, int arg) { return arg < 0 ? -1 : groupSet.indexOf(arg); } + private static String upperAggCallName(AggregateCall aggCall, + int groupingSetIndex) { + String baseName = aggCall.getName(); + if (baseName == null || baseName.isEmpty()) { + baseName = aggCall.getAggregation().getName(); + } + return baseName + "_g" + groupingSetIndex; + } + /** * Converts an aggregate relational expression that contains just one * distinct aggregate function (or perhaps several over the same arguments) diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index 53e4b5feefa8..4369727f3fab 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -2395,6 +2395,45 @@ private void checkSemiOrAntiJoinProjectTranspose(JoinRelType type) { .check(); } + /** Test case for + * [CALCITE-5465] + * Rule of AGGREGATE_EXPAND_DISTINCT_AGGREGATES produces an incorrect plan + * when sql has distinct agg-call with rollup. */ + @Test void testDistinctNonDistinctAggregatesWithGroupingSets() { + final String sql = "SELECT deptno, COUNT(DISTINCT sal)\n" + + "FROM emp\n" + + "GROUP BY ROLLUP(deptno)"; + sql(sql) + .withRule(CoreRules.AGGREGATE_EXPAND_DISTINCT_AGGREGATES) + .check(); + } + + /** Test case for + * [CALCITE-5465] + * Rule of AGGREGATE_EXPAND_DISTINCT_AGGREGATES produces an incorrect plan + * when sql has distinct agg-call with rollup. */ + @Test void testDistinctNonDistinctAggregatesWithGroupingSets2() { + final String sql = "SELECT deptno, COUNT(DISTINCT sal), SUM(sal)\n" + + "FROM emp\n" + + "GROUP BY GROUPING SETS ((deptno), ())"; + sql(sql) + .withRule(CoreRules.AGGREGATE_EXPAND_DISTINCT_AGGREGATES) + .check(); + } + + /** Test case for + * [CALCITE-5465] + * Rule of AGGREGATE_EXPAND_DISTINCT_AGGREGATES produces an incorrect plan + * when sql has distinct agg-call with rollup. */ + @Test void testDistinctNonDistinctAggregatesWithGroupingSets3() { + final String sql = "SELECT deptno, COUNT(DISTINCT sal), SUM(DISTINCT sal), COUNT(*)\n" + + "FROM emp\n" + + "GROUP BY GROUPING SETS ((deptno), ())"; + sql(sql) + .withRule(CoreRules.AGGREGATE_EXPAND_DISTINCT_AGGREGATES) + .check(); + } + @Test void testDistinctNonDistinctAggregates() { final String sql = "select emp.empno, count(*), avg(distinct dept.deptno)\n" + "from sales.emp emp inner join sales.dept dept\n" diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index a9b4bc81a0f6..c63602eae693 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -1800,9 +1800,9 @@ LogicalAggregate(group=[{0}], EXPR$1=[SUM(DISTINCT $1)], EXPR$2=[SUM(DISTINCT $2 ($5, 0)), AND(=($8, 1), >($6, 0)), =($8, 3))]) + LogicalAggregate(group=[{0, 1}], groups=[[{0, 1}, {0}, {}]], EXPR$2_g0=[COUNT($2) FILTER $3], EXPR$2_g1=[COUNT($2) FILTER $5], EXPR$2_g2=[COUNT($2) FILTER $7], $g_present_0=[COUNT() FILTER $4], $g_present_1=[COUNT() FILTER $6], $g_present_2=[COUNT() FILTER $8], $g_final=[GROUPING($0, $1)]) + LogicalProject(DEPTNO=[$0], JOB=[$1], ENAME=[$2], $g_0=[=($3, 0)], $g_1=[=($3, 1)], $g_2=[=($3, 2)], $g_3=[=($3, 3)], $g_6=[=($3, 6)], $g_7=[=($3, 7)]) + LogicalAggregate(group=[{0, 1, 2}], groups=[[{0, 1, 2}, {0, 1}, {0, 2}, {0}, {2}, {}]], $g=[GROUPING($0, $1, $2)]) + LogicalProject(DEPTNO=[$7], JOB=[$2], ENAME=[$1]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) ]]> @@ -3283,11 +3285,13 @@ LogicalAggregate(group=[{0, 1}], groups=[[{0, 1}, {0}, {}]], EXPR$2=[COUNT(DISTI ($8, 0)), AND(=($11, 1), >($9, 0)), =($11, 3))]) + LogicalAggregate(group=[{0, 1}], groups=[[{0, 1}, {0}, {}]], EXPR$2_g0=[COUNT($2) FILTER $4], EXPR$2_g1=[COUNT($2) FILTER $6], EXPR$2_g2=[COUNT($2) FILTER $8], EXPR$3_g0=[MIN($3) FILTER $5], EXPR$3_g1=[MIN($3) FILTER $7], EXPR$3_g2=[MIN($3) FILTER $9], $g_present_0=[COUNT() FILTER $5], $g_present_1=[COUNT() FILTER $7], $g_present_2=[COUNT() FILTER $9], $g_final=[GROUPING($0, $1)]) + LogicalProject(DEPTNO=[$0], JOB=[$1], ENAME=[$2], EXPR$3=[$3], $g_0=[=($4, 0)], $g_1=[=($4, 1)], $g_2=[=($4, 2)], $g_3=[=($4, 3)], $g_6=[=($4, 6)], $g_7=[=($4, 7)]) + LogicalAggregate(group=[{0, 1, 2}], groups=[[{0, 1, 2}, {0, 1}, {0, 2}, {0}, {2}, {}]], EXPR$3=[SUM($3)], $g=[GROUPING($0, $1, $2)]) + LogicalProject(DEPTNO=[$7], JOB=[$2], ENAME=[$1], SAL=[$5]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) ]]> @@ -3307,7 +3311,7 @@ LogicalAggregate(group=[{0}], CDDJ=[COUNT(DISTINCT $0, $1)], S=[SUM($2)]) @@ -3410,10 +3415,11 @@ LogicalAggregate(group=[{}], EXPR$0=[COUNT(DISTINCT $0)], EXPR$1=[COUNT(DISTINCT @@ -3486,10 +3492,11 @@ LogicalAggregate(group=[{}], EXPR$0=[MAX($0)], EXPR$1=[COUNT(DISTINCT $1)]) @@ -3562,6 +3569,81 @@ LogicalAggregate(group=[{0}], EXPR$1=[$SUM0($2)], EXPR$2=[SUM($1)]) LogicalAggregate(group=[{0, 1}], EXPR$1=[COUNT()]) LogicalProject(DEPTNO=[$7], SAL=[$5]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +]]> + + + + + + + + + + + ($3, 0)), =($5, 1))]) + LogicalAggregate(group=[{0}], groups=[[{0}, {}]], EXPR$1_g0=[COUNT($1) FILTER $2], EXPR$1_g1=[COUNT($1) FILTER $4], $g_present_0=[COUNT() FILTER $3], $g_present_1=[COUNT() FILTER $5], $g_final=[GROUPING($0)]) + LogicalProject(DEPTNO=[$0], SAL=[$1], $g_0=[=($2, 0)], $g_1=[=($2, 1)], $g_2=[=($2, 2)], $g_3=[=($2, 3)]) + LogicalAggregate(group=[{0, 1}], groups=[[{0, 1}, {0}, {1}, {}]], $g=[GROUPING($0, $1)]) + LogicalProject(DEPTNO=[$7], SAL=[$5]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +]]> + + + + + + + + + + + ($5, 0)), =($7, 1))]) + LogicalAggregate(group=[{0}], groups=[[{0}, {}]], EXPR$1_g0=[COUNT($1) FILTER $3], EXPR$1_g1=[COUNT($1) FILTER $5], EXPR$2_g0=[MIN($2) FILTER $4], EXPR$2_g1=[MIN($2) FILTER $6], $g_present_0=[COUNT() FILTER $4], $g_present_1=[COUNT() FILTER $6], $g_final=[GROUPING($0)]) + LogicalProject(DEPTNO=[$0], SAL=[$1], EXPR$2=[$2], $g_0=[=($3, 0)], $g_1=[=($3, 1)], $g_2=[=($3, 2)], $g_3=[=($3, 3)]) + LogicalAggregate(group=[{0, 1}], groups=[[{0, 1}, {0}, {1}, {}]], EXPR$2=[SUM($1)], $g=[GROUPING($0, $1)]) + LogicalProject(DEPTNO=[$7], SAL=[$5]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +]]> + + + + + + + + + + + ($7, 0)), =($9, 1))]) + LogicalAggregate(group=[{0}], groups=[[{0}, {}]], EXPR$1_g0=[COUNT($1) FILTER $3], EXPR$1_g1=[COUNT($1) FILTER $5], EXPR$2_g0=[SUM($1) FILTER $3], EXPR$2_g1=[SUM($1) FILTER $5], EXPR$3_g0=[MIN($2) FILTER $4], EXPR$3_g1=[MIN($2) FILTER $6], $g_present_0=[COUNT() FILTER $4], $g_present_1=[COUNT() FILTER $6], $g_final=[GROUPING($0)]) + LogicalProject(DEPTNO=[$0], SAL=[$1], EXPR$3=[$2], $g_0=[=($3, 0)], $g_1=[=($3, 1)], $g_2=[=($3, 2)], $g_3=[=($3, 3)]) + LogicalAggregate(group=[{0, 1}], groups=[[{0, 1}, {0}, {1}, {}]], EXPR$3=[COUNT()], $g=[GROUPING($0, $1)]) + LogicalProject(DEPTNO=[$7], SAL=[$5]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) ]]> @@ -3602,11 +3684,12 @@ LogicalAggregate(group=[{}], EXPR$0=[COUNT(DISTINCT $0) FILTER $1], EXPR$1=[COUN ($5, 1000)], D=[<($5, 500)]) - LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +LogicalProject(EXPR$0=[$0], EXPR$1=[$1]) + LogicalAggregate(group=[{}], EXPR$0_g0=[COUNT($0) FILTER $3], EXPR$1_g0=[COUNT($1) FILTER $2]) + LogicalProject(C=[$0], D=[$1], $g_0_f_0=[AND(=($2, 0), IS TRUE($0))], $g_0_f_1=[AND(=($2, 0), IS TRUE($1))]) + LogicalAggregate(group=[{0, 1}], groups=[[{0, 1}, {}]], $g=[GROUPING($0, $1)]) + LogicalProject(C=[>($5, 1000)], D=[<($5, 500)]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) ]]> @@ -3626,7 +3709,7 @@ LogicalAggregate(group=[{0}], EXPR$1=[SUM($1)], EXPR$2=[COUNT(DISTINCT $2) FILTE ($5, 1000)]) @@ -3648,11 +3731,12 @@ LogicalAggregate(group=[{}], EXPR$0=[SUM($0)], EXPR$1=[COUNT(DISTINCT $1) FILTER ($5, 1000)]) - LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +LogicalProject(EXPR$0=[$0], EXPR$1=[$1]) + LogicalAggregate(group=[{}], EXPR$0_g0=[MIN($1) FILTER $3], EXPR$1_g0=[COUNT($0) FILTER $2]) + LogicalProject(SAL=[$0], EXPR$0=[$2], $g_0_f_1=[AND(=($3, 0), IS TRUE($1))], $g_3=[=($3, 3)]) + LogicalAggregate(group=[{1, 2}], groups=[[{1, 2}, {}]], EXPR$0=[SUM($0)], $g=[GROUPING($1, $2)]) + LogicalProject(COMM=[$6], SAL=[$5], $f2=[>($5, 1000)]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) ]]> diff --git a/core/src/test/resources/sql/agg.iq b/core/src/test/resources/sql/agg.iq index 9afc119d270b..3c8afd63ff3d 100644 --- a/core/src/test/resources/sql/agg.iq +++ b/core/src/test/resources/sql/agg.iq @@ -1838,14 +1838,14 @@ group by deptno; select count(distinct deptno) as cd, count(*) as c from "scott".emp group by cube(deptno); -+----+---+ -| CD | C | -+----+---+ -| 1 | 3 | -| 1 | 5 | -| 1 | 6 | -| 3 | 3 | -+----+---+ ++----+----+ +| CD | C | ++----+----+ +| 1 | 3 | +| 1 | 5 | +| 1 | 6 | +| 3 | 14 | ++----+----+ (4 rows) !ok @@ -2823,7 +2823,7 @@ select count(distinct EMPNO), COUNT(SAL), MIN(SAL), MAX(SAL) from "scott".emp; !ok EnumerableCalc(expr#0..3=[{inputs}], expr#4=[CAST($t1):BIGINT NOT NULL], EXPR$0=[$t0], EXPR$1=[$t4], EXPR$2=[$t2], EXPR$3=[$t3]) - EnumerableAggregate(group=[{}], EXPR$0=[COUNT($0) FILTER $4], EXPR$1=[MIN($1) FILTER $5], EXPR$2=[MIN($2) FILTER $5], EXPR$3=[MIN($3) FILTER $5]) + EnumerableAggregate(group=[{}], EXPR$0_g0=[COUNT($0) FILTER $4], EXPR$1_g0=[MIN($1) FILTER $5], EXPR$2_g0=[MIN($2) FILTER $5], EXPR$3_g0=[MIN($3) FILTER $5]) EnumerableCalc(expr#0..4=[{inputs}], expr#5=[0], expr#6=[=($t4, $t5)], expr#7=[1], expr#8=[=($t4, $t7)], proj#0..3=[{exprs}], $g_0=[$t6], $g_1=[$t8]) EnumerableAggregate(group=[{0}], groups=[[{0}, {}]], EXPR$1=[COUNT($5)], EXPR$2=[MIN($5)], EXPR$3=[MAX($5)], $g=[GROUPING($0)]) EnumerableTableScan(table=[[scott, EMP]]) @@ -2841,7 +2841,7 @@ select count(distinct DEPTNO), COUNT(JOB), MIN(SAL), MAX(SAL) from "scott".emp; !ok EnumerableCalc(expr#0..3=[{inputs}], expr#4=[CAST($t1):BIGINT NOT NULL], EXPR$0=[$t0], EXPR$1=[$t4], EXPR$2=[$t2], EXPR$3=[$t3]) - EnumerableAggregate(group=[{}], EXPR$0=[COUNT($0) FILTER $4], EXPR$1=[MIN($1) FILTER $5], EXPR$2=[MIN($2) FILTER $5], EXPR$3=[MIN($3) FILTER $5]) + EnumerableAggregate(group=[{}], EXPR$0_g0=[COUNT($0) FILTER $4], EXPR$1_g0=[MIN($1) FILTER $5], EXPR$2_g0=[MIN($2) FILTER $5], EXPR$3_g0=[MIN($3) FILTER $5]) EnumerableCalc(expr#0..4=[{inputs}], expr#5=[0], expr#6=[=($t4, $t5)], expr#7=[1], expr#8=[=($t4, $t7)], proj#0..3=[{exprs}], $g_0=[$t6], $g_1=[$t8]) EnumerableAggregate(group=[{7}], groups=[[{7}, {}]], EXPR$1=[COUNT($2)], EXPR$2=[MIN($5)], EXPR$3=[MAX($5)], $g=[GROUPING($7)]) EnumerableTableScan(table=[[scott, EMP]]) @@ -2865,7 +2865,7 @@ select MGR, count(distinct DEPTNO), COUNT(JOB), MIN(SAL), MAX(SAL) from "scott". !ok EnumerableCalc(expr#0..4=[{inputs}], expr#5=[CAST($t2):BIGINT NOT NULL], proj#0..1=[{exprs}], EXPR$2=[$t5], EXPR$3=[$t3], EXPR$4=[$t4]) - EnumerableAggregate(group=[{0}], EXPR$1=[COUNT($1) FILTER $5], EXPR$2=[MIN($2) FILTER $6], EXPR$3=[MIN($3) FILTER $6], EXPR$4=[MIN($4) FILTER $6]) + EnumerableAggregate(group=[{0}], EXPR$1_g0=[COUNT($1) FILTER $5], EXPR$2_g0=[MIN($2) FILTER $6], EXPR$3_g0=[MIN($3) FILTER $6], EXPR$4_g0=[MIN($4) FILTER $6]) EnumerableCalc(expr#0..5=[{inputs}], expr#6=[0], expr#7=[=($t5, $t6)], expr#8=[1], expr#9=[=($t5, $t8)], proj#0..4=[{exprs}], $g_0=[$t7], $g_1=[$t9]) EnumerableAggregate(group=[{3, 7}], groups=[[{3, 7}, {3}]], EXPR$2=[COUNT($2)], EXPR$3=[MIN($5)], EXPR$4=[MAX($5)], $g=[GROUPING($3, $7)]) EnumerableTableScan(table=[[scott, EMP]]) @@ -2888,7 +2888,7 @@ select MGR, count(distinct DEPTNO, JOB), MIN(SAL), MAX(SAL) from "scott".emp gro !ok -EnumerableAggregate(group=[{1}], EXPR$1=[COUNT($2, $0) FILTER $5], EXPR$2=[MIN($3) FILTER $6], EXPR$3=[MIN($4) FILTER $6]) +EnumerableAggregate(group=[{1}], EXPR$1_g0=[COUNT($2, $0) FILTER $5], EXPR$2_g0=[MIN($3) FILTER $6], EXPR$3_g0=[MIN($4) FILTER $6]) EnumerableCalc(expr#0..5=[{inputs}], expr#6=[0], expr#7=[=($t5, $t6)], expr#8=[5], expr#9=[=($t5, $t8)], proj#0..4=[{exprs}], $g_0=[$t7], $g_5=[$t9]) EnumerableAggregate(group=[{2, 3, 7}], groups=[[{2, 3, 7}, {3}]], EXPR$2=[MIN($5)], EXPR$3=[MAX($5)], $g=[GROUPING($2, $3, $7)]) EnumerableTableScan(table=[[scott, EMP]]) @@ -4187,4 +4187,115 @@ EnumerableCalc(expr#0..8=[{inputs}], DEPTNO=[$t0], ENAME=[$t2], SAL_COL=[$t3], F EnumerableTableScan(table=[[scott, EMP]]) !plan +# [CALCITE-5465] Rule of AGGREGATE_EXPAND_DISTINCT_AGGREGATES produces an incorrect plan when sql has distinct agg-call with rollup +!use scott +WITH t1 (id, c1) AS ( + VALUES + ('1', 'A1'), + ('2', 'A2'), + ('3', 'A3'), + ('3', 'A3'), + ('3', 'A2'), + (NULL, 'A4') +) +SELECT id, COUNT(DISTINCT c1) +FROM t1 +GROUP BY ROLLUP(id); ++----+--------+ +| ID | EXPR$1 | ++----+--------+ +| 1 | 1 | +| 2 | 1 | +| 3 | 2 | +| | 1 | +| | 4 | ++----+--------+ +(5 rows) + +!ok + +SELECT deptno, job, COUNT(DISTINCT ename) +FROM "scott".emp +GROUP BY ROLLUP(deptno, job); ++--------+-----------+--------+ +| DEPTNO | JOB | EXPR$2 | ++--------+-----------+--------+ +| 10 | CLERK | 1 | +| 10 | MANAGER | 1 | +| 10 | PRESIDENT | 1 | +| 10 | | 3 | +| 20 | ANALYST | 2 | +| 20 | CLERK | 2 | +| 20 | MANAGER | 1 | +| 20 | | 5 | +| 30 | CLERK | 1 | +| 30 | MANAGER | 1 | +| 30 | SALESMAN | 4 | +| 30 | | 6 | +| | | 14 | ++--------+-----------+--------+ +(13 rows) + +!ok + +SELECT deptno, COUNT(DISTINCT sal) +FROM "scott".emp +GROUP BY GROUPING SETS ((deptno), ()); ++--------+--------+ +| DEPTNO | EXPR$1 | ++--------+--------+ +| 10 | 3 | +| 20 | 4 | +| 30 | 5 | +| | 12 | ++--------+--------+ +(4 rows) + +!ok + +SELECT deptno, COUNT(DISTINCT sal), SUM(sal) +FROM emp +GROUP BY GROUPING SETS ((deptno), ()); ++--------+--------+----------+ +| DEPTNO | EXPR$1 | EXPR$2 | ++--------+--------+----------+ +| 10 | 3 | 8750.00 | +| 20 | 4 | 10875.00 | +| 30 | 5 | 9400.00 | +| | 12 | 29025.00 | ++--------+--------+----------+ +(4 rows) + +!ok + +SELECT deptno, COUNT(DISTINCT sal), COUNT(sal) +FROM emp +GROUP BY GROUPING SETS ((deptno), ()); ++--------+--------+--------+ +| DEPTNO | EXPR$1 | EXPR$2 | ++--------+--------+--------+ +| 10 | 3 | 3 | +| 20 | 4 | 5 | +| 30 | 5 | 6 | +| | 12 | 14 | ++--------+--------+--------+ +(4 rows) + +!ok + +SELECT deptno, COUNT(DISTINCT sal), SUM(DISTINCT sal), COUNT(*) +FROM emp +GROUP BY GROUPING SETS ((deptno), ()); ++--------+--------+----------+--------+ +| DEPTNO | EXPR$1 | EXPR$2 | EXPR$3 | ++--------+--------+----------+--------+ +| 10 | 3 | 8750.00 | 3 | +| 20 | 4 | 7875.00 | 5 | +| 30 | 5 | 8150.00 | 6 | +| | 12 | 24775.00 | 14 | ++--------+--------+----------+--------+ +(4 rows) + +!ok + # End agg.iq diff --git a/core/src/test/resources/sql/sub-query.iq b/core/src/test/resources/sql/sub-query.iq index 1dd2edf4df00..ab9a52fa795a 100644 --- a/core/src/test/resources/sql/sub-query.iq +++ b/core/src/test/resources/sql/sub-query.iq @@ -2814,7 +2814,7 @@ EnumerableCalc(expr#0..13=[{inputs}], expr#14=[<>($t10, $t9)], expr#15=[1], expr EnumerableAggregate(group=[{7}]) EnumerableTableScan(table=[[scott, EMP]]) EnumerableCalc(expr#0..5=[{inputs}], expr#6=[CAST($t1):BIGINT NOT NULL], expr#7=[CAST($t2):BIGINT NOT NULL], expr#8=[CAST($t5):BOOLEAN NOT NULL], DEPTNO=[$t0], c=[$t6], d=[$t7], dd=[$t3], m=[$t4], trueLiteral=[$t8]) - EnumerableAggregate(group=[{1}], c=[MIN($2) FILTER $7], d=[MIN($3) FILTER $7], dd=[COUNT($0) FILTER $6], m=[MIN($4) FILTER $7], trueLiteral=[MIN(true, $5) FILTER $7]) + EnumerableAggregate(group=[{1}], c_g0=[MIN($2) FILTER $7], d_g0=[MIN($3) FILTER $7], dd_g0=[COUNT($0) FILTER $6], m_g0=[MIN($4) FILTER $7], trueLiteral_g0=[MIN(true, $5) FILTER $7]) EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[=($t6, $t7)], expr#9=[2], expr#10=[=($t6, $t9)], proj#0..5=[{exprs}], $g_0=[$t8], $g_2=[$t10]) EnumerableAggregate(group=[{6, 7}], groups=[[{6, 7}, {7}]], c=[COUNT()], d=[COUNT($6)], m=[MAX($6)], trueLiteral=[LITERAL_AGG(true)], $g=[GROUPING($6, $7)]) EnumerableCalc(expr#0..7=[{inputs}], expr#8=[IS NOT NULL($t7)], proj#0..7=[{exprs}], $condition=[$t8]) @@ -2871,7 +2871,7 @@ EnumerableCalc(expr#0..13=[{inputs}], expr#14=[<>($t9, $t8)], expr#15=[1], expr# EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0]) EnumerableTableScan(table=[[scott, EMP]]) EnumerableCalc(expr#0..4=[{inputs}], expr#5=[CAST($t1):BIGINT NOT NULL], expr#6=[CAST($t3):INTEGER NOT NULL], expr#7=[CAST($t4):BOOLEAN NOT NULL], DEPTNO0=[$t0], c=[$t5], dd=[$t2], m=[$t6], trueLiteral=[$t7]) - EnumerableAggregate(group=[{0}], c=[MIN($2) FILTER $6], dd=[COUNT($1) FILTER $5], m=[MIN($3) FILTER $6], trueLiteral=[MIN(true, $4) FILTER $6]) + EnumerableAggregate(group=[{0}], c_g0=[MIN($2) FILTER $6], dd_g0=[COUNT($1) FILTER $5], m_g0=[MIN($3) FILTER $6], trueLiteral_g0=[MIN(true, $4) FILTER $6]) EnumerableCalc(expr#0..5=[{inputs}], expr#6=[0], expr#7=[=($t5, $t6)], expr#8=[1], expr#9=[=($t5, $t8)], proj#0..4=[{exprs}], $g_0=[$t7], $g_1=[$t9]) EnumerableAggregate(group=[{0, 1}], groups=[[{0, 1}, {0}]], c=[COUNT()], m=[MAX($1)], trueLiteral=[LITERAL_AGG(true)], $g=[GROUPING($0, $1)]) EnumerableCalc(expr#0..2=[{inputs}], expr#3=[CAST($t0):SMALLINT NOT NULL], expr#4=[2], DEPTNO0=[$t3], EXPR$0=[$t4]) @@ -2922,7 +2922,7 @@ EnumerableCalc(expr#0..13=[{inputs}], expr#14=[<>($t9, $t8)], expr#15=[1], expr# EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0]) EnumerableTableScan(table=[[scott, EMP]]) EnumerableCalc(expr#0..4=[{inputs}], expr#5=[CAST($t1):BIGINT NOT NULL], expr#6=[CAST($t3):INTEGER NOT NULL], expr#7=[CAST($t4):BOOLEAN NOT NULL], DEPTNO0=[$t0], c=[$t5], dd=[$t2], m=[$t6], trueLiteral=[$t7]) - EnumerableAggregate(group=[{0}], c=[MIN($2) FILTER $6], dd=[COUNT($1) FILTER $5], m=[MIN($3) FILTER $6], trueLiteral=[MIN(true, $4) FILTER $6]) + EnumerableAggregate(group=[{0}], c_g0=[MIN($2) FILTER $6], dd_g0=[COUNT($1) FILTER $5], m_g0=[MIN($3) FILTER $6], trueLiteral_g0=[MIN(true, $4) FILTER $6]) EnumerableCalc(expr#0..5=[{inputs}], expr#6=[0], expr#7=[=($t5, $t6)], expr#8=[1], expr#9=[=($t5, $t8)], proj#0..4=[{exprs}], $g_0=[$t7], $g_1=[$t9]) EnumerableAggregate(group=[{0, 1}], groups=[[{0, 1}, {0}]], c=[COUNT()], m=[MAX($1)], trueLiteral=[LITERAL_AGG(true)], $g=[GROUPING($0, $1)]) EnumerableCalc(expr#0..2=[{inputs}], expr#3=[CAST($t0):SMALLINT NOT NULL], expr#4=[2], DEPTNO0=[$t3], EXPR$0=[$t4]) @@ -2973,7 +2973,7 @@ EnumerableCalc(expr#0..13=[{inputs}], expr#14=[<>($t10, $t9)], expr#15=[1], expr EnumerableAggregate(group=[{5}]) EnumerableTableScan(table=[[scott, EMP]]) EnumerableCalc(expr#0..5=[{inputs}], expr#6=[CAST($t1):BIGINT NOT NULL], expr#7=[CAST($t2):BIGINT NOT NULL], expr#8=[CAST($t5):BOOLEAN NOT NULL], SAL=[$t0], c=[$t6], d=[$t7], dd=[$t3], m=[$t4], trueLiteral=[$t8]) - EnumerableAggregate(group=[{0}], c=[MIN($2) FILTER $7], d=[MIN($3) FILTER $7], dd=[COUNT($1) FILTER $6], m=[MIN($4) FILTER $7], trueLiteral=[MIN(true, $5) FILTER $7]) + EnumerableAggregate(group=[{0}], c_g0=[MIN($2) FILTER $7], d_g0=[MIN($3) FILTER $7], dd_g0=[COUNT($1) FILTER $6], m_g0=[MIN($4) FILTER $7], trueLiteral_g0=[MIN(true, $5) FILTER $7]) EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[=($t6, $t7)], expr#9=[1], expr#10=[=($t6, $t9)], proj#0..5=[{exprs}], $g_0=[$t8], $g_1=[$t10]) EnumerableAggregate(group=[{5, 6}], groups=[[{5, 6}, {5}]], c=[COUNT()], d=[COUNT($6)], m=[MAX($6)], trueLiteral=[LITERAL_AGG(true)], $g=[GROUPING($5, $6)]) EnumerableCalc(expr#0..7=[{inputs}], expr#8=[IS NOT NULL($t5)], proj#0..7=[{exprs}], $condition=[$t8]) diff --git a/spark/src/test/java/org/apache/calcite/test/SparkAdapterTest.java b/spark/src/test/java/org/apache/calcite/test/SparkAdapterTest.java index 39d1bb39bc63..37ac09469bf7 100644 --- a/spark/src/test/java/org/apache/calcite/test/SparkAdapterTest.java +++ b/spark/src/test/java/org/apache/calcite/test/SparkAdapterTest.java @@ -122,7 +122,7 @@ private CalciteAssert.AssertQuery sql(String sql) { final String plan = "PLAN=" + "EnumerableCalc(expr#0..5=[{inputs}], expr#6=[CAST($t1):INTEGER NOT NULL], expr#7=[CAST($t2):CHAR(1) NOT NULL], expr#8=[CAST($t3):CHAR(1) NOT NULL], expr#9=[CAST($t4):BIGINT NOT NULL], SUM_X=[$t6], MIN_Y=[$t7], MAX_Y=[$t8], CNT_Y=[$t9], CNT_DIST_Y=[$t5])\n" - + " EnumerableAggregate(group=[{0}], SUM_X=[MIN($2) FILTER $7], MIN_Y=[MIN($3) FILTER $7], MAX_Y=[MIN($4) FILTER $7], CNT_Y=[MIN($5) FILTER $7], CNT_DIST_Y=[COUNT($1) FILTER $6])\n" + + " EnumerableAggregate(group=[{0}], SUM_X_g0=[MIN($2) FILTER $7], MIN_Y_g0=[MIN($3) FILTER $7], MAX_Y_g0=[MIN($4) FILTER $7], CNT_Y_g0=[MIN($5) FILTER $7], CNT_DIST_Y_g0=[COUNT($1) FILTER $6])\n" + " EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[=($t6, $t7)], expr#9=[1], expr#10=[=($t6, $t9)], proj#0..5=[{exprs}], $g_0=[$t8], $g_1=[$t10])\n" + " EnumerableAggregate(group=[{0, 1}], groups=[[{0, 1}, {0}]], SUM_X=[$SUM0($0)], MIN_Y=[MIN($1)], MAX_Y=[MAX($1)], CNT_Y=[COUNT()], $g=[GROUPING($0, $1)])\n" + " EnumerableValues(tuples=[[{ 1, 'a' }, { 2, 'b' }, { 1, 'b' }, { 2, 'c' }, { 2, 'c' }]])\n"; @@ -143,7 +143,7 @@ private CalciteAssert.AssertQuery sql(String sql) { final String plan = "PLAN=" + "EnumerableCalc(expr#0..4=[{inputs}], expr#5=[CAST($t3):BIGINT NOT NULL], proj#0..2=[{exprs}], CNT_Y=[$t5], CNT_DIST_Y=[$t4])\n" - + " EnumerableAggregate(group=[{}], SUM_X=[MIN($1) FILTER $6], MIN_Y=[MIN($2) FILTER $6], MAX_Y=[MIN($3) FILTER $6], CNT_Y=[MIN($4) FILTER $6], CNT_DIST_Y=[COUNT($0) FILTER $5])\n" + + " EnumerableAggregate(group=[{}], SUM_X_g0=[MIN($1) FILTER $6], MIN_Y_g0=[MIN($2) FILTER $6], MAX_Y_g0=[MIN($3) FILTER $6], CNT_Y_g0=[MIN($4) FILTER $6], CNT_DIST_Y_g0=[COUNT($0) FILTER $5])\n" + " EnumerableCalc(expr#0..5=[{inputs}], expr#6=[0], expr#7=[=($t2, $t6)], expr#8=[null:INTEGER], expr#9=[CASE($t7, $t8, $t1)], expr#10=[=($t5, $t6)], expr#11=[1], expr#12=[=($t5, $t11)], Y=[$t0], SUM_X=[$t9], MIN_Y=[$t3], MAX_Y=[$t4], CNT_Y=[$t2], $g_0=[$t10], $g_1=[$t12])\n" + " EnumerableAggregate(group=[{1}], groups=[[{1}, {}]], SUM_X=[$SUM0($0)], agg#1=[COUNT()], MIN_Y=[MIN($1)], MAX_Y=[MAX($1)], $g=[GROUPING($1)])\n" + " EnumerableValues(tuples=[[{ 1, 'a' }, { 2, 'b' }, { 1, 'b' }, { 2, 'c' }, { 2, 'c' }]])\n"; @@ -179,7 +179,7 @@ private CalciteAssert.AssertQuery sql(String sql) { final String plan = "PLAN=" + "EnumerableSort(sort0=[$0], dir0=[ASC])\n" + " EnumerableCalc(expr#0..4=[{inputs}], expr#5=[CAST($t1):CHAR(1) NOT NULL], expr#6=[CAST($t2):CHAR(1) NOT NULL], expr#7=[CAST($t3):BIGINT NOT NULL], X=[$t0], MIN_Y=[$t5], MAX_Y=[$t6], CNT_Y=[$t7], CNT_DIST_Y=[$t4])\n" - + " EnumerableAggregate(group=[{0}], MIN_Y=[MIN($2) FILTER $6], MAX_Y=[MIN($3) FILTER $6], CNT_Y=[MIN($4) FILTER $6], CNT_DIST_Y=[COUNT($1) FILTER $5])\n" + + " EnumerableAggregate(group=[{0}], MIN_Y_g0=[MIN($2) FILTER $6], MAX_Y_g0=[MIN($3) FILTER $6], CNT_Y_g0=[MIN($4) FILTER $6], CNT_DIST_Y_g0=[COUNT($1) FILTER $5])\n" + " EnumerableCalc(expr#0..5=[{inputs}], expr#6=[0], expr#7=[=($t5, $t6)], expr#8=[1], expr#9=[=($t5, $t8)], proj#0..4=[{exprs}], $g_0=[$t7], $g_1=[$t9])\n" + " EnumerableAggregate(group=[{0, 1}], groups=[[{0, 1}, {0}]], MIN_Y=[MIN($1)], MAX_Y=[MAX($1)], CNT_Y=[COUNT()], $g=[GROUPING($0, $1)])\n" + " EnumerableValues(tuples=[[{ 1, 'a' }, { 2, 'b' }, { 1, 'b' }, { 2, 'c' }, { 2, 'c' }]])\n\n"; @@ -201,7 +201,7 @@ private CalciteAssert.AssertQuery sql(String sql) { final String plan = "PLAN=" + "EnumerableSort(sort0=[$0], dir0=[DESC])\n" + " EnumerableCalc(expr#0..4=[{inputs}], expr#5=[CAST($t1):CHAR(1) NOT NULL], expr#6=[CAST($t2):CHAR(1) NOT NULL], expr#7=[CAST($t3):BIGINT NOT NULL], X=[$t0], MIN_Y=[$t5], MAX_Y=[$t6], CNT_Y=[$t7], CNT_DIST_Y=[$t4])\n" - + " EnumerableAggregate(group=[{0}], MIN_Y=[MIN($2) FILTER $6], MAX_Y=[MIN($3) FILTER $6], CNT_Y=[MIN($4) FILTER $6], CNT_DIST_Y=[COUNT($1) FILTER $5])\n" + + " EnumerableAggregate(group=[{0}], MIN_Y_g0=[MIN($2) FILTER $6], MAX_Y_g0=[MIN($3) FILTER $6], CNT_Y_g0=[MIN($4) FILTER $6], CNT_DIST_Y_g0=[COUNT($1) FILTER $5])\n" + " EnumerableCalc(expr#0..5=[{inputs}], expr#6=[0], expr#7=[=($t5, $t6)], expr#8=[1], expr#9=[=($t5, $t8)], proj#0..4=[{exprs}], $g_0=[$t7], $g_1=[$t9])\n" + " EnumerableAggregate(group=[{0, 1}], groups=[[{0, 1}, {0}]], MIN_Y=[MIN($1)], MAX_Y=[MAX($1)], CNT_Y=[COUNT()], $g=[GROUPING($0, $1)])\n" + " EnumerableValues(tuples=[[{ 1, 'a' }, { 2, 'b' }, { 1, 'b' }, { 2, 'c' }, { 2, 'c' }]])\n\n"; From 2c846a90fd4172b8ec7b344eb4a0c3412a7588f2 Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Fri, 21 Nov 2025 16:27:50 +0800 Subject: [PATCH 026/562] [CALCITE-6963] SqlToRelConverter fails when subquery is in join on clause --- .../calcite/sql2rel/SqlToRelConverter.java | 3 +- .../calcite/test/SqlToRelConverterTest.java | 10 +++++ .../calcite/test/SqlToRelConverterTest.xml | 45 ++++++++++++++----- 3 files changed, 46 insertions(+), 12 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java index 392fcdda1042..da96b98b0655 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java @@ -3453,7 +3453,8 @@ private Pair convertOnCondition( SqlNode condition, RelNode leftRel, RelNode rightRel) { - bb.setRoot(ImmutableList.of(leftRel, rightRel)); + bb.setRoot(ImmutableList.of(leftRel, rightRel), leftRel, + leftRel instanceof LogicalJoin); replaceSubQueries(bb, condition, RelOptUtil.Logic.UNKNOWN_AS_FALSE); final RelNode newRightRel = bb.root == null || bb.registered.isEmpty() diff --git a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java index b332be680e1b..b62b9c247ff4 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java @@ -5976,4 +5976,14 @@ void checkUserDefinedOrderByOver(NullCollation nullCollation) { + " where e1.deptno = e2.deptno)"; sql(sql).withExpand(false).ok(); } + + /** Test case of + * [CALCITE-6963] + * SqlToRelConverter fails when subquery is in join on clause. */ + @Test void testSubqueryInJoinOnClause() { + final String sql = "select t1.* from emp t1\n" + + "left join dept t2 on t1.deptno = t2.deptno\n" + + "and t1.ename in (select t3.ename from emp t3 )"; + sql(sql).ok(); + } } diff --git a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml index 382e39292f7a..a1c7f101b39b 100644 --- a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml @@ -3751,14 +3751,15 @@ from (values (cast(null as int), 1), @@ -3953,11 +3954,13 @@ LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$ @@ -8251,6 +8254,26 @@ LogicalProject(DEPTNO=[$7]) LogicalJoin(condition=[true], joinType=[left]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) LogicalValues(tuples=[[{ 10 }]]) +]]> + + + + + + + + From e857a653c3a4bb429debbf7ac0fb4085d06224fe Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Sun, 23 Nov 2025 22:26:57 +0800 Subject: [PATCH 027/562] Add tests for [CALCITE-6985] to verify AggregateMinMaxToLimitRule handles empty tables correctly --- core/src/test/resources/sql/planner.iq | 48 ++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/core/src/test/resources/sql/planner.iq b/core/src/test/resources/sql/planner.iq index 787ec7fb1a7f..a24181cadba1 100644 --- a/core/src/test/resources/sql/planner.iq +++ b/core/src/test/resources/sql/planner.iq @@ -147,6 +147,54 @@ EnumerableCalc(expr#0..2=[{inputs}], $f0=[$t1], $f1=[$t2]) !plan !set planner-rules original +# Add tests for [CALCITE-6985] to verify AggregateMinMaxToLimitRule handles empty tables correctly +!use blank +create table t_empty (id int); +(0 rows modified) + +!update + +select min(id), max(id) from t_empty; ++--------+--------+ +| EXPR$0 | EXPR$1 | ++--------+--------+ +| | | ++--------+--------+ +(1 row) + +!ok +EnumerableAggregate(group=[{}], EXPR$0=[MIN($0)], EXPR$1=[MAX($0)]) + EnumerableTableScan(table=[[BLANK, T_EMPTY]]) +!plan + +!set planner-rules " ++AGGREGATE_MIN_MAX_TO_LIMIT, +-EnumerableRules.ENUMERABLE_AGGREGATE_RULE, ++PROJECT_SUB_QUERY_TO_CORRELATE" +select min(id), max(id) from t_empty; ++--------+--------+ +| EXPR$0 | EXPR$1 | ++--------+--------+ +| | | ++--------+--------+ +(1 row) + +!ok +EnumerableCalc(expr#0..2=[{inputs}], $f0=[$t1], $f1=[$t2]) + EnumerableNestedLoopJoin(condition=[true], joinType=[left]) + EnumerableNestedLoopJoin(condition=[true], joinType=[left]) + EnumerableValues(tuples=[[{ 1 }]]) + EnumerableLimit(fetch=[1]) + EnumerableSort(sort0=[$0], dir0=[ASC]) + EnumerableCalc(expr#0=[{inputs}], expr#1=[IS NOT NULL($t0)], ID=[$t0], $condition=[$t1]) + EnumerableTableScan(table=[[BLANK, T_EMPTY]]) + EnumerableLimit(fetch=[1]) + EnumerableSort(sort0=[$0], dir0=[DESC]) + EnumerableCalc(expr#0=[{inputs}], expr#1=[IS NOT NULL($t0)], ID=[$t0], $condition=[$t1]) + EnumerableTableScan(table=[[BLANK, T_EMPTY]]) +!plan +!set planner-rules original + # [CALCITE-7000] Extend IntersectToSemiJoinRule to support n-way inputs !set planner-rules " -EnumerableRules.ENUMERABLE_INTERSECT_RULE, From 178c91285a696375a8324454d05728b8fea06146 Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Sat, 22 Nov 2025 23:32:14 +0800 Subject: [PATCH 028/562] [CALCITE-7302] Infinite loop with JoinPushTransitivePredicatesRule --- .../calcite/rel/metadata/RelMdPredicates.java | 60 +++++++++++++++---- .../apache/calcite/test/RelOptRulesTest.java | 13 ++++ .../apache/calcite/test/RelOptRulesTest.xml | 33 +++++++++- 3 files changed, 92 insertions(+), 14 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdPredicates.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdPredicates.java index 44b3f1c1d538..11a670515891 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdPredicates.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdPredicates.java @@ -710,30 +710,55 @@ static class JoinConditionBasedPredicateInference { Mappings.TargetMapping leftMapping = Mappings.createShiftMapping(nSysFields + nFieldsLeft, nSysFields, 0, nFieldsLeft); - leftChildPredicates = + RexNode lcp = leftPredicates.accept( new RexPermuteInputsShuttle(leftMapping, joinRel.getInput(0))); + leftChildPredicates = lcp; - allExprs.add(leftChildPredicates); - for (RexNode r : RelOptUtil.conjunctions(leftChildPredicates)) { - exprFields.put(r, RelOptUtil.InputFinder.bits(r)); - allExprs.add(r); + if (lcp != null) { + allExprs.add(lcp); + for (RexNode r : RelOptUtil.conjunctions(lcp)) { + exprFields.put(r, RelOptUtil.InputFinder.bits(r)); + allExprs.add(r); + } + RexNode simplified = + simplify.simplifyFilterPredicates(RelOptUtil.conjunctions(lcp)); + if (simplified != null && !simplified.equals(lcp)) { + allExprs.add(simplified); + for (RexNode r : RelOptUtil.conjunctions(simplified)) { + exprFields.put(r, RelOptUtil.InputFinder.bits(r)); + allExprs.add(r); + } + } } } + if (rightPredicates == null) { rightChildPredicates = null; } else { Mappings.TargetMapping rightMapping = Mappings.createShiftMapping(nSysFields + nFieldsLeft + nFieldsRight, nSysFields + nFieldsLeft, 0, nFieldsRight); - rightChildPredicates = + RexNode rcp = rightPredicates.accept( new RexPermuteInputsShuttle(rightMapping, joinRel.getInput(1))); + rightChildPredicates = rcp; - allExprs.add(rightChildPredicates); - for (RexNode r : RelOptUtil.conjunctions(rightChildPredicates)) { - exprFields.put(r, RelOptUtil.InputFinder.bits(r)); - allExprs.add(r); + if (rcp != null) { + allExprs.add(rcp); + for (RexNode r : RelOptUtil.conjunctions(rcp)) { + exprFields.put(r, RelOptUtil.InputFinder.bits(r)); + allExprs.add(r); + } + RexNode simplified = + simplify.simplifyFilterPredicates(RelOptUtil.conjunctions(rcp)); + if (simplified != null && !simplified.equals(rcp)) { + allExprs.add(simplified); + for (RexNode r : RelOptUtil.conjunctions(simplified)) { + exprFields.put(r, RelOptUtil.InputFinder.bits(r)); + allExprs.add(r); + } + } } } @@ -864,7 +889,20 @@ public RelOptPredicateList inferPredicates( private void infer(@Nullable RexNode predicates, Set allExprs, List inferredPredicates, boolean includeEqualityInference, ImmutableBitSet inferringFields) { - for (RexNode r : RelOptUtil.conjunctions(predicates)) { + if (predicates == null) { + return; + } + + // Normalize predicates by simplification to ensure consistent deduplication. + // Prevents infinite loops when semantically equivalent predicates are simplified + // to different forms (e.g., AND(>=, <=) to SEARCH). + RexNode normalizedPredicates = + simplify.simplifyFilterPredicates(RelOptUtil.conjunctions(predicates)); + if (normalizedPredicates == null) { + return; + } + + for (RexNode r : RelOptUtil.conjunctions(normalizedPredicates)) { if (!includeEqualityInference && equalityPredicates.contains(r)) { continue; diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index 4369727f3fab..2b429eac3016 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -6521,6 +6521,19 @@ private HepProgram getTransitiveProgram() { sql(sql).withPre(getTransitiveProgram()).withProgram(program).check(); } + /** Test case for + * [CALCITE-7302] + * Infinite loop with JoinPushTransitivePredicatesRule. */ + @Test void testInfiniteLoopWithBetweenAnd() { + final String sql = "With dept_temp as" + + " (SELECT deptno, name FROM dept where deptno between 30 and 50)," + + "emp_temp as" + + " (Select ename, deptno from emp)" + + "select * from dept_temp inner join emp_temp on dept_temp.deptno = emp_temp.deptno "; + sql(sql).withRule(CoreRules.JOIN_PUSH_TRANSITIVE_PREDICATES) + .check(); + } + /** Test case for * [CALCITE-2110] * ArrayIndexOutOfBoundsException in RexSimplify when using diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index c63602eae693..bb53c538bd84 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -6420,11 +6420,38 @@ LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$ + + + + + + + + =($0, 30), <=($0, 50))]) + LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) + LogicalProject(ENAME=[$1], DEPTNO=[$7]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +]]> + + + =($0, 30), <=($0, 50))]) + LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) + LogicalFilter(condition=[SEARCH($1, Sarg[[30..50]])]) + LogicalProject(ENAME=[$1], DEPTNO=[$7]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) ]]> From 330f870fb80acfe47f0ef5142d3920136957a09f Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Fri, 14 Nov 2025 13:12:37 -0800 Subject: [PATCH 029/562] [CALCITE-7289] Select NULL subquery throwing exception Signed-off-by: Mihai Budiu --- .../org/apache/calcite/tools/RelBuilder.java | 7 +++- .../apache/calcite/test/RelOptRulesTest.java | 12 +++++++ .../apache/calcite/test/RelOptRulesTest.xml | 34 +++++++++++++++++++ 3 files changed, 52 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java index b8500177c0a6..154cca84a3b0 100644 --- a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java +++ b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java @@ -1937,8 +1937,13 @@ public RelBuilder filter(Iterable variablesSet, if (config.simplify()) { conjunctionPredicates = simplifier.simplifyFilterPredicates(predicates); } else { + List simplified = new ArrayList<>(); + for (RexNode predicate : predicates) { + RexNode simple = RexSimplify.simplifyComparisonWithNull(predicate, getRexBuilder()); + simplified.add(simple); + } conjunctionPredicates = - RexUtil.composeConjunction(simplifier.rexBuilder, predicates); + RexUtil.composeConjunction(simplifier.rexBuilder, simplified); } if (conjunctionPredicates == null || conjunctionPredicates.isAlwaysFalse()) { diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index 2b429eac3016..af463cbcb4b8 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -100,6 +100,7 @@ import org.apache.calcite.rel.rules.SortProjectTransposeRule; import org.apache.calcite.rel.rules.SortUnionTransposeRule; import org.apache.calcite.rel.rules.SpatialRules; +import org.apache.calcite.rel.rules.SubQueryRemoveRule; import org.apache.calcite.rel.rules.UnionMergeRule; import org.apache.calcite.rel.rules.ValuesReduceRule; import org.apache.calcite.rel.type.RelDataType; @@ -9282,6 +9283,17 @@ private void checkSemiJoinRuleOnAntiJoin(RelOptRule rule) { .check(); } + /** Test case for [CALCITE-7289] + * Select NULL subquery throwing exception. */ + @Test void testNullSelect() { + final String sql = "SELECT 1 from emp WHERE NULL IN (SELECT null)"; + RelBuilder.Config config = + RelBuilder.Config.DEFAULT.withSimplifyValues(false).withSimplify(false); + RelOptRule subQueryFilterRule = + SubQueryRemoveRule.Config.FILTER.withRelBuilderFactory(RelBuilder.proto(config)).toRule(); + sql(sql).withRule(subQueryFilterRule).withLateDecorrelate(true).check(); + } + /** Test case for * [CALCITE-6652] * RelDecorrelator can't decorrelate query with limit 1. diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index bb53c538bd84..a653e7e0d134 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -10723,6 +10723,40 @@ LogicalProject(DEPTNO=[$7]) + + + + + + + + + + + + + + From c3ae5781f3fb902b2506329b4badf07268b5c442 Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Fri, 21 Nov 2025 17:33:20 +0800 Subject: [PATCH 030/562] [CALCITE-6681] NullPointerException in ProjectCorrelateTransposeRule --- .../apache/calcite/test/RelOptRulesTest.java | 46 +++++++++++++ .../apache/calcite/test/RelOptRulesTest.xml | 68 +++++++++++++++++++ 2 files changed, 114 insertions(+) diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index af463cbcb4b8..ed3ffa32b0ef 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -9201,6 +9201,52 @@ private void checkSemiJoinRuleOnAntiJoin(RelOptRule rule) { .checkUnchanged(); } + /** Test case of + * [CALCITE-6681] + * NullPointerException in ProjectCorrelateTransposeRule. */ + @Test void testLateralTransposeWithDecorrelateFalse() { + final String sql = "WITH " + + " t1(a, ts) AS (VALUES('a', 1))," + + " t2(a, ts, x) AS (SELECT ename as a, empno as ts, mgr as x FROM emp)\n" + + "SELECT * FROM t1\n" + + "LEFT JOIN LATERAL (\n" + + " SELECT x FROM t2\n" + + " WHERE t2.a = t1.a AND t2.ts <= t1.ts\n" + + " LIMIT 1\n" + + ") ON true\n" + + "LEFT JOIN LATERAL (\n" + + " SELECT x\n" + + " FROM t2\n" + + " WHERE t2.a = t1.a\n" + + ") ON true"; + sql(sql).withDecorrelate(false) + .withRule(CoreRules.PROJECT_CORRELATE_TRANSPOSE) + .checkUnchanged(); + } + + /** Test case of + * [CALCITE-6681] + * NullPointerException in ProjectCorrelateTransposeRule. */ + @Test void testLateralTransposeWithDecorrelateTrue() { + final String sql = "WITH " + + " t1(a, ts) AS (VALUES('a', 1))," + + " t2(a, ts, x) AS (SELECT ename as a, empno as ts, mgr as x FROM emp)\n" + + "SELECT * FROM t1\n" + + "LEFT JOIN LATERAL (\n" + + " SELECT x FROM t2\n" + + " WHERE t2.a = t1.a AND t2.ts <= t1.ts\n" + + " LIMIT 1\n" + + ") ON true\n" + + "LEFT JOIN LATERAL (\n" + + " SELECT x\n" + + " FROM t2\n" + + " WHERE t2.a = t1.a\n" + + ") ON true"; + sql(sql).withDecorrelate(true) + .withRule(CoreRules.PROJECT_CORRELATE_TRANSPOSE) + .checkUnchanged(); + } + /** Test case for CALCITE-5683 for two level nested decorrelate with standard program * failing during the decorrelation phase. The correlation variable is used at two levels * deep. */ diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index a653e7e0d134..df13f4c78aea 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -8720,6 +8720,74 @@ LogicalProject(EMPNO=[$0], ENAME=[$1], EXPR$0=[$9], EXPR$1=[$10]) LogicalProject(EMPNO=[$0], ENAME=[$1], EXPR$0=[$9], EXPR$1=[$10]) LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7], SLACKER=[$8], $f9=[5], $f10=[5]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +]]> + + + + + + + + + + + + + + + + From 1926fc4a9dcc48b69d66aec63cc086229afa980f Mon Sep 17 00:00:00 2001 From: Thomas Rebele Date: Thu, 20 Nov 2025 14:07:43 +0100 Subject: [PATCH 031/562] [CALCITE-7296] RexSimplify should not simplify IS NULL(CAST(10/0 AS BIGINT)) --- .../org/apache/calcite/rex/RexSimplify.java | 30 ++++++++++--------- .../apache/calcite/rex/RexProgramTest.java | 5 ++++ 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rex/RexSimplify.java b/core/src/main/java/org/apache/calcite/rex/RexSimplify.java index 3b0b04043b51..bb7ccf70505a 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexSimplify.java +++ b/core/src/main/java/org/apache/calcite/rex/RexSimplify.java @@ -1153,20 +1153,21 @@ private RexNode simplifyIs(RexCall call, RexUnknownAs unknownAs) { if (!a.getType().isNullable() && isSafe) { return rexBuilder.makeLiteral(true); } + RexNode simplifiedResult = null; if (RexUtil.isLosslessCast(a)) { - if (!a.getType().isNullable()) { - return rexBuilder.makeLiteral(true); - } - return rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL, RexUtil.removeCast(a)); + a = RexUtil.removeCast(a); + // to keep this simplification, we must return IS NOT NULL(a), + // even if we cannot do anything else + simplifiedResult = rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL, a); } if (predicates.pulledUpPredicates.contains(a)) { return rexBuilder.makeLiteral(true); } if (hasCustomNullabilityRules(a.getKind())) { - return null; + return simplifiedResult; } if (!isSafe) { - return rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL, a); + return simplifiedResult; } switch (Strong.policy(a)) { case NOT_NULL: @@ -1197,7 +1198,7 @@ private RexNode simplifyIs(RexCall call, RexUnknownAs unknownAs) { } case AS_IS: default: - return null; + return simplifiedResult; } } @@ -1212,20 +1213,21 @@ private RexNode simplifyIs(RexCall call, RexUnknownAs unknownAs) { if (!a.getType().isNullable() && isSafe) { return rexBuilder.makeLiteral(false); } + RexNode simplifiedResult = null; if (RexUtil.isLosslessCast(a)) { - if (!a.getType().isNullable()) { - return rexBuilder.makeLiteral(false); - } - return rexBuilder.makeCall(SqlStdOperatorTable.IS_NULL, RexUtil.removeCast(a)); + a = RexUtil.removeCast(a); + // to keep this simplification, we must return IS NULL(a), + // even if we cannot do anything else + simplifiedResult = rexBuilder.makeCall(SqlStdOperatorTable.IS_NULL, a); } if (RexUtil.isNull(a)) { return rexBuilder.makeLiteral(true); } if (hasCustomNullabilityRules(a.getKind())) { - return null; + return simplifiedResult; } if (!isSafe) { - return rexBuilder.makeCall(SqlStdOperatorTable.IS_NULL, a); + return simplifiedResult; } switch (Strong.policy(a)) { case NOT_NULL: @@ -1246,7 +1248,7 @@ private RexNode simplifyIs(RexCall call, RexUnknownAs unknownAs) { return RexUtil.composeDisjunction(rexBuilder, operands, false); case AS_IS: default: - return null; + return simplifiedResult; } } diff --git a/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java b/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java index 5e727d300521..877878aedef6 100644 --- a/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java +++ b/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java @@ -2776,6 +2776,11 @@ trueLiteral, literal(1), checkSimplify( isNotNull(div(vDecimalNotNull(), cast(literal(BigDecimal.valueOf(2.5)), intType))), "true"); + + checkSimplify(isNull(cast(div(vIntNotNull(), literal(0)), tBigInt())), + "IS NULL(/(?0.notNullInt0, 0))"); + checkSimplify(isNotNull(cast(div(vIntNotNull(), literal(0)), tBigInt())), + "IS NOT NULL(/(?0.notNullInt0, 0))"); } /** From 8fe4b590db658b3fbca63655778b1e8aa0abcf2f Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Mon, 1 Dec 2025 06:49:40 +0800 Subject: [PATCH 032/562] [CALCITE-7087] SQLite does not support RIGHT/FULL JOIN until version 3.39.0 --- .../calcite/rel/rel2sql/SqlImplementor.java | 4 + .../calcite/sql/dialect/SqliteSqlDialect.java | 19 +++++ .../rel/rel2sql/RelToSqlConverterTest.java | 85 ++++++++++++++++++- 3 files changed, 107 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java b/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java index 8952736563c6..ece87d0b5c2b 100644 --- a/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java +++ b/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java @@ -194,6 +194,10 @@ public final Result visitRoot(RelNode r) { rules.add(FullToLeftAndRightJoinRule.Config.DEFAULT.toRule()); } + if (!this.dialect.supportsJoinType(JoinRelType.RIGHT)) { + rules.add(CoreRules.JOIN_COMMUTE_RIGHT_TO_LEFT); + } + if (!this.dialect.supportsOrderByLiteral()) { rules.add(CoreRules.SORT_REMOVE_CONSTANT_KEYS); } diff --git a/core/src/main/java/org/apache/calcite/sql/dialect/SqliteSqlDialect.java b/core/src/main/java/org/apache/calcite/sql/dialect/SqliteSqlDialect.java index cd809efa8b11..18e865f7e957 100644 --- a/core/src/main/java/org/apache/calcite/sql/dialect/SqliteSqlDialect.java +++ b/core/src/main/java/org/apache/calcite/sql/dialect/SqliteSqlDialect.java @@ -17,6 +17,7 @@ package org.apache.calcite.sql.dialect; import org.apache.calcite.config.NullCollation; +import org.apache.calcite.rel.core.JoinRelType; import org.apache.calcite.sql.SqlCall; import org.apache.calcite.sql.SqlDialect; import org.apache.calcite.sql.SqlNode; @@ -40,9 +41,27 @@ public class SqliteSqlDialect extends SqlDialect { public static final SqlDialect DEFAULT = new SqliteSqlDialect(DEFAULT_CONTEXT); + private final int majorVersion; + private final int minorVersion; + /** Creates a SqliteSqlDialect. */ public SqliteSqlDialect(SqlDialect.Context context) { super(context); + this.majorVersion = context.databaseMajorVersion(); + this.minorVersion = context.databaseMinorVersion(); + } + + @Override public boolean supportsJoinType(JoinRelType joinType) { + // Unknown version means we conservatively assume support for no join types + if (majorVersion < 0) { + return false; + } + + // For non-RIGHT/FULL join types, SQLite supports them in any version + // For RIGHT/FULL joins, SQLite added support in 3.39.0 + // See: https://www.sqlite.org/releaselog/3_39_0.html + return (joinType != JoinRelType.RIGHT && joinType != JoinRelType.FULL) + || (majorVersion > 3 || (majorVersion == 3 && minorVersion >= 39)); } @Override public boolean supportsAliasedValues() { diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java index fd13824c8733..b96530fc2aa0 100644 --- a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java +++ b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java @@ -3164,10 +3164,86 @@ private SqlDialect nonOrdinalDialect() { .withHsqldb().ok(expectedHsqldb); } + /** Test case for + * [CALCITE-7087] + * SQLite does not support RIGHT/FULL JOIN until version 3.39.0. */ + @Test void testSqliteRightJoinRewrittenToLeftOnOldVersion() { + final String query = "SELECT \"EMP\".\"ENAME\", \"DEPT\".\"DNAME\"\n" + + "FROM \"scott\".\"EMP\"\n" + + "RIGHT JOIN \"scott\".\"DEPT\" ON \"EMP\".\"DEPTNO\" = \"DEPT\".\"DEPTNO\""; + final String expected = "SELECT \"EMP\".\"ENAME\", \"DEPT\".\"DNAME\"\n" + + "FROM \"scott\".\"DEPT\"\n" + + "LEFT JOIN \"scott\".\"EMP\" ON \"DEPT\".\"DEPTNO\" = \"EMP\".\"DEPTNO\""; + sql(query) + .schema(CalciteAssert.SchemaSpec.SCOTT) + .withSQLite(3, 38) + .ok(expected); + } + + /** Test case for + * [CALCITE-7087] + * SQLite does not support RIGHT/FULL JOIN until version 3.39.0. */ + @Test void testSqliteRightJoinKeptOnNewerVersion() { + final String query = "SELECT \"EMP\".\"ENAME\", \"DEPT\".\"DNAME\"\n" + + "FROM \"scott\".\"EMP\"\n" + + "RIGHT JOIN \"scott\".\"DEPT\" ON \"EMP\".\"DEPTNO\" = \"DEPT\".\"DEPTNO\""; + sql(query) + .schema(CalciteAssert.SchemaSpec.SCOTT) + .withSQLite(3, 39) + .ok(query); + } + + /** Test case for + * [CALCITE-7087] + * SQLite does not support RIGHT/FULL JOIN until version 3.39.0. */ + @Test void testSqliteFullJoinThrowsOnOldVersion() { + final String query = "SELECT \"EMP\".\"ENAME\", \"DEPT\".\"DNAME\"\n" + + "FROM \"scott\".\"EMP\"\n" + + "FULL JOIN \"scott\".\"DEPT\" ON \"EMP\".\"DEPTNO\" = \"DEPT\".\"DEPTNO\""; + final String expected = "SELECT \"ENAME\", \"DNAME\"\n" + + "FROM (SELECT *\n" + + "FROM \"scott\".\"EMP\"\n" + + "LEFT JOIN \"scott\".\"DEPT\" ON \"EMP\".\"DEPTNO\" = \"DEPT\".\"DEPTNO\"\n" + + "UNION ALL\n" + + "SELECT *\n" + + "FROM (SELECT \"EMP0\".\"EMPNO\"," + + " \"EMP0\".\"ENAME\"," + + " \"EMP0\".\"JOB\"," + + " \"EMP0\".\"MGR\"," + + " \"EMP0\".\"HIREDATE\"," + + " \"EMP0\".\"SAL\"," + + " \"EMP0\".\"COMM\"," + + " \"EMP0\".\"DEPTNO\"," + + " \"DEPT0\".\"DEPTNO\" AS \"DEPTNO0\"," + + " \"DEPT0\".\"DNAME\"," + + " \"DEPT0\".\"LOC\"\n" + + "FROM \"scott\".\"DEPT\" AS \"DEPT0\"\n" + + "LEFT JOIN \"scott\".\"EMP\" AS \"EMP0\"" + + " ON \"DEPT0\".\"DEPTNO\" = \"EMP0\".\"DEPTNO\") AS \"t\"\n" + + "WHERE \"t\".\"DEPTNO\" = \"t\".\"DEPTNO0\" IS NOT TRUE) AS \"t1\""; + sql(query) + .schema(CalciteAssert.SchemaSpec.SCOTT) + .withSQLite(3, 38) + .ok(expected); + } + + /** Test case for + * [CALCITE-7087] + * SQLite does not support RIGHT/FULL JOIN until version 3.39.0. */ + @Test void testSqliteFullJoinKeptOnNewerVersion() { + final String query = "SELECT *\n" + + "FROM \"scott\".\"EMP\"\n" + + "FULL JOIN \"scott\".\"DEPT\"" + + " ON \"EMP\".\"DEPTNO\" = \"DEPT\".\"DEPTNO\""; + sql(query) + .schema(CalciteAssert.SchemaSpec.SCOTT) + .withSQLite(3, 39) + .ok(query); + } + /** Test case for * [CALCITE-3771] * Support of TRIM function for SPARK dialect and improvement in HIVE Dialect. */ - @Test void testHiveAndSparkTrimWithLeadingChar() { final String query = "SELECT TRIM(LEADING 'a' from 'abcd')\n" + "from \"foodmart\".\"reserve_employee\""; @@ -11027,6 +11103,13 @@ Sql withSQLite() { return dialect(DatabaseProduct.SQLITE.getDialect()); } + Sql withSQLite(int majorVersion, int minorVersion) { + return dialect( + new SqliteSqlDialect(SqliteSqlDialect.DEFAULT_CONTEXT + .withDatabaseMajorVersion(majorVersion) + .withDatabaseMinorVersion(minorVersion))); + } + Sql withSybase() { return dialect(DatabaseProduct.SYBASE.getDialect()); } From c07de903db96bcccfbf96ac062ccd508bb8e1fcf Mon Sep 17 00:00:00 2001 From: xuzifu666 Date: Tue, 2 Dec 2025 17:00:21 +0800 Subject: [PATCH 033/562] [CALCITE-7309] Position is unparsed incorrectly for ClickHouseSqlDialect --- .../calcite/sql/dialect/ClickHouseSqlDialect.java | 12 ++++++++++++ .../calcite/rel/rel2sql/RelToSqlConverterTest.java | 13 +++++++++++++ 2 files changed, 25 insertions(+) diff --git a/core/src/main/java/org/apache/calcite/sql/dialect/ClickHouseSqlDialect.java b/core/src/main/java/org/apache/calcite/sql/dialect/ClickHouseSqlDialect.java index c4bd7802bfa4..884bb81261f8 100644 --- a/core/src/main/java/org/apache/calcite/sql/dialect/ClickHouseSqlDialect.java +++ b/core/src/main/java/org/apache/calcite/sql/dialect/ClickHouseSqlDialect.java @@ -245,6 +245,18 @@ private static SqlDataTypeSpec createSqlDataTypeSpecByName(String typeAlias, } writer.endList(arrayFrame); break; + case POSITION: + final SqlWriter.Frame f = writer.startFunCall("POSITION"); + writer.sep(","); + call.operand(1).unparse(writer, leftPrec, rightPrec); + writer.sep(","); + call.operand(0).unparse(writer, leftPrec, rightPrec); + if (call.operandCount() == 3) { + writer.sep(","); + call.operand(2).unparse(writer, leftPrec, rightPrec); + } + writer.endFunCall(f); + break; case FLOOR: if (call.operandCount() != 2) { super.unparseCall(writer, call, leftPrec, rightPrec); diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java index b96530fc2aa0..da2c2deefc9e 100644 --- a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java +++ b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java @@ -3510,6 +3510,19 @@ private SqlDialect nonOrdinalDialect() { .withDoris().ok(expectedStarRocks); } + /** Test case for + * [CALCITE-7309] + * Position is unparsed incorrectly for ClickHouseSqlDialect. */ + @Test void testPositionForClickHouse() { + final String query = "SELECT POSITION('a' IN 'abca')"; + final String expected = "SELECT POSITION('abca', 'a')"; + sql(query).withClickHouse().ok(expected); + + final String query1 = "SELECT POSITION('a' IN 'abca' FROM 1)"; + final String expected1 = "SELECT POSITION('abca', 'a', 1)"; + sql(query1).withClickHouse().ok(expected1); + } + @Test void testPositionFunctionForSqlite() { final String query = "select position('A' IN 'ABC') from \"product\""; final String expected = "SELECT INSTR('ABC', 'A')\n" From e2aef5831a7b8c1f8416224ebabb9d406f10a0b0 Mon Sep 17 00:00:00 2001 From: Sergey Nuyanzin Date: Wed, 3 Dec 2025 00:50:55 +0100 Subject: [PATCH 034/562] [CALCITE-7312] Alias is not auto generated for LATERAL TABLE --- .../calcite/sql/validate/SqlValidatorImpl.java | 5 +++-- .../apache/calcite/test/SqlValidatorTest.java | 18 +++++++++++++++++- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java index 03e8978efb19..00a02a3cc40e 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java @@ -2623,8 +2623,8 @@ private SqlNode registerFrom( case LATERAL: SqlBasicCall sbc = (SqlBasicCall) node; - registerFrom(parentScope, - usingScope, + newOperand = + registerFrom(parentScope, usingScope, register, ((SqlCall) node).operand(0), enclosingNode, @@ -2635,6 +2635,7 @@ private SqlNode registerFrom( // Put the usingScope which is a JoinScope, // in order to make visible the left items // of the JOIN tree. + sbc.setOperand(0, newOperand); scopes.put(node, usingScope); return sbc; diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index 129261c9e9e7..105e428efe7d 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -9127,13 +9127,23 @@ void testGroupExpressionEquivalenceParams() { /** Test case for * [CALCITE-7217] - * LATERAL is lost after validation. */ + * LATERAL is lost after validation and + * [CALCITE-7312] + * Alias is not auto generated for LATERAL TABLE. + * */ @Test void testCollectionTableWithLateralRewrite() { sql("select * from emp, lateral table(ramp(emp.deptno)), dept") .rewritesTo("SELECT *\n" + "FROM `EMP`,\n" + "LATERAL TABLE(RAMP(`EMP`.`DEPTNO`)),\n" + "`DEPT`"); + // SELECT 1 to save space since test is verifying alias for the case of LATERAL TABLE + sql("select 1 from emp, lateral table(ramp(emp.deptno)), dept") + .withValidatorIdentifierExpansion(true) + .rewritesTo("SELECT 1\n" + + "FROM `CATALOG`.`SALES`.`EMP` AS `EMP`,\n" + + "LATERAL TABLE(RAMP(`EMP`.`DEPTNO`)) AS `EXPR$0`,\n" + + "`CATALOG`.`SALES`.`DEPT` AS `DEPT`"); // As above, with alias sql("select * from emp, lateral table(ramp(emp.deptno)) as t(a), dept") .rewritesTo("SELECT *\n" @@ -9174,6 +9184,12 @@ void testGroupExpressionEquivalenceParams() { .rewritesTo("SELECT *\n" + "FROM LATERAL TABLE(RAMP(1234)),\n" + "`EMP`"); + // SELECT 1 to save space since test is verifying alias for the case of LATERAL TABLE + sql("select 1 from lateral table(ramp(1234)), emp") + .withValidatorIdentifierExpansion(true) + .rewritesTo("SELECT 1\n" + + "FROM LATERAL TABLE(RAMP(1234)) AS `EXPR$0`,\n" + + "`CATALOG`.`SALES`.`EMP` AS `EMP`"); } @Test void testCollectionTableWithCursorParam() { From 2ea6b9c34cc5462ac3282e096ffeca939e9a81e0 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Wed, 3 Dec 2025 14:27:15 -0800 Subject: [PATCH 035/562] Carry source position information through more code rewrites Signed-off-by: Mihai Budiu --- .../org/apache/calcite/rex/RexSimplify.java | 45 ++++--- .../java/org/apache/calcite/rex/RexUtil.java | 11 +- .../sql2rel/StandardConvertletTable.java | 120 +++++++++--------- 3 files changed, 97 insertions(+), 79 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rex/RexSimplify.java b/core/src/main/java/org/apache/calcite/rex/RexSimplify.java index bb7ccf70505a..7065c25c5c7f 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexSimplify.java +++ b/core/src/main/java/org/apache/calcite/rex/RexSimplify.java @@ -358,7 +358,7 @@ RexNode isNotFalse(RexNode e) { ? isNotTrue(((RexCall) e).operands.get(0)) : predicates.isEffectivelyNotNull(e) ? e // would "CAST(e AS BOOLEAN NOT NULL)" better? - : rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_FALSE, e); + : rexBuilder.makeCall(RexUtil.getPos(e), SqlStdOperatorTable.IS_NOT_FALSE, e); } /** Applies IS NOT TRUE to an expression. */ @@ -371,7 +371,7 @@ RexNode isNotTrue(RexNode e) { ? isNotFalse(((RexCall) e).operands.get(0)) : predicates.isEffectivelyNotNull(e) ? not(e) - : rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_TRUE, e); + : rexBuilder.makeCall(RexUtil.getPos(e), SqlStdOperatorTable.IS_NOT_TRUE, e); } /** Applies IS TRUE to an expression. */ @@ -384,7 +384,7 @@ RexNode isTrue(RexNode e) { ? isFalse(((RexCall) e).operands.get(0)) : predicates.isEffectivelyNotNull(e) ? e // would "CAST(e AS BOOLEAN NOT NULL)" better? - : rexBuilder.makeCall(SqlStdOperatorTable.IS_TRUE, e); + : rexBuilder.makeCall(RexUtil.getPos(e), SqlStdOperatorTable.IS_TRUE, e); } /** Applies IS FALSE to an expression. */ @@ -397,7 +397,7 @@ RexNode isFalse(RexNode e) { ? isTrue(((RexCall) e).operands.get(0)) : predicates.isEffectivelyNotNull(e) ? not(e) - : rexBuilder.makeCall(SqlStdOperatorTable.IS_FALSE, e); + : rexBuilder.makeCall(RexUtil.getPos(e), SqlStdOperatorTable.IS_FALSE, e); } /** @@ -1131,13 +1131,13 @@ private RexNode simplifyIs(RexCall call, RexUnknownAs unknownAs) { // because of null values. final SqlOperator notKind = RexUtil.op(kind.negateNullSafe()); final RexNode arg = ((RexCall) a).operands.get(0); - return simplify(rexBuilder.makeCall(notKind, arg), UNKNOWN); + return simplify(rexBuilder.makeCall(RexUtil.getPos(a), notKind, arg), UNKNOWN); default: break; } final RexNode a2 = simplify(a, UNKNOWN); if (a != a2) { - return rexBuilder.makeCall(RexUtil.op(kind), ImmutableList.of(a2)); + return rexBuilder.makeCall(RexUtil.getPos(a), RexUtil.op(kind), ImmutableList.of(a2)); } return null; // cannot be simplified } @@ -1154,11 +1154,12 @@ private RexNode simplifyIs(RexCall call, RexUnknownAs unknownAs) { return rexBuilder.makeLiteral(true); } RexNode simplifiedResult = null; + SqlParserPos pos = RexUtil.getPos(a); if (RexUtil.isLosslessCast(a)) { a = RexUtil.removeCast(a); // to keep this simplification, we must return IS NOT NULL(a), // even if we cannot do anything else - simplifiedResult = rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL, a); + simplifiedResult = rexBuilder.makeCall(pos, SqlStdOperatorTable.IS_NOT_NULL, a); } if (predicates.pulledUpPredicates.contains(a)) { return rexBuilder.makeLiteral(true); @@ -1180,7 +1181,7 @@ private RexNode simplifyIs(RexCall call, RexUnknownAs unknownAs) { final RexNode simplified = simplifyIsNotNull(operand); if (simplified == null) { operands.add( - rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL, operand)); + rexBuilder.makeCall(RexUtil.getPos(a), SqlStdOperatorTable.IS_NOT_NULL, operand)); } else if (simplified.isAlwaysFalse()) { return rexBuilder.makeLiteral(false); } else { @@ -1208,6 +1209,7 @@ private RexNode simplifyIs(RexCall call, RexUnknownAs unknownAs) { // For example, given // "(CASE WHEN FALSE THEN 1 ELSE 2) IS NULL" we first simplify the // argument to "2", and only then we can simplify "2 IS NULL" to "FALSE". + SqlParserPos pos = RexUtil.getPos(a); a = simplify(a, UNKNOWN); boolean isSafe = isSafeExpression(a); if (!a.getType().isNullable() && isSafe) { @@ -1218,7 +1220,7 @@ private RexNode simplifyIs(RexCall call, RexUnknownAs unknownAs) { a = RexUtil.removeCast(a); // to keep this simplification, we must return IS NULL(a), // even if we cannot do anything else - simplifiedResult = rexBuilder.makeCall(SqlStdOperatorTable.IS_NULL, a); + simplifiedResult = rexBuilder.makeCall(pos, SqlStdOperatorTable.IS_NULL, a); } if (RexUtil.isNull(a)) { return rexBuilder.makeLiteral(true); @@ -1240,7 +1242,7 @@ private RexNode simplifyIs(RexCall call, RexUnknownAs unknownAs) { final RexNode simplified = simplifyIsNull(operand); if (simplified == null) { operands.add( - rexBuilder.makeCall(SqlStdOperatorTable.IS_NULL, operand)); + rexBuilder.makeCall(RexUtil.getPos(a), SqlStdOperatorTable.IS_NULL, operand)); } else { operands.add(simplified); } @@ -1359,7 +1361,9 @@ && isSafeExpression(newCond)) { // in this case, last branch and new branch have the same conclusion, // hence we create a new composite condition and we do not add it to // the final branches for the time being - newCond = rexBuilder.makeCall(SqlStdOperatorTable.OR, lastBranch.cond, newCond); + newCond = + rexBuilder.makeCall(call.getParserPosition(), SqlStdOperatorTable.OR, + lastBranch.cond, newCond); conditionNeedsSimplify = true; } else { // if we reach here, the new branch is not mergeable with the last one, @@ -1811,7 +1815,8 @@ RexNode simplifyAnd2(List terms, List notTerms) { for (RexNode notSatisfiableNullable : notSatisfiableNullables) { terms.add( simplifyIs((RexCall) - rexBuilder.makeCall(SqlStdOperatorTable.IS_NULL, notSatisfiableNullable), UNKNOWN)); + rexBuilder.makeCall(RexUtil.getPos(notSatisfiableNullable), + SqlStdOperatorTable.IS_NULL, notSatisfiableNullable), UNKNOWN)); } } // Add the NOT disjunctions back in. @@ -2059,7 +2064,7 @@ private > RexNode simplifyAnd2ForUnknownAsFalse( for (RexNode operand : notNullOperands) { if (!strongOperands.contains(operand)) { terms.add( - rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL, operand)); + rexBuilder.makeCall(RexUtil.getPos(operand), SqlStdOperatorTable.IS_NOT_NULL, operand)); } } return RexUtil.composeConjunction(rexBuilder, terms); @@ -2096,7 +2101,7 @@ private > RexNode simplifyUsingPredicates(RexNode e, // Range is always satisfied given these predicates; but nullability might // be problematic return simplify( - rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL, comparison.ref), + rexBuilder.makeCall(RexUtil.getPos(e), SqlStdOperatorTable.IS_NOT_NULL, comparison.ref), RexUnknownAs.UNKNOWN); } else if (rangeSet2.asRanges().size() == 1 && Iterables.getOnlyElement(rangeSet2.asRanges()).hasLowerBound() @@ -2105,7 +2110,7 @@ private > RexNode simplifyUsingPredicates(RexNode e, .equals(Iterables.getOnlyElement(rangeSet2.asRanges()).upperEndpoint())) { final Range r = Iterables.getOnlyElement(rangeSet2.asRanges()); // range is now a point; it's worth simplifying - return rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, comparison.ref, + return rexBuilder.makeCall(RexUtil.getPos(e), SqlStdOperatorTable.EQUALS, comparison.ref, rexBuilder.makeLiteral(r.lowerEndpoint(), comparison.literal.getType(), comparison.literal.getTypeName())); } else { @@ -2280,13 +2285,13 @@ private RexNode simplifyOrs(List terms, RexUnknownAs unknownAs) { && comparable1.compareTo(comparable2) != 0) { // X <> A OR X <> B => X IS NOT NULL OR NULL final RexNode isNotNull = - rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL, + rexBuilder.makeCall(RexUtil.getPos(term), SqlStdOperatorTable.IS_NOT_NULL, notEqualsComparison.ref); final RexNode constantNull = rexBuilder.makeNullLiteral(trueLiteral.getType()); final RexNode newCondition = simplify( - rexBuilder.makeCall(SqlStdOperatorTable.OR, isNotNull, + rexBuilder.makeCall(RexUtil.getPos(term), SqlStdOperatorTable.OR, isNotNull, constantNull), unknownAs); if (newCondition.isAlwaysTrue()) { @@ -2317,7 +2322,7 @@ private RexNode simplifyOrs(List terms, RexUnknownAs unknownAs) { } final RexNode isNotNull = - rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL, x); + rexBuilder.makeCall(RexUtil.getPos(term), SqlStdOperatorTable.IS_NOT_NULL, x); terms.set(terms.indexOf(x), simplifyIs((RexCall) isNotNull, unknownAs)); terms.set(i, rexBuilder.makeNullLiteral(x.getType())); i--; @@ -2635,6 +2640,7 @@ private RexNode simplifyTrim(RexCall e) { && trimType.equals(simplify(childNode.operands.get(0))) && trimed.equals(simplify(childNode.operands.get(1)))) { return simplifyTrim(childNode); + } } @@ -2642,8 +2648,7 @@ private RexNode simplifyTrim(RexCall e) { rexNodes.add(trimType); rexNodes.add(trimed); rexNodes.add(simplify(e.operands.get(2))); - RexNode rexNode = rexBuilder.makeCall(e.getType(), e.getOperator(), rexNodes); - return rexNode; + return rexBuilder.makeCall(e.getParserPosition(), e.getType(), e.getOperator(), rexNodes); } /** Method that returns whether we can rollup from inner time unit diff --git a/core/src/main/java/org/apache/calcite/rex/RexUtil.java b/core/src/main/java/org/apache/calcite/rex/RexUtil.java index 5904bba9229f..8c45ffc97ae6 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexUtil.java +++ b/core/src/main/java/org/apache/calcite/rex/RexUtil.java @@ -2379,6 +2379,15 @@ public static com.google.common.base.Function notFn( return e -> not(rexBuilder, e); } + /** If the RexNode contains position information, return it. + * Otherwise, return SqlParserPos.ZERO. */ + public static SqlParserPos getPos(RexNode e) { + if (e instanceof RexCall) { + return ((RexCall) e).getParserPosition(); + } + return SqlParserPos.ZERO; + } + /** Applies NOT to an expression. * *

    Unlike {@link #not}, may strengthen the type from {@code BOOLEAN} @@ -2390,7 +2399,7 @@ static RexNode not(final RexBuilder rexBuilder, RexNode input) { ? rexBuilder.makeLiteral(true) : input.getKind() == SqlKind.NOT ? ((RexCall) input).operands.get(0) - : rexBuilder.makeCall(SqlStdOperatorTable.NOT, input); + : rexBuilder.makeCall(getPos(input), SqlStdOperatorTable.NOT, input); } /** Returns whether an expression contains a {@link RexCorrelVariable}. */ diff --git a/core/src/main/java/org/apache/calcite/sql2rel/StandardConvertletTable.java b/core/src/main/java/org/apache/calcite/sql2rel/StandardConvertletTable.java index 7112fda4a89f..e8b1bcbd7a04 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/StandardConvertletTable.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/StandardConvertletTable.java @@ -252,29 +252,29 @@ private StandardConvertletTable() { // Expand "x NOT LIKE y" into "NOT (x LIKE y)" registerOp(SqlStdOperatorTable.NOT_LIKE, (cx, call) -> cx.convertExpression( - SqlStdOperatorTable.NOT.createCall(SqlParserPos.ZERO, - SqlStdOperatorTable.LIKE.createCall(SqlParserPos.ZERO, + SqlStdOperatorTable.NOT.createCall(call.getParserPosition(), + SqlStdOperatorTable.LIKE.createCall(call.getParserPosition(), call.getOperandList())))); // Expand "x NOT ILIKE y" into "NOT (x ILIKE y)" registerOp(SqlLibraryOperators.NOT_ILIKE, (cx, call) -> cx.convertExpression( - SqlStdOperatorTable.NOT.createCall(SqlParserPos.ZERO, - SqlLibraryOperators.ILIKE.createCall(SqlParserPos.ZERO, + SqlStdOperatorTable.NOT.createCall(call.getParserPosition(), + SqlLibraryOperators.ILIKE.createCall(call.getParserPosition(), call.getOperandList())))); // Expand "x NOT RLIKE y" into "NOT (x RLIKE y)" registerOp(SqlLibraryOperators.NOT_RLIKE, (cx, call) -> cx.convertExpression( - SqlStdOperatorTable.NOT.createCall(SqlParserPos.ZERO, - SqlLibraryOperators.RLIKE.createCall(SqlParserPos.ZERO, + SqlStdOperatorTable.NOT.createCall(call.getParserPosition(), + SqlLibraryOperators.RLIKE.createCall(call.getParserPosition(), call.getOperandList())))); // Expand "x NOT SIMILAR y" into "NOT (x SIMILAR y)" registerOp(SqlStdOperatorTable.NOT_SIMILAR_TO, (cx, call) -> cx.convertExpression( - SqlStdOperatorTable.NOT.createCall(SqlParserPos.ZERO, - SqlStdOperatorTable.SIMILAR_TO.createCall(SqlParserPos.ZERO, + SqlStdOperatorTable.NOT.createCall(call.getParserPosition(), + SqlStdOperatorTable.SIMILAR_TO.createCall(call.getParserPosition(), call.getOperandList())))); // Unary "+" has no effect, so expand "+ x" into "x". @@ -312,7 +312,7 @@ private StandardConvertletTable() { // "SQRT(x)" is equivalent to "POWER(x, .5)" registerOp(SqlStdOperatorTable.SQRT, (cx, call) -> cx.convertExpression( - SqlStdOperatorTable.POWER.createCall(SqlParserPos.ZERO, + SqlStdOperatorTable.POWER.createCall(call.getParserPosition(), call.operand(0), SqlLiteral.createExactNumeric("0.5", SqlParserPos.ZERO)))); @@ -320,7 +320,7 @@ private StandardConvertletTable() { // "POSITION(substring IN string)" registerOp(SqlLibraryOperators.STRPOS, (cx, call) -> cx.convertExpression( - SqlStdOperatorTable.POSITION.createCall(SqlParserPos.ZERO, + SqlStdOperatorTable.POSITION.createCall(call.getParserPosition(), call.operand(1), call.operand(0)))); // "INSTR(string, substring, position, occurrence) is equivalent to @@ -388,7 +388,7 @@ private StandardConvertletTable() { if (!getComponentTypeOrThrow(type).isStruct()) { return cx.convertExpression( SqlStdOperatorTable.ELEMENT_SLICE.createCall( - SqlParserPos.ZERO, operand)); + call.getParserPosition(), operand)); } // fallback on default behavior @@ -404,7 +404,7 @@ private StandardConvertletTable() { final SqlNode operand = call.operand(0); final RexNode expr = cx.convertExpression( - SqlStdOperatorTable.ELEMENT.createCall(SqlParserPos.ZERO, + SqlStdOperatorTable.ELEMENT.createCall(call.getParserPosition(), operand)); return cx.getRexBuilder().makeFieldAccess(expr, 0); }); @@ -559,25 +559,25 @@ private static RexNode convertInterval(SqlRexContext cx, SqlCall call) { //~ Methods ---------------------------------------------------------------- - private static RexNode or(RexBuilder rexBuilder, RexNode a0, RexNode a1) { - return rexBuilder.makeCall(SqlStdOperatorTable.OR, a0, a1); + private static RexNode or(SqlParserPos pos, RexBuilder rexBuilder, RexNode a0, RexNode a1) { + return rexBuilder.makeCall(pos, SqlStdOperatorTable.OR, a0, a1); } - private static RexNode eq(RexBuilder rexBuilder, RexNode a0, RexNode a1) { - return rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, a0, a1); + private static RexNode eq(SqlParserPos pos, RexBuilder rexBuilder, RexNode a0, RexNode a1) { + return rexBuilder.makeCall(pos, SqlStdOperatorTable.EQUALS, a0, a1); } - private static RexNode ge(RexBuilder rexBuilder, RexNode a0, RexNode a1) { - return rexBuilder.makeCall(SqlStdOperatorTable.GREATER_THAN_OR_EQUAL, a0, + private static RexNode ge(SqlParserPos pos, RexBuilder rexBuilder, RexNode a0, RexNode a1) { + return rexBuilder.makeCall(pos, SqlStdOperatorTable.GREATER_THAN_OR_EQUAL, a0, a1); } - private static RexNode le(RexBuilder rexBuilder, RexNode a0, RexNode a1) { - return rexBuilder.makeCall(SqlStdOperatorTable.LESS_THAN_OR_EQUAL, a0, a1); + private static RexNode le(SqlParserPos pos, RexBuilder rexBuilder, RexNode a0, RexNode a1) { + return rexBuilder.makeCall(pos, SqlStdOperatorTable.LESS_THAN_OR_EQUAL, a0, a1); } - private static RexNode and(RexBuilder rexBuilder, RexNode a0, RexNode a1) { - return rexBuilder.makeCall(SqlStdOperatorTable.AND, a0, a1); + private static RexNode and(SqlParserPos pos, RexBuilder rexBuilder, RexNode a0, RexNode a1) { + return rexBuilder.makeCall(pos, SqlStdOperatorTable.AND, a0, a1); } private static RexNode divideInt(SqlParserPos pos, RexBuilder rexBuilder, RexNode a0, @@ -749,7 +749,7 @@ protected RexNode convertCast( final SqlNode left = call.operand(0); final SqlNode right = call.operand(1); final SqlLiteral format = call.getOperandList().size() > 2 - ? call.operand(2) : SqlLiteral.createNull(SqlParserPos.ZERO); + ? call.operand(2) : SqlLiteral.createNull(call.getParserPosition()); final RexBuilder rexBuilder = cx.getRexBuilder(); final RexNode arg = cx.convertExpression(left); @@ -840,7 +840,7 @@ protected RexNode convertFloorCeil(SqlRexContext cx, SqlCall call) { final RexBuilder rexBuilder = cx.getRexBuilder(); RexNode zero = rexBuilder.makeExactLiteral(BigDecimal.valueOf(0)); - RexNode cond = ge(rexBuilder, rexInterval, zero); + RexNode cond = ge(pos, rexBuilder, rexInterval, zero); RexNode pad = rexBuilder.makeExactLiteral(val.subtract(BigDecimal.ONE)); @@ -1428,9 +1428,10 @@ public RexNode convertBetween( final RexNode z = list.get(SqlBetweenOperator.UPPER_OPERAND); final RexBuilder rexBuilder = cx.getRexBuilder(); - RexNode ge1 = ge(rexBuilder, x, y); - RexNode le1 = le(rexBuilder, x, z); - RexNode and1 = and(rexBuilder, ge1, le1); + final SqlParserPos pos = call.getParserPosition(); + RexNode ge1 = ge(pos, rexBuilder, x, y); + RexNode le1 = le(pos, rexBuilder, x, z); + RexNode and1 = and(pos, rexBuilder, ge1, le1); RexNode res; final SqlBetweenOperator.Flag symmetric = op.flag; @@ -1439,10 +1440,10 @@ public RexNode convertBetween( res = and1; break; case SYMMETRIC: - RexNode ge2 = ge(rexBuilder, x, z); - RexNode le2 = le(rexBuilder, x, y); - RexNode and2 = and(rexBuilder, ge2, le2); - res = or(rexBuilder, and1, and2); + RexNode ge2 = ge(pos, rexBuilder, x, z); + RexNode le2 = le(pos, rexBuilder, x, y); + RexNode and2 = and(pos, rexBuilder, ge2, le2); + res = or(pos, rexBuilder, and1, and2); break; default: throw Util.unexpected(symmetric); @@ -1551,34 +1552,35 @@ public RexNode convertOverlaps( // Sort end points into start and end, such that (s0 <= e0) and (s1 <= e1). final RexBuilder rexBuilder = cx.getRexBuilder(); - RexNode leftSwap = le(rexBuilder, r0, r1); + final SqlParserPos pos = call.getParserPosition(); + RexNode leftSwap = le(pos, rexBuilder, r0, r1); final RexNode s0 = case_(rexBuilder, leftSwap, r0, r1); final RexNode e0 = case_(rexBuilder, leftSwap, r1, r0); - RexNode rightSwap = le(rexBuilder, r2, r3); + RexNode rightSwap = le(pos, rexBuilder, r2, r3); final RexNode s1 = case_(rexBuilder, rightSwap, r2, r3); final RexNode e1 = case_(rexBuilder, rightSwap, r3, r2); // (e0 >= s1) AND (e1 >= s0) switch (op.kind) { case OVERLAPS: - return and(rexBuilder, - ge(rexBuilder, e0, s1), - ge(rexBuilder, e1, s0)); + return and(pos, rexBuilder, + ge(pos, rexBuilder, e0, s1), + ge(pos, rexBuilder, e1, s0)); case CONTAINS: - return and(rexBuilder, - le(rexBuilder, s0, s1), - ge(rexBuilder, e0, e1)); + return and(pos, rexBuilder, + le(pos, rexBuilder, s0, s1), + ge(pos, rexBuilder, e0, e1)); case PERIOD_EQUALS: - return and(rexBuilder, - eq(rexBuilder, s0, s1), - eq(rexBuilder, e0, e1)); + return and(pos, rexBuilder, + eq(pos, rexBuilder, s0, s1), + eq(pos, rexBuilder, e0, e1)); case PRECEDES: - return le(rexBuilder, e0, s1); + return le(pos, rexBuilder, e0, s1); case IMMEDIATELY_PRECEDES: - return eq(rexBuilder, e0, s1); + return eq(pos, rexBuilder, e0, s1); case SUCCEEDS: - return ge(rexBuilder, s0, e1); + return ge(pos, rexBuilder, s0, e1); case IMMEDIATELY_SUCCEEDS: - return eq(rexBuilder, s0, e1); + return eq(pos, rexBuilder, s0, e1); default: throw new AssertionError(op); } @@ -1657,18 +1659,19 @@ private static class RegrCovarianceConvertlet implements SqlRexConvertlet { final SqlNode expr; final RelDataType type = cx.getValidator().getValidatedNodeType(call); + final SqlParserPos pos = call.getParserPosition(); switch (kind) { case COVAR_POP: - expr = expandCovariance(arg1, arg2, null, type, cx, true); + expr = expandCovariance(pos, arg1, arg2, null, type, cx, true); break; case COVAR_SAMP: - expr = expandCovariance(arg1, arg2, null, type, cx, false); + expr = expandCovariance(pos, arg1, arg2, null, type, cx, false); break; case REGR_SXX: - expr = expandRegrSzz(arg2, arg1, type, cx, true); + expr = expandRegrSzz(pos, arg2, arg1, type, cx, true); break; case REGR_SYY: - expr = expandRegrSzz(arg1, arg2, type, cx, true); + expr = expandRegrSzz(pos, arg1, arg2, type, cx, true); break; default: throw Util.unexpected(kind); @@ -1678,13 +1681,13 @@ private static class RegrCovarianceConvertlet implements SqlRexConvertlet { } private static SqlNode expandRegrSzz( + final SqlParserPos pos, final SqlNode arg1, final SqlNode arg2, final RelDataType avgType, final SqlRexContext cx, boolean variance) { - final SqlParserPos pos = SqlParserPos.ZERO; final SqlNode count = SqlStdOperatorTable.REGR_COUNT.createCall(pos, arg1, arg2); final SqlNode varPop = - expandCovariance(arg1, variance ? arg1 : arg2, arg2, avgType, cx, true); + expandCovariance(pos, arg1, variance ? arg1 : arg2, arg2, avgType, cx, true); final RexNode varPopRex = cx.convertExpression(varPop); final SqlNode varPopCast; varPopCast = getCastedSqlNode(varPop, avgType, pos, varPopRex); @@ -1692,6 +1695,7 @@ private static SqlNode expandRegrSzz( } private static SqlNode expandCovariance( + final SqlParserPos pos, final SqlNode arg0Input, final SqlNode arg1Input, final @Nullable SqlNode dependent, @@ -1705,8 +1709,7 @@ private static SqlNode expandCovariance( // covar_samp(x1, x2) ==> // (sum(x1 * x2) - sum(x1) * sum(x2) / count(x1, x2)) // / (count(x1, x2) - 1) - final SqlParserPos pos = SqlParserPos.ZERO; - final SqlLiteral nullLiteral = SqlLiteral.createNull(SqlParserPos.ZERO); + final SqlLiteral nullLiteral = SqlLiteral.createNull(pos); final RelDataType highPrecision = AvgVarianceConvertlet.highPrecision(cx, varType); final RexNode arg0Rex = cx.convertExpression(arg0Input); @@ -1751,7 +1754,7 @@ private static SqlNode expandCovariance( } else { final SqlNumericLiteral one = SqlLiteral.createExactNumeric("1", pos); denominator = - new SqlCase(SqlParserPos.ZERO, countCasted, + new SqlCase(pos, countCasted, SqlNodeList.of( SqlStdOperatorTable.EQUALS.createCall(pos, countCasted, one)), SqlNodeList.of(getCastedSqlNode(nullLiteral, highPrecision, pos, null)), @@ -2305,12 +2308,13 @@ private static class TimestampAddConvertlet implements SqlRexConvertlet { case 2: if (call.getOperator() == SqlLibraryOperators.ADD_MONTHS) { // Oracle-style 'ADD_MONTHS(date, integer months)' - qualifier = new SqlIntervalQualifier(TimeUnit.MONTH, null, SqlParserPos.ZERO); + qualifier = + new SqlIntervalQualifier(TimeUnit.MONTH, null, call.getParserPosition()); op2 = handleFirstParameter(cx, rexBuilder, call); op1 = handleSecondParameter(cx, rexBuilder, call); } else if (call.getOperator() == SqlLibraryOperators.DATE_ADD_SPARK) { // Spark-style 'DATE_ADD(date, integer days)' - qualifier = new SqlIntervalQualifier(TimeUnit.DAY, null, SqlParserPos.ZERO); + qualifier = new SqlIntervalQualifier(TimeUnit.DAY, null, call.getParserPosition()); op2 = handleFirstParameter(cx, rexBuilder, call); op1 = handleSecondParameter(cx, rexBuilder, call); } else { @@ -2405,7 +2409,7 @@ private static class TimestampSubConvertlet implements SqlRexConvertlet { final RexNode op2; if (call.getOperator() == SqlLibraryOperators.DATE_SUB_SPARK) { // Spark-style 'DATE_SUB(date, integer days)' - qualifier = new SqlIntervalQualifier(TimeUnit.DAY, null, SqlParserPos.ZERO); + qualifier = new SqlIntervalQualifier(TimeUnit.DAY, null, call.getParserPosition()); op2 = handleFirstParameter(cx, rexBuilder, call); op1 = handleSecondParameter(cx, rexBuilder, call); } else { From 2d6c5577eacd9963e327c57e4c27a97401e1cc20 Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Sun, 30 Nov 2025 23:09:17 +0800 Subject: [PATCH 036/562] [CALCITE-7207] Semi Join RelNode cannot be translated into correct MySQL SQL --- .../rel/rel2sql/RelToSqlConverter.java | 4 +- .../rel/rel2sql/RelToSqlConverterTest.java | 61 +++++++++++++++++++ core/src/test/resources/sql/join.iq | 21 +++++++ 3 files changed, 85 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java b/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java index 84fb8329ec2f..54b60c63d72f 100644 --- a/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java +++ b/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java @@ -326,7 +326,9 @@ protected Result visitAntiOrSemiJoin(Join e) { } sqlSelect.setWhere(sqlCondition); - if (leftResult.neededAlias != null && sqlSelect.getFrom() != null) { + if (leftResult.neededAlias != null + && sqlSelect.getFrom() != null + && sqlSelect.getFrom().getKind() != SqlKind.JOIN) { sqlSelect.setFrom(as(sqlSelect.getFrom(), leftResult.neededAlias)); } return result(sqlSelect, ImmutableList.of(Clause.FROM), e, null); diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java index da2c2deefc9e..a4f78852fb80 100644 --- a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java +++ b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java @@ -2051,6 +2051,67 @@ private static String toSql(RelNode root, SqlDialect dialect, assertThat(toSql(root), isLinux(expectedSql)); } + /** Test case for + * [CALCITE-7207] + * Semi Join RelNode cannot be translated into correct MySQL SQL. */ + @Test void testSemiJoinMysqlNoExtraAlias() { + final RelBuilder builder = relBuilder(); + // Create the left side + RelNode leftSubBase = builder + .scan("EMP") + .project(builder.field("EMPNO")) + .build(); + RelNode leftJoin = builder + .push(leftSubBase) + .scan("EMP") + .join( + JoinRelType.INNER, builder.equals( + builder.field(2, 0, "EMPNO"), + builder.field(2, 1, "EMPNO"))) + .project(builder.field(0), builder.field(1)) + .build(); + + // Create the right side + RelNode rightSubBase = builder + .scan("EMP") + .project(builder.field("EMPNO")) + .build(); + RelNode rightJoin = builder + .push(rightSubBase) + .scan("EMP") + .join( + JoinRelType.INNER, builder.equals( + builder.field(2, 0, "EMPNO"), + builder.field(2, 1, "EMPNO"))) + .project(builder.field(0), builder.field(1)) + .build(); + + // Top-level SEMI join + final RelNode root = builder + .push(leftJoin) + .push(rightJoin) + .join( + JoinRelType.SEMI, + builder.and( + builder.equals(builder.field(2, 1, "EMPNO"), builder.field(2, 0, "EMPNO")), + builder.call(SqlStdOperatorTable.LESS_THAN, + builder.field(2, 1, "EMPNO"), builder.field(2, 0, "EMPNO")))) + .project(builder.field(0), builder.field(1)) + .build(); + + final String expected = "SELECT \"t\".\"EMPNO\", \"EMP0\".\"EMPNO\" AS \"EMPNO0\"\n" + + "FROM (SELECT \"EMPNO\"\n" + + "FROM \"scott\".\"EMP\") AS \"t\"\n" + + "INNER JOIN \"scott\".\"EMP\" AS \"EMP0\" ON \"t\".\"EMPNO\" = \"EMP0\".\"EMPNO\"\n" + + "WHERE EXISTS (SELECT 1\n" + + "FROM (SELECT \"t1\".\"EMPNO\", \"EMP2\".\"EMPNO\" AS \"EMPNO0\"\n" + + "FROM (SELECT \"EMPNO\"\nFROM \"scott\".\"EMP\") AS \"t1\"\n" + + "INNER JOIN \"scott\".\"EMP\" AS \"EMP2\"" + + " ON \"t1\".\"EMPNO\" = \"EMP2\".\"EMPNO\") AS \"t2\"\n" + + "WHERE \"t\".\"EMPNO\" = \"t2\".\"EMPNO\" AND \"t\".\"EMPNO\" > \"t2\".\"EMPNO\")"; + assertThat(toSql(root), isLinux(expected)); + } + /** Test case for * [CALCITE-2792] * StackOverflowError while evaluating filter with large number of OR diff --git a/core/src/test/resources/sql/join.iq b/core/src/test/resources/sql/join.iq index 0c7573baee97..7911edbf6b7e 100644 --- a/core/src/test/resources/sql/join.iq +++ b/core/src/test/resources/sql/join.iq @@ -1092,4 +1092,25 @@ on t1.id = t2.id or not exists(select * !ok +# [CALCITE-7207] Semi Join RelNode cannot be translated into correct MySQL SQL +# This SQL comes from RelToSqlConverterTest.testSemiJoinMysqlNoExtraAlias() +!use scott-mysql +SELECT "t"."EMPNO", "EMP0"."EMPNO" AS "EMPNO0" +FROM (SELECT "EMPNO" +FROM "scott"."EMP") AS "t" +INNER JOIN "scott"."EMP" AS "EMP0" ON "t"."EMPNO" = "EMP0"."EMPNO" +WHERE EXISTS (SELECT 1 +FROM (SELECT "t1"."EMPNO", "EMP2"."EMPNO" AS "EMPNO0" +FROM (SELECT "EMPNO" +FROM "scott"."EMP") AS "t1" +INNER JOIN "scott"."EMP" AS "EMP2" ON "t1"."EMPNO" = "EMP2"."EMPNO") AS "t2" +WHERE "t"."EMPNO" = "t2"."EMPNO" AND "t"."EMPNO" > "t2"."EMPNO"); ++-------+--------+ +| EMPNO | EMPNO0 | ++-------+--------+ ++-------+--------+ +(0 rows) + +!ok + # End join.iq From ce2021b71a3dfef9d2daa429069473e2c45909fd Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Fri, 28 Nov 2025 14:32:40 -0800 Subject: [PATCH 037/562] [CALCITE-7305] Subqueries in ASOF JOIN MATCH_CONDITION cause an assertion failure Signed-off-by: Mihai Budiu --- .../sql/validate/IdentifierNamespace.java | 1 + .../sql/validate/SqlValidatorImpl.java | 18 ++++++++++----- .../apache/calcite/test/SqlValidatorTest.java | 22 +++++++++++++++++++ 3 files changed, 35 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql/validate/IdentifierNamespace.java b/core/src/main/java/org/apache/calcite/sql/validate/IdentifierNamespace.java index 9291752b90a3..f23ddec087ec 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/IdentifierNamespace.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/IdentifierNamespace.java @@ -69,6 +69,7 @@ public class IdentifierNamespace extends AbstractNamespace { * @param extendList Extension columns, or null * @param enclosingNode Enclosing node * @param parentScope Parent scope which this namespace turns to in order to + * resolve objects */ IdentifierNamespace(SqlValidatorImpl validator, SqlIdentifier id, @Nullable SqlNodeList extendList, @Nullable SqlNode enclosingNode, diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java index 00a02a3cc40e..96d46174d4a4 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java @@ -2600,6 +2600,9 @@ private SqlNode registerFrom( scopes.putIfAbsent(stripAs(join.getRight()), parentScope); scopes.putIfAbsent(stripAs(join.getLeft()), parentScope); registerSubQueries(joinScope, join.getCondition()); + if (join.getJoinType() == JoinType.ASOF || join.getJoinType() == JoinType.LEFT_ASOF) { + registerSubQueries(joinScope, ((SqlAsofJoin) join).getMatchCondition()); + } final JoinNamespace joinNamespace = new JoinNamespace(this, join); registerNamespace(null, null, joinNamespace, forceNullable); return join; @@ -3702,17 +3705,20 @@ private void checkRollUpInUsing(SqlIdentifier identifier, /** Get the number of scopes referenced by the specified node; the node * represents a computation that will be converted to a Rel node eventually. */ - private int getScopeCount(SqlNode node) { - SqlValidatorScope scope = scopes.get(node); + private int getScopeCount(@Nullable SqlValidatorScope scope) { if (scope == null) { // Not all nodes have an associated scope; count these as "1". // For example, a VALUES node. return 1; - } - if (scope instanceof ListScope) { - ListScope join = (ListScope) scope; + } else if (scope instanceof JoinScope) { + JoinScope join = (JoinScope) scope; return join.children.size(); + } else if (scope instanceof WithScope) { + return getScopeCount(((WithScope) scope).getParent()); } + // We don't need to handle arbitrary scopes here, because the argument scope + // is always from the left side of a join, and the SQL syntax constrains the + // kinds of subqueries that can appear in the left side of a join. return 1; } @@ -3815,7 +3821,7 @@ protected void validateJoin(SqlJoin join, SqlValidatorScope scope) { throw newValidationError(condition, RESOURCE.asofConditionMustBeComparison()); } - int leftScopeCount = getScopeCount(left); + int leftScopeCount = getScopeCount(scopes.get(left)); CompareFromBothSides validateCompare = new CompareFromBothSides(joinScope, leftScopeCount, catalogReader, RESOURCE.asofConditionMustBeComparison()); diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index 105e428efe7d..2de4ae67044d 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -3507,6 +3507,28 @@ void testWinPartClause() { } @Test void testAsOfJoin() { + sql("WITH " + + " T2(id, intt) AS (VALUES(1, 0)),\n" + + " T1(id, intt) as (VALUES(1, 0)),\n" + + " T3(id) AS (VALUES(1))\n" + + "SELECT t1.id, t2.intt\n" + + "FROM T1 LEFT ASOF JOIN T2\n" + + " MATCH_CONDITION t2.intt < t1.intt\n" + + " ON t1.id = t2.id") + .ok(); + + // Test case for [CALCITE-7305] + // Subqueries in ASOF JOIN MATCH_CONDITION cause an assertion failure + sql("WITH T1(id, intt) as (VALUES(1, 0)),\n" + + " T2(id, intt) AS (VALUES(1, 0)),\n" + + " T3(id) AS (VALUES(1))\n" + + "SELECT t1.id, t2.intt\n" + + "FROM T1 LEFT ASOF JOIN T2\n" + + " MATCH_CONDITION ^(t2.intt IN (SELECT id FROM T3))^\n" + + " ON t1.id = t2.id") + .fails("ASOF JOIN MATCH_CONDITION must be a comparison between columns " + + "from the two inputs"); + final String type0 = "RecordType(INTEGER NOT NULL EMPNO, INTEGER NOT NULL DEPTNO) NOT NULL"; final String sql0 = "select emp.empno, dept.deptno from emp asof join dept\n" + "match_condition emp.deptno <= dept.deptno\n" From 7adabd82a2d2408c757d03934a9bb7a4fe1a44eb Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Sun, 7 Dec 2025 17:33:25 +0800 Subject: [PATCH 038/562] [CALCITE-4525] Pull up predicate will lose some predicates when project contains same RexInputRef --- .../org/apache/calcite/test/RelMetadataTest.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java b/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java index 9213bc23454b..e8cc0eb81a5c 100644 --- a/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java @@ -3764,6 +3764,19 @@ private void checkPredicates(RelOptCluster cluster, RelOptTable empTable, assertThat(pulledUpPredicates, sortsAs("[]")); } + /** Test case for + * [CALCITE-4525] + * Pull up predicate will lose some predicates when project contains same RexInputRef. */ + @Test public void testPullUpPredicatesFromProject6() { + final String sql = "select MGR, MGR as manager, MGR as manager1" + + " from (select * from emp where MGR = 0)"; + final RelNode rel = sql(sql).toRel(); + final RelMetadataQuery mq = rel.getCluster().getMetadataQuery(); + RelOptPredicateList inputSet = mq.getPulledUpPredicates(rel); + ImmutableList pulledUpPredicates = inputSet.pulledUpPredicates; + assertThat(pulledUpPredicates, sortsAs("[=($0, 0), =($1, 0), =($2, 0)]")); + } + /** Test case for * [CALCITE-6599] * RelMdPredicates should pull up more predicates from VALUES From cc85ace118705016815f0308ed463d4777057224 Mon Sep 17 00:00:00 2001 From: iwanttobepowerful <745778074@qq.com> Date: Fri, 28 Nov 2025 18:40:13 +0800 Subject: [PATCH 039/562] [CALCITE-7303] Subqueries cannot be decorrelated if filter condition have multi CorrelationId --- .../calcite/sql2rel/RelDecorrelator.java | 14 +++- .../calcite/sql2rel/RelDecorrelatorTest.java | 84 +++++++++++++++++++ core/src/test/resources/sql/sub-query.iq | 62 ++++++++++++++ 3 files changed, 157 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java b/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java index e9d7f498f949..e45edff7e577 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java @@ -1495,17 +1495,25 @@ private Frame decorrelateInputWithValueGenerator(RelNode rel, Frame frame) { } /** Finds a {@link RexInputRef} that is equivalent to a {@link CorRef}, - * and if found, throws a {@link org.apache.calcite.util.Util.FoundOne}. */ + * and if found, throws a {@link org.apache.calcite.util.Util.FoundOne}. + * + *

    The equivalent expression must not contain any {@link RexFieldAccess}, + * ensuring that we only map the correlation variable to a local field or + * expression from the current relational expression (e.g., a {@link RexInputRef}), + * rather than to another correlation variable. + */ private static void findCorrelationEquivalent(CorRef correlation, RexNode e) throws Util.FoundOne { switch (e.getKind()) { case EQUALS: final RexCall call = (RexCall) e; final List operands = call.getOperands(); - if (references(operands.get(0), correlation)) { + if (!RexUtil.containsFieldAccess(operands.get(1)) + && references(operands.get(0), correlation)) { throw new Util.FoundOne(operands.get(1)); } - if (references(operands.get(1), correlation)) { + if (!RexUtil.containsFieldAccess(operands.get(0)) + && references(operands.get(1), correlation)) { throw new Util.FoundOne(operands.get(0)); } break; diff --git a/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java b/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java index c079af7740c9..f6504fdef9d2 100644 --- a/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java +++ b/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java @@ -272,6 +272,90 @@ public static Frameworks.ConfigBuilder config() { assertThat(after, hasTree(planAfter)); } + /** Test case for [CALCITE-7303] + * Subqueries cannot be decorrelated if filter condition have multi CorrelationId. */ + @Test void testCorrelationEquivalent() { + final FrameworkConfig frameworkConfig = config().build(); + final RelBuilder builder = RelBuilder.create(frameworkConfig); + final RelOptCluster cluster = builder.getCluster(); + final Planner planner = Frameworks.getPlanner(frameworkConfig); + final String sql = "SELECT deptno\n" + + "FROM emp e\n" + + "WHERE EXISTS (\n" + + " SELECT *\n" + + " FROM dept d\n" + + " WHERE EXISTS(\n" + + " SELECT *\n" + + " FROM bonus b\n" + + " WHERE b.ename = e.ename\n" + + " AND b.job = d.dname\n" + + " AND d.deptno = e.deptno))"; + final RelNode originalRel; + try { + final SqlNode parse = planner.parse(sql); + final SqlNode validate = planner.validate(parse); + originalRel = planner.rel(validate).rel; + } catch (Exception e) { + throw TestUtil.rethrow(e); + } + + final HepProgram hepProgram = HepProgram.builder() + .addRuleCollection( + ImmutableList.of( + // SubQuery program rules + CoreRules.FILTER_SUB_QUERY_TO_CORRELATE, + CoreRules.PROJECT_SUB_QUERY_TO_CORRELATE, + CoreRules.JOIN_SUB_QUERY_TO_CORRELATE)) + .build(); + final Program program = + Programs.of(hepProgram, true, + requireNonNull(cluster.getMetadataProvider())); + final RelNode before = + program.run(cluster.getPlanner(), originalRel, cluster.traitSet(), + Collections.emptyList(), Collections.emptyList()); + final String planBefore = "" + + "LogicalProject(DEPTNO=[$7])\n" + + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7])\n" + + " LogicalCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{1, 7}])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalAggregate(group=[{0}])\n" + + " LogicalProject(i=[true])\n" + + " LogicalProject(DEPTNO=[$0], DNAME=[$1], LOC=[$2])\n" + + " LogicalCorrelate(correlation=[$cor1], joinType=[inner], requiredColumns=[{0, 1}])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n" + + " LogicalAggregate(group=[{0}])\n" + + " LogicalProject(i=[true])\n" + + " LogicalFilter(condition=[AND(=($0, $cor0.ENAME), =(CAST($1):VARCHAR(14), $cor1.DNAME), =($cor1.DEPTNO, $cor0.DEPTNO))])\n" + + " LogicalTableScan(table=[[scott, BONUS]])\n"; + assertThat(before, hasTree(planBefore)); + + // Decorrelate without any rules, just "purely" decorrelation algorithm on RelDecorrelator + final RelNode after = + RelDecorrelator.decorrelateQuery(before, builder, RuleSets.ofList(Collections.emptyList()), + RuleSets.ofList(Collections.emptyList())); + // Verify plan + final String planAfter = "" + + "LogicalProject(DEPTNO=[$7])\n" + + " LogicalJoin(condition=[AND(=($1, $8), =($7, $9))], joinType=[inner])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalProject(ENAME0=[$0], DEPTNO0=[$1], $f2=[true])\n" + + " LogicalAggregate(group=[{0, 1}])\n" + + " LogicalProject(ENAME0=[$3], DEPTNO0=[$4])\n" + + " LogicalJoin(condition=[AND(=($0, $5), =($1, $6))], joinType=[inner])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n" + + " LogicalProject(ENAME0=[$0], DEPTNO=[$1], DEPTNO0=[$2], DNAME=[$3], $f4=[true])\n" + + " LogicalAggregate(group=[{0, 1, 2, 3}])\n" + + " LogicalProject(ENAME0=[$4], DEPTNO=[$5], DEPTNO0=[$6], DNAME=[$7])\n" + + " LogicalJoin(condition=[AND(=($0, $4), =(CAST($1):VARCHAR(14), $7))], joinType=[inner])\n" + + " LogicalTableScan(table=[[scott, BONUS]])\n" + + " LogicalJoin(condition=[=($2, $1)], joinType=[inner])\n" + + " LogicalProject(ENAME=[$1], DEPTNO=[$7])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalProject(DEPTNO=[$0], DNAME=[$1])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n"; + assertThat(after, hasTree(planAfter)); + } + /** * Test case for * [CALCITE-6468] RelDecorrelator diff --git a/core/src/test/resources/sql/sub-query.iq b/core/src/test/resources/sql/sub-query.iq index ab9a52fa795a..2fa71122fba1 100644 --- a/core/src/test/resources/sql/sub-query.iq +++ b/core/src/test/resources/sql/sub-query.iq @@ -5443,4 +5443,66 @@ ORDER BY deptno; !ok +# [CALCITE-7303] Subqueries cannot be decorrelated if filter condition have multi CorrelationId +SELECT deptno +FROM emp e +WHERE EXISTS + (SELECT * + FROM dept d + WHERE EXISTS + (SELECT * + FROM bonus b + WHERE b.ename = e.ename + AND d.deptno = e.deptno)); ++--------+ +| DEPTNO | ++--------+ ++--------+ +(0 rows) + +!ok + +# [CALCITE-7303] Subqueries cannot be decorrelated if filter condition have multi CorrelationId +SELECT deptno +FROM emp e +WHERE EXISTS ( + SELECT * + FROM dept d + WHERE EXISTS( + SELECT * + FROM bonus b + WHERE b.ename = e.ename + AND b.job = d.dname + AND d.deptno = e.deptno)); ++--------+ +| DEPTNO | ++--------+ ++--------+ +(0 rows) + +!ok + +# [CALCITE-7303] Subqueries cannot be decorrelated if filter condition have multi CorrelationId +WITH t0(t0a, t0b) AS (VALUES (1, 1), (1, 2), (2, 1), (2, 2), (2, 0)), + t1(t1a, t1b, t1c) AS (VALUES (1, 1, 3)), + t2(t2a, t2b, t2c) AS (VALUES (1, 1, 5), (2, 2, 7)) +SELECT t0a +FROM t0 e +WHERE EXISTS + (SELECT * + FROM t1 d + WHERE EXISTS + (SELECT * + FROM t2 b + WHERE b.t2b = e.t0b + AND d.t1a = e.t0a)); ++-----+ +| T0A | ++-----+ +| 1 | +| 1 | ++-----+ +(2 rows) + +!ok # End sub-query.iq From c900c79baa5b731b053a0ac9be98451b6f574fe7 Mon Sep 17 00:00:00 2001 From: TJ Banghart Date: Mon, 1 Dec 2025 10:27:47 -0800 Subject: [PATCH 040/562] [CALCITE-7254] Add rule for sharing trivially equivalent RelNodes within Combine --- .../adapter/enumerable/EnumerableCombine.java | 167 ++++++ .../enumerable/EnumerableCombineRule.java | 53 ++ .../adapter/enumerable/EnumerableRules.java | 6 + .../apache/calcite/plan/SpoolRelOptTable.java | 145 +++++ .../org/apache/calcite/rel/core/Combine.java | 42 +- .../rules/CombineSimpleEquivalenceRule.java | 212 ++++++++ .../apache/calcite/runtime/SqlFunctions.java | 37 ++ .../calcite/test/CombineRelOptRulesTest.java | 509 ++++++++++++++++++ .../apache/calcite/test/SqlFunctionsTest.java | 33 ++ .../enumerable/EnumerableCombineTest.java | 167 ++++++ .../calcite/test/CombineRelOptRulesTest.xml | 371 +++++++++++++ 11 files changed, 1730 insertions(+), 12 deletions(-) create mode 100644 core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableCombine.java create mode 100644 core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableCombineRule.java create mode 100644 core/src/main/java/org/apache/calcite/plan/SpoolRelOptTable.java create mode 100644 core/src/main/java/org/apache/calcite/rel/rules/CombineSimpleEquivalenceRule.java create mode 100644 core/src/test/java/org/apache/calcite/test/CombineRelOptRulesTest.java create mode 100644 core/src/test/java/org/apache/calcite/test/enumerable/EnumerableCombineTest.java create mode 100644 core/src/test/resources/org/apache/calcite/test/CombineRelOptRulesTest.xml diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableCombine.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableCombine.java new file mode 100644 index 000000000000..65340259eb23 --- /dev/null +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableCombine.java @@ -0,0 +1,167 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.adapter.enumerable; + +import org.apache.calcite.linq4j.Enumerable; +import org.apache.calcite.linq4j.Linq4j; +import org.apache.calcite.linq4j.Ord; +import org.apache.calcite.linq4j.tree.BlockBuilder; +import org.apache.calcite.linq4j.tree.Expression; +import org.apache.calcite.linq4j.tree.Expressions; +import org.apache.calcite.linq4j.tree.ParameterExpression; +import org.apache.calcite.linq4j.tree.Types; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelTraitSet; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.Combine; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.runtime.SqlFunctions; +import org.apache.calcite.util.BuiltInMethod; + +import java.util.ArrayList; +import java.util.List; + +/** Implementation of {@link org.apache.calcite.rel.core.Combine} in + * {@link org.apache.calcite.adapter.enumerable.EnumerableConvention enumerable calling convention}. + * + *

    The output format is a wide table where each column corresponds to a query + * (named QUERY_0, QUERY_1, etc.) and each row contains a struct (map) with that + * query's column values for that row index. The number of output rows equals the + * maximum row count across all input queries. Queries with fewer rows have null + * values for the additional rows. + * + *

    Example output for two queries: + * 

    + * QUERY_0                  | QUERY_1
+ * ------------------------ | ------------------------
+ * {empno=100, name=Bill}   | {deptno=10, name=Sales}
+ * {empno=110, name=Eric}   | {deptno=20, name=HR}
+ * {empno=120, name=Ted}    | null
+ *
    + */ +public class EnumerableCombine extends Combine implements EnumerableRel { + public EnumerableCombine(RelOptCluster cluster, RelTraitSet traitSet, + List inputs) { + super(cluster, traitSet, inputs); + } + + @Override public EnumerableCombine copy(RelTraitSet traitSet, List inputs) { + return new EnumerableCombine(getCluster(), traitSet, inputs); + } + + @Override public Result implement(EnumerableRelImplementor implementor, Prefer pref) { + final BlockBuilder builder = new BlockBuilder(); + final RelDataType rowType = getRowType(); + + // Collect all query results as lists of maps + // Each list corresponds to one query, containing maps for each row + final List queryLists = new ArrayList<>(); + + for (Ord ord : Ord.zip(inputs)) { + EnumerableRel input = (EnumerableRel) ord.e; + final Result result = implementor.visitChild(this, ord.i, input, pref); + Expression childExp = + builder.append( + "child" + ord.i, + result.block); + + // Get column names for this input + final List fields = input.getRowType().getFieldList(); + final int fieldCount = fields.size(); + + // Transform each row to a Map with column names as keys + ParameterExpression row = Expressions.parameter(Object.class, "row" + ord.i); + + // Build the arguments for SqlFunctions.map(key1, val1, key2, val2, ...) + List mapArgs = new ArrayList<>(); + for (int i = 0; i < fieldCount; i++) { + String colName = fields.get(i).getName(); + mapArgs.add(Expressions.constant(colName)); + if (fieldCount > 1) { + // Multi-column: access row[i] + mapArgs.add( + Expressions.arrayIndex( + Expressions.convert_(row, Object[].class), + Expressions.constant(i))); + } else { + // Single column: use row directly + mapArgs.add(row); + } + } + + Expression mapCall = + Expressions.call( + SqlFunctions.class, + "map", + Expressions.newArrayInit(Object.class, mapArgs)); + + Expression selectLambda = Expressions.lambda(mapCall, row); + Expression enumerableToConvert = + builder.append("converted" + ord.i, + Expressions.call( + childExp, + BuiltInMethod.SELECT.method, + selectLambda)); + + // Convert Enumerable to List + Expression listExp = + builder.append( + "list" + ord.i, + Expressions.call( + enumerableToConvert, + Types.lookupMethod( + Enumerable.class, + "toList"))); + + queryLists.add(listExp); + } + + // The physical type: each row is Object[] with one element per query + final PhysType physType = + PhysTypeImpl.of( + implementor.getTypeFactory(), + rowType, + pref.prefer(JavaRowFormat.ARRAY)); + + // Create an array of all query result lists + Expression queryListsArray = + builder.append("queryLists", + Expressions.newArrayInit(List.class, queryLists)); + + // Call helper method to combine results into rows + // combineQueryResults(List[] queryLists) -> List + Expression combinedRows = + builder.append("combinedRows", + Expressions.call( + SqlFunctions.class, + "combineQueryResults", + queryListsArray)); + + Expression enumerableExp = + Expressions.call( + Types.lookupMethod( + Linq4j.class, + "asEnumerable", + List.class), + combinedRows); + + builder.add(enumerableExp); + + return implementor.result(physType, builder.toBlock()); + } +} diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableCombineRule.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableCombineRule.java new file mode 100644 index 000000000000..b85e67d3e624 --- /dev/null +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableCombineRule.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.adapter.enumerable; + +import org.apache.calcite.plan.Convention; +import org.apache.calcite.plan.RelTraitSet; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.convert.ConverterRule; +import org.apache.calcite.rel.core.Combine; +import org.apache.calcite.util.Util; + +import java.util.List; + +/** + * Rule to convert a {@link Combine} to an {@link EnumerableCombine}. + * + * @see EnumerableRules#ENUMERABLE_COMBINE_RULE + */ +class EnumerableCombineRule extends ConverterRule { + /** Default configuration. */ + static final Config DEFAULT_CONFIG = Config.INSTANCE + .withConversion(Combine.class, Convention.NONE, + EnumerableConvention.INSTANCE, "EnumerableCombineRule") + .withRuleFactory(EnumerableCombineRule::new); + + /** Called from the Config. */ + protected EnumerableCombineRule(Config config) { + super(config); + } + + @Override public RelNode convert(RelNode rel) { + final Combine combine = (Combine) rel; + final EnumerableConvention out = EnumerableConvention.INSTANCE; + final RelTraitSet traitSet = rel.getCluster().traitSet().replace(out); + final List newInputs = + Util.transform(combine.getInputs(), n -> convert(n, traitSet)); + return new EnumerableCombine(rel.getCluster(), traitSet, newInputs); + } +} diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableRules.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableRules.java index a575d189432e..f33997450ca0 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableRules.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableRules.java @@ -103,6 +103,11 @@ private EnumerableRules() { public static final EnumerableUnionRule ENUMERABLE_UNION_RULE = EnumerableUnionRule.DEFAULT_CONFIG.toRule(EnumerableUnionRule.class); + /** Rule that converts a {@link org.apache.calcite.rel.core.Combine} + * to an {@link EnumerableCombine}. */ + public static final EnumerableCombineRule ENUMERABLE_COMBINE_RULE = + EnumerableCombineRule.DEFAULT_CONFIG.toRule(EnumerableCombineRule.class); + /** Rule that converts a {@link LogicalRepeatUnion} into an * {@link EnumerableRepeatUnion}. */ public static final EnumerableRepeatUnionRule ENUMERABLE_REPEAT_UNION_RULE = @@ -224,6 +229,7 @@ private EnumerableRules() { EnumerableRules.ENUMERABLE_UNCOLLECT_RULE, EnumerableRules.ENUMERABLE_MERGE_UNION_RULE, EnumerableRules.ENUMERABLE_UNION_RULE, + EnumerableRules.ENUMERABLE_COMBINE_RULE, EnumerableRules.ENUMERABLE_REPEAT_UNION_RULE, EnumerableRules.ENUMERABLE_TABLE_SPOOL_RULE, EnumerableRules.ENUMERABLE_INTERSECT_RULE, diff --git a/core/src/main/java/org/apache/calcite/plan/SpoolRelOptTable.java b/core/src/main/java/org/apache/calcite/plan/SpoolRelOptTable.java new file mode 100644 index 000000000000..0909b951c456 --- /dev/null +++ b/core/src/main/java/org/apache/calcite/plan/SpoolRelOptTable.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.plan; + +import org.apache.calcite.linq4j.tree.Expression; +import org.apache.calcite.rel.RelCollation; +import org.apache.calcite.rel.RelDistribution; +import org.apache.calcite.rel.RelDistributions; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.RelReferentialConstraint; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.schema.ColumnStrategy; +import org.apache.calcite.schema.Statistic; +import org.apache.calcite.schema.Statistics; +import org.apache.calcite.schema.Table; +import org.apache.calcite.schema.impl.ListTransientTable; +import org.apache.calcite.util.ImmutableBitSet; + +import com.google.common.collect.ImmutableList; + +import org.checkerframework.checker.nullness.qual.Nullable; + +import java.util.Collections; +import java.util.List; + +/** + * Implementation of {@link RelOptTable} for temporary spool tables. + * + *

    This table represents temporary storage used by spool operators + * during query execution. It's used for planning purposes only and + * will be converted to appropriate physical operators later. + */ +public class SpoolRelOptTable implements RelOptTable { + private final @Nullable RelOptSchema schema; + private final RelDataType rowType; + private final String name; + private final double rowCount; + private final Table table; + + /** + * Creates a SpoolRelOptTable with explicit row count. + * + * @param schema the schema this table belongs to (can be null for temporary tables) + * @param rowType the row type of the data that will be stored in this spool + * @param name optional name for the spool table + * @param rowCount the estimated number of rows that will be materialized in this spool + */ + public SpoolRelOptTable( + @Nullable RelOptSchema schema, + RelDataType rowType, + String name, + double rowCount) { + this.schema = schema; + this.rowType = rowType; + this.name = name; + this.rowCount = rowCount; + // Use standard ListTransientTable with custom statistics for accurate cost estimation + this.table = new ListTransientTable(name, rowType) { + @Override public Statistic getStatistic() { + return Statistics.of(rowCount, ImmutableList.of()); + } + }; + } + + @Override public RelNode toRel(ToRelContext context) { + // This shouldn't be called during planning - spools are created differently + throw new UnsupportedOperationException("SpoolRelOptTable.toRel should not be called"); + } + + @Override public List getQualifiedName() { + return ImmutableList.of("TEMP", name); + } + + @Override public double getRowCount() { + // Return the actual row count of the materialized data in this spool + return rowCount; + } + + @Override public RelDataType getRowType() { + return rowType; + } + + @Override public @Nullable RelOptSchema getRelOptSchema() { + return schema; + } + + @Override public @Nullable RelDistribution getDistribution() { + return RelDistributions.ANY; + } + + @Override public @Nullable List getKeys() { + // Spools typically don't have keys + return ImmutableList.of(); + } + + @Override public @Nullable List getReferentialConstraints() { + // Temporary tables don't have referential constraints + return ImmutableList.of(); + } + + @Override public @Nullable List getCollationList() { + // Could be extended to preserve collations from the input + return ImmutableList.of(); + } + + @Override public boolean isKey(ImmutableBitSet columns) { + return false; + } + + @Override public @Nullable Expression getExpression(Class clazz) { + // Return null so EnumerableTableScanRule won't try to convert spool table scans + // Spool table scans are handled within the spool operator itself + return null; + } + + @Override public RelOptTable extend(List extendedFields) { + throw new UnsupportedOperationException("SpoolRelOptTable.extend should not be called"); + } + + @Override public List getColumnStrategies() { + return Collections.emptyList(); + } + + @Override public @Nullable C unwrap(Class aClass) { + if (aClass.isInstance(table)) { + return aClass.cast(table); + } + return null; + } +} diff --git a/core/src/main/java/org/apache/calcite/rel/core/Combine.java b/core/src/main/java/org/apache/calcite/rel/core/Combine.java index 286facf2c837..6c99a3a4de1d 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/Combine.java +++ b/core/src/main/java/org/apache/calcite/rel/core/Combine.java @@ -35,9 +35,11 @@ /** * A relational operator that combines multiple relational expressions into a single root. * This is used for multi-root optimization in the VolcanoPlanner. + * + * @see org.apache.calcite.adapter.enumerable.EnumerableCombine */ public class Combine extends AbstractRelNode { - protected final ImmutableList inputs; + protected ImmutableList inputs; /** Creates a Combine. */ public static Combine create(RelOptCluster cluster, RelTraitSet traitSet, List inputs) { @@ -54,6 +56,25 @@ public Combine(RelOptCluster cluster, RelTraitSet traitSet, List inputs return inputs; } + @Override public RelNode copy(RelTraitSet traitSet, List inputs) { + return new Combine(getCluster(), traitSet, inputs); + } + + @Override public void replaceInput(int ordinalInParent, RelNode rel) { + // Combine has multiple inputs stored in an immutable list. + // To replace an input, we need to create a new list with the replacement. + ImmutableList.Builder newInputs = ImmutableList.builder(); + for (int i = 0; i < inputs.size(); i++) { + if (i == ordinalInParent) { + newInputs.add(rel); + } else { + newInputs.add(inputs.get(i)); + } + } + inputs = newInputs.build(); + } + + @Override public RelWriter explainTerms(RelWriter pw) { super.explainTerms(pw); for (Ord ord : Ord.zip(inputs)) { @@ -63,21 +84,18 @@ public Combine(RelOptCluster cluster, RelTraitSet traitSet, List inputs } @Override protected RelDataType deriveRowType() { - // Combine represents multiple independent result sets that are not merged. - // Each input maintains its own row type and is accessed independently. - // - // We use a struct type where each field represents one of the input queries. - // This allows metadata and optimization rules to understand the structure - // while making it clear that results are not unified into a single stream. - RelDataTypeFactory typeFactory = getCluster().getTypeFactory(); RelDataTypeFactory.Builder builder = typeFactory.builder(); + // One column per input query (QUERY_0, QUERY_1, etc.) + // Each cell is a nullable MAP representing a struct with column names as keys + RelDataType anyType = typeFactory.createJavaType(Object.class); + RelDataType mapType = + typeFactory.createMapType(typeFactory.createJavaType(String.class), anyType); + RelDataType nullableMapType = typeFactory.createTypeWithNullability(mapType, true); + for (int i = 0; i < inputs.size(); i++) { - RelNode input = inputs.get(i); - // Create a field for each input with its row type - // Field names are "QUERY_0", "QUERY_1", etc. - builder.add("QUERY_" + i, input.getRowType()); + builder.add("QUERY_" + i, nullableMapType); } return builder.build(); diff --git a/core/src/main/java/org/apache/calcite/rel/rules/CombineSimpleEquivalenceRule.java b/core/src/main/java/org/apache/calcite/rel/rules/CombineSimpleEquivalenceRule.java new file mode 100644 index 000000000000..eba15bcddc24 --- /dev/null +++ b/core/src/main/java/org/apache/calcite/rel/rules/CombineSimpleEquivalenceRule.java @@ -0,0 +1,212 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.rel.rules; + +import org.apache.calcite.plan.RelDigest; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptUtil; +import org.apache.calcite.plan.RelRule; +import org.apache.calcite.plan.SpoolRelOptTable; +import org.apache.calcite.rel.RelCommonExpressionBasicSuggester; +import org.apache.calcite.rel.RelHomogeneousShuttle; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.Combine; +import org.apache.calcite.rel.core.RelFactories; +import org.apache.calcite.rel.core.Spool; +import org.apache.calcite.rel.logical.LogicalTableScan; +import org.apache.calcite.rel.logical.LogicalTableSpool; +import org.apache.calcite.rel.metadata.RelMetadataQuery; + +import com.google.common.collect.ImmutableList; + +import org.immutables.value.Value; + +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +/** + * Rule that optimizes a {@link Combine} operator by detecting shared sub-expressions + * across its inputs and introducing {@link Spool}s to avoid redundant computation. + * + *

    This rule identifies structurally equivalent sub-plans within a Combine's inputs + * and replaces them with a spool pattern: the first occurrence becomes a producer + * (TableSpool that materializes the result), and subsequent occurrences become + * consumers (TableScan reading from the spooled data). + * + *

    Example

    + * + *

    Consider two queries combined that share a common filtered table scan: + * + * 

    {@code
+ * -- Query 1: Count high earners
+ * SELECT COUNT(*) FROM EMP WHERE SAL > 2000
+ * -- Query 2: Average salary of high earners
+ * SELECT AVG(SAL) FROM EMP WHERE SAL > 2000
+ * }
    + * + *

    Before this rule applies, the plan looks like: + * + * 

    {@code
+ * Combine
+ *   LogicalAggregate(group=[{}], CNT=[COUNT()])
+ *     LogicalFilter(condition=[>(SAL, 2000)])
+ *       LogicalTableScan(table=[EMP])
+ *   LogicalAggregate(group=[{}], AVG_SAL=[AVG(SAL)])
+ *     LogicalFilter(condition=[>(SAL, 2000)])
+ *       LogicalTableScan(table=[EMP])
+ * }
    + * + *

    After this rule identifies the shared {@code Filter(SAL > 2000) -> TableScan(EMP)} + * sub-expression, the plan becomes: + * + * 

    {@code
+ * Combine
+ *   LogicalAggregate(group=[{}], CNT=[COUNT()])
+ *     LogicalTableSpool(table=[spool_0])        -- Producer: materializes filtered rows
+ *       LogicalFilter(condition=[>(SAL, 2000)])
+ *         LogicalTableScan(table=[EMP])
+ *   LogicalAggregate(group=[{}], AVG_SAL=[AVG(SAL)])
+ *     LogicalTableScan(table=[spool_0])         -- Consumer: reads from spool
+ * }
    + * + * @see Combine + * @see Spool + * @see RelCommonExpressionBasicSuggester + */ +@Value.Enclosing +public class CombineSimpleEquivalenceRule extends RelRule { + + /** Creates a CombineSharedComponentsRule. */ + protected CombineSimpleEquivalenceRule(Config config) { + super(config); + } + + @Override public void onMatch(RelOptRuleCall call) { + RelNode combine = RelOptUtil.stripAll(call.rel(0)); + + // Use the suggester to find shared components + RelCommonExpressionBasicSuggester suggester = new RelCommonExpressionBasicSuggester(); + Collection sharedComponents = suggester.suggest(combine, null); + + // Filter out any components that are already spools or scans from spool tables + // to avoid creating spools of spools + sharedComponents = sharedComponents.stream() + .filter(node -> { + if (node instanceof Spool) { + return false; + } + // Skip if it's a TableScan reading from a spool table + if (node instanceof LogicalTableScan) { + LogicalTableScan scan = (LogicalTableScan) node; + // Check if the underlying table is a SpoolRelOptTable + return !(scan.getTable() instanceof SpoolRelOptTable); + } + return true; + }) + .collect(java.util.stream.Collectors.toList()); + + // If no shared components found, nothing to do + if (sharedComponents.isEmpty()) { + return; + } + + // Map to track which shared component digest gets which spool + Map digestToSpool = new HashMap<>(); + int spoolCounter = 0; + + // Get metadata query for row count estimation + final RelMetadataQuery mq = call.getMetadataQuery(); + + // For each shared component, create a spool + for (RelNode sharedComponent : sharedComponents) { + // Get the actual row count of the shared component being materialized + double actualRowCount = mq.getRowCount(sharedComponent); + + SpoolRelOptTable spoolTable = + new SpoolRelOptTable(null, // no schema needed for temporary tables + sharedComponent.getRowType(), + "spool_" + spoolCounter++, + actualRowCount); // Pass the actual row count for accurate cardinality); + + // Create the TableSpool that will produce/write to this table + LogicalTableSpool spool = + (LogicalTableSpool) RelFactories.DEFAULT_SPOOL_FACTORY.createTableSpool( + sharedComponent, + Spool.Type.LAZY, // Read type + Spool.Type.LAZY, // Write type + spoolTable); + + digestToSpool.put(sharedComponent.getRelDigest(), spool); + } + + combine = + combine.accept(getReplacer(digestToSpool)); + + call.transformTo(combine); + } + + private static RelHomogeneousShuttle getReplacer( + Map digestToSpool) { + Set producers = new HashSet<>(); + + return new RelHomogeneousShuttle() { + @Override public RelNode visit(RelNode node) { + // Check if this node's digest matches any of our shared components + RelDigest nodeDigest = node.getRelDigest(); + if (digestToSpool.containsKey(nodeDigest)) { + LogicalTableSpool spool = digestToSpool.get(nodeDigest); + + if (producers.contains(nodeDigest)) { + // Subsequent occurrence - replace with table scan (consumer) + return LogicalTableScan.create( + node.getCluster(), + spool.getTable(), + ImmutableList.of()); + } else { + // First occurrence - replace with the spool (producer) + producers.add(nodeDigest); + return spool; + } + } + + return super.visit(node); + } + }; + } + + + /** Rule configuration. */ + @Value.Immutable(singleton = true) + public interface Config extends RelRule.Config { + Config DEFAULT = ImmutableCombineSimpleEquivalenceRule.Config.builder() + .build() + .withOperandFor(Combine.class); + + @Override default CombineSimpleEquivalenceRule toRule() { + return new CombineSimpleEquivalenceRule(this); + } + + default Config withOperandFor(Class combineClass) { + return withOperandSupplier(b -> b.operand(combineClass) + .anyInputs()) + .as(Config.class); + } + } +} diff --git a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java index 45c2f7be976a..ede46373b83e 100644 --- a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java +++ b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java @@ -6884,6 +6884,43 @@ public static Map map(Object... args) { return map; } + /** Combines multiple query result lists into rows for the Combine operator. + * + *

    Each input list contains maps representing rows from a query. + * The output is a list of Object arrays, where each array is a row + * with one element per query. The number of output rows equals the + * maximum size across all input lists. Shorter lists are padded with nulls. + * + * @param queryLists array of lists, one per query + * @return list of Object arrays representing combined rows + */ + public static List<@Nullable Object[]> combineQueryResults(List[] queryLists) { + // Find the maximum row count across all queries + int maxRows = 0; + for (List list : queryLists) { + if (list.size() > maxRows) { + maxRows = list.size(); + } + } + + // Build the result rows + List<@Nullable Object[]> result = new ArrayList<>(maxRows); + for (int rowIdx = 0; rowIdx < maxRows; rowIdx++) { + @Nullable Object[] row = new Object[queryLists.length]; + for (int queryIdx = 0; queryIdx < queryLists.length; queryIdx++) { + List queryList = queryLists[queryIdx]; + if (rowIdx < queryList.size()) { + row[queryIdx] = queryList.get(rowIdx); + } else { + row[queryIdx] = null; + } + } + result.add(row); + } + + return result; + } + /** Support the STR_TO_MAP function. */ public static Map strToMap(String string, String stringDelimiter, String keyValueDelimiter) { final Map map = new LinkedHashMap(); diff --git a/core/src/test/java/org/apache/calcite/test/CombineRelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/CombineRelOptRulesTest.java new file mode 100644 index 000000000000..6ca1cbb44972 --- /dev/null +++ b/core/src/test/java/org/apache/calcite/test/CombineRelOptRulesTest.java @@ -0,0 +1,509 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.test; + +import org.apache.calcite.plan.RelOptUtil; +import org.apache.calcite.rel.RelCollationTraitDef; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.Combine; +import org.apache.calcite.rel.core.JoinRelType; +import org.apache.calcite.rel.rules.CombineSimpleEquivalenceRule; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.tools.RelBuilder; + +import org.junit.jupiter.api.Test; + +import java.util.function.Function; + +/** + * Unit tests for {@link Combine} RelNode demonstrating various + * shared component patterns including joins, filters, aggregations, and projections. + */ +class CombineRelOptRulesTest extends RelOptTestBase { + + @Override RelOptFixture fixture() { + return super.fixture() + .withDiffRepos(DiffRepository.lookup(CombineRelOptRulesTest.class)); + } + + @Test void testSharedJoin() { + // Two queries sharing the same EMP-DEPT join + // Query 1: SELECT E.EMPNO, D.DNAME FROM EMP E JOIN DEPT D ON E.DEPTNO = D.DEPTNO + // Query 2: SELECT E.ENAME, D.LOC FROM EMP E JOIN DEPT D ON E.DEPTNO = D.DEPTNO + final Function relFn = b -> { + // Query 1 + b.scan("EMP") + .scan("DEPT") + .join(JoinRelType.INNER, + b.call(SqlStdOperatorTable.EQUALS, + b.field(2, 0, "DEPTNO"), + b.field(2, 1, "DEPTNO"))) + .project(b.field("EMPNO"), b.field("DNAME")); + + // Query 2 + b.scan("EMP") + .scan("DEPT") + .join(JoinRelType.INNER, + b.call(SqlStdOperatorTable.EQUALS, + b.field(2, 0, "DEPTNO"), + b.field(2, 1, "DEPTNO"))) + .project(b.field("ENAME"), b.field("LOC")); + + return b.combine(2).build(); + }; + + relFn(relFn) + .withVolcanoPlanner(false, planner -> { + planner.addRelTraitDef(RelCollationTraitDef.INSTANCE); + RelOptUtil.registerDefaultRules(planner, false, false); + planner.addRule(CombineSimpleEquivalenceRule.Config.DEFAULT.toRule()); + }) + .check(); + } + + @Test void testSharedComplexJoin() { + // Multiple queries sharing a 3-way join: EMP -> DEPT -> SALGRADE + // Query 1: Count employees per department grade + // Query 2: Average salary per department grade + final Function relFn = b -> { + // Query 1: SELECT D.DNAME, S.GRADE, COUNT(*) ... + b.scan("EMP") + .scan("DEPT") + .join(JoinRelType.INNER, + b.call(SqlStdOperatorTable.EQUALS, + b.field(2, 0, "DEPTNO"), + b.field(2, 1, "DEPTNO"))) + .scan("SALGRADE") + .join(JoinRelType.INNER, + b.call(SqlStdOperatorTable.AND, + b.call(SqlStdOperatorTable.GREATER_THAN_OR_EQUAL, + b.field(2, 0, "SAL"), + b.field(2, 1, "LOSAL")), + b.call(SqlStdOperatorTable.LESS_THAN_OR_EQUAL, + b.field(2, 0, "SAL"), + b.field(2, 1, "HISAL")))) + .aggregate( + b.groupKey("DNAME", "GRADE"), + b.count(false, "EMP_COUNT")); + + // Query 2: SELECT D.DNAME, S.GRADE, AVG(SAL) ... + b.scan("EMP") + .scan("DEPT") + .join(JoinRelType.INNER, + b.call(SqlStdOperatorTable.EQUALS, + b.field(2, 0, "DEPTNO"), + b.field(2, 1, "DEPTNO"))) + .scan("SALGRADE") + .join(JoinRelType.INNER, + b.call(SqlStdOperatorTable.AND, + b.call(SqlStdOperatorTable.GREATER_THAN_OR_EQUAL, + b.field(2, 0, "SAL"), + b.field(2, 1, "LOSAL")), + b.call(SqlStdOperatorTable.LESS_THAN_OR_EQUAL, + b.field(2, 0, "SAL"), + b.field(2, 1, "HISAL")))) + .aggregate( + b.groupKey("DNAME", "GRADE"), + b.avg(false, "AVG_SAL", b.field("SAL"))); + + return b.combine(2).build(); + }; + + relFn(relFn) + .withVolcanoPlanner(false, planner -> { + planner.addRelTraitDef(RelCollationTraitDef.INSTANCE); + RelOptUtil.registerDefaultRules(planner, false, false); + planner.addRule(CombineSimpleEquivalenceRule.Config.DEFAULT.toRule()); + }) + .check(); + } + + // ========== Shared Filter Tests ========== + + @Test void testSharedFilter() { + // Two queries sharing the same filter condition + // Query 1: SELECT EMPNO, SAL FROM EMP WHERE SAL > 2000 AND DEPTNO = 10 + // Query 2: SELECT ENAME, JOB FROM EMP WHERE SAL > 2000 AND DEPTNO = 10 + final Function relFn = b -> { + // Query 1 + b.scan("EMP") + .filter( + b.call(SqlStdOperatorTable.AND, + b.call(SqlStdOperatorTable.GREATER_THAN, + b.field("SAL"), + b.literal(2000)), + b.call(SqlStdOperatorTable.EQUALS, + b.field("DEPTNO"), + b.literal(10)))) + .project(b.field("EMPNO"), b.field("SAL")); + + // Query 2 + b.scan("EMP") + .filter( + b.call(SqlStdOperatorTable.AND, + b.call(SqlStdOperatorTable.GREATER_THAN, + b.field("SAL"), + b.literal(2000)), + b.call(SqlStdOperatorTable.EQUALS, + b.field("DEPTNO"), + b.literal(10)))) + .project(b.field("ENAME"), b.field("JOB")); + + return b.combine(2).build(); + }; + + relFn(relFn) + .withVolcanoPlanner(false, planner -> { + planner.addRelTraitDef(RelCollationTraitDef.INSTANCE); + RelOptUtil.registerDefaultRules(planner, false, false); + planner.addRule(CombineSimpleEquivalenceRule.Config.DEFAULT.toRule()); + }) + .check(); + } + + @Test void testSharedFilterWithDifferentProjections() { + // Three queries sharing same filter but different projections + final Function relFn = b -> { + // Shared filter: EMP WHERE SAL BETWEEN 1000 AND 3000 + // Query 1: Count + b.scan("EMP") + .filter( + b.call(SqlStdOperatorTable.AND, + b.call(SqlStdOperatorTable.GREATER_THAN_OR_EQUAL, + b.field("SAL"), + b.literal(1000)), + b.call(SqlStdOperatorTable.LESS_THAN_OR_EQUAL, + b.field("SAL"), + b.literal(3000)))) + .aggregate(b.groupKey(), b.count(false, "CNT")); + + // Query 2: Average salary + b.scan("EMP") + .filter( + b.call(SqlStdOperatorTable.AND, + b.call(SqlStdOperatorTable.GREATER_THAN_OR_EQUAL, + b.field("SAL"), + b.literal(1000)), + b.call(SqlStdOperatorTable.LESS_THAN_OR_EQUAL, + b.field("SAL"), + b.literal(3000)))) + .aggregate(b.groupKey(), b.avg(false, "AVG_SAL", b.field("SAL"))); + + // Query 3: List of names + b.scan("EMP") + .filter( + b.call(SqlStdOperatorTable.AND, + b.call(SqlStdOperatorTable.GREATER_THAN_OR_EQUAL, + b.field("SAL"), + b.literal(1000)), + b.call(SqlStdOperatorTable.LESS_THAN_OR_EQUAL, + b.field("SAL"), + b.literal(3000)))) + .project(b.field("ENAME"), b.field("SAL")); + + return b.combine(3).build(); + }; + + relFn(relFn) + .withVolcanoPlanner(false, planner -> { + planner.addRelTraitDef(RelCollationTraitDef.INSTANCE); + RelOptUtil.registerDefaultRules(planner, false, false); + planner.addRule(CombineSimpleEquivalenceRule.Config.DEFAULT.toRule()); + }) + .check(); + } + + @Test void testSharedAggregationBase() { + // Two queries that could share aggregation computation + // Query 1: SELECT DEPTNO, SUM(SAL), COUNT(*) FROM EMP GROUP BY DEPTNO + // Query 2: SELECT DEPTNO, SUM(SAL) FROM EMP GROUP BY DEPTNO WHERE SUM(SAL) > 10000 + final Function relFn = b -> { + // Query 1: Basic aggregation + b.scan("EMP") + .aggregate( + b.groupKey("DEPTNO"), + b.sum(false, "TOTAL_SAL", b.field("SAL"))); + + // Query 2: Same aggregation with HAVING clause + b.scan("EMP") + .aggregate( + b.groupKey("DEPTNO"), + b.sum(false, "TOTAL_SAL", b.field("SAL"))) + .filter( + b.call(SqlStdOperatorTable.GREATER_THAN, + b.field("TOTAL_SAL"), + b.literal(10000))); + + return b.combine(2).build(); + }; + + relFn(relFn) + .withVolcanoPlanner(false, planner -> { + planner.addRelTraitDef(RelCollationTraitDef.INSTANCE); + RelOptUtil.registerDefaultRules(planner, false, false); + planner.addRule(CombineSimpleEquivalenceRule.Config.DEFAULT.toRule()); + }) + .check(); + } + + @Test void testSharedJoinThenAggregation() { + // Queries sharing join followed by different aggregations + // Query 1: Total salary by department + // Query 2: Employee count by location + final Function relFn = b -> { + // Query 1: SELECT D.DNAME, SUM(E.SAL) FROM EMP E JOIN DEPT D ... GROUP BY D.DNAME + b.scan("EMP") + .scan("DEPT") + .join(JoinRelType.INNER, + b.call(SqlStdOperatorTable.EQUALS, + b.field(2, 0, "DEPTNO"), + b.field(2, 1, "DEPTNO"))) + .aggregate( + b.groupKey("DNAME"), + b.sum(false, "TOTAL_SAL", b.field("SAL"))); + + // Query 2: SELECT D.LOC, COUNT(*) FROM EMP E JOIN DEPT D ... GROUP BY D.LOC + b.scan("EMP") + .scan("DEPT") + .join(JoinRelType.INNER, + b.call(SqlStdOperatorTable.EQUALS, + b.field(2, 0, "DEPTNO"), + b.field(2, 1, "DEPTNO"))) + .aggregate( + b.groupKey("LOC"), + b.count(false, "EMP_CNT")); + + return b.combine(2).build(); + }; + + relFn(relFn) + .withVolcanoPlanner(false, planner -> { + planner.addRelTraitDef(RelCollationTraitDef.INSTANCE); + RelOptUtil.registerDefaultRules(planner, false, false); + planner.addRule(CombineSimpleEquivalenceRule.Config.DEFAULT.toRule()); + }) + .check(); + } + + @Test void testSharedFilterJoinAggregate() { + // Complex pattern: Filter -> Join -> different aggregations + final Function relFn = b -> { + // Query 1: High earners by department with count + b.scan("EMP") + .filter( + b.call(SqlStdOperatorTable.GREATER_THAN, + b.field("SAL"), + b.literal(2000))) + .scan("DEPT") + .join(JoinRelType.INNER, + b.call(SqlStdOperatorTable.EQUALS, + b.field(2, 0, "DEPTNO"), + b.field(2, 1, "DEPTNO"))) + .aggregate( + b.groupKey("DNAME"), + b.count(false, "HIGH_EARNER_CNT")); + + // Query 2: High earners by department with average + b.scan("EMP") + .filter( + b.call(SqlStdOperatorTable.GREATER_THAN, + b.field("SAL"), + b.literal(2000))) + .scan("DEPT") + .join(JoinRelType.INNER, + b.call(SqlStdOperatorTable.EQUALS, + b.field(2, 0, "DEPTNO"), + b.field(2, 1, "DEPTNO"))) + .aggregate( + b.groupKey("DNAME"), + b.avg(false, "AVG_HIGH_SAL", b.field("SAL"))); + + return b.combine(2).build(); + }; + + relFn(relFn) + .withVolcanoPlanner(false, planner -> { + planner.addRelTraitDef(RelCollationTraitDef.INSTANCE); + RelOptUtil.registerDefaultRules(planner, false, false); + planner.addRule(CombineSimpleEquivalenceRule.Config.DEFAULT.toRule()); + }) + .check(); + } + + // ========== Tests WITHOUT Shared Expressions (No Trivial Equivalence) ========== + + @Test void testNoSharedExpressionsDifferentTables() { + // Two queries on completely different tables - no sharing possible + // Query 1: SELECT * FROM EMP + // Query 2: SELECT * FROM SALGRADE + final Function relFn = b -> { + // Query 1: EMP table + b.scan("EMP") + .project(b.field("EMPNO"), b.field("ENAME"), b.field("SAL")); + + // Query 2: SALGRADE table (completely unrelated) + b.scan("SALGRADE") + .project(b.field("GRADE"), b.field("LOSAL"), b.field("HISAL")); + + return b.combine(2).build(); + }; + + relFn(relFn) + .withVolcanoPlanner(false, planner -> { + planner.addRelTraitDef(RelCollationTraitDef.INSTANCE); + RelOptUtil.registerDefaultRules(planner, false, false); + planner.addRule(CombineSimpleEquivalenceRule.Config.DEFAULT.toRule()); + }) + .check(); + } + + @Test void testNoSharedExpressionsDifferentFilters() { + // Two queries on same table but with non-overlapping filters - no sharing + // Query 1: SELECT EMPNO FROM EMP WHERE DEPTNO = 10 + // Query 2: SELECT ENAME FROM EMP WHERE JOB = 'CLERK' + final Function relFn = b -> { + // Query 1: Filter on DEPTNO + b.scan("EMP") + .filter( + b.call(SqlStdOperatorTable.EQUALS, + b.field("DEPTNO"), + b.literal(10))) + .project(b.field("EMPNO")); + + // Query 2: Filter on JOB (different filter, not shareable) + b.scan("EMP") + .filter( + b.call(SqlStdOperatorTable.EQUALS, + b.field("JOB"), + b.literal("CLERK"))) + .project(b.field("ENAME")); + + return b.combine(2).build(); + }; + + relFn(relFn) + .withVolcanoPlanner(false, planner -> { + planner.addRelTraitDef(RelCollationTraitDef.INSTANCE); + RelOptUtil.registerDefaultRules(planner, false, false); + planner.addRule(CombineSimpleEquivalenceRule.Config.DEFAULT.toRule()); + }) + .check(); + } + + @Test void testNoSharedExpressionsDifferentJoins() { + // Two queries with different join conditions - no sharing + // Query 1: EMP JOIN DEPT ON DEPTNO + // Query 2: EMP JOIN SALGRADE ON SAL between LOSAL and HISAL + final Function relFn = b -> { + // Query 1: EMP-DEPT join + b.scan("EMP") + .scan("DEPT") + .join(JoinRelType.INNER, + b.call(SqlStdOperatorTable.EQUALS, + b.field(2, 0, "DEPTNO"), + b.field(2, 1, "DEPTNO"))) + .project(b.field("ENAME"), b.field("DNAME")); + + // Query 2: EMP-SALGRADE join (completely different join) + b.scan("EMP") + .scan("SALGRADE") + .join(JoinRelType.INNER, + b.call(SqlStdOperatorTable.AND, + b.call(SqlStdOperatorTable.GREATER_THAN_OR_EQUAL, + b.field(2, 0, "SAL"), + b.field(2, 1, "LOSAL")), + b.call(SqlStdOperatorTable.LESS_THAN_OR_EQUAL, + b.field(2, 0, "SAL"), + b.field(2, 1, "HISAL")))) + .project(b.field("ENAME"), b.field("GRADE")); + + return b.combine(2).build(); + }; + + relFn(relFn) + .withVolcanoPlanner(false, planner -> { + planner.addRelTraitDef(RelCollationTraitDef.INSTANCE); + RelOptUtil.registerDefaultRules(planner, false, false); + planner.addRule(CombineSimpleEquivalenceRule.Config.DEFAULT.toRule()); + }) + .check(); + } + + @Test void testNoSharedExpressionsDifferentGroupByKeys() { + // Two queries with different GROUP BY keys - no sharing + // Query 1: GROUP BY DEPTNO + // Query 2: GROUP BY JOB + final Function relFn = b -> { + // Query 1: Group by DEPTNO + b.scan("EMP") + .aggregate( + b.groupKey("DEPTNO"), + b.count(false, "CNT")); + + // Query 2: Group by JOB (different key, not shareable) + b.scan("EMP") + .aggregate( + b.groupKey("JOB"), + b.count(false, "CNT")); + + return b.combine(2).build(); + }; + + relFn(relFn) + .withVolcanoPlanner(false, planner -> { + planner.addRelTraitDef(RelCollationTraitDef.INSTANCE); + RelOptUtil.registerDefaultRules(planner, false, false); + planner.addRule(CombineSimpleEquivalenceRule.Config.DEFAULT.toRule()); + }) + .check(); + } + + @Test void testNoSharedExpressionsDifferentJoinTypes() { + // Two queries with same tables but different join types - no sharing + // Query 1: EMP INNER JOIN DEPT + // Query 2: EMP FULL OUTER JOIN DEPT + final Function relFn = b -> { + // Query 1: Inner join + b.scan("EMP") + .scan("DEPT") + .join(JoinRelType.INNER, + b.call(SqlStdOperatorTable.EQUALS, + b.field(2, 0, "DEPTNO"), + b.field(2, 1, "DEPTNO"))) + .project(b.field("EMPNO"), b.field("DNAME")); + + // Query 2: Full outer join (different join type) + b.scan("EMP") + .scan("DEPT") + .join(JoinRelType.FULL, + b.call(SqlStdOperatorTable.EQUALS, + b.field(2, 0, "DEPTNO"), + b.field(2, 1, "DEPTNO"))) + .project(b.field("EMPNO"), b.field("DNAME")); + + return b.combine(2).build(); + }; + + relFn(relFn) + .withVolcanoPlanner(false, planner -> { + planner.addRelTraitDef(RelCollationTraitDef.INSTANCE); + RelOptUtil.registerDefaultRules(planner, false, false); + planner.addRule(CombineSimpleEquivalenceRule.Config.DEFAULT.toRule()); + }) + .check(); + } +} diff --git a/core/src/test/java/org/apache/calcite/test/SqlFunctionsTest.java b/core/src/test/java/org/apache/calcite/test/SqlFunctionsTest.java index 043c23b10525..4c5f6cc2b7f1 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlFunctionsTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlFunctionsTest.java @@ -85,6 +85,7 @@ import static org.hamcrest.CoreMatchers.nullValue; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.closeTo; +import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.hasToString; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -2057,4 +2058,36 @@ private long sqlTimestamp(String str) { assertArrayEquals(new byte[]{(byte) 0x80, (byte) 0x00}, SqlFunctions.leftShift(new byte[]{(byte) 0x40, (byte) 0x00}, 1)); } + + @Test void testCombineQueryResults() { + // Test combining two equal-length lists + List list1 = Arrays.asList(1, 2, 3); + List list2 = Arrays.asList(10, 20, 30); + List result = SqlFunctions.combineQueryResults(new List[]{list1, list2}); + + assertThat(result, hasSize(3)); + assertArrayEquals(new Object[]{1, 10}, result.get(0)); + assertArrayEquals(new Object[]{2, 20}, result.get(1)); + assertArrayEquals(new Object[]{3, 30}, result.get(2)); + + // Test combining lists of different lengths (shorter list padded with nulls) + List listA = Arrays.asList("a", "b"); + List listB = Arrays.asList("x", "y", "z", "w"); + result = SqlFunctions.combineQueryResults(new List[]{listA, listB}); + + assertThat(result, hasSize(4)); + assertArrayEquals(new Object[]{"a", "x"}, result.get(0)); + assertArrayEquals(new Object[]{"b", "y"}, result.get(1)); + assertArrayEquals(new Object[]{null, "z"}, result.get(2)); + assertArrayEquals(new Object[]{null, "w"}, result.get(3)); + + // Test with empty list + List emptyList = Collections.emptyList(); + List nonEmpty = Arrays.asList(100, 200); + result = SqlFunctions.combineQueryResults(new List[]{emptyList, nonEmpty}); + + assertThat(result, hasSize(2)); + assertArrayEquals(new Object[]{null, 100}, result.get(0)); + assertArrayEquals(new Object[]{null, 200}, result.get(1)); + } } diff --git a/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableCombineTest.java b/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableCombineTest.java new file mode 100644 index 000000000000..70c3ed4ec0ea --- /dev/null +++ b/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableCombineTest.java @@ -0,0 +1,167 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.test.enumerable; + +import org.apache.calcite.adapter.java.ReflectiveSchema; +import org.apache.calcite.config.CalciteConnectionProperty; +import org.apache.calcite.config.Lex; +import org.apache.calcite.test.CalciteAssert; +import org.apache.calcite.test.schemata.hr.HrSchema; + +import org.junit.jupiter.api.Test; + +/** + * Unit tests for {@link org.apache.calcite.adapter.enumerable.EnumerableCombine}. + */ +class EnumerableCombineTest { + + /** + * Test that executes two simple queries combined. + * Query 1: Select employee names from department 10 + * Query 2: Select department names + * + *

    The Combine operator returns results in a wide format where each query + * is a column (QUERY_0, QUERY_1, etc.) and each row contains a struct (map) + * for each query. Queries with fewer rows have null values for additional rows. + */ + @Test void testCombineTwoQueries() { + tester(new HrSchema()) + .withRel( + builder -> { + // Query 1: SELECT name FROM emps WHERE deptno = 10 + builder.scan("s", "emps") + .filter( + builder.equals( + builder.field("deptno"), + builder.literal(10))) + .project(builder.field("name")); + + // Query 2: SELECT name FROM depts + builder.scan("s", "depts") + .project(builder.field("name")); + + // Combine both queries + return builder.combine(2).build(); + }) + .returnsOrdered( + "QUERY_0={name=Bill}; QUERY_1={name=Sales}", + "QUERY_0={name=Sebastian}; QUERY_1={name=Marketing}", + "QUERY_0={name=Theodore}; QUERY_1={name=HR}"); + } + + /** + * Test that executes two queries with different row counts. + * Query 1: Select all employee names (4 rows) + * Query 2: Select all department names (3 rows) + * + *

    Since Query 2 returns fewer rows, QUERY1 is null for the extra rows. + */ + @Test void testCombineDifferentRowCounts() { + tester(new HrSchema()) + .withRel( + builder -> { + // Query 1: SELECT name FROM emps (4 rows) + builder.scan("s", "emps") + .project(builder.field("name")); + + // Query 2: SELECT name FROM depts (3 rows) + builder.scan("s", "depts") + .project(builder.field("name")); + + // Combine both queries + return builder.combine(2).build(); + }) + .returnsUnordered( + "QUERY_0={name=Bill}; QUERY_1={name=Sales}", + "QUERY_0={name=Eric}; QUERY_1={name=Marketing}", + "QUERY_0={name=Sebastian}; QUERY_1={name=HR}", + "QUERY_0={name=Theodore}; QUERY_1=null"); + } + + /** + * Test that executes two queries with multiple columns each. + * Query 1: Select empid and name from employees in department 10 + * Query 2: Select deptno and name from departments + */ + @Test void testCombineMultipleColumns() { + tester(new HrSchema()) + .withRel( + builder -> { + // Query 1: SELECT empid, name FROM emps WHERE deptno = 10 + builder.scan("s", "emps") + .filter( + builder.equals( + builder.field("deptno"), + builder.literal(10))) + .project( + builder.field("empid"), + builder.field("name")); + + // Query 2: SELECT deptno, name FROM depts + builder.scan("s", "depts") + .project( + builder.field("deptno"), + builder.field("name")); + + // Combine both queries + return builder.combine(2).build(); + }) + .returnsUnordered( + "QUERY_0={empid=100, name=Bill}; QUERY_1={deptno=10, name=Sales}", + "QUERY_0={empid=150, name=Sebastian}; QUERY_1={deptno=30, name=Marketing}", + "QUERY_0={empid=110, name=Theodore}; QUERY_1={deptno=40, name=HR}"); + } + + /** + * Test that executes two queries returning different numbers of rows. + * Query 1: Select name from depts (3 rows) + * Query 2: Select empid, name, deptno from emps where deptno = 10 (3 rows) + */ + @Test void testCombineDifferentColumnCounts() { + tester(new HrSchema()) + .withRel( + builder -> { + // Query 1: SELECT name FROM depts (1 column) + builder.scan("s", "depts") + .project(builder.field("name")); + + // Query 2: SELECT empid, name, deptno FROM emps WHERE deptno = 10 (3 columns) + builder.scan("s", "emps") + .filter( + builder.equals( + builder.field("deptno"), + builder.literal(10))) + .project( + builder.field("empid"), + builder.field("name"), + builder.field("deptno")); + + // Combine both queries + return builder.combine(2).build(); + }) + .returnsUnordered( + "QUERY_0={name=Sales}; QUERY_1={empid=100, name=Bill, deptno=10}", + "QUERY_0={name=Marketing}; QUERY_1={empid=150, name=Sebastian, deptno=10}", + "QUERY_0={name=HR}; QUERY_1={empid=110, name=Theodore, deptno=10}"); + } + + private CalciteAssert.AssertThat tester(Object schema) { + return CalciteAssert.that() + .with(CalciteConnectionProperty.LEX, Lex.JAVA) + .withSchema("s", new ReflectiveSchema(schema)); + } +} diff --git a/core/src/test/resources/org/apache/calcite/test/CombineRelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/CombineRelOptRulesTest.xml new file mode 100644 index 000000000000..1ae0496621cc --- /dev/null +++ b/core/src/test/resources/org/apache/calcite/test/CombineRelOptRulesTest.xml @@ -0,0 +1,371 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + =($5, $9), <=($5, $10))], joinType=[inner]) + LogicalTableScan(table=[[scott, EMP]]) + LogicalTableScan(table=[[scott, SALGRADE]]) +]]> + + + =($5, $9), <=($5, $10))], joinType=[inner]) + EnumerableTableScan(table=[[scott, EMP]]) + EnumerableTableScan(table=[[scott, SALGRADE]]) +]]> + + + + + + + + + + + + + ($1, 10000)]) + LogicalAggregate(group=[{7}], TOTAL_SAL=[SUM($5)]) + LogicalTableScan(table=[[scott, EMP]]) +]]> + + + ($1, 10000)]) + EnumerableInterpreter + BindableTableScan(table=[[TEMP, spool_0]]) +]]> + + + + + =($5, $12), <=($5, $13))], joinType=[inner]) + LogicalJoin(condition=[=($7, $8)], joinType=[inner]) + LogicalTableScan(table=[[scott, EMP]]) + LogicalTableScan(table=[[scott, DEPT]]) + LogicalTableScan(table=[[scott, SALGRADE]]) + LogicalAggregate(group=[{9, 11}], AVG_SAL=[AVG($5)]) + LogicalJoin(condition=[AND(>=($5, $12), <=($5, $13))], joinType=[inner]) + LogicalJoin(condition=[=($7, $8)], joinType=[inner]) + LogicalTableScan(table=[[scott, EMP]]) + LogicalTableScan(table=[[scott, DEPT]]) + LogicalTableScan(table=[[scott, SALGRADE]]) +]]> + + + =($5, $12), <=($5, $13))], joinType=[inner]) + EnumerableProject(EMPNO=[$3], ENAME=[$4], JOB=[$5], MGR=[$6], HIREDATE=[$7], SAL=[$8], COMM=[$9], DEPTNO=[$10], DEPTNO0=[$0], DNAME=[$1], LOC=[$2]) + EnumerableHashJoin(condition=[=($0, $10)], joinType=[inner]) + EnumerableTableScan(table=[[scott, DEPT]]) + EnumerableTableScan(table=[[scott, EMP]]) + EnumerableTableScan(table=[[scott, SALGRADE]]) + EnumerableProject(DNAME=[$0], GRADE=[$1], AVG_SAL=[CAST(/(CAST(CASE(=($3, 0), null:DECIMAL(19, 2), $2)):DECIMAL(7, 2), $3)):DECIMAL(7, 2)]) + EnumerableAggregate(group=[{9, 11}], agg#0=[$SUM0($5)], agg#1=[COUNT($5)]) + EnumerableInterpreter + BindableTableScan(table=[[TEMP, spool_0]]) +]]> + + + + + ($5, 2000), =($7, 10))]) + LogicalTableScan(table=[[scott, EMP]]) + LogicalProject(ENAME=[$1], JOB=[$2]) + LogicalFilter(condition=[AND(>($5, 2000), =($7, 10))]) + LogicalTableScan(table=[[scott, EMP]]) +]]> + + + ($5, 2000), =($7, 10))]) + EnumerableTableScan(table=[[scott, EMP]]) + EnumerableProject(ENAME=[$1], JOB=[$2]) + EnumerableInterpreter + BindableTableScan(table=[[TEMP, spool_0]]) +]]> + + + + + ($5, 2000)]) + LogicalTableScan(table=[[scott, EMP]]) + LogicalTableScan(table=[[scott, DEPT]]) + LogicalAggregate(group=[{9}], AVG_HIGH_SAL=[AVG($5)]) + LogicalJoin(condition=[=($7, $8)], joinType=[inner]) + LogicalFilter(condition=[>($5, 2000)]) + LogicalTableScan(table=[[scott, EMP]]) + LogicalTableScan(table=[[scott, DEPT]]) +]]> + + + ($5, 2000)]) + EnumerableTableScan(table=[[scott, EMP]]) + EnumerableProject(DNAME=[$0], AVG_HIGH_SAL=[CAST(/(CAST(CASE(=($2, 0), null:DECIMAL(19, 2), $1)):DECIMAL(7, 2), $2)):DECIMAL(7, 2)]) + EnumerableAggregate(group=[{9}], agg#0=[$SUM0($5)], agg#1=[COUNT($5)]) + EnumerableInterpreter + BindableTableScan(table=[[TEMP, spool_0]]) +]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ($5, 1500)]) + LogicalTableScan(table=[[scott, EMP]]) + LogicalProject(EMPNO=[$0], ENAME=[$1]) + LogicalFilter(condition=[=($7, 20)]) + LogicalFilter(condition=[>($5, 1500)]) + LogicalTableScan(table=[[scott, EMP]]) +]]> + + + ($5, 1500), =($7, 20))]) + EnumerableTableScan(table=[[scott, EMP]]) + EnumerableProject(EMPNO=[$0], ENAME=[$1]) + EnumerableInterpreter + BindableTableScan(table=[[TEMP, spool_0]]) +]]> + + + From 2989d1c32a27554ed2fa9268bbe59291b931450b Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Tue, 9 Dec 2025 06:42:44 +0800 Subject: [PATCH 041/562] [CALCITE-7310] Support the syntax SELECT * EXCLUDE(columns) --- babel/src/main/codegen/config.fmpp | 1 + .../apache/calcite/test/BabelParserTest.java | 25 ++++ .../org/apache/calcite/test/BabelTest.java | 51 ++++++++ babel/src/test/resources/sql/select.iq | 53 ++++++++ core/src/main/codegen/default_config.fmpp | 1 + core/src/main/codegen/templates/Parser.jj | 61 +++++++++ .../calcite/runtime/CalciteResource.java | 6 + .../apache/calcite/sql/SqlStarExclude.java | 84 +++++++++++++ .../sql/validate/SqlValidatorImpl.java | 118 +++++++++++++++++- .../runtime/CalciteResource.properties | 2 + site/_docs/reference.md | 12 +- 11 files changed, 406 insertions(+), 8 deletions(-) create mode 100644 core/src/main/java/org/apache/calcite/sql/SqlStarExclude.java diff --git a/babel/src/main/codegen/config.fmpp b/babel/src/main/codegen/config.fmpp index c41f28bd71b4..b9c4a1c6ee61 100644 --- a/babel/src/main/codegen/config.fmpp +++ b/babel/src/main/codegen/config.fmpp @@ -617,6 +617,7 @@ data: { includePosixOperators: true includeParsingStringLiteralAsArrayLiteral: true includeIntervalWithoutQualifier: true + includeStarExclude: true } } diff --git a/babel/src/test/java/org/apache/calcite/test/BabelParserTest.java b/babel/src/test/java/org/apache/calcite/test/BabelParserTest.java index a0fb721aef69..e3d49d2bb6e6 100644 --- a/babel/src/test/java/org/apache/calcite/test/BabelParserTest.java +++ b/babel/src/test/java/org/apache/calcite/test/BabelParserTest.java @@ -136,6 +136,31 @@ class BabelParserTest extends SqlParserTest { + "FROM \"t\""); } + /** Test case for + * [CALCITE-7310] Support the syntax SELECT * EXCLUDE(columns). */ + @Test void testStarExclude() { + final String sql = "select * exclude(empno) from emp"; + final String expected = "SELECT * EXCLUDE (`EMPNO`)\n" + + "FROM `EMP`"; + sql(sql).ok(expected); + + final String sql2 = "select e.* exclude(e.empno, e.ename, e.job, e.mgr, d.deptno)" + + " from emp e join dept d on e.deptno = d.deptno"; + final String expected2 = "SELECT `E`.* EXCLUDE (`E`.`EMPNO`, `E`.`ENAME`," + + " `E`.`JOB`, `E`.`MGR`, `D`.`DEPTNO`)\n" + + "FROM `EMP` AS `E`\n" + + "INNER JOIN `DEPT` AS `D` ON (`E`.`DEPTNO` = `D`.`DEPTNO`)"; + sql(sql2).ok(expected2); + + final String sql3 = "select e.* exclude(e.empno, e.ename, e.job, e.mgr, d.deptno)," + + " d.* exclude(d.dname) from emp e join dept d on e.deptno = d.deptno"; + final String expected3 = "SELECT `E`.* EXCLUDE (`E`.`EMPNO`, `E`.`ENAME`," + + " `E`.`JOB`, `E`.`MGR`, `D`.`DEPTNO`), `D`.* EXCLUDE (`D`.`DNAME`)\n" + + "FROM `EMP` AS `E`\n" + + "INNER JOIN `DEPT` AS `D` ON (`E`.`DEPTNO` = `D`.`DEPTNO`)"; + sql(sql3).ok(expected3); + } + /** Tests that there are no reserved keywords. */ @Disabled @Test void testKeywords() { diff --git a/babel/src/test/java/org/apache/calcite/test/BabelTest.java b/babel/src/test/java/org/apache/calcite/test/BabelTest.java index 1feb2e6020fb..9368a7ab7146 100644 --- a/babel/src/test/java/org/apache/calcite/test/BabelTest.java +++ b/babel/src/test/java/org/apache/calcite/test/BabelTest.java @@ -18,6 +18,7 @@ import org.apache.calcite.config.CalciteConnectionProperty; import org.apache.calcite.rel.type.DelegatingTypeSystem; +import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.rel.type.TimeFrameSet; import org.apache.calcite.sql.SqlOperatorTable; import org.apache.calcite.sql.fun.SqlLibrary; @@ -26,6 +27,8 @@ import org.apache.calcite.sql.parser.babel.SqlBabelParserImpl; import org.apache.calcite.sql.validate.SqlConformanceEnum; +import com.google.common.collect.ImmutableList; + import org.junit.jupiter.api.Test; import java.sql.Connection; @@ -35,8 +38,10 @@ import java.sql.SQLException; import java.sql.Statement; import java.sql.Types; +import java.util.List; import java.util.Properties; import java.util.function.UnaryOperator; +import java.util.stream.Collectors; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; @@ -159,6 +164,52 @@ private void checkInfixCast(Statement statement, String typeName, int sqlType) .fails("(?s).*Encountered \":\" at .*"); } + /** Test case for + * [CALCITE-7310] Support the syntax SELECT * EXCLUDE(columns). */ + @Test void testStarExcludeValidation() { + final SqlValidatorFixture fixture = Fixtures.forValidator() + .withParserConfig(p -> p.withParserFactory(SqlBabelParserImpl.FACTORY)); + + fixture.withSql("select * exclude(empno, deptno) from emp") + .type(type -> { + final List names = type.getFieldList().stream() + .map(RelDataTypeField::getName) + .collect(Collectors.toList()); + assertThat( + names, is( + ImmutableList.of("ENAME", "JOB", "MGR", "HIREDATE", "SAL", "COMM", "SLACKER"))); + }); + + fixture.withSql("select * exclude (empno, ^foo^) from emp") + .fails("SELECT \\* EXCLUDE list contains unknown column\\(s\\): FOO"); + + fixture.withSql("select e.* exclude(e.empno, e.ename, e.job, e.mgr)" + + " from emp e join dept d on e.deptno = d.deptno") + .type(type -> { + final List names = type.getFieldList().stream() + .map(RelDataTypeField::getName) + .collect(Collectors.toList()); + assertThat( + names, is( + ImmutableList.of("HIREDATE", "SAL", "COMM", "DEPTNO", "SLACKER"))); + }); + + fixture.withSql("select e.* exclude(e.empno, e.ename, e.job, e.mgr, ^d.deptno^)" + + " from emp e join dept d on e.deptno = d.deptno") + .fails("SELECT \\* EXCLUDE list contains unknown column\\(s\\): D.DEPTNO"); + + fixture.withSql("select e.* exclude(e.empno, e.ename, e.job, e.mgr), d.* exclude(d.name)" + + " from emp e join dept d on e.deptno = d.deptno") + .type(type -> { + final List names = type.getFieldList().stream() + .map(RelDataTypeField::getName) + .collect(Collectors.toList()); + assertThat( + names, is( + ImmutableList.of("HIREDATE", "SAL", "COMM", "DEPTNO", "SLACKER", "DEPTNO0"))); + }); + } + /** Tests that DATEADD, DATEDIFF, DATEPART, DATE_PART allow custom time * frames. */ @Test void testTimeFrames() { diff --git a/babel/src/test/resources/sql/select.iq b/babel/src/test/resources/sql/select.iq index 9c234d1a04e3..6b02cca8e36a 100755 --- a/babel/src/test/resources/sql/select.iq +++ b/babel/src/test/resources/sql/select.iq @@ -107,4 +107,57 @@ select 1.0 % 2; !ok +# [CALCITE-7310] Support the syntax SELECT * EXCLUDE(columns) +select * exclude(empno, ename, job, mgr) from emp limit 1; ++------------+--------+------+--------+ +| HIREDATE | SAL | COMM | DEPTNO | ++------------+--------+------+--------+ +| 1980-12-17 | 800.00 | | 20 | ++------------+--------+------+--------+ +(1 row) + +!ok + +select * exclude(empno, ename, job, mgr, mgr) from emp limit 1; ++------------+--------+------+--------+ +| HIREDATE | SAL | COMM | DEPTNO | ++------------+--------+------+--------+ +| 1980-12-17 | 800.00 | | 20 | ++------------+--------+------+--------+ +(1 row) + +!ok + +select e.*, d.* from emp e join dept d on e.deptno = d.deptno limit 1; ++-------+-------+---------+------+------------+---------+------+--------+---------+------------+----------+ +| EMPNO | ENAME | JOB | MGR | HIREDATE | SAL | COMM | DEPTNO | DEPTNO0 | DNAME | LOC | ++-------+-------+---------+------+------------+---------+------+--------+---------+------------+----------+ +| 7782 | CLARK | MANAGER | 7839 | 1981-06-09 | 2450.00 | | 10 | 10 | ACCOUNTING | NEW YORK | ++-------+-------+---------+------+------------+---------+------+--------+---------+------------+----------+ +(1 row) + +!ok + +select e.* exclude(e.empno, e.ename, e.job, e.mgr) +from emp e join dept d on e.deptno = d.deptno limit 1; ++------------+--------+------+--------+ +| HIREDATE | SAL | COMM | DEPTNO | ++------------+--------+------+--------+ +| 1980-12-17 | 800.00 | | 20 | ++------------+--------+------+--------+ +(1 row) + +!ok + +select e.* exclude(e.empno, e.ename, e.job, e.mgr), d.* exclude(d.dname) +from emp e join dept d on e.deptno = d.deptno limit 1; ++------------+---------+------+--------+---------+----------+ +| HIREDATE | SAL | COMM | DEPTNO | DEPTNO0 | LOC | ++------------+---------+------+--------+---------+----------+ +| 1981-06-09 | 2450.00 | | 10 | 10 | NEW YORK | ++------------+---------+------+--------+---------+----------+ +(1 row) + +!ok + # End select.iq diff --git a/core/src/main/codegen/default_config.fmpp b/core/src/main/codegen/default_config.fmpp index a2547273cb10..56d17b82798b 100644 --- a/core/src/main/codegen/default_config.fmpp +++ b/core/src/main/codegen/default_config.fmpp @@ -460,4 +460,5 @@ parser: { includeAdditionalDeclarations: false includeParsingStringLiteralAsArrayLiteral: false includeIntervalWithoutQualifier: false + includeStarExclude: false } diff --git a/core/src/main/codegen/templates/Parser.jj b/core/src/main/codegen/templates/Parser.jj index 95d34d4eb6d2..a6f50f1f1280 100644 --- a/core/src/main/codegen/templates/Parser.jj +++ b/core/src/main/codegen/templates/Parser.jj @@ -91,6 +91,7 @@ import org.apache.calcite.sql.SqlRowTypeNameSpec; import org.apache.calcite.sql.SqlSampleSpec; import org.apache.calcite.sql.SqlSelect; import org.apache.calcite.sql.SqlSelectKeyword; +import org.apache.calcite.sql.SqlStarExclude; import org.apache.calcite.sql.SqlSetOption; import org.apache.calcite.sql.SqlSnapshot; import org.apache.calcite.sql.SqlTableRef; @@ -1974,6 +1975,65 @@ void AddSelectItem(List list) : ) } +<#if (parser.includeStarExclude!default.parser.includeStarExclude)> +/** + * Parses one unaliased expression in a select list. + */ +SqlNode SelectExpression() : +{ + SqlNode e; + SqlNodeList excludeList; +} +{ + ( + { + e = SqlIdentifier.star(getPos()); + } + | + e = Expression(ExprContext.ACCEPT_SUB_QUERY) + ) + ( + excludeList = StarExcludeList() { + if (!(e instanceof SqlIdentifier)) { + throw SqlUtil.newContextException(excludeList.getParserPosition(), + RESOURCE.selectExcludeRequiresStar()); + } + final SqlIdentifier sqlIdentifier = (SqlIdentifier) e; + if (!sqlIdentifier.isStar()) { + throw SqlUtil.newContextException(excludeList.getParserPosition(), + RESOURCE.selectExcludeRequiresStar()); + } + final SqlParserPos pos = SqlParserPos.sum( + ImmutableList.of(sqlIdentifier.getParserPosition(), + excludeList.getParserPosition())); + return new SqlStarExclude(pos, sqlIdentifier, excludeList); + } + | + { return e; } + ) +} + +SqlNodeList StarExcludeList() : +{ + final Span s; + final List list = new ArrayList(); + SqlIdentifier id; +} +{ + { s = span(); } + id = CompoundIdentifier() { + list.add(id); + } + ( + id = CompoundIdentifier() { + list.add(id); + } + )* + { + return new SqlNodeList(list, s.end(this)); + } +} +<#else> /** * Parses one unaliased expression in a select list. */ @@ -1990,6 +2050,7 @@ SqlNode SelectExpression() : return e; } } + SqlLiteral Natural() : { diff --git a/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java b/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java index 809bdcce7051..d35f1a030e7f 100644 --- a/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java +++ b/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java @@ -807,6 +807,12 @@ ExInst illegalArgumentForTableFunctionCall(String a0, @BaseMessage("SELECT * requires a FROM clause") ExInst selectStarRequiresFrom(); + @BaseMessage("EXCLUDE clause must follow a STAR expression") + ExInst selectExcludeRequiresStar(); + + @BaseMessage("SELECT * EXCLUDE list contains unknown column(s): {0}") + ExInst selectStarExcludeListContainsUnknownColumns(String columns); + @BaseMessage("Group function ''{0}'' can only appear in GROUP BY clause") ExInst groupFunctionMustAppearInGroupByClause(String funcName); diff --git a/core/src/main/java/org/apache/calcite/sql/SqlStarExclude.java b/core/src/main/java/org/apache/calcite/sql/SqlStarExclude.java new file mode 100644 index 000000000000..884c4a846360 --- /dev/null +++ b/core/src/main/java/org/apache/calcite/sql/SqlStarExclude.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.sql; + +import org.apache.calcite.sql.parser.SqlParserPos; + +import com.google.common.collect.ImmutableList; + +import org.checkerframework.checker.nullness.qual.Nullable; + +import java.util.List; + +import static java.util.Objects.requireNonNull; + +/** + * Represents {@code SELECT * EXCLUDE(...)}. + */ +public class SqlStarExclude extends SqlCall { + public static final SqlOperator OPERATOR = + new SqlSpecialOperator("SELECT_STAR_EXCLUDE", SqlKind.OTHER) { + @SuppressWarnings("argument.type.incompatible") + @Override public SqlCall createCall( + @Nullable SqlLiteral functionQualifier, + SqlParserPos pos, + @Nullable SqlNode... operands) { + return new SqlStarExclude( + pos, + (SqlIdentifier) operands[0], + (SqlNodeList) operands[1]); + } + }; + + private final SqlIdentifier starIdentifier; + private final SqlNodeList excludeList; + + public SqlStarExclude(SqlParserPos pos, SqlIdentifier starIdentifier, + SqlNodeList excludeList) { + super(pos); + this.starIdentifier = requireNonNull(starIdentifier, "starIdentifier"); + this.excludeList = requireNonNull(excludeList, "excludeList"); + } + + public SqlIdentifier getStarIdentifier() { + return starIdentifier; + } + + public SqlNodeList getExcludeList() { + return excludeList; + } + + @Override public SqlOperator getOperator() { + return OPERATOR; + } + + @Override public SqlKind getKind() { + return OPERATOR.getKind(); + } + + @Override public List getOperandList() { + return ImmutableList.of(starIdentifier, excludeList); + } + + @Override public void unparse(SqlWriter writer, int leftPrec, int rightPrec) { + starIdentifier.unparse(writer, leftPrec, rightPrec); + writer.sep("EXCLUDE"); + final SqlWriter.Frame frame = writer.startList("(", ")"); + excludeList.unparse(writer, 0, 0); + writer.endList(frame); + } +} diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java index 96d46174d4a4..172921babf17 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java @@ -79,6 +79,7 @@ import org.apache.calcite.sql.SqlSelect; import org.apache.calcite.sql.SqlSelectKeyword; import org.apache.calcite.sql.SqlSnapshot; +import org.apache.calcite.sql.SqlStarExclude; import org.apache.calcite.sql.SqlSyntax; import org.apache.calcite.sql.SqlTableFunction; import org.apache.calcite.sql.SqlUnknownLiteral; @@ -639,13 +640,26 @@ private static void validateQualifiedCommonColumn(SqlJoin join, private boolean expandStar(List selectItems, Set aliases, PairList fields, boolean includeSystemVars, SelectScope scope, SqlNode node) { - if (!(node instanceof SqlIdentifier)) { + final SqlIdentifier identifier; + final SqlNodeList excludeList; + if (node instanceof SqlStarExclude) { + final SqlStarExclude starExclude = (SqlStarExclude) node; + identifier = starExclude.getStarIdentifier(); + excludeList = starExclude.getExcludeList(); + } else if (node instanceof SqlIdentifier) { + identifier = (SqlIdentifier) node; + excludeList = null; + } else { return false; } - final SqlIdentifier identifier = (SqlIdentifier) node; if (!identifier.isStar()) { return false; } + final List excludeIdentifiers = + excludeList == null ? Collections.emptyList() : extractExcludeIdentifiers(excludeList); + final boolean[] excludeMatched = new boolean[excludeIdentifiers.size()]; + final SqlNameMatcher nameMatcher = + scope.validator.catalogReader.nameMatcher(); final int originalSize = selectItems.size(); final SqlParserPos startPosition = identifier.getParserPosition(); switch (identifier.names.size()) { @@ -687,6 +701,10 @@ private boolean expandStar(List selectItems, Set aliases, new SqlIdentifier( ImmutableList.of(child.name, columnName), startPosition); + recordExcludeMatches(excludeIdentifiers, exp, nameMatcher, excludeMatched); + if (shouldExcludeField(excludeList, exp, nameMatcher)) { + continue; + } // Don't add expanded rolled up columns if (!isRolledUpColumn(exp, scope)) { addOrExpandField( @@ -720,15 +738,16 @@ private boolean expandStar(List selectItems, Set aliases, int offset = Math.min(calculatePermuteOffset(selectItems), originalSize); new Permute(from, offset).permute(selectItems, fields); } + throwIfUnknownExcludeColumns(excludeIdentifiers, excludeMatched); return true; default: final SqlIdentifier prefixId = identifier.skipLast(1); final SqlValidatorScope.ResolvedImpl resolved = new SqlValidatorScope.ResolvedImpl(); - final SqlNameMatcher nameMatcher = + final SqlNameMatcher resolvedNameMatcher = scope.validator.catalogReader.nameMatcher(); - scope.resolve(prefixId.names, nameMatcher, true, resolved); + scope.resolve(prefixId.names, resolvedNameMatcher, true, resolved); if (resolved.count() == 0) { // e.g. "select s.t.* from e" // or "select r.* from e" @@ -749,6 +768,13 @@ private boolean expandStar(List selectItems, Set aliases, for (RelDataTypeField field : rowType.getFieldList()) { String columnName = field.getName(); + final SqlIdentifier columnId = + prefixId.plus(columnName, startPosition); + recordExcludeMatches(excludeIdentifiers, columnId, resolvedNameMatcher, + excludeMatched); + if (shouldExcludeField(excludeList, columnId, resolvedNameMatcher)) { + continue; + } // TODO: do real implicit collation here addOrExpandField( selectItems, @@ -756,12 +782,13 @@ private boolean expandStar(List selectItems, Set aliases, fields, includeSystemVars, scope, - prefixId.plus(columnName, startPosition), + columnId, field); } } else { throw newValidationError(prefixId, RESOURCE.starRequiresRecordType()); } + throwIfUnknownExcludeColumns(excludeIdentifiers, excludeMatched); return true; } } @@ -778,6 +805,86 @@ private static int calculatePermuteOffset(List selectItems) { return 0; } + private static boolean matchesExcludeNames(List identifierNames, + List excludedIdentifierNames, SqlNameMatcher nameMatcher) { + if (excludedIdentifierNames.size() > identifierNames.size()) { + return false; + } + final int offset = identifierNames.size() - excludedIdentifierNames.size(); + for (int i = 0; i < excludedIdentifierNames.size(); i++) { + if (!nameMatcher.matches(identifierNames.get(offset + i), + excludedIdentifierNames.get(i))) { + return false; + } + } + return true; + } + + private static boolean shouldExcludeField(@Nullable SqlNodeList excludeList, + SqlIdentifier columnId, SqlNameMatcher nameMatcher) { + if (excludeList == null) { + return false; + } + for (SqlNode node : excludeList) { + assert node instanceof SqlIdentifier; + if (matchesExcludeIdentifier(columnId, (SqlIdentifier) node, nameMatcher)) { + return true; + } + } + return false; + } + + private static boolean matchesExcludeIdentifier(SqlIdentifier columnId, + SqlIdentifier excludeIdentifier, SqlNameMatcher nameMatcher) { + return matchesExcludeNames(columnId.names, excludeIdentifier.names, nameMatcher); + } + + private static List extractExcludeIdentifiers(@Nullable SqlNodeList excludeList) { + if (excludeList == null) { + return ImmutableList.of(); + } + final ImmutableList.Builder builder = ImmutableList.builder(); + for (SqlNode node : excludeList) { + if (node instanceof SqlIdentifier) { + builder.add((SqlIdentifier) node); + } + } + return builder.build(); + } + + private static void recordExcludeMatches(List excludeIdentifiers, + SqlIdentifier columnId, SqlNameMatcher nameMatcher, boolean[] matched) { + for (int i = 0; i < excludeIdentifiers.size(); i++) { + if (!matched[i] + && matchesExcludeIdentifier(columnId, excludeIdentifiers.get(i), nameMatcher)) { + matched[i] = true; + } + } + } + + private void throwIfUnknownExcludeColumns(List excludeIdentifiers, + boolean[] excludeMatched) { + if (excludeIdentifiers.isEmpty()) { + return; + } + final List unknownExcludeNames = new ArrayList<>(); + int firstUnknownIndex = -1; + for (int i = 0; i < excludeIdentifiers.size(); i++) { + if (!excludeMatched[i]) { + if (firstUnknownIndex < 0) { + firstUnknownIndex = i; + } + unknownExcludeNames.add(excludeIdentifiers.get(i).toString()); + } + } + if (firstUnknownIndex >= 0) { + throw newValidationError( + excludeIdentifiers.get(firstUnknownIndex), + RESOURCE.selectStarExcludeListContainsUnknownColumns( + String.join(", ", unknownExcludeNames))); + } + } + private SqlNode maybeCast(SqlNode node, RelDataType currentType, RelDataType desiredType) { return SqlTypeUtil.equalSansNullability(typeFactory, currentType, desiredType) @@ -801,7 +908,6 @@ private boolean addOrExpandField(List selectItems, Set aliases, scope, starExp); return true; - default: addToSelectList( selectItems, diff --git a/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties b/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties index 124e133a1410..4cc492795e7b 100644 --- a/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties +++ b/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties @@ -266,6 +266,8 @@ CannotStreamResultsForNonStreamingInputs=Cannot stream results of a query with n MinusNotAllowed=MINUS is not allowed under the current SQL conformance level SelectMissingFrom=SELECT must have a FROM clause SelectStarRequiresFrom=SELECT * requires a FROM clause +SelectExcludeRequiresStar=EXCLUDE clause must follow a STAR expression +SelectStarExcludeListContainsUnknownColumns=SELECT * EXCLUDE list contains unknown column(s): {0} GroupFunctionMustAppearInGroupByClause=Group function ''{0}'' can only appear in GROUP BY clause AuxiliaryWithoutMatchingGroupCall=Call to auxiliary group function ''{0}'' must have matching call to group function ''{1}'' in GROUP BY clause PivotAggMalformed=Measure expression in PIVOT must use aggregate function diff --git a/site/_docs/reference.md b/site/_docs/reference.md index b07ba0c8fa1c..7017d4565df8 100644 --- a/site/_docs/reference.md +++ b/site/_docs/reference.md @@ -205,8 +205,8 @@ orderItem: expression [ ASC | DESC ] [ NULLS FIRST | NULLS LAST ] select: - SELECT [ hintComment ] [ STREAM ] [ ALL | DISTINCT ] - { * | projectItem [, projectItem ]* } + SELECT [ hintComment ] [ STREAM ] [ ALL | DISTINCT ] + { starWithExclude | projectItem [, projectItem ]* } FROM tableExpression [ WHERE booleanExpression ] [ GROUP BY [ ALL | DISTINCT ] { groupItem [, groupItem ]* } ] @@ -218,6 +218,14 @@ selectWithoutFrom: SELECT [ ALL | DISTINCT ] { * | projectItem [, projectItem ]* } +starWithExclude: + * + | * EXCLUDE '(' column [, column ]* ')' + +Note: + +* `SELECT * EXCLUDE (...)` is recognized only when the Babel parser is enabled. It sets the generated parser configuration flag `includeStarExclude` to `true` (the standard parser leaves that flag `false`), which allows a `STAR` token followed by `EXCLUDE` and a parenthesized identifier list to be parsed into a `SqlStarExclude` node and ensures validators respect the exclusion list when expanding the projection. Reusing the same parser configuration elsewhere enables the same syntax for other components that need it. + projectItem: expression [ [ AS ] columnAlias ] | tableAlias . * From b68b9c393788ce5fba4751d053852eabedb5fa35 Mon Sep 17 00:00:00 2001 From: Alessandro Solimando Date: Tue, 9 Dec 2025 15:19:56 +0100 Subject: [PATCH 042/562] [CALCITE-7317] SubQueryRemoveRule should skip NULL-safety checks for IN subqueries when both the keys and the subquery columns are NOT NULL --- .../calcite/rel/rules/SubQueryRemoveRule.java | 130 +++++++++------ .../apache/calcite/test/JdbcAdapterTest.java | 4 +- .../apache/calcite/test/RelOptRulesTest.java | 33 ++++ .../apache/calcite/test/RelOptRulesTest.xml | 154 ++++++++++++------ core/src/test/resources/sql/sub-query.iq | 45 ++++- 5 files changed, 260 insertions(+), 106 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rel/rules/SubQueryRemoveRule.java b/core/src/main/java/org/apache/calcite/rel/rules/SubQueryRemoveRule.java index 6f137231704b..8ed8c287b47a 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/SubQueryRemoveRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/SubQueryRemoveRule.java @@ -583,11 +583,11 @@ private static RexNode rewriteIn(RexSubQuery e, Set variablesSet, // // select e.deptno, // case - // when ct.c = 0 then false - // when e.deptno is null then null - // when dt.i is not null then true - // when ct.ck < ct.c then null - // else false + // when ct.c = 0 then false -- (1) empty subquery check + // when e.deptno is null then null -- (2) key NULL check + // when dt.i is not null then true -- (3) match found + // when ct.ck < ct.c then null -- (4) NULLs exist in subquery + // else false -- (5) no match // end // from emp as e // left join ( @@ -595,37 +595,32 @@ private static RexNode rewriteIn(RexSubQuery e, Set variablesSet, // cross join (select distinct deptno, true as i from emp)) as dt // on e.deptno = dt.deptno // - // If keys are not null we can remove "ct" and simplify to + // If both keys (e.deptno) and subquery columns (deptno) are NOT NULL, + // we can drop checks (1), (2), and (4), which eliminates the need for ct: // // select e.deptno, // case - // when dt.i is not null then true - // else false + // when dt.i is not null then true -- (3) match found + // else false -- (5) no match // end // from emp as e // left join (select distinct deptno, true as i from emp) as dt // on e.deptno = dt.deptno // - // We could further simplify to - // - // select e.deptno, - // dt.i is not null - // from emp as e - // left join (select distinct deptno, true as i from emp) as dt - // on e.deptno = dt.deptno + // Check (1) is not needed: if the subquery is empty, all dt.i are NULL, + // and the LEFT JOIN pattern correctly returns FALSE for IN (TRUE for NOT IN). // - // but have not yet. + // NULL-safety checks are required if either the keys or the subquery + // columns are nullable, due to SQL three-valued logic. // - // If the logic is TRUE we can just kill the record if the condition - // evaluates to FALSE or UNKNOWN. Thus the query simplifies to an inner - // join: + // If the logic is TRUE (as opposed to TRUE_FALSE_UNKNOWN), we only care about + // matches, so the query simplifies to an inner join regardless of nullability: // // select e.deptno, // true // from emp as e // inner join (select distinct deptno from emp) as dt // on e.deptno = dt.deptno - // builder.push(e.rel); final List fields = new ArrayList<>(builder.fields()); @@ -671,6 +666,7 @@ private static RexNode rewriteIn(RexSubQuery e, Set variablesSet, final RexLiteral falseLiteral = builder.literal(false); final RexLiteral unknownLiteral = builder.getRexBuilder().makeNullLiteral(trueLiteral.getType()); + boolean needsNullSafety = false; if (allLiterals) { final List conditions = Pair.zip(expressionOperands, fields).stream() @@ -718,39 +714,51 @@ private static RexNode rewriteIn(RexSubQuery e, Set variablesSet, expressionOperands.clear(); fields.clear(); } else { + boolean anyFieldNullable = fields.stream() + .anyMatch(field -> field.getType().isNullable()); + + // we can skip NULL-safety checks only if both keys + // and subquery columns are NOT NULL + needsNullSafety = + (logic == RelOptUtil.Logic.TRUE_FALSE_UNKNOWN + || logic == RelOptUtil.Logic.UNKNOWN_AS_TRUE) + && (!keyIsNulls.isEmpty() || anyFieldNullable); + switch (logic) { case TRUE: builder.aggregate(builder.groupKey(fields)); break; case TRUE_FALSE_UNKNOWN: case UNKNOWN_AS_TRUE: - // Builds the cross join - // Some databases don't support use FILTER clauses for aggregate functions - // like {@code COUNT(*) FILTER (WHERE not(a is null))} - // So use count(*) when only one column - if (builder.fields().size() <= 1) { - builder.aggregate(builder.groupKey(), - builder.count(false, "c"), - builder.count(builder.fields()).as("ck")); - } else { - builder.aggregate(builder.groupKey(), - builder.count(false, "c"), - builder.count() - .filter(builder - .not(builder - .and(builder.fields().stream() - .map(builder::isNull) - .collect(Collectors.toList())))) - .as("ck")); - } - builder.as(ctAlias); - if (!variablesSet.isEmpty()) { - builder.join(JoinRelType.LEFT, trueLiteral, variablesSet); - } else { - builder.join(JoinRelType.INNER, trueLiteral, variablesSet); + if (needsNullSafety) { + // Builds the cross join + // Some databases don't support use FILTER clauses for aggregate functions + // like {@code COUNT(*) FILTER (WHERE not(a is null))} + // So use count(*) when only one column + if (builder.fields().size() <= 1) { + builder.aggregate(builder.groupKey(), + builder.count(false, "c"), + builder.count(builder.fields()).as("ck")); + } else { + builder.aggregate(builder.groupKey(), + builder.count(false, "c"), + builder.count() + .filter(builder + .not(builder + .and(builder.fields().stream() + .map(builder::isNull) + .collect(Collectors.toList())))) + .as("ck")); + } + builder.as(ctAlias); + if (!variablesSet.isEmpty()) { + builder.join(JoinRelType.LEFT, trueLiteral, variablesSet); + } else { + builder.join(JoinRelType.INNER, trueLiteral, variablesSet); + } + offset += 2; + builder.push(e.rel); } - offset += 2; - builder.push(e.rel); // fall through default: builder.aggregate(builder.groupKey(fields), @@ -797,9 +805,12 @@ private static RexNode rewriteIn(RexSubQuery e, Set variablesSet, builder.equals(builder.field(dtAlias, "cs"), falseLiteral), b); } else { - operands.add( - builder.equals(builder.field(ctAlias, "c"), builder.literal(0)), - falseLiteral); + // only reference ctAlias if we created it + if (needsNullSafety) { + operands.add( + builder.equals(builder.field(ctAlias, "c"), builder.literal(0)), + falseLiteral); + } } break; default: @@ -822,17 +833,28 @@ private static RexNode rewriteIn(RexSubQuery e, Set variablesSet, switch (logic) { case TRUE_FALSE_UNKNOWN: case UNKNOWN_AS_TRUE: - operands.add( - builder.lessThan(builder.field(ctAlias, "ck"), - builder.field(ctAlias, "c")), - b); + // only reference ctAlias if we created it + if (needsNullSafety) { + operands.add( + builder.lessThan(builder.field(ctAlias, "ck"), + builder.field(ctAlias, "c")), + b); + } break; default: break; } } operands.add(falseLiteral); - return builder.call(SqlStdOperatorTable.CASE, operands.build()); + RexNode result = builder.call(SqlStdOperatorTable.CASE, operands.build()); + + // When we skip NULL-safety checks, the result might be NOT NULL + // but the original IN expression was nullable, so we need to preserve that + if (e.getType().isNullable() && !result.getType().isNullable()) { + result = builder.getRexBuilder().makeCast(e.getType(), result, false, false); + } + + return result; } /** Returns a reference to a particular field, by offset, across several diff --git a/core/src/test/java/org/apache/calcite/test/JdbcAdapterTest.java b/core/src/test/java/org/apache/calcite/test/JdbcAdapterTest.java index 51e2aca144b9..a94cad903bb6 100644 --- a/core/src/test/java/org/apache/calcite/test/JdbcAdapterTest.java +++ b/core/src/test/java/org/apache/calcite/test/JdbcAdapterTest.java @@ -248,7 +248,7 @@ class JdbcAdapterTest { .query("select * from dept where deptno not in (select deptno from emp)") .explainContains("PLAN=JdbcToEnumerableConverter\n" + " JdbcProject(DEPTNO=[$0], DNAME=[$1], LOC=[$2])\n" - + " JdbcFilter(condition=[OR(=($3, 0), AND(IS NULL($6), >=($4, $3)))])\n" + + " JdbcFilter(condition=[OR(AND(IS NULL($6), >=($4, $3)), =($3, 0))])\n" + " JdbcJoin(condition=[=($0, $5)], joinType=[left])\n" + " JdbcJoin(condition=[true], joinType=[inner])\n" + " JdbcTableScan(table=[[SCOTT, DEPT]])\n" @@ -263,7 +263,7 @@ class JdbcAdapterTest { + "FROM \"SCOTT\".\"EMP\") AS \"t\"\n" + "LEFT JOIN (SELECT \"DEPTNO\", TRUE AS \"i\"\n" + "FROM \"SCOTT\".\"EMP\"\nGROUP BY \"DEPTNO\") AS \"t0\" ON \"DEPT\".\"DEPTNO\" = \"t0\".\"DEPTNO\"\n" - + "WHERE \"t\".\"c\" = 0 OR \"t0\".\"i\" IS NULL AND \"t\".\"ck\" >= \"t\".\"c\""); + + "WHERE \"t0\".\"i\" IS NULL AND \"t\".\"ck\" >= \"t\".\"c\" OR \"t\".\"c\" = 0"); } @Test void testNotPushDownNotIn() { diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index ed3ffa32b0ef..33b72492f77c 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -5621,6 +5621,39 @@ private void checkEmptyJoin(RelOptFixture f) { sql(sql).withSubQueryRules().check(); } + /** Test case for + * [CALCITE-7317] + * SubQueryRemoveRule should skip NULL-safety checks for IN subqueries when + * both the keys and the subquery columns are NOT NULL. */ + @Test void testInOptimizationBothNotNull() { + final String sql = "select * from emp as e1\n" + + "where empno in (\n" + + " select empno from emp e2)"; + sql(sql).withSubQueryRules().check(); + } + + /** Test case for + * [CALCITE-7317] + * SubQueryRemoveRule should skip NULL-safety checks for IN subqueries when + * both the keys and the subquery columns are NOT NULL. */ + @Test void testNotInNullableSubqueryColumn() { + final String sql = "select * from empnullables as e1\n" + + "where coalesce(deptno, 0) not in (\n" + + " select deptno from empnullables e2)"; + sql(sql).withSubQueryRules().check(); + } + + /** Test case for + * [CALCITE-7317] + * SubQueryRemoveRule should skip NULL-safety checks for IN subqueries when + * both the keys and the subquery columns are NOT NULL. */ + @Test void testNotInNullableKey() { + final String sql = "select * from empnullables as e1\n" + + "where deptno not in (\n" + + " select coalesce(deptno, 0) from empnullables e2)"; + sql(sql).withSubQueryRules().check(); + } + @Test void testSomeWithGreaterThanNoRowSubQuery() { final String sql = "select * from dept as d\n" + "where deptno > some(\n" diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index df13f4c78aea..ab4ba1281bee 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -6363,6 +6363,33 @@ LogicalProject(ENAME=[$26], NAME=[$16]) LogicalJoin(condition=[=($9, $1)], joinType=[inner]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) LogicalTableScan(table=[[CATALOG, SALES, BONUS]]) +]]> + + + + + + + + + + + @@ -8493,14 +8520,10 @@ LogicalProject(DEPTNO=[$0]) =($10, $9)))], joinType=[inner]) - LogicalJoin(condition=[=($7, $11)], joinType=[left]) - LogicalJoin(condition=[true], joinType=[inner]) - LogicalTableScan(table=[[CATALOG, SALES, EMP]]) - LogicalProject(c=[$0], ck=[$0]) - LogicalAggregate(group=[{}], c=[COUNT()]) - LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) + LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7], SLACKER=[$8], DEPTNO0=[$11], NAME=[$12]) + LogicalJoin(condition=[IS NULL($10)], joinType=[inner]) + LogicalJoin(condition=[=($7, $9)], joinType=[left]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) LogicalProject(DEPTNO=[$0], i=[true]) LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) @@ -10705,6 +10728,72 @@ LogicalProject(USER=[USER]) LogicalAggregate(group=[{0}], EXPR$1=[SUM($1)]) LogicalProject(NAME=[$1], DEPTNO=[$0]) LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) +]]> + + + + + + + + + + + =($10, $9), IS NOT NULL($7)))]) + LogicalJoin(condition=[=($7, $11)], joinType=[left]) + LogicalJoin(condition=[true], joinType=[inner]) + LogicalTableScan(table=[[CATALOG, SALES, EMPNULLABLES]]) + LogicalProject(c=[$0], ck=[$0]) + LogicalAggregate(group=[{}], c=[COUNT()]) + LogicalTableScan(table=[[CATALOG, SALES, EMPNULLABLES]]) + LogicalAggregate(group=[{0}], i=[LITERAL_AGG(true)]) + LogicalProject(EXPR$0=[CASE(IS NOT NULL($7), CAST($7):INTEGER NOT NULL, 0)]) + LogicalTableScan(table=[[CATALOG, SALES, EMPNULLABLES]]) +]]> + + + + + + + + + + + ($9, 0)))]) + LogicalJoin(condition=[=(CASE(IS NOT NULL($7), CAST($7):INTEGER NOT NULL, 0), $11)], joinType=[left]) + LogicalJoin(condition=[true], joinType=[inner]) + LogicalTableScan(table=[[CATALOG, SALES, EMPNULLABLES]]) + LogicalAggregate(group=[{}], c=[COUNT()], ck=[COUNT($0)]) + LogicalProject(DEPTNO=[$7]) + LogicalTableScan(table=[[CATALOG, SALES, EMPNULLABLES]]) + LogicalAggregate(group=[{0}], i=[LITERAL_AGG(true)]) + LogicalProject(DEPTNO=[$7]) + LogicalTableScan(table=[[CATALOG, SALES, EMPNULLABLES]]) ]]> @@ -20921,17 +21010,9 @@ LogicalProject(DEPTNO=[$0]) @@ -20940,14 +21021,9 @@ LogicalProject(SAL=[$5]) ($2, 2), =($cor0.ENAME, $0))]) - LogicalProject(ENAME=[$1], EMPNO=[$0], R=[$5]) - LogicalTableScan(table=[[CATALOG, SALES, EMP]]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) LogicalFilter(condition=[=($cor0.EMPNO, $0)]) LogicalProject(EMPNO=[$1], i=[true]) LogicalFilter(condition=[AND(>($2, 2), =($cor0.ENAME, $0))]) @@ -20997,19 +21067,9 @@ LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$ ($2, 2)]) - LogicalProject(ENAME=[$1], EMPNO=[$0], R=[$5]) - LogicalTableScan(table=[[CATALOG, SALES, EMP]]) + LogicalFilter(condition=[IS NULL($10)]) + LogicalJoin(condition=[AND(=($0, $9), =($1, $11))], joinType=[left]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) LogicalProject(EMPNO=[$1], i=[true], ENAME=[$0]) LogicalFilter(condition=[>($2, 2)]) LogicalProject(ENAME=[$1], EMPNO=[$0], R=[$5]) diff --git a/core/src/test/resources/sql/sub-query.iq b/core/src/test/resources/sql/sub-query.iq index 2fa71122fba1..59fda9fca2f2 100644 --- a/core/src/test/resources/sql/sub-query.iq +++ b/core/src/test/resources/sql/sub-query.iq @@ -2023,7 +2023,7 @@ select sal from "scott".emp (0 rows) !ok -EnumerableCalc(expr#0..3=[{inputs}], expr#4=[IS NULL($t3)], expr#5=[NOT($t2)], expr#6=[IS NOT NULL($t2)], expr#7=[OR($t5, $t6)], expr#8=[IS NOT TRUE($t7)], expr#9=[OR($t4, $t8)], SAL=[$t1], $condition=[$t9]) +EnumerableCalc(expr#0..3=[{inputs}], expr#4=[NOT($t2)], expr#5=[IS NOT NULL($t2)], expr#6=[OR($t4, $t5)], expr#7=[IS NOT TRUE($t6)], expr#8=[IS NULL($t3)], expr#9=[OR($t7, $t8)], SAL=[$t1], $condition=[$t9]) EnumerableNestedLoopJoin(condition=[true], joinType=[left]) EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], SAL=[$t5]) EnumerableTableScan(table=[[scott, EMP]]) @@ -2128,7 +2128,7 @@ select sal from "scott".emp (0 rows) !ok -EnumerableCalc(expr#0..3=[{inputs}], expr#4=[IS NULL($t3)], expr#5=[NOT($t2)], expr#6=[IS NOT NULL($t2)], expr#7=[OR($t5, $t6)], expr#8=[IS NOT TRUE($t7)], expr#9=[OR($t4, $t8)], SAL=[$t1], $condition=[$t9]) +EnumerableCalc(expr#0..3=[{inputs}], expr#4=[NOT($t2)], expr#5=[IS NOT NULL($t2)], expr#6=[OR($t4, $t5)], expr#7=[IS NOT TRUE($t6)], expr#8=[IS NULL($t3)], expr#9=[OR($t7, $t8)], SAL=[$t1], $condition=[$t9]) EnumerableNestedLoopJoin(condition=[true], joinType=[left]) EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], SAL=[$t5]) EnumerableTableScan(table=[[scott, EMP]]) @@ -2610,6 +2610,44 @@ EnumerableCalc(expr#0..1=[{inputs}], EXPR$0=[$t1]) !use scott +# [CALCITE-7317] SubQueryRemoveRule should skip NULL-safety checks for IN subqueries when both keys and subquery columns are NOT NULL +select * from emp as e1 where empno in (select empno from emp e2); + EMPNO | ENAME | JOB | MGR | HIREDATE | SAL | COMM | DEPTNO +-------+--------+-----------+------+------------+---------+---------+-------- + 7369 | SMITH | CLERK | 7902 | 1980-12-17 | 800.00 | | 20 + 7499 | ALLEN | SALESMAN | 7698 | 1981-02-20 | 1600.00 | 300.00 | 30 + 7521 | WARD | SALESMAN | 7698 | 1981-02-22 | 1250.00 | 500.00 | 30 + 7566 | JONES | MANAGER | 7839 | 1981-02-04 | 2975.00 | | 20 + 7654 | MARTIN | SALESMAN | 7698 | 1981-09-28 | 1250.00 | 1400.00 | 30 + 7698 | BLAKE | MANAGER | 7839 | 1981-01-05 | 2850.00 | | 30 + 7782 | CLARK | MANAGER | 7839 | 1981-06-09 | 2450.00 | | 10 + 7788 | SCOTT | ANALYST | 7566 | 1987-04-19 | 3000.00 | | 20 + 7839 | KING | PRESIDENT | | 1981-11-17 | 5000.00 | | 10 + 7844 | TURNER | SALESMAN | 7698 | 1981-09-08 | 1500.00 | 0.00 | 30 + 7876 | ADAMS | CLERK | 7788 | 1987-05-23 | 1100.00 | | 20 + 7900 | JAMES | CLERK | 7698 | 1981-12-03 | 950.00 | | 30 + 7902 | FORD | ANALYST | 7566 | 1981-12-03 | 3000.00 | | 20 + 7934 | MILLER | CLERK | 7782 | 1982-01-23 | 1300.00 | | 10 +(14 rows) + +!ok + +# [CALCITE-7317] SubQueryRemoveRule should skip NULL-safety checks for IN subqueries when both keys and subquery columns are NOT NULL +select * from emp as e1 where coalesce(deptno, 0) not in (select deptno from emp e2); + EMPNO | ENAME | JOB | MGR | HIREDATE | SAL | COMM | DEPTNO +-------+-------+-----+-----+----------+-----+------+-------- +(0 rows) + +!ok + +# [CALCITE-7317] SubQueryRemoveRule should skip NULL-safety checks for IN subqueries when both keys and subquery columns are NOT NULL +select * from emp as e1 where deptno not in (select coalesce(deptno, 0) from emp e2); + EMPNO | ENAME | JOB | MGR | HIREDATE | SAL | COMM | DEPTNO +-------+-------+-----+-----+----------+-----+------+-------- +(0 rows) + +!ok + # [CALCITE-1513] Correlated NOT IN query throws AssertionError select count(*) as c from "scott".emp as e @@ -3920,7 +3958,7 @@ select * from "scott".emp where empno not in (null, 7782); !ok -EnumerableCalc(expr#0..12=[{inputs}], expr#13=[0], expr#14=[=($t8, $t13)], expr#15=[IS NULL($t12)], expr#16=[>=($t9, $t8)], expr#17=[AND($t15, $t16)], expr#18=[OR($t14, $t17)], proj#0..7=[{exprs}], $condition=[$t18]) +EnumerableCalc(expr#0..12=[{inputs}], expr#13=[IS NULL($t12)], expr#14=[>=($t9, $t8)], expr#15=[AND($t13, $t14)], expr#16=[0], expr#17=[=($t8, $t16)], expr#18=[OR($t15, $t17)], proj#0..7=[{exprs}], $condition=[$t18]) EnumerableMergeJoin(condition=[=($10, $11)], joinType=[left]) EnumerableSort(sort0=[$10], dir0=[ASC]) EnumerableCalc(expr#0..9=[{inputs}], expr#10=[CAST($t0):INTEGER NOT NULL], proj#0..10=[{exprs}]) @@ -5505,4 +5543,5 @@ WHERE EXISTS (2 rows) !ok + # End sub-query.iq From 9f45194cfbd5ed3570537d809acdd5aa86d8b729 Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Wed, 10 Dec 2025 21:04:13 +0800 Subject: [PATCH 043/562] [CALCITE-7319] FILTER_INTO_JOIN rule loses correlation variable context in HepPlanner --- .../calcite/rel/rules/FilterJoinRule.java | 5 ++++ .../rel/rel2sql/RelToSqlConverterTest.java | 29 +++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/core/src/main/java/org/apache/calcite/rel/rules/FilterJoinRule.java b/core/src/main/java/org/apache/calcite/rel/rules/FilterJoinRule.java index 8e58173c4bd9..e80ac4150d92 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/FilterJoinRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/FilterJoinRule.java @@ -84,6 +84,11 @@ protected void perform(RelOptRuleCall call, @Nullable Filter filter, return; } + if (filter != null + && RexUtil.containsCorrelation(filter.getCondition())) { + return; + } + final List aboveFilters = filter != null ? getConjunctions(filter) diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java index a4f78852fb80..6189182132ba 100644 --- a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java +++ b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java @@ -11025,6 +11025,35 @@ private void checkLiteral2(String expression, String expected) { .ok(expected); } + /** Test case of + * [CALCITE-7319] + * FILTER_INTO_JOIN rule loses correlation variable context in HepPlanner. */ + @Test void testFilterIntoJoinMissingVariableCor() { + final String sql = "SELECT E.EMPNO\n" + + "FROM EMP E\n" + + "JOIN DEPT D ON E.DEPTNO = D.DEPTNO\n" + + "WHERE D.DEPTNO = (\n" + + " SELECT MIN(D_INNER.DEPTNO)\n" + + " FROM DEPT D_INNER\n" + + " WHERE D_INNER.DEPTNO = E.DEPTNO)"; + final String expected = "SELECT \"EMP\".\"EMPNO\"\n" + + "FROM \"SCOTT\".\"EMP\"\n" + + "INNER JOIN \"SCOTT\".\"DEPT\" ON \"EMP\".\"DEPTNO\" = \"DEPT\".\"DEPTNO\"\n" + + "WHERE \"DEPT\".\"DEPTNO\" = (((SELECT MIN(\"DEPTNO\")\n" + + "FROM \"SCOTT\".\"DEPT\"\n" + + "WHERE \"DEPTNO\" = \"EMP\".\"DEPTNO\")))"; + + HepProgramBuilder builder = new HepProgramBuilder(); + builder.addRuleClass(FilterJoinRule.FilterIntoJoinRule.class); + HepPlanner hepPlanner = new HepPlanner(builder.build()); + RuleSet rules = RuleSets.ofList(CoreRules.FILTER_INTO_JOIN); + sql(sql) + .schema(CalciteAssert.SchemaSpec.JDBC_SCOTT) + .withCalcite() + .optimize(rules, hepPlanner) + .ok(expected); + } + /** Fluid interface to run tests. */ static class Sql { private final CalciteAssert.SchemaSpec schemaSpec; From 7ac1d7ddcafdc5ac24e55c5eae5d2db4ad2f81a5 Mon Sep 17 00:00:00 2001 From: iwanttobepowerful <745778074@qq.com> Date: Mon, 8 Dec 2025 22:36:52 +0800 Subject: [PATCH 044/562] [CALCITE-7297] The result is incorrect when the GROUP BY key in a subquery is a RexFieldAccess --- .../calcite/sql2rel/RelDecorrelator.java | 12 +-- .../calcite/sql2rel/RelDecorrelatorTest.java | 69 +++++++++++++++++ .../calcite/test/SqlToRelConverterTest.xml | 14 ++-- core/src/test/resources/sql/sub-query.iq | 75 +++++++++++++++++++ 4 files changed, 151 insertions(+), 19 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java b/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java index e45edff7e577..db647da89cba 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java @@ -1217,9 +1217,7 @@ private static void shiftMapping(Map mapping, int startIndex, // If this Project has correlated reference, create value generator // and produce the correlated variables in the new output. - if (cm.mapRefRelToCorRef.containsKey(rel)) { - frame = decorrelateInputWithValueGenerator(rel, frame); - } + frame = maybeAddValueGenerator(rel, frame); // Project projects the original expressions final Map mapOldToNewOutputs = new HashMap<>(); @@ -1609,13 +1607,7 @@ private static boolean isWidening(RelDataType type, RelDataType type1) { // If this Filter has correlated reference, create value generator // and produce the correlated variables in the new output. - if (false) { - if (cm.mapRefRelToCorRef.containsKey(rel)) { - frame = decorrelateInputWithValueGenerator(rel, frame); - } - } else { - frame = maybeAddValueGenerator(rel, frame); - } + frame = maybeAddValueGenerator(rel, frame); final CorelMap cm2 = new CorelMapBuilder().build(rel); diff --git a/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java b/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java index f6504fdef9d2..b92606be5b23 100644 --- a/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java +++ b/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java @@ -356,6 +356,75 @@ public static Frameworks.ConfigBuilder config() { assertThat(after, hasTree(planAfter)); } + /** Test case for [CALCITE-7297] + * The result is incorrect when the GROUP BY key in a subquery is a RexFieldAccess. */ + @Test void testSkipsRedundantValueGenerator() { + final FrameworkConfig frameworkConfig = config().build(); + final RelBuilder builder = RelBuilder.create(frameworkConfig); + final RelOptCluster cluster = builder.getCluster(); + final Planner planner = Frameworks.getPlanner(frameworkConfig); + final String sql = "" + + "SELECT *,\n" + + " (SELECT COUNT(*)\n" + + " FROM \n" + + " (\n" + + " SELECT empno, ename, job\n" + + " FROM emp\n" + + " WHERE emp.deptno = dept.deptno) AS sub\n" + + " GROUP BY deptno) AS num_dept_groups\n" + + "FROM dept"; + final RelNode originalRel; + try { + final SqlNode parse = planner.parse(sql); + final SqlNode validate = planner.validate(parse); + originalRel = planner.rel(validate).rel; + } catch (Exception e) { + throw TestUtil.rethrow(e); + } + + final HepProgram hepProgram = HepProgram.builder() + .addRuleCollection( + ImmutableList.of( + // SubQuery program rules + CoreRules.FILTER_SUB_QUERY_TO_CORRELATE, + CoreRules.PROJECT_SUB_QUERY_TO_CORRELATE, + CoreRules.JOIN_SUB_QUERY_TO_CORRELATE)) + .build(); + final Program program = + Programs.of(hepProgram, true, + requireNonNull(cluster.getMetadataProvider())); + final RelNode before = + program.run(cluster.getPlanner(), originalRel, cluster.traitSet(), + Collections.emptyList(), Collections.emptyList()); + final String planBefore = "" + + "LogicalCorrelate(correlation=[$cor1], joinType=[left], requiredColumns=[{0}])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n" + + " LogicalAggregate(group=[{}], agg#0=[SINGLE_VALUE($0)])\n" + + " LogicalProject(EXPR$0=[$1])\n" + + " LogicalAggregate(group=[{0}], EXPR$0=[COUNT()])\n" + + " LogicalProject($f0=[$cor1.DEPTNO])\n" + + " LogicalFilter(condition=[=($7, $cor1.DEPTNO)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + assertThat(before, hasTree(planBefore)); + + // Decorrelate without any rules, just "purely" decorrelation algorithm on RelDecorrelator + final RelNode after = + RelDecorrelator.decorrelateQuery(before, builder, RuleSets.ofList(Collections.emptyList()), + RuleSets.ofList(Collections.emptyList())); + // Verify plan + final String planAfter = "" + + "LogicalProject(DEPTNO=[$0], DNAME=[$1], LOC=[$2], $f1=[$4])\n" + + " LogicalJoin(condition=[=($0, $3)], joinType=[left])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n" + + " LogicalAggregate(group=[{0}], agg#0=[SINGLE_VALUE($1)])\n" + + " LogicalProject(DEPTNO=[$1], EXPR$0=[$2])\n" + + " LogicalAggregate(group=[{0, 1}], EXPR$0=[COUNT()])\n" + + " LogicalProject($f0=[$7], DEPTNO=[$7])\n" + + " LogicalFilter(condition=[IS NOT NULL($7)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + assertThat(after, hasTree(planAfter)); + } + /** * Test case for * [CALCITE-6468] RelDecorrelator diff --git a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml index a1c7f101b39b..7cdde323145c 100644 --- a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml @@ -1601,15 +1601,11 @@ LogicalProject(C=[$0], D=[$1], C0=[$4], F=[$5]) LogicalJoin(condition=[AND(=($0, $6), =($3, $7))], joinType=[inner]) LogicalProject(C=[$0], D=[$1], C0=[$0], $f3=[+($0, $1)]) LogicalValues(tuples=[[{ 4, 5 }]]) - LogicalProject(C=[$3], F=[*($0, $3)], C0=[$3], $f3=[$2]) - LogicalJoin(condition=[true], joinType=[inner]) - LogicalJoin(condition=[=($2, *($0, $1))], joinType=[inner]) - LogicalValues(tuples=[[{ 2 }]]) - LogicalAggregate(group=[{0, 1}]) - LogicalProject(C=[$0], $f3=[+($0, $1)]) - LogicalValues(tuples=[[{ 4, 5 }]]) - LogicalAggregate(group=[{0}]) - LogicalProject(C=[$0]) + LogicalProject(C=[$1], F=[*($0, $1)], C2=[$1], $f3=[$2]) + LogicalJoin(condition=[=($2, *($0, $1))], joinType=[inner]) + LogicalValues(tuples=[[{ 2 }]]) + LogicalAggregate(group=[{0, 1}]) + LogicalProject(C=[$0], $f3=[+($0, $1)]) LogicalValues(tuples=[[{ 4, 5 }]]) ]]> diff --git a/core/src/test/resources/sql/sub-query.iq b/core/src/test/resources/sql/sub-query.iq index 59fda9fca2f2..a12b087ebb21 100644 --- a/core/src/test/resources/sql/sub-query.iq +++ b/core/src/test/resources/sql/sub-query.iq @@ -5544,4 +5544,79 @@ WHERE EXISTS !ok +# [CALCITE-7297] The result is incorrect when the GROUP BY key in a subquery is a RexFieldAccess +SELECT *, + (SELECT COUNT(*) + FROM + ( + SELECT empno, ename, job + FROM emp + WHERE emp.deptno = dept.deptno) AS sub + GROUP BY deptno) AS num_dept_groups +FROM dept; ++--------+------------+----------+-----------------+ +| DEPTNO | DNAME | LOC | NUM_DEPT_GROUPS | ++--------+------------+----------+-----------------+ +| 10 | ACCOUNTING | NEW YORK | 3 | +| 20 | RESEARCH | DALLAS | 5 | +| 30 | SALES | CHICAGO | 6 | +| 40 | OPERATIONS | BOSTON | | ++--------+------------+----------+-----------------+ +(4 rows) + +!ok + +# [CALCITE-7297] The result is incorrect when the GROUP BY key in a subquery is a RexFieldAccess +SELECT *, + (SELECT COUNT(*) + FROM + ( + SELECT empno, ename, job, comm + FROM emp + WHERE emp.deptno = dept.deptno + ORDER BY empno LIMIT 1 + ) AS sub + GROUP BY sub.comm + ) AS num_dept_groups +FROM dept; ++--------+------------+----------+-----------------+ +| DEPTNO | DNAME | LOC | NUM_DEPT_GROUPS | ++--------+------------+----------+-----------------+ +| 10 | ACCOUNTING | NEW YORK | 1 | +| 20 | RESEARCH | DALLAS | 1 | +| 30 | SALES | CHICAGO | 1 | +| 40 | OPERATIONS | BOSTON | | ++--------+------------+----------+-----------------+ +(4 rows) + +!ok + +# [CALCITE-7297] The result is incorrect when the GROUP BY key in a subquery is a RexFieldAccess +select * from (values (4, 5)) as t(c, d) +cross join lateral +(select c, a*c as f +from (values 2) as s(a) +where c+d=a*c); ++---+---+----+---+ +| C | D | C0 | F | ++---+---+----+---+ ++---+---+----+---+ +(0 rows) + +!ok + +# [CALCITE-7297] The result is incorrect when the GROUP BY key in a subquery is a RexFieldAccess +select * from (values (2,2), (2,2)) as t(c,d) +cross join lateral +(select c,a*c as f from (values 2) as s(a) +where c+d=a*c); ++---+---+----+---+ +| C | D | C0 | F | ++---+---+----+---+ +| 2 | 2 | 2 | 4 | +| 2 | 2 | 2 | 4 | ++---+---+----+---+ +(2 rows) + +!ok # End sub-query.iq From 10f20d9930578ea7f27b8f71781f0ee96679255e Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Wed, 10 Dec 2025 17:47:16 -0800 Subject: [PATCH 045/562] [CALCITE-7326] FILTER_CORRELATE rule loses correlation variable context in HepPlanner Signed-off-by: Mihai Budiu --- .../rel/rules/FilterCorrelateRule.java | 17 +++++++++- .../rel/rel2sql/RelToSqlConverterTest.java | 33 +++++++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/calcite/rel/rules/FilterCorrelateRule.java b/core/src/main/java/org/apache/calcite/rel/rules/FilterCorrelateRule.java index ab60bd9acdbb..4c13baa173e5 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/FilterCorrelateRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/FilterCorrelateRule.java @@ -70,12 +70,24 @@ public FilterCorrelateRule(RelFactories.FilterFactory filterFactory, final Filter filter = call.rel(0); final Correlate corr = call.rel(1); - final List aboveFilters = + List aboveFilters = RelOptUtil.conjunctions(filter.getCondition()); final List leftFilters = new ArrayList<>(); final List rightFilters = new ArrayList<>(); + // Do not consider moving predicates that contain correlation variables + final List ineligible = new ArrayList<>(); + final List eligible = new ArrayList<>(); + for (RexNode f : aboveFilters) { + if (RexUtil.containsCorrelation(f)) { + ineligible.add(f); + } else { + eligible.add(f); + } + } + aboveFilters = eligible; + // Try to push down above filters. These are typically where clause // filters. They can be pushed down if they are not on the NULL // generating side. @@ -89,6 +101,9 @@ public FilterCorrelateRule(RelFactories.FilterFactory filterFactory, leftFilters, rightFilters); + // Add back the ineligible filters + aboveFilters.addAll(ineligible); + if (leftFilters.isEmpty() && rightFilters.isEmpty()) { // no filters got pushed diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java index 6189182132ba..915db7da6e51 100644 --- a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java +++ b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java @@ -38,8 +38,10 @@ import org.apache.calcite.rel.rules.AggregateJoinTransposeRule; import org.apache.calcite.rel.rules.AggregateProjectMergeRule; import org.apache.calcite.rel.rules.CoreRules; +import org.apache.calcite.rel.rules.FilterCorrelateRule; import org.apache.calcite.rel.rules.FilterJoinRule; import org.apache.calcite.rel.rules.FullToLeftAndRightJoinRule; +import org.apache.calcite.rel.rules.JoinToCorrelateRule; import org.apache.calcite.rel.rules.ProjectOverSumToSum0Rule; import org.apache.calcite.rel.rules.ProjectToWindowRule; import org.apache.calcite.rel.rules.PruneEmptyRules; @@ -11054,6 +11056,37 @@ private void checkLiteral2(String expression, String expected) { .ok(expected); } + /** Test case of + * [CALCITE-7326] + * FILTER_CORRELATE rule loses correlation variable context in HepPlanner. */ + @Test void testFilterCorrelateMissingVariableCor() { + final String sql = "SELECT E.EMPNO\n" + + "FROM EMP E\n" + + "JOIN DEPT D ON E.DEPTNO = D.DEPTNO\n" + + "WHERE D.DEPTNO = (\n" + + " SELECT MIN(D_INNER.DEPTNO)\n" + + " FROM DEPT D_INNER\n" + + " WHERE D_INNER.DEPTNO = E.DEPTNO)"; + final String expected = "SELECT \"$cor1\".\"EMPNO\"\n" + + "FROM \"SCOTT\".\"EMP\" AS \"$cor1\",\n" + + "LATERAL (SELECT *\nFROM \"SCOTT\".\"DEPT\"\n" + + "WHERE \"$cor1\".\"DEPTNO\" = \"DEPTNO\") AS \"t\"\n" + + "WHERE \"t\".\"DEPTNO\" = (((SELECT MIN(\"DEPTNO\")\n" + + "FROM \"SCOTT\".\"DEPT\"\nWHERE \"DEPTNO\" = \"$cor1\".\"DEPTNO\")))"; + HepProgramBuilder builder = new HepProgramBuilder(); + builder.addRuleClass(JoinToCorrelateRule.class); + builder.addRuleClass(FilterCorrelateRule.class); + HepPlanner hepPlanner = new HepPlanner(builder.build()); + RuleSet rules = + RuleSets.ofList(CoreRules.JOIN_TO_CORRELATE, + CoreRules.FILTER_CORRELATE); + sql(sql) + .schema(CalciteAssert.SchemaSpec.JDBC_SCOTT) + .withCalcite() + .optimize(rules, hepPlanner) + .ok(expected); + } + /** Fluid interface to run tests. */ static class Sql { private final CalciteAssert.SchemaSpec schemaSpec; From a01f2ca4cc69408f03205d24115b27340ff1dbc4 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Wed, 10 Dec 2025 20:27:59 -0800 Subject: [PATCH 046/562] [CALCITE-7319] FILTER_INTO_JOIN rule loses correlation variable context in HepPlanner Signed-off-by: Mihai Budiu --- .../calcite/rel/rules/FilterJoinRule.java | 21 +++-- .../apache/calcite/test/RelOptRulesTest.java | 17 ++++ .../apache/calcite/test/RelOptRulesTest.xml | 40 +++++++++ .../apache/calcite/adapter/tpch/TpchTest.java | 85 +++++++++++++++++++ 4 files changed, 157 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rel/rules/FilterJoinRule.java b/core/src/main/java/org/apache/calcite/rel/rules/FilterJoinRule.java index e80ac4150d92..1f80126aa869 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/FilterJoinRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/FilterJoinRule.java @@ -84,12 +84,7 @@ protected void perform(RelOptRuleCall call, @Nullable Filter filter, return; } - if (filter != null - && RexUtil.containsCorrelation(filter.getCondition())) { - return; - } - - final List aboveFilters = + List aboveFilters = filter != null ? getConjunctions(filter) : new ArrayList<>(); @@ -107,6 +102,18 @@ protected void perform(RelOptRuleCall call, @Nullable Filter filter, final List leftFilters = new ArrayList<>(); final List rightFilters = new ArrayList<>(); + // Do not consider moving predicates that contain correlation variables + final List ineligible = new ArrayList<>(); + final List eligible = new ArrayList<>(); + for (RexNode f : aboveFilters) { + if (RexUtil.containsCorrelation(f)) { + ineligible.add(f); + } else { + eligible.add(f); + } + } + aboveFilters = eligible; + // TODO - add logic to derive additional filters. E.g., from // (t1.a = 1 AND t2.a = 2) OR (t1.b = 3 AND t2.b = 4), you can // derive table filters: @@ -126,6 +133,8 @@ protected void perform(RelOptRuleCall call, @Nullable Filter filter, leftFilters, rightFilters); + // Add back the ineligible filters + aboveFilters.addAll(ineligible); // Move join filters up if needed validateJoinFilters(aboveFilters, joinFilters, join, joinType); diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index 33b72492f77c..8d432256b4be 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -8275,6 +8275,23 @@ private void checkSemiJoinRuleOnAntiJoin(RelOptRule rule) { .check(); } + /** Test case for [CALCITE-7319] + * FILTER_INTO_JOIN rule loses correlation variable context in HepPlanner. */ + @Test void testFilterIntoJoinMissingVariableCor() { + final String sql = "SELECT E.EMPNO\n" + + "FROM EMP E\n" + + "JOIN DEPT D ON E.DEPTNO = D.DEPTNO\n" + + "WHERE E.EMPNO > 10 AND D.DEPTNO = (\n" + + " SELECT MIN(D_INNER.DEPTNO)\n" + + " FROM DEPT D_INNER\n" + + " WHERE D_INNER.DEPTNO = E.DEPTNO)"; + sql(sql) + .withExpand(false) + .withDecorrelate(false) + .withRule(CoreRules.FILTER_INTO_JOIN) + .check(); + } + /** Test case for * [CALCITE-4616] * AggregateUnionTransposeRule causes row type mismatch when some inputs have diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index ab4ba1281bee..8ed236780f67 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -5594,6 +5594,46 @@ LogicalFilter(condition=[<($0, 20)]) })]) LogicalCalc(expr#0..8=[{inputs}], DEPTNO=[$t7]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +]]> + + + + + 10 AND D.DEPTNO = ( + SELECT MIN(D_INNER.DEPTNO) + FROM DEPT D_INNER + WHERE D_INNER.DEPTNO = E.DEPTNO)]]> + + + ($0, 10), =($9, $SCALAR_QUERY({ +LogicalAggregate(group=[{}], EXPR$0=[MIN($0)]) + LogicalProject(DEPTNO=[$0]) + LogicalFilter(condition=[=($0, $cor0.DEPTNO)]) + LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) +})))], variablesSet=[[$cor0]]) + LogicalJoin(condition=[=($7, $9)], joinType=[inner]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) + LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) +]]> + + + ($0, 10)]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) + LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) ]]> diff --git a/plus/src/test/java/org/apache/calcite/adapter/tpch/TpchTest.java b/plus/src/test/java/org/apache/calcite/adapter/tpch/TpchTest.java index 9ea5cd0ab7e1..5f4e93d9d7a1 100644 --- a/plus/src/test/java/org/apache/calcite/adapter/tpch/TpchTest.java +++ b/plus/src/test/java/org/apache/calcite/adapter/tpch/TpchTest.java @@ -16,8 +16,23 @@ */ package org.apache.calcite.adapter.tpch; +import org.apache.calcite.plan.RelOptPlanner; import org.apache.calcite.plan.RelOptUtil; +import org.apache.calcite.plan.hep.HepPlanner; +import org.apache.calcite.plan.hep.HepProgram; +import org.apache.calcite.plan.hep.HepProgramBuilder; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.RelRoot; +import org.apache.calcite.rel.rules.CoreRules; +import org.apache.calcite.schema.SchemaPlus; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.parser.SqlParseException; import org.apache.calcite.test.CalciteAssert; +import org.apache.calcite.tools.FrameworkConfig; +import org.apache.calcite.tools.Frameworks; +import org.apache.calcite.tools.Planner; +import org.apache.calcite.tools.RelConversionException; +import org.apache.calcite.tools.ValidationException; import org.apache.calcite.util.TestUtil; import com.google.common.collect.ImmutableList; @@ -29,6 +44,8 @@ import java.util.List; import java.util.concurrent.TimeUnit; +import static org.apache.calcite.test.Matchers.containsStringLinux; + import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.not; import static org.hamcrest.MatcherAssert.assertThat; @@ -833,6 +850,74 @@ private CalciteAssert.AssertThat with() { }); } + /** Test case for [CALCITE-7319] + * FILTER_INTO_JOIN rule loses correlation variable context in HepPlanner. */ + @Test public void optimizeQuery2() + throws SqlParseException, ValidationException, RelConversionException { + SchemaPlus rootSchema = Frameworks.createRootSchema(true); + TpchSchema tpchSchema = new TpchSchema(1.0, 0, 1, false); + rootSchema.add("TPCH", tpchSchema); + FrameworkConfig config = Frameworks.newConfigBuilder() + .defaultSchema(rootSchema) + .build(); + + Planner planner = Frameworks.getPlanner(config); + + SqlNode parsed = planner.parse(QUERY_ARRAY[1]); + SqlNode validated = planner.validate(parsed); + RelRoot root = planner.rel(validated); + + final HepProgramBuilder builder = HepProgram.builder(); + builder.addRuleInstance(CoreRules.FILTER_SUB_QUERY_TO_CORRELATE); + builder.addRuleInstance(CoreRules.PROJECT_SUB_QUERY_TO_CORRELATE); + builder.addRuleInstance(CoreRules.JOIN_SUB_QUERY_TO_CORRELATE); + builder.addRuleInstance(CoreRules.FILTER_CORRELATE); + // We are checking that this rule can push some predicates into joins + // Prior to fixing [CALCITE-7319] (second improvement) many joins below + // had a condition=[true]. + builder.addRuleInstance(CoreRules.FILTER_INTO_JOIN); + + RelOptPlanner optPlanner = new HepPlanner(builder.build()); + optPlanner.setRoot(root.rel); + RelNode rel = optPlanner.findBestExp(); + final String expected = "LogicalSort(sort0=[$0], sort1=[$2], sort2=[$1], sort3=[$3], " + + "dir0=[DESC], dir1=[ASC], dir2=[ASC], dir3=[ASC], fetch=[100])\n" + + " LogicalProject(S_ACCTBAL=[$14], S_NAME=[$10], N_NAME=[$22], P_PARTKEY=[$0], " + + "P_MFGR=[$2], S_ADDRESS=[$11], S_PHONE=[$13], S_COMMENT=[$15])\n" + + " LogicalProject(P_PARTKEY=[$0], P_NAME=[$1], P_MFGR=[$2], P_BRAND=[$3], P_TYPE=[$4], " + + "P_SIZE=[$5], P_CONTAINER=[$6], P_RETAILPRICE=[$7], P_COMMENT=[$8], S_SUPPKEY=[$9], " + + "S_NAME=[$10], S_ADDRESS=[$11], S_NATIONKEY=[$12], S_PHONE=[$13], S_ACCTBAL=[$14], " + + "S_COMMENT=[$15], PS_PARTKEY=[$16], PS_SUPPKEY=[$17], PS_AVAILQTY=[$18], " + + "PS_SUPPLYCOST=[$19], PS_COMMENT=[$20], N_NATIONKEY=[$21], N_NAME=[$22], " + + "N_REGIONKEY=[$23], N_COMMENT=[$24], R_REGIONKEY=[$25], R_NAME=[$26], R_COMMENT=[$27])\n" + + " LogicalFilter(condition=[=($19, $28)])\n" + + " LogicalCorrelate(correlation=[$cor0], joinType=[left], requiredColumns=[{0}])\n" + + " LogicalJoin(condition=[=($23, $25)], joinType=[inner])\n" + + " LogicalJoin(condition=[=($12, $21)], joinType=[inner])\n" + + " LogicalJoin(condition=[AND(=($0, $16), =($9, $17))], joinType=[inner])\n" + + " LogicalJoin(condition=[true], joinType=[inner])\n" + + " LogicalFilter(condition=[AND(=(CAST($5):INTEGER, 41), " + + "LIKE($4, '%NICKEL'))])\n" + + " LogicalTableScan(table=[[TPCH, PART]])\n" + + " LogicalTableScan(table=[[TPCH, SUPPLIER]])\n" + + " LogicalTableScan(table=[[TPCH, PARTSUPP]])\n" + + " LogicalTableScan(table=[[TPCH, NATION]])\n" + + " LogicalFilter(condition=[=(CAST($1):VARCHAR, 'EUROPE')])\n" + + " LogicalTableScan(table=[[TPCH, REGION]])\n" + + " LogicalAggregate(group=[{}], EXPR$0=[MIN($0)])\n" + + " LogicalProject(PS_SUPPLYCOST=[$3])\n" + + " LogicalFilter(condition=[=($cor0.P_PARTKEY, $0)])\n" + + " LogicalJoin(condition=[=($14, $16)], joinType=[inner])\n" + + " LogicalJoin(condition=[=($8, $12)], joinType=[inner])\n" + + " LogicalJoin(condition=[=($5, $1)], joinType=[inner])\n" + + " LogicalTableScan(table=[[TPCH, PARTSUPP]])\n" + + " LogicalTableScan(table=[[TPCH, SUPPLIER]])\n" + + " LogicalTableScan(table=[[TPCH, NATION]])\n" + + " LogicalFilter(condition=[=(CAST($1):VARCHAR, 'EUROPE')])\n" + + " LogicalTableScan(table=[[TPCH, REGION]])"; + assertThat(rel.explain(), containsStringLinux(expected)); + } + @Test void testQuery03() { checkQuery(3); } From 273e1583ca9eae3c7dd3a90fffbfabd6c3385452 Mon Sep 17 00:00:00 2001 From: Alessandro Solimando Date: Tue, 9 Dec 2025 14:29:57 +0100 Subject: [PATCH 047/562] [CALCITE-7321] FilesTableFunction throws NumberFormatException on macOS with GNU stat installed --- .../adapter/os/FilesTableFunction.java | 82 ++++++++++++++++--- 1 file changed, 70 insertions(+), 12 deletions(-) diff --git a/plus/src/main/java/org/apache/calcite/adapter/os/FilesTableFunction.java b/plus/src/main/java/org/apache/calcite/adapter/os/FilesTableFunction.java index c7de135d494f..8c7b3cc81555 100644 --- a/plus/src/main/java/org/apache/calcite/adapter/os/FilesTableFunction.java +++ b/plus/src/main/java/org/apache/calcite/adapter/os/FilesTableFunction.java @@ -44,7 +44,8 @@ public class FilesTableFunction { private static final BigDecimal THOUSAND = BigDecimal.valueOf(1000L); - + private static final String SINGLE_QUOTE = + "Path with single quote characters are not supported"; private FilesTableFunction() { } @@ -115,9 +116,9 @@ private Enumerable sourceLinux() { private Enumerable sourceMacOs() { if (path.contains("'")) { - // no injection monkey business - throw new IllegalArgumentException(); + throw new IllegalArgumentException(SINGLE_QUOTE); } + // BSD stat format specifiers: https://man.freebsd.org/cgi/man.cgi?query=stat final String[] args = {"/bin/sh", "-c", "find '" + path + "' | xargs stat -f " + "%a%n" // access_time @@ -144,6 +145,49 @@ private Enumerable sourceMacOs() { return Processes.processLines('\n', args); } + private Enumerable sourceGnuStat() { + if (path.contains("'")) { + throw new IllegalArgumentException(SINGLE_QUOTE); + } + // GNU stat format specifiers: + // https://www.gnu.org/software/coreutils/manual/html_node/stat-invocation.html + // format string must have exactly 20 lines per file to match the schema + final String[] args = {"/bin/sh", "-c", "find '" + path + + "' | xargs stat -c '" + + "%X\n" // access_time + + "%b\n" // block_count + + "%Z\n" // change_time + + "0\n" // depth: computed later based on path + + "%d\n" // device + + "filename\n" // filename: computed later from path + + "%F\n" // fstype (file system type) + + "%G\n" // gname + + "%g\n" // gid + + "dirname\n" // dir_name: computed later from path + + "%i\n\n" // inode (followed by empty line for link) + + "%a\n" // perm + + "%h\n" // hard + + "%n\n" // path + + "%s\n" // size + + "%Y\n" // mod_time + + "%U\n" // user + + "%u\n" // uid + + "%F'" // type + }; + return Processes.processLines('\n', args); + } + + private boolean isGnuStat() { + try { + // BSD stat doesn't support --version, so we use this to detect GNU stat + final String[] args = {"stat", "--version"}; + return Processes.processLines('\n', args) + .any(line -> line.contains("GNU coreutils")); + } catch (RuntimeException e) { + return false; + } + } + @Override public Enumerable<@Nullable Object[]> scan(DataContext root) { JavaTypeFactory typeFactory = root.getTypeFactory(); final RelDataType rowType = getRowType(typeFactory); @@ -154,8 +198,9 @@ private Enumerable sourceMacOs() { Util.discard(osVersion); final Enumerable enumerable; switch (osName) { - case "Mac OS X": // tested on version 10.12.5 - enumerable = sourceMacOs(); + case "Mac OS X": // tested on versions 10.12.5 and 15.6.1 + // detect GNU vs BSD stat and adapt the format accordingly + enumerable = isGnuStat() ? sourceGnuStat() : sourceMacOs(); break; default: enumerable = sourceLinux(); @@ -187,7 +232,7 @@ private Enumerable sourceMacOs() { } switch (osName) { case "Mac OS X": - // Strip leading "./" + // post-process fields: compute filename, dir_name, depth from path String path = requireNonNull((String) current[14]); if (".".equals(path)) { current[14] = path = ""; @@ -207,12 +252,25 @@ private Enumerable sourceMacOs() { current[9] = ""; // dir_name } - // Make type values more like those on Linux - final String type = (String) current[19]; - current[19] = "/".equals(type) ? "d" - : "".equals(type) || "*".equals(type) ? "f" - : "@".equals(type) ? "l" - : type; + // detect output format: BSD outputs single chars, GNU outputs words + final String type = requireNonNull((String) current[19]); + if (type.length() > 1) { + // GNU stat outputs descriptive types like "regular file", "directory" + current[19] = type.contains("directory") ? "d" + : type.contains("regular") ? "f" + : type.contains("symbolic") ? "l" + : type.contains("block") ? "b" + : type.contains("character") ? "c" + : type.contains("fifo") ? "p" + : type.contains("socket") ? "s" + : "?"; + } else { + // BSD stat outputs single characters: "/" "*" "@" or "" + current[19] = "/".equals(type) ? "d" + : "".equals(type) || "*".equals(type) ? "f" + : "@".equals(type) ? "l" + : type; + } break; default: break; From c5f36c988756c32bd73e0116b69ab1b62a7fcd98 Mon Sep 17 00:00:00 2001 From: Jinkun Liu Date: Thu, 11 Dec 2025 21:54:58 +0800 Subject: [PATCH 048/562] [CALCITE-7325] Incorrect VARIANT signatures in SqlItemOperator --- .../java/org/apache/calcite/sql/fun/SqlItemOperator.java | 2 +- .../java/org/apache/calcite/test/SqlValidatorTest.java | 2 +- .../java/org/apache/calcite/test/SqlOperatorTest.java | 9 ++++++++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlItemOperator.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlItemOperator.java index 1a1fb4d2a857..2285b0804aab 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlItemOperator.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlItemOperator.java @@ -183,7 +183,7 @@ private static SqlSingleOperandTypeChecker getChecker(SqlCallBinding callBinding return "[]\n" + "[]\n" + "[|]\n" - + "[]"; + + "[|]"; } else { return "[" + name + "()]"; } diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index 2de4ae67044d..8a6877193dab 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -9331,7 +9331,7 @@ void testGroupExpressionEquivalenceParams() { + "\\)'\\. Supported form\\(s\\): \\[\\]\n" + "\\[\\]\n" + "\\[\\|\\]\n" - + "\\[\\].*"); + + "\\[\\|\\].*"); } /** Test case for diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java index b11a96845827..351f44584f27 100644 --- a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java +++ b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java @@ -13373,7 +13373,14 @@ private static void checkArrayConcatAggFuncFails(SqlOperatorFixture t) { + "\\)'\\. Supported form\\(s\\): \\[\\]\n" + "\\[\\]\n" + "\\[\\|\\]\n" - + "\\[\\]", + + "\\[\\|\\]", + false); + f.checkFails("^CAST(MAP[4.2, 1] AS VARIANT)[4.2]^", + "Cannot apply 'ITEM' to arguments of type 'ITEM\\(, \\)'\\. " + + "Supported form\\(s\\): \\[\\]\n" + + "\\[\\]\n" + + "\\[\\|\\]\n" + + "\\[\\|\\]", false); // Array of INTEGER NOT NULL is interesting because we might be tempted From 0ac585030de5d7e564002e0819497f911181c20b Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Sat, 13 Dec 2025 10:35:30 +0800 Subject: [PATCH 049/562] [CALCITE-7332] SELECT * EXCLUDE list should error when it excludes all columns --- .../org/apache/calcite/test/BabelTest.java | 21 +++++++++++++++++++ .../calcite/runtime/CalciteResource.java | 3 +++ .../sql/validate/SqlValidatorImpl.java | 15 +++++++++++++ .../runtime/CalciteResource.properties | 1 + 4 files changed, 40 insertions(+) diff --git a/babel/src/test/java/org/apache/calcite/test/BabelTest.java b/babel/src/test/java/org/apache/calcite/test/BabelTest.java index 9368a7ab7146..261bcb24791e 100644 --- a/babel/src/test/java/org/apache/calcite/test/BabelTest.java +++ b/babel/src/test/java/org/apache/calcite/test/BabelTest.java @@ -210,6 +210,27 @@ names, is( }); } + /** Test case for [CALCITE-7332] + * SELECT * EXCLUDE list should error when it excludes all columns. */ + @Test void testStarExcludeWithEmptyColumn() { + final SqlValidatorFixture fixture = Fixtures.forValidator() + .withParserConfig(p -> p.withParserFactory(SqlBabelParserImpl.FACTORY)); + + // To verify the scenario where all columns in the exclude list exist + // and the number of columns in the list is equal to the number of columns in the table. + fixture.withSql("select * exclude(deptno, deptno) from dept") + .type(type -> { + final List names = type.getFieldList().stream() + .map(RelDataTypeField::getName) + .collect(Collectors.toList()); + assertThat(names, is(ImmutableList.of("NAME"))); + }); + + // To verify that the exclude list contains all columns in the table + fixture.withSql("select ^*^ exclude(deptno, name) from dept") + .fails("SELECT \\* EXCLUDE list cannot exclude all columns"); + } + /** Tests that DATEADD, DATEDIFF, DATEPART, DATE_PART allow custom time * frames. */ @Test void testTimeFrames() { diff --git a/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java b/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java index d35f1a030e7f..6769f68283f2 100644 --- a/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java +++ b/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java @@ -813,6 +813,9 @@ ExInst illegalArgumentForTableFunctionCall(String a0, @BaseMessage("SELECT * EXCLUDE list contains unknown column(s): {0}") ExInst selectStarExcludeListContainsUnknownColumns(String columns); + @BaseMessage("SELECT * EXCLUDE list cannot exclude all columns") + ExInst selectStarExcludeCannotExcludeAllColumns(); + @BaseMessage("Group function ''{0}'' can only appear in GROUP BY clause") ExInst groupFunctionMustAppearInGroupByClause(String funcName); diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java index 172921babf17..fd737039c4ab 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java @@ -662,6 +662,7 @@ private boolean expandStar(List selectItems, Set aliases, scope.validator.catalogReader.nameMatcher(); final int originalSize = selectItems.size(); final SqlParserPos startPosition = identifier.getParserPosition(); + final int fieldsBeforeStar = fields.size(); switch (identifier.names.size()) { case 1: SqlNode from = scope.getNode().getFrom(); @@ -739,6 +740,8 @@ private boolean expandStar(List selectItems, Set aliases, new Permute(from, offset).permute(selectItems, fields); } throwIfUnknownExcludeColumns(excludeIdentifiers, excludeMatched); + throwIfExcludeEliminatesAllColumns(excludeIdentifiers, fieldsBeforeStar, + fields, identifier); return true; default: @@ -789,6 +792,8 @@ private boolean expandStar(List selectItems, Set aliases, throw newValidationError(prefixId, RESOURCE.starRequiresRecordType()); } throwIfUnknownExcludeColumns(excludeIdentifiers, excludeMatched); + throwIfExcludeEliminatesAllColumns(excludeIdentifiers, fieldsBeforeStar, + fields, identifier); return true; } } @@ -885,6 +890,16 @@ private void throwIfUnknownExcludeColumns(List excludeIdentifiers } } + private void throwIfExcludeEliminatesAllColumns(List excludeIdentifiers, + int fieldsBeforeStar, PairList fields, + SqlIdentifier identifier) { + if (!excludeIdentifiers.isEmpty() + && fields.size() == fieldsBeforeStar) { + throw newValidationError(identifier, + RESOURCE.selectStarExcludeCannotExcludeAllColumns()); + } + } + private SqlNode maybeCast(SqlNode node, RelDataType currentType, RelDataType desiredType) { return SqlTypeUtil.equalSansNullability(typeFactory, currentType, desiredType) diff --git a/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties b/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties index 4cc492795e7b..25531dd2cd5e 100644 --- a/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties +++ b/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties @@ -268,6 +268,7 @@ SelectMissingFrom=SELECT must have a FROM clause SelectStarRequiresFrom=SELECT * requires a FROM clause SelectExcludeRequiresStar=EXCLUDE clause must follow a STAR expression SelectStarExcludeListContainsUnknownColumns=SELECT * EXCLUDE list contains unknown column(s): {0} +SelectStarExcludeCannotExcludeAllColumns=SELECT * EXCLUDE list cannot exclude all columns GroupFunctionMustAppearInGroupByClause=Group function ''{0}'' can only appear in GROUP BY clause AuxiliaryWithoutMatchingGroupCall=Call to auxiliary group function ''{0}'' must have matching call to group function ''{1}'' in GROUP BY clause PivotAggMalformed=Measure expression in PIVOT must use aggregate function From 1d9d43b15f526b04c417553875166b631a462da4 Mon Sep 17 00:00:00 2001 From: Alessandro Solimando Date: Thu, 11 Dec 2025 18:12:32 +0100 Subject: [PATCH 050/562] [CALCITE-7330] AggregateCaseToFilterRule should not be applied on aggregate functions that don't skip NULL inputs Added SqlAggFunction.skipsNullInputs() method to indicate whether an aggregate function skips NULL input values. AggregateCaseToFilterRule now checks this method before applying the CASE-to-FILTER transformation, preventing incorrect optimization for aggregates where NULL inputs are semantically significant This is not a breaking change: the method defaults to true (standard SQL behavior where aggregates skip NULLs), preserving existing behavior for all built-in aggregates. Custom UDAFs that do not skip NULL inputs should override this method to return false --- .../rel/rules/AggregateCaseToFilterRule.java | 3 +- .../apache/calcite/sql/SqlAggFunction.java | 22 ++++++++++++++ .../apache/calcite/test/RelOptRulesTest.java | 29 +++++++++++++++++++ .../apache/calcite/test/RelOptRulesTest.xml | 13 +++++++++ 4 files changed, 66 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/calcite/rel/rules/AggregateCaseToFilterRule.java b/core/src/main/java/org/apache/calcite/rel/rules/AggregateCaseToFilterRule.java index 5922d0503291..7ac593adad6f 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/AggregateCaseToFilterRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/AggregateCaseToFilterRule.java @@ -230,7 +230,8 @@ && isIntLiteral(arg2, BigDecimal.ZERO)) { false, call.rexList, ImmutableList.of(), newProjects.size() - 1, null, RelCollations.EMPTY, dataType, call.getName()); } else if ((RexLiteral.isNullLiteral(arg2) // Case A1 - && call.getAggregation().allowsFilter()) + && call.getAggregation().allowsFilter() + && call.getAggregation().skipsNullInputs()) || (kind == SqlKind.SUM0 // Case A2 && isIntLiteral(arg2, BigDecimal.ZERO))) { newProjects.add(arg1); diff --git a/core/src/main/java/org/apache/calcite/sql/SqlAggFunction.java b/core/src/main/java/org/apache/calcite/sql/SqlAggFunction.java index b7f7df233eeb..59cb7522e6a8 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlAggFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlAggFunction.java @@ -212,6 +212,28 @@ public boolean allowsNullTreatment() { return false; } + /** Returns whether this aggregate function skips NULL input values. + * + *

    Standard SQL aggregate functions skip NULL input values: + * {@code SUM(x)}, {@code AVG(x)}, {@code MIN(x)}, {@code MAX(x)}, + * {@code COUNT(x)}, etc. For example, {@code SUM(x)} only sums non-NULL + * values of x. + * + *

    This property is only relevant for aggregate functions that accept + * value arguments. Functions like {@code COUNT(*)} that count rows rather + * than values are not affected by this property. + * + *

    Custom user-defined aggregate functions may treat NULL values as + * semantically significant inputs. Such functions should override this + * method to return {@code false}. + * + * @return true if NULL input values are skipped (standard SQL behavior), + * false if NULL input values have semantic significance + */ + public boolean skipsNullInputs() { + return true; + } + /** * Gets rollup aggregation function. */ diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index 8d432256b4be..ff3cb03c36e2 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -114,6 +114,7 @@ import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexUtil; +import org.apache.calcite.sql.SqlAggFunction; import org.apache.calcite.sql.SqlBasicFunction; import org.apache.calcite.sql.SqlFunction; import org.apache.calcite.sql.SqlFunctionCategory; @@ -126,6 +127,7 @@ import org.apache.calcite.sql.type.OperandTypes; import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.sql.util.SqlOperatorTables; import org.apache.calcite.sql.validate.SqlConformanceEnum; import org.apache.calcite.sql.validate.SqlMonotonicity; import org.apache.calcite.sql2rel.RelDecorrelator; @@ -140,6 +142,7 @@ import org.apache.calcite.util.DateString; import org.apache.calcite.util.Holder; import org.apache.calcite.util.ImmutableBitSet; +import org.apache.calcite.util.Optionality; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -6186,6 +6189,32 @@ public boolean test(Project project) { sql(sql).withRule(CoreRules.AGGREGATE_CASE_TO_FILTER).checkUnchanged(); } + /** Test case for + * [CALCITE-7330] + * AggregateCaseToFilterRule should not be applied on aggregate functions that + * don't skip NULL inputs. */ + @Test void testAggregateCaseToFilterWithCustomNullAwareUdaf() { + final SqlAggFunction nullAwareAgg = + new SqlAggFunction("NULL_AWARE_SUM", null, SqlKind.SUM, ReturnTypes.ARG0_NULLABLE, null, + OperandTypes.NUMERIC, SqlFunctionCategory.NUMERIC, false, false, + Optionality.FORBIDDEN) { + @Override public boolean skipsNullInputs() { + return false; // NULL values are semantically relevant + } + }; + + final String sql = "select null_aware_sum(case when deptno > 10 then sal else null end)\n" + + "from emp"; + + sql(sql) + .withFactory(t -> + t.withOperatorTable(opTab -> + SqlOperatorTables.chain(opTab, + SqlOperatorTables.of(ImmutableList.of(nullAwareAgg))))) + .withRule(CoreRules.AGGREGATE_CASE_TO_FILTER) + .checkUnchanged(); + } + @Test void testPullAggregateThroughUnion() { final String sql = "select deptno, job from" + " (select deptno, job from emp as e1" diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index 8ed236780f67..e1db15d0ce80 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -85,6 +85,19 @@ from emp]]> LogicalAggregate(group=[{}], SUM_NO_MATCH=[SUM($0)], SUM_NO_MATCH2=[SUM($1)], SUM_NO_MATCH3=[SUM($2)]) LogicalProject($f0=[CASE(=($7, -1), 1, 0)], $f1=[CASE(=($7, -1), 2, 0)], $f2=[CASE(=($7, -1), 3, -1)]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +]]> + + + + + 10 then sal else null end) +from emp]]> + + + ($7, 10), $5, null:INTEGER)]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) ]]> From b753ea7fda139a43e52edcb5dfb2f7a91ff6ada1 Mon Sep 17 00:00:00 2001 From: big face cat <731030576@qq.com> Date: Thu, 11 Dec 2025 12:33:23 +0800 Subject: [PATCH 051/562] Correct several misspellings --- cassandra/src/test/resources/cassandra.yaml | 2 +- .../java/org/apache/calcite/sql/advise/SqlSimpleParser.java | 2 +- .../main/java/org/apache/calcite/sql/dialect/Db2SqlDialect.java | 2 +- core/src/main/java/org/apache/calcite/sql/util/SqlBuilder.java | 2 +- core/src/test/resources/sql/cast-with-format.iq | 2 +- site/_docs/history.md | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cassandra/src/test/resources/cassandra.yaml b/cassandra/src/test/resources/cassandra.yaml index 123b5807dbcf..ab6058bbba06 100644 --- a/cassandra/src/test/resources/cassandra.yaml +++ b/cassandra/src/test/resources/cassandra.yaml @@ -297,7 +297,7 @@ commit_failure_policy: stop # i.e. use bind markers for variable parts. # # Do only change the default value, if you really have more prepared statements than -# fit in the cache. In most cases it is not neccessary to change this value. +# fit in the cache. In most cases it is not necessary to change this value. # Constantly re-preparing statements is a performance penalty. # # Default value ("auto") is 1/256th of the heap or 10MB, whichever is greater diff --git a/core/src/main/java/org/apache/calcite/sql/advise/SqlSimpleParser.java b/core/src/main/java/org/apache/calcite/sql/advise/SqlSimpleParser.java index 29903b312da8..86d4c88e941c 100644 --- a/core/src/main/java/org/apache/calcite/sql/advise/SqlSimpleParser.java +++ b/core/src/main/java/org/apache/calcite/sql/advise/SqlSimpleParser.java @@ -297,7 +297,7 @@ private Token parseQuotedIdentifier() { ++pos; if (c == closeQuote) { if (pos < sql.length() && sql.charAt(pos) == closeQuote) { - // Double close means escaped closing quote is a part of identifer + // Double close means escaped closing quote is a part of identifier ++pos; continue; } diff --git a/core/src/main/java/org/apache/calcite/sql/dialect/Db2SqlDialect.java b/core/src/main/java/org/apache/calcite/sql/dialect/Db2SqlDialect.java index fb335e8aa5c8..27bccd0b9399 100644 --- a/core/src/main/java/org/apache/calcite/sql/dialect/Db2SqlDialect.java +++ b/core/src/main/java/org/apache/calcite/sql/dialect/Db2SqlDialect.java @@ -81,7 +81,7 @@ public Db2SqlDialect(Context context) { // A duration is a positive or negative number representing an interval of time. // If one operand is a date, the other labeled duration of YEARS, MONTHS, or DAYS. // If one operand is a time, the other must be labeled duration of HOURS, MINUTES, or SECONDS. - // If one operand is a timestamp, the other operand can be any of teh duration. + // If one operand is a timestamp, the other operand can be any duration. SqlIntervalLiteral.IntervalValue interval = literal.getValueAs(SqlIntervalLiteral.IntervalValue.class); diff --git a/core/src/main/java/org/apache/calcite/sql/util/SqlBuilder.java b/core/src/main/java/org/apache/calcite/sql/util/SqlBuilder.java index 29aea20cd55d..023728abe0e7 100644 --- a/core/src/main/java/org/apache/calcite/sql/util/SqlBuilder.java +++ b/core/src/main/java/org/apache/calcite/sql/util/SqlBuilder.java @@ -30,7 +30,7 @@ * *

    Using this class helps to prevent SQL injection attacks, incorrectly * quoted identifiers and strings. These problems occur when you build SQL by - * concatenating strings, and you forget to treat identifers and string literals + * concatenating strings, and you forget to treat identifiers and string literals * correctly. SqlBuilder has special methods for appending identifiers and * literals. */ diff --git a/core/src/test/resources/sql/cast-with-format.iq b/core/src/test/resources/sql/cast-with-format.iq index 04588143d741..f6b8d5026d11 100644 --- a/core/src/test/resources/sql/cast-with-format.iq +++ b/core/src/test/resources/sql/cast-with-format.iq @@ -2860,7 +2860,7 @@ select cast(cast("1985-12-02" as date) as varchar format "\"free text\""); No datetime tokens provided. !error -# FX modifier not at the begining of the format. +# FX modifier not at the beginning of the format. select cast("2001-03-01 00:10:02" as timestamp format "YYYY-MM-DD FXHH12:MI:SS"); FX modifier should be at the beginning of the format string. diff --git a/site/_docs/history.md b/site/_docs/history.md index 0066dde468e2..399d0bfcec92 100644 --- a/site/_docs/history.md +++ b/site/_docs/history.md @@ -1577,7 +1577,7 @@ other software versions as specified in gradle.properties. `AssertionError`: "Conversion to relational algebra failed to preserve datatypes" when union `VARCHAR` literal and `CAST(null AS INTEGER)` * [CALCITE-6178] - `WITH RECURSIVE` query when cloned using `SqlShuttle` looses `RECURSIVE` property + `WITH RECURSIVE` query when cloned using `SqlShuttle` loses `RECURSIVE` property * [CALCITE-6332] Optimization `CoreRules.AGGREGATE_EXPAND_DISTINCT_AGGREGATES_TO_JOIN` produces incorrect results for aggregates with groupSets From 733a2d86caa80fd6b54e348150eb1601133148ae Mon Sep 17 00:00:00 2001 From: Silun Dong Date: Thu, 6 Nov 2025 00:36:58 +0800 Subject: [PATCH 052/562] [CALCITE-7031] Implement the general decorrelation algorithm (Neumann & Kemper) --- .../enumerable/EnumerableJoinRule.java | 10 +- .../org/apache/calcite/plan/RelOptUtil.java | 25 +- .../rel/core/ConditionalCorrelate.java | 76 ++ .../apache/calcite/rel/core/Correlate.java | 7 + .../org/apache/calcite/rel/core/Join.java | 17 +- .../apache/calcite/rel/core/JoinRelType.java | 32 +- .../apache/calcite/rel/core/RelFactories.java | 49 + .../logical/LogicalConditionalCorrelate.java | 90 ++ .../calcite/rel/metadata/RelMdSize.java | 18 +- .../calcite/rel/metadata/RelMdUtil.java | 3 + .../apache/calcite/rel/rules/CoreRules.java | 14 + .../rel/rules/MarkToSemiOrAntiJoinRule.java | 143 +++ .../calcite/rel/rules/SubQueryRemoveRule.java | 134 ++ .../sql/validate/SqlValidatorUtil.java | 11 + .../sql2rel/TopDownGeneralDecorrelator.java | 1125 +++++++++++++++++ .../org/apache/calcite/tools/RelBuilder.java | 14 +- .../apache/calcite/test/RelOptRulesTest.java | 263 ++++ .../apache/calcite/test/RelOptRulesTest.xml | 728 +++++++++++ .../apache/calcite/test/RelOptFixture.java | 41 +- 19 files changed, 2767 insertions(+), 33 deletions(-) create mode 100644 core/src/main/java/org/apache/calcite/rel/core/ConditionalCorrelate.java create mode 100644 core/src/main/java/org/apache/calcite/rel/logical/LogicalConditionalCorrelate.java create mode 100644 core/src/main/java/org/apache/calcite/rel/rules/MarkToSemiOrAntiJoinRule.java create mode 100644 core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableJoinRule.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableJoinRule.java index 57a6778e0d09..fe40d17dc451 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableJoinRule.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableJoinRule.java @@ -21,10 +21,14 @@ import org.apache.calcite.rel.convert.ConverterRule; import org.apache.calcite.rel.core.Join; import org.apache.calcite.rel.core.JoinInfo; +import org.apache.calcite.rel.core.JoinRelType; import org.apache.calcite.rel.logical.LogicalJoin; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexUtil; +import org.apache.calcite.util.Bug; + +import org.checkerframework.checker.nullness.qual.Nullable; import java.util.ArrayList; import java.util.Arrays; @@ -48,8 +52,12 @@ protected EnumerableJoinRule(Config config) { super(config); } - @Override public RelNode convert(RelNode rel) { + @Override public @Nullable RelNode convert(RelNode rel) { Join join = (Join) rel; + if (!Bug.TODO_FIXED && join.getJoinType() == JoinRelType.LEFT_MARK) { + // TODO implement LEFT MARK join + return null; + } List newInputs = new ArrayList<>(); for (RelNode input : join.getInputs()) { if (!(input.getConvention() instanceof EnumerableConvention)) { diff --git a/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java b/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java index 6698ec9b0d86..34724a8adfbc 100644 --- a/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java +++ b/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java @@ -3960,12 +3960,25 @@ public static RelNode pushDownJoinConditions(Join originalJoin, joinCond, left, right, joinType, originalJoin.isSemiJoinDone())); } if (!extraLeftExprs.isEmpty() || !extraRightExprs.isEmpty()) { - final int totalFields = joinType.projectsRight() - ? leftCount + extraLeftExprs.size() + rightCount + extraRightExprs.size() - : leftCount + extraLeftExprs.size(); - final int[] mappingRanges = joinType.projectsRight() - ? new int[] { 0, 0, leftCount, leftCount, leftCount + extraLeftExprs.size(), rightCount } - : new int[] { 0, 0, leftCount }; + final int totalFields; + final int[] mappingRanges; + switch (joinType) { + case SEMI: + case ANTI: + totalFields = leftCount + extraLeftExprs.size(); + mappingRanges = new int[] { 0, 0, leftCount }; + break; + case LEFT_MARK: + totalFields = leftCount + extraLeftExprs.size() + 1; + mappingRanges + = new int[] { 0, 0, leftCount, leftCount, leftCount + extraLeftExprs.size(), 1 }; + break; + default: + totalFields = leftCount + extraLeftExprs.size() + rightCount + extraRightExprs.size(); + mappingRanges = + new int[] { 0, 0, leftCount, leftCount, leftCount + extraLeftExprs.size(), rightCount }; + break; + } Mappings.TargetMapping mapping = Mappings.createShiftMapping( totalFields, diff --git a/core/src/main/java/org/apache/calcite/rel/core/ConditionalCorrelate.java b/core/src/main/java/org/apache/calcite/rel/core/ConditionalCorrelate.java new file mode 100644 index 000000000000..a6527f23fdc1 --- /dev/null +++ b/core/src/main/java/org/apache/calcite/rel/core/ConditionalCorrelate.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.rel.core; + +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelTraitSet; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.RelWriter; +import org.apache.calcite.rel.hint.RelHint; +import org.apache.calcite.rel.rules.CoreRules; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.util.ImmutableBitSet; + +import java.util.List; + +/** + * This is a extension of {@link Correlate} that contains a condition. + * When removing SOME/IN subqueries, the condition need to be retained in the left mark type + * Correlate (it cannot be pulled up or pushed down). This is why ConditionalCorrelate extends + * the condition. + * + * @see CoreRules#FILTER_SUB_QUERY_TO_MARK_CORRELATE + * @see CoreRules#PROJECT_SUB_QUERY_TO_MARK_CORRELATE + */ +public abstract class ConditionalCorrelate extends Correlate { + + private final RexNode condition; + + protected ConditionalCorrelate( + RelOptCluster cluster, + RelTraitSet traitSet, + List hints, + RelNode left, + RelNode right, + CorrelationId correlationId, + ImmutableBitSet requiredColumns, + JoinRelType joinType, + RexNode condition) { + super(cluster, traitSet, hints, left, right, correlationId, requiredColumns, joinType); + this.condition = condition; + assert joinType == JoinRelType.LEFT_MARK; + } + + @Override public ConditionalCorrelate copy(RelTraitSet traitSet, List inputs) { + assert inputs.size() == 2; + return copy(traitSet, inputs.get(0), inputs.get(1), correlationId, + requiredColumns, joinType, condition); + } + + public abstract ConditionalCorrelate copy(RelTraitSet traitSet, RelNode left, RelNode right, + CorrelationId correlationId, ImmutableBitSet requiredColumns, JoinRelType joinType, + RexNode condition); + + @Override public RelWriter explainTerms(RelWriter pw) { + return super.explainTerms(pw) + .itemIf("condition", condition, !condition.isAlwaysTrue()); + } + + @Override public RexNode getCondition() { + return condition; + } +} diff --git a/core/src/main/java/org/apache/calcite/rel/core/Correlate.java b/core/src/main/java/org/apache/calcite/rel/core/Correlate.java index 752dd9bf15a2..e9d2adbccdd4 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/Correlate.java +++ b/core/src/main/java/org/apache/calcite/rel/core/Correlate.java @@ -29,6 +29,7 @@ import org.apache.calcite.rel.hint.RelHint; import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rex.RexNode; import org.apache.calcite.sql.validate.SqlValidatorUtil; import org.apache.calcite.util.ImmutableBitSet; import org.apache.calcite.util.Litmus; @@ -169,6 +170,7 @@ public JoinRelType getJoinType() { switch (joinType) { case LEFT: case INNER: + case LEFT_MARK: return SqlValidatorUtil.deriveJoinRowType(left.getRowType(), right.getRowType(), joinType, getCluster().getTypeFactory(), null, @@ -211,6 +213,10 @@ public ImmutableBitSet getRequiredColumns() { return requiredColumns; } + public RexNode getCondition() { + return getCluster().getRexBuilder().makeLiteral(true); + } + @Override public Set getVariablesSet() { return ImmutableSet.of(correlationId); } @@ -220,6 +226,7 @@ public ImmutableBitSet getRequiredColumns() { switch (joinType) { case SEMI: case ANTI: + case LEFT_MARK: return leftRowCount; default: return leftRowCount * mq.getRowCount(right); diff --git a/core/src/main/java/org/apache/calcite/rel/core/Join.java b/core/src/main/java/org/apache/calcite/rel/core/Join.java index 8016e9f36a6c..1a56edbed022 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/Join.java +++ b/core/src/main/java/org/apache/calcite/rel/core/Join.java @@ -152,10 +152,19 @@ public JoinRelType getJoinType() { if (!super.isValid(litmus, context)) { return false; } - if (getRowType().getFieldCount() - != getSystemFieldList().size() - + left.getRowType().getFieldCount() - + (joinType.projectsRight() ? right.getRowType().getFieldCount() : 0)) { + int expectedFieldCount = left.getRowType().getFieldCount(); + switch (joinType) { + case SEMI: + case ANTI: + break; + case LEFT_MARK: + expectedFieldCount += 1; + break; + default: + expectedFieldCount += right.getRowType().getFieldCount(); + break; + } + if (getRowType().getFieldCount() != expectedFieldCount) { return litmus.fail("field count mismatch"); } if (condition != null) { diff --git a/core/src/main/java/org/apache/calcite/rel/core/JoinRelType.java b/core/src/main/java/org/apache/calcite/rel/core/JoinRelType.java index dbb2891ce2e8..ffe4a3d10372 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/JoinRelType.java +++ b/core/src/main/java/org/apache/calcite/rel/core/JoinRelType.java @@ -91,7 +91,32 @@ public enum JoinRelType { /** * The left version of an ASOF join, where each row from the left table is part of the output. */ - LEFT_ASOF; + LEFT_ASOF, + + /** + * An LEFT MARK JOIN will keep all rows from the left side and creates a new attribute to mark a + * tuple as having join partners from right side or not. Refer to + * + * The Complete Story of Joins (in HyPer). + * + *

    Example: + * 

    +   * SELECT EMPNO FROM EMP
+   * WHERE EXISTS (SELECT 1 FROM DEPT
+   *     WHERE DEPT.DEPTNO = EMP.DEPTNO)
+   *     OR EMPNO > 1
+   *
+   * LogicalProject(EMPNO=[$0])
+   *   LogicalFilter(condition=[OR($9, >($0, 1))])
+   *     LogicalJoin(condition=[IS NOT DISTINCT FROM($7, $9)], joinType=[left_mark])
+   *       LogicalTableScan(table=[[CATALOG, SALES, EMP]])
+   *       LogicalTableScan(table=[[CATALOG, SALES, DEPT]])
+   *
    + * + *

    If the marker is used on only conjunctive predicates the optimizer will try to translate + * the mark join into semi or anti join. + */ + LEFT_MARK; /** Lower-case name. */ public final String lowerName = name().toLowerCase(Locale.ROOT); @@ -173,7 +198,7 @@ public JoinRelType cancelNullsOnRight() { } public boolean projectsRight() { - return this != SEMI && this != ANTI; + return this != SEMI && this != ANTI && this != LEFT_MARK; } /** Returns whether this join type accepts pushing predicates from above into its predicate. */ @@ -185,7 +210,8 @@ public boolean canPushIntoFromAbove() { /** Returns whether this join type accepts pushing predicates from above into its left input. */ @API(since = "1.28", status = API.Status.EXPERIMENTAL) public boolean canPushLeftFromAbove() { - return (this == INNER) || (this == LEFT) || (this == SEMI) || (this == ANTI); + return (this == INNER) || (this == LEFT) || (this == SEMI) + || (this == ANTI) || (this == LEFT_MARK); } /** Returns whether this join type accepts pushing predicates from above into its right input. */ diff --git a/core/src/main/java/org/apache/calcite/rel/core/RelFactories.java b/core/src/main/java/org/apache/calcite/rel/core/RelFactories.java index 9844219ecb31..e2a096762f77 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/RelFactories.java +++ b/core/src/main/java/org/apache/calcite/rel/core/RelFactories.java @@ -29,6 +29,7 @@ import org.apache.calcite.rel.hint.RelHint; import org.apache.calcite.rel.logical.LogicalAggregate; import org.apache.calcite.rel.logical.LogicalAsofJoin; +import org.apache.calcite.rel.logical.LogicalConditionalCorrelate; import org.apache.calcite.rel.logical.LogicalCorrelate; import org.apache.calcite.rel.logical.LogicalExchange; import org.apache.calcite.rel.logical.LogicalFilter; @@ -91,6 +92,9 @@ public class RelFactories { public static final CorrelateFactory DEFAULT_CORRELATE_FACTORY = new CorrelateFactoryImpl(); + public static final ConditionalCorrelateFactory DEFAULT_CORRELATE_PLUS_FACTORY = + new ConditionalCorrelateFactoryImpl(); + public static final SortFactory DEFAULT_SORT_FACTORY = new SortFactoryImpl(); @@ -144,6 +148,7 @@ public class RelFactories { DEFAULT_JOIN_FACTORY, DEFAULT_ASOFJOIN_FACTORY, DEFAULT_CORRELATE_FACTORY, + DEFAULT_CORRELATE_PLUS_FACTORY, DEFAULT_VALUES_FACTORY, DEFAULT_TABLE_SCAN_FACTORY, DEFAULT_TABLE_FUNCTION_SCAN_FACTORY, @@ -485,6 +490,44 @@ private static class CorrelateFactoryImpl implements CorrelateFactory { } } + /** + * Can create a ConditionalCorrelate of the appropriate type for a rule's calling + * convention. + * + *

    The result is typically a {@link ConditionalCorrelate}. + */ + public interface ConditionalCorrelateFactory { + + /** + * Creates a ConditionalCorrelate. + * + * @param left Left input + * @param right Right input + * @param hints Hints + * @param correlationId Variable name for the row of left input + * @param requiredColumns Required columns + * @param joinType Join type + * @param condition Join condition + */ + RelNode createConditionalCorrelate(RelNode left, RelNode right, List hints, + CorrelationId correlationId, ImmutableBitSet requiredColumns, + JoinRelType joinType, RexNode condition); + } + + /** + * Implementation of {@link ConditionalCorrelateFactory} that returns a vanilla + * {@link LogicalConditionalCorrelate}. + */ + private static class ConditionalCorrelateFactoryImpl implements ConditionalCorrelateFactory { + + @Override public RelNode createConditionalCorrelate(RelNode left, RelNode right, + List hints, CorrelationId correlationId, ImmutableBitSet requiredColumns, + JoinRelType joinType, RexNode condition) { + return LogicalConditionalCorrelate.create(left, right, hints, correlationId, + requiredColumns, joinType, condition); + } + } + /** * Can create a semi-join of the appropriate type for a rule's calling * convention. @@ -739,6 +782,7 @@ public static class Struct { public final JoinFactory joinFactory; public final AsofJoinFactory asofJoinFactory; public final CorrelateFactory correlateFactory; + public final ConditionalCorrelateFactory conditionalCorrelateFactory; public final ValuesFactory valuesFactory; public final TableScanFactory scanFactory; public final TableFunctionScanFactory tableFunctionScanFactory; @@ -759,6 +803,7 @@ private Struct(FilterFactory filterFactory, JoinFactory joinFactory, AsofJoinFactory asofJoinFactory, CorrelateFactory correlateFactory, + ConditionalCorrelateFactory conditionalCorrelateFactory, ValuesFactory valuesFactory, TableScanFactory scanFactory, TableFunctionScanFactory tableFunctionScanFactory, @@ -778,6 +823,8 @@ private Struct(FilterFactory filterFactory, this.joinFactory = requireNonNull(joinFactory, "joinFactory"); this.asofJoinFactory = requireNonNull(asofJoinFactory, "asofJoinFactory"); this.correlateFactory = requireNonNull(correlateFactory, "correlateFactory"); + this.conditionalCorrelateFactory = + requireNonNull(conditionalCorrelateFactory, "conditionalCorrelateFactory"); this.valuesFactory = requireNonNull(valuesFactory, "valuesFactory"); this.scanFactory = requireNonNull(scanFactory, "scanFactory"); this.tableFunctionScanFactory = @@ -816,6 +863,8 @@ public static Struct fromContext(Context context) { .orElse(DEFAULT_ASOFJOIN_FACTORY), context.maybeUnwrap(CorrelateFactory.class) .orElse(DEFAULT_CORRELATE_FACTORY), + context.maybeUnwrap(ConditionalCorrelateFactory.class) + .orElse(DEFAULT_CORRELATE_PLUS_FACTORY), context.maybeUnwrap(ValuesFactory.class) .orElse(DEFAULT_VALUES_FACTORY), context.maybeUnwrap(TableScanFactory.class) diff --git a/core/src/main/java/org/apache/calcite/rel/logical/LogicalConditionalCorrelate.java b/core/src/main/java/org/apache/calcite/rel/logical/LogicalConditionalCorrelate.java new file mode 100644 index 000000000000..2df2f839c5f3 --- /dev/null +++ b/core/src/main/java/org/apache/calcite/rel/logical/LogicalConditionalCorrelate.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.rel.logical; + +import org.apache.calcite.plan.Convention; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelTraitSet; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.ConditionalCorrelate; +import org.apache.calcite.rel.core.Correlate; +import org.apache.calcite.rel.core.CorrelationId; +import org.apache.calcite.rel.core.JoinRelType; +import org.apache.calcite.rel.hint.RelHint; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.util.ImmutableBitSet; + +import java.util.List; + +/** + * Sub-class of {@link ConditionalCorrelate} not targeted at any particular engine or calling convention. + */ +public final class LogicalConditionalCorrelate extends ConditionalCorrelate { + //~ Instance fields -------------------------------------------------------- + + //~ Constructors ----------------------------------------------------------- + + /** + * Creates a LogicalConditionalCorrelate. + * + * @param cluster Cluster this relational expression belongs to + * @param left Left input relational expression + * @param right Right input relational expression + * @param correlationId Variable name for the row of left input + * @param requiredColumns Required columns + * @param joinType Join type + * @param condition Join condition + */ + public LogicalConditionalCorrelate( + RelOptCluster cluster, + RelTraitSet traitSet, + List hints, + RelNode left, + RelNode right, + CorrelationId correlationId, + ImmutableBitSet requiredColumns, + JoinRelType joinType, + RexNode condition) { + super(cluster, traitSet, hints, left, right, correlationId, + requiredColumns, joinType, condition); + } + + /** Creates a LogicalConditionalCorrelate. */ + public static LogicalConditionalCorrelate create(RelNode left, RelNode right, List hints, + CorrelationId correlationId, ImmutableBitSet requiredColumns, JoinRelType joinType, + RexNode condition) { + final RelOptCluster cluster = left.getCluster(); + final RelTraitSet traitSet = cluster.traitSetOf(Convention.NONE); + return new LogicalConditionalCorrelate(cluster, traitSet, hints, left, right, correlationId, + requiredColumns, joinType, condition); + } + + @Override public ConditionalCorrelate copy(RelTraitSet traitSet, RelNode left, RelNode right, + CorrelationId correlationId, ImmutableBitSet requiredColumns, JoinRelType joinType, + RexNode condition) { + assert traitSet.containsIfApplicable(Convention.NONE); + return new LogicalConditionalCorrelate(getCluster(), traitSet, hints, left, right, + correlationId, requiredColumns, joinType, condition); + } + + @Override public Correlate copy(RelTraitSet traitSet, + RelNode left, RelNode right, CorrelationId correlationId, + ImmutableBitSet requiredColumns, JoinRelType joinType) { + // This method does not provide the condition as an argument, so it should never be called + throw new RuntimeException("This method should not be called"); + } +} diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdSize.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdSize.java index b6a687e91d68..675aa1e3639e 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdSize.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdSize.java @@ -25,6 +25,7 @@ import org.apache.calcite.rel.core.Filter; import org.apache.calcite.rel.core.Intersect; import org.apache.calcite.rel.core.Join; +import org.apache.calcite.rel.core.JoinRelType; import org.apache.calcite.rel.core.Minus; import org.apache.calcite.rel.core.Project; import org.apache.calcite.rel.core.Sort; @@ -45,6 +46,7 @@ import org.apache.calcite.util.Pair; import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; import org.checkerframework.checker.nullness.qual.Nullable; @@ -206,14 +208,24 @@ protected RelMdSize() {} return averageJoinColumnSizes(rel, mq); } - private static @Nullable List<@Nullable Double> averageJoinColumnSizes(Join rel, + private @Nullable List<@Nullable Double> averageJoinColumnSizes(Join rel, RelMetadataQuery mq) { boolean semiOrAntijoin = !rel.getJoinType().projectsRight(); final RelNode left = rel.getLeft(); final RelNode right = rel.getRight(); final @Nullable List<@Nullable Double> lefts = mq.getAverageColumnSizes(left); - final @Nullable List<@Nullable Double> rights = - semiOrAntijoin ? null : mq.getAverageColumnSizes(right); + final @Nullable List<@Nullable Double> rights; + if (semiOrAntijoin) { + if (rel.getJoinType() == JoinRelType.LEFT_MARK) { + RelDataTypeField markColType = + rel.getRowType().getFieldList().get(rel.getRowType().getFieldCount() - 1); + rights = Lists.newArrayList(averageFieldValueSize(markColType)); + } else { + rights = null; + } + } else { + rights = mq.getAverageColumnSizes(right); + } if (lefts == null && rights == null) { return null; } diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdUtil.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdUtil.java index dc12983f3f14..227e19a85209 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdUtil.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdUtil.java @@ -813,6 +813,9 @@ public static double getMinusRowCount(RelMetadataQuery mq, Minus minus) { public static @Nullable Double getJoinRowCount(RelMetadataQuery mq, Join join, RexNode condition) { if (!join.getJoinType().projectsRight()) { + if (join.getJoinType() == JoinRelType.LEFT_MARK) { + return mq.getRowCount(join.getLeft()); + } // Create a RexNode representing the selectivity of the // semijoin filter and pass it to getSelectivity RexNode semiJoinSelectivity = diff --git a/core/src/main/java/org/apache/calcite/rel/rules/CoreRules.java b/core/src/main/java/org/apache/calcite/rel/rules/CoreRules.java index 938df4685b67..c604f713580f 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/CoreRules.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/CoreRules.java @@ -498,6 +498,20 @@ private CoreRules() {} public static final SubQueryRemoveRule JOIN_SUB_QUERY_TO_CORRELATE = SubQueryRemoveRule.Config.JOIN.toRule(); + /** Rule that converts sub-queries from filter expressions into + * {@link Correlate} instances. It will rewrite SOME/EXISTS/IN to a LEFT MARK type Correlate. */ + public static final SubQueryRemoveRule FILTER_SUB_QUERY_TO_MARK_CORRELATE = + SubQueryRemoveRule.Config.FILTER_ENABLE_MARK_JOIN.toRule(); + + /** Rule that converts sub-queries from project expressions into + * {@link Correlate} instances. It will rewrite SOME/EXISTS/IN to a LEFT MARK type Correlate. */ + public static final SubQueryRemoveRule PROJECT_SUB_QUERY_TO_MARK_CORRELATE = + SubQueryRemoveRule.Config.PROJECT_ENABLE_MARK_JOIN.toRule(); + + /** Rule that converts mark join to semi/anti join. */ + public static final MarkToSemiOrAntiJoinRule MARK_TO_SEMI_OR_ANTI_JOIN_RULE = + MarkToSemiOrAntiJoinRule.Config.DEFAULT.toRule(); + /** Rule that converts SUM to SUM0 in OVER clauses in a project list. */ public static final ProjectOverSumToSum0Rule PROJECT_OVER_SUM_TO_SUM0_RULE = ProjectOverSumToSum0Rule.Config.DEFAULT.toRule(); diff --git a/core/src/main/java/org/apache/calcite/rel/rules/MarkToSemiOrAntiJoinRule.java b/core/src/main/java/org/apache/calcite/rel/rules/MarkToSemiOrAntiJoinRule.java new file mode 100644 index 000000000000..9c4230db3875 --- /dev/null +++ b/core/src/main/java/org/apache/calcite/rel/rules/MarkToSemiOrAntiJoinRule.java @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.rel.rules; + +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptUtil; +import org.apache.calcite.plan.RelRule; +import org.apache.calcite.plan.Strong; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.Filter; +import org.apache.calcite.rel.core.Join; +import org.apache.calcite.rel.core.JoinRelType; +import org.apache.calcite.rel.core.Project; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.tools.RelBuilder; +import org.apache.calcite.util.ImmutableBitSet; + +import org.immutables.value.Value; + +import java.util.ArrayList; +import java.util.List; + +import static org.apache.calcite.plan.RelOptUtil.conjunctions; + +/** + * Rule to simplify a mark join to semi join or anti join. This rule is applies by default after + * general decorrelation. + * + * @see org.apache.calcite.sql2rel.TopDownGeneralDecorrelator + */ +@Value.Enclosing +public class MarkToSemiOrAntiJoinRule + extends RelRule + implements TransformationRule { + + + /** Creates a MarkToSemiOrAntiJoinRule. */ + protected MarkToSemiOrAntiJoinRule(Config config) { + super(config); + } + + @Override public void onMatch(RelOptRuleCall call) { + final Project project = call.rel(0); + final Filter filter = call.rel(1); + final Join join = call.rel(2); + final RelBuilder builder = call.builder(); + + int markIndex = join.getRowType().getFieldCount() - 1; + ImmutableBitSet projectColumns = RelOptUtil.InputFinder.bits(project.getProjects(), null); + ImmutableBitSet filterColumns = RelOptUtil.InputFinder.bits(filter.getCondition()); + if (projectColumns.get(markIndex) || !filterColumns.get(markIndex)) { + return; + } + + // Proj <- no result of the project depends on marker + // Filter <- condition depends on marker + // Join <- mark join + // After expressing the filter condition as a conjunction, there are only two cases to simplify: + // 1. only reference the marker, simplify to semi join + // 2. NOT(marker), and the join condition will only return TRUE/FALSE + // (will not return NULL values), simplify to anti join + boolean toSemi = false; + boolean toAnti = false; + List filterConditions = RelOptUtil.conjunctions(filter.getCondition()); + List newFilterConditions = new ArrayList<>(); + for (RexNode condition : filterConditions) { + final ImmutableBitSet inputBits = RelOptUtil.InputFinder.bits(condition); + // marker is not referenced + if (!inputBits.get(markIndex)) { + newFilterConditions.add(condition); + continue; + } + + // only reference the marker, to semi join + if (condition instanceof RexInputRef && !toAnti) { + toSemi = true; + continue; + } + // NOT(marker), and the join condition will only return TRUE/FALSE, to anti join + if (condition instanceof RexCall + && condition.isA(SqlKind.NOT) + && ((RexCall) condition).getOperands().get(0) instanceof RexInputRef + && isJoinConditionNotStrong(join.getCondition()) + && !toSemi) { + toAnti = true; + continue; + } + // other forms cannot be simplified, for example, disjunction + return; + } + JoinRelType newJoinType = toSemi ? JoinRelType.SEMI : JoinRelType.ANTI; + RelNode result + = builder.push(join.getLeft()).push(join.getRight()) + .join(newJoinType, join.getCondition()) + .filter(newFilterConditions) + .project(project.getProjects()) + .build(); + call.transformTo(result); + } + + private static boolean isJoinConditionNotStrong(RexNode condition) { + List conjunctions = conjunctions(condition); + for (RexNode expr : conjunctions) { + if (Strong.isStrong(expr)) { + return false; + } + } + return true; + } + + /** Rule configuration. */ + @Value.Immutable + public interface Config extends RelRule.Config { + Config DEFAULT = ImmutableMarkToSemiOrAntiJoinRule.Config.of() + .withOperandSupplier(b1 -> + b1.operand(Project.class).oneInput(b2 -> + b2.operand(Filter.class).oneInput(b3 -> + b3.operand(Join.class).predicate(join -> + join.getJoinType() == JoinRelType.LEFT_MARK).anyInputs()))); + + @Override default MarkToSemiOrAntiJoinRule toRule() { + return new MarkToSemiOrAntiJoinRule(this); + } + } + +} diff --git a/core/src/main/java/org/apache/calcite/rel/rules/SubQueryRemoveRule.java b/core/src/main/java/org/apache/calcite/rel/rules/SubQueryRemoveRule.java index 8ed8c287b47a..37e92aa23aef 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/SubQueryRemoveRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/SubQueryRemoveRule.java @@ -41,6 +41,7 @@ import org.apache.calcite.rex.RexUtil; import org.apache.calcite.sql.SqlAggFunction; import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.fun.SqlQuantifyOperator; import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql2rel.RelDecorrelator; @@ -1078,6 +1079,123 @@ private static void matchJoin(SubQueryRemoveRule rule, RelOptRuleCall call) { call.transformTo(builder.build()); } + private static void matchFilterEnableMarkJoin(SubQueryRemoveRule rule, RelOptRuleCall call) { + final Filter filter = call.rel(0); + final Set variablesSet = filter.getVariablesSet(); + final RelBuilder builder = call.builder(); + builder.push(filter.getInput()); + List newCondition + = rule.applyEnableMarkJoin(variablesSet, ImmutableList.of(filter.getCondition()), builder); + assert newCondition.size() == 1; + builder.filter(newCondition.get(0)); + builder.project(fields(builder, filter.getRowType().getFieldCount())); + call.transformTo(builder.build()); + } + + private static void matchProjectEnableMarkJoin(SubQueryRemoveRule rule, RelOptRuleCall call) { + final Project project = call.rel(0); + final Set variablesSet = project.getVariablesSet(); + final RelBuilder builder = call.builder(); + builder.push(project.getInput()); + List newProjects + = rule.applyEnableMarkJoin(variablesSet, project.getProjects(), builder); + builder.project(newProjects, project.getRowType().getFieldNames()); + call.transformTo(builder.build()); + } + + private List applyEnableMarkJoin(Set variablesSetOfRelNode, + List expressions, RelBuilder builder) { + List newExpressions = new ArrayList<>(expressions); + int count = 0; + while (true) { + final RexSubQuery e = RexUtil.SubQueryFinder.find(newExpressions); + if (e == null) { + assert count > 0; + break; + } + ++count; + final Set variablesSet = RelOptUtil.getVariablesUsed(e.rel); + // Only keep the correlation that are defined in the current RelNode level, to avoid creating + // wrong Correlate node. + variablesSet.retainAll(variablesSetOfRelNode); + + RexNode target; + // rewrite EXISTS/IN/SOME to left mark join/correlate + switch (e.getKind()) { + case EXISTS: + case IN: + case SOME: + target = + rewriteToMarkJoin(e, variablesSet, builder, + builder.peek().getRowType().getFieldCount()); + break; + case SCALAR_QUERY: + target = + rewriteScalarQuery(e, variablesSet, builder, 1, + builder.peek().getRowType().getFieldCount()); + break; + case ARRAY_QUERY_CONSTRUCTOR: + case MAP_QUERY_CONSTRUCTOR: + case MULTISET_QUERY_CONSTRUCTOR: + target = + rewriteCollection(e, variablesSet, builder, 1, + builder.peek().getRowType().getFieldCount()); + break; + case UNIQUE: + target = rewriteUnique(e, builder); + break; + default: + throw new AssertionError(e.getKind()); + } + final RexShuttle shuttle = new ReplaceSubQueryShuttle(e, target); + newExpressions = shuttle.apply(newExpressions); + } + return newExpressions; + } + + /** + * Rewrites a IN/SOME/EXISTS RexSubQuery into a {@link Join} of LEFT MARK type. + * + * @param e IN/SOME/EXISTS Sub-query to rewrite + * @param variablesSet A set of variables used by a relational + * expression of the specified RexSubQuery + * @param builder Builder + * @param offset Offset to shift {@link RexInputRef} + * @return Expression that may be used to replace the RexSubQuery + */ + private static RexNode rewriteToMarkJoin(RexSubQuery e, Set variablesSet, + RelBuilder builder, int offset) { + builder.push(e.rel); + final List rightShiftRef = RexUtil.shift(builder.fields(), offset); + final List externalPredicate = new ArrayList<>(); + final SqlOperator externalOperator; + switch (e.getKind()) { + case SOME: + SqlQuantifyOperator op = (SqlQuantifyOperator) e.op; + externalOperator = RelOptUtil.op(op.comparisonKind, SqlStdOperatorTable.EQUALS); + break; + case IN: + externalOperator = SqlStdOperatorTable.EQUALS; + break; + case EXISTS: + externalOperator = SqlStdOperatorTable.EQUALS; + assert e.getOperands().isEmpty(); + break; + default: + throw new IllegalArgumentException("Only IN/SOME/EXISTS sub-query can be rewritten to " + + "left mark join, but got: " + e.getKind()); + } + Pair.zip(e.getOperands(), rightShiftRef, false).stream() + .map(pair -> builder.call(externalOperator, pair.left, pair.right)) + .forEach(externalPredicate::add); + + builder.join( + JoinRelType.LEFT_MARK, + RexUtil.composeConjunction(builder.getRexBuilder(), externalPredicate), + variablesSet); + return last(builder.fields()); + } + /** Shuttle that replaces occurrences of a given * {@link org.apache.calcite.rex.RexSubQuery} with a replacement * expression. */ @@ -1122,6 +1240,22 @@ public interface Config extends RelRule.Config { .anyInputs()) .withDescription("SubQueryRemoveRule:Join"); + Config PROJECT_ENABLE_MARK_JOIN = ImmutableSubQueryRemoveRule.Config.builder() + .withMatchHandler(SubQueryRemoveRule::matchProjectEnableMarkJoin) + .build() + .withOperandSupplier(b -> + b.operand(Project.class) + .predicate(RexUtil.SubQueryFinder::containsSubQuery).anyInputs()) + .withDescription("SubQueryRemoveRule:ProjectEnableMarkJoin"); + + Config FILTER_ENABLE_MARK_JOIN = ImmutableSubQueryRemoveRule.Config.builder() + .withMatchHandler(SubQueryRemoveRule::matchFilterEnableMarkJoin) + .build() + .withOperandSupplier(b -> + b.operand(Filter.class) + .predicate(RexUtil.SubQueryFinder::containsSubQuery).anyInputs()) + .withDescription("SubQueryRemoveRule:FilterEnableMarkJoin"); + @Override default SubQueryRemoveRule toRule() { return new SubQueryRemoveRule(this); } diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorUtil.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorUtil.java index ebfc011b3ef0..a01367057a32 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorUtil.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorUtil.java @@ -57,6 +57,7 @@ import org.apache.calcite.sql.SqlUtil; import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.parser.SqlParserPos; +import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.sql.type.SqlTypeUtil; import org.apache.calcite.util.ImmutableBitSet; import org.apache.calcite.util.Litmus; @@ -67,6 +68,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; +import com.google.common.collect.Sets; import org.checkerframework.checker.nullness.qual.Nullable; @@ -555,6 +557,15 @@ public static RelDataType deriveJoinRowType( case ANTI: rightType = null; break; + case LEFT_MARK: + final String markColName = + SqlValidatorUtil.uniquify("markCol", Sets.newHashSet(leftType.getFieldNames()), + SqlValidatorUtil.EXPR_SUGGESTER); + rightType = + typeFactory.createStructType( + ImmutableList.of(typeFactory.createSqlType(SqlTypeName.BOOLEAN)), + ImmutableList.of(markColName)); + break; default: break; } diff --git a/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java b/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java new file mode 100644 index 000000000000..cea5e5d5a461 --- /dev/null +++ b/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java @@ -0,0 +1,1125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.sql2rel; + +import org.apache.calcite.linq4j.function.Experimental; +import org.apache.calcite.plan.RelOptUtil; +import org.apache.calcite.plan.Strong; +import org.apache.calcite.plan.hep.HepPlanner; +import org.apache.calcite.plan.hep.HepProgram; +import org.apache.calcite.rel.RelCollation; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.Aggregate; +import org.apache.calcite.rel.core.AggregateCall; +import org.apache.calcite.rel.core.Correlate; +import org.apache.calcite.rel.core.CorrelationId; +import org.apache.calcite.rel.core.Filter; +import org.apache.calcite.rel.core.Join; +import org.apache.calcite.rel.core.JoinRelType; +import org.apache.calcite.rel.core.Project; +import org.apache.calcite.rel.core.SetOp; +import org.apache.calcite.rel.core.Sort; +import org.apache.calcite.rel.rules.CoreRules; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexCorrelVariable; +import org.apache.calcite.rex.RexFieldAccess; +import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexShuttle; +import org.apache.calcite.rex.RexUtil; +import org.apache.calcite.rex.RexWindow; +import org.apache.calcite.sql.SqlAggFunction; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.fun.SqlCountAggFunction; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql2rel.RelDecorrelator.CorDef; +import org.apache.calcite.sql2rel.RelDecorrelator.Frame; +import org.apache.calcite.tools.RelBuilder; +import org.apache.calcite.util.ImmutableBitSet; +import org.apache.calcite.util.Litmus; +import org.apache.calcite.util.Pair; +import org.apache.calcite.util.ReflectUtil; +import org.apache.calcite.util.ReflectiveVisitor; +import org.apache.calcite.util.mapping.Mappings; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; + +import org.checkerframework.checker.nullness.qual.Nullable; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.NavigableMap; +import java.util.NavigableSet; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.stream.IntStream; + +import static java.util.Objects.requireNonNull; + +/** + * A top‑down, generic decorrelation algorithm that can handle deep nestings of correlated + * subqueries and that generalizes to complex query constructs. More details are in paper: + * + * Improving Unnesting of Complex Queries. It's an improved version of the paper: + * + * Unnesting Arbitrary Queries. + * + *

    Usage notes for TopDownGeneralDecorrelator: + * + *

    TopDownGeneralDecorrelator is not yet integrated into other modules and needs to be called + * separately. If you want to use it to replace {@link RelDecorrelator}, we recommend: + * + *

      + *
    1. When generating the initial plan by {@link SqlToRelConverter}, do not remove subqueries + * and do not enable decorrelation.
      2. + *
    2. Build a {@link HepPlanner} and apply rules for removing subqueries to the initial + * plan. With subqueries removed correctly, TopDownGeneralDecorrelator can in theory eliminate all + * correlation. We recommend using {@link CoreRules#FILTER_SUB_QUERY_TO_MARK_CORRELATE} + * and {@link CoreRules#PROJECT_SUB_QUERY_TO_MARK_CORRELATE} to remove subqueries from Filter and + * Project. These rules produce LEFT MARK Join/Correlate which are better suited for + * TopDownGeneralDecorrelator. There is not yet a corresponding, specially tailored rule for + * Join; you may choose to use {@link CoreRules#JOIN_SUB_QUERY_TO_CORRELATE}. Alternatively, for + * greater stability, you can run TopDownGeneralDecorrelator first and then apply + * {@link CoreRules#JOIN_SUB_QUERY_TO_CORRELATE} together with {@link RelDecorrelator}.
      3. + *
    3. Call {@link TopDownGeneralDecorrelator#decorrelateQuery(RelNode, RelBuilder)} to obtain + * the decorrelated plan.
      4. + *
    4. Continue with other optimizations.
      5. + *
    + * + *

    See + * org.apache.calcite.test.RelOptRulesTest#testTopDownGeneralDecorrelateForFilterExists() + * and org.apache.calcite.test.RelOptFixture#checkPlanning(boolean) for + * working examples. + */ +@Experimental +public class TopDownGeneralDecorrelator implements ReflectiveVisitor { + + private final RelBuilder builder; + + // record the CorDef in the current context (including those in the parent Correlate). + // NavigableSet is used to ensure a stable iteration order. + private final NavigableSet corDefs; + + // a map from RelNode to whether existing correlated expressions (according to corDefs). + private final Map hasCorrelatedExpressions; + + // a map from RelNode to its UnnestedQuery. + private final Map mapRelToUnnestedQuery; + + private final boolean hasParent; + + // the domain of the free variables (i.e. corDefs) D, it's duplicate free. + private DedupFreeVarsNode dedupFreeVarsNode; + + // invokes using reflection a method named unnestInternal based on the + // runtime type of the argument. + @SuppressWarnings("method.invocation.invalid") + private final ReflectUtil.MethodDispatcher dispatcher = + ReflectUtil.createMethodDispatcher( + RelNode.class, getVisitor(), "unnestInternal", RelNode.class, boolean.class); + + /** + * Creates a TopDownGeneralDecorrelator. If parent context arguments are provided, + * they are reused/merged into this instance. + * + * @param builder RelBuilder + * @param hasParent whether has parent decorrelator + * @param parentCorDefs corDefs from parent decorrelator + * @param parentHasCorrelatedExpressions a map from RelNode to whether existing correlated + * expressions + * @param parentMapRelToUnnestedQuery a map from RelNode to its UnnestedQuery + */ + @SuppressWarnings("initialization.fields.uninitialized") + private TopDownGeneralDecorrelator( + RelBuilder builder, + boolean hasParent, + @Nullable Set parentCorDefs, + @Nullable Map parentHasCorrelatedExpressions, + @Nullable Map parentMapRelToUnnestedQuery) { + this.builder = builder; + this.hasParent = hasParent; + this.corDefs = new TreeSet<>(); + if (parentCorDefs != null) { + this.corDefs.addAll(parentCorDefs); + } + this.hasCorrelatedExpressions = parentHasCorrelatedExpressions == null + ? new HashMap<>() + : parentHasCorrelatedExpressions; + this.mapRelToUnnestedQuery = parentMapRelToUnnestedQuery == null + ? new HashMap<>() + : parentMapRelToUnnestedQuery; + } + + public static TopDownGeneralDecorrelator createEmptyDecorrelator(RelBuilder builder) { + return new TopDownGeneralDecorrelator(builder, false, null, null, null); + } + + private TopDownGeneralDecorrelator createSubDecorrelator() { + TopDownGeneralDecorrelator subDecorrelator = + new TopDownGeneralDecorrelator( + builder, + true, + corDefs, + hasCorrelatedExpressions, + mapRelToUnnestedQuery); + subDecorrelator.dedupFreeVarsNode = this.dedupFreeVarsNode; + return subDecorrelator; + } + + /** + * Decorrelates a query. This is the entry point for this class. + * + * @param rel Root node of the query + * @param builder RelBuilder + * @return Equivalent node without correlation + */ + public static RelNode decorrelateQuery(RelNode rel, RelBuilder builder) { + HepProgram preProgram = HepProgram.builder() + .addRuleCollection( + ImmutableList.of( + CoreRules.FILTER_PROJECT_TRANSPOSE, + CoreRules.FILTER_INTO_JOIN, + CoreRules.FILTER_CORRELATE)) + .build(); + HepPlanner prePlanner = new HepPlanner(preProgram); + prePlanner.setRoot(rel); + RelNode preparedRel = prePlanner.findBestExp(); + + // start decorrelating + TopDownGeneralDecorrelator decorrelator = createEmptyDecorrelator(builder); + RelNode decorrelateNode = rel; + try { + decorrelateNode = decorrelator.correlateElimination(preparedRel, true); + } catch (UnsupportedOperationException e) { + // if the correlation exists in an unsupported operator, retain the original plan. + } + + HepProgram postProgram = HepProgram.builder() + .addRuleCollection( + ImmutableList.of( + CoreRules.FILTER_PROJECT_TRANSPOSE, + CoreRules.FILTER_INTO_JOIN, + CoreRules.MARK_TO_SEMI_OR_ANTI_JOIN_RULE, + CoreRules.PROJECT_MERGE, + CoreRules.PROJECT_REMOVE)) + .build(); + HepPlanner postPlanner = new HepPlanner(postProgram); + postPlanner.setRoot(decorrelateNode); + return postPlanner.findBestExp(); + } + + /** + * Eliminates Correlate. + * + * @param rel RelNode + * @param allowEmptyOutputFromRewrite whether allow empty output resulting from + * decorrelate rewriting. + * @return Equivalent RelNode without Correlate + */ + private RelNode correlateElimination(RelNode rel, boolean allowEmptyOutputFromRewrite) { + if (!(rel instanceof Correlate)) { + for (int i = 0; i < rel.getInputs().size(); i++) { + rel.replaceInput(i, correlateElimination(rel.getInput(i), allowEmptyOutputFromRewrite)); + } + return rel; + } + + final Correlate correlate = (Correlate) rel; + final RelNode newLeft; + if (hasParent) { + // if the current decorrelator has a parent, it means that the Correlate must have + // correlation from above. + assert hasCorrelatedExpressions.containsKey(correlate) + && hasCorrelatedExpressions.get(correlate); + newLeft = unnest(correlate.getLeft(), allowEmptyOutputFromRewrite); + } else { + // otherwise, start a new decorrelation for the left side. + newLeft = decorrelateQuery(correlate.getLeft(), builder); + } + + // create or update UnnestedQuery of left side and corDefs of this decorrelator. + UnnestedQuery leftInfo = mapRelToUnnestedQuery.get(correlate.getLeft()); + TreeMap corDefOutputs = new TreeMap<>(); + Map oldToNewOutputs = new HashMap<>(); + for (int i = 0; i < correlate.getLeft().getRowType().getFieldCount(); i++) { + int newColumnIndex = leftInfo == null ? i : requireNonNull(leftInfo.oldToNewOutputs.get(i)); + oldToNewOutputs.put(i, newColumnIndex); + if (correlate.getRequiredColumns().get(i)) { + CorDef corDef = new CorDef(correlate.getCorrelationId(), i); + corDefs.add(corDef); + corDefOutputs.put(corDef, newColumnIndex); + } + } + if (leftInfo != null) { + corDefOutputs.putAll(leftInfo.corDefOutputs); + } + leftInfo = new UnnestedQuery(correlate.getLeft(), newLeft, corDefOutputs, oldToNewOutputs); + dedupFreeVarsNode = DedupFreeVarsNode.create(newLeft, leftInfo, corDefs, builder); + + // decorrelate right side + detectCorrelatedExpressions(correlate.getRight()); + allowEmptyOutputFromRewrite &= correlate.getJoinType() != JoinRelType.LEFT_MARK; + RelNode newRight = unnest(correlate.getRight(), allowEmptyOutputFromRewrite); + UnnestedQuery rightInfo = requireNonNull(mapRelToUnnestedQuery.get(correlate.getRight())); + + // rewrite condition, adding the natural join condition between the left side and + // the domain D that is produced from the right side. This the fundamental equation from the + // paper Improving Unnesting of Complex Queries, shown in Section 2.2 + // + // Correlate(condition=[p]) + // / \ + // L ... with correlation + // \ + // R + // => + // Join(condition=[p AND (L is not distinct from D)]) + // / \ + // L ... without correlation + // \ + // x + // / \ + // D R + builder.push(newLeft).push(newRight); + RexNode unnestedJoinCondition = + UnnestedQuery.createUnnestedJoinCondition(correlate.getCondition(), leftInfo, rightInfo, + true, builder, corDefs); + RelNode unnestedRel = builder.join(correlate.getJoinType(), unnestedJoinCondition).build(); + + if (!hasParent) { + // ensure that the fields are in the same order as in the original plan. + builder.push(unnestedRel); + UnnestedQuery unnestedQuery = + UnnestedQuery.createJoinUnnestInfo( + leftInfo, + rightInfo, + correlate, + unnestedRel, + correlate.getJoinType()); + List projects + = builder.fields(new ArrayList<>(unnestedQuery.oldToNewOutputs.values())); + unnestedRel = builder.project(projects).build(); + } + return unnestedRel; + } + + /** + * Detects whether any expression in the relational tree rooted at {@code rel} refers to any + * variables that appear in {@link #corDefs} and populates the {@link #hasCorrelatedExpressions}. + * + *

    It is necessary to detect correlation for every node, for example: + * + * 

    +   *      Union
+   *    /   |   \
+   *  r1    r2   r3     all with correlation
+   *  |     |     |
+   * r11   r22   r33    all without correlation
+   *
    + * + *

    If we stop after detecting correlation in the r1 branch, we lose correlation information + * for the r2 and r3 branches. Without that information we cannot know the correct stopping point + * when pushing down D to r2/r3 branches. In addition, accurately knowing the correlation of each + * input enables useful optimizations when pushing down D to Join. + * + * @param rel RelNode + * @return true when there are correlated expressions + */ + private boolean detectCorrelatedExpressions(RelNode rel) { + if (!hasParent && hasCorrelatedExpressions.containsKey(rel)) { + // for shared sub-trees, check the map hasCorrelatedExpressions first. However, this is only + // valid when there is no parent decorrelator. For example: + // Correlate0 => cor0 + // / \ + // r1 r2 + // \ + // Correlate1 => cor1 + // / \ + // r3 with cor0 r5 with cor1 + // / \ + // r4 r6 + // for the parent decorrelator-0 of Correlate0, r5 doesn't have correlation. However, for the + // decorrelator-1 of Correlate1, its construction merge information from the parent + // decorrelation-0, at this point, r5 still doesn't have correlation. Once the decorrelator-1 + // completes the decorrelation on the left side of Correlate1, it need to detect the + // correlation on Correlate1 right side (based on cor0 and cor1). Now r5 has correlation, + // and the value in hasCorrelatedExpressions will change from FALSE to TRUE. + return hasCorrelatedExpressions.get(rel); + } + boolean hasCorrelation = false; + for (RelNode input : rel.getInputs()) { + hasCorrelation |= detectCorrelatedExpressions(input); + } + if (!hasCorrelation) { + RelOptUtil.VariableUsedVisitor variableUsedVisitor = + new RelOptUtil.VariableUsedVisitor(null); + rel.accept(variableUsedVisitor); + Set corrIdSet + = corDefs.stream() + .map(corDef -> corDef.corr) + .collect(ImmutableSet.toImmutableSet()); + hasCorrelation = + !variableUsedVisitor.variables.isEmpty() + && !Collections.disjoint(corrIdSet, variableUsedVisitor.variables); + } + hasCorrelatedExpressions.put(rel, hasCorrelation); + return hasCorrelation; + } + + /** + * Unnests a RelNode. If there is no correlation in the node, create the cross product + * with domain D; otherwise, dispatch to specific method to push down D based on the type of rel. + * + * @param rel RelNode + * @param allowEmptyOutputFromRewrite whether allow empty output resulting from + * decorrelate rewriting. + * @return new node (contains domain D) without correlation + */ + private RelNode unnest(RelNode rel, boolean allowEmptyOutputFromRewrite) { + if (!requireNonNull(hasCorrelatedExpressions.get(rel))) { + RelNode newRel + = builder.push(decorrelateQuery(rel, builder)) + .push(dedupFreeVarsNode.r) + .join(JoinRelType.INNER) + .build(); + Map oldToNewOutputs = new HashMap<>(); + IntStream.range(0, rel.getRowType().getFieldCount()) + .forEach(i -> oldToNewOutputs.put(i, i)); + + int offset = rel.getRowType().getFieldCount(); + TreeMap corDefOutputs = new TreeMap<>(); + for (CorDef corDef : corDefs) { + corDefOutputs.put(corDef, offset++); + } + + UnnestedQuery unnestedQuery + = new UnnestedQuery(rel, newRel, corDefOutputs, oldToNewOutputs); + mapRelToUnnestedQuery.put(rel, unnestedQuery); + return newRel; + } + return dispatcher.invoke(rel, allowEmptyOutputFromRewrite); + } + + public RelNode unnestInternal(Filter filter, boolean allowEmptyOutputFromRewrite) { + Map oldToNewOutputs = new HashMap<>(); + TreeMap corDefOutputs = new TreeMap<>(); + List newConditions = new ArrayList<>(); + // try to replace all free variables to input refs according to equi-conditions + if (tryReplaceFreeVarsToInputRef(filter, corDefOutputs, newConditions)) { + // all free variables can be replaced, no need to push down D, that is, D is eliminated. + builder.push(filter.getInput()).filter(newConditions); + for (int i = 0; i < filter.getRowType().getFieldCount(); i++) { + oldToNewOutputs.put(i, i); + } + } else { + // push down D + RelNode newInput = unnest(filter.getInput(), allowEmptyOutputFromRewrite); + UnnestedQuery inputInfo = requireNonNull(mapRelToUnnestedQuery.get(filter.getInput())); + RexNode newCondition = + CorrelatedExprRewriter.rewrite(filter.getCondition(), inputInfo); + builder.push(newInput).filter(newCondition); + oldToNewOutputs = inputInfo.oldToNewOutputs; + corDefOutputs.putAll(inputInfo.corDefOutputs); + } + RelNode newFilter = builder.build(); + UnnestedQuery unnestedQuery = + new UnnestedQuery(filter, newFilter, corDefOutputs, oldToNewOutputs); + mapRelToUnnestedQuery.put(filter, unnestedQuery); + return newFilter; + } + + public RelNode unnestInternal(Project project, boolean allowEmptyOutputFromRewrite) { + for (RexNode expr : project.getProjects()) { + if (!allowEmptyOutputFromRewrite) { + break; + } + allowEmptyOutputFromRewrite &= Strong.isStrong(expr); + } + RelNode newInput = unnest(project.getInput(), allowEmptyOutputFromRewrite); + UnnestedQuery inputInfo = requireNonNull(mapRelToUnnestedQuery.get(project.getInput())); + List newProjects + = CorrelatedExprRewriter.rewrite(project.getProjects(), inputInfo); + + int oriFieldCount = newProjects.size(); + Map oldToNewOutputs = new HashMap<>(); + IntStream.range(0, oriFieldCount).forEach(i -> oldToNewOutputs.put(i, i)); + + builder.push(newInput); + TreeMap corDefOutputs = new TreeMap<>(); + for (CorDef corDef : corDefs) { + newProjects.add(builder.field(requireNonNull(inputInfo.corDefOutputs.get(corDef)))); + corDefOutputs.put(corDef, oriFieldCount++); + } + RelNode newProject = builder.project(newProjects, ImmutableList.of(), true).build(); + UnnestedQuery unnestedQuery + = new UnnestedQuery(project, newProject, corDefOutputs, oldToNewOutputs); + mapRelToUnnestedQuery.put(project, unnestedQuery); + return newProject; + } + + public RelNode unnestInternal(Aggregate aggregate, boolean allowEmptyOutputFromRewrite) { + RelNode newInput = unnest(aggregate.getInput(), allowEmptyOutputFromRewrite); + UnnestedQuery inputUnnestedQuery = + requireNonNull(mapRelToUnnestedQuery.get(aggregate.getInput())); + builder.push(newInput); + + // create new groupSet and groupSets, adding the fields in D to group keys + ImmutableBitSet.Builder corKeyBuilder = ImmutableBitSet.builder(); + for (CorDef corDef : corDefs) { + int corKeyIndex = requireNonNull(inputUnnestedQuery.corDefOutputs.get(corDef)); + corKeyBuilder.set(corKeyIndex); + } + ImmutableBitSet corKeyBitSet = corKeyBuilder.build(); + ImmutableBitSet newGroupSet + = aggregate.getGroupSet().permute(inputUnnestedQuery.oldToNewOutputs) + .union(corKeyBitSet); + List newGroupSets = new ArrayList<>(); + for (ImmutableBitSet bitSet : aggregate.getGroupSets()) { + ImmutableBitSet newBitSet + = bitSet.permute(inputUnnestedQuery.oldToNewOutputs).union(corKeyBitSet); + newGroupSets.add(newBitSet); + } + + // create new aggregate functions + boolean hasCountFunction = false; + List permutedAggCalls = new ArrayList<>(); + Mappings.TargetMapping targetMapping = + Mappings.target( + inputUnnestedQuery.oldToNewOutputs, + inputUnnestedQuery.oldRel.getRowType().getFieldCount(), + inputUnnestedQuery.r.getRowType().getFieldCount()); + for (AggregateCall aggCall : aggregate.getAggCallList()) { + hasCountFunction |= aggCall.getAggregation() instanceof SqlCountAggFunction; + permutedAggCalls.add(aggCall.transform(targetMapping)); + } + // create new Aggregate node + RelNode newAggregate + = builder.aggregate(builder.groupKey(newGroupSet, newGroupSets), permutedAggCalls).build(); + + // create UnnestedQuery + Map oldToNewOutputs = new HashMap<>(); + for (int groupKey : aggregate.getGroupSet()) { + int oriIndex = aggregate.getGroupSet().indexOf(groupKey); + int newIndex = newGroupSet.indexOf(groupKey); + oldToNewOutputs.put(oriIndex, newIndex); + } + for (int i = 0; i < aggregate.getAggCallList().size(); i++) { + oldToNewOutputs.put( + aggregate.getGroupCount() + i, + newGroupSet.cardinality() + i); + } + TreeMap corDefOutputs = new TreeMap<>(); + for (CorDef corDef : corDefs) { + int index = requireNonNull(inputUnnestedQuery.corDefOutputs.get(corDef)); + corDefOutputs.put(corDef, newGroupSet.indexOf(index)); + } + + if (aggregate.hasEmptyGroup() + && (!allowEmptyOutputFromRewrite || hasCountFunction)) { + // create a left join with D to avoid rewriting from non-empty to empty output + builder.push(dedupFreeVarsNode.r).push(newAggregate); + List leftJoinConditions = new ArrayList<>(); + int freeVarsIndex = 0; + for (CorDef corDef : corDefs) { + RexNode notDistinctFrom = + builder.isNotDistinctFrom( + builder.field(2, 0, freeVarsIndex), + builder.field(2, 1, requireNonNull(corDefOutputs.get(corDef)))); + leftJoinConditions.add(notDistinctFrom); + + corDefOutputs.put(corDef, freeVarsIndex++); + } + builder.join(JoinRelType.LEFT, leftJoinConditions); + + // replace the reference to COUNT with CASE WHEN COUNT(*) IS NULL THEN 0 ELSE COUNT(*) END + List aggCallProjects = new ArrayList<>(); + final int aggCallStartIndex = + dedupFreeVarsNode.r.getRowType().getFieldCount() + newGroupSet.cardinality(); + for (int i = 0; i < permutedAggCalls.size(); i++) { + int index = aggCallStartIndex + i; + SqlAggFunction aggregation = permutedAggCalls.get(i).getAggregation(); + if (aggregation instanceof SqlCountAggFunction) { + RexNode caseWhenRewrite = + builder.call( + SqlStdOperatorTable.CASE, + builder.isNotNull(builder.field(index)), + builder.field(index), + builder.literal(0)); + aggCallProjects.add(caseWhenRewrite); + } else { + aggCallProjects.add(builder.field(index)); + } + } + List projects = + new ArrayList<>(builder.fields(ImmutableBitSet.range(0, aggCallStartIndex))); + projects.addAll(aggCallProjects); + newAggregate = builder.project(projects).build(); + + + for (Map.Entry entry : oldToNewOutputs.entrySet()) { + int value = requireNonNull(entry.getValue()); + entry.setValue(value + corDefs.size()); + } + } + UnnestedQuery unnestedQuery + = new UnnestedQuery(aggregate, newAggregate, corDefOutputs, oldToNewOutputs); + mapRelToUnnestedQuery.put(aggregate, unnestedQuery); + return newAggregate; + } + + public RelNode unnestInternal(Sort sort, boolean allowEmptyOutputFromRewrite) { + RelNode newInput = unnest(sort.getInput(), allowEmptyOutputFromRewrite); + UnnestedQuery inputInfo = + requireNonNull(mapRelToUnnestedQuery.get(sort.getInput())); + Mappings.TargetMapping targetMapping = + Mappings.target( + inputInfo.oldToNewOutputs, + inputInfo.oldRel.getRowType().getFieldCount(), + inputInfo.r.getRowType().getFieldCount()); + RelCollation shiftCollation = sort.getCollation().apply(targetMapping); + builder.push(newInput); + + if (!sort.collation.getFieldCollations().isEmpty() + && (sort.offset != null || sort.fetch != null)) { + // the Sort with ORDER BY and LIMIT or OFFSET have to be changed during rewriting because + // now the limit has to be enforced per value of the outer bindings instead of globally. + // It can be rewritten using ROW_NUMBER() window function and filtering on it, + // see section 4.4 in paper Improving Unnesting of Complex Queries + List partitionKeys = new ArrayList<>(); + for (CorDef corDef : corDefs) { + int partitionKeyIndex = requireNonNull(inputInfo.corDefOutputs.get(corDef)); + partitionKeys.add(builder.field(partitionKeyIndex)); + } + RexNode rowNumber = builder.aggregateCall(SqlStdOperatorTable.ROW_NUMBER) + .over() + .partitionBy(partitionKeys) + .orderBy(builder.fields(shiftCollation)) + .toRex(); + List projectsWithRowNumber = new ArrayList<>(builder.fields()); + projectsWithRowNumber.add(rowNumber); + builder.project(projectsWithRowNumber); + + List conditions = new ArrayList<>(); + if (sort.offset != null) { + RexNode greaterThenLowerBound = + builder.call( + SqlStdOperatorTable.GREATER_THAN, + builder.field(projectsWithRowNumber.size() - 1), + sort.offset); + conditions.add(greaterThenLowerBound); + } + if (sort.fetch != null) { + RexNode upperBound = sort.offset == null + ? sort.fetch + : builder.call(SqlStdOperatorTable.PLUS, sort.offset, sort.fetch); + RexNode lessThenOrEqualUpperBound = + builder.call( + SqlStdOperatorTable.LESS_THAN_OR_EQUAL, + builder.field(projectsWithRowNumber.size() - 1), + upperBound); + conditions.add(lessThenOrEqualUpperBound); + } + builder.filter(conditions); + } else { + builder.sortLimit(sort.offset, sort.fetch, builder.fields(shiftCollation)); + } + RelNode newSort = builder.build(); + UnnestedQuery unnestedQuery + = new UnnestedQuery(sort, newSort, inputInfo.corDefOutputs, inputInfo.oldToNewOutputs); + mapRelToUnnestedQuery.put(sort, unnestedQuery); + return newSort; + } + + public RelNode unnestInternal(Correlate correlate, boolean allowEmptyOutputFromRewrite) { + // when nesting Correlate and there are still correlation, create a sub-decorrelator and merge + // this decorrelator's context to the sub-decorrelator. + TopDownGeneralDecorrelator subDecorrelator = createSubDecorrelator(); + Join newJoin = + (Join) subDecorrelator.correlateElimination(correlate, allowEmptyOutputFromRewrite); + + UnnestedQuery leftInfo + = requireNonNull(subDecorrelator.mapRelToUnnestedQuery.get(correlate.getLeft())); + UnnestedQuery rightInfo + = requireNonNull(subDecorrelator.mapRelToUnnestedQuery.get(correlate.getRight())); + UnnestedQuery unnestedQuery = + UnnestedQuery.createJoinUnnestInfo(leftInfo, rightInfo, correlate, + newJoin, correlate.getJoinType()); + mapRelToUnnestedQuery.put(correlate, unnestedQuery); + return newJoin; + } + + public RelNode unnestInternal(Join join, boolean allowEmptyOutputFromRewrite) { + boolean leftHasCorrelation = + requireNonNull(hasCorrelatedExpressions.get(join.getLeft())); + boolean rightHasCorrelation = + requireNonNull(hasCorrelatedExpressions.get(join.getRight())); + boolean pushDownToLeft = false; + boolean pushDownToRight = false; + RelNode newLeft; + RelNode newRight; + UnnestedQuery leftInfo; + UnnestedQuery rightInfo; + + if (!leftHasCorrelation && !join.getJoinType().generatesNullsOnRight() + && join.getJoinType().projectsRight()) { + // there is no need to push down domain D to left side when both following conditions + // are satisfied: + // 1. there is no correlation on left side + // 2. join type will not generate NULL values on right side and will project right + // In this case, the left side will start a decorrelation independently + newLeft = decorrelateQuery(join.getLeft(), builder); + Map leftOldToNewOutputs = new HashMap<>(); + IntStream.range(0, newLeft.getRowType().getFieldCount()) + .forEach(i -> leftOldToNewOutputs.put(i, i)); + leftInfo = new UnnestedQuery(join.getLeft(), newLeft, new TreeMap<>(), leftOldToNewOutputs); + } else { + newLeft = unnest(join.getLeft(), allowEmptyOutputFromRewrite); + pushDownToLeft = true; + leftInfo = requireNonNull(mapRelToUnnestedQuery.get(join.getLeft())); + } + if (!rightHasCorrelation && !join.getJoinType().generatesNullsOnLeft()) { + // there is no need to push down domain D to right side when both following conditions + // are satisfied: + // 1. there is no correlation on right side + // 2. join type will not generate NULL values on left side + // In this case, the right side will start a decorrelation independently + newRight = decorrelateQuery(join.getRight(), builder); + Map rightOldToNewOutputs = new HashMap<>(); + IntStream.range(0, newRight.getRowType().getFieldCount()) + .forEach(i -> rightOldToNewOutputs.put(i, i)); + rightInfo = + new UnnestedQuery(join.getRight(), newRight, new TreeMap<>(), rightOldToNewOutputs); + } else { + allowEmptyOutputFromRewrite &= join.getJoinType() != JoinRelType.LEFT_MARK; + newRight = unnest(join.getRight(), allowEmptyOutputFromRewrite); + pushDownToRight = true; + rightInfo = requireNonNull(mapRelToUnnestedQuery.get(join.getRight())); + } + + builder.push(newLeft).push(newRight); + // if domain D is pushed down to both sides, the new join condition need to add the natural + // condition between D + RexNode newJoinCondition = + UnnestedQuery.createUnnestedJoinCondition( + join.getCondition(), + leftInfo, + rightInfo, + pushDownToLeft && pushDownToRight, + builder, + corDefs); + RelNode newJoin = builder.join(join.getJoinType(), newJoinCondition).build(); + UnnestedQuery unnestedQuery = + UnnestedQuery.createJoinUnnestInfo( + leftInfo, + rightInfo, + join, + newJoin, + join.getJoinType()); + mapRelToUnnestedQuery.put(join, unnestedQuery); + return newJoin; + } + + public RelNode unnestInternal(SetOp setOp, boolean allowEmptyOutputFromRewrite) { + List newInputs = new ArrayList<>(); + for (RelNode input : setOp.getInputs()) { + // push down the domain D to each input + RelNode newInput = unnest(input, allowEmptyOutputFromRewrite); + builder.push(newInput); + UnnestedQuery inputInfo = requireNonNull(mapRelToUnnestedQuery.get(input)); + // ensure that the rowType remains consistent after each input is rewritten: + // [original fields that maintain their original order, the domain D] + List projectIndexes = new ArrayList<>(); + for (int i = 0; i < inputInfo.oldRel.getRowType().getFieldCount(); i++) { + projectIndexes.add(requireNonNull(inputInfo.oldToNewOutputs.get(i))); + } + for (CorDef corDef : corDefs) { + projectIndexes.add(requireNonNull(inputInfo.corDefOutputs.get(corDef))); + } + builder.project(builder.fields(projectIndexes)); + newInputs.add(builder.build()); + } + builder.pushAll(newInputs); + switch (setOp.kind) { + case UNION: + builder.union(setOp.all, newInputs.size()); + break; + case INTERSECT: + builder.intersect(setOp.all, newInputs.size()); + break; + case EXCEPT: + builder.minus(setOp.all, newInputs.size()); + break; + default: + throw new AssertionError("Not a set op: " + setOp); + } + RelNode newSetOp = builder.build(); + + int oriSetOpFieldCount = setOp.getRowType().getFieldCount(); + Map oldToNewOutputs = new HashMap<>(); + IntStream.range(0, oriSetOpFieldCount).forEach(i -> oldToNewOutputs.put(i, i)); + TreeMap corDefOutputs = new TreeMap<>(); + for (CorDef corDef : corDefs) { + corDefOutputs.put(corDef, oriSetOpFieldCount++); + } + UnnestedQuery unnestedQuery = + new UnnestedQuery(setOp, newSetOp, corDefOutputs, oldToNewOutputs); + mapRelToUnnestedQuery.put(setOp, unnestedQuery); + return newSetOp; + } + + public RelNode unnestInternal(RelNode other) { + throw new UnsupportedOperationException("Top-down general decorrelator does not support: " + + other.getClass().getSimpleName()); + } + + /** + * Try to replace all free variables (i.e. the attributes of domain D) with RexInputRef. + * When the decorrelation process reaches: + * + * 

    {@code
+   *              Filter    with correlation
+   *                |
+   *              Input     without correlation
+   * }
    + * + *

    It will introduce the domain D by creating a cross product with Input. However, if all free + * variables in the condition are filtered using equality conditions with local attributes, then + * we can instead derive the domain from the local attributes. This substitution results in a + * superset (compared to creating a cross product between D and input), because the filter effect + * of the equality conditions is removed. However, this does not affect the final result, + * because the filter will still happen at a later stage (at the original Correlate). Although + * this substitution will result in more intermediate results, we assume that introducing a join + * is more costly. See section 3.3 in paper Unnesting Arbitrary Queries. + * + * @param filter Filter node + * @param corDefToInputIndex a map from CorDef to input index + * @param newConditions new conditions after replacing free variables + * @return true when all free variables are replaced + */ + private boolean tryReplaceFreeVarsToInputRef( + Filter filter, + Map corDefToInputIndex, + List newConditions) { + if (requireNonNull(hasCorrelatedExpressions.get(filter.getInput()))) { + return false; + } + List oriConditions = RelOptUtil.conjunctions(filter.getCondition()); + for (RexNode condition : oriConditions) { + if (RexUtil.containsCorrelation(condition)) { + Pair pair = getPairOfFreeVarAndInputRefInEqui(condition); + if (pair != null) { + // equi-condition will filter NULL values, so need to add IS NOT NULL for input ref + if (condition.isA(SqlKind.EQUALS)) { + newConditions.add(builder.isNotNull(pair.right)); + } + corDefToInputIndex.put(pair.left, pair.right.getIndex()); + continue; + } + // if the condition is correlated but it's not an equi-condition between free variable + // and input ref, then cannot replaced. + return false; + } else { + newConditions.add(condition); + } + } + Set replacedCorDef = corDefToInputIndex.keySet(); + // ensure all free variables can be replaced + return replacedCorDef.size() == corDefs.size() && corDefs.containsAll(replacedCorDef); + } + + private @Nullable Pair getPairOfFreeVarAndInputRefInEqui(RexNode condition) { + if (!condition.isA(SqlKind.EQUALS) && !condition.isA(SqlKind.IS_NOT_DISTINCT_FROM)) { + return null; + } + RexCall equiCond = (RexCall) condition; + RexNode left = equiCond.getOperands().get(0); + RexNode right = equiCond.getOperands().get(1); + CorDef leftCorDef = unwrapCorDef(left); + CorDef rightCorDef = unwrapCorDef(right); + if (left instanceof RexInputRef && rightCorDef != null) { + return Pair.of(rightCorDef, (RexInputRef) left); + } + if (right instanceof RexInputRef && leftCorDef != null) { + return Pair.of(leftCorDef, (RexInputRef) right); + } + return null; + } + + private @Nullable CorDef unwrapCorDef(RexNode expr) { + if (expr instanceof RexFieldAccess) { + RexFieldAccess fieldAccess = (RexFieldAccess) expr; + if (fieldAccess.getReferenceExpr() instanceof RexCorrelVariable) { + RexCorrelVariable v = (RexCorrelVariable) fieldAccess.getReferenceExpr(); + CorDef corDef = new CorDef(v.id, fieldAccess.getField().getIndex()); + return corDefs.contains(corDef) ? corDef : null; + } + } + return null; + } + + /** + * Rewrites correlated expressions, window function and shift input references. + */ + static class CorrelatedExprRewriter extends RexShuttle { + final UnnestedQuery unnestedQuery; + + CorrelatedExprRewriter(UnnestedQuery unnestedQuery) { + this.unnestedQuery = unnestedQuery; + } + + static RexNode rewrite( + RexNode expr, + UnnestedQuery unnestedQuery) { + CorrelatedExprRewriter rewriter = new CorrelatedExprRewriter(unnestedQuery); + return expr.accept(rewriter); + } + + static List rewrite( + List exprs, + UnnestedQuery unnestedQuery) { + CorrelatedExprRewriter rewriter = new CorrelatedExprRewriter(unnestedQuery); + return new ArrayList<>(rewriter.apply(exprs)); + } + + @Override public RexNode visitInputRef(RexInputRef inputRef) { + int newIndex = requireNonNull(unnestedQuery.oldToNewOutputs.get(inputRef.getIndex())); + if (newIndex == inputRef.getIndex()) { + return inputRef; + } + return new RexInputRef(newIndex, inputRef.getType()); + } + + @Override public RexNode visitFieldAccess(RexFieldAccess fieldAccess) { + if (fieldAccess.getReferenceExpr() instanceof RexCorrelVariable) { + RexCorrelVariable v = + (RexCorrelVariable) fieldAccess.getReferenceExpr(); + CorDef corDef = new CorDef(v.id, fieldAccess.getField().getIndex()); + int newIndex = requireNonNull(unnestedQuery.corDefOutputs.get(corDef)); + return new RexInputRef(newIndex, fieldAccess.getType()); + } + return super.visitFieldAccess(fieldAccess); + } + + @Override public RexWindow visitWindow(RexWindow window) { + RexWindow shiftedWindow = super.visitWindow(window); + List newPartitionKeys = new ArrayList<>(shiftedWindow.partitionKeys); + for (Integer corIndex : unnestedQuery.corDefOutputs.values()) { + RexInputRef inputRef = + new RexInputRef( + corIndex, + unnestedQuery.r.getRowType().getFieldList().get(corIndex).getType()); + newPartitionKeys.add(inputRef); + } + return unnestedQuery.r.getCluster().getRexBuilder().makeWindow( + newPartitionKeys, + window.orderKeys, + window.getLowerBound(), + window.getUpperBound(), + window.isRows(), + window.getExclude()); + } + } + + public TopDownGeneralDecorrelator getVisitor() { + return this; + } + + /** + * Unnesting information. + */ + static class UnnestedQuery extends Frame { + final RelNode oldRel; + + /** + * Creates a UnnestedQuery. + * + * @param oldRel old node before unnesting + * @param r new node after unnesting + * @param corDefOutputs a sorted map from CorDef to output index in new node + * @param oldToNewOutputs a map from old node output index to new node output index + */ + UnnestedQuery(RelNode oldRel, RelNode r, NavigableMap corDefOutputs, + Map oldToNewOutputs) { + super(oldRel, r, corDefOutputs, oldToNewOutputs); + this.oldRel = oldRel; + } + + /** + * Create UnnestedQuery for Join/Correlate after decorrelating. + * + * @param leftInfo UnnestedQuery of the left side + * @param rightInfo UnnestedQuery of the right side + * @param oriJoinNode original Join/Correlate node + * @param unnestedJoinNode new node after decorrelating + * @param joinRelType join type of original Join/Correlate + * @return UnnestedQuery + */ + private static UnnestedQuery createJoinUnnestInfo( + UnnestedQuery leftInfo, + UnnestedQuery rightInfo, + RelNode oriJoinNode, + RelNode unnestedJoinNode, + JoinRelType joinRelType) { + Map oldToNewOutputs = new HashMap<>(); + oldToNewOutputs.putAll(leftInfo.oldToNewOutputs); + int oriLeftFieldCount = leftInfo.oldRel.getRowType().getFieldCount(); + int newLeftFieldCount = leftInfo.r.getRowType().getFieldCount(); + switch (joinRelType) { + case SEMI: + case ANTI: + break; + case LEFT_MARK: + oldToNewOutputs.put(oriLeftFieldCount, newLeftFieldCount); + break; + default: + rightInfo.oldToNewOutputs.forEach((oriIndex, newIndex) -> + oldToNewOutputs.put( + requireNonNull(oriIndex, "oriIndex") + oriLeftFieldCount, + requireNonNull(newIndex, "newIndex") + newLeftFieldCount)); + break; + } + + TreeMap corDefOutputs = new TreeMap<>(); + if (!leftInfo.corDefOutputs.isEmpty()) { + corDefOutputs.putAll(leftInfo.corDefOutputs); + } else if (!rightInfo.corDefOutputs.isEmpty()) { + Litmus.THROW.check(joinRelType.projectsRight(), + "If the joinType doesn't project right, its left side must have UnnestInfo."); + rightInfo.corDefOutputs.forEach((corDef, index) -> + corDefOutputs.put(corDef, index + newLeftFieldCount)); + } else { + throw new IllegalArgumentException("The UnnestInfo for both sides of Join/Correlate that " + + "has correlation should not all be empty."); + } + return new UnnestedQuery(oriJoinNode, unnestedJoinNode, corDefOutputs, oldToNewOutputs); + } + + /** + * Create the new join condition after decorrelating. + * + * @param oriCondition original Correlate/Join condition + * @param leftInfo UnnestedQuery of the left side + * @param rightInfo UnnestedQuery of the right side + * @param needNaturalJoinCondition whether need to add the natural join condition for domain D + * @param builder RelBuilder + * @param corDefs the CorDef in the current decorrelator context + * @return the new join condition + */ + private static RexNode createUnnestedJoinCondition( + RexNode oriCondition, + UnnestedQuery leftInfo, + UnnestedQuery rightInfo, + boolean needNaturalJoinCondition, + RelBuilder builder, + NavigableSet corDefs) { + // create a temporary inner join and its UnnestedQuery to help rewrite the + // original condition by CorrelatedExprRewriter + Map temporaryOldToNewOutputs = new HashMap<>(); + int oriLeftFieldCount = leftInfo.oldRel.getRowType().getFieldCount(); + int newLeftFieldCount = leftInfo.r.getRowType().getFieldCount(); + temporaryOldToNewOutputs.putAll(leftInfo.oldToNewOutputs); + rightInfo.oldToNewOutputs.forEach((oriIndex, newIndex) -> + temporaryOldToNewOutputs.put( + requireNonNull(oriIndex, "oriIndex") + oriLeftFieldCount, + requireNonNull(newIndex, "newIndex") + newLeftFieldCount)); + + TreeMap temporaryCorDefOutputs = new TreeMap<>(); + if (!leftInfo.corDefOutputs.isEmpty()) { + temporaryCorDefOutputs.putAll(leftInfo.corDefOutputs); + } else if (!rightInfo.corDefOutputs.isEmpty()) { + rightInfo.corDefOutputs.forEach((corDef, index) -> + temporaryCorDefOutputs.put(corDef, index + newLeftFieldCount)); + } else { + throw new IllegalArgumentException("The UnnestInfo for both sides of Join/Correlate that " + + "has correlation should not all be empty."); + } + RelNode temporaryOldRel = builder.push(leftInfo.oldRel).push(rightInfo.oldRel) + .join(JoinRelType.INNER) + .build(); + RelNode temporaryNewRel = builder.push(leftInfo.r).push(rightInfo.r) + .join(JoinRelType.INNER) + .build(); + UnnestedQuery temporaryInfo = + new UnnestedQuery(temporaryOldRel, temporaryNewRel, + temporaryCorDefOutputs, temporaryOldToNewOutputs); + RexNode rewriteOriCondition = CorrelatedExprRewriter.rewrite(oriCondition, temporaryInfo); + List unnestedJoinConditions = new ArrayList<>(); + unnestedJoinConditions.add(rewriteOriCondition); + + if (needNaturalJoinCondition) { + for (CorDef corDef : corDefs) { + int leftIndex = requireNonNull(leftInfo.corDefOutputs.get(corDef)); + RelDataType leftColumnType + = leftInfo.r.getRowType().getFieldList().get(leftIndex).getType(); + int rightIndex = requireNonNull(rightInfo.corDefOutputs.get(corDef)); + RelDataType rightColumnType + = rightInfo.r.getRowType().getFieldList().get(rightIndex).getType(); + RexNode notDistinctFrom = + builder.isNotDistinctFrom( + new RexInputRef(leftIndex, leftColumnType), + new RexInputRef(rightIndex + newLeftFieldCount, rightColumnType)); + unnestedJoinConditions.add(notDistinctFrom); + } + } + return RexUtil.composeConjunction(builder.getRexBuilder(), unnestedJoinConditions); + } + + } + + /** + * The domain of the free variables. It's duplicate free. Corresponds to a relation denoted + * by D in the paper. + */ + static class DedupFreeVarsNode { + final RelNode r; + + DedupFreeVarsNode(RelNode r) { + this.r = r; + } + + /** + * Generate the domain of the free variables D. + * + * @param newLeft the left side (without correlation) of Correlate + * @param leftInfo the UnnestedQuery of the left side of Correlate + * @param corDefs the CorDef in the current decorrelator context + * @param builder RelBuilder + * @return the domain of the free variables D + */ + static DedupFreeVarsNode create( + RelNode newLeft, + UnnestedQuery leftInfo, + NavigableSet corDefs, + RelBuilder builder) { + List columnIndexes = new ArrayList<>(); + for (CorDef corDef : corDefs) { + int fieldIndex = requireNonNull(leftInfo.corDefOutputs.get(corDef)); + columnIndexes.add(fieldIndex); + } + List inputRefs = builder.push(newLeft) + .fields(columnIndexes); + RelNode rel = builder.project(inputRefs).distinct().build(); + return new DedupFreeVarsNode(rel); + } + } + +} diff --git a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java index 154cca84a3b0..8f9e11797453 100644 --- a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java +++ b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java @@ -3329,6 +3329,8 @@ public RelBuilder join(JoinRelType joinType, RexNode condition, filter(condition.accept(new Shifter(left.rel, id, right.rel))); right = stack.pop(); break; + case LEFT_MARK: + break; case INNER: // For INNER, we can defer. postCondition = condition; @@ -3337,9 +3339,15 @@ public RelBuilder join(JoinRelType joinType, RexNode condition, throw new IllegalArgumentException("Correlated " + joinType + " join is not supported"); } final ImmutableBitSet requiredColumns = RelOptUtil.correlationColumns(id, right.rel); - join = - struct.correlateFactory.createCorrelate(left.rel, right.rel, ImmutableList.of(), id, - requiredColumns, joinType); + if (joinType == JoinRelType.LEFT_MARK) { + join = + struct.conditionalCorrelateFactory.createConditionalCorrelate(left.rel, right.rel, + ImmutableList.of(), id, requiredColumns, joinType, condition); + } else { + join = + struct.correlateFactory.createCorrelate(left.rel, right.rel, ImmutableList.of(), id, + requiredColumns, joinType); + } } else { RelNode join0 = struct.joinFactory.createJoin(left.rel, right.rel, diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index ff3cb03c36e2..ac7a55984f26 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -11699,4 +11699,267 @@ private void checkLoptOptimizeJoinRule(LoptOptimizeJoinRule rule) { }) .check(); } + + /** Test case of + * [CALCITE-7031] + * Implement the general decorrelation algorithm (Neumann & Kemper). */ + @Test void testTopDownGeneralDecorrelateForFilterExists() { + final String sql = "select empno from emp where " + + "exists(select * from dept where dept.deptno = emp.deptno)"; + + sql(sql) + .withRule( + CoreRules.FILTER_SUB_QUERY_TO_MARK_CORRELATE, + CoreRules.PROJECT_MERGE, + CoreRules.PROJECT_REMOVE) + .withLateDecorrelate(true) + .withTopDownGeneralDecorrelate(true) + .check(); + } + + @Test void testTopDownGeneralDecorrelateForFilterSome() { + final String sql = "select empno from emp where " + + "empno > SOME(select empno from emp_b where emp.ename = emp_b.ename)"; + + sql(sql) + .withRule( + CoreRules.FILTER_SUB_QUERY_TO_MARK_CORRELATE, + CoreRules.PROJECT_MERGE, + CoreRules.PROJECT_REMOVE) + .withLateDecorrelate(true) + .withTopDownGeneralDecorrelate(true) + .check(); + } + + @Test void testTopDownGeneralDecorrelateForFilterNotIn() { + final String sql = "select empno from emp where " + + "empno not in (select empno from emp_b where emp.ename = emp_b.ename)"; + + sql(sql) + .withRule( + CoreRules.FILTER_SUB_QUERY_TO_MARK_CORRELATE, + CoreRules.PROJECT_MERGE, + CoreRules.PROJECT_REMOVE) + .withLateDecorrelate(true) + .withTopDownGeneralDecorrelate(true) + .check(); + } + + @Test void testTopDownGeneralDecorrelateForFilterNotExists() { + final String sql = "select empno from emp where " + + "not exists(select * from emp_b where emp.ename = emp_b.ename)"; + + sql(sql) + .withRule( + CoreRules.FILTER_SUB_QUERY_TO_MARK_CORRELATE, + CoreRules.PROJECT_MERGE, + CoreRules.PROJECT_REMOVE) + .withLateDecorrelate(true) + .withTopDownGeneralDecorrelate(true) + .check(); + } + + @Test void testTopDownGeneralDecorrelateForFilterScalar() { + final String sql = "select empno from emp where " + + "sal > (select avg(sal) from emp_b where emp.ename = emp_b.ename)"; + + sql(sql) + .withRule( + CoreRules.FILTER_SUB_QUERY_TO_MARK_CORRELATE, + CoreRules.PROJECT_MERGE, + CoreRules.PROJECT_REMOVE) + .withLateDecorrelate(true) + .withTopDownGeneralDecorrelate(true) + .check(); + } + + @Test void testTopDownGeneralDecorrelateForSubqueryWithSetOp() { + final String sql = "select empno, (select sum(deptno) from (" + + "select deptno from emp_b where emp.empno = emp_b.empno " + + "union all select deptno from empnullables where emp.empno = empnullables.empno))" + + " from emp"; + + sql(sql) + .withRule( + CoreRules.PROJECT_SUB_QUERY_TO_MARK_CORRELATE, + CoreRules.PROJECT_MERGE, + CoreRules.PROJECT_REMOVE) + .withLateDecorrelate(true) + .withTopDownGeneralDecorrelate(true) + .check(); + } + + @Test void testTopDownGeneralDecorrelateForSubqueryWithJoin() { + final String sql = "select empno from emp where sal > SOME(" + + "select sal from empnullables, (select empno from emp_b where emp.deptno = emp_b.deptno)" + + " b where empnullables.empno = b.empno)"; + + sql(sql) + .withRule( + CoreRules.FILTER_SUB_QUERY_TO_MARK_CORRELATE, + CoreRules.PROJECT_MERGE, + CoreRules.PROJECT_REMOVE) + .withLateDecorrelate(true) + .withTopDownGeneralDecorrelate(true) + .check(); + } + + @Test void testTopDownGeneralDecorrelateForCountScalar() { + final String sql = "select deptno, " + + "(select count(empno) from emp where dept.deptno = emp.deptno) from dept"; + + sql(sql) + .withRule( + CoreRules.PROJECT_SUB_QUERY_TO_MARK_CORRELATE, + CoreRules.PROJECT_MERGE, + CoreRules.PROJECT_REMOVE) + .withLateDecorrelate(true) + .withTopDownGeneralDecorrelate(true) + .check(); + } + + @Test void testTopDownGeneralDecorrelateForProjectScalar() { + final String sql = "SELECT empno, sal + " + + "(SELECT avg(sal) FROM empdefaults where emp.deptno = empdefaults.deptno) " + + "FROM emp"; + + sql(sql) + .withRule( + CoreRules.PROJECT_SUB_QUERY_TO_MARK_CORRELATE, + CoreRules.PROJECT_MERGE, + CoreRules.PROJECT_REMOVE) + .withLateDecorrelate(true) + .withTopDownGeneralDecorrelate(true) + .check(); + } + + @Test void testTopDownGeneralDecorrelateForProjectExists() { + final String sql = "SELECT dept.deptno, EXISTS ( SELECT 1 FROM emp e " + + "WHERE e.deptno = dept.deptno ) AS has_employees FROM dept"; + + sql(sql) + .withRule( + CoreRules.PROJECT_SUB_QUERY_TO_MARK_CORRELATE, + CoreRules.PROJECT_MERGE, + CoreRules.PROJECT_REMOVE) + .withLateDecorrelate(true) + .withTopDownGeneralDecorrelate(true) + .check(); + } + + @Test void testTopDownGeneralDecorrelateForProjectIn() { + final String sql = "SELECT emp.deptno, emp.deptno IN (SELECT dept.deptno FROM dept " + + "where dept.deptno < emp.empno ) FROM emp"; + + sql(sql) + .withRule( + CoreRules.PROJECT_SUB_QUERY_TO_MARK_CORRELATE, + CoreRules.PROJECT_MERGE, + CoreRules.PROJECT_REMOVE) + .withLateDecorrelate(true) + .withTopDownGeneralDecorrelate(true) + .check(); + } + + @Test void testTopDownGeneralDecorrelateForLateralJoin() { + final String sql = "select empno from emp,\n" + + " LATERAL (select * from dept where emp.deptno = dept.deptno)"; + + sql(sql) + .withRule( + CoreRules.PROJECT_MERGE, + CoreRules.PROJECT_REMOVE) + .withLateDecorrelate(true) + .withTopDownGeneralDecorrelate(true) + .check(); + } + + @Test void testTopDownGeneralDecorrelateForTwoLevelCorrelate() { + final String sql = "select empno from emp where " + + "exists(select * from emp_b where emp.ename = emp_b.ename and " + + "exists(select * from empnullables where emp.empno = empnullables.empno and " + + "emp_b.deptno = empnullables.deptno))"; + + sql(sql) + .withRule( + CoreRules.FILTER_SUB_QUERY_TO_MARK_CORRELATE, + CoreRules.PROJECT_MERGE, + CoreRules.PROJECT_REMOVE) + .withLateDecorrelate(true) + .withTopDownGeneralDecorrelate(true) + .check(); + } + + @Test void testTopDownGeneralDecorrelateForTwoLevelCorrelate2() { + final String sql = "select empno from emp where " + + "exists(select * from emp_b where emp.ename = emp_b.ename and " + + "exists(select * from empnullables where emp.empno = empnullables.empno))"; + + sql(sql) + .withRule( + CoreRules.FILTER_SUB_QUERY_TO_MARK_CORRELATE, + CoreRules.PROJECT_MERGE, + CoreRules.PROJECT_REMOVE) + .withLateDecorrelate(true) + .withTopDownGeneralDecorrelate(true) + .check(); + } + + @Test void testTopDownGeneralDecorrelateForTwoLevelCorrelate3() { + final String sql = "SELECT deptno FROM emp e WHERE EXISTS (SELECT * FROM dept d WHERE EXISTS " + + "(SELECT * FROM bonus ea WHERE ea.ENAME = e.ENAME AND d.deptno = e.deptno))"; + + sql(sql) + .withRule( + CoreRules.FILTER_SUB_QUERY_TO_MARK_CORRELATE, + CoreRules.PROJECT_MERGE, + CoreRules.PROJECT_REMOVE) + .withLateDecorrelate(true) + .withTopDownGeneralDecorrelate(true) + .check(); + } + + @Test void testTopDownGeneralDecorrelateForCannotRemoveD() { + final String sql = "select empno from emp where " + + "exists(select * from empnullables where emp.deptno > empnullables.deptno)"; + + sql(sql) + .withRule( + CoreRules.FILTER_SUB_QUERY_TO_MARK_CORRELATE, + CoreRules.PROJECT_MERGE, + CoreRules.PROJECT_REMOVE) + .withLateDecorrelate(true) + .withTopDownGeneralDecorrelate(true) + .check(); + } + + @Test void testTopDownGeneralDecorrelateForSubqueryWithSort() { + final String sql = "select empno from emp where " + + "sal > SOME(select sal from emp_b where emp.deptno = emp_b.deptno " + + "order by emp_b.sal limit 5)"; + + sql(sql) + .withRule( + CoreRules.FILTER_SUB_QUERY_TO_MARK_CORRELATE, + CoreRules.PROJECT_MERGE, + CoreRules.PROJECT_REMOVE) + .withLateDecorrelate(true) + .withTopDownGeneralDecorrelate(true) + .check(); + } + + @Test void testTopDownGeneralDecorrelateForSubqueryWithCube() { + final String sql = "select empno from emp where " + + "sal < SOME(select avg(sal) from emp_b where emp.job = emp_b.job group by cube(deptno))"; + + sql(sql) + .withRule( + CoreRules.FILTER_SUB_QUERY_TO_MARK_CORRELATE, + CoreRules.PROJECT_MERGE, + CoreRules.PROJECT_REMOVE) + .withLateDecorrelate(true) + .withTopDownGeneralDecorrelate(true) + .check(); + } + } diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index e1db15d0ce80..73f362b3f0b8 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -19587,6 +19587,734 @@ LogicalProject(EMPNO=[$0]) LogicalJoin(condition=[=($7, $8)], joinType=[semi]) LogicalTableScan(table=[[scott, EMP]]) LogicalTableScan(table=[[scott, DEPT]]) +]]> + + + + + empnullables.deptno)]]> + + + ($cor0.DEPTNO, $7)]) + LogicalTableScan(table=[[CATALOG, SALES, EMPNULLABLES]]) +})], variablesSet=[[$cor0]]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +]]> + + + ($cor0.DEPTNO, $7)]) + LogicalTableScan(table=[[CATALOG, SALES, EMPNULLABLES]]) +]]> + + + ($9, $7)], joinType=[inner]) + LogicalTableScan(table=[[CATALOG, SALES, EMPNULLABLES]]) + LogicalAggregate(group=[{0}]) + LogicalProject(DEPTNO=[$7]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + (select avg(sal) from emp_b where emp.ename = emp_b.ename)]]> + + + ($5, $SCALAR_QUERY({ +LogicalAggregate(group=[{}], EXPR$0=[AVG($0)]) + LogicalProject(SAL=[$5]) + LogicalFilter(condition=[=($cor0.ENAME, $1)]) + LogicalTableScan(table=[[CATALOG, SALES, EMP_B]]) +}))], variablesSet=[[$cor0]]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +]]> + + + ($5, $9)]) + LogicalCorrelate(correlation=[$cor0], joinType=[left], requiredColumns=[{1}]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) + LogicalAggregate(group=[{}], EXPR$0=[AVG($0)]) + LogicalProject(SAL=[$5]) + LogicalFilter(condition=[=($cor0.ENAME, $1)]) + LogicalTableScan(table=[[CATALOG, SALES, EMP_B]]) +]]> + + + ($5, $10))], joinType=[inner]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) + LogicalAggregate(group=[{1}], EXPR$0=[AVG($0)]) + LogicalProject(SAL=[$5], ENAME=[$1]) + LogicalTableScan(table=[[CATALOG, SALES, EMP_B]]) +]]> + + + + + SOME(select empno from emp_b where emp.ename = emp_b.ename)]]> + + + SOME($0, { +LogicalProject(EMPNO=[$0]) + LogicalFilter(condition=[=($cor0.ENAME, $1)]) + LogicalTableScan(table=[[CATALOG, SALES, EMP_B]]) +})], variablesSet=[[$cor0]]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +]]> + + + ($0, $9)]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) + LogicalProject(EMPNO=[$0]) + LogicalFilter(condition=[=($cor0.ENAME, $1)]) + LogicalTableScan(table=[[CATALOG, SALES, EMP_B]]) +]]> + + + ($0, $9), IS NOT DISTINCT FROM($1, $10))], joinType=[semi]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) + LogicalProject(EMPNO=[$0], ENAME=[$1]) + LogicalTableScan(table=[[CATALOG, SALES, EMP_B]]) +]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + SOME(select sal from empnullables, (select empno from emp_b where emp.deptno = emp_b.deptno) b where empnullables.empno = b.empno)]]> + + + SOME($5, { +LogicalProject(SAL=[$5]) + LogicalFilter(condition=[=($0, $9)]) + LogicalJoin(condition=[true], joinType=[inner]) + LogicalTableScan(table=[[CATALOG, SALES, EMPNULLABLES]]) + LogicalProject(EMPNO=[$0]) + LogicalFilter(condition=[=($cor0.DEPTNO, $7)]) + LogicalTableScan(table=[[CATALOG, SALES, EMP_B]]) +})], variablesSet=[[$cor0]]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +]]> + + + ($5, $9)]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) + LogicalProject(SAL=[$5]) + LogicalFilter(condition=[=($0, $9)]) + LogicalJoin(condition=[true], joinType=[inner]) + LogicalTableScan(table=[[CATALOG, SALES, EMPNULLABLES]]) + LogicalProject(EMPNO=[$0]) + LogicalFilter(condition=[=($cor0.DEPTNO, $7)]) + LogicalTableScan(table=[[CATALOG, SALES, EMP_B]]) +]]> + + + ($5, $9), IS NOT DISTINCT FROM($7, $10))], joinType=[semi]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) + LogicalProject(SAL=[$5], DEPTNO0=[$10]) + LogicalJoin(condition=[=($0, $9)], joinType=[inner]) + LogicalTableScan(table=[[CATALOG, SALES, EMPNULLABLES]]) + LogicalProject(EMPNO=[$0], DEPTNO=[$7]) + LogicalTableScan(table=[[CATALOG, SALES, EMP_B]]) +]]> + + + + + + + + + + + + + + + + + + + SOME(select sal from emp_b where emp.deptno = emp_b.deptno order by emp_b.sal limit 5)]]> + + + SOME($5, { +LogicalSort(sort0=[$0], dir0=[ASC], fetch=[5]) + LogicalProject(SAL=[$5]) + LogicalFilter(condition=[=($cor0.DEPTNO, $7)]) + LogicalTableScan(table=[[CATALOG, SALES, EMP_B]]) +})], variablesSet=[[$cor0]]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +]]> + + + ($5, $9)]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) + LogicalSort(sort0=[$0], dir0=[ASC], fetch=[5]) + LogicalProject(SAL=[$5]) + LogicalFilter(condition=[=($cor0.DEPTNO, $7)]) + LogicalTableScan(table=[[CATALOG, SALES, EMP_B]]) +]]> + + + ($5, $9), IS NOT DISTINCT FROM($7, $10))], joinType=[semi]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) + LogicalFilter(condition=[<=($2, 5)]) + LogicalProject(SAL=[$5], DEPTNO=[$7], $f2=[ROW_NUMBER() OVER (PARTITION BY $7 ORDER BY $5 NULLS LAST)]) + LogicalTableScan(table=[[CATALOG, SALES, EMP_B]]) +]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/testkit/src/main/java/org/apache/calcite/test/RelOptFixture.java b/testkit/src/main/java/org/apache/calcite/test/RelOptFixture.java index 55e8ba283cd1..68a6111d0629 100644 --- a/testkit/src/main/java/org/apache/calcite/test/RelOptFixture.java +++ b/testkit/src/main/java/org/apache/calcite/test/RelOptFixture.java @@ -41,6 +41,7 @@ import org.apache.calcite.sql.validate.SqlConformance; import org.apache.calcite.sql2rel.RelDecorrelator; import org.apache.calcite.sql2rel.SqlToRelConverter; +import org.apache.calcite.sql2rel.TopDownGeneralDecorrelator; import org.apache.calcite.test.catalog.MockCatalogReaderDynamic; import org.apache.calcite.tools.RelBuilder; import org.apache.calcite.util.Closer; @@ -81,7 +82,7 @@ public class RelOptFixture { static final RelOptFixture DEFAULT = new RelOptFixture(SqlToRelFixture.TESTER, SqlTestFactory.INSTANCE, null, RelSupplier.NONE, null, null, - ImmutableMap.of(), (f, r) -> r, (f, r) -> r, false, false) + ImmutableMap.of(), (f, r) -> r, (f, r) -> r, false, false, false) .withFactory(f -> f.withValidatorConfig(c -> c.withIdentifierExpansion(true)) .withSqlToRelConfig(c -> c.withExpand(false))) @@ -102,6 +103,7 @@ public class RelOptFixture { final BiFunction after; final boolean decorrelate; final boolean lateDecorrelate; + final boolean topDownGeneralDecorrelate; RelOptFixture(SqlTester tester, SqlTestFactory factory, @Nullable DiffRepository diffRepos, RelSupplier relSupplier, @@ -109,7 +111,7 @@ public class RelOptFixture { ImmutableMap> hooks, BiFunction before, BiFunction after, - boolean decorrelate, boolean lateDecorrelate) { + boolean decorrelate, boolean lateDecorrelate, boolean topDownGeneralDecorrelate) { this.tester = requireNonNull(tester, "tester"); this.factory = factory; this.diffRepos = diffRepos; @@ -121,6 +123,7 @@ public class RelOptFixture { this.hooks = requireNonNull(hooks, "hooks"); this.decorrelate = decorrelate; this.lateDecorrelate = lateDecorrelate; + this.topDownGeneralDecorrelate = topDownGeneralDecorrelate; } public RelOptFixture withDiffRepos(DiffRepository diffRepos) { @@ -129,7 +132,7 @@ public RelOptFixture withDiffRepos(DiffRepository diffRepos) { } return new RelOptFixture(tester, factory, diffRepos, relSupplier, preProgram, planner, hooks, before, after, decorrelate, - lateDecorrelate); + lateDecorrelate, topDownGeneralDecorrelate); } public RelOptFixture withRelSupplier(RelSupplier relSupplier) { @@ -138,7 +141,7 @@ public RelOptFixture withRelSupplier(RelSupplier relSupplier) { } return new RelOptFixture(tester, factory, diffRepos, relSupplier, preProgram, planner, hooks, before, after, decorrelate, - lateDecorrelate); + lateDecorrelate, topDownGeneralDecorrelate); } public RelOptFixture sql(String sql) { @@ -156,7 +159,7 @@ public RelOptFixture withBefore( (sql, r) -> transform.apply(this, before0.apply(this, r)); return new RelOptFixture(tester, factory, diffRepos, relSupplier, preProgram, planner, hooks, before, after, decorrelate, - lateDecorrelate); + lateDecorrelate, topDownGeneralDecorrelate); } public RelOptFixture withAfter( @@ -166,7 +169,7 @@ public RelOptFixture withAfter( (sql, r) -> transform.apply(this, after0.apply(this, r)); return new RelOptFixture(tester, factory, diffRepos, relSupplier, preProgram, planner, hooks, before, after, decorrelate, - lateDecorrelate); + lateDecorrelate, topDownGeneralDecorrelate); } public RelOptFixture withDynamicTable() { @@ -180,7 +183,7 @@ public RelOptFixture withFactory(UnaryOperator transform) { } return new RelOptFixture(tester, factory, diffRepos, relSupplier, preProgram, planner, hooks, before, after, decorrelate, - lateDecorrelate); + lateDecorrelate, topDownGeneralDecorrelate); } public RelOptFixture withPre(HepProgram preProgram) { @@ -189,7 +192,7 @@ public RelOptFixture withPre(HepProgram preProgram) { } return new RelOptFixture(tester, factory, diffRepos, relSupplier, preProgram, planner, hooks, before, after, decorrelate, - lateDecorrelate); + lateDecorrelate, topDownGeneralDecorrelate); } public RelOptFixture withPreRule(RelOptRule... rules) { @@ -206,7 +209,7 @@ public RelOptFixture withPlanner(RelOptPlanner planner) { } return new RelOptFixture(tester, factory, diffRepos, relSupplier, preProgram, planner, hooks, before, after, decorrelate, - lateDecorrelate); + lateDecorrelate, topDownGeneralDecorrelate); } public RelOptFixture withProgram(HepProgram program) { @@ -235,7 +238,7 @@ public RelOptFixture withHook(Hook hook, Consumer handler) { } return new RelOptFixture(tester, factory, diffRepos, relSupplier, preProgram, planner, hooks, before, after, decorrelate, - lateDecorrelate); + lateDecorrelate, topDownGeneralDecorrelate); } public RelOptFixture withProperty(Hook hook, V value) { @@ -270,7 +273,16 @@ public RelOptFixture withLateDecorrelate(final boolean lateDecorrelate) { } return new RelOptFixture(tester, factory, diffRepos, relSupplier, preProgram, planner, hooks, before, after, decorrelate, - lateDecorrelate); + lateDecorrelate, topDownGeneralDecorrelate); + } + + public RelOptFixture withTopDownGeneralDecorrelate(final boolean topDownGeneralDecorrelate) { + if (topDownGeneralDecorrelate == this.topDownGeneralDecorrelate) { + return this; + } + return new RelOptFixture(tester, factory, diffRepos, relSupplier, + preProgram, planner, hooks, before, after, decorrelate, + lateDecorrelate, topDownGeneralDecorrelate); } public RelOptFixture withDecorrelate(final boolean decorrelate) { @@ -279,7 +291,7 @@ public RelOptFixture withDecorrelate(final boolean decorrelate) { } return new RelOptFixture(tester, factory, diffRepos, relSupplier, preProgram, planner, hooks, before, after, decorrelate, - lateDecorrelate); + lateDecorrelate, topDownGeneralDecorrelate); } public RelOptFixture withTrim(final boolean trim) { @@ -389,7 +401,10 @@ private void checkPlanning(boolean unchanged) { assertThat(r3, relIsValid()); final RelBuilder relBuilder = RelFactories.LOGICAL_BUILDER.create(cluster, null); - r4 = RelDecorrelator.decorrelateQuery(r3, relBuilder); + r4 = + topDownGeneralDecorrelate + ? TopDownGeneralDecorrelator.decorrelateQuery(r3, relBuilder) + : RelDecorrelator.decorrelateQuery(r3, relBuilder); } else { r4 = r3; } From 61819fc90d0b0f00ae620a059f8123670773595a Mon Sep 17 00:00:00 2001 From: xuzifu666 Date: Wed, 10 Dec 2025 11:59:07 +0800 Subject: [PATCH 053/562] [CALCITE-7322] The POSITION function in MySQL is missing the FROM clause --- .../apache/calcite/sql/dialect/MysqlSqlDialect.java | 12 ++++++++---- .../calcite/rel/rel2sql/RelToSqlConverterTest.java | 10 +++++++++- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql/dialect/MysqlSqlDialect.java b/core/src/main/java/org/apache/calcite/sql/dialect/MysqlSqlDialect.java index f828e89fd7f9..7b4475283ea0 100644 --- a/core/src/main/java/org/apache/calcite/sql/dialect/MysqlSqlDialect.java +++ b/core/src/main/java/org/apache/calcite/sql/dialect/MysqlSqlDialect.java @@ -257,12 +257,16 @@ public MysqlSqlDialect(Context context) { int leftPrec, int rightPrec) { switch (call.getKind()) { case POSITION: - final SqlWriter.Frame frame = writer.startFunCall("INSTR"); - writer.sep(","); - call.operand(1).unparse(writer, leftPrec, rightPrec); + final SqlWriter.Frame f = writer.startFunCall("LOCATE"); writer.sep(","); call.operand(0).unparse(writer, leftPrec, rightPrec); - writer.endFunCall(frame); + writer.sep(","); + call.operand(1).unparse(writer, leftPrec, rightPrec); + if (call.operandCount() == 3) { + writer.sep(","); + call.operand(2).unparse(writer, leftPrec, rightPrec); + } + writer.endFunCall(f); break; case FLOOR: if (call.operandCount() != 2) { diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java index 915db7da6e51..1b34d42c2887 100644 --- a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java +++ b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java @@ -3600,11 +3600,19 @@ private SqlDialect nonOrdinalDialect() { sql(query).withHive().ok(expected); } + /** Test case for + * [CALCITE-7322] + * The POSITION function in MySQL is missing the FROM clause. */ @Test void testPositionFunctionForMySql() { final String query = "select position('A' IN 'ABC') from \"product\""; - final String expected = "SELECT INSTR('ABC', 'A')\n" + final String expected = "SELECT LOCATE('A', 'ABC')\n" + "FROM `foodmart`.`product`"; sql(query).withMysql().ok(expected); + + final String query1 = "select position('A' IN 'ABC' FROM '2') from \"product\""; + final String expected1 = "SELECT LOCATE('A', 'ABC', 2)\n" + + "FROM `foodmart`.`product`"; + sql(query1).withMysql().ok(expected1); } @Test void testPositionFunctionForBigQuery() { From 0b3e6ae21288218ad638cf85a1062b9562a45e6f Mon Sep 17 00:00:00 2001 From: Silun Date: Tue, 16 Dec 2025 12:54:58 +0800 Subject: [PATCH 054/562] Site: Add Silun Dong as committer --- site/_data/contributors.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/site/_data/contributors.yml b/site/_data/contributors.yml index c5f34d0a8aac..858e5137f9dc 100644 --- a/site/_data/contributors.yml +++ b/site/_data/contributors.yml @@ -314,6 +314,11 @@ githubId: suez1224 org: Uber role: Committer +- name: Silun Dong + apacheId: silun + githubId: silundong + org: + role: Committer - name: Slim Bouguerra apacheId: bslim githubId: b-slim From 0dfc2e8aebababd395d5f622db424e183e408bdf Mon Sep 17 00:00:00 2001 From: xuzifu666 Date: Tue, 16 Dec 2025 13:37:47 +0800 Subject: [PATCH 055/562] Site: Add Yu Xu as committer --- site/_data/contributors.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/site/_data/contributors.yml b/site/_data/contributors.yml index 858e5137f9dc..55a4b85b10dd 100644 --- a/site/_data/contributors.yml +++ b/site/_data/contributors.yml @@ -384,6 +384,11 @@ pronouns: he/him org: Hikvision role: PMC +- name: Yu Xu + apacheId: xuzifu666 + githubId: xuzifu666 + org: + role: Committer - name: Yong Liu apacheId: jackylau githubId: liuyongvs From 7e70edf7c97462b3e56f18f108455c1aa50eaeaa Mon Sep 17 00:00:00 2001 From: xuzifu666 Date: Tue, 16 Dec 2025 15:23:20 +0800 Subject: [PATCH 056/562] Site: Change the Position of Yu Xu for Committer message --- site/_data/contributors.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/site/_data/contributors.yml b/site/_data/contributors.yml index 55a4b85b10dd..f47ae3247bb3 100644 --- a/site/_data/contributors.yml +++ b/site/_data/contributors.yml @@ -384,17 +384,17 @@ pronouns: he/him org: Hikvision role: PMC -- name: Yu Xu - apacheId: xuzifu666 - githubId: xuzifu666 - org: - role: Committer - name: Yong Liu apacheId: jackylau githubId: liuyongvs pronouns: he/him org: Ant Financial role: Committer +- name: Yu Xu + apacheId: xuzifu666 + githubId: xuzifu666 + org: + role: Committer - name: Zhaohui Xu apacheId: zhaohui githubId: xy2953396112 From 76572f14eef6865c2bc7c243c0e08bb833a333ae Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Wed, 17 Dec 2025 07:06:44 +0800 Subject: [PATCH 057/562] [CALCITE-5347] Add 'SELECT ... BY', a syntax extension that is shorthand for GROUP BY and ORDER BY --- babel/src/main/codegen/config.fmpp | 3 +- .../org/apache/calcite/test/BabelTest.java | 141 ++++++++++++++++++ babel/src/test/resources/sql/select.iq | 74 +++++++++ core/src/main/codegen/templates/Parser.jj | 68 ++++++++- .../calcite/runtime/CalciteResource.java | 6 + .../org/apache/calcite/sql/SqlByRewriter.java | 83 +++++++++++ .../org/apache/calcite/sql/SqlSelect.java | 10 ++ .../java/org/apache/calcite/sql/SqlUtil.java | 44 ++++++ .../sql/validate/SqlValidatorImpl.java | 11 +- .../runtime/CalciteResource.properties | 2 + .../calcite/sql/test/SqlAdvisorTest.java | 1 + core/src/test/resources/sql/agg.iq | 6 +- site/_docs/reference.md | 26 +++- 13 files changed, 459 insertions(+), 16 deletions(-) create mode 100644 core/src/main/java/org/apache/calcite/sql/SqlByRewriter.java diff --git a/babel/src/main/codegen/config.fmpp b/babel/src/main/codegen/config.fmpp index b9c4a1c6ee61..001bdf2e1034 100644 --- a/babel/src/main/codegen/config.fmpp +++ b/babel/src/main/codegen/config.fmpp @@ -119,7 +119,7 @@ data: { "BOOLEAN" "BOTH" "BREADTH" - "BY" +# "BY" # "CALL" "CALLED" "CARDINALITY" @@ -618,6 +618,7 @@ data: { includeParsingStringLiteralAsArrayLiteral: true includeIntervalWithoutQualifier: true includeStarExclude: true + includeSelectBy: true } } diff --git a/babel/src/test/java/org/apache/calcite/test/BabelTest.java b/babel/src/test/java/org/apache/calcite/test/BabelTest.java index 261bcb24791e..09da75550854 100644 --- a/babel/src/test/java/org/apache/calcite/test/BabelTest.java +++ b/babel/src/test/java/org/apache/calcite/test/BabelTest.java @@ -17,15 +17,27 @@ package org.apache.calcite.test; import org.apache.calcite.config.CalciteConnectionProperty; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.rel2sql.RelToSqlConverter; import org.apache.calcite.rel.type.DelegatingTypeSystem; import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.rel.type.TimeFrameSet; +import org.apache.calcite.schema.SchemaPlus; +import org.apache.calcite.sql.SqlDialect; +import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlOperatorTable; +import org.apache.calcite.sql.dialect.CalciteSqlDialect; import org.apache.calcite.sql.fun.SqlLibrary; import org.apache.calcite.sql.fun.SqlLibraryOperatorTableFactory; +import org.apache.calcite.sql.parser.SqlParser; import org.apache.calcite.sql.parser.SqlParserFixture; import org.apache.calcite.sql.parser.babel.SqlBabelParserImpl; import org.apache.calcite.sql.validate.SqlConformanceEnum; +import org.apache.calcite.tools.FrameworkConfig; +import org.apache.calcite.tools.Frameworks; +import org.apache.calcite.tools.Planner; +import org.apache.calcite.tools.Programs; +import org.apache.calcite.util.TestUtil; import com.google.common.collect.ImmutableList; @@ -43,6 +55,8 @@ import java.util.function.UnaryOperator; import java.util.stream.Collectors; +import static org.apache.calcite.test.Matchers.isLinux; + import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; @@ -341,6 +355,133 @@ names, is( .type("RecordType(VARCHAR(10) NOT NULL NAME) NOT NULL"); } + /** Test case for + * [CALCITE-5347] + * Support parse "SELECT ... BY" in Babel parser. */ + @Test void testByClause() { + final SqlValidatorFixture v = Fixtures.forValidator() + .withParserConfig(c -> c.withParserFactory(SqlBabelParserImpl.FACTORY)) + .withConformance(SqlConformanceEnum.BABEL); + + // Test basic BY clause: SELECT a BY b is sugar for SELECT b, a GROUP BY b ORDER BY b + v.withSql("select ename, empno by deptno from emp").ok(); + // Test BY clause with alias + v.withSql("select ename, empno by deptno as dept from emp").ok(); + // Test BY clause with DESC modifier + v.withSql("select ename, empno by deptno DESC from emp").ok(); + // Test BY clause with multiple columns + v.withSql("select ename, empno by deptno, job from emp").ok(); + // Test complex BY clause example from the feature proposal + v.withSql("select e.ename, e.empno by d.name as dept DESC, e.job as title " + + "from emp as e join dept as d on e.deptno = d.deptno where d.name = 'SALES'") + .ok(); + + // Test SELECT BY cannot be used with GROUP BY + v.withSql("select ename by deptno from emp ^group by empno^") + .fails("SELECT BY cannot be used with GROUP BY"); + // Test SELECT BY cannot be used with ORDER BY + v.withSql("select ename by deptno from emp ^order by empno^") + .fails("SELECT BY cannot be used with ORDER BY"); + } + + /** Test case of + * [CALCITE-5347] + * Add 'SELECT ... BY', a syntax extension that is shorthand for GROUP BY and ORDER BY. */ + @Test void testByClauseConversion() { + // Test basic BY clause: SELECT a BY b is sugar for SELECT b, a GROUP BY b ORDER BY b + final String sql = "select ename, empno by deptno from emp"; + final String expected = "SELECT \"DEPTNO\"," + + " ANY_VALUE(\"ENAME\") AS \"ENAME\"," + + " ANY_VALUE(\"EMPNO\") AS \"EMPNO\"\n" + + "FROM \"SCOTT\".\"EMP\"\n" + + "GROUP BY \"DEPTNO\"\n" + + "ORDER BY \"DEPTNO\""; + checkSqlConversion(sql, expected); + + // Test BY clause with alias + final String sql2 = "select ename, empno by deptno as dept from emp"; + final String expected2 = "SELECT \"DEPTNO\" AS \"DEPT\"," + + " ANY_VALUE(\"ENAME\") AS \"ENAME\"," + + " ANY_VALUE(\"EMPNO\") AS \"EMPNO\"\n" + + "FROM \"SCOTT\".\"EMP\"\n" + + "GROUP BY \"DEPTNO\"\n" + + "ORDER BY \"DEPTNO\""; + checkSqlConversion(sql2, expected2); + + // Test BY clause with DESC modifier + final String sql3 = "select ename, empno by deptno DESC from emp"; + final String expected3 = "SELECT \"DEPTNO\"," + + " ANY_VALUE(\"ENAME\") AS \"ENAME\"," + + " ANY_VALUE(\"EMPNO\") AS \"EMPNO\"\n" + + "FROM \"SCOTT\".\"EMP\"\n" + + "GROUP BY \"DEPTNO\"\n" + + "ORDER BY \"DEPTNO\" DESC"; + checkSqlConversion(sql3, expected3); + + // Test BY clause with multiple columns + final String sql4 = "select ename, empno by deptno, job from emp"; + final String expected4 = "SELECT \"DEPTNO\", \"JOB\"," + + " ANY_VALUE(\"ENAME\") AS \"ENAME\"," + + " ANY_VALUE(\"EMPNO\") AS \"EMPNO\"\n" + + "FROM \"SCOTT\".\"EMP\"\n" + + "GROUP BY \"DEPTNO\", \"JOB\"\n" + + "ORDER BY \"DEPTNO\", \"JOB\""; + checkSqlConversion(sql4, expected4); + + // Test complex BY clause example from the feature proposal + final String sql5 = "SELECT e.ename, e.empno BY d.dname AS dept DESC, e.job AS title\n" + + "FROM emp AS e\n" + + " JOIN dept AS d ON e.deptno = d.deptno\n" + + "WHERE d.loc = 'CHICAGO'"; + final String expected5 = "SELECT \"DEPT\".\"DNAME\" AS \"DEPT\"," + + " \"EMP\".\"JOB\" AS \"TITLE\"," + + " ANY_VALUE(\"EMP\".\"ENAME\") AS \"ENAME\"," + + " ANY_VALUE(\"EMP\".\"EMPNO\") AS \"EMPNO\"\n" + + "FROM \"SCOTT\".\"EMP\"\n" + + "INNER JOIN \"SCOTT\".\"DEPT\" ON \"EMP\".\"DEPTNO\" = \"DEPT\".\"DEPTNO\"\n" + + "WHERE \"DEPT\".\"LOC\" = 'CHICAGO'\n" + + "GROUP BY \"DEPT\".\"DNAME\", \"EMP\".\"JOB\"\n" + + "ORDER BY \"DEPT\".\"DNAME\" DESC, \"EMP\".\"JOB\""; + checkSqlConversion(sql5, expected5); + } + + private void checkSqlConversion(String sql, String expected) { + try { + final SqlParser.Config parserConfig = SqlParser.config() + .withParserFactory(SqlBabelParserImpl.FACTORY); + + final SchemaPlus rootSchema = Frameworks.createRootSchema(true); + final SchemaPlus defaultSchema = + CalciteAssert.addSchema(rootSchema, CalciteAssert.SchemaSpec.JDBC_SCOTT); + + final FrameworkConfig config = Frameworks.newConfigBuilder() + .parserConfig(parserConfig) + .defaultSchema(defaultSchema) + .programs(Programs.standard()) + .build(); + + final Planner planner = Frameworks.getPlanner(config); + final SqlNode parse = planner.parse(sql); + final SqlNode validate = planner.validate(parse); + RelNode rel = planner.rel(validate).project(); + + final SqlDialect dialect = CalciteSqlDialect.DEFAULT; + final RelToSqlConverter converter = new RelToSqlConverter(dialect); + final SqlNode sqlNode = converter.visitRoot(rel).asStatement(); + final String actual = sqlNode.toSqlString(c -> + c.withDialect(dialect) + .withAlwaysUseParentheses(false) + .withSelectListItemsOnSeparateLines(false) + .withUpdateSetListNewline(false) + .withIndentation(0)) + .getSql(); + + assertThat(actual, isLinux(expected)); + } catch (Exception e) { + throw TestUtil.rethrow(e); + } + } + private void checkSqlResult(String funLibrary, String query, String result) { CalciteAssert.that() .with(CalciteConnectionProperty.PARSER_FACTORY, diff --git a/babel/src/test/resources/sql/select.iq b/babel/src/test/resources/sql/select.iq index 6b02cca8e36a..073daf8cd2b1 100755 --- a/babel/src/test/resources/sql/select.iq +++ b/babel/src/test/resources/sql/select.iq @@ -160,4 +160,78 @@ from emp e join dept d on e.deptno = d.deptno limit 1; !ok +# Test basic BY clause: SELECT a BY b is sugar for SELECT b, a GROUP BY b ORDER BY b +select ename, empno by deptno from emp; ++--------+--------+-------+ +| DEPTNO | ENAME | EMPNO | ++--------+--------+-------+ +| 10 | MILLER | 7934 | +| 20 | SMITH | 7902 | +| 30 | WARD | 7900 | ++--------+--------+-------+ +(3 rows) + +!ok + +# Test BY clause with alias +select ename, empno by deptno as dept from emp; ++------+--------+-------+ +| DEPT | ENAME | EMPNO | ++------+--------+-------+ +| 10 | MILLER | 7934 | +| 20 | SMITH | 7902 | +| 30 | WARD | 7900 | ++------+--------+-------+ +(3 rows) + +!ok + +# Test BY clause with DESC modifier +select ename, empno by deptno DESC from emp; ++--------+--------+-------+ +| DEPTNO | ENAME | EMPNO | ++--------+--------+-------+ +| 30 | WARD | 7900 | +| 20 | SMITH | 7902 | +| 10 | MILLER | 7934 | ++--------+--------+-------+ +(3 rows) + +!ok + +# Test BY clause with multiple columns +select ename, empno by deptno, job from emp; ++--------+-----------+--------+-------+ +| DEPTNO | JOB | ENAME | EMPNO | ++--------+-----------+--------+-------+ +| 10 | CLERK | MILLER | 7934 | +| 10 | MANAGER | CLARK | 7782 | +| 10 | PRESIDENT | KING | 7839 | +| 20 | ANALYST | SCOTT | 7902 | +| 20 | CLERK | SMITH | 7876 | +| 20 | MANAGER | JONES | 7566 | +| 30 | CLERK | JAMES | 7900 | +| 30 | MANAGER | BLAKE | 7698 | +| 30 | SALESMAN | WARD | 7844 | ++--------+-----------+--------+-------+ +(9 rows) + +!ok + +# Test complex BY clause example from the feature proposal +SELECT e.ename, e.empno BY d.dname AS dept DESC, e.job AS title +FROM emp AS e + JOIN dept AS d ON e.deptno = d.deptno +WHERE d.loc = 'CHICAGO'; ++-------+----------+-------+-------+ +| DEPT | TITLE | ENAME | EMPNO | ++-------+----------+-------+-------+ +| SALES | CLERK | JAMES | 7900 | +| SALES | MANAGER | BLAKE | 7698 | +| SALES | SALESMAN | WARD | 7844 | ++-------+----------+-------+-------+ +(3 rows) + +!ok + # End select.iq diff --git a/core/src/main/codegen/templates/Parser.jj b/core/src/main/codegen/templates/Parser.jj index a6f50f1f1280..ee3c1c759a98 100644 --- a/core/src/main/codegen/templates/Parser.jj +++ b/core/src/main/codegen/templates/Parser.jj @@ -90,6 +90,7 @@ import org.apache.calcite.sql.SqlPrefixOperator; import org.apache.calcite.sql.SqlRowTypeNameSpec; import org.apache.calcite.sql.SqlSampleSpec; import org.apache.calcite.sql.SqlSelect; +import org.apache.calcite.sql.SqlByRewriter; import org.apache.calcite.sql.SqlSelectKeyword; import org.apache.calcite.sql.SqlStarExclude; import org.apache.calcite.sql.SqlSetOption; @@ -724,6 +725,12 @@ SqlNode OrderByLimitOpt(SqlNode e) : ] { if (orderBy != null || offsetFetch[0] != null || offsetFetch[1] != null) { + if (orderBy != null + && e instanceof SqlSelect + && ((SqlSelect) e).hasByClause()) { + throw SqlUtil.newContextException(orderBy.getParserPosition(), + RESOURCE.selectByCannotWithOrderBy()); + } return new SqlOrderBy(getPos(), e, Util.first(orderBy, SqlNodeList.EMPTY), offsetFetch[0], offsetFetch[1]); @@ -1343,6 +1350,7 @@ SqlSelect SqlSelect() : final SqlNode having; final SqlNodeList windowDecls; final SqlNode qualify; + final SqlNodeList by; final List hints = new ArrayList(); final Span s; } @@ -1363,6 +1371,11 @@ SqlSelect SqlSelect() : } AddSelectItem(selectList) ( AddSelectItem(selectList) )* +<#if parser.includeSelectBy!false> + ( by = SqlSelectBy() | { by = null; } ) +<#else> + { by = null; } + ( fromClause = FromClause() ( where = Where() | { where = null; } ) @@ -1381,10 +1394,12 @@ SqlSelect SqlSelect() : } ) { - return new SqlSelect(s.end(this), keywordList, + final SqlSelect select = new SqlSelect(s.end(this), keywordList, new SqlNodeList(selectList, Span.of(selectList).pos()), fromClause, where, groupBy, having, windowDecls, qualify, null, null, null, new SqlNodeList(hints, getPos())); + SqlByRewriter.rewrite(select, by); + return select; } } @@ -2969,16 +2984,43 @@ SqlNodeList OrderBy(boolean accept) : throw SqlUtil.newContextException(s.pos(), RESOURCE.illegalOrderBy()); } } - AddOrderItem(list) - ( - // NOTE jvs 6-Feb-2004: See comments at top of file for why - // hint is necessary here. - LOOKAHEAD(2) AddOrderItem(list) - )* + OrderItemList(list) + { + return new SqlNodeList(list, s.addAll(list).pos()); + } +} + +<#if parser.includeSelectBy!false> +/** + * Parses a BY clause for SELECT (syntactic sugar for GROUP BY ... ORDER BY). + */ +SqlNodeList SqlSelectBy() : +{ + final List list = new ArrayList(); + final Span s; +} +{ + { s = span(); } + OrderItemList(list) { return new SqlNodeList(list, s.addAll(list).pos()); } } + + + +/** + * Parses a list of ORDER BY items. + */ +void OrderItemList(List list) : +{ +} +{ + AddOrderItem(list) + ( + LOOKAHEAD(2) AddOrderItem(list) + )* +} /** * Parses one item in an ORDER BY clause, and adds it to a list. @@ -2986,9 +3028,21 @@ SqlNodeList OrderBy(boolean accept) : void AddOrderItem(List list) : { SqlNode e; + final SqlIdentifier id; } { e = Expression(ExprContext.ACCEPT_SUB_QUERY) + ( + [ + ( + id = SimpleIdentifier() + | + LOOKAHEAD(1) + id = SimpleIdentifierFromStringLiteral() + ) + { e = SqlStdOperatorTable.AS.createCall(span().end(e), e, id); } + ] + ) ( | { diff --git a/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java b/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java index 6769f68283f2..33fd588a02d4 100644 --- a/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java +++ b/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java @@ -1171,4 +1171,10 @@ ExInst multipleCapturingGroupsForRegexpFunctions(String value, @BaseMessage("Cannot infer return type for {0}; operand types: {1}") ExInst cannotInferReturnType(String operator, String types); + + @BaseMessage("SELECT BY cannot be used with GROUP BY") + ExInst selectByCannotWithGroupBy(); + + @BaseMessage("SELECT BY cannot be used with ORDER BY") + ExInst selectByCannotWithOrderBy(); } diff --git a/core/src/main/java/org/apache/calcite/sql/SqlByRewriter.java b/core/src/main/java/org/apache/calcite/sql/SqlByRewriter.java new file mode 100644 index 000000000000..b8d6cc910e2e --- /dev/null +++ b/core/src/main/java/org/apache/calcite/sql/SqlByRewriter.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.sql; + +import org.checkerframework.checker.nullness.qual.Nullable; + +import java.util.ArrayList; +import java.util.List; + +import static org.apache.calcite.util.Static.RESOURCE; + +/** + * Rewrites the parser-level {@code SELECT ... BY} clause into equivalent + * {@code GROUP BY} and {@code ORDER BY} lists, and updates {@link SqlSelect} + * accordingly. + */ +public final class SqlByRewriter { + private SqlByRewriter() {} + + /** + * Rewrites {@code by} when it is specified on {@code select}. No action is + * taken when {@code by} is null or empty. + */ + public static void rewrite(SqlSelect select, @Nullable SqlNodeList by) { + if (by == null || by.isEmpty()) { + return; + } + ensureNoGroupBy(select); + ensureNoOrderBy(select); + + select.setHasByClause(true); + + final SqlNodeList selectList = select.getSelectList(); + final SqlNodeList groupBy = new SqlNodeList(by.getParserPosition()); + final SqlNodeList orderBy = new SqlNodeList(by.getParserPosition()); + final List extraSelectItems = new ArrayList<>(); + + for (SqlNode node : by) { + final SqlNode selectItem = SqlUtil.stripOrderModifiers(node); + extraSelectItems.add(selectItem); + + final SqlNode groupItem = SqlUtil.stripAs(selectItem); + groupBy.add(groupItem.clone(groupItem.getParserPosition())); + + final SqlNode orderItem = SqlUtil.stripAsFromOrder(node); + orderBy.add(orderItem.clone(orderItem.getParserPosition())); + } + + selectList.addAll(0, extraSelectItems); + select.setGroupBy(groupBy); + select.setOrderBy(orderBy); + } + + private static void ensureNoGroupBy(SqlSelect select) { + final SqlNodeList groupList = select.getGroup(); + if (groupList != null && !groupList.isEmpty()) { + throw SqlUtil.newContextException(groupList.getParserPosition(), + RESOURCE.selectByCannotWithGroupBy()); + } + } + + private static void ensureNoOrderBy(SqlSelect select) { + final SqlNodeList orderList = select.getOrderList(); + if (orderList != null && !orderList.isEmpty()) { + throw SqlUtil.newContextException(orderList.getParserPosition(), + RESOURCE.selectByCannotWithOrderBy()); + } + } +} diff --git a/core/src/main/java/org/apache/calcite/sql/SqlSelect.java b/core/src/main/java/org/apache/calcite/sql/SqlSelect.java index 3203bc84d5a8..0faa1d024f57 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlSelect.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlSelect.java @@ -56,6 +56,7 @@ public class SqlSelect extends SqlCall { @Nullable SqlNode offset; @Nullable SqlNode fetch; @Nullable SqlNodeList hints; + boolean hasByClause; //~ Constructors ----------------------------------------------------------- @@ -87,6 +88,7 @@ public SqlSelect(SqlParserPos pos, this.offset = offset; this.fetch = fetch; this.hints = hints; + this.hasByClause = false; } /** deprecated, without {@code qualify}. */ @@ -245,6 +247,14 @@ public void setOrderBy(@Nullable SqlNodeList orderBy) { this.orderBy = orderBy; } + void setHasByClause(boolean hasByClause) { + this.hasByClause = hasByClause; + } + + public boolean hasByClause() { + return hasByClause; + } + @Pure public final @Nullable SqlNode getOffset() { return offset; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlUtil.java b/core/src/main/java/org/apache/calcite/sql/SqlUtil.java index 107125af5f4e..342ed7752fe6 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlUtil.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlUtil.java @@ -1294,6 +1294,50 @@ public static boolean containsCall(SqlNode node, } } + /** + * Returns a copy of the sql node with ordering modifiers removed. + * + *

    Does not modify the input. Strips outermost {@code DESC}, + * {@code NULLS FIRST}, and {@code NULLS LAST} modifiers recursively. + */ + public static SqlNode stripOrderModifiers(SqlNode node) { + SqlNode expr = node; + while (expr instanceof SqlCall) { + final SqlCall call = (SqlCall) expr; + final SqlKind kind = call.getKind(); + if (kind == SqlKind.DESCENDING + || kind == SqlKind.NULLS_FIRST + || kind == SqlKind.NULLS_LAST) { + expr = call.operand(0); + } else { + break; + } + } + return expr; + } + + /** + * Returns a copy of the ORDER BY item with {@code AS} alias removed, + * preserving sort modifiers (DESC, NULLS FIRST/LAST). + */ + public static SqlNode stripAsFromOrder(SqlNode node) { + if (node instanceof SqlCall) { + SqlCall call = (SqlCall) node; + SqlKind kind = call.getKind(); + if (kind == SqlKind.DESCENDING + || kind == SqlKind.NULLS_FIRST + || kind == SqlKind.NULLS_LAST) { + SqlNode operand = call.operand(0); + SqlNode stripped = stripAsFromOrder(operand); + if (stripped != operand) { + return call.getOperator().createCall(call.getParserPosition(), stripped); + } + return node; + } + } + return stripAs(node); + } + //~ Inner Classes ---------------------------------------------------------- /** diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java index fd737039c4ab..1f9415ff7548 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java @@ -489,9 +489,10 @@ private boolean expandSelectItem(final SqlNode selectItem, SqlSelect select, selectScope = getSelectScope(select); expanded = expandSelectExpr(selectItem, scope, select, expansions); - // Non-strict GROUP BY: wrap non-aggregated, non-grouped columns in ANY_VALUE() + // Non-strict GROUP BY or BY clause: wrap non-aggregated, non-grouped columns in ANY_VALUE() if (isAggregate(select) - && config.conformance().isNonStrictGroupBy() + && (config.conformance().isNonStrictGroupBy() + || select.hasByClause()) && isNonAggregatedNonGroupedColumn(expanded, select)) { expanded = SqlStdOperatorTable.ANY_VALUE.createCall(expanded.getParserPosition(), expanded); @@ -554,7 +555,11 @@ private boolean isNonAggregatedNonGroupedColumn(SqlNode node, SqlSelect select) } if (node instanceof SqlCall) { - return ((SqlCall) node).getOperandList().stream() + final SqlCall call = (SqlCall) node; + if (call.getKind() == SqlKind.AS) { + return isNonAggregatedNonGroupedColumn(call.operand(0), select); + } + return call.getOperandList().stream() .anyMatch(operand -> isNonAggregatedNonGroupedColumn(operand, select)); } else if (node instanceof SqlLiteral) { return true; diff --git a/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties b/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties index 25531dd2cd5e..6cff62359856 100644 --- a/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties +++ b/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties @@ -383,4 +383,6 @@ IllegalRowIndexValue=ROW type does not have a field with index {0,number}; legal UnequalRowSizes=Unequal number of entries in ROW expressions IllegalRowIndex=Index in ROW type does not have a constant integer or string value CannotInferReturnType=Cannot infer return type for {0}; operand types: {1} +SelectByCannotWithGroupBy=SELECT BY cannot be used with GROUP BY +SelectByCannotWithOrderBy=SELECT BY cannot be used with ORDER BY # End CalciteResource.properties diff --git a/core/src/test/java/org/apache/calcite/sql/test/SqlAdvisorTest.java b/core/src/test/java/org/apache/calcite/sql/test/SqlAdvisorTest.java index e7a57630d146..1e24c9ede649 100644 --- a/core/src/test/java/org/apache/calcite/sql/test/SqlAdvisorTest.java +++ b/core/src/test/java/org/apache/calcite/sql/test/SqlAdvisorTest.java @@ -272,6 +272,7 @@ class SqlAdvisorTest extends SqlValidatorTestCase { private static final List ORDER_KEYWORDS = Arrays.asList( "KEYWORD(,)", + "KEYWORD(AS)", "KEYWORD(ASC)", "KEYWORD(DESC)", "KEYWORD(NULLS)"); diff --git a/core/src/test/resources/sql/agg.iq b/core/src/test/resources/sql/agg.iq index 3c8afd63ff3d..51e6221f1b78 100644 --- a/core/src/test/resources/sql/agg.iq +++ b/core/src/test/resources/sql/agg.iq @@ -4181,9 +4181,9 @@ from emp group by deptno, sal_col; (12 rows) !ok -EnumerableCalc(expr#0..8=[{inputs}], DEPTNO=[$t0], ENAME=[$t2], SAL_COL=[$t3], FULL_IDENTIFIER=[$t4], TWO_COLS_ONLY_ONE_IN_GROUP_BY=[$t5], SAL_PLUS_CONSTANT=[$t6], CONSTANT_3=[$t7], AGG_FUNC=[$t8]) - EnumerableAggregate(group=[{0, 1}], ENAME=[ANY_VALUE($2)], SAL_COL=[ANY_VALUE($1)], FULL_IDENTIFIER=[ANY_VALUE($3)], TWO_COLS_ONLY_ONE_IN_GROUP_BY=[ANY_VALUE($4)], SAL_PLUS_CONSTANT=[ANY_VALUE($5)], CONSTANT_3=[ANY_VALUE($6)], AGG_FUNC=[MAX($1)]) - EnumerableCalc(expr#0..7=[{inputs}], expr#8=[+($t5, $t6)], expr#9=[1], expr#10=[+($t5, $t9)], expr#11=[3], DEPTNO=[$t7], SAL=[$t5], ENAME=[$t1], JOB=[$t2], $f4=[$t8], $f5=[$t10], $f6=[$t11]) +EnumerableCalc(expr#0..7=[{inputs}], DEPTNO=[$t0], ENAME=[$t2], SAL_COL=[$t1], FULL_IDENTIFIER=[$t3], TWO_COLS_ONLY_ONE_IN_GROUP_BY=[$t4], SAL_PLUS_CONSTANT=[$t5], CONSTANT_3=[$t6], AGG_FUNC=[$t7]) + EnumerableAggregate(group=[{0, 1}], ENAME=[ANY_VALUE($2)], FULL_IDENTIFIER=[ANY_VALUE($3)], TWO_COLS_ONLY_ONE_IN_GROUP_BY=[ANY_VALUE($4)], SAL_PLUS_CONSTANT=[ANY_VALUE($5)], CONSTANT_3=[ANY_VALUE($6)], AGG_FUNC=[MAX($1)]) + EnumerableCalc(expr#0..7=[{inputs}], expr#8=[+($t5, $t6)], expr#9=[1], expr#10=[+($t5, $t9)], expr#11=[3], DEPTNO=[$t7], SAL_COL=[$t5], ENAME=[$t1], JOB=[$t2], $f4=[$t8], $f5=[$t10], $f6=[$t11]) EnumerableTableScan(table=[[scott, EMP]]) !plan diff --git a/site/_docs/reference.md b/site/_docs/reference.md index 7017d4565df8..617ce1a83bec 100644 --- a/site/_docs/reference.md +++ b/site/_docs/reference.md @@ -205,8 +205,9 @@ orderItem: expression [ ASC | DESC ] [ NULLS FIRST | NULLS LAST ] select: - SELECT [ hintComment ] [ STREAM ] [ ALL | DISTINCT ] - { starWithExclude | projectItem [, projectItem ]* } + SELECT [ hintComment ] [ STREAM ] [ ALL | DISTINCT ] + { starWithExclude | projectItem [, projectItem ]* } + [ BY expression [, expression ]* ] FROM tableExpression [ WHERE booleanExpression ] [ GROUP BY [ ALL | DISTINCT ] { groupItem [, groupItem ]* } ] @@ -214,6 +215,27 @@ select: [ WINDOW windowName AS windowSpec [, windowName AS windowSpec ]* ] [ QUALIFY booleanExpression ] +The optional, non-standard `BY` clause groups and orders the query by +the specified expressions, and automatically adds them to the SELECT list +for naming and positional reference. But `SELECT ... BY` cannot be combined +with an explicit `GROUP BY` or `ORDER BY` clause in the same query. +`SELECT ... BY` is recognized only when the Babel parser is enabled. It sets the generated parser configuration flag `includeSelectBy` to `true`. + +For example: + +{% highlight sql %} +SELECT ename, empno BY deptno FROM emp +{% endhighlight %} + +is equivalent to: + +{% highlight sql %} +SELECT deptno, ename, empno +FROM emp +GROUP BY deptno +ORDER BY deptno +{% endhighlight %} + selectWithoutFrom: SELECT [ ALL | DISTINCT ] { * | projectItem [, projectItem ]* } From 3ef8ba9226df5051f5571c402ec7a6c25bcd1c4e Mon Sep 17 00:00:00 2001 From: xuzifu666 Date: Wed, 10 Dec 2025 19:20:26 +0800 Subject: [PATCH 058/562] [CALCITE-7323] Result of cast Number to Boolean is not correct --- .../enumerable/RexToLixTranslator.java | 14 +++++++++ .../apache/calcite/runtime/SqlFunctions.java | 12 ++++++++ .../apache/calcite/util/BuiltInMethod.java | 1 + .../rel/rel2sql/RelToSqlConverterTest.java | 4 +++ .../apache/calcite/rex/RexProgramTest.java | 4 ++- core/src/test/resources/sql/cast.iq | 24 +++++++++++++++ .../apache/calcite/test/SqlOperatorTest.java | 30 +++++++++++++++++++ 7 files changed, 88 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java index e82cde56a9af..7e5f1948ab41 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java @@ -428,6 +428,20 @@ private Expression getConvertExpression( case CHAR: case VARCHAR: return Expressions.call(BuiltInMethod.STRING_TO_BOOLEAN.method, operand); + // Numberic type handle + case TINYINT: + case SMALLINT: + case INTEGER: + case BIGINT: + case UTINYINT: + case USMALLINT: + case UINTEGER: + case UBIGINT: + case DECIMAL: + case FLOAT: + case REAL: + case DOUBLE: + return Expressions.call(BuiltInMethod.NUMBER_TO_BOOLEAN.method, operand); default: return defaultExpression.get(); diff --git a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java index ede46373b83e..52031cf494af 100644 --- a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java +++ b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java @@ -4735,6 +4735,18 @@ public static boolean toBoolean(String s) { } public static boolean toBoolean(Number number) { + if (number instanceof BigDecimal) { + BigDecimal decimal = (BigDecimal) number; + return decimal.compareTo(BigDecimal.ZERO) != 0; + } + if (number instanceof Double) { + Double d = (Double) number; + return !d.equals(Double.valueOf(0)); + } + if (number instanceof Float) { + Float f = (Float) number; + return !f.equals(Float.valueOf(0)); + } return !number.equals(0); } diff --git a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java index 5fec119d65bb..14cb12d3e7eb 100644 --- a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java +++ b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java @@ -682,6 +682,7 @@ public enum BuiltInMethod { "getModifiableCollection"), SCANNABLE_TABLE_SCAN(ScannableTable.class, "scan", DataContext.class), STRING_TO_BOOLEAN(SqlFunctions.class, "toBoolean", String.class), + NUMBER_TO_BOOLEAN(SqlFunctions.class, "toBoolean", Number.class), INTERNAL_TO_DATE(SqlFunctions.class, "internalToDate", int.class), INTERNAL_TO_TIME(SqlFunctions.class, "internalToTime", int.class), INTERNAL_TO_TIMESTAMP(SqlFunctions.class, "internalToTimestamp", long.class), diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java index 1b34d42c2887..d2125c0ec8b0 100644 --- a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java +++ b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java @@ -771,6 +771,10 @@ private static String toSql(RelNode root, SqlDialect dialect, query = "select FALSE = 0.0e0"; expected = "SELECT *\nFROM (VALUES (TRUE)) AS \"t\" (\"EXPR$0\")"; sql(query).ok(expected); + + query = "select cast(\"product_id\" as BOOLEAN) from \"product\""; + expected = "SELECT \"product_id\" <> 0\nFROM \"foodmart\".\"product\""; + sql(query).ok(expected); } @Test void testSelectQueryWithWhereClauseOfBasicOperators() { diff --git a/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java b/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java index 877878aedef6..3daf4b896226 100644 --- a/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java +++ b/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java @@ -3152,7 +3152,9 @@ private SqlOperator getNoDeterministicOperator() { checkSimplify(cast(literal(1), varcharType), "'1':VARCHAR(10)"); checkSimplifyUnchanged(cast(literalAbc, booleanType)); checkSimplify(cast(literal(1), booleanType), - "false"); // different from Hive + "true"); + checkSimplify(cast(literal(0), booleanType), + "false"); checkSimplifyUnchanged(cast(literalAbc, dateType)); checkSimplify(cast(literal(1), dateType), "1970-01-02"); // different from Hive diff --git a/core/src/test/resources/sql/cast.iq b/core/src/test/resources/sql/cast.iq index 95d11e5b5a44..de08cd380792 100644 --- a/core/src/test/resources/sql/cast.iq +++ b/core/src/test/resources/sql/cast.iq @@ -1978,4 +1978,28 @@ SELECT ARRAY[cast(null as integer), cast(null as integer), cast(null as integer) EXPR$0 INTEGER ARRAY NOT NULL !type +# [CALCITE-7323] Result of cast Number to Boolean is not correct +select cast(mod(deptno, 3) as BOOLEAN), mod(deptno, 3) from emp; ++--------+--------+ +| EXPR$0 | EXPR$1 | ++--------+--------+ +| false | 0 | +| false | 0 | +| false | 0 | +| false | 0 | +| false | 0 | +| false | 0 | +| true | 1 | +| true | 1 | +| true | 1 | +| true | 2 | +| true | 2 | +| true | 2 | +| true | 2 | +| true | 2 | ++--------+--------+ +(14 rows) + +!ok + # End cast.iq diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java index 351f44584f27..90839a1daa7c 100644 --- a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java +++ b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java @@ -825,6 +825,36 @@ void testCastToExactNumeric(CastType castType, SqlOperatorFixture f) { f.checkNull("CAST(CAST(NULL AS VARCHAR) AS VARBINARY)"); } + /** + * Test case for + * Result of cast Number to Boolean is not correct. */ + @Test public void testNumbericBooleanCast() { + SqlOperatorFixture f = fixture(); + f.setFor(SqlStdOperatorTable.MAX, VM_EXPAND); + String[] values = {"0", "CAST(null AS INTEGER)", "2", "2"}; + f.checkAgg("cast(max(x) as BOOLEAN)", values, isSingle(true)); + String[] values1 = {"0", "CAST(null AS INTEGER)", "0", "-2"}; + f.checkAgg("cast(max(x) as BOOLEAN)", values1, isSingle(false)); + f.checkScalar("CAST(1 AS BOOLEAN)", "true", "BOOLEAN NOT NULL"); + f.checkScalar("CAST(2 AS BOOLEAN)", "true", "BOOLEAN NOT NULL"); + f.checkScalar("CAST(abs(2) AS BOOLEAN)", "true", "BOOLEAN NOT NULL"); + f.checkScalar("CAST(abs(0) AS BOOLEAN)", "false", "BOOLEAN NOT NULL"); + f.checkScalar("CAST(-1 AS BOOLEAN)", "true", "BOOLEAN NOT NULL"); + f.checkScalar("CAST(1.2 AS BOOLEAN)", "true", "BOOLEAN NOT NULL"); + f.checkScalar("CAST(CAST(100.5e0 AS DECIMAL(4, 1)) AS BOOLEAN)", "true", "BOOLEAN NOT NULL"); + f.checkScalar("CAST(0 AS BOOLEAN)", "false", "BOOLEAN NOT NULL"); + f.checkScalar("CAST(0.0 AS BOOLEAN)", "false", "BOOLEAN NOT NULL"); + f.checkScalar("CAST(0.0e0 AS BOOLEAN)", "false", "BOOLEAN NOT NULL"); + f.checkScalar("CAST(0.01e0 AS BOOLEAN)", "true", "BOOLEAN NOT NULL"); + f.checkScalar("CAST(0e0 AS BOOLEAN)", "false", "BOOLEAN NOT NULL"); + f.checkScalar("CAST(-0e0 AS BOOLEAN)", "false", "BOOLEAN NOT NULL"); + f.checkNull("CAST(NULL AS BOOLEAN)"); + f.checkScalar("CAST(CAST(1.2 AS FLOAT) AS BOOLEAN)", "true", "BOOLEAN NOT NULL"); + f.checkScalar("CAST(CAST(1.2 AS Double) AS BOOLEAN)", "true", "BOOLEAN NOT NULL"); + f.checkScalar("CAST(cast(1 as INTEGER UNSIGNED) AS BOOLEAN)", "true", "BOOLEAN NOT NULL"); + f.checkScalar("CAST(cast(1 as TINYINT UNSIGNED) AS BOOLEAN)", "true", "BOOLEAN NOT NULL"); + } + @ParameterizedTest @MethodSource("safeParameters") void testCastStringToDecimal(CastType castType, SqlOperatorFixture f) { From 0609ef51af8d3ec9677b84659f0955018b0a258d Mon Sep 17 00:00:00 2001 From: Silun Date: Thu, 11 Dec 2025 14:42:12 +0800 Subject: [PATCH 059/562] [CALCITE-7327] Support IS NOT DISTINCT FROM as equi condition of hash join --- .../enumerable/EnumerableHashJoin.java | 12 +- .../enumerable/EnumerableMergeJoin.java | 10 ++ .../enumerable/EnumerableMergeJoinRule.java | 5 +- .../calcite/adapter/enumerable/PhysType.java | 9 ++ .../adapter/enumerable/PhysTypeImpl.java | 50 +++++++ .../org/apache/calcite/plan/RelOptUtil.java | 21 ++- .../org/apache/calcite/rel/core/Join.java | 2 +- .../org/apache/calcite/rel/core/JoinInfo.java | 36 +++-- .../rel/rules/LoptOptimizeJoinRule.java | 2 +- .../rel/rules/LoptSemiJoinOptimizer.java | 2 +- .../calcite/rel/rules/SemiJoinRule.java | 4 +- .../org/apache/calcite/runtime/FlatLists.java | 13 ++ .../apache/calcite/util/BuiltInMethod.java | 1 + .../org/apache/calcite/test/JdbcTest.java | 2 +- .../enumerable/EnumerableHashJoinTest.java | 131 ++++++++++++++++++ core/src/test/resources/sql/blank.iq | 4 +- core/src/test/resources/sql/planner.iq | 17 ++- core/src/test/resources/sql/sub-query.iq | 32 ++--- .../calcite/linq4j/EnumerableDefaults.java | 52 +++---- 19 files changed, 325 insertions(+), 80 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableHashJoin.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableHashJoin.java index 8bd9ffbf08db..e37173137aef 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableHashJoin.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableHashJoin.java @@ -218,8 +218,10 @@ private Result implementHashSemiJoin(EnumerableRelImplementor implementor, Prefe Expressions.list( leftExpression, rightExpression, - leftResult.physType.generateAccessorWithoutNulls(joinInfo.leftKeys), - rightResult.physType.generateAccessorWithoutNulls(joinInfo.rightKeys), + leftResult.physType.generateNullAwareAccessor( + joinInfo.leftKeys, joinInfo.nullExclusionFlags), + rightResult.physType.generateNullAwareAccessor( + joinInfo.rightKeys, joinInfo.nullExclusionFlags), Util.first(keyPhysType.comparer(), Expressions.constant(null)), predicate))) @@ -264,8 +266,10 @@ private Result implementHashJoin(EnumerableRelImplementor implementor, Prefer pr BuiltInMethod.HASH_JOIN.method, Expressions.list( rightExpression, - leftResult.physType.generateAccessorWithoutNulls(joinInfo.leftKeys), - rightResult.physType.generateAccessorWithoutNulls(joinInfo.rightKeys), + leftResult.physType.generateNullAwareAccessor( + joinInfo.leftKeys, joinInfo.nullExclusionFlags), + rightResult.physType.generateNullAwareAccessor( + joinInfo.rightKeys, joinInfo.nullExclusionFlags), EnumUtils.joinSelector(joinType, physType, ImmutableList.of( diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeJoin.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeJoin.java index 838e37064f45..557362e6bc29 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeJoin.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeJoin.java @@ -34,6 +34,7 @@ import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.CorrelationId; import org.apache.calcite.rel.core.Join; +import org.apache.calcite.rel.core.JoinInfo; import org.apache.calcite.rel.core.JoinRelType; import org.apache.calcite.rel.metadata.RelMdCollation; import org.apache.calcite.rel.metadata.RelMetadataQuery; @@ -71,6 +72,9 @@ * {@link EnumerableConvention enumerable calling convention} using * a merge algorithm. */ public class EnumerableMergeJoin extends Join implements EnumerableRel { + @SuppressWarnings("HidingField") + private final JoinInfo joinInfo; + protected EnumerableMergeJoin( RelOptCluster cluster, RelTraitSet traits, @@ -80,6 +84,12 @@ protected EnumerableMergeJoin( Set variablesSet, JoinRelType joinType) { super(cluster, traits, ImmutableList.of(), left, right, condition, variablesSet, joinType); + // TODO: support IS NOT DISTINCT FROM condition as join keys of MergeJoin + // EnumerableMergeJoin cannot use IS NOT DISTINCT FROM condition as join keys + // (In the algorithm of MergeJoin in Enumerable convention, it will stop + // when leftKey or rightKey is NULL), so we create a new JoinInfo that only + // considers EQUALS. + this.joinInfo = JoinInfo.createWithStrictEquality(left, right, condition); assert getConvention() instanceof EnumerableConvention; final List leftCollations = getCollations(left.getTraitSet()); final List rightCollations = getCollations(right.getTraitSet()); diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeJoinRule.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeJoinRule.java index 4137f6b74bb1..624db0a60483 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeJoinRule.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeJoinRule.java @@ -60,7 +60,10 @@ protected EnumerableMergeJoinRule(Config config) { @Override public @Nullable RelNode convert(RelNode rel) { Join join = (Join) rel; - final JoinInfo info = join.analyzeCondition(); + // EnumerableMergeJoin cannot use IS NOT DISTINCT FROM condition as join keys. More details + // in EnumerableMergeJoin.java. + final JoinInfo info = + JoinInfo.createWithStrictEquality(join.getLeft(), join.getRight(), join.getCondition()); if (!EnumerableMergeJoin.isMergeJoinSupported(join.getJoinType())) { // EnumerableMergeJoin only supports certain join types. return null; diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/PhysType.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/PhysType.java index 8d4eeb176ce9..c50d6237dfe5 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/PhysType.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/PhysType.java @@ -134,6 +134,15 @@ Expression fieldReference(Expression expression, int field, */ Expression generateAccessorWithoutNulls(List fields); + /** + * Similar to {@link #generateAccessor(List)} and {@link #generateAccessorWithoutNulls(List)}, + * but it's null-aware. It returns a Expression which evaluates to null (if one of + * field is null and it isn't null-safe) or a list of + * fields that may contain null (no field is null, or there are fields with null but they are + * null-safe) at runtime. + */ + Expression generateNullAwareAccessor(List fields, List nullExclusionFlags); + /** Generates a selector for the given fields from an expression, with the * default row format. */ Expression generateSelector( diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/PhysTypeImpl.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/PhysTypeImpl.java index ab51dd3e35da..e3fdd4d36656 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/PhysTypeImpl.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/PhysTypeImpl.java @@ -642,6 +642,21 @@ private List fieldReferences( } } + private static Expression getListExpressionAllowSingleElement( + Expressions.FluentList list) { + assert list.size() > 0; + + if (list.size() == 1) { + return Expressions.call( + List.class, + null, + BuiltInMethod.LIST1.method, + list); + } else { + return getListExpression(list); + } + } + private static Expression getListExpression(Expressions.FluentList list) { assert list.size() >= 2; @@ -713,6 +728,41 @@ private static Expression getListExpression(Expressions.FluentList l return Expressions.lambda(Function1.class, exp, v1); } + @Override public Expression generateNullAwareAccessor( + List fields, + List nullExclusionFlags) { + assert fields.size() == nullExclusionFlags.size(); + ParameterExpression v1 = Expressions.parameter(javaRowClass, "v1"); + if (fields.isEmpty()) { + return Expressions.lambda( + Function1.class, + Expressions.field( + null, + BuiltInMethod.COMPARABLE_EMPTY_LIST.field), + v1); + } + Expressions.FluentList list = Expressions.list(); + for (int field : fields) { + list.add(fieldReference(v1, field)); + } + + // in the HashJoin key selector scenario, when there is exactly one join key and it is + // null-safe, a row whose join key is null must still be correctly recognized and extracted. + // Therefore, when list.size() == 1, this method returns a list containing a single + // element (which may be null) rather than returning the element directly. + Expression exp = getListExpressionAllowSingleElement(list); + for (int i = list.size() - 1; i >= 0; i--) { + if (nullExclusionFlags.get(i)) { + exp = + Expressions.condition( + Expressions.equal(list.get(i), Expressions.constant(null)), + Expressions.constant(null), + exp); + } + } + return Expressions.lambda(Function1.class, exp, v1); + } + @Override public Expression fieldReference( Expression expression, int field) { return fieldReference(expression, field, null); diff --git a/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java b/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java index 34724a8adfbc..274e91f14de1 100644 --- a/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java +++ b/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java @@ -1470,11 +1470,24 @@ private static void splitJoinCondition( nonEquiList.add(condition); } - /** Builds an equi-join condition from a set of left and right keys. */ + /** Builds an equi-join condition by conjoining EQUALS operator for each corresponding pair of + * leftKeys and rightKeys. */ public static RexNode createEquiJoinCondition( final RelNode left, final List leftKeys, final RelNode right, final List rightKeys, final RexBuilder rexBuilder) { + List filterNulls = Collections.nCopies(leftKeys.size(), Boolean.TRUE); + return createHashJoinCondition(left, leftKeys, right, rightKeys, + filterNulls, rexBuilder); + } + + /** Builds an equi-join condition by conjoining operators for each corresponding pair of + * leftKeys and rightKeys. The operator is EQUALS if filterNulls is true for that + * position, otherwise IS NOT DISTINCT FROM. */ + public static RexNode createHashJoinCondition( + final RelNode left, final List leftKeys, + final RelNode right, final List rightKeys, + final List filterNulls, final RexBuilder rexBuilder) { final List leftTypes = RelOptUtil.getFieldTypeList(left.getRowType()); final List rightTypes = @@ -1484,7 +1497,11 @@ public static RexNode createEquiJoinCondition( @Override public RexNode get(int index) { final int leftKey = leftKeys.get(index); final int rightKey = rightKeys.get(index); - return rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, + final SqlOperator operator = + filterNulls.get(index) + ? SqlStdOperatorTable.EQUALS + : SqlStdOperatorTable.IS_NOT_DISTINCT_FROM; + return rexBuilder.makeCall(operator, rexBuilder.makeInputRef(leftTypes.get(leftKey), leftKey), rexBuilder.makeInputRef(rightTypes.get(rightKey), leftTypes.size() + rightKey)); diff --git a/core/src/main/java/org/apache/calcite/rel/core/Join.java b/core/src/main/java/org/apache/calcite/rel/core/Join.java index 1a56edbed022..1b81717aedde 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/Join.java +++ b/core/src/main/java/org/apache/calcite/rel/core/Join.java @@ -104,7 +104,7 @@ protected Join( this.condition = requireNonNull(condition, "condition"); this.variablesSet = ImmutableSet.copyOf(variablesSet); this.joinType = requireNonNull(joinType, "joinType"); - this.joinInfo = JoinInfo.createWithStrictEquality(left, right, condition); + this.joinInfo = JoinInfo.of(left, right, condition); this.hints = ImmutableList.copyOf(hints); } diff --git a/core/src/main/java/org/apache/calcite/rel/core/JoinInfo.java b/core/src/main/java/org/apache/calcite/rel/core/JoinInfo.java index f092a73ea939..0d470cdde2d2 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/JoinInfo.java +++ b/core/src/main/java/org/apache/calcite/rel/core/JoinInfo.java @@ -29,6 +29,7 @@ import com.google.common.collect.ImmutableList; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import static java.util.Objects.requireNonNull; @@ -36,9 +37,10 @@ /** An analyzed join condition. * *

    It is useful for the many algorithms that care whether a join has an - * equi-join condition. + * equi-join condition (contains EQUALS and IS NOT DISTINCT FROM). * - *

    You can create one using {@link #createWithStrictEquality}, or call + *

    You can create one using {@link #of(RelNode, RelNode, RexNode)}, + * {@link #createWithStrictEquality}, or call * {@link Join#analyzeCondition()}; many kinds of join cache their * join info, especially those that are equi-joins. * @@ -46,28 +48,36 @@ public class JoinInfo { public final ImmutableIntList leftKeys; public final ImmutableIntList rightKeys; + // for each join key, whether it filters out nulls. If TRUE, the join key uses EQUALS semantics + // (not null-safe); if FALSE, it uses IS NOT DISTINCT FROM semantics (null-safe). + public final ImmutableList nullExclusionFlags; + // non-equi parts of join condition. + // after CALCITE-7327, IS NOT DISTINCT FROM can be treated as a hash join key and is no longer + // part of nonEquiConditions. public final ImmutableList nonEquiConditions; /** Creates a JoinInfo. */ protected JoinInfo(ImmutableIntList leftKeys, ImmutableIntList rightKeys, - ImmutableList nonEquiConditions) { + ImmutableList nullExclusionFlags, ImmutableList nonEquiConditions) { this.leftKeys = requireNonNull(leftKeys, "leftKeys"); this.rightKeys = requireNonNull(rightKeys, "rightKeys"); + this.nullExclusionFlags = requireNonNull(nullExclusionFlags, "nullExclusionFlags"); this.nonEquiConditions = requireNonNull(nonEquiConditions, "nonEquiConditions"); - assert leftKeys.size() == rightKeys.size(); + assert leftKeys.size() == rightKeys.size() && leftKeys.size() == nullExclusionFlags.size(); } /** Creates a {@code JoinInfo} by analyzing a condition. */ public static JoinInfo of(RelNode left, RelNode right, RexNode condition) { final List leftKeys = new ArrayList<>(); final List rightKeys = new ArrayList<>(); - final List filterNulls = new ArrayList<>(); + final List nullExclusionFlags = new ArrayList<>(); final List nonEquiList = new ArrayList<>(); RelOptUtil.splitJoinCondition(left, right, condition, leftKeys, rightKeys, - filterNulls, nonEquiList); + nullExclusionFlags, nonEquiList); return new JoinInfo(ImmutableIntList.copyOf(leftKeys), - ImmutableIntList.copyOf(rightKeys), ImmutableList.copyOf(nonEquiList)); + ImmutableIntList.copyOf(rightKeys), ImmutableList.copyOf(nullExclusionFlags), + ImmutableList.copyOf(nonEquiList)); } /** Creates a {@code JoinInfo} by analyzing a condition. @@ -82,14 +92,18 @@ public static JoinInfo createWithStrictEquality(RelNode left, final List nonEquiList = new ArrayList<>(); RelOptUtil.splitJoinCondition(left, right, condition, leftKeys, rightKeys, null, nonEquiList); + List nullExclusionFlags = Collections.nCopies(leftKeys.size(), Boolean.TRUE); return new JoinInfo(ImmutableIntList.copyOf(leftKeys), - ImmutableIntList.copyOf(rightKeys), ImmutableList.copyOf(nonEquiList)); + ImmutableIntList.copyOf(rightKeys), ImmutableList.copyOf(nullExclusionFlags), + ImmutableList.copyOf(nonEquiList)); } - /** Creates an equi-join. */ + /** Creates an equi-join (only considers EQUALS operations). */ public static JoinInfo of(ImmutableIntList leftKeys, ImmutableIntList rightKeys) { - return new JoinInfo(leftKeys, rightKeys, ImmutableList.of()); + List nullExclusionFlags = Collections.nCopies(leftKeys.size(), Boolean.TRUE); + return new JoinInfo(leftKeys, rightKeys, + ImmutableList.copyOf(nullExclusionFlags), ImmutableList.of()); } /** Returns whether this is an equi-join. */ @@ -117,7 +131,7 @@ public RexNode getRemaining(RexBuilder rexBuilder) { public RexNode getEquiCondition(RelNode left, RelNode right, RexBuilder rexBuilder) { - return RelOptUtil.createEquiJoinCondition(left, leftKeys, right, rightKeys, + return RelOptUtil.createHashJoinCondition(left, leftKeys, right, rightKeys, nullExclusionFlags, rexBuilder); } diff --git a/core/src/main/java/org/apache/calcite/rel/rules/LoptOptimizeJoinRule.java b/core/src/main/java/org/apache/calcite/rel/rules/LoptOptimizeJoinRule.java index 75056dd98121..16cb367ac2fe 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/LoptOptimizeJoinRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/LoptOptimizeJoinRule.java @@ -2053,7 +2053,7 @@ public static boolean isRemovableSelfJoin(Join joinRel) { */ private static boolean areSelfJoinKeysUnique(RelMetadataQuery mq, RelNode leftRel, RelNode rightRel, RexNode joinFilters) { - final JoinInfo joinInfo = JoinInfo.createWithStrictEquality(leftRel, rightRel, joinFilters); + final JoinInfo joinInfo = JoinInfo.of(leftRel, rightRel, joinFilters); // Make sure each key on the left maps to the same simple column as the // corresponding key on the right diff --git a/core/src/main/java/org/apache/calcite/rel/rules/LoptSemiJoinOptimizer.java b/core/src/main/java/org/apache/calcite/rel/rules/LoptSemiJoinOptimizer.java index e9d5020b0dc9..724884641dec 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/LoptSemiJoinOptimizer.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/LoptSemiJoinOptimizer.java @@ -264,7 +264,7 @@ private static int isSuitableFilter( RelNode factRel = multiJoin.getJoinFactor(factIdx); RelNode dimRel = multiJoin.getJoinFactor(dimIdx); - final JoinInfo joinInfo = JoinInfo.createWithStrictEquality(factRel, dimRel, semiJoinCondition); + final JoinInfo joinInfo = JoinInfo.of(factRel, dimRel, semiJoinCondition); assert !joinInfo.leftKeys.isEmpty(); // mutable copies diff --git a/core/src/main/java/org/apache/calcite/rel/rules/SemiJoinRule.java b/core/src/main/java/org/apache/calcite/rel/rules/SemiJoinRule.java index 2a07d7fb953c..4a10ec533a20 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/SemiJoinRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/SemiJoinRule.java @@ -108,9 +108,9 @@ protected void perform(RelOptRuleCall call, @Nullable Project project, final ImmutableIntList newRightKeys = ImmutableIntList.copyOf(newRightKeyBuilder); relBuilder.push(aggregate.getInput()); final RexNode newCondition = - RelOptUtil.createEquiJoinCondition(relBuilder.peek(2, 0), + RelOptUtil.createHashJoinCondition(relBuilder.peek(2, 0), joinInfo.leftKeys, relBuilder.peek(2, 1), newRightKeys, - rexBuilder); + joinInfo.nullExclusionFlags, rexBuilder); relBuilder.semiJoin(newCondition).hints(join.getHints()); break; diff --git a/core/src/main/java/org/apache/calcite/runtime/FlatLists.java b/core/src/main/java/org/apache/calcite/runtime/FlatLists.java index 544a8926f066..872bc76a49a6 100644 --- a/core/src/main/java/org/apache/calcite/runtime/FlatLists.java +++ b/core/src/main/java/org/apache/calcite/runtime/FlatLists.java @@ -58,6 +58,19 @@ public static List of(T t0) { return new Flat1List<>(t0); } + /** + * Creates a flat list with 1 element. This is different from {@link #of(Object)}, because + * it prevents overload from {@link #of(List)}. This may be useful when you create a flat list + * with 1 element of List type. + * + * @param t0 Element + * @param Element type + * @return List containing the given members + */ + public static List ofSingle(T t0) { + return new Flat1List<>(t0); + } + /** Creates a flat list with 2 elements. */ public static List of(T t0, T t1) { return new Flat2List<>(t0, t1); diff --git a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java index 14cb12d3e7eb..38344096404a 100644 --- a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java +++ b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java @@ -306,6 +306,7 @@ public enum BuiltInMethod { FlatProductInputType[].class), FLAT_LIST(SqlFunctions.class, "flatList"), LIST_N(FlatLists.class, "copyOf", Comparable[].class), + LIST1(FlatLists.class, "ofSingle", Object.class), LIST2(FlatLists.class, "of", Object.class, Object.class), LIST3(FlatLists.class, "of", Object.class, Object.class, Object.class), LIST4(FlatLists.class, "of", Object.class, Object.class, Object.class, diff --git a/core/src/test/java/org/apache/calcite/test/JdbcTest.java b/core/src/test/java/org/apache/calcite/test/JdbcTest.java index 6ef217fb0f8f..e25c6f31ccf2 100644 --- a/core/src/test/java/org/apache/calcite/test/JdbcTest.java +++ b/core/src/test/java/org/apache/calcite/test/JdbcTest.java @@ -4155,7 +4155,7 @@ public void checkOrderBy(final boolean desc, + "on \"t1\".\"commission\" is not distinct from \"t2\".\"commission\""; CalciteAssert.hr() .query(sql) - .explainContains("NestedLoopJoin(condition=[IS NOT DISTINCT FROM($0, $1)]") + .explainContains("HashJoin(condition=[IS NOT DISTINCT FROM($0, $1)]") .returnsUnordered("commission=1000", "commission=250", "commission=500", diff --git a/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableHashJoinTest.java b/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableHashJoinTest.java index 614ed98e031e..c589f6eea5e5 100644 --- a/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableHashJoinTest.java +++ b/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableHashJoinTest.java @@ -320,6 +320,137 @@ class EnumerableHashJoinTest { "empid=200"); } + @Test void hashJoinWithIsNotDistinctFrom() { + String tempTableSql = "WITH t1(id, sal) as ( VALUES (1,10), (2,NULL), (3,30), (5, NULL))," + + "t2(id, sal) as ( VALUES (1,10), (2,NULL), (4,40), (5, 50) ) "; + // t1 t2 + // id | sal id | sal + // 1 | 10 1 | 10 + // 2 | NULL 2 | NULL + // 3 | 30 4 | 40 + // 5 | NULL 5 | 50 + + // inner join: t1.sal IS NOT DISTINCT FROM t2.sal + tester(false, new HrSchema()) + .query( + tempTableSql + + "select t1.id, t1.sal, t2.id, t2.sal from t1 join t2" + + " on t1.sal is not distinct from t2.sal") + .withHook(Hook.PLANNER, (Consumer) planner -> + planner.removeRule(EnumerableRules.ENUMERABLE_MERGE_JOIN_RULE)) + .explainContains( + "EnumerableHashJoin(condition=[IS NOT DISTINCT FROM($1, $3)], joinType=[inner])\n" + + " EnumerableValues(tuples=[[{ 1, 10 }, { 2, null }, { 3, 30 }, { 5, null }]])\n" + + " EnumerableValues(tuples=[[{ 1, 10 }, { 2, null }, { 4, 40 }, { 5, 50 }]])\n") + .returnsUnordered( + "id=1; sal=10; id=1; sal=10", + "id=2; sal=null; id=2; sal=null", + "id=5; sal=null; id=2; sal=null"); + + // inner join: t1.sal = t2.sal + tester(false, new HrSchema()) + .query( + tempTableSql + + "select t1.id, t1.sal, t2.id, t2.sal from t1 join t2" + + " on t1.sal = t2.sal") + .withHook(Hook.PLANNER, (Consumer) planner -> + planner.removeRule(EnumerableRules.ENUMERABLE_MERGE_JOIN_RULE)) + .explainContains("EnumerableHashJoin(condition=[=($1, $3)], joinType=[inner])\n" + + " EnumerableValues(tuples=[[{ 1, 10 }, { 2, null }, { 3, 30 }, { 5, null }]])\n" + + " EnumerableValues(tuples=[[{ 1, 10 }, { 2, null }, { 4, 40 }, { 5, 50 }]])\n") + .returnsUnordered( + "id=1; sal=10; id=1; sal=10"); + + // inner join: t1.id = t2.id && t1.sal IS NOT DISTINCT FROM t2.sal + tester(false, new HrSchema()) + .query( + tempTableSql + + "select t1.id, t1.sal, t2.id, t2.sal from t1 join t2" + + " on t1.id = t2.id and t1.sal is not distinct from t2.sal") + .withHook(Hook.PLANNER, (Consumer) planner -> + planner.removeRule(EnumerableRules.ENUMERABLE_MERGE_JOIN_RULE)) + .explainContains( + "EnumerableHashJoin(condition=[AND(=($0, $2), IS NOT DISTINCT FROM($1, $3))], joinType=[inner])\n" + + " EnumerableValues(tuples=[[{ 1, 10 }, { 2, null }, { 3, 30 }, { 5, null }]])\n" + + " EnumerableValues(tuples=[[{ 1, 10 }, { 2, null }, { 4, 40 }, { 5, 50 }]])\n") + .returnsUnordered( + "id=1; sal=10; id=1; sal=10", + "id=2; sal=null; id=2; sal=null"); + + // semi join: t1.sal IS NOT DISTINCT FROM t2.sal + tester(true, new HrSchema()) + .withRel(builder -> { + builder + .values(new String[]{"id1", "sal1"}, 1, 10, 2, null, 3, 30, 5, null) + .values(new String[]{"id2", "sal2"}, 1, 10, 2, null, 4, 40, 5, 50) + .semiJoin( + builder.isNotDistinctFrom( + builder.field(2, 0, "sal1"), + builder.field(2, 1, "sal2"))); + return builder.build(); + }) + .withHook(Hook.PLANNER, (Consumer) planner -> { + planner.removeRule(EnumerableRules.ENUMERABLE_MERGE_JOIN_RULE); + }) + .explainHookMatches( + "EnumerableHashJoin(condition=[IS NOT DISTINCT FROM($1, $3)], joinType=[semi])\n" + + " EnumerableValues(tuples=[[{ 1, 10 }, { 2, null }, { 3, 30 }, { 5, null }]])\n" + + " EnumerableValues(tuples=[[{ 1, 10 }, { 2, null }, { 4, 40 }, { 5, 50 }]])\n") + .returnsUnordered( + "id1=1; sal1=10", + "id1=2; sal1=null", + "id1=5; sal1=null"); + + // semi join: t1.sal = t2.sal + tester(true, new HrSchema()) + .withRel(builder -> { + builder + .values(new String[]{"id1", "sal1"}, 1, 10, 2, null, 3, 30, 5, null) + .values(new String[]{"id2", "sal2"}, 1, 10, 2, null, 4, 40, 5, 50) + .semiJoin( + builder.equals( + builder.field(2, 0, "sal1"), + builder.field(2, 1, "sal2"))); + return builder.build(); + }) + .withHook(Hook.PLANNER, (Consumer) planner -> { + planner.removeRule(EnumerableRules.ENUMERABLE_MERGE_JOIN_RULE); + }) + .explainHookMatches( + "EnumerableHashJoin(condition=[=($1, $3)], joinType=[semi])\n" + + " EnumerableValues(tuples=[[{ 1, 10 }, { 2, null }, { 3, 30 }, { 5, null }]])\n" + + " EnumerableValues(tuples=[[{ 1, 10 }, { 2, null }, { 4, 40 }, { 5, 50 }]])\n") + .returnsUnordered( + "id1=1; sal1=10"); + + // semi join: t1.id = t2.id && t1.sal IS NOT DISTINCT FROM t2.sal + tester(true, new HrSchema()) + .withRel(builder -> { + builder + .values(new String[]{"id1", "sal1"}, 1, 10, 2, null, 3, 30, 5, null) + .values(new String[]{"id2", "sal2"}, 1, 10, 2, null, 4, 40, 5, 50) + .semiJoin( + builder.and( + builder.equals( + builder.field(2, 0, "id1"), + builder.field(2, 1, "id2")), + builder.isNotDistinctFrom( + builder.field(2, 0, "sal1"), + builder.field(2, 1, "sal2")))); + return builder.build(); + }) + .withHook(Hook.PLANNER, (Consumer) planner -> { + planner.removeRule(EnumerableRules.ENUMERABLE_MERGE_JOIN_RULE); + }) + .explainHookMatches( + "EnumerableHashJoin(condition=[AND(=($0, $2), IS NOT DISTINCT FROM($1, $3))], joinType=[semi])\n" + + " EnumerableValues(tuples=[[{ 1, 10 }, { 2, null }, { 3, 30 }, { 5, null }]])\n" + + " EnumerableValues(tuples=[[{ 1, 10 }, { 2, null }, { 4, 40 }, { 5, 50 }]])\n") + .returnsUnordered( + "id1=1; sal1=10", + "id1=2; sal1=null"); + } + private CalciteAssert.AssertThat tester(boolean forceDecorrelate, Object schema) { return CalciteAssert.that() diff --git a/core/src/test/resources/sql/blank.iq b/core/src/test/resources/sql/blank.iq index 882d430412ef..9c200caf7e2f 100644 --- a/core/src/test/resources/sql/blank.iq +++ b/core/src/test/resources/sql/blank.iq @@ -92,10 +92,10 @@ select i, j from table1 where table1.j NOT IN (select i from table2 where table1 EnumerableCalc(expr#0..7=[{inputs}], expr#8=[0], expr#9=[=($t3, $t8)], expr#10=[IS NULL($t1)], expr#11=[IS NOT NULL($t7)], expr#12=[<($t4, $t3)], expr#13=[OR($t10, $t11, $t12)], expr#14=[IS NOT TRUE($t13)], expr#15=[OR($t9, $t14)], proj#0..1=[{exprs}], $condition=[$t15]) EnumerableMergeJoin(condition=[AND(=($0, $6), =($1, $5))], joinType=[left]) EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) - EnumerableNestedLoopJoin(condition=[IS NOT DISTINCT FROM($0, $2)], joinType=[left]) + EnumerableHashJoin(condition=[IS NOT DISTINCT FROM($0, $2)], joinType=[left]) EnumerableTableScan(table=[[BLANK, TABLE1]]) EnumerableCalc(expr#0..3=[{inputs}], expr#4=[IS NOT NULL($t2)], expr#5=[0], expr#6=[CASE($t4, $t2, $t5)], expr#7=[IS NOT NULL($t3)], expr#8=[CASE($t7, $t3, $t5)], J=[$t0], c=[$t6], ck=[$t8]) - EnumerableNestedLoopJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left]) + EnumerableHashJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left]) EnumerableAggregate(group=[{0}]) EnumerableTableScan(table=[[BLANK, TABLE1]]) EnumerableAggregate(group=[{1}], c=[COUNT()], ck=[COUNT($0)]) diff --git a/core/src/test/resources/sql/planner.iq b/core/src/test/resources/sql/planner.iq index a24181cadba1..0461cc644b76 100644 --- a/core/src/test/resources/sql/planner.iq +++ b/core/src/test/resources/sql/planner.iq @@ -54,7 +54,7 @@ select * from t as t2 where t2.i > 0; !ok -EnumerableNestedLoopJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[semi]) +EnumerableHashJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[semi]) EnumerableValues(tuples=[[{ 0 }, { 1 }]]) EnumerableCalc(expr#0=[{inputs}], expr#1=[0], expr#2=[>($t0, $t1)], EXPR$0=[$t0], $condition=[$t2]) EnumerableValues(tuples=[[{ 0 }, { 1 }]]) @@ -74,7 +74,7 @@ select * from t as t2 where t2.i > 0; !ok -EnumerableNestedLoopJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[semi]) +EnumerableHashJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[semi]) EnumerableValues(tuples=[[{ 0 }, { 1 }]]) EnumerableCalc(expr#0=[{inputs}], expr#1=[0], expr#2=[>($t0, $t1)], EXPR$0=[$t0], $condition=[$t2]) EnumerableValues(tuples=[[{ 0 }, { 1 }]]) @@ -215,14 +215,13 @@ select a from (values (1.0), (4.0), (null)) as t3 (a); !ok EnumerableAggregate(group=[{0}]) - EnumerableNestedLoopJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[semi]) + EnumerableHashJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[semi]) EnumerableCalc(expr#0=[{inputs}], expr#1=[CAST($t0):DECIMAL(11, 1)], A=[$t1]) - EnumerableAggregate(group=[{0}]) - EnumerableNestedLoopJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[semi]) - EnumerableCalc(expr#0=[{inputs}], expr#1=[CAST($t0):DECIMAL(11, 1) NOT NULL], A=[$t1]) - EnumerableValues(tuples=[[{ 1.0 }, { 2.0 }, { 3.0 }, { 4.0 }, { 5.0 }]]) - EnumerableCalc(expr#0=[{inputs}], expr#1=[CAST($t0):DECIMAL(11, 1) NOT NULL], A=[$t1]) - EnumerableValues(tuples=[[{ 1 }, { 2 }]]) + EnumerableHashJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[semi]) + EnumerableCalc(expr#0=[{inputs}], expr#1=[CAST($t0):DECIMAL(11, 1)], A=[$t1]) + EnumerableValues(tuples=[[{ 1.0 }, { 2.0 }, { 3.0 }, { 4.0 }, { 5.0 }]]) + EnumerableCalc(expr#0=[{inputs}], expr#1=[CAST($t0):DECIMAL(11, 1)], A=[$t1]) + EnumerableValues(tuples=[[{ 1 }, { 2 }]]) EnumerableCalc(expr#0=[{inputs}], expr#1=[CAST($t0):DECIMAL(11, 1)], A=[$t1]) EnumerableValues(tuples=[[{ 1.0 }, { 4.0 }, { null }]]) !plan diff --git a/core/src/test/resources/sql/sub-query.iq b/core/src/test/resources/sql/sub-query.iq index a12b087ebb21..9f320c7ef024 100644 --- a/core/src/test/resources/sql/sub-query.iq +++ b/core/src/test/resources/sql/sub-query.iq @@ -536,7 +536,7 @@ EnumerableCalc(expr#0..9=[{inputs}], expr#10=[0], expr#11=[=($t5, $t10)], expr#1 EnumerableTableScan(table=[[scott, DEPT]]) EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableCalc(expr#0..3=[{inputs}], expr#4=[IS NOT NULL($t2)], expr#5=[0], expr#6=[CASE($t4, $t2, $t5)], expr#7=[IS NOT NULL($t3)], expr#8=[CASE($t7, $t3, $t5)], DEPTNO0=[$t0], c=[$t6], ck=[$t8]) - EnumerableNestedLoopJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left]) + EnumerableHashJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left]) EnumerableAggregate(group=[{1}]) EnumerableHashJoin(condition=[=($1, $2)], joinType=[semi]) EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], DEPTNO=[$t7]) @@ -2666,11 +2666,11 @@ EnumerableAggregate(group=[{}], C=[COUNT()]) EnumerableMergeJoin(condition=[AND(=($3, $5), =($4, $6))], joinType=[left]) EnumerableSort(sort0=[$3], sort1=[$4], dir0=[ASC], dir1=[ASC]) EnumerableCalc(expr#0..6=[{inputs}], expr#7=[100], expr#8=[+($t2, $t7)], expr#9=[CAST($t1):VARCHAR(14)], SAL=[$t2], c=[$t4], ck=[$t5], $f5=[$t8], ENAME0=[$t9]) - EnumerableNestedLoopJoin(condition=[IS NOT DISTINCT FROM($3, $6)], joinType=[left]) + EnumerableHashJoin(condition=[IS NOT DISTINCT FROM($3, $6)], joinType=[left]) EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t1):VARCHAR(14)], proj#0..1=[{exprs}], SAL=[$t5], ENAME0=[$t8]) EnumerableTableScan(table=[[scott, EMP]]) EnumerableCalc(expr#0..2=[{inputs}], expr#3=[IS NOT NULL($t2)], expr#4=[0], expr#5=[CASE($t3, $t2, $t4)], c=[$t5], ck=[$t5], DNAME=[$t0]) - EnumerableNestedLoopJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left]) + EnumerableHashJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left]) EnumerableAggregate(group=[{0}]) EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t1):VARCHAR(14)], ENAME0=[$t8]) EnumerableTableScan(table=[[scott, EMP]]) @@ -2686,9 +2686,9 @@ select empno from "scott".emp as e where e.empno > ANY( select 2 from "scott".dept e2 where e2.deptno = e.deptno) ; EnumerableCalc(expr#0..6=[{inputs}], EMPNO=[$t5]) - EnumerableNestedLoopJoin(condition=[AND(IS NOT DISTINCT FROM($6, $4), OR(AND(>($5, $0), IS NOT TRUE(OR(IS NULL($3), =($1, 0)))), AND(>($5, $0), IS NOT TRUE(OR(IS NULL($3), =($1, 0))), IS NOT TRUE(>($5, $0)), <=($1, $2))))], joinType=[inner]) + EnumerableHashJoin(condition=[AND(IS NOT DISTINCT FROM($4, $6), OR(AND(>($5, $0), IS NOT TRUE(OR(IS NULL($3), =($1, 0)))), AND(>($5, $0), IS NOT TRUE(OR(IS NULL($3), =($1, 0))), IS NOT TRUE(>($5, $0)), <=($1, $2))))], joinType=[inner]) EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NOT NULL($t3)], expr#6=[0], expr#7=[CASE($t5, $t3, $t6)], m=[$t2], c=[$t7], d=[$t7], trueLiteral=[$t4], DEPTNO=[$t0]) - EnumerableNestedLoopJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left]) + EnumerableHashJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left]) EnumerableAggregate(group=[{7}]) EnumerableTableScan(table=[[scott, EMP]]) EnumerableCalc(expr#0..2=[{inputs}], expr#3=[2], expr#4=[1:BIGINT], expr#5=[true], DEPTNO=[$t0], EXPR$0=[$t3], $f2=[$t4], $f3=[$t5]) @@ -2726,7 +2726,7 @@ EnumerableCalc(expr#0..6=[{inputs}], expr#7=[>($t1, $t2)], expr#8=[IS TRUE($t7)] EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], DEPTNO=[$t7]) EnumerableTableScan(table=[[scott, EMP]]) EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NOT NULL($t3)], expr#6=[0], expr#7=[CASE($t5, $t3, $t6)], m=[$t2], c=[$t7], d=[$t7], trueLiteral=[$t4], DEPTNO0=[$t0]) - EnumerableNestedLoopJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left]) + EnumerableHashJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left]) EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0]) EnumerableTableScan(table=[[scott, EMP]]) EnumerableAggregate(group=[{0}], m=[MIN($1)], c=[COUNT()], trueLiteral=[LITERAL_AGG(true)]) @@ -2801,7 +2801,7 @@ EnumerableCalc(expr#0..6=[{inputs}], expr#7=[<>($t2, $t1)], expr#8=[1], expr#9=[ EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0]) EnumerableTableScan(table=[[scott, EMP]]) EnumerableCalc(expr#0..5=[{inputs}], expr#6=[IS NOT NULL($t2)], expr#7=[0], expr#8=[CASE($t6, $t2, $t7)], expr#9=[IS NOT NULL($t3)], expr#10=[CASE($t9, $t3, $t7)], c=[$t8], d=[$t8], dd=[$t10], m=[$t4], trueLiteral=[$t5], EMPNO1=[$t0]) - EnumerableNestedLoopJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left]) + EnumerableHashJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left]) EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0]) EnumerableTableScan(table=[[scott, EMP]]) EnumerableCalc(expr#0..7=[{inputs}], expr#8=[1:BIGINT], expr#9=[true], EMPNO1=[$t0], $f1=[$t8], $f2=[$t8], EMPNO=[$t0], $f4=[$t9]) @@ -2845,10 +2845,10 @@ select * from "scott".emp emp1 where empno <> some (select comm from "scott".emp where deptno = emp1.deptno); EnumerableCalc(expr#0..13=[{inputs}], expr#14=[<>($t10, $t9)], expr#15=[1], expr#16=[<=($t11, $t15)], expr#17=[AND($t14, $t16)], expr#18=[=($t11, $t15)], expr#19=[OR($t17, $t18)], expr#20=[<>($t0, $t12)], expr#21=[IS NULL($t13)], expr#22=[0], expr#23=[=($t9, $t22)], expr#24=[OR($t21, $t23)], expr#25=[IS NOT TRUE($t24)], expr#26=[AND($t19, $t20, $t25)], expr#27=[IS NOT TRUE($t19)], expr#28=[AND($t25, $t27)], expr#29=[OR($t26, $t28)], proj#0..7=[{exprs}], $condition=[$t29]) - EnumerableNestedLoopJoin(condition=[IS NOT DISTINCT FROM($7, $8)], joinType=[left]) + EnumerableHashJoin(condition=[IS NOT DISTINCT FROM($7, $8)], joinType=[left]) EnumerableTableScan(table=[[scott, EMP]]) EnumerableCalc(expr#0..6=[{inputs}], expr#7=[IS NOT NULL($t2)], expr#8=[0], expr#9=[CASE($t7, $t2, $t8)], expr#10=[IS NOT NULL($t3)], expr#11=[CASE($t10, $t3, $t8)], expr#12=[IS NOT NULL($t4)], expr#13=[CASE($t12, $t4, $t8)], DEPTNO=[$t0], c=[$t9], d=[$t11], dd=[$t13], m=[$t5], trueLiteral=[$t6]) - EnumerableNestedLoopJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left]) + EnumerableHashJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left]) EnumerableAggregate(group=[{7}]) EnumerableTableScan(table=[[scott, EMP]]) EnumerableCalc(expr#0..5=[{inputs}], expr#6=[CAST($t1):BIGINT NOT NULL], expr#7=[CAST($t2):BIGINT NOT NULL], expr#8=[CAST($t5):BOOLEAN NOT NULL], DEPTNO=[$t0], c=[$t6], d=[$t7], dd=[$t3], m=[$t4], trueLiteral=[$t8]) @@ -2905,7 +2905,7 @@ EnumerableCalc(expr#0..13=[{inputs}], expr#14=[<>($t9, $t8)], expr#15=[1], expr# EnumerableHashJoin(condition=[=($0, $13)], joinType=[left]) EnumerableTableScan(table=[[scott, EMP]]) EnumerableCalc(expr#0..5=[{inputs}], expr#6=[IS NOT NULL($t2)], expr#7=[0], expr#8=[CASE($t6, $t2, $t7)], expr#9=[IS NOT NULL($t3)], expr#10=[CASE($t9, $t3, $t7)], c=[$t8], d=[$t8], dd=[$t10], m=[$t4], trueLiteral=[$t5], DEPTNO0=[$t0]) - EnumerableNestedLoopJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left]) + EnumerableHashJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left]) EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0]) EnumerableTableScan(table=[[scott, EMP]]) EnumerableCalc(expr#0..4=[{inputs}], expr#5=[CAST($t1):BIGINT NOT NULL], expr#6=[CAST($t3):INTEGER NOT NULL], expr#7=[CAST($t4):BOOLEAN NOT NULL], DEPTNO0=[$t0], c=[$t5], dd=[$t2], m=[$t6], trueLiteral=[$t7]) @@ -2956,7 +2956,7 @@ EnumerableCalc(expr#0..13=[{inputs}], expr#14=[<>($t9, $t8)], expr#15=[1], expr# EnumerableHashJoin(condition=[=($0, $13)], joinType=[left]) EnumerableTableScan(table=[[scott, EMP]]) EnumerableCalc(expr#0..5=[{inputs}], expr#6=[IS NOT NULL($t2)], expr#7=[0], expr#8=[CASE($t6, $t2, $t7)], expr#9=[IS NOT NULL($t3)], expr#10=[CASE($t9, $t3, $t7)], c=[$t8], d=[$t8], dd=[$t10], m=[$t4], trueLiteral=[$t5], DEPTNO0=[$t0]) - EnumerableNestedLoopJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left]) + EnumerableHashJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left]) EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0]) EnumerableTableScan(table=[[scott, EMP]]) EnumerableCalc(expr#0..4=[{inputs}], expr#5=[CAST($t1):BIGINT NOT NULL], expr#6=[CAST($t3):INTEGER NOT NULL], expr#7=[CAST($t4):BOOLEAN NOT NULL], DEPTNO0=[$t0], c=[$t5], dd=[$t2], m=[$t6], trueLiteral=[$t7]) @@ -3004,10 +3004,10 @@ select * from "scott".emp emp1 where emp1.comm <> some (select comm from "scott".emp emp2 where emp2.sal = emp1.sal); EnumerableCalc(expr#0..13=[{inputs}], expr#14=[<>($t10, $t9)], expr#15=[1], expr#16=[<=($t11, $t15)], expr#17=[AND($t14, $t16)], expr#18=[=($t11, $t15)], expr#19=[OR($t17, $t18)], expr#20=[<>($t6, $t12)], expr#21=[IS NULL($t13)], expr#22=[IS NULL($t6)], expr#23=[0], expr#24=[=($t9, $t23)], expr#25=[OR($t21, $t22, $t24)], expr#26=[IS NOT TRUE($t25)], expr#27=[AND($t19, $t20, $t26)], expr#28=[IS NOT TRUE($t19)], expr#29=[AND($t26, $t28)], expr#30=[OR($t27, $t29)], proj#0..7=[{exprs}], $condition=[$t30]) - EnumerableNestedLoopJoin(condition=[IS NOT DISTINCT FROM($5, $8)], joinType=[left]) + EnumerableHashJoin(condition=[IS NOT DISTINCT FROM($5, $8)], joinType=[left]) EnumerableTableScan(table=[[scott, EMP]]) EnumerableCalc(expr#0..6=[{inputs}], expr#7=[IS NOT NULL($t2)], expr#8=[0], expr#9=[CASE($t7, $t2, $t8)], expr#10=[IS NOT NULL($t3)], expr#11=[CASE($t10, $t3, $t8)], expr#12=[IS NOT NULL($t4)], expr#13=[CASE($t12, $t4, $t8)], SAL=[$t0], c=[$t9], d=[$t11], dd=[$t13], m=[$t5], trueLiteral=[$t6]) - EnumerableNestedLoopJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left]) + EnumerableHashJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left]) EnumerableAggregate(group=[{5}]) EnumerableTableScan(table=[[scott, EMP]]) EnumerableCalc(expr#0..5=[{inputs}], expr#6=[CAST($t1):BIGINT NOT NULL], expr#7=[CAST($t2):BIGINT NOT NULL], expr#8=[CAST($t5):BOOLEAN NOT NULL], SAL=[$t0], c=[$t6], d=[$t7], dd=[$t3], m=[$t4], trueLiteral=[$t8]) @@ -5388,10 +5388,10 @@ select * from emp where deptno <> (select count(deptno) from dept where dept.dep !ok EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t4)], expr#6=[CAST($t1):BIGINT], expr#7=[0:BIGINT], expr#8=[<>($t6, $t7)], expr#9=[AND($t5, $t8)], expr#10=[<>($t6, $t4)], expr#11=[OR($t9, $t10)], proj#0..2=[{exprs}], $condition=[$t11]) - EnumerableNestedLoopJoin(condition=[IS NOT DISTINCT FROM($1, $3)], joinType=[left]) + EnumerableHashJoin(condition=[IS NOT DISTINCT FROM($1, $3)], joinType=[left]) EnumerableValues(tuples=[[{ 'Jane ', 10, 'F' }, { 'Bob ', 10, 'M' }, { 'Eric ', 20, 'M' }, { 'Susan', 30, 'F' }, { 'Alice', 30, 'F' }, { 'Adam ', 50, 'M' }, { 'Eve ', 50, 'F' }, { 'Grace', 60, 'F' }, { 'Wilma', null, 'F' }]]) EnumerableCalc(expr#0..2=[{inputs}], expr#3=[IS NOT NULL($t2)], expr#4=[0], expr#5=[CASE($t3, $t2, $t4)], DEPTNO=[$t0], EXPR$0=[$t5]) - EnumerableNestedLoopJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left]) + EnumerableHashJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left]) EnumerableAggregate(group=[{0}]) EnumerableValues(tuples=[[{ 10 }, { 10 }, { 20 }, { 30 }, { 30 }, { 50 }, { 50 }, { 60 }, { null }]]) EnumerableCalc(expr#0..1=[{inputs}], expr#2=[1:BIGINT], DEPTNO=[$t0], $f1=[$t2]) @@ -5414,7 +5414,7 @@ select * from emp where deptno <> (select count(deptno) + 10 from dept where de !ok EnumerableCalc(expr#0..4=[{inputs}], expr#5=[CAST($t1):BIGINT], expr#6=[IS NULL($t4)], expr#7=[0:BIGINT], expr#8=[CASE($t6, $t7, $t4)], expr#9=[10], expr#10=[+($t8, $t9)], expr#11=[<>($t5, $t10)], proj#0..2=[{exprs}], $condition=[$t11]) - EnumerableNestedLoopJoin(condition=[IS NOT DISTINCT FROM($1, $3)], joinType=[left]) + EnumerableHashJoin(condition=[IS NOT DISTINCT FROM($1, $3)], joinType=[left]) EnumerableValues(tuples=[[{ 'Jane ', 10, 'F' }, { 'Bob ', 10, 'M' }, { 'Eric ', 20, 'M' }, { 'Susan', 30, 'F' }, { 'Alice', 30, 'F' }, { 'Adam ', 50, 'M' }, { 'Eve ', 50, 'F' }, { 'Grace', 60, 'F' }, { 'Wilma', null, 'F' }]]) EnumerableCalc(expr#0=[{inputs}], expr#1=[0], expr#2=[CAST($t1):BIGINT NOT NULL], DEPTNO=[$t0], $f1=[$t2]) EnumerableAggregate(group=[{0}]) diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java b/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java index 335fbc89b264..14309bdfd652 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java @@ -1543,18 +1543,15 @@ private static Enumerable hashEquiJoin } final TSource outer = outers.current(); final Enumerable innerEnumerable; - if (outer == null) { + // if the key is null-safe, still extract outerKey and probe even if outer is NULL. + final TKey outerKey = outerKeySelector.apply(outer); + if (outerKey == null) { innerEnumerable = null; } else { - final TKey outerKey = outerKeySelector.apply(outer); - if (outerKey == null) { - innerEnumerable = null; - } else { - if (unmatchedKeys != null) { - unmatchedKeys.remove(outerKey); - } - innerEnumerable = innerLookup.get(outerKey); + if (unmatchedKeys != null) { + unmatchedKeys.remove(outerKey); } + innerEnumerable = innerLookup.get(outerKey); } if (innerEnumerable == null || !innerEnumerable.any()) { @@ -1639,30 +1636,27 @@ private static Enumerable hashJoinWith } final TSource outer = outers.current(); Enumerable innerEnumerable; - if (outer == null) { + // if the key is null-safe, still extract outerKey and probe even if outer is NULL. + final TKey outerKey = outerKeySelector.apply(outer); + if (outerKey == null) { innerEnumerable = null; } else { - final TKey outerKey = outerKeySelector.apply(outer); - if (outerKey == null) { - innerEnumerable = null; - } else { - innerEnumerable = innerLookup.get(outerKey); - // apply predicate to filter per-row - if (innerEnumerable != null) { - final List matchedInners = new ArrayList<>(); - try (Enumerator innerEnumerator = - innerEnumerable.enumerator()) { - while (innerEnumerator.moveNext()) { - final TInner inner = innerEnumerator.current(); - if (predicate.apply(outer, inner)) { - matchedInners.add(inner); - } + innerEnumerable = innerLookup.get(outerKey); + // apply predicate to filter per-row + if (innerEnumerable != null) { + final List matchedInners = new ArrayList<>(); + try (Enumerator innerEnumerator = + innerEnumerable.enumerator()) { + while (innerEnumerator.moveNext()) { + final TInner inner = innerEnumerator.current(); + if (predicate.apply(outer, inner)) { + matchedInners.add(inner); } } - innerEnumerable = Linq4j.asEnumerable(matchedInners); - if (innersUnmatched != null) { - innersUnmatched.removeAll(matchedInners); - } + } + innerEnumerable = Linq4j.asEnumerable(matchedInners); + if (innersUnmatched != null) { + innersUnmatched.removeAll(matchedInners); } } } From 4aa2098a3abc6ad3ff575c80f1f4cf19355654fb Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Wed, 17 Dec 2025 10:00:26 +0800 Subject: [PATCH 060/562] [CALCITE-7335] RelToSqlConverter generate sql containing Scala subqueries includes redundant parentheses --- .../calcite/sql/fun/SqlStdOperatorTable.java | 14 +++++-- .../rel/rel2sql/RelToSqlConverterTest.java | 41 +++++++++++++------ 2 files changed, 39 insertions(+), 16 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java index 074c3103f299..2a26a78929cf 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java @@ -2391,7 +2391,7 @@ public class SqlStdOperatorTable extends ReflectiveSqlOperatorTable { new SqlInternalOperator( "$SCALAR_QUERY", SqlKind.SCALAR_QUERY, - 0, + 100, // High precedence to prevent SqlCall from adding extra parentheses false, ReturnTypes.RECORD_TO_SCALAR, null, @@ -2401,9 +2401,15 @@ public class SqlStdOperatorTable extends ReflectiveSqlOperatorTable { SqlCall call, int leftPrec, int rightPrec) { - final SqlWriter.Frame frame = writer.startList("(", ")"); - call.operand(0).unparse(writer, 0, 0); - writer.endList(frame); + final SqlNode operand = call.operand(0); + if (operand.getKind() == SqlKind.SELECT) { + operand.unparse(writer, leftPrec, rightPrec); + } else { + final SqlWriter.Frame frame = + writer.startList(SqlWriter.FrameTypeEnum.SUB_QUERY, "(", ")"); + operand.unparse(writer, 0, 0); + writer.endList(frame); + } } @Override public boolean argumentMustBeScalar(int ordinal) { diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java index d2125c0ec8b0..06e1b93358bb 100644 --- a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java +++ b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java @@ -8627,9 +8627,9 @@ private void checkLiteral2(String expression, String expected) { + "HAVING \"t1\".\"department_id\" = MIN(\"t1\".\"department_id\")) \"t4\" ON \"employee\".\"department_id\" = \"t4\".\"department_id0\""; final String expectedNoExpand = "SELECT \"department_id\"\n" + "FROM \"foodmart\".\"employee\"\n" - + "WHERE \"department_id\" = (((SELECT MIN(\"employee\".\"department_id\")\n" + + "WHERE \"department_id\" = (SELECT MIN(\"employee\".\"department_id\")\n" + "FROM \"foodmart\".\"department\"\n" - + "WHERE 1 = 2)))"; + + "WHERE 1 = 2)"; final String expected = "SELECT \"employee\".\"department_id\"\n" + "FROM \"foodmart\".\"employee\"\n" + "INNER JOIN (SELECT \"t1\".\"department_id\" AS \"department_id0\", MIN(\"t1\".\"department_id\") AS \"EXPR$0\"\n" @@ -9802,17 +9802,17 @@ private void checkLiteral2(String expression, String expected) { final String sql0 = "update \"foodmart\".\"product\" a set \"product_id\" = " + "(select \"product_class_id\" from \"foodmart\".\"product_class\" b " + "where a.\"product_class_id\" = b.\"product_class_id\")"; - final String expected0 = "UPDATE \"foodmart\".\"product\" SET \"product_id\" = (((SELECT " + final String expected0 = "UPDATE \"foodmart\".\"product\" SET \"product_id\" = (SELECT " + "\"product_class_id\"\nFROM \"foodmart\".\"product_class\"\nWHERE \"product\"" - + ".\"product_class_id\" = \"product_class_id\")))"; + + ".\"product_class_id\" = \"product_class_id\")"; sql(sql0).ok(expected0); final String sql1 = "update \"foodmart\".\"product\" a set \"brand_name\" = " + "(select cast(\"product_category\" as varchar(60)) from \"foodmart\".\"product_class\" b " + "where a.\"product_class_id\" = b.\"product_class_id\")"; - final String expected1 = "UPDATE \"foodmart\".\"product\" SET \"brand_name\" = (((SELECT CAST" + final String expected1 = "UPDATE \"foodmart\".\"product\" SET \"brand_name\" = (SELECT CAST" + "(\"product_category\" AS VARCHAR(60) CHARACTER SET \"ISO-8859-1\")\nFROM \"foodmart\"" - + ".\"product_class\"\nWHERE \"product\".\"product_class_id\" = \"product_class_id\")))"; + + ".\"product_class\"\nWHERE \"product\".\"product_class_id\" = \"product_class_id\")"; sql(sql1).ok(expected1); final String sql2 = "update \"foodmart\".\"product\"\n" @@ -10972,10 +10972,10 @@ private void checkLiteral2(String expression, String expected) { final String expected = "SELECT " + "\"DEPTNO\", " + "\"DNAME\", " - + "(((SELECT COUNT(*) AS \"COUNT\"\n" + + "(SELECT COUNT(*) AS \"COUNT\"\n" + "FROM \"scott\".\"EMP\"\n" + "GROUP BY \"DEPTNO\"\n" - + "HAVING \"DEPTNO\" = \"DEPT\".\"DEPTNO\"))) AS \"$f2\"\n" + + "HAVING \"DEPTNO\" = \"DEPT\".\"DEPTNO\") AS \"$f2\"\n" + "FROM \"scott\".\"DEPT\""; relFn(relFn).ok(expected); @@ -11053,9 +11053,9 @@ private void checkLiteral2(String expression, String expected) { final String expected = "SELECT \"EMP\".\"EMPNO\"\n" + "FROM \"SCOTT\".\"EMP\"\n" + "INNER JOIN \"SCOTT\".\"DEPT\" ON \"EMP\".\"DEPTNO\" = \"DEPT\".\"DEPTNO\"\n" - + "WHERE \"DEPT\".\"DEPTNO\" = (((SELECT MIN(\"DEPTNO\")\n" + + "WHERE \"DEPT\".\"DEPTNO\" = (SELECT MIN(\"DEPTNO\")\n" + "FROM \"SCOTT\".\"DEPT\"\n" - + "WHERE \"DEPTNO\" = \"EMP\".\"DEPTNO\")))"; + + "WHERE \"DEPTNO\" = \"EMP\".\"DEPTNO\")"; HepProgramBuilder builder = new HepProgramBuilder(); builder.addRuleClass(FilterJoinRule.FilterIntoJoinRule.class); @@ -11083,8 +11083,8 @@ private void checkLiteral2(String expression, String expected) { + "FROM \"SCOTT\".\"EMP\" AS \"$cor1\",\n" + "LATERAL (SELECT *\nFROM \"SCOTT\".\"DEPT\"\n" + "WHERE \"$cor1\".\"DEPTNO\" = \"DEPTNO\") AS \"t\"\n" - + "WHERE \"t\".\"DEPTNO\" = (((SELECT MIN(\"DEPTNO\")\n" - + "FROM \"SCOTT\".\"DEPT\"\nWHERE \"DEPTNO\" = \"$cor1\".\"DEPTNO\")))"; + + "WHERE \"t\".\"DEPTNO\" = (SELECT MIN(\"DEPTNO\")\n" + + "FROM \"SCOTT\".\"DEPT\"\nWHERE \"DEPTNO\" = \"$cor1\".\"DEPTNO\")"; HepProgramBuilder builder = new HepProgramBuilder(); builder.addRuleClass(JoinToCorrelateRule.class); builder.addRuleClass(FilterCorrelateRule.class); @@ -11099,6 +11099,23 @@ private void checkLiteral2(String expression, String expected) { .ok(expected); } + /** Test case of + * [CALCITE-7335] + * RelToSqlConverter generate sql containing Scala subqueries + * includes redundant parentheses. */ + @Test void testScalarSubqueryInSelectList() { + final String sql = "SELECT\n" + + " (SELECT COUNT(*)\n" + + " FROM \"employee\"\n" + + " WHERE v.\"product_id\" >= 2), 3\n" + + "FROM \"product\" AS v"; + final String expected = "SELECT (SELECT COUNT(*)\n" + + "FROM \"foodmart\".\"employee\"\n" + + "WHERE \"product\".\"product_id\" >= 2), 3\n" + + "FROM \"foodmart\".\"product\""; + sql(sql).ok(expected); + } + /** Fluid interface to run tests. */ static class Sql { private final CalciteAssert.SchemaSpec schemaSpec; From e9abe3dc326818d7c838c77ee5d1f99fc9a7845f Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Wed, 17 Dec 2025 23:27:49 +0800 Subject: [PATCH 061/562] [CALCITE-2152] SQL parser unable to parse SQL with nested joins produced by RelToSqlConverter --- .../rel/rel2sql/RelToSqlConverterTest.java | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java index 06e1b93358bb..43e855592422 100644 --- a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java +++ b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java @@ -256,6 +256,65 @@ private static String toSql(RelNode root, SqlDialect dialect, sql(query).withMysql().ok(expected); } + /** Test case for + * [CALCITE-2152] + * SQL parser unable to parse SQL with nested joins produced by RelToSqlConverter. */ + @Test public void testNestedJoin() { + final String query = "select *\n" + + "from \"sales_fact_1997\"\n" + + "inner join (select * from \"customer\"\n" + + "inner join \"employee\" on (\"customer\".\"city\" = \"employee\".\"store_id\") ) AS \"customer_employee\"\n" + + "on (\"sales_fact_1997\".\"store_id\" = \"customer_employee\".\"city\")"; + String expected = "SELECT \"sales_fact_1997\".\"product_id\", \"sales_fact_1997\".\"time_id\"," + + " \"sales_fact_1997\".\"customer_id\", \"sales_fact_1997\".\"promotion_id\"," + + " \"sales_fact_1997\".\"store_id\", \"sales_fact_1997\".\"store_sales\"," + + " \"sales_fact_1997\".\"store_cost\", \"sales_fact_1997\".\"unit_sales\"," + + " \"t0\".\"customer_id\" AS \"customer_id0\", \"t0\".\"account_num\", \"t0\".\"lname\"," + + " \"t0\".\"fname\", \"t0\".\"mi\", \"t0\".\"address1\", \"t0\".\"address2\"," + + " \"t0\".\"address3\", \"t0\".\"address4\", \"t0\".\"city\", \"t0\".\"state_province\"," + + " \"t0\".\"postal_code\", \"t0\".\"country\", \"t0\".\"customer_region_id\"," + + " \"t0\".\"phone1\", \"t0\".\"phone2\", \"t0\".\"birthdate\", \"t0\".\"marital_status\"," + + " \"t0\".\"yearly_income\", \"t0\".\"gender\", \"t0\".\"total_children\"," + + " \"t0\".\"num_children_at_home\", \"t0\".\"education\", \"t0\".\"date_accnt_opened\"," + + " \"t0\".\"member_card\", \"t0\".\"occupation\", \"t0\".\"houseowner\"," + + " \"t0\".\"num_cars_owned\", \"t0\".\"fullname\", \"t0\".\"employee_id\"," + + " \"t0\".\"full_name\", \"t0\".\"first_name\", \"t0\".\"last_name\"," + + " \"t0\".\"position_id\", \"t0\".\"position_title\", \"t0\".\"store_id\" AS \"store_id0\"," + + " \"t0\".\"department_id\", \"t0\".\"birth_date\", \"t0\".\"hire_date\"," + + " \"t0\".\"end_date\", \"t0\".\"salary\", \"t0\".\"supervisor_id\"," + + " \"t0\".\"education_level\", \"t0\".\"marital_status0\", \"t0\".\"gender0\"," + + " \"t0\".\"management_role\"\n" + + "FROM \"foodmart\".\"sales_fact_1997\"\n" + + "INNER JOIN (SELECT \"t\".\"customer_id\", \"t\".\"account_num\", \"t\".\"lname\"," + + " \"t\".\"fname\", \"t\".\"mi\", \"t\".\"address1\", \"t\".\"address2\"," + + " \"t\".\"address3\", \"t\".\"address4\", \"t\".\"city\", \"t\".\"state_province\"," + + " \"t\".\"postal_code\", \"t\".\"country\", \"t\".\"customer_region_id\"," + + " \"t\".\"phone1\", \"t\".\"phone2\", \"t\".\"birthdate\", \"t\".\"marital_status\"," + + " \"t\".\"yearly_income\", \"t\".\"gender\", \"t\".\"total_children\"," + + " \"t\".\"num_children_at_home\", \"t\".\"education\", \"t\".\"date_accnt_opened\"," + + " \"t\".\"member_card\", \"t\".\"occupation\", \"t\".\"houseowner\"," + + " \"t\".\"num_cars_owned\", \"t\".\"fullname\", \"employee\".\"employee_id\"," + + " \"employee\".\"full_name\", \"employee\".\"first_name\", \"employee\".\"last_name\"," + + " \"employee\".\"position_id\", \"employee\".\"position_title\", \"employee\".\"store_id\"," + + " \"employee\".\"department_id\", \"employee\".\"birth_date\", \"employee\".\"hire_date\"," + + " \"employee\".\"end_date\", \"employee\".\"salary\", \"employee\".\"supervisor_id\"," + + " \"employee\".\"education_level\", \"employee\".\"marital_status\" AS \"marital_status0\"," + + " \"employee\".\"gender\" AS \"gender0\", \"employee\".\"management_role\"," + + " CAST(\"t\".\"city\" AS INTEGER) AS \"city0\"\n" + + "FROM (SELECT \"customer_id\"," + + " \"account_num\", \"lname\", \"fname\", \"mi\", \"address1\", \"address2\"," + + " \"address3\", \"address4\", \"city\", \"state_province\", \"postal_code\"," + + " \"country\", \"customer_region_id\", \"phone1\", \"phone2\", \"birthdate\"," + + " \"marital_status\", \"yearly_income\", \"gender\", \"total_children\"," + + " \"num_children_at_home\", \"education\", \"date_accnt_opened\", \"member_card\"," + + " \"occupation\", \"houseowner\", \"num_cars_owned\", \"fullname\"," + + " CAST(\"city\" AS INTEGER) AS \"city0\"\n" + + "FROM \"foodmart\".\"customer\") AS \"t\"\n" + + "INNER JOIN \"foodmart\".\"employee\" ON \"t\".\"city0\" = \"employee\".\"store_id\") AS \"t0\"" + + " ON \"sales_fact_1997\".\"store_id\" = \"t0\".\"city0\""; + sql(query).ok(expected); + } + /** * Test for [CALCITE-4723] * Check whether JDBC adapter generates "GROUP BY ()" against Oracle, DB2, MSSQL. From 43617c17f96a814bc5e97779a505601b201bdbc3 Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Thu, 18 Dec 2025 16:46:01 +0800 Subject: [PATCH 062/562] [CALCITE-7311] Support the syntax ROW(*) to create a nested ROW type with all columns --- core/src/main/codegen/templates/Parser.jj | 52 ++++++++++++---- .../calcite/runtime/CalciteResource.java | 3 + .../sql/validate/SqlValidatorImpl.java | 60 ++++++++++++++++++- .../runtime/CalciteResource.properties | 1 + .../apache/calcite/test/SqlValidatorTest.java | 9 +++ core/src/test/resources/sql/struct.iq | 39 ++++++++++++ 6 files changed, 151 insertions(+), 13 deletions(-) diff --git a/core/src/main/codegen/templates/Parser.jj b/core/src/main/codegen/templates/Parser.jj index ee3c1c759a98..2069882cc223 100644 --- a/core/src/main/codegen/templates/Parser.jj +++ b/core/src/main/codegen/templates/Parser.jj @@ -170,6 +170,20 @@ public class ${parser.class} extends SqlAbstractParserImpl private Casing quotedCasing; private int identifierMaxLength; private SqlConformance conformance; + private int rowValueStarCount; + + private void pushRowValueStar() { + rowValueStarCount++; + } + + private void popRowValueStar() { + assert rowValueStarCount > 0; + rowValueStarCount--; + } + + private boolean allowRowValueStar() { + return rowValueStarCount > 0; + } /** * {@link SqlParserImplFactory} implementation for creating parser. @@ -4040,27 +4054,38 @@ SqlNode Expression3(ExprContext exprContext) : LOOKAHEAD(3) { s = span(); + pushRowValueStar(); } list = ParenthesizedQueryOrCommaList(exprContext) { - if (exprContext != ExprContext.ACCEPT_ALL - && exprContext != ExprContext.ACCEPT_CURSOR - && !this.conformance.allowExplicitRowValueConstructor()) - { - throw SqlUtil.newContextException(s.end(list), - RESOURCE.illegalRowExpression()); + try { + if (exprContext != ExprContext.ACCEPT_ALL + && exprContext != ExprContext.ACCEPT_CURSOR + && !this.conformance.allowExplicitRowValueConstructor()) + { + throw SqlUtil.newContextException(s.end(list), + RESOURCE.illegalRowExpression()); + } + return SqlStdOperatorTable.ROW.createCall(list); + } finally { + popRowValueStar(); } - return SqlStdOperatorTable.ROW.createCall(list); } | ( - { rowSpan = span(); } + { rowSpan = span(); pushRowValueStar(); } | { rowSpan = null; } ) list1 = ParenthesizedQueryOrCommaList(exprContext) { - if (rowSpan != null) { - // interpret as row constructor - return SqlStdOperatorTable.ROW.createCall(rowSpan.end(list1), - (List) list1); + try { + if (rowSpan != null) { + // interpret as row constructor + return SqlStdOperatorTable.ROW.createCall(rowSpan.end(list1), + (List) list1); + } + } finally { + if (rowSpan != null) { + popRowValueStar(); + } } } [ @@ -4391,6 +4416,9 @@ SqlNode AtomicRowExpression() : e = ContextVariable() | e = CompoundIdentifier() + | + LOOKAHEAD({ allowRowValueStar() }) + { return SqlIdentifier.star(getPos()); } | e = NewSpecification() | diff --git a/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java b/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java index 33fd588a02d4..e3bc081496ea 100644 --- a/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java +++ b/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java @@ -1169,6 +1169,9 @@ ExInst multipleCapturingGroupsForRegexpFunctions(String value, @BaseMessage("Unequal number of entries in ROW expressions") ExInst unequalRowSizes(); + @BaseMessage("Star is not allowed in ROW constructor outside of query context") + ExInst rowStarNotAllowed(); + @BaseMessage("Cannot infer return type for {0}; operand types: {1}") ExInst cannotInferReturnType(String operator, String types); diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java index 1f9415ff7548..0fc14cf6dc2a 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java @@ -7412,11 +7412,69 @@ public static boolean isAmbiguousException(Exception ex) { CallCopyingArgHandler argHandler = new CallCopyingArgHandler(call, false); call.getOperator().acceptCall(this, call, true, argHandler); - final SqlNode result = argHandler.result(); + final SqlNode result = expandStarInRow(argHandler.result()); validator.setOriginal(result, call); return result; } + /** + * Expands star (*) within ROW constructors. + * For example, transforms {@code ROW(*)} or {@code ROW(t.*)} into + * {@code ROW(col1, col2, ...)} based on available columns in scope. + * + * @param node Node to potentially expand + * @return Original node if not a ROW with stars, otherwise expanded ROW + */ + private SqlNode expandStarInRow(SqlNode node) { + if (!(node instanceof SqlCall)) { + return node; + } + final SqlCall call = (SqlCall) node; + if (call.getKind() != SqlKind.ROW) { + return node; + } + final SqlValidatorScope scope = getScope(); + if (!(scope instanceof SelectScope)) { + // Check if any operand is a star identifier before throwing error + for (SqlNode operand : call.getOperandList()) { + if (operand instanceof SqlIdentifier + && ((SqlIdentifier) operand).isStar()) { + throw validator.newValidationError(node, + RESOURCE.rowStarNotAllowed()); + } + } + return node; + } + final SelectScope selectScope = (SelectScope) scope; + final List expandedOperands = new ArrayList<>(); + boolean expanded = false; + for (SqlNode operand : call.getOperandList()) { + if (operand instanceof SqlIdentifier) { + final SqlIdentifier identifier = (SqlIdentifier) operand; + if (identifier.isStar()) { + final boolean expandedStar = + validator.expandStar(expandedOperands, + validator.catalogReader.nameMatcher().createSet(), + PairList.of(), + false, + selectScope, + identifier); + if (!expandedStar) { + throw new AssertionError("Row star expansion failed for " + identifier); + } + expanded = true; + continue; + } + } + expandedOperands.add(operand); + } + if (!expanded) { + return node; + } + return SqlStdOperatorTable.ROW.createCall( + call.getParserPosition(), expandedOperands); + } + protected SqlNode expandDynamicStar(SqlIdentifier id, SqlIdentifier fqId) { if (DynamicRecordType.isDynamicStarColName(Util.last(fqId.names)) && !DynamicRecordType.isDynamicStarColName(Util.last(id.names))) { diff --git a/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties b/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties index 6cff62359856..cf600b98cf77 100644 --- a/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties +++ b/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties @@ -381,6 +381,7 @@ AsofCannotBeCorrelated=ASOF JOIN does not support correlated subqueries UnknownRowField=ROW type does not have a field named ''{0}'': {1} IllegalRowIndexValue=ROW type does not have a field with index {0,number}; legal range is 1 to {1,number} UnequalRowSizes=Unequal number of entries in ROW expressions +RowStarNotAllowed=Star is not allowed in ROW constructor outside of query context IllegalRowIndex=Index in ROW type does not have a constant integer or string value CannotInferReturnType=Cannot infer return type for {0}; operand types: {1} SelectByCannotWithGroupBy=SELECT BY cannot be used with GROUP BY diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index 8a6877193dab..98c7ea665ec4 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -2101,6 +2101,15 @@ void testLikeAndSimilarFails() { .columnType("INTEGER NOT NULL"); } + /** Test case for + * [CALCITE-7311] + * Support the syntax ROW(*) to create a nested ROW type with all columns. */ + @Test void testRowWildcardExpansion() { + sql("select row(*) from emp").ok(); + sql("select row(emp.*) from emp").ok(); + sql("select row(emp.*, dept.*) from emp join dept on emp.deptno = dept.deptno").ok(); + } + @Test void testRowWithValidDot() { sql("select ((1,2),(3,4,5)).\"EXPR$1\".\"EXPR$2\"\n from dept") .columnType("INTEGER NOT NULL"); diff --git a/core/src/test/resources/sql/struct.iq b/core/src/test/resources/sql/struct.iq index 0706980cc412..61c892157302 100644 --- a/core/src/test/resources/sql/struct.iq +++ b/core/src/test/resources/sql/struct.iq @@ -146,6 +146,45 @@ select !ok +# [CALCITE-7311] Support the syntax ROW(*) to create a nested ROW type with all columns +select row(*) from emp limit 1; ++----------------------------------------------------------+ +| EXPR$0 | ++----------------------------------------------------------+ +| {7369, SMITH, CLERK, 7902, 1980-12-17, 800.00, null, 20} | ++----------------------------------------------------------+ +(1 row) + +!ok + +select row(emp.*) from emp limit 1; ++----------------------------------------------------------+ +| EXPR$0 | ++----------------------------------------------------------+ +| {7369, SMITH, CLERK, 7902, 1980-12-17, 800.00, null, 20} | ++----------------------------------------------------------+ +(1 row) + +!ok +select row(emp.*, dept.*) from emp join dept on emp.deptno = dept.deptno limit 1; ++---------------------------------------------------------------------------------------+ +| EXPR$0 | ++---------------------------------------------------------------------------------------+ +| {7782, CLARK, MANAGER, 7839, 1981-06-09, 2450.00, null, 10, 10, ACCOUNTING, NEW YORK} | ++---------------------------------------------------------------------------------------+ +(1 row) + +!ok + +select row(d.*, row(d.*)) from dept d limit 1; ++--------------------------------------------------------+ +| EXPR$0 | ++--------------------------------------------------------+ +| {10, ACCOUNTING, NEW YORK, {10, ACCOUNTING, NEW YORK}} | ++--------------------------------------------------------+ +(1 row) + +!ok # End struct.iq From 5add0defb03d57ab7394b54a968fc7c65dfcc3ba Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Thu, 18 Dec 2025 22:24:14 -0800 Subject: [PATCH 063/562] Add Feldera to 'powered by Calcite' page Signed-off-by: Mihai Budiu --- site/_docs/powered_by.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/site/_docs/powered_by.md b/site/_docs/powered_by.md index 809c01ca93f6..ca4f769f0dae 100644 --- a/site/_docs/powered_by.md +++ b/site/_docs/powered_by.md @@ -117,6 +117,13 @@ component provides a SQL interface to Dremio uses Calcite for SQL parsing and cost-based query optimization. +### Feldera + +Feldera's incremental view +maintenance engine uses Calcite for SQL parsing and high-level plan +optimizations. The Feldera SQL compiler generates Rust programs, +which then are run continuously in streaming mode. + ### HerdDB HerdDB From ec888f98c1c4d174c912635e4ff5ebbde5b9630a Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Fri, 19 Dec 2025 15:05:15 +0800 Subject: [PATCH 064/562] Minor optimizations to the document --- site/_docs/algebra.md | 35 ++++++------- site/_docs/materialized_views.md | 88 ++++++++++++++++---------------- 2 files changed, 61 insertions(+), 62 deletions(-) diff --git a/site/_docs/algebra.md b/site/_docs/algebra.md index 30a6b8175bd8..ed4b9cc9627a 100644 --- a/site/_docs/algebra.md +++ b/site/_docs/algebra.md @@ -155,7 +155,6 @@ straight, you can remove expressions from the stack. For example, here we are building a bushy join: {% highlight text %} -. join / \ join join @@ -212,8 +211,8 @@ each of the scalar expressions. The field names of an operator are guaranteed to be unique, but sometimes that means that the names are not exactly what you expect. For example, when you -join EMP to DEPT, one of the output fields will be called DEPTNO and another -will be called something like DEPTNO_1. +join `EMP` to `DEPT`, one of the output fields will be called `DEPTNO` and another +will be called something like `DEPTNO_1`. Some relational expression methods give you more control over field names: @@ -239,39 +238,39 @@ When you are building a relational expression that accepts multiple inputs, you need to build field references that take that into account. This occurs most often when building join conditions. -Suppose you are building a join on EMP, -which has 8 fields [EMPNO, ENAME, JOB, MGR, HIREDATE, SAL, COMM, DEPTNO] -and DEPT, -which has 3 fields [DEPTNO, DNAME, LOC]. +Suppose you are building a join on `EMP`, +which has 8 fields [`EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`] +and `DEPT`, +which has 3 fields [`DEPTNO`, `DNAME`, `LOC`]. Internally, Calcite represents those fields as offsets into a combined input row with 11 fields: the first field of the left input is -field #0 (0-based, remember), and the first field of the right input is -field #8. +field `#0 (0-based, remember)`, and the first field of the right input is +field `#8`. But through the builder API, you specify which field of which input. -To reference "SAL", internal field #5, +To reference `SAL`, internal field `#5`, write `builder.field(2, 0, "SAL")`, `builder.field(2, "EMP", "SAL")`, or `builder.field(2, 0, 5)`. -This means "the field #5 of input #0 of two inputs". +This means "the field `#5` of input `#0` of two inputs". (Why does it need to know that there are two inputs? Because they are stored on -the stack; input #1 is at the top of the stack, and input #0 is below it. +the stack; input `#1` is at the top of the stack, and input `#0` is below it. If we did not tell the builder that were two inputs, it would not know how deep -to go for input #0.) +to go for input `#0`.) -Similarly, to reference "DNAME", internal field #9 (8 + 1), +Similarly, to reference `DNAME`, internal field `#9 (8 + 1)`, write `builder.field(2, 1, "DNAME")`, `builder.field(2, "DEPT", "DNAME")`, or `builder.field(2, 1, 1)`. ### Recursive Queries Warning: The current API is experimental and subject to change without notice. -A SQL recursive query, e.g. this one that generates the sequence 1, 2, 3, ...10: +A SQL recursive query, e.g. this one that generates the sequence 1, 2, 3, ..., 10: {% highlight sql %} WITH RECURSIVE aux(i) AS ( VALUES (1) UNION ALL - SELECT i+1 FROM aux WHERE i < 10 + SELECT i + 1 FROM aux WHERE i < 10 ) SELECT * FROM aux {% endhighlight %} @@ -413,7 +412,7 @@ The following methods return a scalar expression ([RexNode]({{ site.apiRoot }}/org/apache/calcite/rex/RexNode.html)). Many of them use the contents of the stack. For example, `field("DEPTNO")` -returns a reference to the "DEPTNO" field of the relational expression just +returns a reference to the `DEPTNO` field of the relational expression just added to the stack. | Method | Description @@ -448,7 +447,7 @@ added to the stack. The following methods convert a sub-query into a scalar value (a `BOOLEAN` in the case of `in`, `exists`, `some`, `all`, `unique`; -any scalar type for `scalarQuery`). +any scalar type for `scalarQuery`), an `ARRAY` for `arrayQuery`, a `MAP` for `mapQuery`, and a `MULTISET` for `multisetQuery`). diff --git a/site/_docs/materialized_views.md b/site/_docs/materialized_views.md index 419835fa8b45..7d5862847f8a 100644 --- a/site/_docs/materialized_views.md +++ b/site/_docs/materialized_views.md @@ -77,7 +77,7 @@ To produce a larger number of rewritings, the rule relies on the information exp Let us illustrate with some examples the coverage of the view rewriting algorithm implemented in `MaterializedViewRule`. The examples are based on the following database schema. -```sql +{% highlight sql %} CREATE TABLE depts( deptno INT NOT NULL, deptname VARCHAR(20), @@ -98,7 +98,7 @@ CREATE TABLE emps( FOREIGN KEY (deptno) REFERENCES depts(deptno), FOREIGN KEY (locationid) REFERENCES locations(locationid) ); -``` +{% endhighlight %} ###### Join rewriting @@ -106,7 +106,7 @@ The rewriting can handle different join orders in the query and the view definit * Query: -```sql +{% highlight sql %} SELECT empid FROM depts JOIN ( @@ -114,80 +114,80 @@ JOIN ( FROM emps WHERE empid = 1) AS subq ON depts.deptno = subq.deptno -``` +{% endhighlight %} * Materialized view definition: -```sql +{% highlight sql %} SELECT empid FROM emps JOIN depts USING (deptno) -``` +{% endhighlight %} * Rewriting: -```sql +{% highlight sql %} SELECT empid FROM mv WHERE empid = 1 -``` +{% endhighlight %} ###### Aggregate rewriting * Query: -```sql +{% highlight sql %} SELECT deptno FROM emps WHERE deptno > 10 GROUP BY deptno -``` +{% endhighlight %} * Materialized view definition: -```sql +{% highlight sql %} SELECT empid, deptno FROM emps WHERE deptno > 5 GROUP BY empid, deptno -``` +{% endhighlight %} * Rewriting: -```sql +{% highlight sql %} SELECT deptno FROM mv WHERE deptno > 10 GROUP BY deptno -``` +{% endhighlight %} ###### Aggregate rewriting (with aggregation rollup) * Query: -```sql +{% highlight sql %} SELECT deptno, COUNT(*) AS c, SUM(salary) AS s FROM emps GROUP BY deptno -``` +{% endhighlight %} * Materialized view definition: -```sql +{% highlight sql %} SELECT empid, deptno, COUNT(*) AS c, SUM(salary) AS s FROM emps GROUP BY empid, deptno -``` +{% endhighlight %} * Rewriting: -```sql +{% highlight sql %} SELECT deptno, SUM(c), SUM(s) FROM mv GROUP BY deptno -``` +{% endhighlight %} ###### Query partial rewriting @@ -196,84 +196,84 @@ Through the declared constraints, the rule can detect joins that only append col * Query: -```sql +{% highlight sql %} SELECT deptno, COUNT(*) FROM emps GROUP BY deptno -``` +{% endhighlight %} * Materialized view definition: -```sql +{% highlight sql %} SELECT empid, depts.deptno, COUNT(*) AS c, SUM(salary) AS s FROM emps JOIN depts USING (deptno) GROUP BY empid, depts.deptno -``` +{% endhighlight %} * Rewriting: -```sql +{% highlight sql %} SELECT deptno, SUM(c) FROM mv GROUP BY deptno -``` +{% endhighlight %} ###### View partial rewriting * Query: -```sql +{% highlight sql %} SELECT deptname, state, SUM(salary) AS s FROM emps JOIN depts ON emps.deptno = depts.deptno JOIN locations ON emps.locationid = locations.locationid GROUP BY deptname, state -``` +{% endhighlight %} * Materialized view definition: -```sql +{% highlight sql %} SELECT empid, deptno, state, SUM(salary) AS s FROM emps JOIN locations ON emps.locationid = locations.locationid GROUP BY empid, deptno, state -``` +{% endhighlight %} * Rewriting: -```sql +{% highlight sql %} SELECT deptname, state, SUM(s) FROM mv JOIN depts ON mv.deptno = depts.deptno GROUP BY deptname, state -``` +{% endhighlight %} ###### Union rewriting * Query: -```sql +{% highlight sql %} SELECT empid, deptname FROM emps JOIN depts ON emps.deptno = depts.deptno WHERE salary > 10000 -``` +{% endhighlight %} * Materialized view definition: -```sql +{% highlight sql %} SELECT empid, deptname FROM emps JOIN depts ON emps.deptno = depts.deptno WHERE salary > 12000 -``` +{% endhighlight %} * Rewriting: -```sql +{% highlight sql %} SELECT empid, deptname FROM mv UNION ALL @@ -281,34 +281,34 @@ SELECT empid, deptname FROM emps JOIN depts ON emps.deptno = depts.deptno WHERE salary > 10000 AND salary <= 12000 -``` +{% endhighlight %} ###### Union rewriting with aggregate * Query: -```sql +{% highlight sql %} SELECT empid, deptname, SUM(salary) AS s FROM emps JOIN depts ON emps.deptno = depts.deptno WHERE salary > 10000 GROUP BY empid, deptname -``` +{% endhighlight %} * Materialized view definition: -```sql +{% highlight sql %} SELECT empid, deptname, SUM(salary) AS s FROM emps JOIN depts ON emps.deptno = depts.deptno WHERE salary > 12000 GROUP BY empid, deptname -``` +{% endhighlight %} * Rewriting: -```sql +{% highlight sql %} SELECT empid, deptname, SUM(s) FROM ( SELECT empid, deptname, s @@ -320,7 +320,7 @@ FROM ( WHERE salary > 10000 AND salary <= 12000 GROUP BY empid, deptname) AS subq GROUP BY empid, deptname -``` +{% endhighlight %} ##### Limitations From eaedb8a3e1a80008d2f5b2474c6df5955e518cd5 Mon Sep 17 00:00:00 2001 From: Issac Date: Mon, 15 Dec 2025 16:11:31 +0100 Subject: [PATCH 065/562] [CALCITE-7307] A new added dependency conflicts with Java 9+ and JPMS --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 7d021d2f344a..974408d02b58 100644 --- a/gradle.properties +++ b/gradle.properties @@ -132,7 +132,7 @@ jmh.version=1.12 jna.version=5.14.0 jna-platform.version=5.14.0 joda-time.version=2.8.1 -joou.version=0.9.4 +joou.version=0.9.5 json-path.version=2.10.0 json-smart.version=2.6.0 jsr305.version=3.0.2 From df823fb5c07edf2d513120494ffb01fa2afb888c Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Fri, 19 Dec 2025 11:22:41 -0800 Subject: [PATCH 066/562] Change current PMC chair Signed-off-by: Mihai Budiu --- site/_data/contributors.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/site/_data/contributors.yml b/site/_data/contributors.yml index f47ae3247bb3..14d83ab3ac1f 100644 --- a/site/_data/contributors.yml +++ b/site/_data/contributors.yml @@ -249,7 +249,8 @@ apacheId: mbudiu githubId: mihaibudiu org: Feldera.com - role: PMC + role: PMC Chair + homepage: https://mihaibudiu.github.io/work/index.html - name: Milinda Pathirage apacheId: milinda githubId: milinda @@ -292,7 +293,7 @@ apacheId: rubenql githubId: rubenada org: Voltron Data - role: PMC Chair + role: PMC - name: Rui Wang apacheId: amaliujia githubId: amaliujia From 652de497609f4c204eef08eee17300d26a5b4218 Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Sun, 21 Dec 2025 22:15:07 +0800 Subject: [PATCH 067/562] [CALCITE-3128] Joining two tables producing only NULLs will return 0 rows --- core/src/test/resources/sql/join.iq | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/core/src/test/resources/sql/join.iq b/core/src/test/resources/sql/join.iq index 7911edbf6b7e..ea04212757aa 100644 --- a/core/src/test/resources/sql/join.iq +++ b/core/src/test/resources/sql/join.iq @@ -1113,4 +1113,28 @@ WHERE "t"."EMPNO" = "t2"."EMPNO" AND "t"."EMPNO" > "t2"."EMPNO"); !ok +# [CALCITE-3128] Joining two tables producing only NULLs will return 0 rows +SELECT * FROM (SELECT NULLIF(5, 5)) a , (SELECT NULLIF(5, 5)) b; ++--------+---------+ +| EXPR$0 | EXPR$00 | ++--------+---------+ +| | | ++--------+---------+ +(1 row) + +!ok + +SELECT * FROM (VALUES (NULLIF(5, 5)), (NULLIF(5, 5))) a, (VALUES (NULLIF(5, 5)), (NULLIF(5, 5))) b; ++---+---+ +| A | B | ++---+---+ +| | | +| | | +| | | +| | | ++---+---+ +(4 rows) + +!ok + # End join.iq From d36c3ccb713e35522b56674fc91f8de225e50987 Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Sat, 20 Dec 2025 09:48:10 +0800 Subject: [PATCH 068/562] [CALCITE-7338] Window hints are not propagated to window rel nodes --- .../calcite/rel/logical/LogicalWindow.java | 32 +++++++++--- .../rel/logical/ToLogicalConverter.java | 4 +- .../calcite/rel/mutable/MutableRels.java | 2 +- .../calcite/rel/rules/CalcRelSplitter.java | 49 ++++++++++++++++++- .../rel/rules/ProjectToWindowRule.java | 13 +++-- .../rel/rules/ProjectWindowTransposeRule.java | 4 +- .../rel/rules/ReduceExpressionsRule.java | 2 +- .../calcite/test/SqlHintsConverterTest.java | 24 +++++++++ 8 files changed, 110 insertions(+), 20 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rel/logical/LogicalWindow.java b/core/src/main/java/org/apache/calcite/rel/logical/LogicalWindow.java index d17b8aef859e..815b8f8ca596 100644 --- a/core/src/main/java/org/apache/calcite/rel/logical/LogicalWindow.java +++ b/core/src/main/java/org/apache/calcite/rel/logical/LogicalWindow.java @@ -102,36 +102,54 @@ public LogicalWindow(RelOptCluster cluster, RelTraitSet traitSet, RelNode input, @Override public LogicalWindow copy(RelTraitSet traitSet, List inputs) { - return new LogicalWindow(getCluster(), traitSet, sole(inputs), constants, + return new LogicalWindow(getCluster(), traitSet, hints, sole(inputs), constants, getRowType(), groups); } @Override public Window copy(List constants) { - return new LogicalWindow(getCluster(), getTraitSet(), getInput(), + return new LogicalWindow(getCluster(), getTraitSet(), hints, getInput(), constants, getRowType(), groups); } + @Deprecated // to be removed before 2.0 + public static LogicalWindow create(RelTraitSet traitSet, RelNode input, + List constants, RelDataType rowType, List groups) { + return create(traitSet, Collections.emptyList(), input, constants, rowType, groups); + } + /** * Creates a LogicalWindow. * - * @param input Input relational expression * @param traitSet Trait set + * @param hints Hints + * @param input Input relational expression * @param constants List of constants that are additional inputs * @param rowType Output row type * @param groups Window groups */ - public static LogicalWindow create(RelTraitSet traitSet, RelNode input, - List constants, RelDataType rowType, List groups) { - return new LogicalWindow(input.getCluster(), traitSet, input, constants, + public static LogicalWindow create(RelTraitSet traitSet, List hints, + RelNode input, List constants, RelDataType rowType, + List groups) { + return new LogicalWindow(input.getCluster(), traitSet, hints, input, constants, rowType, groups); } /** * Creates a LogicalWindow by parsing a {@link RexProgram}. */ + @Deprecated // to be removed before 2.0 public static RelNode create(RelOptCluster cluster, RelTraitSet traitSet, RelBuilder relBuilder, RelNode child, final RexProgram program) { + return create(cluster, traitSet, relBuilder, child, program, Collections.emptyList()); + } + + /** + * Creates a LogicalWindow by parsing a {@link RexProgram}. + */ + public static RelNode create(RelOptCluster cluster, + RelTraitSet traitSet, RelBuilder relBuilder, RelNode child, + final RexProgram program, List hints) { final RelDataType outRowType = program.getOutputRowType(); // Build a list of distinct groups, partitions and aggregate // functions. @@ -288,7 +306,7 @@ public static RelNode create(RelOptCluster cluster, }; final LogicalWindow window = - LogicalWindow.create(traitSet, child, constants, intermediateRowType, + LogicalWindow.create(traitSet, hints, child, constants, intermediateRowType, groups); // The order that the "over" calls occur in the groups and diff --git a/core/src/main/java/org/apache/calcite/rel/logical/ToLogicalConverter.java b/core/src/main/java/org/apache/calcite/rel/logical/ToLogicalConverter.java index eaf85133146d..86d08f5641e3 100644 --- a/core/src/main/java/org/apache/calcite/rel/logical/ToLogicalConverter.java +++ b/core/src/main/java/org/apache/calcite/rel/logical/ToLogicalConverter.java @@ -144,8 +144,8 @@ public ToLogicalConverter(RelBuilder relBuilder) { if (relNode instanceof Window) { final Window window = (Window) relNode; final RelNode input = visit(window.getInput()); - return LogicalWindow.create(input.getTraitSet(), input, window.constants, - window.getRowType(), window.groups); + return LogicalWindow.create(input.getTraitSet(), window.getHints(), + input, window.constants, window.getRowType(), window.groups); } if (relNode instanceof Calc) { diff --git a/core/src/main/java/org/apache/calcite/rel/mutable/MutableRels.java b/core/src/main/java/org/apache/calcite/rel/mutable/MutableRels.java index 54761eeef003..ed509b6d60b6 100644 --- a/core/src/main/java/org/apache/calcite/rel/mutable/MutableRels.java +++ b/core/src/main/java/org/apache/calcite/rel/mutable/MutableRels.java @@ -262,7 +262,7 @@ public static RelNode fromMutable(MutableRel node, RelBuilder relBuilder) { case WINDOW: { final MutableWindow window = (MutableWindow) node; final RelNode child = fromMutable(window.getInput(), relBuilder); - return LogicalWindow.create(child.getTraitSet(), + return LogicalWindow.create(child.getTraitSet(), ImmutableList.of(), child, window.constants, window.rowType, window.groups); } case MATCH: { diff --git a/core/src/main/java/org/apache/calcite/rel/rules/CalcRelSplitter.java b/core/src/main/java/org/apache/calcite/rel/rules/CalcRelSplitter.java index fde19567e8c6..67bbbae24626 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/CalcRelSplitter.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/CalcRelSplitter.java @@ -21,6 +21,8 @@ import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Calc; +import org.apache.calcite.rel.hint.Hintable; +import org.apache.calcite.rel.hint.RelHint; import org.apache.calcite.rel.logical.LogicalCalc; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; @@ -88,6 +90,7 @@ public abstract class CalcRelSplitter { //~ Instance fields -------------------------------------------------------- protected final RexProgram program; + protected final List hints; private final RelDataTypeFactory typeFactory; private final List relTypes; @@ -108,6 +111,7 @@ public abstract class CalcRelSplitter { CalcRelSplitter(Calc calc, RelBuilder relBuilder, RelType[] relTypes) { this.relBuilder = relBuilder; this.program = calc.getProgram(); + this.hints = calc.getHints(); this.cluster = calc.getCluster(); this.traits = calc.getTraitSet(); this.typeFactory = calc.getCluster().getTypeFactory(); @@ -224,8 +228,35 @@ RelNode execute() { projectExprOrdinals, conditionExprOrdinal, outputRowType); + + // Propagate hints to each level. Since CalcRelSplitter builds a vertical stack of + // relational expressions (bottom-up), the relative depth of a level from the + // original top-level Calc determines how many '0's must be appended to the + // hint's inheritPath to maintain correct mapping. + // + // Example: SELECT /*+ Hint message */ SUM(v1) OVER(P1), SUM(v2) OVER(P2) FROM t + // split into 3 levels: + // LogicalProject, relativeDepth = 0, path = [] + // LogicalWindow (P2), relativeDepth = 1, path = [0] + // LogicalWindow (P1), relativeDepth = 2, path = [0, 0] + final List levelHints; + final int relativeDepth = (levelCount - 1) - level; + if (hints.isEmpty() || relativeDepth == 0) { + levelHints = hints; + } else { + levelHints = new ArrayList<>(hints.size()); + for (RelHint hint : hints) { + List newPath = new ArrayList<>(hint.inheritPath.size() + relativeDepth); + newPath.addAll(hint.inheritPath); + for (int i = 0; i < relativeDepth; i++) { + newPath.add(0); + } + levelHints.add(hint.copy(newPath)); + } + } + rel = - relType.makeRel(cluster, traits, relBuilder, rel, program1); + relType.makeRel(cluster, traits, relBuilder, rel, program1, levelHints); // Sometimes a level's program merely projects its inputs. We don't // want these. They cause an explosion in the search space. @@ -757,10 +788,24 @@ protected boolean supportsCondition() { return true; } + @Deprecated // to be removed before 2.0 protected RelNode makeRel(RelOptCluster cluster, RelTraitSet traitSet, RelBuilder relBuilder, RelNode input, RexProgram program) { - return LogicalCalc.create(input, program); + return makeRel(cluster, traitSet, relBuilder, input, program, ImmutableList.of()); + } + + protected RelNode makeRel(RelOptCluster cluster, + RelTraitSet traitSet, + RelBuilder relBuilder, + RelNode input, + RexProgram program, + List hints) { + RelNode rel = LogicalCalc.create(input, program); + if (!hints.isEmpty()) { + rel = ((Hintable) rel).withHints(hints); + } + return rel; } /** diff --git a/core/src/main/java/org/apache/calcite/rel/rules/ProjectToWindowRule.java b/core/src/main/java/org/apache/calcite/rel/rules/ProjectToWindowRule.java index 37ef7b888c43..931169135fd4 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/ProjectToWindowRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/ProjectToWindowRule.java @@ -23,6 +23,7 @@ import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Calc; import org.apache.calcite.rel.core.Project; +import org.apache.calcite.rel.hint.RelHint; import org.apache.calcite.rel.logical.LogicalCalc; import org.apache.calcite.rel.logical.LogicalWindow; import org.apache.calcite.rex.RexBiVisitorImpl; @@ -156,7 +157,9 @@ public ProjectToLogicalProjectAndWindowRule( project.getRowType(), project.getCluster().getRexBuilder()); // temporary LogicalCalc, never registered - final LogicalCalc calc = LogicalCalc.create(input, program); + final LogicalCalc calc = + new LogicalCalc(project.getCluster(), project.getTraitSet(), + project.getHints(), input, program); final CalcRelSplitter transform = new WindowedAggRelSplitter(calc, call.builder()) { @Override protected RelNode handle(RelNode rel) { @@ -226,11 +229,11 @@ static class WindowedAggRelSplitter extends CalcRelSplitter { @Override protected RelNode makeRel(RelOptCluster cluster, RelTraitSet traitSet, RelBuilder relBuilder, RelNode input, - RexProgram program) { + RexProgram program, List hints) { assert !program.containsAggs(); program = program.normalize(cluster.getRexBuilder(), null); return super.makeRel(cluster, traitSet, relBuilder, input, - program); + program, hints); } }, new RelType("WinAggRelType") { @@ -255,11 +258,11 @@ static class WindowedAggRelSplitter extends CalcRelSplitter { } @Override protected RelNode makeRel(RelOptCluster cluster, RelTraitSet traitSet, - RelBuilder relBuilder, RelNode input, RexProgram program) { + RelBuilder relBuilder, RelNode input, RexProgram program, List hints) { checkArgument(program.getCondition() == null, "WindowedAggregateRel cannot accept a condition"); return LogicalWindow.create(cluster, traitSet, relBuilder, input, - program); + program, hints); } } }; diff --git a/core/src/main/java/org/apache/calcite/rel/rules/ProjectWindowTransposeRule.java b/core/src/main/java/org/apache/calcite/rel/rules/ProjectWindowTransposeRule.java index fac0f459accb..e606fc8b726d 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/ProjectWindowTransposeRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/ProjectWindowTransposeRule.java @@ -172,8 +172,8 @@ public ProjectWindowTransposeRule(RelBuilderFactory relBuilderFactory) { } final LogicalWindow newLogicalWindow = - LogicalWindow.create(window.getTraitSet(), projectBelowWindow, - window.constants, outputBuilder.build(), groups); + LogicalWindow.create(window.getTraitSet(), window.getHints(), + projectBelowWindow, window.constants, outputBuilder.build(), groups); // Modify the top LogicalProject final List topProjExps = diff --git a/core/src/main/java/org/apache/calcite/rel/rules/ReduceExpressionsRule.java b/core/src/main/java/org/apache/calcite/rel/rules/ReduceExpressionsRule.java index fa6864ac42c9..88ffaf72fcdb 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/ReduceExpressionsRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/ReduceExpressionsRule.java @@ -625,7 +625,7 @@ public WindowReduceExpressionsRule(Class windowClass, } if (reduced) { call.transformTo(LogicalWindow - .create(window.getTraitSet(), window.getInput(), + .create(window.getTraitSet(), window.getHints(), window.getInput(), window.getConstants(), window.getRowType(), groups)); call.getPlanner().prune(window); } diff --git a/core/src/test/java/org/apache/calcite/test/SqlHintsConverterTest.java b/core/src/test/java/org/apache/calcite/test/SqlHintsConverterTest.java index 9a75c50ada20..cff0c2047459 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlHintsConverterTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlHintsConverterTest.java @@ -58,6 +58,7 @@ import org.apache.calcite.rel.logical.LogicalUnion; import org.apache.calcite.rel.logical.LogicalValues; import org.apache.calcite.rel.rules.CoreRules; +import org.apache.calcite.rel.rules.ProjectToWindowRule; import org.apache.calcite.sql.SqlDelete; import org.apache.calcite.sql.SqlInsert; import org.apache.calcite.sql.SqlMerge; @@ -270,6 +271,29 @@ public final Fixture sql(String sql) { sql(sql).ok(); } + /** Test case for + * [CALCITE-7338] + * Window hints are not propagated to window rel nodes. */ + @Test void testWindowHintsPropagateAfterProjectToWindowRule() { + final String sql = "select /*+ mini_batch */ last_value(deptno)\n" + + "over (order by empno rows 2 following) from emp"; + + // Build rel with the same HintStrategyTable as this class + final RelNode rel = sql(sql).toRel(); + + // Run the rule that materializes LogicalWindow + HepProgramBuilder builder = new HepProgramBuilder(); + builder.addRuleClass(ProjectToWindowRule.class); + HepPlanner planner = new HepPlanner(builder.build()); + planner.addRule(CoreRules.PROJECT_TO_LOGICAL_PROJECT_AND_WINDOW); + planner.setRoot(rel); + RelNode transformed = planner.findBestExp(); + + // Expect the hint to be on LogicalWindow after the rule. + final RelHint expected = RelHint.builder("MINI_BATCH").inheritPath(0).build(); + new ValidateHintVisitor(expected, Window.class).go(transformed); + } + @Test void testHintsInSubQueryWithDecorrelation() { final String sql = "select /*+ resource(parallelism='3'), AGG_STRATEGY(TWO_PHASE) */\n" + "sum(e1.empno) from emp e1, dept d1\n" From 4dc0c0aa1a0fa0f696599d4544c8054feeb22f8d Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Sat, 20 Dec 2025 21:39:43 +0800 Subject: [PATCH 069/562] [CALCITE-2274] Filter predicates aren't inferred while using dynamic star in subquery --- .../apache/calcite/test/RelOptRulesTest.java | 11 ++++++++ .../apache/calcite/test/RelOptRulesTest.xml | 28 +++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index ac7a55984f26..959cdada0bc9 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -6455,6 +6455,17 @@ private HepProgram getTransitiveProgram() { .check(); } + /** Test case for + * [CALCITE-2274] + * Filter predicates aren't inferred while using dynamic star in subquery. */ + @Test void testTransitiveInferenceJoinUsingStar() { + final String sql = "SELECT * FROM sales.emp d JOIN\n" + + "(SELECT * FROM sales.emp WHERE deptno = 4) e\n" + + "ON e.deptno = d.deptno"; + sql(sql).withPre(getTransitiveProgram()) + .withRule(CoreRules.JOIN_PUSH_TRANSITIVE_PREDICATES).check(); + } + /** Tests propagation of a filter derived from an "IS NOT DISTINCT FROM" * predicate. * diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index 73f362b3f0b8..e651fcd74e6c 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -20521,6 +20521,34 @@ LogicalProject(EXPR$0=[1]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) LogicalFilter(condition=[>($7, 7)]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +]]> + + + + + + + + + + + From d0c77e91e07f6d0e5a9cd2bb6bf8890024d43b94 Mon Sep 17 00:00:00 2001 From: iwanttobepowerful <745778074@qq.com> Date: Mon, 15 Dec 2025 17:32:41 +0800 Subject: [PATCH 070/562] [CALCITE-7272] Subqueries cannot be decorrelated if have set op --- .../calcite/sql2rel/RelDecorrelator.java | 216 ++- .../calcite/sql2rel/RelDecorrelatorTest.java | 415 ++++++ .../apache/calcite/test/RelOptRulesTest.xml | 10 +- core/src/test/resources/sql/sub-query.iq | 1185 +++++++++++++++++ 4 files changed, 1779 insertions(+), 47 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java b/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java index db647da89cba..c5986208d22d 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java @@ -44,6 +44,7 @@ import org.apache.calcite.rel.core.JoinRelType; import org.apache.calcite.rel.core.Project; import org.apache.calcite.rel.core.RelFactories; +import org.apache.calcite.rel.core.SetOp; import org.apache.calcite.rel.core.Sort; import org.apache.calcite.rel.logical.LogicalAggregate; import org.apache.calcite.rel.logical.LogicalCorrelate; @@ -817,8 +818,7 @@ protected RexNode removeCorrelationExpr( } } - if (rel.getGroupType() == Aggregate.Group.SIMPLE - && rel.getGroupSet().isEmpty() + if ((rel.hasEmptyGroup() || rel.getGroupSet().isEmpty()) && !frame.corDefOutputs.isEmpty() && !parentPropagatesNullValues) { newRel = rewriteScalarAggregate(rel, newRel, outputMap, corDefOutputs); @@ -930,71 +930,63 @@ private RelNode rewriteScalarAggregate(Aggregate oldRel, RelNode newRel, Map outputMap, NavigableMap corDefOutputs) { - final Pair outerFramePair = requireNonNull(this.frameStack.peek()); - final Frame outFrame = outerFramePair.right; - RexBuilder rexBuilder = relBuilder.getRexBuilder(); + final CorelMap localCorelMap = new CorelMapBuilder().build(oldRel); + final List corVarList = new ArrayList<>(localCorelMap.mapRefRelToCorRef.values()); + Collections.sort(corVarList); - int groupKeySize = (int) corDefOutputs.keySet().stream() - .filter(a -> a.corr.equals(outerFramePair.left)) - .count(); - List newRelFields = newRel.getRowType().getFieldList(); - ImmutableBitSet.Builder corFieldBuilder = ImmutableBitSet.builder(); + final NavigableMap valueGenCorDefOutputs = new TreeMap<>(); + final RelNode valueGen = + requireNonNull(createValueGenerator(corVarList, 0, valueGenCorDefOutputs)); + final int valueGenFieldCount = valueGen.getRowType().getFieldCount(); - // Here we record the mapping between the original index and the new project. - // For the count, we map it as `case when x is null then 0 else x`. + // Build join conditions final Map newProjectMap = new HashMap<>(); final List conditions = new ArrayList<>(); for (Map.Entry corDefOutput : corDefOutputs.entrySet()) { - CorDef corDef = corDefOutput.getKey(); - Integer corIndex = corDefOutput.getValue(); - if (corDef.corr.equals(outerFramePair.left)) { - int newIdx = requireNonNull(outFrame.oldToNewOutputs.get(corDef.field)); - corFieldBuilder.set(newIdx); - - RelDataType type = outFrame.r.getRowType().getFieldList().get(newIdx).getType(); - RexNode left = new RexInputRef(corFieldBuilder.cardinality() - 1, type); - newProjectMap.put(corIndex + groupKeySize, left); - conditions.add( - relBuilder.isNotDistinctFrom(left, - new RexInputRef(corIndex + groupKeySize, - newRelFields.get(corIndex).getType()))); - } - } - - ImmutableBitSet groupSet = corFieldBuilder.build(); - // Build [09] LogicalAggregate(group=[{0}]) to obtain the distinct set of - // corVar from outFrame. - relBuilder.push(outFrame.r) - .aggregate(relBuilder.groupKey(groupSet)); + final CorDef corDef = corDefOutput.getKey(); + final int leftPos = requireNonNull(valueGenCorDefOutputs.get(corDef)); + final int rightPos = corDefOutput.getValue(); + final RelDataType leftType = valueGen.getRowType().getFieldList().get(leftPos).getType(); + final RelDataType rightType = newRel.getRowType().getFieldList().get(rightPos).getType(); + final RexNode leftRef = new RexInputRef(leftPos, leftType); + final RexNode rightRef = new RexInputRef(valueGenFieldCount + rightPos, rightType); + conditions.add(relBuilder.isNotDistinctFrom(leftRef, rightRef)); + newProjectMap.put(valueGenFieldCount + rightPos, leftRef); + } + final RexNode joinCond = RexUtil.composeConjunction(relBuilder.getRexBuilder(), conditions); // Build [08] LogicalJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left]) // to ensure each corVar's aggregate result is output. - final RelNode join = relBuilder.push(newRel) - .join(JoinRelType.LEFT, conditions).build(); + final RelNode join = relBuilder.push(valueGen).push(newRel) + .join(JoinRelType.LEFT, joinCond).build(); + RelDataType joinRowType = join.getRowType(); + RexBuilder rexBuilder = relBuilder.getRexBuilder(); + // Here we record the mapping between the original index and the new project. + // For the count, we map it as `case when x is null then 0 else x`. for (int i1 = 0; i1 < oldRel.getAggCallList().size(); i1++) { AggregateCall aggCall = oldRel.getAggCallList().get(i1); if (aggCall.getAggregation() instanceof SqlCountAggFunction) { int index = requireNonNull(outputMap.get(i1 + oldRel.getGroupSet().size())); - final RexInputRef ref = RexInputRef.of(index + groupKeySize, join.getRowType()); - RexNode specificCountValue = - rexBuilder.makeCall(SqlStdOperatorTable.CASE, - ImmutableList.of(relBuilder.isNotNull(ref), ref, relBuilder.literal(0))); + final RexInputRef ref = RexInputRef.of(index + valueGenFieldCount, joinRowType); + ImmutableList exprs = + ImmutableList.of(relBuilder.isNotNull(ref), ref, relBuilder.literal(0)); + RexNode specificCountValue = rexBuilder.makeCall(SqlStdOperatorTable.CASE, exprs); newProjectMap.put(ref.getIndex(), specificCountValue); } } + // Build [07] LogicalProject(DEPTNO=[$0], EXPR$0=[CASE(IS NOT NULL($2), $2, 0)]) + // to handle COUNT function by converting nulls to zero. final List newProjects = new ArrayList<>(); - for (int index : ImmutableBitSet.range(groupKeySize, join.getRowType().getFieldCount())) { + for (int index : ImmutableBitSet.range(valueGenFieldCount, joinRowType.getFieldCount())) { if (newProjectMap.containsKey(index)) { newProjects.add(requireNonNull(newProjectMap.get(index))); } else { - newProjects.add(RexInputRef.of(index, join.getRowType())); + newProjects.add(RexInputRef.of(index, joinRowType)); } } - // Build [07] LogicalProject(DEPTNO=[$0], EXPR$0=[CASE(IS NOT NULL($2), $2, 0)]) - // to handle COUNT function by converting nulls to zero. return relBuilder.push(join) .project(newProjects, newRel.getRowType().getFieldNames()) .build(); @@ -1184,6 +1176,144 @@ private static void shiftMapping(Map mapping, int startIndex, return null; } + /** + * Given the SQL: + * SELECT ename, + * (SELECT sum(c) + * FROM + * (SELECT deptno AS c + * FROM dept + * WHERE dept.deptno = emp.deptno + * UNION ALL + * SELECT 2 AS c + * FROM bonus + * WHERE bonus.job = emp.job) AS union_subquery + * ) AS correlated_sum + * FROM emp; + * + *

    from: + * LogicalUnion(all=[true]) + * LogicalProject(C=[CAST($0):INTEGER NOT NULL]) + * LogicalFilter(condition=[=($0, $cor0.DEPTNO)]) + * LogicalTableScan(table=[[scott, DEPT]]) + * LogicalProject(C=[2]) + * LogicalFilter(condition=[=($1, $cor0.JOB)]) + * LogicalTableScan(table=[[scott, BONUS]]) + * + *

    to: + * LogicalUnion(all=[true]) + * LogicalProject(JOB=[$0], DEPTNO=[$1], C=[$2]) + * LogicalJoin(condition=[IS NOT DISTINCT FROM($1, $3)], joinType=[inner]) + * LogicalAggregate(group=[{0, 1}]) + * LogicalProject(JOB=[$2], DEPTNO=[$7]) + * LogicalTableScan(table=[[scott, EMP]]) + * LogicalProject(C=[CAST($0):INTEGER NOT NULL], DEPTNO=[$0]) + * LogicalTableScan(table=[[scott, DEPT]]) + * LogicalProject(JOB=[$0], DEPTNO=[$1], C=[$2]) + * LogicalJoin(condition=[IS NOT DISTINCT FROM($0, $3)], joinType=[inner]) + * LogicalAggregate(group=[{0, 1}]) + * LogicalProject(JOB=[$2], DEPTNO=[$7]) + * LogicalTableScan(table=[[scott, EMP]]) + * LogicalProject(C=[2], JOB=[$1]) + * LogicalFilter(condition=[IS NOT NULL($1)]) + * LogicalTableScan(table=[[scott, BONUS]]) + */ + public @Nullable Frame decorrelateRel(SetOp rel, boolean isCorVarDefined, + boolean parentPropagatesNullValues) { + if (!isCorVarDefined) { + return decorrelateRel((RelNode) rel, false, parentPropagatesNullValues); + } + + final CorelMap localCorelMap = new CorelMapBuilder().build(rel); + final List corVarList = new ArrayList<>(localCorelMap.mapRefRelToCorRef.values()); + Collections.sort(corVarList); + + final NavigableMap valueGenCorDefOutputs = new TreeMap<>(); + final RelNode valueGen = + requireNonNull(createValueGenerator(corVarList, 0, valueGenCorDefOutputs)); + final int valueGenFieldCount = valueGen.getRowType().getFieldCount(); + // Original SetOp payload width. + final int payloadFieldCount = rel.getRowType().getFieldCount(); + final List newInputs = new ArrayList<>(); + final Map setOpOldToNewOutputs = new HashMap<>(); + final NavigableMap setOpCorDefOutputs = new TreeMap<>(); + + for (int i = 0; i < rel.getInputs().size(); i++) { + RelNode oldInput = rel.getInput(i); + Frame frame = getInvoke(oldInput, true, rel, parentPropagatesNullValues); + if (frame == null) { + // If input has not been rewritten, do not rewrite this rel. + return null; + } + + // Build join conditions: for each CorDef of this branch that belongs + // to the current outFrameCorrId, equate valueGen(col) with branch(col). + final List conditions = new ArrayList<>(); + for (Map.Entry e : frame.corDefOutputs.entrySet()) { + final CorDef corDef = e.getKey(); + final int leftPos = requireNonNull(valueGenCorDefOutputs.get(corDef)); + final int rightPos = e.getValue(); + final RelDataType leftType = valueGen.getRowType().getFieldList().get(leftPos).getType(); + final RelDataType rightType = frame.r.getRowType().getFieldList().get(rightPos).getType(); + final RexNode leftRef = new RexInputRef(leftPos, leftType); + final RexNode rightRef = new RexInputRef(valueGenFieldCount + rightPos, rightType); + conditions.add(relBuilder.isNotDistinctFrom(leftRef, rightRef)); + } + final RexNode joinCondition = + RexUtil.composeConjunction(relBuilder.getRexBuilder(), conditions); + RelNode join = relBuilder.push(valueGen).push(frame.r) + .join(JoinRelType.INNER, joinCondition).build(); + + final List joinFields = join.getRowType().getFieldList(); + + // Build the final projection for this branch: + // all correlated columns (from valueGen), original payload columns (from branch) + final PairList projects = PairList.of(); + final Map childOldToNew = new HashMap<>(); + final NavigableMap childCorDefOutputs = new TreeMap<>(); + + // a) Correlated columns, in the order of valueGenCorDefOutputs. + int newPos = 0; + for (Map.Entry e : valueGenCorDefOutputs.entrySet()) { + final int srcIndex = e.getValue(); + RexInputRef inputRef = RexInputRef.of(srcIndex, join.getRowType()); + String name = joinFields.get(srcIndex).getName(); + + projects.add(inputRef, name); + childCorDefOutputs.put(e.getKey(), newPos); + newPos++; + } + + // b) Original SetOp payload columns. + for (int oldIndex = 0; oldIndex < payloadFieldCount; oldIndex++) { + final Integer srcInFrame = requireNonNull(frame.oldToNewOutputs.get(oldIndex)); + final int srcInJoin = valueGenFieldCount + srcInFrame; + RexInputRef inputRef = RexInputRef.of(srcInJoin, join.getRowType()); + String name = joinFields.get(srcInJoin).getName(); + + projects.add(inputRef, name); + childOldToNew.put(oldIndex, newPos); + newPos++; + } + + final RelNode newInput = relBuilder.push(join) + .projectNamed(projects.leftList(), projects.rightList(), true) + .build(); + newInputs.add(newInput); + + register(oldInput, newInput, childOldToNew, childCorDefOutputs); + + // Use the first branch as prototype for the SetOp's frame mappings. + if (i == 0) { + setOpOldToNewOutputs.putAll(childOldToNew); + setOpCorDefOutputs.putAll(childCorDefOutputs); + } + } + + final SetOp newSetOp = rel.copy(rel.getTraitSet(), newInputs, rel.all); + return register(rel, newSetOp, setOpOldToNewOutputs, setOpCorDefOutputs); + } + public @Nullable Frame decorrelateRel(LogicalProject rel, boolean isCorVarDefined, boolean parentPropagatesNullValues) { return decorrelateRel((Project) rel, isCorVarDefined, parentPropagatesNullValues); diff --git a/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java b/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java index b92606be5b23..80a3dda64651 100644 --- a/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java +++ b/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java @@ -425,6 +425,421 @@ public static Frameworks.ConfigBuilder config() { assertThat(after, hasTree(planAfter)); } + @Test void testCorrelationInSetOp0() { + final FrameworkConfig frameworkConfig = config().build(); + final RelBuilder builder = RelBuilder.create(frameworkConfig); + final RelOptCluster cluster = builder.getCluster(); + final Planner planner = Frameworks.getPlanner(frameworkConfig); + final String sql = "SELECT ename,\n" + + " (SELECT sum(c)\n" + + " FROM\n" + + " (SELECT deptno AS c\n" + + " FROM dept\n" + + " WHERE dept.deptno = emp.deptno\n" + + " UNION ALL\n" + + " SELECT 2 AS c\n" + + " FROM bonus) AS union_subquery\n" + + " ) AS correlated_sum\n" + + "FROM emp\n" + + "ORDER BY ename"; + final RelNode originalRel; + try { + final SqlNode parse = planner.parse(sql); + final SqlNode validate = planner.validate(parse); + originalRel = planner.rel(validate).rel; + } catch (Exception e) { + throw TestUtil.rethrow(e); + } + + final HepProgram hepProgram = HepProgram.builder() + .addRuleCollection( + ImmutableList.of( + // SubQuery program rules + CoreRules.FILTER_SUB_QUERY_TO_CORRELATE, + CoreRules.PROJECT_SUB_QUERY_TO_CORRELATE, + CoreRules.JOIN_SUB_QUERY_TO_CORRELATE)) + .build(); + final Program program = + Programs.of(hepProgram, true, + requireNonNull(cluster.getMetadataProvider())); + final RelNode before = + program.run(cluster.getPlanner(), originalRel, cluster.traitSet(), + Collections.emptyList(), Collections.emptyList()); + final String planBefore = "" + + "LogicalSort(sort0=[$0], dir0=[ASC])\n" + + " LogicalProject(ENAME=[$1], CORRELATED_SUM=[$8])\n" + + " LogicalCorrelate(correlation=[$cor0], joinType=[left], requiredColumns=[{7}])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalAggregate(group=[{}], EXPR$0=[SUM($0)])\n" + + " LogicalUnion(all=[true])\n" + + " LogicalProject(C=[CAST($0):INTEGER NOT NULL])\n" + + " LogicalFilter(condition=[=($0, $cor0.DEPTNO)])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n" + + " LogicalProject(C=[2])\n" + + " LogicalTableScan(table=[[scott, BONUS]])\n"; + assertThat(before, hasTree(planBefore)); + + // Decorrelate without any rules, just "purely" decorrelation algorithm on RelDecorrelator + final RelNode after = + RelDecorrelator.decorrelateQuery(before, builder, RuleSets.ofList(Collections.emptyList()), + RuleSets.ofList(Collections.emptyList())); + // Verify plan + final String planAfter = "" + + "LogicalSort(sort0=[$0], dir0=[ASC])\n" + + " LogicalProject(ENAME=[$1], CORRELATED_SUM=[$9])\n" + + " LogicalJoin(condition=[IS NOT DISTINCT FROM($7, $8)], joinType=[left])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalAggregate(group=[{0}], EXPR$0=[SUM($1)])\n" + + " LogicalProject(DEPTNO=[$0], C=[$1])\n" + + " LogicalUnion(all=[true])\n" + + " LogicalProject(DEPTNO=[$0], C=[$1])\n" + + " LogicalJoin(condition=[IS NOT DISTINCT FROM($0, $2)], joinType=[inner])\n" + + " LogicalAggregate(group=[{0}])\n" + + " LogicalProject(DEPTNO=[$7])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalProject(C=[CAST($0):INTEGER NOT NULL], DEPTNO=[$0])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n" + + " LogicalProject(DEPTNO=[$0], C=[$1])\n" + + " LogicalJoin(condition=[true], joinType=[inner])\n" + + " LogicalAggregate(group=[{0}])\n" + + " LogicalProject(DEPTNO=[$7])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalProject(C=[2])\n" + + " LogicalTableScan(table=[[scott, BONUS]])\n"; + assertThat(after, hasTree(planAfter)); + } + + /** Test case for [CALCITE-7272] + * Subqueries cannot be decorrelated if have set op. */ + @Test void testCorrelationInSetOp1() { + final FrameworkConfig frameworkConfig = config().build(); + final RelBuilder builder = RelBuilder.create(frameworkConfig); + final RelOptCluster cluster = builder.getCluster(); + final Planner planner = Frameworks.getPlanner(frameworkConfig); + final String sql = "SELECT ename,\n" + + " (SELECT sum(c)\n" + + " FROM\n" + + " (SELECT deptno AS c\n" + + " FROM dept\n" + + " WHERE dept.deptno = emp.deptno\n" + + " UNION ALL\n" + + " SELECT 2 AS c\n" + + " FROM bonus\n" + + " WHERE bonus.job = emp.job) AS union_subquery\n" + + " ) AS correlated_sum\n" + + "FROM emp\n" + + "ORDER BY ename"; + + final RelNode originalRel; + try { + final SqlNode parse = planner.parse(sql); + final SqlNode validate = planner.validate(parse); + originalRel = planner.rel(validate).rel; + } catch (Exception e) { + throw TestUtil.rethrow(e); + } + + final HepProgram hepProgram = HepProgram.builder() + .addRuleCollection( + ImmutableList.of( + // SubQuery program rules + CoreRules.FILTER_SUB_QUERY_TO_CORRELATE, + CoreRules.PROJECT_SUB_QUERY_TO_CORRELATE, + CoreRules.JOIN_SUB_QUERY_TO_CORRELATE)) + .build(); + final Program program = + Programs.of(hepProgram, true, + requireNonNull(cluster.getMetadataProvider())); + final RelNode before = + program.run(cluster.getPlanner(), originalRel, cluster.traitSet(), + Collections.emptyList(), Collections.emptyList()); + final String planBefore = "" + + "LogicalSort(sort0=[$0], dir0=[ASC])\n" + + " LogicalProject(ENAME=[$1], CORRELATED_SUM=[$8])\n" + + " LogicalCorrelate(correlation=[$cor0], joinType=[left], requiredColumns=[{2, 7}])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalAggregate(group=[{}], EXPR$0=[SUM($0)])\n" + + " LogicalUnion(all=[true])\n" + + " LogicalProject(C=[CAST($0):INTEGER NOT NULL])\n" + + " LogicalFilter(condition=[=($0, $cor0.DEPTNO)])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n" + + " LogicalProject(C=[2])\n" + + " LogicalFilter(condition=[=($1, $cor0.JOB)])\n" + + " LogicalTableScan(table=[[scott, BONUS]])\n"; + assertThat(before, hasTree(planBefore)); + + // Decorrelate without any rules, just "purely" decorrelation algorithm on RelDecorrelator + final RelNode after = + RelDecorrelator.decorrelateQuery(before, builder, RuleSets.ofList(Collections.emptyList()), + RuleSets.ofList(Collections.emptyList())); + // Verify plan + final String planAfter = "" + + "LogicalSort(sort0=[$0], dir0=[ASC])\n" + + " LogicalProject(ENAME=[$1], CORRELATED_SUM=[$10])\n" + + " LogicalJoin(condition=[AND(IS NOT DISTINCT FROM($2, $8), IS NOT DISTINCT FROM($7, $9))], joinType=[left])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalAggregate(group=[{0, 1}], EXPR$0=[SUM($2)])\n" + + " LogicalProject(JOB=[$0], DEPTNO=[$1], C=[$2])\n" + + " LogicalUnion(all=[true])\n" + + " LogicalProject(JOB=[$0], DEPTNO=[$1], C=[$2])\n" + + " LogicalJoin(condition=[IS NOT DISTINCT FROM($1, $3)], joinType=[inner])\n" + + " LogicalAggregate(group=[{0, 1}])\n" + + " LogicalProject(JOB=[$2], DEPTNO=[$7])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalProject(C=[CAST($0):INTEGER NOT NULL], DEPTNO=[$0])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n" + + " LogicalProject(JOB=[$0], DEPTNO=[$1], C=[$2])\n" + + " LogicalJoin(condition=[IS NOT DISTINCT FROM($0, $3)], joinType=[inner])\n" + + " LogicalAggregate(group=[{0, 1}])\n" + + " LogicalProject(JOB=[$2], DEPTNO=[$7])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalProject(C=[2], JOB=[$1])\n" + + " LogicalFilter(condition=[IS NOT NULL($1)])\n" + + " LogicalTableScan(table=[[scott, BONUS]])\n"; + assertThat(after, hasTree(planAfter)); + } + + /** Test case for [CALCITE-7272] + * Subqueries cannot be decorrelated if have set op. */ + @Test void testCorrelationInSetOp2() { + final FrameworkConfig frameworkConfig = config().build(); + final RelBuilder builder = RelBuilder.create(frameworkConfig); + final RelOptCluster cluster = builder.getCluster(); + final Planner planner = Frameworks.getPlanner(frameworkConfig); + final String sql = "SELECT d.dname\n" + + "FROM dept d\n" + + "WHERE EXISTS (\n" + + " SELECT 1\n" + + " FROM emp e\n" + + " WHERE e.deptno = d.deptno\n" + + " AND (\n" + + " SELECT SUM(x)\n" + + " FROM (\n" + + " SELECT COUNT(*) as x\n" + + " FROM bonus b\n" + + " WHERE b.ename = e.ename\n" + + " UNION ALL\n" + + " SELECT COUNT(*) as x\n" + + " FROM emp e2\n" + + " WHERE e2.deptno = d.deptno\n" + + " ) t\n" + + " ) > 5)"; + final RelNode originalRel; + try { + final SqlNode parse = planner.parse(sql); + final SqlNode validate = planner.validate(parse); + originalRel = planner.rel(validate).rel; + } catch (Exception e) { + throw TestUtil.rethrow(e); + } + + final HepProgram hepProgram = HepProgram.builder() + .addRuleCollection( + ImmutableList.of( + // SubQuery program rules + CoreRules.FILTER_SUB_QUERY_TO_CORRELATE, + CoreRules.PROJECT_SUB_QUERY_TO_CORRELATE, + CoreRules.JOIN_SUB_QUERY_TO_CORRELATE)) + .build(); + final Program program = + Programs.of(hepProgram, true, + requireNonNull(cluster.getMetadataProvider())); + final RelNode before = + program.run(cluster.getPlanner(), originalRel, cluster.traitSet(), + Collections.emptyList(), Collections.emptyList()); + final String planBefore = "" + + "LogicalProject(DNAME=[$1])\n" + + " LogicalProject(DEPTNO=[$0], DNAME=[$1], LOC=[$2])\n" + + " LogicalCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{0}])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n" + + " LogicalAggregate(group=[{0}])\n" + + " LogicalProject(i=[true])\n" + + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7])\n" + + " LogicalFilter(condition=[AND(=($7, $cor0.DEPTNO), >($8, 5))])\n" + + " LogicalCorrelate(correlation=[$cor1], joinType=[left], requiredColumns=[{1}])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalAggregate(group=[{}], EXPR$0=[SUM($0)])\n" + + " LogicalUnion(all=[true])\n" + + " LogicalAggregate(group=[{}], X=[COUNT()])\n" + + " LogicalFilter(condition=[=($0, $cor1.ENAME)])\n" + + " LogicalTableScan(table=[[scott, BONUS]])\n" + + " LogicalAggregate(group=[{}], X=[COUNT()])\n" + + " LogicalFilter(condition=[=($7, $cor0.DEPTNO)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + assertThat(before, hasTree(planBefore)); + + // Decorrelate without any rules, just "purely" decorrelation algorithm on RelDecorrelator + final RelNode after = + RelDecorrelator.decorrelateQuery(before, builder, RuleSets.ofList(Collections.emptyList()), + RuleSets.ofList(Collections.emptyList())); + // Verify plan + final String planAfter = "" + + "LogicalProject(DNAME=[$1])\n" + + " LogicalJoin(condition=[=($0, $3)], joinType=[inner])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n" + + " LogicalProject(DEPTNO0=[$0], $f1=[true])\n" + + " LogicalAggregate(group=[{0}])\n" + + " LogicalProject(DEPTNO0=[$8])\n" + + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7], DEPTNO0=[CAST($8):TINYINT], ENAME0=[$9], EXPR$0=[CAST($10):BIGINT])\n" + + " LogicalJoin(condition=[AND(IS NOT DISTINCT FROM($1, $9), =($7, $8))], joinType=[inner])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalFilter(condition=[>($2, 5)])\n" + + " LogicalAggregate(group=[{0, 1}], EXPR$0=[SUM($2)])\n" + + " LogicalProject(DEPTNO=[$0], ENAME=[$1], X=[$2])\n" + + " LogicalUnion(all=[true])\n" + + " LogicalProject(DEPTNO=[$0], ENAME=[$1], X=[$3])\n" + + " LogicalJoin(condition=[IS NOT DISTINCT FROM($1, $2)], joinType=[inner])\n" + + " LogicalJoin(condition=[true], joinType=[inner])\n" + + " LogicalProject(DEPTNO=[$0])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n" + + " LogicalProject(ENAME=[$1])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalProject(ENAME=[$0], X=[CASE(IS NOT NULL($2), $2, 0)])\n" + + " LogicalJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left])\n" + + " LogicalProject(ENAME=[$1])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalAggregate(group=[{0}], X=[COUNT()])\n" + + " LogicalProject(ENAME=[$0])\n" + + " LogicalFilter(condition=[IS NOT NULL($0)])\n" + + " LogicalTableScan(table=[[scott, BONUS]])\n" + + " LogicalProject(DEPTNO=[$0], ENAME=[$1], X=[$3])\n" + + " LogicalJoin(condition=[IS NOT DISTINCT FROM($0, $2)], joinType=[inner])\n" + + " LogicalJoin(condition=[true], joinType=[inner])\n" + + " LogicalProject(DEPTNO=[$0])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n" + + " LogicalProject(ENAME=[$1])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalProject(DEPTNO=[$0], X=[CASE(IS NOT NULL($2), $2, 0)])\n" + + " LogicalJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left])\n" + + " LogicalProject(DEPTNO=[$0])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n" + + " LogicalAggregate(group=[{0}], X=[COUNT()])\n" + + " LogicalProject(DEPTNO=[$7])\n" + + " LogicalFilter(condition=[IS NOT NULL($7)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + assertThat(after, hasTree(planAfter)); + } + + /** Test case for [CALCITE-7272] + * Subqueries cannot be decorrelated if have set op. */ + @Test void testCorrelationInSetOp3() { + final FrameworkConfig frameworkConfig = config().build(); + final RelBuilder builder = RelBuilder.create(frameworkConfig); + final RelOptCluster cluster = builder.getCluster(); + final Planner planner = Frameworks.getPlanner(frameworkConfig); + final String sql = "SELECT d.dname\n" + + "FROM dept d\n" + + "WHERE EXISTS (\n" + + " SELECT 1\n" + + " FROM emp e\n" + + " WHERE (\n" + + " SELECT SUM(x)\n" + + " FROM (\n" + + " SELECT COUNT(*) as x\n" + + " FROM bonus b\n" + + " WHERE b.ename = e.ename\n" + + " UNION ALL\n" + + " SELECT COUNT(*) as x\n" + + " FROM emp e2\n" + + " WHERE e2.deptno = d.deptno\n" + + " ) t\n" + + " ) > 5)"; + final RelNode originalRel; + try { + final SqlNode parse = planner.parse(sql); + final SqlNode validate = planner.validate(parse); + originalRel = planner.rel(validate).rel; + } catch (Exception e) { + throw TestUtil.rethrow(e); + } + + final HepProgram hepProgram = HepProgram.builder() + .addRuleCollection( + ImmutableList.of( + // SubQuery program rules + CoreRules.FILTER_SUB_QUERY_TO_CORRELATE, + CoreRules.PROJECT_SUB_QUERY_TO_CORRELATE, + CoreRules.JOIN_SUB_QUERY_TO_CORRELATE)) + .build(); + final Program program = + Programs.of(hepProgram, true, + requireNonNull(cluster.getMetadataProvider())); + final RelNode before = + program.run(cluster.getPlanner(), originalRel, cluster.traitSet(), + Collections.emptyList(), Collections.emptyList()); + final String planBefore = "" + + "LogicalProject(DNAME=[$1])\n" + + " LogicalProject(DEPTNO=[$0], DNAME=[$1], LOC=[$2])\n" + + " LogicalCorrelate(correlation=[$cor1], joinType=[inner], requiredColumns=[{0}])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n" + + " LogicalAggregate(group=[{0}])\n" + + " LogicalProject(i=[true])\n" + + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7])\n" + + " LogicalFilter(condition=[>($8, 5)])\n" + + " LogicalCorrelate(correlation=[$cor0], joinType=[left], requiredColumns=[{1}])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalAggregate(group=[{}], EXPR$0=[SUM($0)])\n" + + " LogicalUnion(all=[true])\n" + + " LogicalAggregate(group=[{}], X=[COUNT()])\n" + + " LogicalFilter(condition=[=($0, $cor0.ENAME)])\n" + + " LogicalTableScan(table=[[scott, BONUS]])\n" + + " LogicalAggregate(group=[{}], X=[COUNT()])\n" + + " LogicalFilter(condition=[=($7, $cor1.DEPTNO)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + assertThat(before, hasTree(planBefore)); + + // Decorrelate without any rules, just "purely" decorrelation algorithm on RelDecorrelator + final RelNode after = + RelDecorrelator.decorrelateQuery(before, builder, RuleSets.ofList(Collections.emptyList()), + RuleSets.ofList(Collections.emptyList())); + // Verify plan + final String planAfter = "" + + "LogicalProject(DNAME=[$1])\n" + + " LogicalJoin(condition=[IS NOT DISTINCT FROM($0, $3)], joinType=[inner])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n" + + " LogicalProject(DEPTNO0=[$0], $f1=[true])\n" + + " LogicalAggregate(group=[{0}])\n" + + " LogicalProject(DEPTNO0=[$9])\n" + + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7], ENAME0=[$8], DEPTNO0=[CAST($9):TINYINT], EXPR$0=[CAST($10):BIGINT])\n" + + " LogicalJoin(condition=[IS NOT DISTINCT FROM($1, $8)], joinType=[inner])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalFilter(condition=[>($2, 5)])\n" + + " LogicalAggregate(group=[{0, 1}], EXPR$0=[SUM($2)])\n" + + " LogicalProject(ENAME=[$0], DEPTNO=[$1], X=[$2])\n" + + " LogicalUnion(all=[true])\n" + + " LogicalProject(ENAME=[$0], DEPTNO=[$1], X=[$3])\n" + + " LogicalJoin(condition=[IS NOT DISTINCT FROM($0, $2)], joinType=[inner])\n" + + " LogicalJoin(condition=[true], joinType=[inner])\n" + + " LogicalProject(ENAME=[$1])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalProject(DEPTNO=[$0])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n" + + " LogicalProject(ENAME=[$0], X=[CASE(IS NOT NULL($2), $2, 0)])\n" + + " LogicalJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left])\n" + + " LogicalProject(ENAME=[$1])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalAggregate(group=[{0}], X=[COUNT()])\n" + + " LogicalProject(ENAME=[$0])\n" + + " LogicalFilter(condition=[IS NOT NULL($0)])\n" + + " LogicalTableScan(table=[[scott, BONUS]])\n" + + " LogicalProject(ENAME=[$0], DEPTNO=[$1], X=[$3])\n" + + " LogicalJoin(condition=[IS NOT DISTINCT FROM($1, $2)], joinType=[inner])\n" + + " LogicalJoin(condition=[true], joinType=[inner])\n" + + " LogicalProject(ENAME=[$1])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalProject(DEPTNO=[$0])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n" + + " LogicalProject(DEPTNO=[$0], X=[CASE(IS NOT NULL($2), $2, 0)])\n" + + " LogicalJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left])\n" + + " LogicalProject(DEPTNO=[$0])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n" + + " LogicalAggregate(group=[{0}], X=[COUNT()])\n" + + " LogicalProject(DEPTNO=[$7])\n" + + " LogicalFilter(condition=[IS NOT NULL($7)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + assertThat(after, hasTree(planAfter)); + } + /** * Test case for * [CALCITE-6468] RelDecorrelator diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index e651fcd74e6c..2eafe44bd82f 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -17372,8 +17372,9 @@ LogicalProject(EXPR$0=[CAST(OR(AND(IS TRUE(>($0, $9)), IS NOT TRUE(OR(IS NULL($1 LogicalTableScan(table=[[CATALOG, SALES, EMP]]) LogicalProject(m=[$2], c=[CASE(IS NOT NULL($3), $3, 0)], d=[CASE(IS NOT NULL($3), $3, 0)], trueLiteral=[$4], NAME=[$0]) LogicalJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left]) - LogicalAggregate(group=[{2}]) - LogicalTableScan(table=[[CATALOG, SALES, EMP]]) + LogicalAggregate(group=[{0}]) + LogicalProject(JOB=[$2]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) LogicalAggregate(group=[{0}], m=[MIN($1)], c=[COUNT()], trueLiteral=[LITERAL_AGG(true)]) LogicalProject(NAME=[$1], DEPTNO=[$0]) LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) @@ -21626,8 +21627,9 @@ LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$ LogicalTableScan(table=[[CATALOG, SALES, EMP]]) LogicalProject(m=[$2], c=[CASE(IS NOT NULL($3), $3, 0)], d=[CASE(IS NOT NULL($3), $3, 0)], trueLiteral=[$4], NAME=[$0]) LogicalJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left]) - LogicalAggregate(group=[{2}]) - LogicalTableScan(table=[[CATALOG, SALES, EMP]]) + LogicalAggregate(group=[{0}]) + LogicalProject(JOB=[$2]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) LogicalAggregate(group=[{0}], m=[MIN($1)], c=[COUNT()], trueLiteral=[LITERAL_AGG(true)]) LogicalProject(NAME=[$1], DEPTNO=[$0]) LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) diff --git a/core/src/test/resources/sql/sub-query.iq b/core/src/test/resources/sql/sub-query.iq index 9f320c7ef024..47f8f6497de4 100644 --- a/core/src/test/resources/sql/sub-query.iq +++ b/core/src/test/resources/sql/sub-query.iq @@ -5619,4 +5619,1189 @@ where c+d=a*c); (2 rows) !ok + +# [CALCITE-7272] Subqueries cannot be decorrelated if have set op +WITH + customers(id, name, city) AS ( + VALUES + (1, 'Alice', 'New York'), + (2, 'Bob', 'San Francisco'), + (3, 'Charlie', 'Los Angeles') + ), + orders(id, customer_id, total_amount) AS ( + VALUES + (100, 1, 500.00), + (101, 2, 150.00), + (102, 1, 300.00) + ), + lineitems(id, order_id, product_name, price) AS ( + VALUES + (1, 100, 'Laptop', 1000.00), + (2, 100, 'Mouse', 20.00), + (3, 100, 'Keyboard', 50.00), + (4, 101, 'Monitor', 150.00) + ), + payments(id, customer_id, amount) AS ( + VALUES + (1, 1, 200.00), + (2, 1, 200.00), + (3, 1, 100.00), + (4, 2, 150.00) + ) +SELECT c.id, c.name +FROM customers c +WHERE EXISTS ( + SELECT 1 + FROM orders o + WHERE o.customer_id = c.id + AND ( + SELECT SUM(cnt) + FROM ( + SELECT COUNT(*) AS cnt + FROM lineitems li + WHERE li.order_id = o.id + UNION ALL + SELECT COUNT(*) AS cnt + FROM payments p + WHERE p.customer_id = c.id + ) AS union_sub + ) > 5 +); ++----+---------+ +| ID | NAME | ++----+---------+ +| 1 | Alice | ++----+---------+ +(1 row) + +!ok + +# [CALCITE-7272] Subqueries cannot be decorrelated if have set op +SELECT d.dname +FROM dept d +WHERE EXISTS ( + SELECT 1 + FROM emp e + WHERE e.deptno = d.deptno + AND ( + SELECT SUM(x) + FROM ( + SELECT COUNT(*) as x + FROM bonus b + WHERE b.ename = e.ename + UNION ALL + SELECT COUNT(*) as x + FROM emp e2 + WHERE e2.deptno = d.deptno + ) t + ) > 5 +); ++-------+ +| DNAME | ++-------+ +| SALES | ++-------+ +(1 row) + +!ok + +# [CALCITE-7272] Subqueries cannot be decorrelated if have set op +SELECT d.dname +FROM dept d +WHERE EXISTS ( + SELECT 1 + FROM emp e + WHERE ( + SELECT SUM(x) + FROM ( + SELECT COUNT(*) as x + FROM bonus b + WHERE b.ename = e.ename + UNION ALL + SELECT COUNT(*) as x + FROM emp e2 + WHERE e2.deptno = d.deptno + ) t + ) > 5 +); ++-------+ +| DNAME | ++-------+ +| SALES | ++-------+ +(1 row) + +!ok + +# [CALCITE-7272] Subqueries cannot be decorrelated if have set op +SELECT ename, + (SELECT sum(c) + FROM + (SELECT deptno AS c + FROM dept + WHERE dept.deptno = emp.deptno + UNION ALL + SELECT 2 AS c + FROM bonus + WHERE bonus.job = emp.job) AS union_subquery + ) AS correlated_sum +FROM emp +ORDER BY ename; ++--------+----------------+ +| ENAME | CORRELATED_SUM | ++--------+----------------+ +| ADAMS | 20 | +| ALLEN | 30 | +| BLAKE | 30 | +| CLARK | 10 | +| FORD | 20 | +| JAMES | 30 | +| JONES | 20 | +| KING | 10 | +| MARTIN | 30 | +| MILLER | 10 | +| SCOTT | 20 | +| SMITH | 20 | +| TURNER | 30 | +| WARD | 30 | ++--------+----------------+ +(14 rows) + +!ok + +# [CALCITE-7272] Subqueries cannot be decorrelated if have set op +SELECT ename, + (SELECT sum(c) + FROM + (SELECT deptno AS c + FROM dept + WHERE dept.deptno = emp.deptno + UNION ALL + SELECT 2 AS c + FROM bonus) AS union_subquery + ) AS correlated_sum +FROM emp +ORDER BY ename; ++--------+----------------+ +| ENAME | CORRELATED_SUM | ++--------+----------------+ +| ADAMS | 20 | +| ALLEN | 30 | +| BLAKE | 30 | +| CLARK | 10 | +| FORD | 20 | +| JAMES | 30 | +| JONES | 20 | +| KING | 10 | +| MARTIN | 30 | +| MILLER | 10 | +| SCOTT | 20 | +| SMITH | 20 | +| TURNER | 30 | +| WARD | 30 | ++--------+----------------+ +(14 rows) + +!ok + +# [CALCITE-7272] Subqueries cannot be decorrelated if have set op +SELECT *, + (SELECT COUNT(*) + FROM ( + SELECT * FROM emp WHERE emp.deptno = dept.deptno + UNION ALL + SELECT * FROM emp) AS sub + GROUP BY deptno) AS num_dept_groups +FROM dept; +more than one value in agg SINGLE_VALUE +!error + +# [CALCITE-7272] Subqueries cannot be decorrelated if have set op +WITH t0(t0a, t0b) AS (VALUES (1, 1), (2, 0)), + t1(t1a, t1b, t1c) AS (VALUES (1, 1, 3)), + t2(t2a, t2b, t2c) AS (VALUES (1, 1, 5), (2, 2, 7)) +SELECT t0a, (SELECT sum(c) FROM + (SELECT t1c as c + FROM t1 + WHERE t1a = t0a + UNION ALL + SELECT t2c as c + FROM t2 + WHERE t2b = t0b) as tmp +) +FROM t0; ++-----+--------+ +| T0A | EXPR$1 | ++-----+--------+ +| 1 | 8 | +| 2 | | ++-----+--------+ +(2 rows) + +!ok + +# [CALCITE-7272] Subqueries cannot be decorrelated if have set op +WITH t0(t0a, t0b) AS (VALUES (1, 1), (2, 0)), + t1(t1a, t1b, t1c) AS (VALUES (1, 1, 3)), + t2(t2a, t2b, t2c) AS (VALUES (1, 1, 5), (2, 2, 7)) +SELECT * FROM t0 WHERE t0a < +(SELECT sum(c) FROM + (SELECT t1c as c + FROM t1 + WHERE t1a = t0a + UNION ALL + SELECT t2c as c + FROM t2 + WHERE t2b = t0b) as tmp +); ++-----+-----+ +| T0A | T0B | ++-----+-----+ +| 1 | 1 | ++-----+-----+ +(1 row) + +!ok + +# [CALCITE-7272] Subqueries cannot be decorrelated if have set op +WITH t0(t0a, t0b) AS (VALUES (1, 1), (2, 0)), + t1(t1a, t1b, t1c) AS (VALUES (1, 1, 3)), + t2(t2a, t2b, t2c) AS (VALUES (1, 1, 5), (2, 2, 7)) +SELECT t0a, (SELECT sum(c) FROM + (SELECT t1c as c + FROM t1 + WHERE t1a = t0a + UNION ALL + SELECT t2c as c + FROM t2 + WHERE t2a = t0a) as tmp +) +FROM t0; ++-----+--------+ +| T0A | EXPR$1 | ++-----+--------+ +| 1 | 8 | +| 2 | 7 | ++-----+--------+ +(2 rows) + +!ok + +# [CALCITE-7272] Subqueries cannot be decorrelated if have set op +WITH t0(t0a, t0b) AS (VALUES (1, 1), (2, 0)), + t1(t1a, t1b, t1c) AS (VALUES (1, 1, 3)), + t2(t2a, t2b, t2c) AS (VALUES (1, 1, 5), (2, 2, 7)) +SELECT t0a, (SELECT sum(c) FROM + (SELECT t1c as c + FROM t1 + WHERE t1a > t0a + UNION ALL + SELECT t2c as c + FROM t2 + WHERE t2b <= t0b) as tmp +) +FROM t0; ++-----+--------+ +| T0A | EXPR$1 | ++-----+--------+ +| 1 | 5 | +| 2 | | ++-----+--------+ +(2 rows) + +!ok + +# [CALCITE-7272] Subqueries cannot be decorrelated if have set op +WITH t0(t0a, t0b) AS (VALUES (1, 1), (2, 0)), + t1(t1a, t1b, t1c) AS (VALUES (1, 1, 3)), + t2(t2a, t2b, t2c) AS (VALUES (1, 1, 5), (2, 2, 7)) +SELECT t0a, (SELECT sum(t1c) FROM + (SELECT t1c + FROM t1 + WHERE t1a = t0a + UNION ALL + SELECT t2c + FROM t2 + WHERE t2b = t0b) as tmp +) +FROM t0; ++-----+--------+ +| T0A | EXPR$1 | ++-----+--------+ +| 1 | 8 | +| 2 | | ++-----+--------+ +(2 rows) + +!ok + +# [CALCITE-7272] Subqueries cannot be decorrelated if have set op +WITH t0(t0a, t0b) AS (VALUES (1, 1), (2, 0)), + t1(t1a, t1b, t1c) AS (VALUES (1, 1, 3)), + t2(t2a, t2b, t2c) AS (VALUES (1, 1, 5), (2, 2, 7)) +SELECT t0a, (SELECT count(t1c) FROM + (SELECT t1c + FROM t1 + WHERE t1a = t0a + UNION ALL + SELECT t2c + FROM t2 + WHERE t2b = t0b) as tmp +) +FROM t0; ++-----+--------+ +| T0A | EXPR$1 | ++-----+--------+ +| 1 | 2 | +| 2 | 0 | ++-----+--------+ +(2 rows) + +!ok + +# [CALCITE-7272] Subqueries cannot be decorrelated if have set op +WITH t0(t0a, t0b) AS (VALUES (1, 1), (2, 0)), + t1(t1a, t1b, t1c) AS (VALUES (1, 1, 3)), + t2(t2a, t2b, t2c) AS (VALUES (1, 1, 5), (2, 2, 7)) +SELECT t0a, (SELECT sum(d) FROM + (SELECT t1a - t0a as d + FROM t1 + UNION ALL + SELECT t2a - t0a as d + FROM t2) as tmp +) +FROM t0; ++-----+--------+ +| T0A | EXPR$1 | ++-----+--------+ +| 1 | 1 | +| 2 | -2 | ++-----+--------+ +(2 rows) + +!ok + +# [CALCITE-7272] Subqueries cannot be decorrelated if have set op +WITH t0(t0a, t0b) AS (VALUES (1, 1), (2, 0)), + t1(t1a, t1b, t1c) AS (VALUES (1, 1, 3)), + t2(t2a, t2b, t2c) AS (VALUES (1, 1, 5), (2, 2, 7)) +SELECT t0a, (SELECT sum(c) FROM + (SELECT t1c as c + FROM t1 + WHERE t1a = t0a + UNION DISTINCT + SELECT t2c as c + FROM t2 + WHERE t2b = t0b) as tmp +) +FROM t0; ++-----+--------+ +| T0A | EXPR$1 | ++-----+--------+ +| 1 | 8 | +| 2 | | ++-----+--------+ +(2 rows) + +!ok + +# [CALCITE-7272] Subqueries cannot be decorrelated if have set op +WITH t0(t0a, t0b) AS (VALUES (1, 1), (2, 0)), + t1(t1a, t1b, t1c) AS (VALUES (1, 1, 3)), + t2(t2a, t2b, t2c) AS (VALUES (1, 1, 5), (2, 2, 7)) +SELECT * FROM t0 WHERE t0a < +(SELECT sum(c) FROM + (SELECT t1c as c + FROM t1 + WHERE t1a = t0a + UNION DISTINCT + SELECT t2c as c + FROM t2 + WHERE t2b = t0b) as tmp +); ++-----+-----+ +| T0A | T0B | ++-----+-----+ +| 1 | 1 | ++-----+-----+ +(1 row) + +!ok + +# [CALCITE-7272] Subqueries cannot be decorrelated if have set op +WITH t0(t0a, t0b) AS (VALUES (1, 1), (2, 0)), + t1(t1a, t1b, t1c) AS (VALUES (1, 1, 3)), + t2(t2a, t2b, t2c) AS (VALUES (1, 1, 5), (2, 2, 7)) +SELECT t0a, (SELECT sum(c) FROM + (SELECT t1c as c + FROM t1 + WHERE t1a = t0a + UNION DISTINCT + SELECT t2c as c + FROM t2 + WHERE t2a = t0a) as tmp +) +FROM t0; ++-----+--------+ +| T0A | EXPR$1 | ++-----+--------+ +| 1 | 8 | +| 2 | 7 | ++-----+--------+ +(2 rows) + +!ok + +# [CALCITE-7272] Subqueries cannot be decorrelated if have set op +WITH t0(t0a, t0b) AS (VALUES (1, 1), (2, 0)), + t1(t1a, t1b, t1c) AS (VALUES (1, 1, 3)), + t2(t2a, t2b, t2c) AS (VALUES (1, 1, 5), (2, 2, 7)) +SELECT t0a, (SELECT sum(c) FROM + (SELECT t1c as c + FROM t1 + WHERE t1a > t0a + UNION DISTINCT + SELECT t2c as c + FROM t2 + WHERE t2b <= t0b) as tmp +) +FROM t0; ++-----+--------+ +| T0A | EXPR$1 | ++-----+--------+ +| 1 | 5 | +| 2 | | ++-----+--------+ +(2 rows) + +!ok + +# [CALCITE-7272] Subqueries cannot be decorrelated if have set op +WITH t0(t0a, t0b) AS (VALUES (1, 1), (2, 0)), + t1(t1a, t1b, t1c) AS (VALUES (1, 1, 3)), + t2(t2a, t2b, t2c) AS (VALUES (1, 1, 5), (2, 2, 7)) +SELECT t0a, (SELECT sum(t1c) FROM + (SELECT t1c + FROM t1 + WHERE t1a = t0a + UNION DISTINCT + SELECT t2c + FROM t2 + WHERE t2b = t0b) as tmp +) +FROM t0; ++-----+--------+ +| T0A | EXPR$1 | ++-----+--------+ +| 1 | 8 | +| 2 | | ++-----+--------+ +(2 rows) + +!ok + +# [CALCITE-7272] Subqueries cannot be decorrelated if have set op +WITH t0(t0a, t0b) AS (VALUES (1, 1), (2, 0)), + t1(t1a, t1b, t1c) AS (VALUES (1, 1, 3)), + t2(t2a, t2b, t2c) AS (VALUES (1, 1, 5), (2, 2, 7)) +SELECT t0a, (SELECT count(t1c) FROM + (SELECT t1c + FROM t1 + WHERE t1a = t0a + UNION DISTINCT + SELECT t2c + FROM t2 + WHERE t2b = t0b) as tmp +) +FROM t0; ++-----+--------+ +| T0A | EXPR$1 | ++-----+--------+ +| 1 | 2 | +| 2 | 0 | ++-----+--------+ +(2 rows) + +!ok + +# [CALCITE-7272] Subqueries cannot be decorrelated if have set op +WITH t0(t0a, t0b) AS (VALUES (1, 1), (2, 0)), + t1(t1a, t1b, t1c) AS (VALUES (1, 1, 3)), + t2(t2a, t2b, t2c) AS (VALUES (1, 1, 5), (2, 2, 7)) +SELECT t0a, (SELECT sum(d) FROM + (SELECT t1a - t0a as d + FROM t1 + UNION DISTINCT + SELECT t2a - t0a as d + FROM t2) as tmp +) +FROM t0; ++-----+--------+ +| T0A | EXPR$1 | ++-----+--------+ +| 1 | 1 | +| 2 | -1 | ++-----+--------+ +(2 rows) + +!ok + +# [CALCITE-7272] Subqueries cannot be decorrelated if have set op +WITH t0(t0a, t0b) AS (VALUES (1, 1), (2, 0)), + t1(t1a, t1b, t1c) AS (VALUES (1, 1, 3)), + t2(t2a, t2b, t2c) AS (VALUES (1, 1, 5), (2, 2, 7)) +SELECT t0a, (SELECT sum(c) FROM + (SELECT t1c as c + FROM t1 + WHERE t1a = t0a + INTERSECT ALL + SELECT t2c as c + FROM t2 + WHERE t2b = t0b) as tmp +) +FROM t0; ++-----+--------+ +| T0A | EXPR$1 | ++-----+--------+ +| 1 | | +| 2 | | ++-----+--------+ +(2 rows) + +!ok + +# [CALCITE-7272] Subqueries cannot be decorrelated if have set op +WITH t0(t0a, t0b) AS (VALUES (1, 1), (2, 0)), + t1(t1a, t1b, t1c) AS (VALUES (1, 1, 3)), + t2(t2a, t2b, t2c) AS (VALUES (1, 1, 5), (2, 2, 7)) +SELECT * FROM t0 WHERE t0a < +(SELECT sum(c) FROM + (SELECT t1c as c + FROM t1 + WHERE t1a = t0a + INTERSECT ALL + SELECT t2c as c + FROM t2 + WHERE t2b = t0b) as tmp +); ++-----+-----+ +| T0A | T0B | ++-----+-----+ ++-----+-----+ +(0 rows) + +!ok + +# [CALCITE-7272] Subqueries cannot be decorrelated if have set op +WITH t0(t0a, t0b) AS (VALUES (1, 1), (2, 0)), + t1(t1a, t1b, t1c) AS (VALUES (1, 1, 3)), + t2(t2a, t2b, t2c) AS (VALUES (1, 1, 5), (2, 2, 7)) +SELECT t0a, (SELECT sum(c) FROM + (SELECT t1c as c + FROM t1 + WHERE t1a = t0a + INTERSECT ALL + SELECT t2c as c + FROM t2 + WHERE t2a = t0a) as tmp +) +FROM t0; ++-----+--------+ +| T0A | EXPR$1 | ++-----+--------+ +| 1 | | +| 2 | | ++-----+--------+ +(2 rows) + +!ok + +# [CALCITE-7272] Subqueries cannot be decorrelated if have set op +WITH t0(t0a, t0b) AS (VALUES (1, 1), (2, 0)), + t1(t1a, t1b, t1c) AS (VALUES (1, 1, 3)), + t2(t2a, t2b, t2c) AS (VALUES (1, 1, 5), (2, 2, 7)) +SELECT t0a, (SELECT sum(c) FROM + (SELECT t1c as c + FROM t1 + WHERE t1a > t0a + INTERSECT ALL + SELECT t2c as c + FROM t2 + WHERE t2b <= t0b) as tmp +) +FROM t0; ++-----+--------+ +| T0A | EXPR$1 | ++-----+--------+ +| 1 | | +| 2 | | ++-----+--------+ +(2 rows) + +!ok + +# [CALCITE-7272] Subqueries cannot be decorrelated if have set op +WITH t0(t0a, t0b) AS (VALUES (1, 1), (2, 0)), + t1(t1a, t1b, t1c) AS (VALUES (1, 1, 3)), + t2(t2a, t2b, t2c) AS (VALUES (1, 1, 5), (2, 2, 7)) +SELECT t0a, (SELECT sum(t1c) FROM + (SELECT t1c + FROM t1 + WHERE t1a = t0a + INTERSECT ALL + SELECT t2c + FROM t2 + WHERE t2b = t0b) as tmp +) +FROM t0; ++-----+--------+ +| T0A | EXPR$1 | ++-----+--------+ +| 1 | | +| 2 | | ++-----+--------+ +(2 rows) + +!ok + +# [CALCITE-7272] Subqueries cannot be decorrelated if have set op +WITH t0(t0a, t0b) AS (VALUES (1, 1), (2, 0)), + t1(t1a, t1b, t1c) AS (VALUES (1, 1, 3)), + t2(t2a, t2b, t2c) AS (VALUES (1, 1, 5), (2, 2, 7)) +SELECT t0a, (SELECT count(t1c) FROM + (SELECT t1c + FROM t1 + WHERE t1a = t0a + INTERSECT ALL + SELECT t2c + FROM t2 + WHERE t2b = t0b) as tmp +) +FROM t0; ++-----+--------+ +| T0A | EXPR$1 | ++-----+--------+ +| 1 | 0 | +| 2 | 0 | ++-----+--------+ +(2 rows) + +!ok + +# [CALCITE-7272] Subqueries cannot be decorrelated if have set op +WITH t0(t0a, t0b) AS (VALUES (1, 1), (2, 0)), + t1(t1a, t1b, t1c) AS (VALUES (1, 1, 3)), + t2(t2a, t2b, t2c) AS (VALUES (1, 1, 5), (2, 2, 7)) +SELECT t0a, (SELECT sum(d) FROM + (SELECT t1a - t0a as d + FROM t1 + INTERSECT ALL + SELECT t2a - t0a as d + FROM t2) as tmp +) +FROM t0; ++-----+--------+ +| T0A | EXPR$1 | ++-----+--------+ +| 1 | 0 | +| 2 | -1 | ++-----+--------+ +(2 rows) + +!ok + +# [CALCITE-7272] Subqueries cannot be decorrelated if have set op +WITH t0(t0a, t0b) AS (VALUES (1, 1), (2, 0)), + t1(t1a, t1b, t1c) AS (VALUES (1, 1, 3)), + t2(t2a, t2b, t2c) AS (VALUES (1, 1, 5), (2, 2, 7)) +SELECT t0a, (SELECT sum(c) FROM + (SELECT t1c as c + FROM t1 + WHERE t1a = t0a + INTERSECT DISTINCT + SELECT t2c as c + FROM t2 + WHERE t2b = t0b) as tmp +) +FROM t0; ++-----+--------+ +| T0A | EXPR$1 | ++-----+--------+ +| 1 | | +| 2 | | ++-----+--------+ +(2 rows) + +!ok + +# [CALCITE-7272] Subqueries cannot be decorrelated if have set op +WITH t0(t0a, t0b) AS (VALUES (1, 1), (2, 0)), + t1(t1a, t1b, t1c) AS (VALUES (1, 1, 3)), + t2(t2a, t2b, t2c) AS (VALUES (1, 1, 5), (2, 2, 7)) +SELECT * FROM t0 WHERE t0a < +(SELECT sum(c) FROM + (SELECT t1c as c + FROM t1 + WHERE t1a = t0a + INTERSECT DISTINCT + SELECT t2c as c + FROM t2 + WHERE t2b = t0b) as tmp +); ++-----+-----+ +| T0A | T0B | ++-----+-----+ ++-----+-----+ +(0 rows) + +!ok + +# [CALCITE-7272] Subqueries cannot be decorrelated if have set op +WITH t0(t0a, t0b) AS (VALUES (1, 1), (2, 0)), + t1(t1a, t1b, t1c) AS (VALUES (1, 1, 3)), + t2(t2a, t2b, t2c) AS (VALUES (1, 1, 5), (2, 2, 7)) +SELECT t0a, (SELECT sum(c) FROM + (SELECT t1c as c + FROM t1 + WHERE t1a = t0a + INTERSECT DISTINCT + SELECT t2c as c + FROM t2 + WHERE t2a = t0a) as tmp +) +FROM t0; ++-----+--------+ +| T0A | EXPR$1 | ++-----+--------+ +| 1 | | +| 2 | | ++-----+--------+ +(2 rows) + +!ok + +# [CALCITE-7272] Subqueries cannot be decorrelated if have set op +WITH t0(t0a, t0b) AS (VALUES (1, 1), (2, 0)), + t1(t1a, t1b, t1c) AS (VALUES (1, 1, 3)), + t2(t2a, t2b, t2c) AS (VALUES (1, 1, 5), (2, 2, 7)) +SELECT t0a, (SELECT sum(c) FROM + (SELECT t1c as c + FROM t1 + WHERE t1a > t0a + INTERSECT DISTINCT + SELECT t2c as c + FROM t2 + WHERE t2b <= t0b) as tmp +) +FROM t0; ++-----+--------+ +| T0A | EXPR$1 | ++-----+--------+ +| 1 | | +| 2 | | ++-----+--------+ +(2 rows) + +!ok + +# [CALCITE-7272] Subqueries cannot be decorrelated if have set op +WITH t0(t0a, t0b) AS (VALUES (1, 1), (2, 0)), + t1(t1a, t1b, t1c) AS (VALUES (1, 1, 3)), + t2(t2a, t2b, t2c) AS (VALUES (1, 1, 5), (2, 2, 7)) +SELECT t0a, (SELECT sum(t1c) FROM + (SELECT t1c + FROM t1 + WHERE t1a = t0a + INTERSECT DISTINCT + SELECT t2c + FROM t2 + WHERE t2b = t0b) as tmp +) +FROM t0; ++-----+--------+ +| T0A | EXPR$1 | ++-----+--------+ +| 1 | | +| 2 | | ++-----+--------+ +(2 rows) + +!ok + +# [CALCITE-7272] Subqueries cannot be decorrelated if have set op +WITH t0(t0a, t0b) AS (VALUES (1, 1), (2, 0)), + t1(t1a, t1b, t1c) AS (VALUES (1, 1, 3)), + t2(t2a, t2b, t2c) AS (VALUES (1, 1, 5), (2, 2, 7)) +SELECT t0a, (SELECT count(t1c) FROM + (SELECT t1c + FROM t1 + WHERE t1a = t0a + INTERSECT DISTINCT + SELECT t2c + FROM t2 + WHERE t2b = t0b) as tmp +) +FROM t0; ++-----+--------+ +| T0A | EXPR$1 | ++-----+--------+ +| 1 | 0 | +| 2 | 0 | ++-----+--------+ +(2 rows) + +!ok + +# [CALCITE-7272] Subqueries cannot be decorrelated if have set op +WITH t0(t0a, t0b) AS (VALUES (1, 1), (2, 0)), + t1(t1a, t1b, t1c) AS (VALUES (1, 1, 3)), + t2(t2a, t2b, t2c) AS (VALUES (1, 1, 5), (2, 2, 7)) +SELECT t0a, (SELECT sum(d) FROM + (SELECT t1a - t0a as d + FROM t1 + INTERSECT DISTINCT + SELECT t2a - t0a as d + FROM t2) as tmp +) +FROM t0; ++-----+--------+ +| T0A | EXPR$1 | ++-----+--------+ +| 1 | 0 | +| 2 | -1 | ++-----+--------+ +(2 rows) + +!ok + +# [CALCITE-7272] Subqueries cannot be decorrelated if have set op +WITH t0(t0a, t0b) AS (VALUES (1, 1), (2, 0)), + t1(t1a, t1b, t1c) AS (VALUES (1, 1, 3)), + t2(t2a, t2b, t2c) AS (VALUES (1, 1, 5), (2, 2, 7)) +SELECT t0a, (SELECT sum(c) FROM + (SELECT t1c as c + FROM t1 + WHERE t1a = t0a + EXCEPT ALL + SELECT t2c as c + FROM t2 + WHERE t2b = t0b) as tmp +) +FROM t0; ++-----+--------+ +| T0A | EXPR$1 | ++-----+--------+ +| 1 | 3 | +| 2 | | ++-----+--------+ +(2 rows) + +!ok + +# [CALCITE-7272] Subqueries cannot be decorrelated if have set op +WITH t0(t0a, t0b) AS (VALUES (1, 1), (2, 0)), + t1(t1a, t1b, t1c) AS (VALUES (1, 1, 3)), + t2(t2a, t2b, t2c) AS (VALUES (1, 1, 5), (2, 2, 7)) +SELECT * FROM t0 WHERE t0a < +(SELECT sum(c) FROM + (SELECT t1c as c + FROM t1 + WHERE t1a = t0a + EXCEPT ALL + SELECT t2c as c + FROM t2 + WHERE t2b = t0b) as tmp +); ++-----+-----+ +| T0A | T0B | ++-----+-----+ +| 1 | 1 | ++-----+-----+ +(1 row) + +!ok + +# [CALCITE-7272] Subqueries cannot be decorrelated if have set op +WITH t0(t0a, t0b) AS (VALUES (1, 1), (2, 0)), + t1(t1a, t1b, t1c) AS (VALUES (1, 1, 3)), + t2(t2a, t2b, t2c) AS (VALUES (1, 1, 5), (2, 2, 7)) +SELECT t0a, (SELECT sum(c) FROM + (SELECT t1c as c + FROM t1 + WHERE t1a = t0a + EXCEPT ALL + SELECT t2c as c + FROM t2 + WHERE t2a = t0a) as tmp +) +FROM t0; ++-----+--------+ +| T0A | EXPR$1 | ++-----+--------+ +| 1 | 3 | +| 2 | | ++-----+--------+ +(2 rows) + +!ok + +# [CALCITE-7272] Subqueries cannot be decorrelated if have set op +WITH t0(t0a, t0b) AS (VALUES (1, 1), (2, 0)), + t1(t1a, t1b, t1c) AS (VALUES (1, 1, 3)), + t2(t2a, t2b, t2c) AS (VALUES (1, 1, 5), (2, 2, 7)) +SELECT t0a, (SELECT sum(c) FROM + (SELECT t1c as c + FROM t1 + WHERE t1a > t0a + EXCEPT ALL + SELECT t2c as c + FROM t2 + WHERE t2b <= t0b) as tmp +) +FROM t0; ++-----+--------+ +| T0A | EXPR$1 | ++-----+--------+ +| 1 | | +| 2 | | ++-----+--------+ +(2 rows) + +!ok + +# [CALCITE-7272] Subqueries cannot be decorrelated if have set op +WITH t0(t0a, t0b) AS (VALUES (1, 1), (2, 0)), + t1(t1a, t1b, t1c) AS (VALUES (1, 1, 3)), + t2(t2a, t2b, t2c) AS (VALUES (1, 1, 5), (2, 2, 7)) +SELECT t0a, (SELECT sum(t1c) FROM + (SELECT t1c + FROM t1 + WHERE t1a = t0a + EXCEPT ALL + SELECT t2c + FROM t2 + WHERE t2b = t0b) as tmp +) +FROM t0; ++-----+--------+ +| T0A | EXPR$1 | ++-----+--------+ +| 1 | 3 | +| 2 | | ++-----+--------+ +(2 rows) + +!ok + +# [CALCITE-7272] Subqueries cannot be decorrelated if have set op +WITH t0(t0a, t0b) AS (VALUES (1, 1), (2, 0)), + t1(t1a, t1b, t1c) AS (VALUES (1, 1, 3)), + t2(t2a, t2b, t2c) AS (VALUES (1, 1, 5), (2, 2, 7)) +SELECT t0a, (SELECT count(t1c) FROM + (SELECT t1c + FROM t1 + WHERE t1a = t0a + EXCEPT ALL + SELECT t2c + FROM t2 + WHERE t2b = t0b) as tmp +) +FROM t0; ++-----+--------+ +| T0A | EXPR$1 | ++-----+--------+ +| 1 | 1 | +| 2 | 0 | ++-----+--------+ +(2 rows) + +!ok + +# [CALCITE-7272] Subqueries cannot be decorrelated if have set op +WITH t0(t0a, t0b) AS (VALUES (1, 1), (2, 0)), + t1(t1a, t1b, t1c) AS (VALUES (1, 1, 3)), + t2(t2a, t2b, t2c) AS (VALUES (1, 1, 5), (2, 2, 7)) +SELECT t0a, (SELECT sum(d) FROM + (SELECT t1a - t0a as d + FROM t1 + EXCEPT ALL + SELECT t2a - t0a as d + FROM t2) as tmp +) +FROM t0; ++-----+--------+ +| T0A | EXPR$1 | ++-----+--------+ +| 1 | | +| 2 | | ++-----+--------+ +(2 rows) + +!ok + +# [CALCITE-7272] Subqueries cannot be decorrelated if have set op +WITH t0(t0a, t0b) AS (VALUES (1, 1), (2, 0)), + t1(t1a, t1b, t1c) AS (VALUES (1, 1, 3)), + t2(t2a, t2b, t2c) AS (VALUES (1, 1, 5), (2, 2, 7)) +SELECT t0a, (SELECT sum(c) FROM + (SELECT t1c as c + FROM t1 + WHERE t1a = t0a + EXCEPT DISTINCT + SELECT t2c as c + FROM t2 + WHERE t2b = t0b) as tmp +) +FROM t0; ++-----+--------+ +| T0A | EXPR$1 | ++-----+--------+ +| 1 | 3 | +| 2 | | ++-----+--------+ +(2 rows) + +!ok + +# [CALCITE-7272] Subqueries cannot be decorrelated if have set op +WITH t0(t0a, t0b) AS (VALUES (1, 1), (2, 0)), + t1(t1a, t1b, t1c) AS (VALUES (1, 1, 3)), + t2(t2a, t2b, t2c) AS (VALUES (1, 1, 5), (2, 2, 7)) +SELECT * FROM t0 WHERE t0a < +(SELECT sum(c) FROM + (SELECT t1c as c + FROM t1 + WHERE t1a = t0a + EXCEPT DISTINCT + SELECT t2c as c + FROM t2 + WHERE t2b = t0b) as tmp +); ++-----+-----+ +| T0A | T0B | ++-----+-----+ +| 1 | 1 | ++-----+-----+ +(1 row) + +!ok + +# [CALCITE-7272] Subqueries cannot be decorrelated if have set op +WITH t0(t0a, t0b) AS (VALUES (1, 1), (2, 0)), + t1(t1a, t1b, t1c) AS (VALUES (1, 1, 3)), + t2(t2a, t2b, t2c) AS (VALUES (1, 1, 5), (2, 2, 7)) +SELECT t0a, (SELECT sum(c) FROM + (SELECT t1c as c + FROM t1 + WHERE t1a = t0a + EXCEPT DISTINCT + SELECT t2c as c + FROM t2 + WHERE t2a = t0a) as tmp +) +FROM t0; ++-----+--------+ +| T0A | EXPR$1 | ++-----+--------+ +| 1 | 3 | +| 2 | | ++-----+--------+ +(2 rows) + +!ok + +# [CALCITE-7272] Subqueries cannot be decorrelated if have set op +WITH t0(t0a, t0b) AS (VALUES (1, 1), (2, 0)), + t1(t1a, t1b, t1c) AS (VALUES (1, 1, 3)), + t2(t2a, t2b, t2c) AS (VALUES (1, 1, 5), (2, 2, 7)) +SELECT t0a, (SELECT sum(c) FROM + (SELECT t1c as c + FROM t1 + WHERE t1a > t0a + EXCEPT DISTINCT + SELECT t2c as c + FROM t2 + WHERE t2b <= t0b) as tmp +) +FROM t0; ++-----+--------+ +| T0A | EXPR$1 | ++-----+--------+ +| 1 | | +| 2 | | ++-----+--------+ +(2 rows) + +!ok + +# [CALCITE-7272] Subqueries cannot be decorrelated if have set op +WITH t0(t0a, t0b) AS (VALUES (1, 1), (2, 0)), + t1(t1a, t1b, t1c) AS (VALUES (1, 1, 3)), + t2(t2a, t2b, t2c) AS (VALUES (1, 1, 5), (2, 2, 7)) +SELECT t0a, (SELECT sum(t1c) FROM + (SELECT t1c + FROM t1 + WHERE t1a = t0a + EXCEPT DISTINCT + SELECT t2c + FROM t2 + WHERE t2b = t0b) as tmp +) +FROM t0; ++-----+--------+ +| T0A | EXPR$1 | ++-----+--------+ +| 1 | 3 | +| 2 | | ++-----+--------+ +(2 rows) + +!ok + +# [CALCITE-7272] Subqueries cannot be decorrelated if have set op +WITH t0(t0a, t0b) AS (VALUES (1, 1), (2, 0)), + t1(t1a, t1b, t1c) AS (VALUES (1, 1, 3)), + t2(t2a, t2b, t2c) AS (VALUES (1, 1, 5), (2, 2, 7)) +SELECT t0a, (SELECT count(t1c) FROM + (SELECT t1c + FROM t1 + WHERE t1a = t0a + EXCEPT DISTINCT + SELECT t2c + FROM t2 + WHERE t2b = t0b) as tmp +) +FROM t0; ++-----+--------+ +| T0A | EXPR$1 | ++-----+--------+ +| 1 | 1 | +| 2 | 0 | ++-----+--------+ +(2 rows) + +!ok + +# [CALCITE-7272] Subqueries cannot be decorrelated if have set op +WITH t0(t0a, t0b) AS (VALUES (1, 1), (2, 0)), + t1(t1a, t1b, t1c) AS (VALUES (1, 1, 3)), + t2(t2a, t2b, t2c) AS (VALUES (1, 1, 5), (2, 2, 7)) +SELECT t0a, (SELECT sum(d) FROM + (SELECT t1a - t0a as d + FROM t1 + EXCEPT DISTINCT + SELECT t2a - t0a as d + FROM t2) as tmp +) +FROM t0; ++-----+--------+ +| T0A | EXPR$1 | ++-----+--------+ +| 1 | | +| 2 | | ++-----+--------+ +(2 rows) + +!ok + # End sub-query.iq From 7559b2860f5364c0a90950caaf33ec7dc4cda807 Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Tue, 23 Dec 2025 07:06:07 +0800 Subject: [PATCH 071/562] [CALCITE-4813] ANY_VALUE assumes that arguments should be comparable --- .../apache/calcite/runtime/SqlFunctions.java | 62 +++++++++++++-- core/src/test/resources/sql/blank.iq | 77 +++++++++++++++++++ .../calcite/linq4j/function/Functions.java | 75 ++++++++++++++---- 3 files changed, 191 insertions(+), 23 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java index 52031cf494af..dc4c6c258784 100644 --- a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java +++ b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java @@ -2242,7 +2242,11 @@ public static boolean lt(List b0, List b1) { return Functions.compareLists(b0, b1) < 0; } - public static boolean lt(Object[] b0, Object[] b1) { + public static boolean lt(Map b0, Map b1) { + return Functions.compareMaps(b0, b1) < 0; + } + + public static boolean lt(@Nullable Object @Nullable [] b0, @Nullable Object @Nullable [] b1) { return Functions.compareObjectArrays(b0, b1) < 0; } @@ -2292,7 +2296,7 @@ public static boolean le(List b0, List b1) { } /** SQL ≤ operator applied to Object[] values. */ - public static boolean le(Object[] b0, Object[] b1) { + public static boolean le(@Nullable Object @Nullable [] b0, @Nullable Object @Nullable [] b1) { return Functions.compareObjectArrays(b0, b1) <= 0; } @@ -2375,7 +2379,11 @@ public static boolean gt(List b0, List b1) { return Functions.compareLists(b0, b1) > 0; } - public static boolean gt(Object[] b0, Object[] b1) { + public static boolean gt(Map b0, Map b1) { + return Functions.compareMaps(b0, b1) > 0; + } + + public static boolean gt(@Nullable Object @Nullable [] b0, @Nullable Object @Nullable [] b1) { return Functions.compareObjectArrays(b0, b1) > 0; } @@ -2426,7 +2434,7 @@ public static boolean ge(List b0, List b1) { } /** SQL ≥ operator applied to Object[] values. */ - public static boolean ge(Object[] b0, Object[] b1) { + public static boolean ge(@Nullable Object @Nullable [] b0, @Nullable Object @Nullable [] b1) { return Functions.compareObjectArrays(b0, b1) >= 0; } @@ -4641,8 +4649,27 @@ public static double lesser(double b0, double b1) { return b0 > b1 ? b1 : b0; } - public static @Nullable > List lesser( - @Nullable List b0, @Nullable List b1) { + public static @Nullable List lesser(@Nullable List b0, @Nullable List b1) { + if (b0 == null) { + return b1; + } + if (b1 == null) { + return b0; + } + return lt(b0, b1) ? b0 : b1; + } + + public static @Nullable Map lesser(@Nullable Map b0, @Nullable Map b1) { + if (b0 == null) { + return b1; + } + if (b1 == null) { + return b0; + } + return lt(b0, b1) ? b0 : b1; + } + + public static @Nullable Object[] lesser(@Nullable Object[] b0, @Nullable Object[] b1) { if (b0 == null) { return b1; } @@ -4652,8 +4679,27 @@ public static double lesser(double b0, double b1) { return lt(b0, b1) ? b0 : b1; } - public static @Nullable > List greater( - @Nullable List b0, @Nullable List b1) { + public static @Nullable List greater(@Nullable List b0, @Nullable List b1) { + if (b0 == null) { + return b1; + } + if (b1 == null) { + return b0; + } + return gt(b0, b1) ? b0 : b1; + } + + public static @Nullable Map greater(@Nullable Map b0, @Nullable Map b1) { + if (b0 == null) { + return b1; + } + if (b1 == null) { + return b0; + } + return gt(b0, b1) ? b0 : b1; + } + + public static @Nullable Object[] greater(@Nullable Object[] b0, @Nullable Object[] b1) { if (b0 == null) { return b1; } diff --git a/core/src/test/resources/sql/blank.iq b/core/src/test/resources/sql/blank.iq index 9c200caf7e2f..c44c39530a4d 100644 --- a/core/src/test/resources/sql/blank.iq +++ b/core/src/test/resources/sql/blank.iq @@ -154,4 +154,81 @@ select * from table1 where j not in (select i from table2) or j = 3; !ok +# [CALCITE-4813] ANY_VALUE assumes that arguments should be comparable +select any_value(r) over(), s from(select array[f, s] r, s from (select 1 as f, 2 as s) t) t; ++--------+---+ +| EXPR$0 | S | ++--------+---+ +| [1, 2] | 2 | ++--------+---+ +(1 row) + +!ok + +select any_value(r) over(), s from(select map[f, s] r, s from (select 1 as f, 2 as s) t) t; ++--------+---+ +| EXPR$0 | S | ++--------+---+ +| {1=2} | 2 | ++--------+---+ +(1 row) + +!ok + +select any_value(r) over(), s from(select row(f, s) r, s from (select 1 as f, 2 as s) t) t; ++--------+---+ +| EXPR$0 | S | ++--------+---+ +| {1, 2} | 2 | ++--------+---+ +(1 row) + +!ok + + +CREATE TABLE complex_t ( + a INTEGER ARRAY, + m MAP, + r ROW(r1 VARCHAR, r2 INTEGER, r3 VARCHAR) +); +(0 rows modified) + +!update + +INSERT INTO complex_t VALUES ( + ARRAY[1, 2, 3, 4, 5], + MAP['math', 95.5, 'science', 88.0, 'english', 92.3], + ROW('Alice Johnson', 30, 'a') +), +( + ARRAY[10, 20, 30, 40, 50, 60], + MAP['physics', 96.2, 'chemistry', 91.8, 'biology', 89.5, 'computer_science', 98.7], + ROW('Bob Smith', 25, 'b') +), +( + ARRAY[100, 200, 300], + MAP['leadership', 88.9, 'teamwork', 94.2, 'communication', 91.5, 'problem_solving', 97.8], + ROW('Charlie Chen', 35, 'c') +); +(3 rows modified) + +!update + +select + max(a) as max_a, + max(m) as max_m, + max(r) as max_r, + min(a) as min_a, + min(m) as min_m, + min(r) as min_r +from complex_t; ++-----------------+----------------------------------------------------------------------------------------------+-----------------------+-----------------+------------------------------------------------------------------------------------------+------------------------+ +| MAX_A | MAX_M | MAX_R | MIN_A | MIN_M | MIN_R | ++-----------------+----------------------------------------------------------------------------------------------+-----------------------+-----------------+------------------------------------------------------------------------------------------+------------------------+ +| [100, 200, 300] | {physics =96.2, chemistry =91.8, biology =89.5, computer_science=98.7} | {Charlie Chen, 35, c} | [1, 2, 3, 4, 5] | {leadership =88.9, teamwork =94.2, communication =91.5, problem_solving=97.8} | {Alice Johnson, 30, a} | ++-----------------+----------------------------------------------------------------------------------------------+-----------------------+-----------------+------------------------------------------------------------------------------------------+------------------------+ +(1 row) + +!ok + # End blank.iq diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/function/Functions.java b/linq4j/src/main/java/org/apache/calcite/linq4j/function/Functions.java index 329e758b948c..5f7aa258d4c3 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/function/Functions.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/function/Functions.java @@ -16,8 +16,6 @@ */ package org.apache.calcite.linq4j.function; -import com.google.common.collect.Lists; - import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.framework.qual.DefaultQualifier; import org.checkerframework.framework.qual.TypeUseLocation; @@ -31,6 +29,7 @@ import java.util.Collections; import java.util.Comparator; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Objects; @@ -573,9 +572,7 @@ private static class NullsLastComparator } else if (o1 instanceof List && o2 instanceof List) { return compareLists((List) o1, (List) o2); } else if (o1 instanceof Object[] && o2 instanceof Object[]) { - final List list1 = Lists.newArrayList((Object[]) o1); - final List list2 = Lists.newArrayList((Object[]) o2); - return compareLists(list1, list2); + return compareObjectArrays((Object[]) o1, (Object[]) o2); } else { throw new IllegalArgumentException(); } @@ -601,9 +598,7 @@ private static class NullsFirstReverseComparator } else if (o1 instanceof List && o2 instanceof List) { return -compareLists((List) o1, (List) o2); } else if (o1 instanceof Object[] && o2 instanceof Object[]) { - final List list1 = Lists.newArrayList((Object[]) o1); - final List list2 = Lists.newArrayList((Object[]) o2); - return -compareLists(list1, list2); + return -compareObjectArrays((Object[]) o1, (Object[]) o2); } else { throw new IllegalArgumentException(); } @@ -611,6 +606,9 @@ private static class NullsFirstReverseComparator } public static int compareLists(List b0, List b1) { + if (b0 == b1) { + return 0; + } if (b0.isEmpty() && b1.isEmpty()) { return 0; } @@ -623,10 +621,46 @@ public static int compareLists(List b0, List b1) { return Integer.compare(b0.size(), b1.size()); } + /** + * Compares two maps. + * + *

    Since maps in Calcite are implemented using {@link java.util.LinkedHashMap}, + * which guarantees insertion order, this method follows DuckDB's behavior by + * comparing entries in order. For each entry, it first compares the key and + * then the value. + */ + public static int compareMaps(Map b0, Map b1) { + if (b0 == b1) { + return 0; + } + final Iterator> i0 = b0.entrySet().iterator(); + final Iterator> i1 = b1.entrySet().iterator(); + while (i0.hasNext() && i1.hasNext()) { + Map.Entry e0 = i0.next(); + Map.Entry e1 = i1.next(); + int c = compareListItems(e0.getKey(), e1.getKey()); + if (c != 0) { + return c; + } + c = compareListItems(e0.getValue(), e1.getValue()); + if (c != 0) { + return c; + } + } + if (i0.hasNext()) { + return 1; + } + if (i1.hasNext()) { + return -1; + } + return 0; + } + private static int compareListItems(@Nullable Object item0, @Nullable Object item1) { - if (item0 == null && item1 == null) { + if (item0 == item1) { return 0; - } else if (item0 == null) { + } + if (item0 == null) { return 1; } else if (item1 == null) { return -1; @@ -635,6 +669,8 @@ private static int compareListItems(@Nullable Object item0, @Nullable Object ite final List b0ItemList = (List) item0; final List b1ItemList = (List) item1; return compareLists(b0ItemList, b1ItemList); + } else if (item0 instanceof Map && item1 instanceof Map) { + return compareMaps((Map) item0, (Map) item1); } else if (item0 instanceof Object[] && item1 instanceof Object[]) { return compareObjectArrays((Object[]) item0, (Object[]) item1); } else if (item0.getClass().equals(item1.getClass()) && item0 instanceof Comparable) { @@ -642,14 +678,23 @@ private static int compareListItems(@Nullable Object item0, @Nullable Object ite final Comparable b1Comparable = (Comparable) item1; return b0Comparable.compareTo(b1Comparable); } else { - throw new IllegalArgumentException("Item types do not match"); + throw new IllegalArgumentException("Item types do not match: " + + item0.getClass() + " vs " + item1.getClass()); } } - public static int compareObjectArrays(Object[] b0, Object[] b1) { - final List b0List = Lists.newArrayList(b0); - final List b1List = Lists.newArrayList(b1); - return compareLists(b0List, b1List); + public static int compareObjectArrays(@Nullable Object @Nullable [] b0, + @Nullable Object @Nullable [] b1) { + if (b0 == b1) { + return 0; + } + if (b0 == null) { + return 1; + } + if (b1 == null) { + return -1; + } + return compareLists(Arrays.asList(b0), Arrays.asList(b1)); } /** Nulls last reverse comparator. */ From d1803a4efe13c10fc54c37847f368780c62a97d9 Mon Sep 17 00:00:00 2001 From: iwanttobepowerful <745778074@qq.com> Date: Tue, 16 Dec 2025 19:15:11 +0800 Subject: [PATCH 072/562] [CALCITE-6942] Support decorrelated for sub-queries with LIMIT 1 and OFFSET --- .../calcite/sql2rel/RelDecorrelator.java | 127 +++-- .../calcite/sql2rel/RelDecorrelatorTest.java | 199 ++++++++ .../apache/calcite/test/RelOptRulesTest.xml | 22 +- .../calcite/test/SqlToRelConverterTest.xml | 13 +- core/src/test/resources/sql/sub-query.iq | 440 ++++++++++++++++-- site/_docs/history.md | 3 + 6 files changed, 687 insertions(+), 117 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java b/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java index c5986208d22d..ad6d12ca98f6 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java @@ -74,6 +74,7 @@ import org.apache.calcite.rex.RexSubQuery; import org.apache.calcite.rex.RexUtil; import org.apache.calcite.rex.RexVisitorImpl; +import org.apache.calcite.rex.RexWindowBounds; import org.apache.calcite.runtime.PairList; import org.apache.calcite.sql.SqlAggFunction; import org.apache.calcite.sql.SqlExplainFormat; @@ -575,16 +576,33 @@ protected RexNode removeCorrelationExpr( } if (isCorVarDefined && (rel.fetch != null || rel.offset != null)) { - if (rel.fetch != null - && rel.offset == null - && RexLiteral.intValue(rel.fetch) == 1) { - return decorrelateFetchOneSort(rel, frame); - } - // Can not decorrelate if the sort has per-correlate-key attributes like - // offset or fetch limit, because these attributes scope would change to - // global after decorrelation. They should take effect within the scope - // of the correlation key actually. - return null; + if (rel.offset == null && rel.fetch instanceof RexLiteral) { + final RexLiteral fetchLiteral = (RexLiteral) requireNonNull(rel.fetch, "fetch"); + final BigDecimal fetch = fetchLiteral.getValueAs(BigDecimal.class); + assert fetch != null; + if (fetch.equals(BigDecimal.ZERO)) { + return null; + } + } + + // + // Rewrite logic: + // + // For correlated Sort with LIMIT/OFFSET: + // Special case: if OFFSET is null and FETCH = 1, + // we may rewrite as an Aggregate using MIN/MAX. + Frame aggFrame = decorrelateSortAsAggregate(rel, frame); + if (aggFrame != null) { + return aggFrame; + } + + // General case: rewrite as + // Project(original_fields..., corVars..., rn) + // where rn = ROW_NUMBER() OVER (PARTITION BY corVars ORDER BY sortExprs + // ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) + // Filter(rn > offset, rn <= offset + fetch) + // This preserves per-corVar LIMIT/OFFSET semantics. + return decorrelateSortWithRowNumber(rel, frame); } final RelNode newInput = frame.r; @@ -1036,30 +1054,7 @@ private static void shiftMapping(Map mapping, int startIndex, return null; } - protected @Nullable Frame decorrelateFetchOneSort(Sort sort, final Frame frame) { - Frame aggFrame = decorrelateSortAsAggregate(sort, frame); - if (aggFrame != null) { - return aggFrame; - } - // - // Rewrite logic: - // - // If sorted without offset and fetch = 1 (enforced by the caller), rewrite the sort to be - // Aggregate(group=(corVar.. , field..)) - // project(first_value(field) over (partition by corVar order by (sort collation))) - // input - // - // 1. For the original sorted input, apply the FIRST_VALUE window function to produce - // the result of sorting with LIMIT 1, and the same as the decorrelate of aggregate, - // add correlated variables in partition list to maintain semantic consistency. - // 2. To ensure that there is at most one row of output for - // any combination of correlated variables, distinct for correlated variables. - // 3. Since we have partitioned by all correlated variables - // in the sorted output field window, so for any combination of correlated variables, - // all other field values are unique. So the following two are equivalent: - // - group by corVar1, covVar2, field1, field2 - // - any_value(fields1), any_value(fields2) group by corVar1, covVar2 - // Here we use the first. + protected @Nullable Frame decorrelateSortWithRowNumber(Sort sort, final Frame frame) { final Map mapOldToNewOutputs = new HashMap<>(); final NavigableMap corDefOutputs = new TreeMap<>(); @@ -1091,29 +1086,63 @@ private static void shiftMapping(Map mapping, int startIndex, for (RelDataTypeField field : sort.getRowType().getFieldList()) { final int newIdx = requireNonNull(frame.oldToNewOutputs.get(field.getIndex())); - - RelBuilder.AggCall aggCall = - relBuilder.aggregateCall(SqlStdOperatorTable.FIRST_VALUE, - RexInputRef.of(newIdx, fieldList)); - - // Convert each field from the sorted output to a window function that partitions by - // correlated variables, orders by the collation, and return the first_value. - RexNode winCall = aggCall.over() - .orderBy(sortExprs) - .partitionBy(corVarProjects.leftList()) - .toRex(); mapOldToNewOutputs.put(newProjExprs.size(), newProjExprs.size()); - newProjExprs.add(winCall, field.getName()); + newProjExprs.add(RexInputRef.of(newIdx, fieldList), field.getName()); } newProjExprs.addAll(corVarProjects); - RelNode result = relBuilder.push(frame.r) - .project(newProjExprs.leftList(), newProjExprs.rightList()) - .distinct().build(); + relBuilder.push(frame.r); + + RexNode rowNumberCall = relBuilder.aggregateCall(SqlStdOperatorTable.ROW_NUMBER) + .over() + .partitionBy(corVarProjects.leftList()) + .orderBy(sortExprs) + .let(c -> c.rowsBetween(RexWindowBounds.UNBOUNDED_PRECEDING, RexWindowBounds.CURRENT_ROW)) + .toRex(); + newProjExprs.add(rowNumberCall, "rn"); // Add the row number column + relBuilder.project(newProjExprs.leftList(), newProjExprs.rightList()); + + List conditions = new ArrayList<>(); + if (sort.offset != null) { + RexNode greaterThenLowerBound = + relBuilder.call( + SqlStdOperatorTable.GREATER_THAN, + relBuilder.field(newProjExprs.size() - 1), + sort.offset); + conditions.add(greaterThenLowerBound); + } + if (sort.fetch != null) { + RexNode upperBound = sort.offset == null + ? sort.fetch + : relBuilder.call(SqlStdOperatorTable.PLUS, sort.offset, sort.fetch); + RexNode lessThenOrEqualUpperBound = + relBuilder.call( + SqlStdOperatorTable.LESS_THAN_OR_EQUAL, + relBuilder.field(newProjExprs.size() - 1), + upperBound); + conditions.add(lessThenOrEqualUpperBound); + } + + RelNode result; + if (!conditions.isEmpty()) { + result = relBuilder.filter(conditions).build(); + } else { + result = relBuilder.build(); + } return register(sort, result, mapOldToNewOutputs, corDefOutputs); } protected @Nullable Frame decorrelateSortAsAggregate(Sort sort, final Frame frame) { + if (sort.offset != null || sort.fetch == null) { + return null; + } + + final BigDecimal fetch = ((RexLiteral) sort.fetch).getValueAs(BigDecimal.class); + assert fetch != null; + if (!fetch.equals(BigDecimal.ONE)) { + return null; + } + final Map mapOldToNewOutputs = new HashMap<>(); final NavigableMap corDefOutputs = new TreeMap<>(); if (sort.getCollation().getFieldCollations().size() == 1 diff --git a/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java b/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java index 80a3dda64651..83bf5780cc57 100644 --- a/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java +++ b/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java @@ -999,4 +999,203 @@ public static Frameworks.ConfigBuilder config() { + " LogicalTableScan(table=[[scott, DEPT]])\n"; assertThat(decorrelatedNoRules, hasTree(planDecorrelatedNoRules)); } + + @Test void testDecorrelateCorrelatedOrderByLimitToRowNumber() { + final FrameworkConfig frameworkConfig = config().build(); + final RelBuilder builder = RelBuilder.create(frameworkConfig); + final RelOptCluster cluster = builder.getCluster(); + final Planner planner = Frameworks.getPlanner(frameworkConfig); + final String sql = "" + + "SELECT dname FROM dept WHERE 2000 > (\n" + + "SELECT emp.sal FROM emp where dept.deptno = emp.deptno\n" + + "ORDER BY year(hiredate), emp.sal limit 1)"; + final RelNode originalRel; + try { + final SqlNode parse = planner.parse(sql); + final SqlNode validate = planner.validate(parse); + originalRel = planner.rel(validate).rel; + } catch (Exception e) { + throw TestUtil.rethrow(e); + } + + final HepProgram hepProgram = HepProgram.builder() + .addRuleCollection( + ImmutableList.of( + // SubQuery program rules + CoreRules.FILTER_SUB_QUERY_TO_CORRELATE, + CoreRules.PROJECT_SUB_QUERY_TO_CORRELATE, + CoreRules.JOIN_SUB_QUERY_TO_CORRELATE)) + .build(); + final Program program = + Programs.of(hepProgram, true, + requireNonNull(cluster.getMetadataProvider())); + final RelNode before = + program.run(cluster.getPlanner(), originalRel, cluster.traitSet(), + Collections.emptyList(), Collections.emptyList()); + final String planBefore = "" + + "LogicalProject(DNAME=[$1])\n" + + " LogicalProject(DEPTNO=[$0], DNAME=[$1], LOC=[$2])\n" + + " LogicalFilter(condition=[>(2000.00, CAST($3):DECIMAL(12, 2))])\n" + + " LogicalCorrelate(correlation=[$cor0], joinType=[left], requiredColumns=[{0}])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n" + + " LogicalProject(SAL=[$0])\n" + + " LogicalSort(sort0=[$1], sort1=[$0], dir0=[ASC], dir1=[ASC], fetch=[1])\n" + + " LogicalProject(SAL=[$5], EXPR$1=[EXTRACT(FLAG(YEAR), $4)])\n" + + " LogicalFilter(condition=[=($cor0.DEPTNO, $7)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + assertThat(before, hasTree(planBefore)); + + // Decorrelate without any rules, just "purely" decorrelation algorithm on RelDecorrelator + final RelNode after = + RelDecorrelator.decorrelateQuery(before, builder, RuleSets.ofList(Collections.emptyList()), + RuleSets.ofList(Collections.emptyList())); + // Verify plan + final String planAfter = "" + + "LogicalProject(DNAME=[$1])\n" + + " LogicalJoin(condition=[=($0, $4)], joinType=[inner])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n" + + " LogicalFilter(condition=[>(2000.00, CAST($0):DECIMAL(12, 2))])\n" + + " LogicalProject(SAL=[$0], DEPTNO=[$2])\n" + + " LogicalFilter(condition=[<=($3, 1)])\n" + + " LogicalProject(SAL=[$5], EXPR$1=[EXTRACT(FLAG(YEAR), $4)], DEPTNO=[$7], rn=[ROW_NUMBER() OVER (PARTITION BY $7 ORDER BY EXTRACT(FLAG(YEAR), $4) NULLS LAST, $5 NULLS LAST)])\n" + + " LogicalFilter(condition=[IS NOT NULL($7)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + assertThat(after, hasTree(planAfter)); + } + + @Test void testDecorrelateCorrelatedOrderByLimitToRowNumber2() { + final FrameworkConfig frameworkConfig = config().build(); + final RelBuilder builder = RelBuilder.create(frameworkConfig); + final RelOptCluster cluster = builder.getCluster(); + final Planner planner = Frameworks.getPlanner(frameworkConfig); + final String sql = "" + + "SELECT *\n" + + "FROM dept d\n" + + "WHERE d.deptno IN (\n" + + " SELECT e.deptno\n" + + " FROM emp e\n" + + " WHERE d.deptno = e.deptno\n" + + " LIMIT 10\n" + + " OFFSET 2\n" + + ")\n" + + "LIMIT 2\n" + + "OFFSET 1"; + final RelNode originalRel; + try { + final SqlNode parse = planner.parse(sql); + final SqlNode validate = planner.validate(parse); + originalRel = planner.rel(validate).rel; + } catch (Exception e) { + throw TestUtil.rethrow(e); + } + + final HepProgram hepProgram = HepProgram.builder() + .addRuleCollection( + ImmutableList.of( + // SubQuery program rules + CoreRules.FILTER_SUB_QUERY_TO_CORRELATE, + CoreRules.PROJECT_SUB_QUERY_TO_CORRELATE, + CoreRules.JOIN_SUB_QUERY_TO_CORRELATE)) + .build(); + final Program program = + Programs.of(hepProgram, true, + requireNonNull(cluster.getMetadataProvider())); + final RelNode before = + program.run(cluster.getPlanner(), originalRel, cluster.traitSet(), + Collections.emptyList(), Collections.emptyList()); + final String planBefore = "" + + "LogicalSort(offset=[1], fetch=[2])\n" + + " LogicalProject(DEPTNO=[$0], DNAME=[$1], LOC=[$2])\n" + + " LogicalProject(DEPTNO=[$0], DNAME=[$1], LOC=[$2])\n" + + " LogicalFilter(condition=[=($0, $3)])\n" + + " LogicalCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{0}])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n" + + " LogicalAggregate(group=[{0}])\n" + + " LogicalSort(offset=[2], fetch=[10])\n" + + " LogicalProject(DEPTNO=[$7])\n" + + " LogicalFilter(condition=[=($cor0.DEPTNO, $7)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + assertThat(before, hasTree(planBefore)); + + // Decorrelate without any rules, just "purely" decorrelation algorithm on RelDecorrelator + final RelNode after = + RelDecorrelator.decorrelateQuery(before, builder, RuleSets.ofList(Collections.emptyList()), + RuleSets.ofList(Collections.emptyList())); + // Verify plan + final String planAfter = "" + + "LogicalSort(offset=[1], fetch=[2])\n" + + " LogicalProject(DEPTNO=[$0], DNAME=[$1], LOC=[$2])\n" + + " LogicalJoin(condition=[=($0, $4)], joinType=[inner])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n" + + " LogicalFilter(condition=[=($1, $0)])\n" + + " LogicalAggregate(group=[{0, 1}])\n" + + " LogicalProject(DEPTNO=[$0], DEPTNO1=[$1])\n" + + " LogicalFilter(condition=[AND(>($2, 2), <=($2, +(2, 10)))])\n" + + " LogicalProject(DEPTNO=[$7], DEPTNO1=[$7], rn=[ROW_NUMBER() OVER (PARTITION BY $7)])\n" + + " LogicalFilter(condition=[IS NOT NULL($7)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + assertThat(after, hasTree(planAfter)); + } + + @Test void testDecorrelateCorrelatedOrderByLimitToRowNumber3() { + final FrameworkConfig frameworkConfig = config().build(); + final RelBuilder builder = RelBuilder.create(frameworkConfig); + final RelOptCluster cluster = builder.getCluster(); + final Planner planner = Frameworks.getPlanner(frameworkConfig); + final String sql = "" + + "SELECT deptno FROM dept WHERE 1000.00 >\n" + + "(SELECT sal FROM emp WHERE dept.deptno = emp.deptno\n" + + "order by emp.sal limit 1 offset 10)"; + final RelNode originalRel; + try { + final SqlNode parse = planner.parse(sql); + final SqlNode validate = planner.validate(parse); + originalRel = planner.rel(validate).rel; + } catch (Exception e) { + throw TestUtil.rethrow(e); + } + + final HepProgram hepProgram = HepProgram.builder() + .addRuleCollection( + ImmutableList.of( + // SubQuery program rules + CoreRules.FILTER_SUB_QUERY_TO_CORRELATE, + CoreRules.PROJECT_SUB_QUERY_TO_CORRELATE, + CoreRules.JOIN_SUB_QUERY_TO_CORRELATE)) + .build(); + final Program program = + Programs.of(hepProgram, true, + requireNonNull(cluster.getMetadataProvider())); + final RelNode before = + program.run(cluster.getPlanner(), originalRel, cluster.traitSet(), + Collections.emptyList(), Collections.emptyList()); + final String planBefore = "" + + "LogicalProject(DEPTNO=[$0])\n" + + " LogicalProject(DEPTNO=[$0], DNAME=[$1], LOC=[$2])\n" + + " LogicalFilter(condition=[>(1000.00, $3)])\n" + + " LogicalCorrelate(correlation=[$cor0], joinType=[left], requiredColumns=[{0}])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n" + + " LogicalSort(sort0=[$0], dir0=[ASC], offset=[10], fetch=[1])\n" + + " LogicalProject(SAL=[$5])\n" + + " LogicalFilter(condition=[=($cor0.DEPTNO, $7)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + assertThat(before, hasTree(planBefore)); + + // Decorrelate without any rules, just "purely" decorrelation algorithm on RelDecorrelator + final RelNode after = + RelDecorrelator.decorrelateQuery(before, builder, RuleSets.ofList(Collections.emptyList()), + RuleSets.ofList(Collections.emptyList())); + // Verify plan + final String planAfter = "" + + "LogicalProject(DEPTNO=[$0])\n" + + " LogicalProject(DEPTNO=[$0], DNAME=[$1], LOC=[$2], SAL=[$3], DEPTNO0=[$4], rn=[CAST($5):BIGINT])\n" + + " LogicalJoin(condition=[=($0, $4)], joinType=[inner])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n" + + " LogicalFilter(condition=[>(1000.00, $0)])\n" + + " LogicalFilter(condition=[AND(>($2, 10), <=($2, +(10, 1)))])\n" + + " LogicalProject(SAL=[$5], DEPTNO=[$7], rn=[ROW_NUMBER() OVER (PARTITION BY $7 ORDER BY $5 NULLS LAST)])\n" + + " LogicalFilter(condition=[IS NOT NULL($7)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + assertThat(after, hasTree(planAfter)); + } } diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index 2eafe44bd82f..2b2deaa226b2 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -2680,12 +2680,12 @@ LogicalProject(NAME=[$1]) (10, $0)]) - LogicalAggregate(group=[{0, 1}]) - LogicalProject(SAL=[FIRST_VALUE($5) OVER (PARTITION BY $7 ORDER BY $5 DESC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)], DEPTNO=[$7]) + LogicalFilter(condition=[<=($2, 1)]) + LogicalProject(SAL=[$5], DEPTNO=[$7], rn=[ROW_NUMBER() OVER (PARTITION BY $7 ORDER BY $5 DESC NULLS FIRST)]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) ]]> @@ -2729,8 +2729,8 @@ LogicalProject(NAME=[$1]) LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) LogicalFilter(condition=[>(10, $0)]) LogicalProject(SAL=[$0], DEPTNO=[$2]) - LogicalAggregate(group=[{0, 1, 2}]) - LogicalProject(SAL=[FIRST_VALUE($5) OVER (PARTITION BY $7 ORDER BY EXTRACT(FLAG(YEAR), $4) NULLS LAST, $5 DESC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)], EXPR$1=[FIRST_VALUE(EXTRACT(FLAG(YEAR), $4)) OVER (PARTITION BY $7 ORDER BY EXTRACT(FLAG(YEAR), $4) NULLS LAST, $5 DESC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)], DEPTNO=[$7]) + LogicalFilter(condition=[<=($3, 1)]) + LogicalProject(SAL=[$5], EXPR$1=[EXTRACT(FLAG(YEAR), $4)], DEPTNO=[$7], rn=[ROW_NUMBER() OVER (PARTITION BY $7 ORDER BY EXTRACT(FLAG(YEAR), $4) NULLS LAST, $5 DESC NULLS FIRST)]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) ]]> @@ -2843,8 +2843,8 @@ LogicalProject(NAME=[$1], EXPR$1=[$2]) LogicalJoin(condition=[=($0, $3)], joinType=[left]) LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) LogicalProject(SAL=[$0], DEPTNO=[$2]) - LogicalAggregate(group=[{0, 1, 2}]) - LogicalProject(SAL=[FIRST_VALUE($5) OVER (PARTITION BY $7 ORDER BY EXTRACT(FLAG(YEAR), $4) NULLS LAST, $5 NULLS LAST RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)], EXPR$1=[FIRST_VALUE(EXTRACT(FLAG(YEAR), $4)) OVER (PARTITION BY $7 ORDER BY EXTRACT(FLAG(YEAR), $4) NULLS LAST, $5 NULLS LAST RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)], DEPTNO=[$7]) + LogicalFilter(condition=[<=($3, 1)]) + LogicalProject(SAL=[$5], EXPR$1=[EXTRACT(FLAG(YEAR), $4)], DEPTNO=[$7], rn=[ROW_NUMBER() OVER (PARTITION BY $7 ORDER BY EXTRACT(FLAG(YEAR), $4) NULLS LAST, $5 NULLS LAST)]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) ]]> @@ -2884,8 +2884,8 @@ LogicalProject(NAME=[$1], EXPR$1=[$4]) LogicalProject(DEPTNO=[$0], NAME=[$1], DEPTNO0=[$0], NAME0=[CAST($1):VARCHAR(20) NOT NULL]) LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) LogicalProject(SAL=[$0], DEPTNO=[$2], ENAME=[$3]) - LogicalAggregate(group=[{0, 1, 2, 3}]) - LogicalProject(SAL=[FIRST_VALUE($5) OVER (PARTITION BY $7, $1 ORDER BY EXTRACT(FLAG(YEAR), $4) NULLS LAST, $5 NULLS LAST RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)], EXPR$1=[FIRST_VALUE(EXTRACT(FLAG(YEAR), $4)) OVER (PARTITION BY $7, $1 ORDER BY EXTRACT(FLAG(YEAR), $4) NULLS LAST, $5 NULLS LAST RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)], DEPTNO=[$7], ENAME=[$1]) + LogicalFilter(condition=[<=($4, 1)]) + LogicalProject(SAL=[$5], EXPR$1=[EXTRACT(FLAG(YEAR), $4)], DEPTNO=[$7], ENAME=[$1], rn=[ROW_NUMBER() OVER (PARTITION BY $7, $1 ORDER BY EXTRACT(FLAG(YEAR), $4) NULLS LAST, $5 NULLS LAST)]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) ]]> @@ -8855,8 +8855,8 @@ LogicalProject(A=[$0], TS=[$1], X=[$2], X0=[$4]) LogicalJoin(condition=[AND(=($1, $5), =($2, $6))], joinType=[left]) LogicalProject(EXPR$0=[$0], EXPR$1=[$1], EXPR$00=[CAST($0):VARCHAR(20) NOT NULL], EXPR$10=[$1]) LogicalValues(tuples=[[{ 'a', 1 }]]) - LogicalAggregate(group=[{0, 1, 2}]) - LogicalProject(X=[FIRST_VALUE($2) OVER (PARTITION BY $3, $4)], EXPR$1=[$3], EXPR$00=[$4]) + LogicalFilter(condition=[<=($3, 1)]) + LogicalProject(X=[$2], EXPR$1=[$3], EXPR$00=[$4], rn=[ROW_NUMBER() OVER (PARTITION BY $3, $4)]) LogicalJoin(condition=[AND(=($0, $4), <=($1, $3))], joinType=[inner]) LogicalProject(A=[$1], TS=[$0], X=[$3]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) diff --git a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml index 7cdde323145c..cbb228b0ed10 100644 --- a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml @@ -931,14 +931,13 @@ FROM @@ -2196,8 +2195,8 @@ LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$ LogicalTableScan(table=[[CATALOG, SALES, EMP]]) LogicalAggregate(group=[{0}], agg#0=[MIN($1)]) LogicalProject(DEPTNO=[$1], $f0=[true]) - LogicalAggregate(group=[{0, 1}]) - LogicalProject(EXPR$0=[FIRST_VALUE(1) OVER (PARTITION BY $0)], DEPTNO=[$0]) + LogicalFilter(condition=[<=($2, 1)]) + LogicalProject(EXPR$0=[1], DEPTNO=[$0], rn=[ROW_NUMBER() OVER (PARTITION BY $0)]) LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) ]]> diff --git a/core/src/test/resources/sql/sub-query.iq b/core/src/test/resources/sql/sub-query.iq index 47f8f6497de4..f64feb0a7932 100644 --- a/core/src/test/resources/sql/sub-query.iq +++ b/core/src/test/resources/sql/sub-query.iq @@ -654,11 +654,10 @@ SELECT dname FROM "scott".dept WHERE 2000 > (SELECT emp.sal FROM "scott".emp whe EnumerableCalc(expr#0..3=[{inputs}], DNAME=[$t3]) EnumerableHashJoin(condition=[=($1, $2)], joinType=[inner]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[2000.00:DECIMAL(12, 2)], expr#4=[CAST($t1):DECIMAL(12, 2)], expr#5=[>($t3, $t4)], SAL=[$t1], DEPTNO=[$t0], $condition=[$t5]) - EnumerableAggregate(group=[{1, 3, 4}]) - EnumerableWindow(window#0=[window(partition {1} order by [2, 0] range between UNBOUNDED PRECEDING and UNBOUNDED FOLLOWING aggs [FIRST_VALUE($0), FIRST_VALUE($2)])]) - EnumerableCalc(expr#0..7=[{inputs}], expr#8=[FLAG(YEAR)], expr#9=[EXTRACT($t8, $t4)], expr#10=[IS NOT NULL($t7)], SAL=[$t5], DEPTNO=[$t7], $2=[$t9], $condition=[$t10]) - EnumerableTableScan(table=[[scott, EMP]]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[1], expr#5=[<=($t3, $t4)], expr#6=[2000.00:DECIMAL(12, 2)], expr#7=[CAST($t0):DECIMAL(12, 2)], expr#8=[>($t6, $t7)], expr#9=[AND($t5, $t8)], proj#0..1=[{exprs}], $condition=[$t9]) + EnumerableWindow(window#0=[window(partition {1} order by [2, 0] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + EnumerableCalc(expr#0..7=[{inputs}], expr#8=[FLAG(YEAR)], expr#9=[EXTRACT($t8, $t4)], expr#10=[IS NOT NULL($t7)], SAL=[$t5], DEPTNO=[$t7], $2=[$t9], $condition=[$t10]) + EnumerableTableScan(table=[[scott, EMP]]) EnumerableCalc(expr#0..2=[{inputs}], proj#0..1=[{exprs}]) EnumerableTableScan(table=[[scott, DEPT]]) !plan @@ -678,11 +677,10 @@ SELECT dname FROM "scott".dept WHERE 2000 > (SELECT emp.sal FROM "scott".emp whe EnumerableCalc(expr#0..3=[{inputs}], DNAME=[$t3]) EnumerableHashJoin(condition=[=($1, $2)], joinType=[inner]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[2000.00:DECIMAL(12, 2)], expr#4=[CAST($t1):DECIMAL(12, 2)], expr#5=[>($t3, $t4)], SAL=[$t1], DEPTNO=[$t0], $condition=[$t5]) - EnumerableAggregate(group=[{1, 3, 4}]) - EnumerableWindow(window#0=[window(partition {1} order by [2, 0] range between UNBOUNDED PRECEDING and UNBOUNDED FOLLOWING aggs [FIRST_VALUE($0), FIRST_VALUE($2)])]) - EnumerableCalc(expr#0..7=[{inputs}], expr#8=[FLAG(YEAR)], expr#9=[EXTRACT($t8, $t4)], expr#10=[CAST($t3):INTEGER], expr#11=[8000], expr#12=[>($t10, $t11)], expr#13=[IS NOT NULL($t7)], expr#14=[AND($t12, $t13)], SAL=[$t5], DEPTNO=[$t7], $2=[$t9], $condition=[$t14]) - EnumerableTableScan(table=[[scott, EMP]]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[1], expr#5=[<=($t3, $t4)], expr#6=[2000.00:DECIMAL(12, 2)], expr#7=[CAST($t0):DECIMAL(12, 2)], expr#8=[>($t6, $t7)], expr#9=[AND($t5, $t8)], proj#0..1=[{exprs}], $condition=[$t9]) + EnumerableWindow(window#0=[window(partition {1} order by [2, 0] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + EnumerableCalc(expr#0..7=[{inputs}], expr#8=[FLAG(YEAR)], expr#9=[EXTRACT($t8, $t4)], expr#10=[CAST($t3):INTEGER], expr#11=[8000], expr#12=[>($t10, $t11)], expr#13=[IS NOT NULL($t7)], expr#14=[AND($t12, $t13)], SAL=[$t5], DEPTNO=[$t7], $2=[$t9], $condition=[$t14]) + EnumerableTableScan(table=[[scott, EMP]]) EnumerableCalc(expr#0..2=[{inputs}], proj#0..1=[{exprs}]) EnumerableTableScan(table=[[scott, DEPT]]) !plan @@ -722,16 +720,14 @@ EnumerableCalc(expr#0..3=[{inputs}], DNAME=[$t1], EXPR$1=[$t3]) # subquery contains null SELECT dname, (SELECT emp.comm FROM "scott".emp where dept.deptno = emp.deptno ORDER BY emp.comm desc limit 1) FROM "scott".dept; -EnumerableCalc(expr#0..3=[{inputs}], DNAME=[$t1], EXPR$1=[$t2]) - EnumerableMergeJoin(condition=[=($0, $3)], joinType=[left]) +EnumerableCalc(expr#0..4=[{inputs}], DNAME=[$t1], EXPR$1=[$t2]) + EnumerableHashJoin(condition=[=($0, $3)], joinType=[left]) EnumerableCalc(expr#0..2=[{inputs}], proj#0..1=[{exprs}]) EnumerableTableScan(table=[[scott, DEPT]]) - EnumerableSort(sort0=[$1], dir0=[ASC]) - EnumerableCalc(expr#0..1=[{inputs}], w0$o0=[$t1], DEPTNO=[$t0]) - EnumerableAggregate(group=[{7, 8}]) - EnumerableWindow(window#0=[window(partition {7} order by [6 DESC] range between UNBOUNDED PRECEDING and UNBOUNDED FOLLOWING aggs [FIRST_VALUE($6)])]) - EnumerableCalc(expr#0..7=[{inputs}], expr#8=[IS NOT NULL($t7)], proj#0..7=[{exprs}], $condition=[$t8]) - EnumerableTableScan(table=[[scott, EMP]]) + EnumerableCalc(expr#0..8=[{inputs}], expr#9=[1], expr#10=[<=($t8, $t9)], COMM=[$t6], DEPTNO=[$t7], $2=[$t8], $condition=[$t10]) + EnumerableWindow(window#0=[window(partition {7} order by [6 DESC] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + EnumerableCalc(expr#0..7=[{inputs}], expr#8=[IS NOT NULL($t7)], proj#0..7=[{exprs}], $condition=[$t8]) + EnumerableTableScan(table=[[scott, EMP]]) !plan +------------+--------+ | DNAME | EXPR$1 | @@ -799,15 +795,13 @@ EnumerableCalc(expr#0..3=[{inputs}], DNAME=[$t1], EXPR$1=[$t3]) SELECT dname, (SELECT emp.sal FROM "scott".emp where dept.deptno = emp.deptno ORDER BY year(hiredate), emp.sal limit 1) FROM "scott".dept; EnumerableCalc(expr#0..3=[{inputs}], DNAME=[$t1], EXPR$1=[$t2]) - EnumerableMergeJoin(condition=[=($0, $3)], joinType=[left]) + EnumerableHashJoin(condition=[=($0, $3)], joinType=[left]) EnumerableCalc(expr#0..2=[{inputs}], proj#0..1=[{exprs}]) EnumerableTableScan(table=[[scott, DEPT]]) - EnumerableSort(sort0=[$1], dir0=[ASC]) - EnumerableCalc(expr#0..2=[{inputs}], SAL=[$t1], DEPTNO=[$t0]) - EnumerableAggregate(group=[{1, 3, 4}]) - EnumerableWindow(window#0=[window(partition {1} order by [2, 0] range between UNBOUNDED PRECEDING and UNBOUNDED FOLLOWING aggs [FIRST_VALUE($0), FIRST_VALUE($2)])]) - EnumerableCalc(expr#0..7=[{inputs}], expr#8=[FLAG(YEAR)], expr#9=[EXTRACT($t8, $t4)], expr#10=[IS NOT NULL($t7)], SAL=[$t5], DEPTNO=[$t7], $2=[$t9], $condition=[$t10]) - EnumerableTableScan(table=[[scott, EMP]]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[1], expr#5=[<=($t3, $t4)], proj#0..1=[{exprs}], $condition=[$t5]) + EnumerableWindow(window#0=[window(partition {1} order by [2, 0] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + EnumerableCalc(expr#0..7=[{inputs}], expr#8=[FLAG(YEAR)], expr#9=[EXTRACT($t8, $t4)], expr#10=[IS NOT NULL($t7)], SAL=[$t5], DEPTNO=[$t7], $2=[$t9], $condition=[$t10]) + EnumerableTableScan(table=[[scott, EMP]]) !plan +------------+---------+ | DNAME | EXPR$1 | @@ -826,15 +820,13 @@ EnumerableCalc(expr#0..3=[{inputs}], DNAME=[$t1], EXPR$1=[$t2]) SELECT dname, (SELECT emp.sal FROM "scott".emp where dept.deptno = emp.deptno and mgr > 8000 ORDER BY year(hiredate), emp.sal limit 1) FROM "scott".dept; EnumerableCalc(expr#0..3=[{inputs}], DNAME=[$t1], EXPR$1=[$t2]) - EnumerableMergeJoin(condition=[=($0, $3)], joinType=[left]) + EnumerableHashJoin(condition=[=($0, $3)], joinType=[left]) EnumerableCalc(expr#0..2=[{inputs}], proj#0..1=[{exprs}]) EnumerableTableScan(table=[[scott, DEPT]]) - EnumerableSort(sort0=[$1], dir0=[ASC]) - EnumerableCalc(expr#0..2=[{inputs}], SAL=[$t1], DEPTNO=[$t0]) - EnumerableAggregate(group=[{1, 3, 4}]) - EnumerableWindow(window#0=[window(partition {1} order by [2, 0] range between UNBOUNDED PRECEDING and UNBOUNDED FOLLOWING aggs [FIRST_VALUE($0), FIRST_VALUE($2)])]) - EnumerableCalc(expr#0..7=[{inputs}], expr#8=[FLAG(YEAR)], expr#9=[EXTRACT($t8, $t4)], expr#10=[CAST($t3):INTEGER], expr#11=[8000], expr#12=[>($t10, $t11)], expr#13=[IS NOT NULL($t7)], expr#14=[AND($t12, $t13)], SAL=[$t5], DEPTNO=[$t7], $2=[$t9], $condition=[$t14]) - EnumerableTableScan(table=[[scott, EMP]]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[1], expr#5=[<=($t3, $t4)], proj#0..1=[{exprs}], $condition=[$t5]) + EnumerableWindow(window#0=[window(partition {1} order by [2, 0] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + EnumerableCalc(expr#0..7=[{inputs}], expr#8=[FLAG(YEAR)], expr#9=[EXTRACT($t8, $t4)], expr#10=[CAST($t3):INTEGER], expr#11=[8000], expr#12=[>($t10, $t11)], expr#13=[IS NOT NULL($t7)], expr#14=[AND($t12, $t13)], SAL=[$t5], DEPTNO=[$t7], $2=[$t9], $condition=[$t14]) + EnumerableTableScan(table=[[scott, EMP]]) !plan +------------+--------+ | DNAME | EXPR$1 | @@ -2238,14 +2230,14 @@ select sal from "scott".emp e (0 rows) !ok -EnumerableCalc(expr#0..4=[{inputs}], expr#5=[RAND()], expr#6=[CAST($t5):INTEGER NOT NULL], expr#7=[2], expr#8=[MOD($t6, $t7)], expr#9=[3], expr#10=[=($t8, $t9)], expr#11=[OR($t10, $t3)], SAL=[$t1], $condition=[$t11]) +EnumerableCalc(expr#0..5=[{inputs}], expr#6=[RAND()], expr#7=[CAST($t6):INTEGER NOT NULL], expr#8=[2], expr#9=[MOD($t7, $t8)], expr#10=[3], expr#11=[=($t9, $t10)], expr#12=[OR($t11, $t3)], SAL=[$t1], $condition=[$t12]) EnumerableMergeJoin(condition=[=($2, $4)], joinType=[left]) EnumerableSort(sort0=[$2], dir0=[ASC]) EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], SAL=[$t5], DEPTNO=[$t7]) EnumerableTableScan(table=[[scott, EMP]]) EnumerableSort(sort0=[$1], dir0=[ASC]) - EnumerableCalc(expr#0..1=[{inputs}], cs=[$t1], DEPTNO=[$t0]) - EnumerableWindow(window#0=[window(partition {0} aggs [FIRST_VALUE($1)])], constants=[[false]]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[false], expr#3=[1], expr#4=[<=($t1, $t3)], cs=[$t2], DEPTNO=[$t0], rn=[$t1], $condition=[$t4]) + EnumerableWindow(window#0=[window(partition {0} rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])], constants=[[false]]) EnumerableCalc(expr#0..2=[{inputs}], DEPTNO=[$t0]) EnumerableTableScan(table=[[scott, DEPT]]) !plan @@ -2328,14 +2320,14 @@ select sal from "scott".emp e (0 rows) !ok -EnumerableCalc(expr#0..4=[{inputs}], expr#5=[NOT($t3)], expr#6=[IS NOT NULL($t3)], expr#7=[OR($t5, $t6)], expr#8=[IS NOT TRUE($t7)], SAL=[$t1], $condition=[$t8]) +EnumerableCalc(expr#0..5=[{inputs}], expr#6=[NOT($t3)], expr#7=[IS NOT NULL($t3)], expr#8=[OR($t6, $t7)], expr#9=[IS NOT TRUE($t8)], SAL=[$t1], $condition=[$t9]) EnumerableMergeJoin(condition=[=($2, $4)], joinType=[left]) EnumerableSort(sort0=[$2], dir0=[ASC]) EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], SAL=[$t5], DEPTNO=[$t7]) EnumerableTableScan(table=[[scott, EMP]]) EnumerableSort(sort0=[$1], dir0=[ASC]) - EnumerableCalc(expr#0..1=[{inputs}], cs=[$t1], DEPTNO=[$t0]) - EnumerableWindow(window#0=[window(partition {0} aggs [FIRST_VALUE($1)])], constants=[[false]]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[false], expr#3=[1], expr#4=[<=($t1, $t3)], cs=[$t2], DEPTNO=[$t0], rn=[$t1], $condition=[$t4]) + EnumerableWindow(window#0=[window(partition {0} rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])], constants=[[false]]) EnumerableCalc(expr#0..2=[{inputs}], DEPTNO=[$t0]) EnumerableTableScan(table=[[scott, DEPT]]) !plan @@ -2400,16 +2392,17 @@ select sal from "scott".emp e (11 rows) !ok -EnumerableCalc(expr#0..4=[{inputs}], expr#5=[NOT($t3)], expr#6=[IS NOT NULL($t3)], expr#7=[OR($t5, $t6)], expr#8=[IS NOT TRUE($t7)], SAL=[$t1], $condition=[$t8]) +EnumerableCalc(expr#0..5=[{inputs}], expr#6=[NOT($t3)], expr#7=[IS NOT NULL($t3)], expr#8=[OR($t6, $t7)], expr#9=[IS NOT TRUE($t8)], SAL=[$t1], $condition=[$t9]) EnumerableMergeJoin(condition=[=($2, $4)], joinType=[left]) EnumerableSort(sort0=[$2], dir0=[ASC]) EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], SAL=[$t5], DEPTNO=[$t7]) EnumerableTableScan(table=[[scott, EMP]]) - EnumerableSort(sort0=[$1], dir0=[ASC]) - EnumerableCalc(expr#0..1=[{inputs}], cs=[$t1], DEPTNO1=[$t0]) - EnumerableWindow(window#0=[window(partition {0} aggs [FIRST_VALUE($1)])], constants=[[true]]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[10], expr#4=[CAST($t0):INTEGER NOT NULL], expr#5=[=($t3, $t4)], DEPTNO=[$t0], $condition=[$t5]) - EnumerableTableScan(table=[[scott, DEPT]]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1], expr#4=[<=($t2, $t3)], proj#0..2=[{exprs}], $condition=[$t4]) + EnumerableSort(sort0=[$1], dir0=[ASC]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[true], cs=[$t2], DEPTNO1=[$t0], rn=[$t1]) + EnumerableWindow(window#0=[window(partition {0} rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])], constants=[[true]]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[10], expr#4=[CAST($t0):INTEGER NOT NULL], expr#5=[=($t3, $t4)], DEPTNO=[$t0], $condition=[$t5]) + EnumerableTableScan(table=[[scott, DEPT]]) !plan # Test filter literal NOT IN nullable correlated @@ -2433,16 +2426,17 @@ select sal from "scott".emp e (11 rows) !ok -EnumerableCalc(expr#0..4=[{inputs}], expr#5=[NOT($t3)], expr#6=[IS NOT NULL($t3)], expr#7=[OR($t5, $t6)], expr#8=[IS NOT TRUE($t7)], SAL=[$t1], $condition=[$t8]) +EnumerableCalc(expr#0..5=[{inputs}], expr#6=[NOT($t3)], expr#7=[IS NOT NULL($t3)], expr#8=[OR($t6, $t7)], expr#9=[IS NOT TRUE($t8)], SAL=[$t1], $condition=[$t9]) EnumerableMergeJoin(condition=[=($2, $4)], joinType=[left]) EnumerableSort(sort0=[$2], dir0=[ASC]) EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], SAL=[$t5], DEPTNO=[$t7]) EnumerableTableScan(table=[[scott, EMP]]) - EnumerableSort(sort0=[$1], dir0=[ASC]) - EnumerableCalc(expr#0..2=[{inputs}], cs=[$t2], DEPTNO=[$t0]) - EnumerableWindow(window#0=[window(partition {0} order by [1 DESC] range between UNBOUNDED PRECEDING and UNBOUNDED FOLLOWING aggs [FIRST_VALUE($1)])]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[CAST($t0):INTEGER NOT NULL], expr#4=[0], expr#5=[>($t3, $t4)], expr#6=[null:TINYINT], expr#7=[CASE($t5, $t0, $t6)], expr#8=[IS NOT NULL($t7)], expr#9=[CAST($t7):INTEGER], expr#10=[Sarg[10; NULL AS TRUE]], expr#11=[SEARCH($t9, $t10)], DEPTNO=[$t0], $1=[$t8], $condition=[$t11]) - EnumerableTableScan(table=[[scott, DEPT]]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1], expr#4=[<=($t2, $t3)], proj#0..2=[{exprs}], $condition=[$t4]) + EnumerableSort(sort0=[$1], dir0=[ASC]) + EnumerableCalc(expr#0..2=[{inputs}], cs=[$t1], DEPTNO=[$t0], rn=[$t2]) + EnumerableWindow(window#0=[window(partition {0} order by [1 DESC] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[CAST($t0):INTEGER NOT NULL], expr#4=[0], expr#5=[>($t3, $t4)], expr#6=[null:TINYINT], expr#7=[CASE($t5, $t0, $t6)], expr#8=[IS NOT NULL($t7)], expr#9=[CAST($t7):INTEGER], expr#10=[Sarg[10; NULL AS TRUE]], expr#11=[SEARCH($t9, $t10)], DEPTNO=[$t0], $1=[$t8], $condition=[$t11]) + EnumerableTableScan(table=[[scott, DEPT]]) !plan # Test filter null IN required is unknown correlated @@ -6804,4 +6798,350 @@ FROM t0; !ok +# [CALCITE-6942] Support decorrelated for sub-queries with LIMIT 1 and OFFSET +select * from emp +where exists ( +select 1 from dept where emp.deptno=dept.deptno limit 1); ++-------+--------+-----------+------+------------+---------+---------+--------+ +| EMPNO | ENAME | JOB | MGR | HIREDATE | SAL | COMM | DEPTNO | ++-------+--------+-----------+------+------------+---------+---------+--------+ +| 7369 | SMITH | CLERK | 7902 | 1980-12-17 | 800.00 | | 20 | +| 7499 | ALLEN | SALESMAN | 7698 | 1981-02-20 | 1600.00 | 300.00 | 30 | +| 7521 | WARD | SALESMAN | 7698 | 1981-02-22 | 1250.00 | 500.00 | 30 | +| 7566 | JONES | MANAGER | 7839 | 1981-02-04 | 2975.00 | | 20 | +| 7654 | MARTIN | SALESMAN | 7698 | 1981-09-28 | 1250.00 | 1400.00 | 30 | +| 7698 | BLAKE | MANAGER | 7839 | 1981-01-05 | 2850.00 | | 30 | +| 7782 | CLARK | MANAGER | 7839 | 1981-06-09 | 2450.00 | | 10 | +| 7788 | SCOTT | ANALYST | 7566 | 1987-04-19 | 3000.00 | | 20 | +| 7839 | KING | PRESIDENT | | 1981-11-17 | 5000.00 | | 10 | +| 7844 | TURNER | SALESMAN | 7698 | 1981-09-08 | 1500.00 | 0.00 | 30 | +| 7876 | ADAMS | CLERK | 7788 | 1987-05-23 | 1100.00 | | 20 | +| 7900 | JAMES | CLERK | 7698 | 1981-12-03 | 950.00 | | 30 | +| 7902 | FORD | ANALYST | 7566 | 1981-12-03 | 3000.00 | | 20 | +| 7934 | MILLER | CLERK | 7782 | 1982-01-23 | 1300.00 | | 10 | ++-------+--------+-----------+------+------------+---------+---------+--------+ +(14 rows) + +!ok + +# [CALCITE-6942] Support decorrelated for sub-queries with LIMIT 1 and OFFSET +SELECT deptno, ename +FROM + (SELECT DISTINCT deptno FROM emp) t1, + LATERAL ( + SELECT ename, sal + FROM emp + WHERE deptno = t1.deptno + ORDER BY sal + DESC LIMIT 3) s; ++--------+--------+ +| DEPTNO | ENAME | ++--------+--------+ +| 10 | CLARK | +| 10 | KING | +| 10 | MILLER | +| 20 | FORD | +| 20 | JONES | +| 20 | SCOTT | +| 30 | ALLEN | +| 30 | BLAKE | +| 30 | TURNER | ++--------+--------+ +(9 rows) + +!ok + +# [CALCITE-6942] Support decorrelated for sub-queries with LIMIT 1 and OFFSET +SELECT * +FROM dept d +WHERE d.deptno IN ( + SELECT e.deptno + FROM emp e + WHERE d.deptno = e.deptno +) +LIMIT 2 +OFFSET 1; ++--------+----------+---------+ +| DEPTNO | DNAME | LOC | ++--------+----------+---------+ +| 20 | RESEARCH | DALLAS | +| 30 | SALES | CHICAGO | ++--------+----------+---------+ +(2 rows) + +!ok + +# [CALCITE-6942] Support decorrelated for sub-queries with LIMIT 1 and OFFSET +SELECT * +FROM dept d +WHERE d.deptno IN ( + SELECT e.deptno + FROM emp e + WHERE d.deptno = e.deptno + LIMIT 10 + OFFSET 2 +) +LIMIT 2 +OFFSET 1; ++--------+----------+---------+ +| DEPTNO | DNAME | LOC | ++--------+----------+---------+ +| 20 | RESEARCH | DALLAS | +| 30 | SALES | CHICAGO | ++--------+----------+---------+ +(2 rows) + +!ok + +# [CALCITE-6942] Support decorrelated for sub-queries with LIMIT 1 and OFFSET +SELECT * +FROM dept d +WHERE d.deptno IN ( + SELECT e.deptno + FROM emp e + WHERE d.deptno = e.deptno +) +OFFSET 1; ++--------+----------+---------+ +| DEPTNO | DNAME | LOC | ++--------+----------+---------+ +| 20 | RESEARCH | DALLAS | +| 30 | SALES | CHICAGO | ++--------+----------+---------+ +(2 rows) + +!ok + +# [CALCITE-6942] Support decorrelated for sub-queries with LIMIT 1 and OFFSET +SELECT * +FROM dept d +WHERE d.deptno IN ( + SELECT e.deptno + FROM emp e + WHERE d.deptno = e.deptno + OFFSET 2 +) +OFFSET 1; ++--------+----------+---------+ +| DEPTNO | DNAME | LOC | ++--------+----------+---------+ +| 20 | RESEARCH | DALLAS | +| 30 | SALES | CHICAGO | ++--------+----------+---------+ +(2 rows) + +!ok + +# [CALCITE-6942] Support decorrelated for sub-queries with LIMIT 1 and OFFSET +SELECT * +FROM dept d +WHERE EXISTS ( + SELECT * + FROM emp e + WHERE d.deptno = e.deptno + OFFSET 2 +); ++--------+------------+----------+ +| DEPTNO | DNAME | LOC | ++--------+------------+----------+ +| 10 | ACCOUNTING | NEW YORK | +| 20 | RESEARCH | DALLAS | +| 30 | SALES | CHICAGO | ++--------+------------+----------+ +(3 rows) + +!ok + +# [CALCITE-6942] Support decorrelated for sub-queries with LIMIT 1 and OFFSET +SELECT * +FROM dept d +JOIN LATERAL ( + SELECT * + FROM emp e + WHERE e.deptno = d.deptno + OFFSET 2 +) s ON TRUE; ++--------+------------+----------+-------+--------+----------+------+------------+---------+---------+---------+ +| DEPTNO | DNAME | LOC | EMPNO | ENAME | JOB | MGR | HIREDATE | SAL | COMM | DEPTNO0 | ++--------+------------+----------+-------+--------+----------+------+------------+---------+---------+---------+ +| 10 | ACCOUNTING | NEW YORK | 7934 | MILLER | CLERK | 7782 | 1982-01-23 | 1300.00 | | 10 | +| 20 | RESEARCH | DALLAS | 7788 | SCOTT | ANALYST | 7566 | 1987-04-19 | 3000.00 | | 20 | +| 20 | RESEARCH | DALLAS | 7876 | ADAMS | CLERK | 7788 | 1987-05-23 | 1100.00 | | 20 | +| 20 | RESEARCH | DALLAS | 7902 | FORD | ANALYST | 7566 | 1981-12-03 | 3000.00 | | 20 | +| 30 | SALES | CHICAGO | 7654 | MARTIN | SALESMAN | 7698 | 1981-09-28 | 1250.00 | 1400.00 | 30 | +| 30 | SALES | CHICAGO | 7698 | BLAKE | MANAGER | 7839 | 1981-01-05 | 2850.00 | | 30 | +| 30 | SALES | CHICAGO | 7844 | TURNER | SALESMAN | 7698 | 1981-09-08 | 1500.00 | 0.00 | 30 | +| 30 | SALES | CHICAGO | 7900 | JAMES | CLERK | 7698 | 1981-12-03 | 950.00 | | 30 | ++--------+------------+----------+-------+--------+----------+------+------------+---------+---------+---------+ +(8 rows) + +!ok + +# [CALCITE-6942] Support decorrelated for sub-queries with LIMIT 1 and OFFSET +SELECT * +FROM dept d +WHERE d.deptno IN ( + SELECT e.deptno + FROM emp e + OFFSET 2 +); ++--------+------------+----------+ +| DEPTNO | DNAME | LOC | ++--------+------------+----------+ +| 10 | ACCOUNTING | NEW YORK | +| 20 | RESEARCH | DALLAS | +| 30 | SALES | CHICAGO | ++--------+------------+----------+ +(3 rows) + +!ok + +# [CALCITE-6942] Support decorrelated for sub-queries with LIMIT 1 and OFFSET +SELECT * +FROM dept d +WHERE ( + SELECT SUM(e.sal) + FROM emp e + WHERE e.deptno = d.deptno + OFFSET 2 +) > 2; ++--------+-------+-----+ +| DEPTNO | DNAME | LOC | ++--------+-------+-----+ ++--------+-------+-----+ +(0 rows) + +!ok + +# [CALCITE-6942] Support decorrelated for sub-queries with LIMIT 1 and OFFSET +SELECT e.ename +FROM emp e +WHERE EXISTS ( + SELECT MAX(d.deptno) AS a + FROM dept d + WHERE d.deptno = e.deptno + GROUP BY d.loc + ORDER BY d.loc + OFFSET 1 +); ++-------+ +| ENAME | ++-------+ ++-------+ +(0 rows) + +!ok + +# [CALCITE-6942] Support decorrelated for sub-queries with LIMIT 1 and OFFSET +SELECT e.ename +FROM emp e +JOIN LATERAL ( + SELECT MAX(d.deptno) AS a + FROM dept d + WHERE d.deptno = e.deptno + GROUP BY d.loc + ORDER BY d.loc + OFFSET 1 +) s ON TRUE; ++-------+ +| ENAME | ++-------+ ++-------+ +(0 rows) + +!ok + +# [CALCITE-6942] Support decorrelated for sub-queries with LIMIT 1 and OFFSET +SELECT e.ename +FROM emp e +JOIN LATERAL ( + SELECT MAX(d.deptno) AS a + FROM dept d + WHERE d.deptno = e.deptno + GROUP BY d.loc + ORDER BY d.loc + LIMIT 2 + OFFSET 1 +) s ON TRUE; ++-------+ +| ENAME | ++-------+ ++-------+ +(0 rows) + +!ok + +# [CALCITE-6942] Support decorrelated for sub-queries with LIMIT 1 and OFFSET +SELECT e.ename +FROM emp e +WHERE EXISTS ( + SELECT MAX(d.deptno) AS a + FROM dept d + WHERE d.deptno = e.deptno + GROUP BY d.loc + ORDER BY d.loc + LIMIT 2 + OFFSET 1 +); ++-------+ +| ENAME | ++-------+ ++-------+ +(0 rows) + +!ok + +# [CALCITE-6942] Support decorrelated for sub-queries with LIMIT 1 and OFFSET +SELECT * +FROM dept d +JOIN LATERAL ( + SELECT * + FROM emp e + WHERE e.deptno = d.deptno + LIMIT 1 + OFFSET 2 +) s ON TRUE; ++--------+------------+----------+-------+--------+----------+------+------------+---------+---------+---------+ +| DEPTNO | DNAME | LOC | EMPNO | ENAME | JOB | MGR | HIREDATE | SAL | COMM | DEPTNO0 | ++--------+------------+----------+-------+--------+----------+------+------------+---------+---------+---------+ +| 10 | ACCOUNTING | NEW YORK | 7934 | MILLER | CLERK | 7782 | 1982-01-23 | 1300.00 | | 10 | +| 20 | RESEARCH | DALLAS | 7788 | SCOTT | ANALYST | 7566 | 1987-04-19 | 3000.00 | | 20 | +| 30 | SALES | CHICAGO | 7654 | MARTIN | SALESMAN | 7698 | 1981-09-28 | 1250.00 | 1400.00 | 30 | ++--------+------------+----------+-------+--------+----------+------+------------+---------+---------+---------+ +(3 rows) + +!ok + +# [CALCITE-6942] Support decorrelated for sub-queries with LIMIT 1 and OFFSET +SELECT * +FROM dept d +WHERE EXISTS ( + SELECT * + FROM emp e + WHERE e.deptno = d.deptno + LIMIT 1 + OFFSET 2 +); ++--------+------------+----------+ +| DEPTNO | DNAME | LOC | ++--------+------------+----------+ +| 10 | ACCOUNTING | NEW YORK | +| 20 | RESEARCH | DALLAS | +| 30 | SALES | CHICAGO | ++--------+------------+----------+ +(3 rows) + +!ok + +# [CALCITE-6942] Support decorrelated for sub-queries with LIMIT 1 and OFFSET +SELECT deptno FROM dept WHERE 1000.00 > +(SELECT sal FROM emp WHERE dept.deptno = emp.deptno order by emp.sal limit 1 offset 10); ++--------+ +| DEPTNO | ++--------+ ++--------+ +(0 rows) + +!ok + # End sub-query.iq diff --git a/site/_docs/history.md b/site/_docs/history.md index 399d0bfcec92..ea6fd1372674 100644 --- a/site/_docs/history.md +++ b/site/_docs/history.md @@ -49,6 +49,9 @@ other software versions as specified in gradle.properties. #### Breaking Changes {: #breaking-1-42-0} +* [CALCITE-6942] +Rename the method `decorrelateFetchOneSort` to `decorrelateSortWithRowNumber`. + #### New features {: #new-features-1-42-0} From d1d6431989ab74a1c55758b80c408c8f7b5e1ad9 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Thu, 18 Dec 2025 17:12:08 -0800 Subject: [PATCH 073/562] [CALCITE-7339] Most classes in SqlDdlNodes use an incorrect SqlCallFactory Signed-off-by: Mihai Budiu --- .../org/apache/calcite/sql/SqlCollation.java | 23 ++++ .../org/apache/calcite/sql/SqlLambda.java | 7 ++ .../sql/ddl/SqlAttributeDefinition.java | 24 ++++- .../calcite/sql/ddl/SqlCheckConstraint.java | 14 ++- .../calcite/sql/ddl/SqlColumnDeclaration.java | 32 ++++-- .../sql/ddl/SqlCreateForeignSchema.java | 21 +++- .../calcite/sql/ddl/SqlCreateFunction.java | 25 ++++- .../sql/ddl/SqlCreateMaterializedView.java | 20 +++- .../calcite/sql/ddl/SqlCreateSchema.java | 19 +++- .../calcite/sql/ddl/SqlCreateTable.java | 19 +++- .../calcite/sql/ddl/SqlCreateTableLike.java | 23 +++- .../apache/calcite/sql/ddl/SqlCreateType.java | 17 ++- .../apache/calcite/sql/ddl/SqlCreateView.java | 17 ++- .../calcite/sql/ddl/SqlDropFunction.java | 16 ++- .../sql/ddl/SqlDropMaterializedView.java | 17 ++- .../apache/calcite/sql/ddl/SqlDropObject.java | 5 +- .../apache/calcite/sql/ddl/SqlDropSchema.java | 19 +++- .../apache/calcite/sql/ddl/SqlDropTable.java | 16 ++- .../apache/calcite/sql/ddl/SqlDropType.java | 16 ++- .../apache/calcite/sql/ddl/SqlDropView.java | 16 ++- .../calcite/sql/ddl/SqlKeyConstraint.java | 25 ++++- .../calcite/sql/ddl/SqlTruncateTable.java | 27 +++-- .../org/apache/calcite/util/UtilTest.java | 12 +++ .../calcite/server/ServerDdlExecutor.java | 9 +- .../apache/calcite/test/ServerParserTest.java | 101 ++++++++++++++++++ 25 files changed, 483 insertions(+), 57 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql/SqlCollation.java b/core/src/main/java/org/apache/calcite/sql/SqlCollation.java index b02035f79109..5f6e4805f764 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlCollation.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlCollation.java @@ -17,8 +17,10 @@ package org.apache.calcite.sql; import org.apache.calcite.config.CalciteSystemProperty; +import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.sql.parser.SqlParserUtil; import org.apache.calcite.util.Glossary; +import org.apache.calcite.util.ImmutableNullableList; import org.apache.calcite.util.SerializableCharset; import org.apache.calcite.util.Util; @@ -37,6 +39,8 @@ import static org.apache.calcite.util.Static.RESOURCE; +import static java.util.Objects.requireNonNull; + /** * A SqlCollation is an object representing a Collate * statement. It is immutable. @@ -113,6 +117,25 @@ public SqlCollation( this.collationName = generateCollationName(charset); } + /** Encode all the information require to reconstruct a SqlCollection in a SqlList object. */ + public SqlNodeList asList() { + return new SqlNodeList( + ImmutableNullableList.of( + SqlLiteral.createCharString(this.getCollationName(), SqlParserPos.ZERO), + SqlLiteral.createSymbol(coercibility, SqlParserPos.ZERO)), + SqlParserPos.ZERO); + } + + /** The inverse of the {@link #asList} function. */ + public static SqlCollation fromSqlList(SqlNodeList list) { + assert list.size() == 2; + String name = ((SqlLiteral) list.get(0)).getValueAs(String.class); + Coercibility coercibility = ((SqlLiteral) list.get(1)).symbolValue(Coercibility.class); + return new SqlCollation( + name, + requireNonNull(coercibility, "coercibility")); + } + /** * Creates a Collation by its coercibility, locale, charset and strength. */ diff --git a/core/src/main/java/org/apache/calcite/sql/SqlLambda.java b/core/src/main/java/org/apache/calcite/sql/SqlLambda.java index c0f8b2b145e6..696753b4f4e5 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlLambda.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlLambda.java @@ -107,6 +107,13 @@ private static class SqlLambdaOperator extends SqlSpecialOperator { super("->", SqlKind.LAMBDA); } + @Override public SqlCall createCall( + @Nullable SqlLiteral functionQualifier, SqlParserPos pos, @Nullable SqlNode... operands) { + return new SqlLambda(pos, + (SqlNodeList) requireNonNull(operands[0], "parameters"), + requireNonNull(operands[1], "expression")); + } + @Override public RelDataType deriveType( SqlValidator validator, SqlValidatorScope scope, SqlCall call) { final SqlLambda lambdaExpr = (SqlLambda) call; diff --git a/core/src/main/java/org/apache/calcite/sql/ddl/SqlAttributeDefinition.java b/core/src/main/java/org/apache/calcite/sql/ddl/SqlAttributeDefinition.java index a25528784608..494c262959a0 100644 --- a/core/src/main/java/org/apache/calcite/sql/ddl/SqlAttributeDefinition.java +++ b/core/src/main/java/org/apache/calcite/sql/ddl/SqlAttributeDefinition.java @@ -21,25 +21,37 @@ import org.apache.calcite.sql.SqlDataTypeSpec; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlLiteral; import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlNodeList; import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.SqlSpecialOperator; import org.apache.calcite.sql.SqlWriter; import org.apache.calcite.sql.parser.SqlParserPos; - -import com.google.common.collect.ImmutableList; +import org.apache.calcite.util.ImmutableNullableList; import org.checkerframework.checker.nullness.qual.Nullable; import java.util.List; +import static java.util.Objects.requireNonNull; + /** * Parse tree for SqlAttributeDefinition, * which is part of a {@link SqlCreateType}. */ public class SqlAttributeDefinition extends SqlCall { - private static final SqlSpecialOperator OPERATOR = - new SqlSpecialOperator("ATTRIBUTE_DEF", SqlKind.ATTRIBUTE_DEF); + private static final SqlOperator OPERATOR = + new SqlSpecialOperator("ATTRIBUTE_DEF", SqlKind.ATTRIBUTE_DEF) { + @Override public SqlCall createCall(@Nullable SqlLiteral functionQualifier, + SqlParserPos pos, @Nullable SqlNode... operands) { + return new SqlAttributeDefinition(pos, + (SqlIdentifier) requireNonNull(operands[0], "name"), + (SqlDataTypeSpec) requireNonNull(operands[1], "dataType"), + operands[2], + operands[3] != null ? SqlCollation.fromSqlList((SqlNodeList) operands[3]) : null); + } + }; public final SqlIdentifier name; public final SqlDataTypeSpec dataType; @@ -60,8 +72,10 @@ public class SqlAttributeDefinition extends SqlCall { return OPERATOR; } + @SuppressWarnings("nullness") @Override public List getOperandList() { - return ImmutableList.of(name, dataType); + return ImmutableNullableList.of(name, dataType, expression, + collation != null ? collation.asList() : null); } @Override public void unparse(SqlWriter writer, int leftPrec, int rightPrec) { diff --git a/core/src/main/java/org/apache/calcite/sql/ddl/SqlCheckConstraint.java b/core/src/main/java/org/apache/calcite/sql/ddl/SqlCheckConstraint.java index a74ed3e90ef7..8106f2d1e81c 100644 --- a/core/src/main/java/org/apache/calcite/sql/ddl/SqlCheckConstraint.java +++ b/core/src/main/java/org/apache/calcite/sql/ddl/SqlCheckConstraint.java @@ -19,6 +19,7 @@ import org.apache.calcite.sql.SqlCall; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlLiteral; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.SqlSpecialOperator; @@ -30,14 +31,23 @@ import java.util.List; +import static java.util.Objects.requireNonNull; + /** * Parse tree for {@code UNIQUE}, {@code PRIMARY KEY} constraints. * *

    And {@code FOREIGN KEY}, when we support it. */ public class SqlCheckConstraint extends SqlCall { - private static final SqlSpecialOperator OPERATOR = - new SqlSpecialOperator("CHECK", SqlKind.CHECK); + private static final SqlOperator OPERATOR = + new SqlSpecialOperator("CHECK", SqlKind.CHECK) { + @Override public SqlCall createCall(@Nullable SqlLiteral functionQualifier, + SqlParserPos pos, @Nullable SqlNode... operands) { + return new SqlCheckConstraint(pos, + (SqlIdentifier) operands[0], + requireNonNull(operands[1], "expression")); + } + }; private final @Nullable SqlIdentifier name; private final SqlNode expression; diff --git a/core/src/main/java/org/apache/calcite/sql/ddl/SqlColumnDeclaration.java b/core/src/main/java/org/apache/calcite/sql/ddl/SqlColumnDeclaration.java index e37075f63123..d57fcb502b2b 100644 --- a/core/src/main/java/org/apache/calcite/sql/ddl/SqlColumnDeclaration.java +++ b/core/src/main/java/org/apache/calcite/sql/ddl/SqlColumnDeclaration.java @@ -21,36 +21,50 @@ import org.apache.calcite.sql.SqlDataTypeSpec; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlLiteral; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.SqlSpecialOperator; import org.apache.calcite.sql.SqlWriter; import org.apache.calcite.sql.parser.SqlParserPos; - -import com.google.common.collect.ImmutableList; +import org.apache.calcite.util.ImmutableNullableList; import org.checkerframework.checker.nullness.qual.Nullable; import java.util.List; +import static java.util.Objects.requireNonNull; + /** * Parse tree for {@code UNIQUE}, {@code PRIMARY KEY} constraints. * *

    And {@code FOREIGN KEY}, when we support it. */ public class SqlColumnDeclaration extends SqlCall { - private static final SqlSpecialOperator OPERATOR = - new SqlSpecialOperator("COLUMN_DECL", SqlKind.COLUMN_DECL); + private static final SqlOperator OPERATOR = + new SqlSpecialOperator("COLUMN_DECL", SqlKind.COLUMN_DECL) { + @Override public SqlCall createCall(@Nullable SqlLiteral functionQualifier, + SqlParserPos pos, @Nullable SqlNode... operands) { + return new SqlColumnDeclaration(pos, + (SqlIdentifier) requireNonNull(operands[0], "name"), + (SqlDataTypeSpec) requireNonNull(operands[1], "dataType"), + operands[2], + operands[3] != null + ? ColumnStrategy.valueOf(((SqlIdentifier) operands[3]).getSimple()) + : null); + } + }; public final SqlIdentifier name; public final SqlDataTypeSpec dataType; public final @Nullable SqlNode expression; - public final ColumnStrategy strategy; + // The Babel parser can supply null for the strategy + public final @Nullable ColumnStrategy strategy; /** Creates a SqlColumnDeclaration; use {@link SqlDdlNodes#column}. */ SqlColumnDeclaration(SqlParserPos pos, SqlIdentifier name, SqlDataTypeSpec dataType, @Nullable SqlNode expression, - ColumnStrategy strategy) { + @Nullable ColumnStrategy strategy) { super(pos); this.name = name; this.dataType = dataType; @@ -62,8 +76,10 @@ public class SqlColumnDeclaration extends SqlCall { return OPERATOR; } + @SuppressWarnings("nullness") @Override public List getOperandList() { - return ImmutableList.of(name, dataType); + return ImmutableNullableList.of(name, dataType, expression, + strategy != null ? new SqlIdentifier(strategy.name(), SqlParserPos.ZERO) : null); } @Override public void unparse(SqlWriter writer, int leftPrec, int rightPrec) { @@ -73,7 +89,7 @@ public class SqlColumnDeclaration extends SqlCall { writer.keyword("NOT NULL"); } SqlNode expression = this.expression; - if (expression != null) { + if (expression != null && strategy != null) { switch (strategy) { case VIRTUAL: case STORED: diff --git a/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateForeignSchema.java b/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateForeignSchema.java index 53e106b4d8fe..e7a4993d8d8d 100644 --- a/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateForeignSchema.java +++ b/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateForeignSchema.java @@ -16,9 +16,11 @@ */ package org.apache.calcite.sql.ddl; +import org.apache.calcite.sql.SqlCall; import org.apache.calcite.sql.SqlCreate; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlLiteral; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlNodeList; import org.apache.calcite.sql.SqlOperator; @@ -51,8 +53,18 @@ public class SqlCreateForeignSchema extends SqlCreate { private final @Nullable SqlNodeList optionList; private static final SqlOperator OPERATOR = - new SqlSpecialOperator("CREATE FOREIGN SCHEMA", - SqlKind.CREATE_FOREIGN_SCHEMA); + new SqlSpecialOperator("CREATE FOREIGN SCHEMA", SqlKind.CREATE_FOREIGN_SCHEMA) { + @Override public SqlCall createCall(@Nullable SqlLiteral functionQualifier, + SqlParserPos pos, @Nullable SqlNode... operands) { + return new SqlCreateForeignSchema(pos, + ((SqlLiteral) requireNonNull(operands[0], "replace")).booleanValue(), + ((SqlLiteral) requireNonNull(operands[1], "ifNotExists")).booleanValue(), + (SqlIdentifier) requireNonNull(operands[2], "name"), + operands[3], + operands[4], + (SqlNodeList) operands[5]); + } + }; /** Creates a SqlCreateForeignSchema. */ SqlCreateForeignSchema(SqlParserPos pos, boolean replace, boolean ifNotExists, @@ -69,7 +81,10 @@ public class SqlCreateForeignSchema extends SqlCreate { @SuppressWarnings("nullness") @Override public List getOperandList() { - return ImmutableNullableList.of(name, type, library, optionList); + return ImmutableNullableList.of( + SqlLiteral.createBoolean(getReplace(), SqlParserPos.ZERO), + SqlLiteral.createBoolean(ifNotExists, SqlParserPos.ZERO), + name, type, library, optionList); } @Override public void unparse(SqlWriter writer, int leftPrec, int rightPrec) { diff --git a/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateFunction.java b/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateFunction.java index ae0abc289822..fb859b2013d0 100644 --- a/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateFunction.java @@ -16,6 +16,7 @@ */ package org.apache.calcite.sql.ddl; +import org.apache.calcite.sql.SqlCall; import org.apache.calcite.sql.SqlCreate; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlKind; @@ -29,7 +30,10 @@ import org.apache.calcite.util.Pair; import org.apache.calcite.util.Util; -import java.util.Arrays; +import com.google.common.collect.ImmutableList; + +import org.checkerframework.checker.nullness.qual.Nullable; + import java.util.List; import static com.google.common.base.Preconditions.checkArgument; @@ -44,8 +48,18 @@ public class SqlCreateFunction extends SqlCreate { private final SqlNode className; private final SqlNodeList usingList; - private static final SqlSpecialOperator OPERATOR = - new SqlSpecialOperator("CREATE FUNCTION", SqlKind.CREATE_FUNCTION); + private static final SqlOperator OPERATOR = + new SqlSpecialOperator("CREATE FUNCTION", SqlKind.CREATE_FUNCTION) { + @Override public SqlCall createCall(@Nullable SqlLiteral functionQualifier, + SqlParserPos pos, @Nullable SqlNode... operands) { + return new SqlCreateFunction(pos, + ((SqlLiteral) requireNonNull(operands[0], "replace")).booleanValue(), + ((SqlLiteral) requireNonNull(operands[1], "ifNotExists")).booleanValue(), + (SqlIdentifier) requireNonNull(operands[2], "name"), + requireNonNull(operands[3], "className"), + (SqlNodeList) requireNonNull(operands[4], "usingList")); + } + }; /** Creates a SqlCreateFunction. */ public SqlCreateFunction(SqlParserPos pos, boolean replace, @@ -91,6 +105,9 @@ private List> pairs() { } @Override public List getOperandList() { - return Arrays.asList(name, className, usingList); + return ImmutableList.of( + SqlLiteral.createBoolean(getReplace(), SqlParserPos.ZERO), + SqlLiteral.createBoolean(ifNotExists, SqlParserPos.ZERO), + name, className, usingList); } } diff --git a/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateMaterializedView.java b/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateMaterializedView.java index a5ccd904db8a..3894cc276971 100644 --- a/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateMaterializedView.java +++ b/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateMaterializedView.java @@ -16,9 +16,11 @@ */ package org.apache.calcite.sql.ddl; +import org.apache.calcite.sql.SqlCall; import org.apache.calcite.sql.SqlCreate; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlLiteral; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlNodeList; import org.apache.calcite.sql.SqlOperator; @@ -42,8 +44,17 @@ public class SqlCreateMaterializedView extends SqlCreate { public final SqlNode query; private static final SqlOperator OPERATOR = - new SqlSpecialOperator("CREATE MATERIALIZED VIEW", - SqlKind.CREATE_MATERIALIZED_VIEW); + new SqlSpecialOperator("CREATE MATERIALIZED VIEW", SqlKind.CREATE_MATERIALIZED_VIEW) { + @Override public SqlCall createCall(@Nullable SqlLiteral functionQualifier, + SqlParserPos pos, @Nullable SqlNode... operands) { + return new SqlCreateMaterializedView(pos, + ((SqlLiteral) requireNonNull(operands[0], "replace")).booleanValue(), + ((SqlLiteral) requireNonNull(operands[1], "ifNotExists")).booleanValue(), + (SqlIdentifier) requireNonNull(operands[2], "name"), + (SqlNodeList) operands[3], + requireNonNull(operands[4], "query")); + } + }; /** Creates a SqlCreateView. */ SqlCreateMaterializedView(SqlParserPos pos, boolean replace, @@ -57,7 +68,10 @@ public class SqlCreateMaterializedView extends SqlCreate { @SuppressWarnings("nullness") @Override public List getOperandList() { - return ImmutableNullableList.of(name, columnList, query); + return ImmutableNullableList.of( + SqlLiteral.createBoolean(getReplace(), SqlParserPos.ZERO), + SqlLiteral.createBoolean(ifNotExists, SqlParserPos.ZERO), + name, columnList, query); } @Override public void unparse(SqlWriter writer, int leftPrec, int rightPrec) { diff --git a/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateSchema.java b/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateSchema.java index ee405b0c2df1..d661daadc219 100644 --- a/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateSchema.java +++ b/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateSchema.java @@ -16,9 +16,11 @@ */ package org.apache.calcite.sql.ddl; +import org.apache.calcite.sql.SqlCall; import org.apache.calcite.sql.SqlCreate; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlLiteral; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.SqlSpecialOperator; @@ -26,6 +28,8 @@ import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.util.ImmutableNullableList; +import org.checkerframework.checker.nullness.qual.Nullable; + import java.util.List; import static java.util.Objects.requireNonNull; @@ -37,7 +41,15 @@ public class SqlCreateSchema extends SqlCreate { public final SqlIdentifier name; private static final SqlOperator OPERATOR = - new SqlSpecialOperator("CREATE SCHEMA", SqlKind.CREATE_SCHEMA); + new SqlSpecialOperator("CREATE SCHEMA", SqlKind.CREATE_SCHEMA) { + @Override public SqlCall createCall(@Nullable SqlLiteral functionQualifier, + SqlParserPos pos, @Nullable SqlNode... operands) { + return new SqlCreateSchema(pos, + ((SqlLiteral) requireNonNull(operands[0], "replace")).booleanValue(), + ((SqlLiteral) requireNonNull(operands[1], "ifNotExists")).booleanValue(), + (SqlIdentifier) requireNonNull(operands[2], "name")); + } + }; /** Creates a SqlCreateSchema. */ SqlCreateSchema(SqlParserPos pos, boolean replace, boolean ifNotExists, @@ -47,7 +59,10 @@ public class SqlCreateSchema extends SqlCreate { } @Override public List getOperandList() { - return ImmutableNullableList.of(name); + return ImmutableNullableList.of( + SqlLiteral.createBoolean(getReplace(), SqlParserPos.ZERO), + SqlLiteral.createBoolean(ifNotExists, SqlParserPos.ZERO), + name); } @Override public void unparse(SqlWriter writer, int leftPrec, int rightPrec) { diff --git a/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateTable.java b/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateTable.java index 7e7dd144cbed..1bf283bf0cd6 100644 --- a/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateTable.java +++ b/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateTable.java @@ -16,9 +16,11 @@ */ package org.apache.calcite.sql.ddl; +import org.apache.calcite.sql.SqlCall; import org.apache.calcite.sql.SqlCreate; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlLiteral; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlNodeList; import org.apache.calcite.sql.SqlOperator; @@ -42,7 +44,17 @@ public class SqlCreateTable extends SqlCreate { public final @Nullable SqlNode query; private static final SqlOperator OPERATOR = - new SqlSpecialOperator("CREATE TABLE", SqlKind.CREATE_TABLE); + new SqlSpecialOperator("CREATE TABLE", SqlKind.CREATE_TABLE) { + @Override public SqlCall createCall(@Nullable SqlLiteral functionQualifier, + SqlParserPos pos, @Nullable SqlNode... operands) { + return new SqlCreateTable(pos, + ((SqlLiteral) requireNonNull(operands[0], "replace")).booleanValue(), + ((SqlLiteral) requireNonNull(operands[1], "ifNotExists")).booleanValue(), + (SqlIdentifier) requireNonNull(operands[2], "name"), + (SqlNodeList) requireNonNull(operands[3], "columnList"), + operands[4]); + } + }; /** Creates a SqlCreateTable. */ protected SqlCreateTable(SqlParserPos pos, boolean replace, boolean ifNotExists, @@ -55,7 +67,10 @@ protected SqlCreateTable(SqlParserPos pos, boolean replace, boolean ifNotExists, @SuppressWarnings("nullness") @Override public List getOperandList() { - return ImmutableNullableList.of(name, columnList, query); + return ImmutableNullableList.of( + SqlLiteral.createBoolean(getReplace(), SqlParserPos.ZERO), + SqlLiteral.createBoolean(ifNotExists, SqlParserPos.ZERO), + name, columnList, query); } @Override public void unparse(SqlWriter writer, int leftPrec, int rightPrec) { diff --git a/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateTableLike.java b/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateTableLike.java index 957f87531dd4..225ae2d8d855 100644 --- a/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateTableLike.java +++ b/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateTableLike.java @@ -16,6 +16,7 @@ */ package org.apache.calcite.sql.ddl; +import org.apache.calcite.sql.SqlCall; import org.apache.calcite.sql.SqlCreate; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlKind; @@ -29,6 +30,8 @@ import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.util.ImmutableNullableList; +import org.checkerframework.checker.nullness.qual.Nullable; + import java.util.HashSet; import java.util.List; import java.util.Set; @@ -36,12 +39,25 @@ import static com.google.common.base.Preconditions.checkArgument; +import static java.util.Objects.requireNonNull; + /** * Parse tree for {@code CREATE TABLE LIKE} statement. */ public class SqlCreateTableLike extends SqlCreate { private static final SqlOperator OPERATOR = - new SqlSpecialOperator("CREATE TABLE LIKE", SqlKind.CREATE_TABLE_LIKE); + new SqlSpecialOperator("CREATE TABLE LIKE", SqlKind.CREATE_TABLE_LIKE) { + @Override public SqlCall createCall(@Nullable SqlLiteral functionQualifier, + SqlParserPos pos, @Nullable SqlNode... operands) { + return new SqlCreateTableLike(pos, + ((SqlLiteral) requireNonNull(operands[0], "replace")).booleanValue(), + ((SqlLiteral) requireNonNull(operands[1], "ifNotExists")).booleanValue(), + (SqlIdentifier) requireNonNull(operands[2], "name"), + (SqlIdentifier) requireNonNull(operands[3], "sourceTable"), + (SqlNodeList) requireNonNull(operands[4], "includingOptions"), + (SqlNodeList) requireNonNull(operands[5], "excludingOptions")); + } + }; /** * The LikeOption specify which additional properties of the original table to copy. @@ -83,7 +99,10 @@ public SqlCreateTableLike(SqlParserPos pos, boolean replace, boolean ifNotExists } @Override public List getOperandList() { - return ImmutableNullableList.of(name, sourceTable, includingOptions, + return ImmutableNullableList.of( + SqlLiteral.createBoolean(getReplace(), SqlParserPos.ZERO), + SqlLiteral.createBoolean(ifNotExists, SqlParserPos.ZERO), + name, sourceTable, includingOptions, excludingOptions); } diff --git a/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateType.java b/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateType.java index 67188f736bfe..7e4dc1f632d8 100644 --- a/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateType.java +++ b/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateType.java @@ -16,10 +16,12 @@ */ package org.apache.calcite.sql.ddl; +import org.apache.calcite.sql.SqlCall; import org.apache.calcite.sql.SqlCreate; import org.apache.calcite.sql.SqlDataTypeSpec; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlLiteral; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlNodeList; import org.apache.calcite.sql.SqlOperator; @@ -43,7 +45,16 @@ public class SqlCreateType extends SqlCreate { public final @Nullable SqlDataTypeSpec dataType; private static final SqlOperator OPERATOR = - new SqlSpecialOperator("CREATE TYPE", SqlKind.CREATE_TYPE); + new SqlSpecialOperator("CREATE TYPE", SqlKind.CREATE_TYPE) { + @Override public SqlCall createCall(@Nullable SqlLiteral functionQualifier, + SqlParserPos pos, @Nullable SqlNode... operands) { + return new SqlCreateType(pos, + ((SqlLiteral) requireNonNull(operands[0], "replace")).booleanValue(), + (SqlIdentifier) requireNonNull(operands[1], "name"), + (SqlNodeList) operands[2], + (SqlDataTypeSpec) operands[3]); + } + }; /** Creates a SqlCreateType. */ SqlCreateType(SqlParserPos pos, boolean replace, SqlIdentifier name, @@ -56,7 +67,9 @@ public class SqlCreateType extends SqlCreate { @SuppressWarnings("nullness") @Override public List getOperandList() { - return ImmutableNullableList.of(name, attributeDefs); + return ImmutableNullableList.of( + SqlLiteral.createBoolean(getReplace(), SqlParserPos.ZERO), + name, attributeDefs, dataType); } @Override public void unparse(SqlWriter writer, int leftPrec, int rightPrec) { diff --git a/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateView.java b/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateView.java index 662e86496693..38297f83e6a3 100644 --- a/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateView.java +++ b/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateView.java @@ -16,9 +16,11 @@ */ package org.apache.calcite.sql.ddl; +import org.apache.calcite.sql.SqlCall; import org.apache.calcite.sql.SqlCreate; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlLiteral; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlNodeList; import org.apache.calcite.sql.SqlOperator; @@ -42,7 +44,16 @@ public class SqlCreateView extends SqlCreate { public final SqlNode query; private static final SqlOperator OPERATOR = - new SqlSpecialOperator("CREATE VIEW", SqlKind.CREATE_VIEW); + new SqlSpecialOperator("CREATE VIEW", SqlKind.CREATE_VIEW) { + @Override public SqlCall createCall(@Nullable SqlLiteral functionQualifier, + SqlParserPos pos, @Nullable SqlNode... operands) { + return new SqlCreateView(pos, + ((SqlLiteral) requireNonNull(operands[0], "replace")).booleanValue(), + (SqlIdentifier) requireNonNull(operands[1], "name"), + (SqlNodeList) operands[3], + requireNonNull(operands[4], "query")); + } + }; /** Creates a SqlCreateView. */ SqlCreateView(SqlParserPos pos, boolean replace, SqlIdentifier name, @@ -55,7 +66,9 @@ public class SqlCreateView extends SqlCreate { @SuppressWarnings("nullness") @Override public List getOperandList() { - return ImmutableNullableList.of(name, columnList, query); + return ImmutableNullableList.of( + SqlLiteral.createBoolean(getReplace(), SqlParserPos.ZERO), + name, columnList, query); } @Override public void unparse(SqlWriter writer, int leftPrec, int rightPrec) { diff --git a/core/src/main/java/org/apache/calcite/sql/ddl/SqlDropFunction.java b/core/src/main/java/org/apache/calcite/sql/ddl/SqlDropFunction.java index 9b24b030b9c9..a8601ba9d0f8 100644 --- a/core/src/main/java/org/apache/calcite/sql/ddl/SqlDropFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/ddl/SqlDropFunction.java @@ -16,18 +16,32 @@ */ package org.apache.calcite.sql.ddl; +import org.apache.calcite.sql.SqlCall; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlLiteral; +import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.SqlSpecialOperator; import org.apache.calcite.sql.parser.SqlParserPos; +import org.checkerframework.checker.nullness.qual.Nullable; + +import static java.util.Objects.requireNonNull; + /** * Parse tree for {@code DROP FUNCTION} statement. */ public class SqlDropFunction extends SqlDropObject { private static final SqlOperator OPERATOR = - new SqlSpecialOperator("DROP FUNCTION", SqlKind.DROP_FUNCTION); + new SqlSpecialOperator("DROP FUNCTION", SqlKind.DROP_FUNCTION) { + @Override public SqlCall createCall(@Nullable SqlLiteral functionQualifier, + SqlParserPos pos, @Nullable SqlNode... operands) { + return new SqlDropFunction(pos, + ((SqlLiteral) requireNonNull(operands[0], "ifExists")).booleanValue(), + (SqlIdentifier) requireNonNull(operands[1], "name")); + } + }; /** Creates a SqlDropFunction. */ public SqlDropFunction(SqlParserPos pos, boolean ifExists, diff --git a/core/src/main/java/org/apache/calcite/sql/ddl/SqlDropMaterializedView.java b/core/src/main/java/org/apache/calcite/sql/ddl/SqlDropMaterializedView.java index 374167478a57..09807cbc96a8 100644 --- a/core/src/main/java/org/apache/calcite/sql/ddl/SqlDropMaterializedView.java +++ b/core/src/main/java/org/apache/calcite/sql/ddl/SqlDropMaterializedView.java @@ -16,19 +16,32 @@ */ package org.apache.calcite.sql.ddl; +import org.apache.calcite.sql.SqlCall; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlLiteral; +import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.SqlSpecialOperator; import org.apache.calcite.sql.parser.SqlParserPos; +import org.checkerframework.checker.nullness.qual.Nullable; + +import static java.util.Objects.requireNonNull; + /** * Parse tree for {@code DROP MATERIALIZED VIEW} statement. */ public class SqlDropMaterializedView extends SqlDropObject { private static final SqlOperator OPERATOR = - new SqlSpecialOperator("DROP MATERIALIZED VIEW", - SqlKind.DROP_MATERIALIZED_VIEW); + new SqlSpecialOperator("DROP MATERIALIZED VIEW", SqlKind.DROP_MATERIALIZED_VIEW) { + @Override public SqlCall createCall(@Nullable SqlLiteral functionQualifier, + SqlParserPos pos, @Nullable SqlNode... operands) { + return new SqlDropMaterializedView(pos, + ((SqlLiteral) requireNonNull(operands[0], "ifExists")).booleanValue(), + (SqlIdentifier) requireNonNull(operands[1], "name")); + } + }; /** Creates a SqlDropMaterializedView. */ SqlDropMaterializedView(SqlParserPos pos, boolean ifExists, diff --git a/core/src/main/java/org/apache/calcite/sql/ddl/SqlDropObject.java b/core/src/main/java/org/apache/calcite/sql/ddl/SqlDropObject.java index 278bde6ac82d..541fe7c4aac4 100644 --- a/core/src/main/java/org/apache/calcite/sql/ddl/SqlDropObject.java +++ b/core/src/main/java/org/apache/calcite/sql/ddl/SqlDropObject.java @@ -19,6 +19,7 @@ import org.apache.calcite.jdbc.CalcitePrepare; import org.apache.calcite.sql.SqlDrop; import org.apache.calcite.sql.SqlIdentifier; +import org.apache.calcite.sql.SqlLiteral; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.SqlWriter; @@ -43,7 +44,9 @@ public abstract class SqlDropObject extends SqlDrop { } @Override public List getOperandList() { - return ImmutableList.of(name); + return ImmutableList.of( + SqlLiteral.createBoolean(ifExists, SqlParserPos.ZERO), + name); } @Override public void unparse(SqlWriter writer, int leftPrec, int rightPrec) { diff --git a/core/src/main/java/org/apache/calcite/sql/ddl/SqlDropSchema.java b/core/src/main/java/org/apache/calcite/sql/ddl/SqlDropSchema.java index b06022977b67..95689c29ef25 100644 --- a/core/src/main/java/org/apache/calcite/sql/ddl/SqlDropSchema.java +++ b/core/src/main/java/org/apache/calcite/sql/ddl/SqlDropSchema.java @@ -16,6 +16,7 @@ */ package org.apache.calcite.sql.ddl; +import org.apache.calcite.sql.SqlCall; import org.apache.calcite.sql.SqlDrop; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlKind; @@ -28,8 +29,12 @@ import com.google.common.collect.ImmutableList; +import org.checkerframework.checker.nullness.qual.Nullable; + import java.util.List; +import static java.util.Objects.requireNonNull; + /** * Parse tree for {@code DROP SCHEMA} statement. */ @@ -38,7 +43,15 @@ public class SqlDropSchema extends SqlDrop { public final SqlIdentifier name; private static final SqlOperator OPERATOR = - new SqlSpecialOperator("DROP SCHEMA", SqlKind.DROP_SCHEMA); + new SqlSpecialOperator("DROP SCHEMA", SqlKind.DROP_SCHEMA) { + @Override public SqlCall createCall(@Nullable SqlLiteral functionQualifier, + SqlParserPos pos, @Nullable SqlNode... operands) { + return new SqlDropSchema(pos, + ((SqlLiteral) requireNonNull(operands[0], "foreign")).booleanValue(), + ((SqlLiteral) requireNonNull(operands[1], "ifExists")).booleanValue(), + (SqlIdentifier) requireNonNull(operands[2], "name")); + } + }; /** Creates a SqlDropSchema. */ SqlDropSchema(SqlParserPos pos, boolean foreign, boolean ifExists, @@ -50,7 +63,9 @@ public class SqlDropSchema extends SqlDrop { @Override public List getOperandList() { return ImmutableList.of( - SqlLiteral.createBoolean(foreign, SqlParserPos.ZERO), name); + SqlLiteral.createBoolean(foreign, SqlParserPos.ZERO), + SqlLiteral.createBoolean(ifExists, SqlParserPos.ZERO), + name); } @Override public void unparse(SqlWriter writer, int leftPrec, int rightPrec) { diff --git a/core/src/main/java/org/apache/calcite/sql/ddl/SqlDropTable.java b/core/src/main/java/org/apache/calcite/sql/ddl/SqlDropTable.java index 4eab052ff07a..314686f1e310 100644 --- a/core/src/main/java/org/apache/calcite/sql/ddl/SqlDropTable.java +++ b/core/src/main/java/org/apache/calcite/sql/ddl/SqlDropTable.java @@ -16,18 +16,32 @@ */ package org.apache.calcite.sql.ddl; +import org.apache.calcite.sql.SqlCall; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlLiteral; +import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.SqlSpecialOperator; import org.apache.calcite.sql.parser.SqlParserPos; +import org.checkerframework.checker.nullness.qual.Nullable; + +import static java.util.Objects.requireNonNull; + /** * Parse tree for {@code DROP TABLE} statement. */ public class SqlDropTable extends SqlDropObject { private static final SqlOperator OPERATOR = - new SqlSpecialOperator("DROP TABLE", SqlKind.DROP_TABLE); + new SqlSpecialOperator("DROP TABLE", SqlKind.DROP_TABLE) { + @Override public SqlCall createCall(@Nullable SqlLiteral functionQualifier, + SqlParserPos pos, @Nullable SqlNode... operands) { + return new SqlDropTable(pos, + ((SqlLiteral) requireNonNull(operands[0], "ifExists")).booleanValue(), + (SqlIdentifier) requireNonNull(operands[1], "name")); + } + }; /** Creates a SqlDropTable. */ SqlDropTable(SqlParserPos pos, boolean ifExists, SqlIdentifier name) { diff --git a/core/src/main/java/org/apache/calcite/sql/ddl/SqlDropType.java b/core/src/main/java/org/apache/calcite/sql/ddl/SqlDropType.java index 17e29c287955..06ecf7427038 100644 --- a/core/src/main/java/org/apache/calcite/sql/ddl/SqlDropType.java +++ b/core/src/main/java/org/apache/calcite/sql/ddl/SqlDropType.java @@ -16,18 +16,32 @@ */ package org.apache.calcite.sql.ddl; +import org.apache.calcite.sql.SqlCall; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlLiteral; +import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.SqlSpecialOperator; import org.apache.calcite.sql.parser.SqlParserPos; +import org.checkerframework.checker.nullness.qual.Nullable; + +import static java.util.Objects.requireNonNull; + /** * Parse tree for {@code DROP TYPE} statement. */ public class SqlDropType extends SqlDropObject { private static final SqlOperator OPERATOR = - new SqlSpecialOperator("DROP TYPE", SqlKind.DROP_TYPE); + new SqlSpecialOperator("DROP TYPE", SqlKind.DROP_TYPE) { + @Override public SqlCall createCall(@Nullable SqlLiteral functionQualifier, + SqlParserPos pos, @Nullable SqlNode... operands) { + return new SqlDropType(pos, + ((SqlLiteral) requireNonNull(operands[0], "ifExists")).booleanValue(), + (SqlIdentifier) requireNonNull(operands[1], "name")); + } + }; SqlDropType(SqlParserPos pos, boolean ifExists, SqlIdentifier name) { super(OPERATOR, pos, ifExists, name); diff --git a/core/src/main/java/org/apache/calcite/sql/ddl/SqlDropView.java b/core/src/main/java/org/apache/calcite/sql/ddl/SqlDropView.java index b8de41d5ad0b..ff2d669a6810 100644 --- a/core/src/main/java/org/apache/calcite/sql/ddl/SqlDropView.java +++ b/core/src/main/java/org/apache/calcite/sql/ddl/SqlDropView.java @@ -16,18 +16,32 @@ */ package org.apache.calcite.sql.ddl; +import org.apache.calcite.sql.SqlCall; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlLiteral; +import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.SqlSpecialOperator; import org.apache.calcite.sql.parser.SqlParserPos; +import org.checkerframework.checker.nullness.qual.Nullable; + +import static java.util.Objects.requireNonNull; + /** * Parse tree for {@code DROP VIEW} statement. */ public class SqlDropView extends SqlDropObject { private static final SqlOperator OPERATOR = - new SqlSpecialOperator("DROP VIEW", SqlKind.DROP_VIEW); + new SqlSpecialOperator("DROP VIEW", SqlKind.DROP_VIEW) { + @Override public SqlCall createCall(@Nullable SqlLiteral functionQualifier, + SqlParserPos pos, @Nullable SqlNode... operands) { + return new SqlDropView(pos, + ((SqlLiteral) requireNonNull(operands[0], "ifExists")).booleanValue(), + (SqlIdentifier) requireNonNull(operands[1], "name")); + } + }; /** Creates a SqlDropView. */ SqlDropView(SqlParserPos pos, boolean ifExists, SqlIdentifier name) { diff --git a/core/src/main/java/org/apache/calcite/sql/ddl/SqlKeyConstraint.java b/core/src/main/java/org/apache/calcite/sql/ddl/SqlKeyConstraint.java index bc526e1d1516..37ad08be223e 100644 --- a/core/src/main/java/org/apache/calcite/sql/ddl/SqlKeyConstraint.java +++ b/core/src/main/java/org/apache/calcite/sql/ddl/SqlKeyConstraint.java @@ -19,6 +19,7 @@ import org.apache.calcite.sql.SqlCall; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlLiteral; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlNodeList; import org.apache.calcite.sql.SqlOperator; @@ -31,17 +32,33 @@ import java.util.List; +import static java.util.Objects.requireNonNull; + /** * Parse tree for {@code UNIQUE}, {@code PRIMARY KEY} constraints. * *

    And {@code FOREIGN KEY}, when we support it. */ public class SqlKeyConstraint extends SqlCall { - private static final SqlSpecialOperator UNIQUE = - new SqlSpecialOperator("UNIQUE", SqlKind.UNIQUE); + private static final SqlOperator UNIQUE = + new SqlSpecialOperator("UNIQUE", SqlKind.UNIQUE) { + @Override public SqlCall createCall(@Nullable SqlLiteral functionQualifier, + SqlParserPos pos, @Nullable SqlNode... operands) { + return unique(pos, + (SqlIdentifier) requireNonNull(operands[0], "name"), + (SqlNodeList) requireNonNull(operands[1], "columnList")); + } + }; - protected static final SqlSpecialOperator PRIMARY = - new SqlSpecialOperator("PRIMARY KEY", SqlKind.PRIMARY_KEY); + protected static final SqlOperator PRIMARY = + new SqlSpecialOperator("PRIMARY KEY", SqlKind.PRIMARY_KEY) { + @Override public SqlCall createCall(@Nullable SqlLiteral functionQualifier, + SqlParserPos pos, @Nullable SqlNode... operands) { + return primary(pos, + (SqlIdentifier) requireNonNull(operands[0], "name"), + (SqlNodeList) requireNonNull(operands[1], "columnList")); + } + }; private final @Nullable SqlIdentifier name; private final SqlNodeList columnList; diff --git a/core/src/main/java/org/apache/calcite/sql/ddl/SqlTruncateTable.java b/core/src/main/java/org/apache/calcite/sql/ddl/SqlTruncateTable.java index 2d82107efc61..1c633c819f72 100644 --- a/core/src/main/java/org/apache/calcite/sql/ddl/SqlTruncateTable.java +++ b/core/src/main/java/org/apache/calcite/sql/ddl/SqlTruncateTable.java @@ -16,8 +16,10 @@ */ package org.apache.calcite.sql.ddl; +import org.apache.calcite.sql.SqlCall; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlLiteral; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.SqlSpecialOperator; @@ -27,40 +29,51 @@ import com.google.common.collect.ImmutableList; +import org.checkerframework.checker.nullness.qual.Nullable; + import java.util.List; +import static java.util.Objects.requireNonNull; + /** * Parse tree for {@code TRUNCATE TABLE} statement. */ public class SqlTruncateTable extends SqlTruncate { private static final SqlOperator OPERATOR = - new SqlSpecialOperator("TRUNCATE TABLE", SqlKind.TRUNCATE_TABLE); + new SqlSpecialOperator("TRUNCATE TABLE", SqlKind.TRUNCATE_TABLE) { + @Override public SqlCall createCall(@Nullable SqlLiteral functionQualifier, + SqlParserPos pos, @Nullable SqlNode... operands) { + return new SqlTruncateTable(pos, + (SqlIdentifier) requireNonNull(operands[0], "name"), + ((SqlLiteral) requireNonNull(operands[1], "continueIdentity")).booleanValue()); + } + }; + public final SqlIdentifier name; - public final boolean continueIdentify; + public final boolean continueIdentity; /** * Creates a SqlTruncateTable. */ - public SqlTruncateTable(SqlParserPos pos, SqlIdentifier name, boolean continueIdentify) { + public SqlTruncateTable(SqlParserPos pos, SqlIdentifier name, boolean continueIdentity) { super(OPERATOR, pos); this.name = name; - this.continueIdentify = continueIdentify; + this.continueIdentity = continueIdentity; } @Override public List getOperandList() { - return ImmutableList.of(name); + return ImmutableList.of(name, SqlLiteral.createBoolean(continueIdentity, SqlParserPos.ZERO)); } @Override public void unparse(SqlWriter writer, int leftPrec, int rightPrec) { writer.keyword("TRUNCATE"); writer.keyword("TABLE"); name.unparse(writer, leftPrec, rightPrec); - if (continueIdentify) { + if (continueIdentity) { writer.keyword("CONTINUE IDENTITY"); } else { writer.keyword("RESTART IDENTITY"); - } } } diff --git a/core/src/test/java/org/apache/calcite/util/UtilTest.java b/core/src/test/java/org/apache/calcite/util/UtilTest.java index 45c964beedbb..7d8797f19caf 100644 --- a/core/src/test/java/org/apache/calcite/util/UtilTest.java +++ b/core/src/test/java/org/apache/calcite/util/UtilTest.java @@ -30,6 +30,7 @@ import org.apache.calcite.runtime.SqlFunctions; import org.apache.calcite.runtime.Utilities; import org.apache.calcite.sql.SqlCollation; +import org.apache.calcite.sql.SqlNodeList; import org.apache.calcite.sql.dialect.CalciteSqlDialect; import org.apache.calcite.sql.fun.SqlLibrary; import org.apache.calcite.sql.util.IdPair; @@ -56,6 +57,7 @@ import org.hamcrest.StringDescription; import org.hamcrest.TypeSafeMatcher; import org.junit.jupiter.api.Test; +import org.locationtech.jts.util.Assert; import java.io.PrintWriter; import java.io.Serializable; @@ -2961,6 +2963,16 @@ private void checkNameMultimap(String s, NameMultimap map) { assertThat(s2, hasToString(s.toString())); } + @Test void testCollationEncoding() { + SqlCollation collation = + new SqlCollation( + SqlCollation.Coercibility.COERCIBLE, Locale.ENGLISH, + Util.getDefaultCharset(), "primary"); + SqlNodeList list = collation.asList(); + SqlCollation decoded = SqlCollation.fromSqlList(list); + Assert.equals(collation, decoded); + } + @Test void testXmlOutput() { final StringWriter w = new StringWriter(); final XmlOutput o = new XmlOutput(w); diff --git a/server/src/main/java/org/apache/calcite/server/ServerDdlExecutor.java b/server/src/main/java/org/apache/calcite/server/ServerDdlExecutor.java index 3fb4545643ba..8bd55b91519f 100644 --- a/server/src/main/java/org/apache/calcite/server/ServerDdlExecutor.java +++ b/server/src/main/java/org/apache/calcite/server/ServerDdlExecutor.java @@ -387,7 +387,7 @@ public void execute(SqlTruncateTable truncate, RESOURCE.tableNotFound(pair.right)); } - if (!truncate.continueIdentify) { + if (!truncate.continueIdentity) { // Calcite not support RESTART IDENTIFY throw new UnsupportedOperationException("RESTART IDENTIFY is not supported"); } @@ -519,7 +519,12 @@ public void execute(SqlCreateTable create, if (d.strategy != ColumnStrategy.VIRTUAL) { storedBuilder.add(d.name.getSimple(), type); } - b.add(ColumnDef.of(d.expression, type, d.strategy)); + final ColumnStrategy strategy = d.strategy != null + ? d.strategy + : type.isNullable() + ? ColumnStrategy.NULLABLE + : ColumnStrategy.NOT_NULLABLE; + b.add(ColumnDef.of(d.expression, type, strategy)); } else if (c.e instanceof SqlIdentifier) { final SqlIdentifier id = (SqlIdentifier) c.e; if (queryRowType == null) { diff --git a/server/src/test/java/org/apache/calcite/test/ServerParserTest.java b/server/src/test/java/org/apache/calcite/test/ServerParserTest.java index aeb1c0752569..99fa4f37d664 100644 --- a/server/src/test/java/org/apache/calcite/test/ServerParserTest.java +++ b/server/src/test/java/org/apache/calcite/test/ServerParserTest.java @@ -16,12 +16,36 @@ */ package org.apache.calcite.test; +import org.apache.calcite.sql.SqlCall; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.ddl.SqlCreateFunction; +import org.apache.calcite.sql.ddl.SqlCreateSchema; +import org.apache.calcite.sql.ddl.SqlCreateTable; +import org.apache.calcite.sql.ddl.SqlCreateType; +import org.apache.calcite.sql.ddl.SqlDropFunction; +import org.apache.calcite.sql.ddl.SqlDropMaterializedView; +import org.apache.calcite.sql.ddl.SqlDropSchema; +import org.apache.calcite.sql.ddl.SqlDropTable; +import org.apache.calcite.sql.ddl.SqlDropType; +import org.apache.calcite.sql.ddl.SqlDropView; +import org.apache.calcite.sql.ddl.SqlTruncateTable; +import org.apache.calcite.sql.parser.SqlParseException; import org.apache.calcite.sql.parser.SqlParserFixture; +import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.sql.parser.SqlParserTest; import org.apache.calcite.sql.parser.ddl.SqlDdlParserImpl; +import org.apache.calcite.sql.util.SqlShuttle; +import org.checkerframework.checker.nullness.qual.Nullable; import org.junit.jupiter.api.Test; +import java.util.function.BiConsumer; +import java.util.function.Function; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +import static java.util.Objects.requireNonNull; + /** * Tests SQL parser extensions for DDL. * @@ -86,6 +110,83 @@ class ServerParserTest extends SqlParserTest { sql(sql).ok(expected); } + /** Test case for [CALCITE-7339] + * Most classes in SqlDdlNodes use an incorrect SqlCallFactory. */ + @Test void testShuttle() throws SqlParseException { + // A shuttle which modified a SqlCall's position + final SqlShuttle shuttle = new SqlShuttle() { + @Override public @Nullable SqlNode visit(SqlCall call) { + SqlNode newCall = super.visit(call); + return requireNonNull(newCall, "newCall").clone(SqlParserPos.ZERO); + } + }; + + BiConsumer> tester = + (fixture, function) -> { + SqlNode node = fixture.node(); + assertTrue(function.apply(node)); + SqlNode newNode = shuttle.visitNode(node); + assertTrue(function.apply(newNode)); + }; + + String sql = "CREATE TYPE T AS (x INT)"; + SqlParserFixture fixture = sql(sql); + fixture.ok("CREATE TYPE `T` AS (`X` INTEGER)"); + tester.accept(fixture, n -> n instanceof SqlCreateType); + + // The following also checks SqlCheckConstraint, SqlColumnDeclaration, SqlAttributeDefinition + sql = "CREATE TABLE X (I INTEGER NOT NULL, CONSTRAINT C1 CHECK (I < 10), J INTEGER)"; + fixture = sql(sql); + fixture.ok("CREATE TABLE `X` (`I` INTEGER NOT NULL, " + + "CONSTRAINT `C1` CHECK (`I` < 10), `J` INTEGER)"); + tester.accept(fixture, n -> n instanceof SqlCreateTable); + + sql = "CREATE FUNCTION F AS 'a.b'"; + fixture = sql(sql); + fixture.ok("CREATE FUNCTION `F` AS 'a.b'"); + tester.accept(fixture, n -> n instanceof SqlCreateFunction); + + sql = "CREATE SCHEMA F"; + fixture = sql(sql); + fixture.ok("CREATE SCHEMA `F`"); + tester.accept(fixture, n -> n instanceof SqlCreateSchema); + + sql = "DROP FUNCTION IF EXISTS F"; + fixture = sql(sql); + fixture.ok("DROP FUNCTION IF EXISTS `F`"); + tester.accept(fixture, n -> n instanceof SqlDropFunction); + + sql = "DROP VIEW IF EXISTS V"; + fixture = sql(sql); + fixture.ok("DROP VIEW IF EXISTS `V`"); + tester.accept(fixture, n -> n instanceof SqlDropView); + + sql = "DROP TABLE T"; + fixture = sql(sql); + fixture.ok("DROP TABLE `T`"); + tester.accept(fixture, n -> n instanceof SqlDropTable); + + sql = "DROP SCHEMA IF EXISTS S"; + fixture = sql(sql); + fixture.ok("DROP SCHEMA IF EXISTS `S`"); + tester.accept(fixture, n -> n instanceof SqlDropSchema); + + sql = "DROP TYPE IF EXISTS T"; + fixture = sql(sql); + fixture.ok("DROP TYPE IF EXISTS `T`"); + tester.accept(fixture, n -> n instanceof SqlDropType); + + sql = "DROP MATERIALIZED VIEW IF EXISTS V"; + fixture = sql(sql); + fixture.ok("DROP MATERIALIZED VIEW IF EXISTS `V`"); + tester.accept(fixture, n -> n instanceof SqlDropMaterializedView); + + sql = "TRUNCATE TABLE T CONTINUE IDENTITY"; + fixture = sql(sql); + fixture.ok("TRUNCATE TABLE `T` CONTINUE IDENTITY"); + tester.accept(fixture, n -> n instanceof SqlTruncateTable); + } + @Test void testCreateForeignSchema2() { final String sql = "create or replace foreign schema x\n" + "library 'com.example.ExampleSchemaFactory'\n" From f098a518349827592f891a9cbf71de99c74a70ad Mon Sep 17 00:00:00 2001 From: iwanttobepowerful <745778074@qq.com> Date: Thu, 25 Dec 2025 10:03:20 +0800 Subject: [PATCH 074/562] [CALCITE-7257] Subqueries cannot be decorrelated if join condition contains RexFieldAccess --- .../calcite/sql2rel/RelDecorrelator.java | 78 ++-- .../calcite/sql2rel/RelDecorrelatorTest.java | 74 ++++ core/src/test/resources/sql/sub-query.iq | 361 ++++++++++++++++++ 3 files changed, 489 insertions(+), 24 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java b/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java index ad6d12ca98f6..80f1f95335b9 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java @@ -1543,17 +1543,17 @@ private RelNode getCorRel(CorRef corVar) { /** Adds a value generator to satisfy the correlating variables used by * a relational expression, if those variables are not already provided by * its input. */ - private Frame maybeAddValueGenerator(RelNode rel, Frame frame) { - final CorelMap cm1 = new CorelMapBuilder().build(frame.r, rel); + private Frame maybeAddValueGenerator(RelNode rel, Frame inputFrame) { + final CorelMap cm1 = new CorelMapBuilder().build(inputFrame.r, rel); if (!cm1.mapRefRelToCorRef.containsKey(rel)) { - return frame; + return inputFrame; } final Collection needs = cm1.mapRefRelToCorRef.get(rel); - final ImmutableSortedSet haves = frame.corDefOutputs.keySet(); + final ImmutableSortedSet haves = inputFrame.corDefOutputs.keySet(); if (hasAll(needs, haves)) { - return frame; + return inputFrame; } - return decorrelateInputWithValueGenerator(rel, frame); + return decorrelateInputWithValueGenerator(rel, inputFrame); } /** Returns whether all of a collection of {@link CorRef}s are satisfied @@ -1579,13 +1579,13 @@ private static boolean has(Collection corDefs, CorRef corr) { return false; } - private Frame decorrelateInputWithValueGenerator(RelNode rel, Frame frame) { + private Frame decorrelateInputWithValueGenerator(RelNode rel, Frame inputFrame) { // currently only handles one input assert rel.getInputs().size() == 1; - RelNode oldInput = frame.r; + RelNode oldInput = inputFrame.r; final NavigableMap corDefOutputs = - new TreeMap<>(frame.corDefOutputs); + new TreeMap<>(inputFrame.corDefOutputs); final Collection corVarList = cm.mapRefRelToCorRef.get(rel); @@ -1606,8 +1606,7 @@ private Frame decorrelateInputWithValueGenerator(RelNode rel, Frame frame) { if (node instanceof RexInputRef) { map.put(def, ((RexInputRef) node).getIndex()); } else { - map.put(def, - frame.r.getRowType().getFieldCount() + projects.size()); + map.put(def, inputFrame.r.getRowType().getFieldCount() + projects.size()); projects.add((RexNode) node); } } @@ -1615,7 +1614,7 @@ private Frame decorrelateInputWithValueGenerator(RelNode rel, Frame frame) { // If all correlation variables are now satisfied, skip creating a value // generator. if (map.size() == corVarList.size()) { - map.putAll(frame.corDefOutputs); + map.putAll(inputFrame.corDefOutputs); final RelNode r; if (!projects.isEmpty()) { relBuilder.push(oldInput) @@ -1624,17 +1623,40 @@ private Frame decorrelateInputWithValueGenerator(RelNode rel, Frame frame) { } else { r = oldInput; } - return register(rel.getInput(0), r, - frame.oldToNewOutputs, map); + return register(rel.getInput(0), r, inputFrame.oldToNewOutputs, map); } } - int leftInputOutputCount = frame.r.getRowType().getFieldCount(); + return createFrameWithValueGenerator(rel.getInput(0), inputFrame, corVarList, corDefOutputs); + } + + /** + * Creates a new {@link Frame} for the given rel by joining its current + * decorrelated rel with a value generator that produces the required + * correlation variables. + * + *

    The value generator is built from {@code corVarList} and joined with + * {@code frame.r} using an INNER join. The provided + * {@code corDefOutputs} map is updated to reflect the positions of all + * correlation definitions in the join output, and the resulting frame is + * registered for {@code rel}. + * + * @param rel target RelNode whose frame is updated to use the join of + * {@code frame.r} and the value generator + * @param frame existing Frame of the rel + * @param corVarList correlated variables that still need to be produced + * @param corDefOutputs mapping from {@link CorDef} to output positions; updated in place + * to include positions in the new join + * @return a new Frame describing {@code rel} after attaching the value generator + */ + private Frame createFrameWithValueGenerator(RelNode rel, Frame frame, + Collection corVarList, NavigableMap corDefOutputs) { + int leftFieldCount = frame.r.getRowType().getFieldCount(); // can directly add positions into corDefOutputs since join // does not change the output ordering from the inputs. final RelNode valueGen = - createValueGenerator(corVarList, leftInputOutputCount, corDefOutputs); + createValueGenerator(corVarList, leftFieldCount, corDefOutputs); requireNonNull(valueGen, "valueGen"); RelNode join = @@ -1647,8 +1669,7 @@ private Frame decorrelateInputWithValueGenerator(RelNode rel, Frame frame) { // Join or Filter does not change the old input ordering. All // input fields from newLeftInput (i.e. the original input to the old // Filter) are in the output and in the same position. - return register(rel.getInput(0), join, frame.oldToNewOutputs, - corDefOutputs); + return register(rel, join, frame.oldToNewOutputs, corDefOutputs); } /** Finds a {@link RexInputRef} that is equivalent to a {@link CorRef}, @@ -1931,8 +1952,19 @@ private static boolean isWidening(RelDataType type, RelDataType type1) { return null; } + Frame newLeftFrame = leftFrame; + boolean joinConditionContainsFieldAccess = RexUtil.containsFieldAccess(rel.getCondition()); + if (joinConditionContainsFieldAccess && isCorVarDefined) { + final CorelMap localCorelMap = new CorelMapBuilder().build(rel); + final List corVarList = new ArrayList<>(localCorelMap.mapRefRelToCorRef.values()); + Collections.sort(corVarList); + + final NavigableMap corDefOutputs = new TreeMap<>(); + newLeftFrame = createFrameWithValueGenerator(oldLeft, leftFrame, corVarList, corDefOutputs); + } + RelNode newJoin = relBuilder - .push(leftFrame.r) + .push(newLeftFrame.r) .push(rightFrame.r) .join(rel.getJoinType(), decorrelateExpr(castNonNull(currentRel), map, cm, rel.getCondition()), @@ -1944,7 +1976,7 @@ private static boolean isWidening(RelDataType type, RelDataType type1) { Map mapOldToNewOutputs = new HashMap<>(); int oldLeftFieldCount = oldLeft.getRowType().getFieldCount(); - int newLeftFieldCount = leftFrame.r.getRowType().getFieldCount(); + int newLeftFieldCount = newLeftFrame.r.getRowType().getFieldCount(); int oldRightFieldCount = oldRight.getRowType().getFieldCount(); //noinspection AssertWithSideEffects @@ -1952,8 +1984,7 @@ private static boolean isWidening(RelDataType type, RelDataType type1) { == oldLeftFieldCount + oldRightFieldCount; // Left input positions are not changed. - mapOldToNewOutputs.putAll(leftFrame.oldToNewOutputs); - + mapOldToNewOutputs.putAll(newLeftFrame.oldToNewOutputs); // Right input positions are shifted by newLeftFieldCount. for (int i = 0; i < oldRightFieldCount; i++) { mapOldToNewOutputs.put(i + oldLeftFieldCount, @@ -1961,8 +1992,7 @@ private static boolean isWidening(RelDataType type, RelDataType type1) { } final NavigableMap corDefOutputs = - new TreeMap<>(leftFrame.corDefOutputs); - + new TreeMap<>(newLeftFrame.corDefOutputs); // Right input positions are shifted by newLeftFieldCount. for (Map.Entry entry : rightFrame.corDefOutputs.entrySet()) { diff --git a/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java b/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java index 83bf5780cc57..45ef07cc45ed 100644 --- a/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java +++ b/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java @@ -1198,4 +1198,78 @@ public static Frameworks.ConfigBuilder config() { + " LogicalTableScan(table=[[scott, EMP]])\n"; assertThat(after, hasTree(planAfter)); } + + /** Test case for [CALCITE-7257] + * Subqueries cannot be decorrelated if join condition contains RexFieldAccess. */ + @Test void testJoinConditionContainsRexFieldAccess() { + final FrameworkConfig frameworkConfig = config().build(); + final RelBuilder builder = RelBuilder.create(frameworkConfig); + final RelOptCluster cluster = builder.getCluster(); + final Planner planner = Frameworks.getPlanner(frameworkConfig); + final String sql = "" + + "SELECT E1.* \n" + + "FROM\n" + + " EMP E1\n" + + "WHERE\n" + + " E1.EMPNO = (\n" + + " SELECT D1.DEPTNO FROM DEPT D1\n" + + " WHERE E1.ENAME IN (SELECT B1.ENAME FROM BONUS B1))"; + final RelNode originalRel; + try { + final SqlNode parse = planner.parse(sql); + final SqlNode validate = planner.validate(parse); + originalRel = planner.rel(validate).rel; + } catch (Exception e) { + throw TestUtil.rethrow(e); + } + + final HepProgram hepProgram = HepProgram.builder() + .addRuleCollection( + ImmutableList.of( + // SubQuery program rules + CoreRules.FILTER_SUB_QUERY_TO_CORRELATE, + CoreRules.PROJECT_SUB_QUERY_TO_CORRELATE, + CoreRules.JOIN_SUB_QUERY_TO_CORRELATE)) + .build(); + final Program program = + Programs.of(hepProgram, true, + requireNonNull(cluster.getMetadataProvider())); + final RelNode before = + program.run(cluster.getPlanner(), originalRel, cluster.traitSet(), + Collections.emptyList(), Collections.emptyList()); + final String planBefore = "" + + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7])\n" + + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7])\n" + + " LogicalFilter(condition=[=($0, CAST($8):SMALLINT)])\n" + + " LogicalCorrelate(correlation=[$cor0], joinType=[left], requiredColumns=[{1}])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalAggregate(group=[{}], agg#0=[SINGLE_VALUE($0)])\n" + + " LogicalProject(DEPTNO=[$0])\n" + + " LogicalProject(DEPTNO=[$0], DNAME=[$1], LOC=[$2])\n" + + " LogicalJoin(condition=[=($cor0.ENAME, $3)], joinType=[inner])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n" + + " LogicalProject(ENAME=[$0])\n" + + " LogicalTableScan(table=[[scott, BONUS]])\n"; + assertThat(before, hasTree(planBefore)); + + // Decorrelate without any rules, just "purely" decorrelation algorithm on RelDecorrelator + final RelNode after = + RelDecorrelator.decorrelateQuery(before, builder, RuleSets.ofList(Collections.emptyList()), + RuleSets.ofList(Collections.emptyList())); + final String planAfter = "" + + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7])\n" + + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7], ENAME0=[$8], $f1=[CAST($9):TINYINT])\n" + + " LogicalJoin(condition=[AND(=($1, $8), =($0, CAST($9):SMALLINT))], joinType=[inner])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalAggregate(group=[{0}], agg#0=[SINGLE_VALUE($1)])\n" + + " LogicalProject(ENAME=[$3], DEPTNO=[$0])\n" + + " LogicalJoin(condition=[=($3, $4)], joinType=[inner])\n" + + " LogicalJoin(condition=[true], joinType=[inner])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n" + + " LogicalProject(ENAME=[$1])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalProject(ENAME=[$0])\n" + + " LogicalTableScan(table=[[scott, BONUS]])\n"; + assertThat(after, hasTree(planAfter)); + } } diff --git a/core/src/test/resources/sql/sub-query.iq b/core/src/test/resources/sql/sub-query.iq index f64feb0a7932..6aa385690afc 100644 --- a/core/src/test/resources/sql/sub-query.iq +++ b/core/src/test/resources/sql/sub-query.iq @@ -6798,6 +6798,367 @@ FROM t0; !ok +# [CALCITE-7257] Subqueries cannot be decorrelated if join condition contains RexFieldAccess +SELECT E1.ENAME +FROM EMP E1 +WHERE + E1.SAL > (SELECT D1.DEPTNO FROM + DEPT D1 JOIN EMP E2 + ON E1.DEPTNO = E2.DEPTNO); +more than one value in agg SINGLE_VALUE +!error + +# [CALCITE-7257] Subqueries cannot be decorrelated if join condition contains RexFieldAccess +SELECT E1.ENAME +FROM EMP E1 +WHERE + E1.SAL > (SELECT D1.DEPTNO FROM + DEPT D1 LEFT JOIN EMP E2 + ON E1.DEPTNO = E2.DEPTNO); +more than one value in agg SINGLE_VALUE +!error + +# [CALCITE-7257] Subqueries cannot be decorrelated if join condition contains RexFieldAccess +SELECT + E1.* +FROM + EMP E1 +WHERE + E1.EMPNO = ( + SELECT D1.DEPTNO FROM DEPT D1 + WHERE E1.ENAME IN (SELECT B1.ENAME FROM BONUS B1)); ++-------+-------+-----+-----+----------+-----+------+--------+ +| EMPNO | ENAME | JOB | MGR | HIREDATE | SAL | COMM | DEPTNO | ++-------+-------+-----+-----+----------+-----+------+--------+ ++-------+-------+-----+-----+----------+-----+------+--------+ +(0 rows) + +!ok + +# [CALCITE-7257] Subqueries cannot be decorrelated if join condition contains RexFieldAccess +SELECT E1.ENAME +FROM EMP E1 +WHERE EXISTS ( + SELECT 1 + FROM DEPT D + JOIN BONUS B + ON D.LOC = 'NEW YORK' OR B.JOB = E1.JOB +); ++-------+ +| ENAME | ++-------+ ++-------+ +(0 rows) + +!ok + +# [CALCITE-7257] Subqueries cannot be decorrelated if join condition contains RexFieldAccess +SELECT E1.ENAME +FROM EMP E1 +WHERE EXISTS ( + SELECT 1 + FROM DEPT D + LEFT JOIN BONUS B + ON D.LOC = 'NEW YORK' OR B.JOB = E1.JOB +); ++--------+ +| ENAME | ++--------+ +| ADAMS | +| ALLEN | +| BLAKE | +| CLARK | +| FORD | +| JAMES | +| JONES | +| KING | +| MARTIN | +| MILLER | +| SCOTT | +| SMITH | +| TURNER | +| WARD | ++--------+ +(14 rows) + +!ok + +# [CALCITE-7257] Subqueries cannot be decorrelated if join condition contains RexFieldAccess +SELECT E1.EMPNO, E1.SAL +FROM EMP E1 +WHERE E1.COMM > ( + SELECT COUNT(*) + FROM BONUS B + JOIN DEPT D + ON B.SAL > E1.SAL AND D.DEPTNO = 10 +); ++-------+---------+ +| EMPNO | SAL | ++-------+---------+ +| 7499 | 1600.00 | +| 7521 | 1250.00 | +| 7654 | 1250.00 | ++-------+---------+ +(3 rows) + +!ok + +# [CALCITE-7257] Subqueries cannot be decorrelated if join condition contains RexFieldAccess +SELECT E1.EMPNO, E1.SAL +FROM EMP E1 +WHERE E1.COMM > ( + SELECT COUNT(*) + FROM BONUS B + LEFT JOIN DEPT D + ON B.SAL > E1.SAL AND D.DEPTNO = 10 +); ++-------+---------+ +| EMPNO | SAL | ++-------+---------+ +| 7499 | 1600.00 | +| 7521 | 1250.00 | +| 7654 | 1250.00 | ++-------+---------+ +(3 rows) + +!ok + +# [CALCITE-7257] Subqueries cannot be decorrelated if join condition contains RexFieldAccess +SELECT E1.ENAME +FROM EMP E1 +WHERE EXISTS ( + SELECT 1 + FROM DEPT D + JOIN BONUS B + ON D.DNAME = B.ENAME AND B.JOB = E1.JOB + WHERE B.ENAME IS NULL +); ++-------+ +| ENAME | ++-------+ ++-------+ +(0 rows) + +!ok + +# [CALCITE-7257] Subqueries cannot be decorrelated if join condition contains RexFieldAccess +SELECT E1.ENAME +FROM EMP E1 +WHERE EXISTS ( + SELECT 1 + FROM DEPT D + LEFT JOIN BONUS B + ON D.DNAME = B.ENAME AND B.JOB = E1.JOB + WHERE B.ENAME IS NULL +); ++--------+ +| ENAME | ++--------+ +| ADAMS | +| ALLEN | +| BLAKE | +| CLARK | +| FORD | +| JAMES | +| JONES | +| KING | +| MARTIN | +| MILLER | +| SCOTT | +| SMITH | +| TURNER | +| WARD | ++--------+ +(14 rows) + +!ok + +# [CALCITE-7257] Subqueries cannot be decorrelated if join condition contains RexFieldAccess +SELECT E1.ENAME +FROM EMP E1 +WHERE EXISTS ( + SELECT 1 + FROM DEPT D + LEFT JOIN BONUS B + ON D.DNAME = B.ENAME AND B.JOB = E1.JOB +); ++--------+ +| ENAME | ++--------+ +| ADAMS | +| ALLEN | +| BLAKE | +| CLARK | +| FORD | +| JAMES | +| JONES | +| KING | +| MARTIN | +| MILLER | +| SCOTT | +| SMITH | +| TURNER | +| WARD | ++--------+ +(14 rows) + +!ok + +# [CALCITE-7257] Subqueries cannot be decorrelated if join condition contains RexFieldAccess +SELECT E1.ENAME +FROM EMP E1 +WHERE E1.SAL IN ( + SELECT B.SAL + FROM BONUS B + JOIN DEPT D + ON D.DEPTNO = E1.DEPTNO + AND B.SAL = (CASE WHEN E1.COMM IS NULL THEN 0 ELSE E1.COMM END + 100) +); ++-------+ +| ENAME | ++-------+ ++-------+ +(0 rows) + +!ok + +# [CALCITE-7257] Subqueries cannot be decorrelated if join condition contains RexFieldAccess +SELECT E1.ENAME +FROM EMP E1 +WHERE E1.SAL IN ( + SELECT B.SAL + FROM BONUS B + LEFT JOIN DEPT D + ON D.DEPTNO = E1.DEPTNO + AND B.SAL = (CASE WHEN E1.COMM IS NULL THEN 0 ELSE E1.COMM END + 100) +); ++-------+ +| ENAME | ++-------+ ++-------+ +(0 rows) + +!ok + +# [CALCITE-7257] Subqueries cannot be decorrelated if join condition contains RexFieldAccess +SELECT E1.* +FROM EMP E1 +WHERE NOT EXISTS ( + SELECT 1 + FROM EMP E2 + JOIN BONUS B + ON E2.SAL = E1.SAL AND B.JOB = E1.JOB + WHERE E2.EMPNO <> E1.EMPNO +); ++-------+--------+-----------+------+------------+---------+---------+--------+ +| EMPNO | ENAME | JOB | MGR | HIREDATE | SAL | COMM | DEPTNO | ++-------+--------+-----------+------+------------+---------+---------+--------+ +| 7369 | SMITH | CLERK | 7902 | 1980-12-17 | 800.00 | | 20 | +| 7499 | ALLEN | SALESMAN | 7698 | 1981-02-20 | 1600.00 | 300.00 | 30 | +| 7521 | WARD | SALESMAN | 7698 | 1981-02-22 | 1250.00 | 500.00 | 30 | +| 7566 | JONES | MANAGER | 7839 | 1981-02-04 | 2975.00 | | 20 | +| 7654 | MARTIN | SALESMAN | 7698 | 1981-09-28 | 1250.00 | 1400.00 | 30 | +| 7698 | BLAKE | MANAGER | 7839 | 1981-01-05 | 2850.00 | | 30 | +| 7782 | CLARK | MANAGER | 7839 | 1981-06-09 | 2450.00 | | 10 | +| 7788 | SCOTT | ANALYST | 7566 | 1987-04-19 | 3000.00 | | 20 | +| 7839 | KING | PRESIDENT | | 1981-11-17 | 5000.00 | | 10 | +| 7844 | TURNER | SALESMAN | 7698 | 1981-09-08 | 1500.00 | 0.00 | 30 | +| 7876 | ADAMS | CLERK | 7788 | 1987-05-23 | 1100.00 | | 20 | +| 7900 | JAMES | CLERK | 7698 | 1981-12-03 | 950.00 | | 30 | +| 7902 | FORD | ANALYST | 7566 | 1981-12-03 | 3000.00 | | 20 | +| 7934 | MILLER | CLERK | 7782 | 1982-01-23 | 1300.00 | | 10 | ++-------+--------+-----------+------+------------+---------+---------+--------+ +(14 rows) + +!ok + +# [CALCITE-7257] Subqueries cannot be decorrelated if join condition contains RexFieldAccess +SELECT E1.* +FROM EMP E1 +WHERE NOT EXISTS ( + SELECT 1 + FROM EMP E2 + JOIN BONUS B + ON E2.SAL = E1.SAL AND B.JOB = E1.JOB + WHERE E2.EMPNO <> E1.EMPNO +); ++-------+--------+-----------+------+------------+---------+---------+--------+ +| EMPNO | ENAME | JOB | MGR | HIREDATE | SAL | COMM | DEPTNO | ++-------+--------+-----------+------+------------+---------+---------+--------+ +| 7369 | SMITH | CLERK | 7902 | 1980-12-17 | 800.00 | | 20 | +| 7499 | ALLEN | SALESMAN | 7698 | 1981-02-20 | 1600.00 | 300.00 | 30 | +| 7521 | WARD | SALESMAN | 7698 | 1981-02-22 | 1250.00 | 500.00 | 30 | +| 7566 | JONES | MANAGER | 7839 | 1981-02-04 | 2975.00 | | 20 | +| 7654 | MARTIN | SALESMAN | 7698 | 1981-09-28 | 1250.00 | 1400.00 | 30 | +| 7698 | BLAKE | MANAGER | 7839 | 1981-01-05 | 2850.00 | | 30 | +| 7782 | CLARK | MANAGER | 7839 | 1981-06-09 | 2450.00 | | 10 | +| 7788 | SCOTT | ANALYST | 7566 | 1987-04-19 | 3000.00 | | 20 | +| 7839 | KING | PRESIDENT | | 1981-11-17 | 5000.00 | | 10 | +| 7844 | TURNER | SALESMAN | 7698 | 1981-09-08 | 1500.00 | 0.00 | 30 | +| 7876 | ADAMS | CLERK | 7788 | 1987-05-23 | 1100.00 | | 20 | +| 7900 | JAMES | CLERK | 7698 | 1981-12-03 | 950.00 | | 30 | +| 7902 | FORD | ANALYST | 7566 | 1981-12-03 | 3000.00 | | 20 | +| 7934 | MILLER | CLERK | 7782 | 1982-01-23 | 1300.00 | | 10 | ++-------+--------+-----------+------+------------+---------+---------+--------+ +(14 rows) + +!ok + +# [CALCITE-7257] Subqueries cannot be decorrelated if join condition contains RexFieldAccess +WITH t0(t0a, t0b) AS (VALUES (1, 1), (2, 0)), + t1(t1a, t1b, t1c) AS (VALUES (1, 1, 3)), + t2(t2a, t2b, t2c) AS (VALUES (1, 1, 5), (2, 2, 7)) +SELECT * FROM t0 WHERE t0a < +(SELECT sum(t1c) FROM + (SELECT t1c + FROM t1 JOIN t2 ON (t1a = t0a AND t2b = t1b)) +); ++-----+-----+ +| T0A | T0B | ++-----+-----+ +| 1 | 1 | ++-----+-----+ +(1 row) + +!ok + +# [CALCITE-7257] Subqueries cannot be decorrelated if join condition contains RexFieldAccess +WITH t0(t0a, t0b) AS (VALUES (1, 1), (2, 0)), + t1(t1a, t1b, t1c) AS (VALUES (1, 1, 3)), + t2(t2a, t2b, t2c) AS (VALUES (1, 1, 5), (2, 2, 7)) +SELECT * FROM t0 WHERE t0a < +(SELECT sum(t1c) FROM + (SELECT t1c + FROM t1 JOIN t2 ON (t1a < t0a AND t2b >= t1b)) +); ++-----+-----+ +| T0A | T0B | ++-----+-----+ +| 2 | 0 | ++-----+-----+ +(1 row) + +!ok + +# [CALCITE-7257] Subqueries cannot be decorrelated if join condition contains RexFieldAccess +WITH t0(t0a, t0b) AS (VALUES (1, 1), (2, 0)), + t1(t1a, t1b, t1c) AS (VALUES (1, 1, 3)), + t2(t2a, t2b, t2c) AS (VALUES (1, 1, 5), (2, 2, 7)) +SELECT * FROM t0 WHERE t0a < +(SELECT sum(t1c) FROM + (SELECT t1c + FROM t1 LEFT JOIN t2 ON (t1a = t0a AND t2b = t0b)) +); ++-----+-----+ +| T0A | T0B | ++-----+-----+ +| 1 | 1 | +| 2 | 0 | ++-----+-----+ +(2 rows) + +!ok + # [CALCITE-6942] Support decorrelated for sub-queries with LIMIT 1 and OFFSET select * from emp where exists ( From bc28d19fa8f72a3a39f8e5aabb2b89daacf79a10 Mon Sep 17 00:00:00 2001 From: nobigo Date: Thu, 25 Dec 2025 17:05:14 +0800 Subject: [PATCH 075/562] [CALCITE-7345] Quidem test support for Field Trimmer --- .../org/apache/calcite/tools/Programs.java | 5 ++++ core/src/test/resources/sql/join.iq | 26 +++++++++++++++++++ .../org/apache/calcite/test/QuidemTest.java | 12 +++++++++ 3 files changed, 43 insertions(+) diff --git a/core/src/main/java/org/apache/calcite/tools/Programs.java b/core/src/main/java/org/apache/calcite/tools/Programs.java index af6d62a5cb14..d473e54f4fdd 100644 --- a/core/src/main/java/org/apache/calcite/tools/Programs.java +++ b/core/src/main/java/org/apache/calcite/tools/Programs.java @@ -286,6 +286,11 @@ public static Program standard() { return standard(DefaultRelMetadataProvider.INSTANCE, true); } + /** Returns the standard program with enableFieldTrimming config. */ + public static Program standard(Boolean enableFieldTrimming) { + return standard(DefaultRelMetadataProvider.INSTANCE, enableFieldTrimming); + } + /** Returns the standard program with user metadata provider. */ public static Program standard(RelMetadataProvider metadataProvider) { return standard(metadataProvider, true); diff --git a/core/src/test/resources/sql/join.iq b/core/src/test/resources/sql/join.iq index ea04212757aa..35363424e9a2 100644 --- a/core/src/test/resources/sql/join.iq +++ b/core/src/test/resources/sql/join.iq @@ -1137,4 +1137,30 @@ SELECT * FROM (VALUES (NULLIF(5, 5)), (NULLIF(5, 5))) a, (VALUES (NULLIF(5, 5)), !ok +# [CALCITE-7345] Quidem test support for Field Trimmer +!use scott +!set trimfields false + +select distinct dept.deptno, emp.deptno +from "scott".emp join "scott".dept using (deptno); + +EnumerableAggregate(group=[{0, 10}]) + EnumerableHashJoin(condition=[=($0, $10)], joinType=[inner]) + EnumerableTableScan(table=[[scott, DEPT]]) + EnumerableTableScan(table=[[scott, EMP]]) +!plan + +!set trimfields true + +select distinct dept.deptno, emp.deptno +from "scott".emp join "scott".dept using (deptno); + +EnumerableAggregate(group=[{0, 2}]) + EnumerableHashJoin(condition=[=($0, $2)], joinType=[inner]) + EnumerableCalc(expr#0..2=[{inputs}], DEPTNO=[$t0]) + EnumerableTableScan(table=[[scott, DEPT]]) + EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], DEPTNO=[$t7]) + EnumerableTableScan(table=[[scott, EMP]]) +!plan + # End join.iq diff --git a/testkit/src/main/java/org/apache/calcite/test/QuidemTest.java b/testkit/src/main/java/org/apache/calcite/test/QuidemTest.java index d4ec9d2ba9dc..bc0b94f14f02 100644 --- a/testkit/src/main/java/org/apache/calcite/test/QuidemTest.java +++ b/testkit/src/main/java/org/apache/calcite/test/QuidemTest.java @@ -39,6 +39,7 @@ import org.apache.calcite.sql.parser.SqlParserImplFactory; import org.apache.calcite.sql.pretty.SqlPrettyWriter; import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.sql2rel.SqlToRelConverter; import org.apache.calcite.test.schemata.catchall.CatchallSchema; import org.apache.calcite.tools.Frameworks; import org.apache.calcite.tools.Planner; @@ -246,6 +247,17 @@ protected void checkRun(String path) throws Exception { int thresholdValue = ((BigDecimal) value).intValue(); closer.add(Prepare.THREAD_INSUBQUERY_THRESHOLD.push(thresholdValue)); } + if (propertyName.equals("trimfields")) { + final boolean b = value instanceof Boolean + && (Boolean) value; + closer.add( + Hook.SQL2REL_CONVERTER_CONFIG_BUILDER.addThread( + (Consumer>) configHolder -> + configHolder.set(configHolder.get().withTrimUnusedFields(b)))); + closer.add( + Hook.PROGRAM.addThread((Consumer>) + holder -> holder.set(Programs.standard(b)))); + } // Configures query planner rules via "!set planner-rules" command. // The value can be set as follows: // - Add rule: "+EnumerableRules.ENUMERABLE_INTERSECT_RULE" From bd5b88320137e5c783a3e9a757d4732d3118a365 Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Tue, 30 Dec 2025 20:39:16 +0800 Subject: [PATCH 076/562] [CALCITE-7342] Quidem test support for TopDownGeneralDecorrelator --- .../config/CalciteConnectionConfig.java | 3 + .../config/CalciteConnectionConfigImpl.java | 5 + .../config/CalciteConnectionProperty.java | 3 + .../calcite/prepare/CalcitePrepareImpl.java | 10 ++ .../apache/calcite/prepare/PlannerImpl.java | 28 +++-- .../rel/rules/FilterProjectTransposeRule.java | 5 +- .../calcite/sql2rel/SqlToRelConverter.java | 11 ++ .../org/apache/calcite/tools/Programs.java | 25 ++++- .../apache/calcite/test/CoreQuidemTest.java | 56 +++++----- .../apache/calcite/test/CoreQuidemTest2.java | 70 ++++++++++++ core/src/test/resources/sql/blank.iq | 16 +++ core/src/test/resources/sql/conditions.iq | 17 +++ core/src/test/resources/sql/hep.iq | 13 +++ core/src/test/resources/sql/planner.iq | 11 ++ .../apache/calcite/test/MockDdlExecutor.java | 2 + .../org/apache/calcite/test/QuidemTest.java | 104 ++++++++++-------- 16 files changed, 294 insertions(+), 85 deletions(-) create mode 100644 core/src/test/java/org/apache/calcite/test/CoreQuidemTest2.java diff --git a/core/src/main/java/org/apache/calcite/config/CalciteConnectionConfig.java b/core/src/main/java/org/apache/calcite/config/CalciteConnectionConfig.java index 3999a6ce9b64..52c8d4cd5663 100644 --- a/core/src/main/java/org/apache/calcite/config/CalciteConnectionConfig.java +++ b/core/src/main/java/org/apache/calcite/config/CalciteConnectionConfig.java @@ -110,6 +110,9 @@ public interface CalciteConnectionConfig extends ConnectionConfig { boolean lenientOperatorLookup(); /** Returns the value of {@link CalciteConnectionProperty#TOPDOWN_OPT}. */ boolean topDownOpt(); + /** Returns the value of + * {@link CalciteConnectionProperty#TOPDOWN_GENERAL_DECORRELATION_ENABLED}. */ + boolean topDownGeneralDecorrelationEnabled(); /** Returns the value of {@link CalciteConnectionProperty#META_TABLE_FACTORY}, * or a default meta table factory if not set. If diff --git a/core/src/main/java/org/apache/calcite/config/CalciteConnectionConfigImpl.java b/core/src/main/java/org/apache/calcite/config/CalciteConnectionConfigImpl.java index 7d9f9f76d1cc..221259cef959 100644 --- a/core/src/main/java/org/apache/calcite/config/CalciteConnectionConfigImpl.java +++ b/core/src/main/java/org/apache/calcite/config/CalciteConnectionConfigImpl.java @@ -215,6 +215,11 @@ public boolean isSet(CalciteConnectionProperty property) { .getBoolean(); } + @Override public boolean topDownGeneralDecorrelationEnabled() { + return CalciteConnectionProperty.TOPDOWN_GENERAL_DECORRELATION_ENABLED.wrap(properties) + .getBoolean(); + } + @Override public @PolyNull T metaTableFactory( Class metaTableFactoryClass, @PolyNull T defaultMetaTableFactory) { diff --git a/core/src/main/java/org/apache/calcite/config/CalciteConnectionProperty.java b/core/src/main/java/org/apache/calcite/config/CalciteConnectionProperty.java index a9c7627025cd..32079be2cfbc 100644 --- a/core/src/main/java/org/apache/calcite/config/CalciteConnectionProperty.java +++ b/core/src/main/java/org/apache/calcite/config/CalciteConnectionProperty.java @@ -154,6 +154,9 @@ public enum CalciteConnectionProperty implements ConnectionProperty { * If true (the default), Calcite de-correlates the plan. */ FORCE_DECORRELATE("forceDecorrelate", Type.BOOLEAN, true, false), + TOPDOWN_GENERAL_DECORRELATION_ENABLED("topDownGeneralDecorrelationEnabled", + Type.BOOLEAN, false, false), + /** Type system. The name of a class that implements * {@link org.apache.calcite.rel.type.RelDataTypeSystem} and has a public * default constructor or an {@code INSTANCE} constant. */ diff --git a/core/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java b/core/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java index 6bb21e1ae2af..8439fabd5f3c 100644 --- a/core/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java +++ b/core/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java @@ -107,8 +107,10 @@ import org.apache.calcite.sql2rel.SqlRexConvertletTable; import org.apache.calcite.sql2rel.SqlToRelConverter; import org.apache.calcite.sql2rel.StandardConvertletTable; +import org.apache.calcite.sql2rel.TopDownGeneralDecorrelator; import org.apache.calcite.tools.FrameworkConfig; import org.apache.calcite.tools.Frameworks; +import org.apache.calcite.tools.RelBuilder; import org.apache.calcite.util.ImmutableIntList; import org.apache.calcite.util.Pair; import org.apache.calcite.util.Util; @@ -1091,6 +1093,9 @@ private PreparedResult prepare_(Supplier fn, SqlValidator validator, CatalogReader catalogReader, SqlToRelConverter.Config config) { + config = + config.withTopDownGeneralDecorrelationEnabled( + context.config().topDownGeneralDecorrelationEnabled()); return new SqlToRelConverter(this, validator, catalogReader, cluster, convertletTable, config); } @@ -1107,6 +1112,11 @@ private PreparedResult prepare_(Supplier fn, @Override protected RelNode decorrelate(SqlToRelConverter sqlToRelConverter, SqlNode query, RelNode rootRel) { + if (context.config().topDownGeneralDecorrelationEnabled()) { + final RelBuilder relBuilder = + sqlToRelConverter.config().getRelBuilderFactory().create(rootRel.getCluster(), null); + return TopDownGeneralDecorrelator.decorrelateQuery(rootRel, relBuilder); + } return sqlToRelConverter.decorrelate(query, rootRel); } diff --git a/core/src/main/java/org/apache/calcite/prepare/PlannerImpl.java b/core/src/main/java/org/apache/calcite/prepare/PlannerImpl.java index ff2903047d03..5fece8fa9770 100644 --- a/core/src/main/java/org/apache/calcite/prepare/PlannerImpl.java +++ b/core/src/main/java/org/apache/calcite/prepare/PlannerImpl.java @@ -51,6 +51,7 @@ import org.apache.calcite.sql2rel.RelDecorrelator; import org.apache.calcite.sql2rel.SqlRexConvertletTable; import org.apache.calcite.sql2rel.SqlToRelConverter; +import org.apache.calcite.sql2rel.TopDownGeneralDecorrelator; import org.apache.calcite.tools.FrameworkConfig; import org.apache.calcite.tools.Planner; import org.apache.calcite.tools.Program; @@ -262,8 +263,10 @@ private void ready() { final RelOptCluster cluster = RelOptCluster.create(requireNonNull(planner, "planner"), rexBuilder); - final SqlToRelConverter.Config config = - sqlToRelConverterConfig.withTrimUnusedFields(false); + final SqlToRelConverter.Config config = sqlToRelConverterConfig + .withTrimUnusedFields(false) + .withTopDownGeneralDecorrelationEnabled( + connectionConfig.topDownGeneralDecorrelationEnabled()); final SqlToRelConverter sqlToRelConverter = new SqlToRelConverter(this, validator, createCatalogReader(), cluster, convertletTable, config); @@ -272,8 +275,9 @@ private void ready() { root = root.withRel(sqlToRelConverter.flattenTypes(root.rel, true)); final RelBuilder relBuilder = config.getRelBuilderFactory().create(cluster, null); - root = - root.withRel(RelDecorrelator.decorrelateQuery(root.rel, relBuilder)); + root = config.isTopDownGeneralDecorrelationEnabled() + ? root.withRel(TopDownGeneralDecorrelator.decorrelateQuery(root.rel, relBuilder)) + : root.withRel(RelDecorrelator.decorrelateQuery(root.rel, relBuilder)); state = State.STATE_5_CONVERTED; return root; } @@ -314,20 +318,22 @@ public class ViewExpanderImpl implements ViewExpander { final RexBuilder rexBuilder = createRexBuilder(); final RelOptCluster cluster = RelOptCluster.create(planner, rexBuilder); - final SqlToRelConverter.Config config = - sqlToRelConverterConfig.withTrimUnusedFields(false); + final SqlToRelConverter.Config config = sqlToRelConverterConfig + .withTrimUnusedFields(false) + .withTopDownGeneralDecorrelationEnabled( + connectionConfig.topDownGeneralDecorrelationEnabled()); final SqlToRelConverter sqlToRelConverter = new SqlToRelConverter(this, validator, catalogReader, cluster, convertletTable, config); - final RelRoot root = + RelRoot root = sqlToRelConverter.convertQuery(sqlNode, true, false); - final RelRoot root2 = - root.withRel(sqlToRelConverter.flattenTypes(root.rel, true)); + root = root.withRel(sqlToRelConverter.flattenTypes(root.rel, true)); final RelBuilder relBuilder = config.getRelBuilderFactory().create(cluster, null); - return root2.withRel( - RelDecorrelator.decorrelateQuery(root.rel, relBuilder)); + return config.isTopDownGeneralDecorrelationEnabled() + ? root.withRel(TopDownGeneralDecorrelator.decorrelateQuery(root.rel, relBuilder)) + : root.withRel(RelDecorrelator.decorrelateQuery(root.rel, relBuilder)); } // CalciteCatalogReader is stateless; no need to store one diff --git a/core/src/main/java/org/apache/calcite/rel/rules/FilterProjectTransposeRule.java b/core/src/main/java/org/apache/calcite/rel/rules/FilterProjectTransposeRule.java index 726c07bb142b..3e3cf46a1117 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/FilterProjectTransposeRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/FilterProjectTransposeRule.java @@ -175,11 +175,10 @@ protected FilterProjectTransposeRule( final RelNode input = project.getInput(); final RelTraitSet traitSet = filter.getTraitSet() .replaceIfs(RelCollationTraitDef.INSTANCE, - () -> Collections.singletonList( - input.getTraitSet().getTrait(RelCollationTraitDef.INSTANCE))) + () -> input.getTraitSet().getTraits(RelCollationTraitDef.INSTANCE)) .replaceIfs(RelDistributionTraitDef.INSTANCE, () -> Collections.singletonList( - input.getTraitSet().getTrait(RelDistributionTraitDef.INSTANCE))); + input.getTraitSet().getTrait(RelDistributionTraitDef.INSTANCE))); newCondition = RexUtil.removeNullabilityCast(relBuilder.getTypeFactory(), newCondition); newFilterRel = filter.copy(traitSet, input, newCondition); } else { diff --git a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java index da96b98b0655..dd97b2a8fdbf 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java @@ -3966,6 +3966,9 @@ protected boolean enableDecorrelation() { } protected RelNode decorrelateQuery(RelNode rootRel) { + if (config.isTopDownGeneralDecorrelationEnabled()) { + return TopDownGeneralDecorrelator.decorrelateQuery(rootRel, relBuilder); + } return RelDecorrelator.decorrelateQuery(rootRel, relBuilder); } @@ -6496,6 +6499,14 @@ public interface Config { /** Sets {@link #isDecorrelationEnabled()}. */ Config withDecorrelationEnabled(boolean decorrelationEnabled); + /** Returns whether to use the top-down general decorrelator. */ + @Value.Default default boolean isTopDownGeneralDecorrelationEnabled() { + return false; + } + + /** Sets {@link #isTopDownGeneralDecorrelationEnabled()}. */ + Config withTopDownGeneralDecorrelationEnabled(boolean topDownGeneralDecorrelationEnabled); + /** Returns the {@code trimUnusedFields} option. Controls whether to trim * unused fields as part of the conversion process. */ @Value.Default default boolean isTrimUnusedFields() { diff --git a/core/src/main/java/org/apache/calcite/tools/Programs.java b/core/src/main/java/org/apache/calcite/tools/Programs.java index d473e54f4fdd..4974db3c20b5 100644 --- a/core/src/main/java/org/apache/calcite/tools/Programs.java +++ b/core/src/main/java/org/apache/calcite/tools/Programs.java @@ -46,6 +46,7 @@ import org.apache.calcite.sql2rel.RelDecorrelator; import org.apache.calcite.sql2rel.RelFieldTrimmer; import org.apache.calcite.sql2rel.SqlToRelConverter; +import org.apache.calcite.sql2rel.TopDownGeneralDecorrelator; import org.apache.calcite.util.Util; import com.google.common.collect.ImmutableList; @@ -259,7 +260,26 @@ public static Program subQuery(RelMetadataProvider metadataProvider) { CoreRules.PROJECT_SUB_QUERY_TO_CORRELATE, CoreRules.JOIN_SUB_QUERY_TO_CORRELATE, CoreRules.PROJECT_OVER_SUM_TO_SUM0_RULE)); - return of(builder.build(), true, metadataProvider); + final Program oldProgram = of(builder.build(), true, metadataProvider); + + final HepProgramBuilder newBuilder = HepProgram.builder(); + newBuilder.addRuleCollection( + ImmutableList.of(CoreRules.FILTER_SUB_QUERY_TO_MARK_CORRELATE, + CoreRules.PROJECT_SUB_QUERY_TO_MARK_CORRELATE, + CoreRules.JOIN_SUB_QUERY_TO_CORRELATE, + CoreRules.PROJECT_OVER_SUM_TO_SUM0_RULE)); + final Program newProgram = of(newBuilder.build(), true, metadataProvider); + + return (planner, rel, requiredOutputTraits, materializations, lattices) -> { + final CalciteConnectionConfig config = + planner.getContext().maybeUnwrap(CalciteConnectionConfig.class) + .orElse(CalciteConnectionConfig.DEFAULT); + final Program program = config.topDownGeneralDecorrelationEnabled() + ? newProgram + : oldProgram; + return program.run(planner, rel, requiredOutputTraits, materializations, + lattices); + }; } public static Program measure(RelMetadataProvider metadataProvider) { @@ -430,6 +450,9 @@ private static class DecorrelateProgram implements Program { if (config.forceDecorrelate()) { final RelBuilder relBuilder = RelFactories.LOGICAL_BUILDER.create(rel.getCluster(), null); + if (config.topDownGeneralDecorrelationEnabled()) { + return TopDownGeneralDecorrelator.decorrelateQuery(rel, relBuilder); + } return RelDecorrelator.decorrelateQuery(rel, relBuilder); } return rel; diff --git a/core/src/test/java/org/apache/calcite/test/CoreQuidemTest.java b/core/src/test/java/org/apache/calcite/test/CoreQuidemTest.java index 1aa6ab187137..b878bfb085e0 100644 --- a/core/src/test/java/org/apache/calcite/test/CoreQuidemTest.java +++ b/core/src/test/java/org/apache/calcite/test/CoreQuidemTest.java @@ -41,7 +41,7 @@ /** * Test that runs every Quidem file in the "core" module as a test. */ -class CoreQuidemTest extends QuidemTest { +public class CoreQuidemTest extends QuidemTest { /** Runs a test from the command line. * *

    For example: @@ -57,6 +57,12 @@ public static void main(String[] args) throws Exception { /** For {@link QuidemTest#test(String)} parameters. */ @Override public Collection getPath() { + return data(); + } + + /** Returns the list of Quidem files to run. + * Subclasses can override this method to gradually add files. */ + protected Collection data() { // Start with a test file we know exists, then find the directory and list // its files. final String first = "sql/agg.iq"; @@ -68,31 +74,31 @@ public static void main(String[] args) throws Exception { @Override public Connection connect(String name, boolean reference) throws Exception { switch (name) { case "blank": - return CalciteAssert.that() + return customize(CalciteAssert.that() .with(CalciteConnectionProperty.PARSER_FACTORY, ExtensionDdlExecutor.class.getName() + "#PARSER_FACTORY") - .with(CalciteAssert.SchemaSpec.BLANK) + .with(CalciteAssert.SchemaSpec.BLANK)) .connect(); case "scott": - return CalciteAssert.that() + return customize(CalciteAssert.that() .with(CalciteConnectionProperty.PARSER_FACTORY, ExtensionDdlExecutor.class.getName() + "#PARSER_FACTORY") .with(CalciteConnectionProperty.FUN, SqlLibrary.CALCITE.fun) - .with(CalciteAssert.Config.SCOTT) + .with(CalciteAssert.Config.SCOTT)) .connect(); case "scott-spark": discard(CustomTypeSystems.SPARK_TYPE_SYSTEM); - return CalciteAssert.that() + return customize(CalciteAssert.that() .with(CalciteConnectionProperty.PARSER_FACTORY, ExtensionDdlExecutor.class.getName() + "#PARSER_FACTORY") .with(CalciteConnectionProperty.FUN, SqlLibrary.CALCITE.fun) .with(CalciteConnectionProperty.TYPE_SYSTEM, CustomTypeSystems.class.getName() + "#SPARK_TYPE_SYSTEM") - .with(CalciteAssert.Config.SCOTT) + .with(CalciteAssert.Config.SCOTT)) .connect(); case "scott-checked-rounding-half-up": discard(CustomTypeSystems.ROUNDING_MODE_HALF_UP); - return CalciteAssert.that() + return customize(CalciteAssert.that() .with(CalciteConnectionProperty.PARSER_FACTORY, ExtensionDdlExecutor.class.getName() + "#PARSER_FACTORY") // Use bigquery conformance, which forces checked arithmetic @@ -100,84 +106,84 @@ public static void main(String[] args) throws Exception { .with(CalciteConnectionProperty.FUN, SqlLibrary.CALCITE.fun) .with(CalciteConnectionProperty.TYPE_SYSTEM, CustomTypeSystems.class.getName() + "#ROUNDING_MODE_HALF_UP") - .with(CalciteAssert.Config.SCOTT) + .with(CalciteAssert.Config.SCOTT)) .connect(); case "scott-negative-scale": discard(CustomTypeSystems.NEGATIVE_SCALE); - return CalciteAssert.that() + return customize(CalciteAssert.that() .with(CalciteConnectionProperty.PARSER_FACTORY, ExtensionDdlExecutor.class.getName() + "#PARSER_FACTORY") .with(CalciteConnectionProperty.FUN, SqlLibrary.CALCITE.fun) .with(CalciteConnectionProperty.TYPE_SYSTEM, CustomTypeSystems.class.getName() + "#NEGATIVE_SCALE") - .with(CalciteAssert.Config.SCOTT) + .with(CalciteAssert.Config.SCOTT)) .connect(); case "scott-negative-scale-rounding-half-up": discard(CustomTypeSystems.NEGATIVE_SCALE_ROUNDING_MODE_HALF_UP); - return CalciteAssert.that() + return customize(CalciteAssert.that() .with(CalciteConnectionProperty.PARSER_FACTORY, ExtensionDdlExecutor.class.getName() + "#PARSER_FACTORY") .with(CalciteConnectionProperty.FUN, SqlLibrary.CALCITE.fun) .with(CalciteConnectionProperty.TYPE_SYSTEM, CustomTypeSystems.class.getName() + "#NEGATIVE_SCALE_ROUNDING_MODE_HALF_UP") - .with(CalciteAssert.Config.SCOTT) + .with(CalciteAssert.Config.SCOTT)) .connect(); case "scott-lenient": // Same as "scott", but uses LENIENT conformance. // TODO: add a way to change conformance without defining a new // connection - return CalciteAssert.that() + return customize(CalciteAssert.that() .with(CalciteConnectionProperty.PARSER_FACTORY, ExtensionDdlExecutor.class.getName() + "#PARSER_FACTORY") .with(CalciteConnectionProperty.CONFORMANCE, SqlConformanceEnum.LENIENT) - .with(CalciteAssert.Config.SCOTT) + .with(CalciteAssert.Config.SCOTT)) .connect(); case "scott-babel": // Same as "scott", but uses BABEL conformance. // connection - return CalciteAssert.that() + return customize(CalciteAssert.that() .with(CalciteConnectionProperty.PARSER_FACTORY, ExtensionDdlExecutor.class.getName() + "#PARSER_FACTORY") .with(CalciteConnectionProperty.CONFORMANCE, SqlConformanceEnum.BABEL) - .with(CalciteAssert.Config.SCOTT) + .with(CalciteAssert.Config.SCOTT)) .connect(); case "scott-mysql": // Same as "scott", but uses MySQL conformance. - return CalciteAssert.that() + return customize(CalciteAssert.that() .with(CalciteConnectionProperty.PARSER_FACTORY, ExtensionDdlExecutor.class.getName() + "#PARSER_FACTORY") .with(CalciteConnectionProperty.CONFORMANCE, SqlConformanceEnum.MYSQL_5) - .with(CalciteAssert.Config.SCOTT) + .with(CalciteAssert.Config.SCOTT)) .connect(); case "scott-oracle": // Same as "scott", but uses Oracle conformance. - return CalciteAssert.that() + return customize(CalciteAssert.that() .with(CalciteConnectionProperty.PARSER_FACTORY, ExtensionDdlExecutor.class.getName() + "#PARSER_FACTORY") .with(CalciteConnectionProperty.CONFORMANCE, SqlConformanceEnum.ORACLE_10) - .with(CalciteAssert.Config.SCOTT) + .with(CalciteAssert.Config.SCOTT)) .connect(); case "scott-mssql": // Same as "scott", but uses SQL_SERVER_2008 conformance. - return CalciteAssert.that() + return customize(CalciteAssert.that() .with(CalciteConnectionProperty.PARSER_FACTORY, ExtensionDdlExecutor.class.getName() + "#PARSER_FACTORY") .with(CalciteConnectionProperty.CONFORMANCE, SqlConformanceEnum.SQL_SERVER_2008) - .with(CalciteAssert.Config.SCOTT) + .with(CalciteAssert.Config.SCOTT)) .connect(); case "steelwheels": - return CalciteAssert.that() + return customize(CalciteAssert.that() .with(CalciteConnectionProperty.PARSER_FACTORY, ExtensionDdlExecutor.class.getName() + "#PARSER_FACTORY") .with(CalciteConnectionProperty.FUN, SqlLibrary.CALCITE.fun) .with(CalciteAssert.SchemaSpec.STEELWHEELS) - .with(Lex.BIG_QUERY) + .with(Lex.BIG_QUERY)) .connect(); default: return super.connect(name, reference); diff --git a/core/src/test/java/org/apache/calcite/test/CoreQuidemTest2.java b/core/src/test/java/org/apache/calcite/test/CoreQuidemTest2.java new file mode 100644 index 000000000000..406bd3bc8f34 --- /dev/null +++ b/core/src/test/java/org/apache/calcite/test/CoreQuidemTest2.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.test; + +import org.apache.calcite.config.CalciteConnectionProperty; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * Test that runs Quidem files with the top-down decorrelator enabled. + */ +public class CoreQuidemTest2 extends CoreQuidemTest { + /** Runs a test from the command line. + * + *

    For example: + * + *

    + * java CoreQuidemTest2 sql/dummy.iq + *
    */ + public static void main(String[] args) throws Exception { + for (String arg : args) { + new CoreQuidemTest2().test(arg); + } + } + + @Override protected Collection data() { + final List paths = new ArrayList<>(super.data()); + // These remove operations are temporary and will be deleted + // once the new decorrelator can adapt to all scenarios. + + // TODO: The following files involves UNNEST and LEFT_MARK JOIN + paths.remove("sql/agg.iq"); + paths.remove("sql/measure.iq"); + paths.remove("sql/unnest.iq"); + paths.remove("sql/lateral.iq"); + paths.remove("sql/some.iq"); + paths.remove("sql/sub-query.iq"); + paths.remove("sql/scalar.iq"); + paths.remove("sql/join.iq"); + paths.remove("sql/spatial.iq"); + paths.remove("sql/measure-paper.iq"); + paths.remove("sql/misc.iq"); + return paths; + } + + @Override protected CalciteAssert.AssertThat customize(CalciteAssert.AssertThat assertThat) { + return super.customize(assertThat) + .with(CalciteConnectionProperty.TOPDOWN_GENERAL_DECORRELATION_ENABLED, true); + } + + @Override protected boolean useTopDownGeneralDecorrelator() { + return true; + } +} diff --git a/core/src/test/resources/sql/blank.iq b/core/src/test/resources/sql/blank.iq index c44c39530a4d..4053cbb26b87 100644 --- a/core/src/test/resources/sql/blank.iq +++ b/core/src/test/resources/sql/blank.iq @@ -89,6 +89,7 @@ insert into table2 values (NULL, 1), (2, 1); # Checked on Oracle !set lateDecorrelate true select i, j from table1 where table1.j NOT IN (select i from table2 where table1.i=table2.j); +!if (use_old_decorr) { EnumerableCalc(expr#0..7=[{inputs}], expr#8=[0], expr#9=[=($t3, $t8)], expr#10=[IS NULL($t1)], expr#11=[IS NOT NULL($t7)], expr#12=[<($t4, $t3)], expr#13=[OR($t10, $t11, $t12)], expr#14=[IS NOT TRUE($t13)], expr#15=[OR($t9, $t14)], proj#0..1=[{exprs}], $condition=[$t15]) EnumerableMergeJoin(condition=[AND(=($0, $6), =($1, $5))], joinType=[left]) EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) @@ -113,7 +114,15 @@ EnumerableCalc(expr#0..7=[{inputs}], expr#8=[0], expr#9=[=($t3, $t8)], expr#10=[ (0 rows) !ok +!} +# TODO: This error needs to be fixed +!if (use_new_decorr) { +Unable to convert LEFT_MARK to Linq4j JoinType +!error +!} + +!if (use_old_decorr) { select * from table1 where j not in (select i from table2); +---+---+ | I | J | @@ -153,6 +162,13 @@ select * from table1 where j not in (select i from table2) or j = 3; (1 row) !ok +!} + +# TODO: This error needs to be fixed +!if (use_new_decorr) { +Unable to convert LEFT_MARK to Linq4j JoinType +!error +!} # [CALCITE-4813] ANY_VALUE assumes that arguments should be comparable select any_value(r) over(), s from(select array[f, s] r, s from (select 1 as f, 2 as s) t) t; diff --git a/core/src/test/resources/sql/conditions.iq b/core/src/test/resources/sql/conditions.iq index 5fa94d08891e..7ce0c78c9c20 100644 --- a/core/src/test/resources/sql/conditions.iq +++ b/core/src/test/resources/sql/conditions.iq @@ -418,6 +418,7 @@ where empno = 7369; !ok +!if (use_old_decorr) { EnumerableCalc(expr#0..2=[{inputs}], EMPNO=[$t0]) EnumerableNestedLoopJoin(condition=[true], joinType=[left]) EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t0):INTEGER NOT NULL], expr#9=[7369], expr#10=[=($t8, $t9)], EMPNO=[$t0], $condition=[$t10]) @@ -430,6 +431,22 @@ EnumerableCalc(expr#0..2=[{inputs}], EMPNO=[$t0]) EnumerableAggregate(group=[{}], agg#0=[COUNT()]) EnumerableTableScan(table=[[scott, EMP]]) !plan +!} + +!if (use_new_decorr) { +EnumerableCalc(expr#0..1=[{inputs}], EMPNO=[$t0]) + EnumerableNestedLoopJoin(condition=[true], joinType=[left]) + EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t0):INTEGER NOT NULL], expr#9=[7369], expr#10=[=($t8, $t9)], EMPNO=[$t0], $condition=[$t10]) + EnumerableTableScan(table=[[scott, EMP]]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[0], DUMMY=[$t2]) + EnumerableNestedLoopJoin(condition=[true], joinType=[inner]) + EnumerableCalc(expr#0..2=[{inputs}], DEPTNO=[$t0]) + EnumerableTableScan(table=[[scott, DEPT]]) + EnumerableCalc(expr#0=[{inputs}], expr#1=[0:BIGINT], expr#2=[>($t0, $t1)], $f0=[$t0], $condition=[$t2]) + EnumerableAggregate(group=[{}], agg#0=[COUNT()]) + EnumerableTableScan(table=[[scott, EMP]]) +!plan +!} # sub-query return true with Equal condition select r.empno, s.deptno diff --git a/core/src/test/resources/sql/hep.iq b/core/src/test/resources/sql/hep.iq index baa57592fa90..556d10f721af 100644 --- a/core/src/test/resources/sql/hep.iq +++ b/core/src/test/resources/sql/hep.iq @@ -141,6 +141,8 @@ WHERE e1.mgr > 12 (5 rows) !ok + +!if (use_old_decorr) { EnumerableCalc(expr#0..3=[{inputs}], MGR=[$t1], COMM=[$t2]) EnumerableHashJoin(condition=[=($1, $3)], joinType=[inner]) EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t3):INTEGER], expr#9=[12], expr#10=[>($t8, $t9)], EMPNO=[$t0], MGR=[$t3], COMM=[$t6], $condition=[$t10]) @@ -149,6 +151,17 @@ EnumerableCalc(expr#0..3=[{inputs}], MGR=[$t1], COMM=[$t2]) EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t6):DECIMAL(12, 2)], expr#9=[5.00:DECIMAL(12, 2)], expr#10=[>($t8, $t9)], expr#11=[IS NOT NULL($t3)], expr#12=[AND($t10, $t11)], MGR=[$t3], $condition=[$t12]) EnumerableTableScan(table=[[scott, EMP]]) !plan +!} + +!if (use_new_decorr) { +EnumerableCalc(expr#0..2=[{inputs}], MGR=[$t1], COMM=[$t2]) + EnumerableHashJoin(condition=[IS NOT DISTINCT FROM($1, $4)], joinType=[semi]) + EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t3):INTEGER], expr#9=[12], expr#10=[>($t8, $t9)], EMPNO=[$t0], MGR=[$t3], COMM=[$t6], $condition=[$t10]) + EnumerableTableScan(table=[[scott, EMP]]) + EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t6):DECIMAL(12, 2)], expr#9=[5.00:DECIMAL(12, 2)], expr#10=[>($t8, $t9)], expr#11=[IS NOT NULL($t3)], expr#12=[AND($t10, $t11)], EMPNO=[$t0], MGR=[$t3], COMM=[$t6], $condition=[$t12]) + EnumerableTableScan(table=[[scott, EMP]]) +!plan +!} !set hep-rules original # End hep.iq diff --git a/core/src/test/resources/sql/planner.iq b/core/src/test/resources/sql/planner.iq index 0461cc644b76..1147a534eff5 100644 --- a/core/src/test/resources/sql/planner.iq +++ b/core/src/test/resources/sql/planner.iq @@ -359,6 +359,7 @@ or !ok +!if (use_old_decorr) { EnumerableHashJoin(condition=[AND(=($0, $6), OR(AND(>($1, 11), <=($7, 32)), AND(<($5, 255), >=($8, 344))))], joinType=[inner]) EnumerableMergeJoin(condition=[AND(=($0, $3), OR(>($1, 11), <($5, 255)))], joinType=[inner]) EnumerableValues(tuples=[[{ 1, 11, 111 }, { 2, 12, 122 }, { 3, 13, 133 }, { 4, 14, 144 }, { 5, 15, 155 }]]) @@ -366,6 +367,16 @@ EnumerableHashJoin(condition=[AND(=($0, $6), OR(AND(>($1, 11), <=($7, 32)), AND( EnumerableCalc(expr#0..2=[{inputs}], expr#3=[32], expr#4=[<=($t1, $t3)], expr#5=[344], expr#6=[>=($t2, $t5)], expr#7=[OR($t4, $t6)], proj#0..2=[{exprs}], $condition=[$t7]) EnumerableValues(tuples=[[{ 1, 31, 311 }, { 2, 32, 322 }, { 3, 33, 333 }, { 4, 34, 344 }, { 5, 35, 355 }]]) !plan +!} + +!if (use_new_decorr) { +EnumerableMergeJoin(condition=[AND(=($0, $6), OR(AND(>($1, 11), <=($7, 32)), AND(<($5, 255), >=($8, 344))))], joinType=[inner]) + EnumerableMergeJoin(condition=[=($0, $3)], joinType=[inner]) + EnumerableValues(tuples=[[{ 1, 11, 111 }, { 2, 12, 122 }, { 3, 13, 133 }, { 4, 14, 144 }, { 5, 15, 155 }]]) + EnumerableValues(tuples=[[{ 1, 21, 211 }, { 2, 22, 222 }, { 3, 23, 233 }, { 4, 24, 244 }, { 5, 25, 255 }]]) + EnumerableValues(tuples=[[{ 1, 31, 311 }, { 2, 32, 322 }, { 3, 33, 333 }, { 4, 34, 344 }, { 5, 35, 355 }]]) +!plan +!} !set planner-rules original # [CALCITE-7086] Implement a rule that performs the inverse operation of AggregateCaseToFilterRule diff --git a/testkit/src/main/java/org/apache/calcite/test/MockDdlExecutor.java b/testkit/src/main/java/org/apache/calcite/test/MockDdlExecutor.java index c12618e13aec..8d744d829bee 100644 --- a/testkit/src/main/java/org/apache/calcite/test/MockDdlExecutor.java +++ b/testkit/src/main/java/org/apache/calcite/test/MockDdlExecutor.java @@ -25,6 +25,7 @@ import org.apache.calcite.linq4j.QueryProvider; import org.apache.calcite.linq4j.Queryable; import org.apache.calcite.linq4j.tree.Expression; +import org.apache.calcite.plan.Contexts; import org.apache.calcite.rel.RelRoot; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; @@ -207,6 +208,7 @@ protected static void populate(SqlIdentifier name, SqlNode query, requireNonNull( Schemas.subSchema(context.getRootSchema(), context.getDefaultSchemaPath())).plus()) + .context(Contexts.of(context.config())) .build(); final Planner planner = Frameworks.getPlanner(config); try { diff --git a/testkit/src/main/java/org/apache/calcite/test/QuidemTest.java b/testkit/src/main/java/org/apache/calcite/test/QuidemTest.java index bc0b94f14f02..67b43ff462d8 100644 --- a/testkit/src/main/java/org/apache/calcite/test/QuidemTest.java +++ b/testkit/src/main/java/org/apache/calcite/test/QuidemTest.java @@ -145,10 +145,18 @@ public static class ExplainValidatedCommand extends AbstractCommand { private static final Pattern PATTERN = Pattern.compile("\\.iq$"); // Saved original planner rules - private static @Nullable List originalRules; + private @Nullable List originalRules; - private static @Nullable Object getEnv(String varName) { + protected boolean useTopDownGeneralDecorrelator() { + return false; + } + + private @Nullable Object getEnv(String varName) { switch (varName) { + case "use_old_decorr": + return !useTopDownGeneralDecorrelator(); + case "use_new_decorr": + return useTopDownGeneralDecorrelator(); case "jdk18": return System.getProperty("java.version").startsWith("1.8"); case "fixed": @@ -200,7 +208,7 @@ protected static Collection data(String first) { final List paths = new ArrayList<>(); final FilenameFilter filter = new PatternFilenameFilter(".*\\.iq$"); for (File f : Util.first(dir.listFiles(filter), new File[0])) { - paths.add(f.getAbsolutePath().substring(commonPrefixLength)); + paths.add(n2u(f.getAbsolutePath().substring(commonPrefixLength))); } return paths; } @@ -221,7 +229,8 @@ protected void checkRun(String path) throws Exception { // outFile = "/home/fred/calcite/core/build/quidem/test/sql/agg.iq" final URL inUrl = QuidemTest.class.getResource("/" + n2u(path)); inFile = Sources.of(requireNonNull(inUrl, "inUrl")).file(); - outFile = replaceDir(inFile, "resources", "quidem"); + outFile = replaceDir(inFile, "resources", "quidem/" + + getClass().getSimpleName()); } Util.discard(outFile.getParentFile().mkdirs()); try (Reader reader = Util.reader(inFile); @@ -266,7 +275,7 @@ protected void checkRun(String path) throws Exception { // - Reset defaults: "original" if (propertyName.equals("planner-rules")) { if (value.equals("original")) { - closer.add(Hook.PLANNER.addThread(QuidemTest::resetPlanner)); + closer.add(Hook.PLANNER.addThread(this::resetPlanner)); } else { closer.add( Hook.PLANNER.addThread((Consumer) @@ -314,7 +323,7 @@ protected void checkRun(String path) throws Exception { } } }) - .withEnv(QuidemTest::getEnv) + .withEnv(this::getEnv) .build(); new Quidem(config).execute(); } @@ -337,7 +346,7 @@ private static void updatePlanner(RelOptPlanner planner, String value) { rulesAdd.forEach(planner::addRule); } - private static void resetPlanner(RelOptPlanner planner) { + private void resetPlanner(RelOptPlanner planner) { if (originalRules != null) { planner.getRules().forEach(planner::removeRule); originalRules.forEach(planner::addRule); @@ -457,6 +466,11 @@ private static File replaceDir(File file, String target, String replacement) { n2u('/' + replacement + '/'))); } + /** Allows subclasses to customize the connection. */ + protected CalciteAssert.AssertThat customize(CalciteAssert.AssertThat assertThat) { + return assertThat; + } + /** Creates a command handler. */ protected CommandHandler createCommandHandler() { return Quidem.EMPTY_COMMAND_HANDLER; @@ -501,7 +515,7 @@ public void test(String path) throws Exception { protected abstract Collection getPath(); /** Quidem connection factory for Calcite's built-in test schemas. */ - protected static class QuidemConnectionFactory + protected class QuidemConnectionFactory implements Quidem.ConnectionFactory { public Connection connect(String name) throws Exception { return connect(name, false); @@ -523,89 +537,89 @@ public Connection connect(String name) throws Exception { } switch (name) { case "hr": - return CalciteAssert.hr() + return customize(CalciteAssert.hr()) .connect(); case "aux": - return CalciteAssert.hr() - .with(CalciteAssert.Config.AUX) + return customize(CalciteAssert.hr() + .with(CalciteAssert.Config.AUX)) .connect(); case "foodmart": - return CalciteAssert.that() - .with(CalciteAssert.Config.FOODMART_CLONE) + return customize(CalciteAssert.that() + .with(CalciteAssert.Config.FOODMART_CLONE)) .connect(); case "geo": - return CalciteAssert.that() - .with(CalciteAssert.Config.GEO) + return customize(CalciteAssert.that() + .with(CalciteAssert.Config.GEO)) .connect(); case "scott": - return CalciteAssert.that() - .with(CalciteAssert.Config.SCOTT) + return customize(CalciteAssert.that() + .with(CalciteAssert.Config.SCOTT)) .connect(); case "jdbc_scott": - return CalciteAssert.that() - .with(CalciteAssert.Config.JDBC_SCOTT) + return customize(CalciteAssert.that() + .with(CalciteAssert.Config.JDBC_SCOTT)) .connect(); case "steelwheels": - return CalciteAssert.that() - .with(CalciteAssert.SchemaSpec.STEELWHEELS) + return customize(CalciteAssert.that() + .with(CalciteAssert.SchemaSpec.STEELWHEELS)) .connect(); case "jdbc_steelwheels": - return CalciteAssert.that() - .with(CalciteAssert.SchemaSpec.JDBC_STEELWHEELS) + return customize(CalciteAssert.that() + .with(CalciteAssert.SchemaSpec.JDBC_STEELWHEELS)) .connect(); case "post": - return CalciteAssert.that() + return customize(CalciteAssert.that() .with(CalciteAssert.Config.REGULAR) - .with(CalciteAssert.SchemaSpec.POST) + .with(CalciteAssert.SchemaSpec.POST)) .connect(); case "post-postgresql": - return CalciteAssert.that() + return customize(CalciteAssert.that() .with(CalciteConnectionProperty.FUN, "standard,postgresql") .with(CalciteAssert.Config.REGULAR) - .with(CalciteAssert.SchemaSpec.POST) + .with(CalciteAssert.SchemaSpec.POST)) .connect(); case "post-big-query": - return CalciteAssert.that() + return customize(CalciteAssert.that() .with(CalciteConnectionProperty.FUN, "standard,bigquery") .with(CalciteAssert.Config.REGULAR) - .with(CalciteAssert.SchemaSpec.POST) + .with(CalciteAssert.SchemaSpec.POST)) .connect(); case "mysqlfunc": - return CalciteAssert.that() + return customize(CalciteAssert.that() .with(CalciteConnectionProperty.FUN, "mysql") .with(CalciteAssert.Config.REGULAR) - .with(CalciteAssert.SchemaSpec.POST) + .with(CalciteAssert.SchemaSpec.POST)) .connect(); case "sparkfunc": - return CalciteAssert.that() + return customize(CalciteAssert.that() .with(CalciteConnectionProperty.FUN, "spark") .with(CalciteAssert.Config.REGULAR) - .with(CalciteAssert.SchemaSpec.POST) + .with(CalciteAssert.SchemaSpec.POST)) .connect(); case "oraclefunc": - return CalciteAssert.that() + return customize(CalciteAssert.that() .with(CalciteConnectionProperty.FUN, "oracle") - .with(CalciteAssert.Config.REGULAR) + .with(CalciteAssert.Config.REGULAR)) .connect(); case "mssqlfunc": - return CalciteAssert.that() + return customize(CalciteAssert.that() .with(CalciteConnectionProperty.FUN, "mssql") - .with(CalciteAssert.Config.REGULAR) + .with(CalciteAssert.Config.REGULAR)) .connect(); case "catchall": - return CalciteAssert.that() + return customize(CalciteAssert.that() .with(CalciteConnectionProperty.TIME_ZONE, "UTC") .withSchema("s", new ReflectiveSchemaWithoutRowCount( - new CatchallSchema())) + new CatchallSchema()))) .connect(); case "orinoco": - return CalciteAssert.that() - .with(CalciteAssert.SchemaSpec.ORINOCO) + return customize(CalciteAssert.that() + .with(CalciteAssert.SchemaSpec.ORINOCO)) .connect(); case "seq": - final Connection connection = CalciteAssert.that() - .withSchema("s", new AbstractSchema()) + final Connection connection = customize(CalciteAssert.that() + .withSchema("s", new AbstractSchema())) .connect(); connection.unwrap(CalciteConnection.class).getRootSchema() .subSchemas().get("s") @@ -623,8 +637,8 @@ public Connection connect(String name) throws Exception { }); return connection; case "bookstore": - return CalciteAssert.that() - .with(CalciteAssert.SchemaSpec.BOOKSTORE) + return customize(CalciteAssert.that() + .with(CalciteAssert.SchemaSpec.BOOKSTORE)) .connect(); default: throw new RuntimeException("unknown connection '" + name + "'"); From d172f090e1ee63cc9ef316064417a4fbd231ad87 Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Tue, 30 Dec 2025 20:50:56 +0800 Subject: [PATCH 077/562] [CALCITE-7346] Prevent overflow in metadata row-count when LIMIT/OFFSET literal exceeds Long range --- .../rel/metadata/RelMdMaxRowCount.java | 14 +++++----- .../rel/metadata/RelMdMinRowCount.java | 14 +++++----- .../calcite/rel/metadata/RelMdRowCount.java | 13 +++++----- .../calcite/rel/metadata/RelMdUtil.java | 26 +++++++++++++++++++ .../apache/calcite/test/RelMetadataTest.java | 11 ++++++++ 5 files changed, 57 insertions(+), 21 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdMaxRowCount.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdMaxRowCount.java index 8f666051ab45..e728c22e1ede 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdMaxRowCount.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdMaxRowCount.java @@ -41,6 +41,8 @@ import org.checkerframework.checker.nullness.qual.Nullable; +import static org.apache.calcite.rel.metadata.RelMdUtil.literalValueApproximatedByDouble; + /** * RelMdMaxRowCount supplies a default implementation of * {@link RelMetadataQuery#getMaxRowCount} for the standard logical algebra. @@ -115,11 +117,10 @@ public Double getMaxRowCount(Sort rel, RelMetadataQuery mq) { rowCount = Double.POSITIVE_INFINITY; } - final long offset = rel.offset instanceof RexLiteral ? RexLiteral.longValue(rel.offset) : 0; + final double offset = literalValueApproximatedByDouble(rel.offset, 0D); rowCount = Math.max(rowCount - offset, 0D); - final double limit = - rel.fetch instanceof RexLiteral ? RexLiteral.longValue(rel.fetch) : rowCount; + final double limit = literalValueApproximatedByDouble(rel.fetch, rowCount); return limit < rowCount ? limit : rowCount; } @@ -129,11 +130,10 @@ public Double getMaxRowCount(EnumerableLimit rel, RelMetadataQuery mq) { rowCount = Double.POSITIVE_INFINITY; } - final long offset = rel.offset instanceof RexLiteral ? RexLiteral.longValue(rel.offset) : 0; + final double offset = literalValueApproximatedByDouble(rel.offset, 0D); rowCount = Math.max(rowCount - offset, 0D); - final double limit = - rel.fetch instanceof RexLiteral ? RexLiteral.longValue(rel.fetch) : rowCount; + final double limit = literalValueApproximatedByDouble(rel.fetch, rowCount); return limit < rowCount ? limit : rowCount; } @@ -214,7 +214,7 @@ public Double getMaxRowCount(RelSubset rel, RelMetadataQuery mq) { if (node instanceof Sort) { Sort sort = (Sort) node; if (sort.fetch instanceof RexLiteral) { - return (double) RexLiteral.longValue(sort.fetch); + return literalValueApproximatedByDouble(sort.fetch, Double.POSITIVE_INFINITY); } } } diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdMinRowCount.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdMinRowCount.java index f4280ee7196d..869d34333547 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdMinRowCount.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdMinRowCount.java @@ -39,6 +39,8 @@ import org.checkerframework.checker.nullness.qual.Nullable; +import static org.apache.calcite.rel.metadata.RelMdUtil.literalValueApproximatedByDouble; + /** * RelMdMinRowCount supplies a default implementation of * {@link RelMetadataQuery#getMinRowCount} for the standard logical algebra. @@ -114,11 +116,10 @@ public Double getMinRowCount(Sort rel, RelMetadataQuery mq) { rowCount = 0D; } - final long offset = rel.offset instanceof RexLiteral ? RexLiteral.longValue(rel.offset) : 0; + final double offset = literalValueApproximatedByDouble(rel.offset, 0D); rowCount = Math.max(rowCount - offset, 0D); - final double limit = - rel.fetch instanceof RexLiteral ? RexLiteral.longValue(rel.fetch) : rowCount; + final double limit = literalValueApproximatedByDouble(rel.fetch, rowCount); return limit < rowCount ? limit : rowCount; } @@ -128,11 +129,10 @@ public Double getMinRowCount(EnumerableLimit rel, RelMetadataQuery mq) { rowCount = 0D; } - final long offset = rel.offset instanceof RexLiteral ? RexLiteral.longValue(rel.offset) : 0; + final double offset = literalValueApproximatedByDouble(rel.offset, 0D); rowCount = Math.max(rowCount - offset, 0D); - final double limit = - rel.fetch instanceof RexLiteral ? RexLiteral.longValue(rel.fetch) : rowCount; + final double limit = literalValueApproximatedByDouble(rel.fetch, rowCount); return limit < rowCount ? limit : rowCount; } @@ -174,7 +174,7 @@ public Double getMinRowCount(RelSubset rel, RelMetadataQuery mq) { if (node instanceof Sort) { Sort sort = (Sort) node; if (sort.fetch instanceof RexLiteral) { - return (double) RexLiteral.longValue(sort.fetch); + return literalValueApproximatedByDouble(sort.fetch, Double.POSITIVE_INFINITY); } } } diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdRowCount.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdRowCount.java index 637e8777cc93..f853fd647a05 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdRowCount.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdRowCount.java @@ -36,7 +36,6 @@ import org.apache.calcite.rel.core.TableScan; import org.apache.calcite.rel.core.Union; import org.apache.calcite.rel.core.Values; -import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.util.Bug; import org.apache.calcite.util.ImmutableBitSet; import org.apache.calcite.util.NumberUtil; @@ -44,6 +43,8 @@ import org.checkerframework.checker.nullness.qual.Nullable; +import static org.apache.calcite.rel.metadata.RelMdUtil.literalValueApproximatedByDouble; + /** * RelMdRowCount supplies a default implementation of * {@link RelMetadataQuery#getRowCount} for the standard logical algebra. @@ -164,11 +165,10 @@ public Double getRowCount(Calc rel, RelMetadataQuery mq) { return null; } - final long offset = rel.offset instanceof RexLiteral ? RexLiteral.longValue(rel.offset) : 0; + final double offset = literalValueApproximatedByDouble(rel.offset, 0D); rowCount = Math.max(rowCount - offset, 0D); - final double limit = - rel.fetch instanceof RexLiteral ? RexLiteral.longValue(rel.fetch) : rowCount; + final double limit = literalValueApproximatedByDouble(rel.fetch, rowCount); return limit < rowCount ? limit : rowCount; } @@ -178,11 +178,10 @@ public Double getRowCount(Calc rel, RelMetadataQuery mq) { return null; } - final long offset = rel.offset instanceof RexLiteral ? RexLiteral.longValue(rel.offset) : 0; + final double offset = literalValueApproximatedByDouble(rel.offset, 0D); rowCount = Math.max(rowCount - offset, 0D); - final double limit = - rel.fetch instanceof RexLiteral ? RexLiteral.longValue(rel.fetch) : rowCount; + final double limit = literalValueApproximatedByDouble(rel.fetch, rowCount); return limit < rowCount ? limit : rowCount; } diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdUtil.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdUtil.java index 227e19a85209..3412e70eee20 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdUtil.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdUtil.java @@ -464,6 +464,32 @@ public static double capInfinity(Double d) { return d.isInfinite() ? Double.MAX_VALUE : d; } + /** + * Returns the numeric value stored in a literal as a double. + * + *

    Throws when the literal exceeds {@link Double#MAX_VALUE} instead of + * silently rounding to infinity. Doubles still approximate large integers (53 + * bits of mantissa), so the returned value becomes only an approximation + * when numbers are very large. + */ + public static double literalValueApproximatedByDouble(@Nullable RexNode node, + double defaultValue) { + if (!(node instanceof RexLiteral)) { + return defaultValue; + } + final Number number = RexLiteral.numberValue(node); + final BigDecimal decimal = NumberUtil.toBigDecimal(number); + if (decimal == null) { + throw new IllegalArgumentException( + "literal value " + number + " cannot be converted to BigDecimal"); + } + if (decimal.abs().compareTo(BigDecimal.valueOf(Double.MAX_VALUE)) > 0) { + throw new IllegalArgumentException( + "literal value " + decimal + " exceeds double range"); + } + return decimal.doubleValue(); + } + /** * Returns default estimates for selectivities, in the absence of stats. * diff --git a/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java b/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java index e8cc0eb81a5c..acb04f176282 100644 --- a/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java @@ -1324,6 +1324,17 @@ void testColumnOriginsUnion() { fixture.assertThatRowCount(is(EMP_SIZE), is(0D), is(123456D)); } + /** Test case for + * [CALCITE-7346] + * Prevent overflow in metadata row-count when LIMIT/OFFSET literal exceeds Long range. */ + @Test void testRowCountSortLimitBeyondLong() { + final BigDecimal fetch = BigDecimal.valueOf(Long.MAX_VALUE).add(BigDecimal.ONE); + final double fetchDouble = fetch.doubleValue(); + final String sql = "select * from emp order by ename limit " + fetchDouble; + final RelMetadataFixture fixture = sql(sql); + fixture.assertThatRowCount(is(EMP_SIZE), is(0D), is(fetchDouble)); + } + @Test void testRowCountSortHighOffset() { final String sql = "select * from emp order by ename offset 123456"; final RelMetadataFixture fixture = sql(sql); From fd940b4bdbae4ee13f5742008b233ac281edb49b Mon Sep 17 00:00:00 2001 From: nobigo Date: Sun, 28 Dec 2025 08:26:55 +0800 Subject: [PATCH 078/562] [CALCITE-7274] RexFieldAccess has wrong index when use trim unused fields --- .../calcite/sql/validate/SelectScope.java | 11 ++ .../calcite/sql2rel/RelFieldTrimmer.java | 57 ++++++- .../calcite/sql2rel/SqlToRelConverter.java | 15 +- .../apache/calcite/test/RelOptRulesTest.xml | 8 +- .../calcite/test/SqlToRelConverterTest.xml | 4 +- core/src/test/resources/sql/sub-query.iq | 144 ++++++++++++++++++ 6 files changed, 230 insertions(+), 9 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SelectScope.java b/core/src/main/java/org/apache/calcite/sql/validate/SelectScope.java index e4cfce77d91e..376a7c6482f0 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SelectScope.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SelectScope.java @@ -222,4 +222,15 @@ public boolean existingWindowName(String winName) { public void setExpandedSelectList(@Nullable List selectList) { expandedSelectList = selectList; } + + @Override public boolean isWithin(SqlValidatorScope scope2) { + if (this == scope2) { + return true; + } + // go from the JOIN to the enclosing SELECT + if (scope2 instanceof JoinScope) { + return isWithin(requireNonNull(((JoinScope) scope2).getUsingScope(), "usingScope")); + } + return false; + } } diff --git a/core/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java b/core/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java index be08ee7ec265..0c9c761b333d 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java @@ -233,6 +233,58 @@ protected TrimResult trimChild( return dispatchTrimFields(input, fieldsUsedBuilder.build(), extraFields); } + /** + * Trims the fields of an input relational expression for RelNode with multiple inputs. + * + * @param rel Relational expression + * @param input Input relational expression, whose fields to trim + * @param startIndex Start index of the field range to process + * @param endIndex End index of the field range to process (exclusive) + * @param fieldsUsed Bitmap of fields needed by the consumer + * @return New relational expression and its field mapping + */ + protected TrimResult trimChild( + RelNode rel, + RelNode input, + int startIndex, + int endIndex, + final ImmutableBitSet fieldsUsed, + Set extraFields) { + final ImmutableBitSet.Builder fieldsUsedBuilder = fieldsUsed.rebuild(); + + // Fields that define the collation cannot be discarded. + final RelMetadataQuery mq = rel.getCluster().getMetadataQuery(); + final ImmutableList collations = mq.collations(input); + if (collations != null) { + for (RelCollation collation : collations) { + for (RelFieldCollation fieldCollation : collation.getFieldCollations()) { + fieldsUsedBuilder.set(fieldCollation.getFieldIndex()); + } + } + } + + // Correlating variables are a means for other relational expressions to use + // fields. + for (final CorrelationId correlation : rel.getVariablesSet()) { + rel.accept( + new CorrelationReferenceFinder() { + @Override protected RexNode handle(RexFieldAccess fieldAccess) { + final RexCorrelVariable v = + (RexCorrelVariable) fieldAccess.getReferenceExpr(); + if (v.id.equals(correlation)) { + if (fieldAccess.getField().getIndex() >= startIndex + && fieldAccess.getField().getIndex() < endIndex) { + fieldsUsedBuilder.set(fieldAccess.getField().getIndex() - startIndex); + } + } + return fieldAccess; + } + }); + } + + return dispatchTrimFields(input, fieldsUsedBuilder.build(), extraFields); + } + /** * Trims a child relational expression, then adds back a dummy project to * restore the fields that were removed. @@ -865,7 +917,8 @@ public TrimResult trimFields( : combinedInputExtraFields; inputExtraFieldCounts.add(inputExtraFields.size()); TrimResult trimResult = - trimChild(join, input, inputFieldsUsed.build(), inputExtraFields); + trimChild(join, input, offset, offset + inputFieldCount, + inputFieldsUsed.build(), inputExtraFields); newInputs.add(trimResult.left); if (trimResult.left != input) { ++changeCount; @@ -946,7 +999,7 @@ public TrimResult trimFields( requireNonNull(newMatchConditionExpr, "newMatchConditionExpr")); break; default: - relBuilder.join(join.getJoinType(), newConditionExpr); + relBuilder.join(join.getJoinType(), newConditionExpr, join.getVariablesSet()); break; } return result(relBuilder.build(), mapping, join); diff --git a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java index dd97b2a8fdbf..1ad87eaed62b 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java @@ -3103,12 +3103,25 @@ protected RelNode createJoin( p.id, requiredCols, joinType); } - final RelNode node = + RelNode node = relBuilder.push(leftRel) .push(rightRel) .join(joinType, joinCond) .build(); + final CorrelationUse correlationUseInJoin = getCorrelationUse(bb, node); + if (correlationUseInJoin != null) { + assert correlationUseInJoin.r instanceof Join; + Join joinRelTemp = (Join) correlationUseInJoin.r; + node = + LogicalJoin.create(joinRelTemp.getLeft(), + joinRelTemp.getRight(), + joinRelTemp.getHints(), + joinRelTemp.getCondition(), + ImmutableSet.of(correlationUseInJoin.id), + joinRelTemp.getJoinType()); + } + // If join conditions are pushed down, update the leaves. if (node instanceof Project) { final Join newJoin = (Join) node.getInputs().get(0); diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index 2b2deaa226b2..7075571d9550 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -8502,7 +8502,7 @@ LogicalProject(ID=[$0], ID0=[$1]) LogicalJoin(condition=[AND(=($0, $1), NOT(EXISTS({ LogicalFilter(condition=[=($0, $cor0.ID)]) LogicalValues(tuples=[[{ 3 }]]) -})))], joinType=[left]) +})))], joinType=[left], variablesSet=[[$cor0]]) LogicalValues(tuples=[[{ 1 }, { 2 }]]) LogicalValues(tuples=[[{ 2 }]]) ]]> @@ -8536,7 +8536,7 @@ LogicalProject(ID=[$0], ID0=[$1]) LogicalJoin(condition=[NOT(EXISTS({ LogicalFilter(condition=[=($0, $cor0.ID0)]) LogicalValues(tuples=[[{ 3 }]]) -}))], joinType=[left]) +}))], joinType=[left], variablesSet=[[$cor0]]) LogicalValues(tuples=[[{ 1 }]]) LogicalValues(tuples=[[{ 2 }]]) ]]> @@ -8598,7 +8598,7 @@ LogicalProject(ID=[$0], ID0=[$1]) LogicalJoin(condition=[OR(=($0, $1), EXISTS({ LogicalFilter(condition=[=($0, $cor0.ID0)]) LogicalValues(tuples=[[{ 3 }]]) -}))], joinType=[left]) +}))], joinType=[left], variablesSet=[[$cor0]]) LogicalValues(tuples=[[{ 1 }]]) LogicalValues(tuples=[[{ 2 }]]) ]]> @@ -8632,7 +8632,7 @@ LogicalProject(ID=[$0], ID0=[$1]) LogicalJoin(condition=[OR(=($0, $1), NOT(EXISTS({ LogicalFilter(condition=[=($0, $cor0.ID0)]) LogicalValues(tuples=[[{ 3 }]]) -})))], joinType=[left]) +})))], joinType=[left], variablesSet=[[$cor0]]) LogicalValues(tuples=[[{ 1 }]]) LogicalValues(tuples=[[{ 2 }]]) ]]> diff --git a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml index cbb228b0ed10..0c8ce2192814 100644 --- a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml @@ -3765,7 +3765,7 @@ LogicalAggregate(group=[{}], EXPR$0=[AVG($0)]) LogicalProject(SAL=[$5]) LogicalFilter(condition=[=($7, $cor0.DEPTNO)]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) -})))], joinType=[inner]) +})))], joinType=[inner], variablesSet=[[$cor0]]) LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) ]]> @@ -3986,7 +3986,7 @@ LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$ LogicalJoin(condition=[OR(=($0, 1), EXISTS({ LogicalFilter(condition=[>($0, +($cor0.DEPTNO0, 5))]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) -}))], joinType=[left]) +}))], joinType=[left], variablesSet=[[$cor0]]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) ]]> diff --git a/core/src/test/resources/sql/sub-query.iq b/core/src/test/resources/sql/sub-query.iq index 6aa385690afc..971e8c0065fd 100644 --- a/core/src/test/resources/sql/sub-query.iq +++ b/core/src/test/resources/sql/sub-query.iq @@ -7505,4 +7505,148 @@ SELECT deptno FROM dept WHERE 1000.00 > !ok +# [CALCITE-7274] RexFieldAccess has wrong index when use trim unused fields +!set trimfields true + +SELECT empno + FROM emp AS e + LEFT JOIN dept AS d + ON d.deptno = e.deptno + AND (EXISTS ( + SELECT e2.deptno FROM emp AS e2 + WHERE e2.deptno = d.deptno + GROUP BY e2.deptno + HAVING SUM(e2.sal) > 1000000)); +EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0]) + EnumerableTableScan(table=[[scott, EMP]]) +!plan ++-------+ +| EMPNO | ++-------+ +| 7369 | +| 7499 | +| 7521 | +| 7566 | +| 7654 | +| 7698 | +| 7782 | +| 7788 | +| 7839 | +| 7844 | +| 7876 | +| 7900 | +| 7902 | +| 7934 | ++-------+ +(14 rows) + +!ok + +SELECT empno + FROM emp AS e + LEFT JOIN dept AS d + ON d.dname = e.ename + AND (EXISTS ( + SELECT e2.deptno FROM emp AS e2 + WHERE e2.deptno = e.deptno + GROUP BY e2.deptno + HAVING SUM(e2.sal) > 1000000)); + +EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0]) + EnumerableTableScan(table=[[scott, EMP]]) +!plan ++-------+ +| EMPNO | ++-------+ +| 7369 | +| 7499 | +| 7521 | +| 7566 | +| 7654 | +| 7698 | +| 7782 | +| 7788 | +| 7839 | +| 7844 | +| 7876 | +| 7900 | +| 7902 | +| 7934 | ++-------+ +(14 rows) + +!ok + +# Same as previous; but don't trim fields +!set trimfields false + +SELECT empno + FROM emp AS e + LEFT JOIN dept AS d + ON d.deptno = e.deptno + AND (EXISTS ( + SELECT e2.deptno FROM emp AS e2 + WHERE e2.deptno = d.deptno + GROUP BY e2.deptno + HAVING SUM(e2.sal) > 1000000)); +EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0]) + EnumerableTableScan(table=[[scott, EMP]]) +!plan ++-------+ +| EMPNO | ++-------+ +| 7369 | +| 7499 | +| 7521 | +| 7566 | +| 7654 | +| 7698 | +| 7782 | +| 7788 | +| 7839 | +| 7844 | +| 7876 | +| 7900 | +| 7902 | +| 7934 | ++-------+ +(14 rows) + +!ok + +SELECT empno + FROM emp AS e + LEFT JOIN dept AS d + ON d.dname = e.ename + AND (EXISTS ( + SELECT e2.deptno FROM emp AS e2 + WHERE e2.deptno = e.deptno + GROUP BY e2.deptno + HAVING SUM(e2.sal) > 1000000)); + +EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0]) + EnumerableTableScan(table=[[scott, EMP]]) +!plan ++-------+ +| EMPNO | ++-------+ +| 7369 | +| 7499 | +| 7521 | +| 7566 | +| 7654 | +| 7698 | +| 7782 | +| 7788 | +| 7839 | +| 7844 | +| 7876 | +| 7900 | +| 7902 | +| 7934 | ++-------+ +(14 rows) + +!ok + # End sub-query.iq From 02521843abf09c10bcb2ea90aeba43f22aafd977 Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Wed, 31 Dec 2025 18:28:50 +0800 Subject: [PATCH 079/562] Move ExpandDisjunctionForJoinInputsRule test from planner.iq to hep.iq --- core/src/test/resources/sql/hep.iq | 74 ++++++++++++++++++++++++++ core/src/test/resources/sql/planner.iq | 69 ------------------------ 2 files changed, 74 insertions(+), 69 deletions(-) diff --git a/core/src/test/resources/sql/hep.iq b/core/src/test/resources/sql/hep.iq index 556d10f721af..8dd530234735 100644 --- a/core/src/test/resources/sql/hep.iq +++ b/core/src/test/resources/sql/hep.iq @@ -164,4 +164,78 @@ EnumerableCalc(expr#0..2=[{inputs}], MGR=[$t1], COMM=[$t2]) !} !set hep-rules original +# Test predicate push down with/without expand disjunction. +with t1 (id1, col11, col12) as (values (1, 11, 111), (2, 12, 122), (3, 13, 133), (4, 14, 144), (5, 15, 155)), +t2 (id2, col21, col22) as (values (1, 21, 211), (2, 22, 222), (3, 23, 233), (4, 24, 244), (5, 25, 255)), +t3 (id3, col31, col32) as (values (1, 31, 311), (2, 32, 322), (3, 33, 333), (4, 34, 344), (5, 35, 355)) +select * from t1, t2, t3 where id1 = id2 and id1 = id3 and +( +(col11 > 11 and col31 <= 32) +or +(col22 < 255 and col32 >= 344) +); ++-----+-------+-------+-----+-------+-------+-----+-------+-------+ +| ID1 | COL11 | COL12 | ID2 | COL21 | COL22 | ID3 | COL31 | COL32 | ++-----+-------+-------+-----+-------+-------+-----+-------+-------+ +| 2 | 12 | 122 | 2 | 22 | 222 | 2 | 32 | 322 | +| 4 | 14 | 144 | 4 | 24 | 244 | 4 | 34 | 344 | ++-----+-------+-------+-----+-------+-------+-----+-------+-------+ +(2 rows) + +!ok + +EnumerableMergeJoin(condition=[AND(=($0, $6), OR(AND(>($1, 11), <=($7, 32)), AND(<($5, 255), >=($8, 344))))], joinType=[inner]) + EnumerableMergeJoin(condition=[=($0, $3)], joinType=[inner]) + EnumerableValues(tuples=[[{ 1, 11, 111 }, { 2, 12, 122 }, { 3, 13, 133 }, { 4, 14, 144 }, { 5, 15, 155 }]]) + EnumerableValues(tuples=[[{ 1, 21, 211 }, { 2, 22, 222 }, { 3, 23, 233 }, { 4, 24, 244 }, { 5, 25, 255 }]]) + EnumerableValues(tuples=[[{ 1, 31, 311 }, { 2, 32, 322 }, { 3, 33, 333 }, { 4, 34, 344 }, { 5, 35, 355 }]]) +!plan + +!set hep-rules " ++CoreRules.EXPAND_FILTER_DISJUNCTION_LOCAL, ++CoreRules.EXPAND_JOIN_DISJUNCTION_LOCAL, ++CoreRules.JOIN_CONDITION_PUSH, ++CoreRules.FILTER_INTO_JOIN" + +with t1 (id1, col11, col12) as (values (1, 11, 111), (2, 12, 122), (3, 13, 133), (4, 14, 144), (5, 15, 155)), +t2 (id2, col21, col22) as (values (1, 21, 211), (2, 22, 222), (3, 23, 233), (4, 24, 244), (5, 25, 255)), +t3 (id3, col31, col32) as (values (1, 31, 311), (2, 32, 322), (3, 33, 333), (4, 34, 344), (5, 35, 355)) +select * from t1, t2, t3 where id1 = id2 and id1 = id3 and +( +(col11 > 11 and col31 <= 32) +or +(col22 < 255 and col32 >= 344) +); ++-----+-------+-------+-----+-------+-------+-----+-------+-------+ +| ID1 | COL11 | COL12 | ID2 | COL21 | COL22 | ID3 | COL31 | COL32 | ++-----+-------+-------+-----+-------+-------+-----+-------+-------+ +| 2 | 12 | 122 | 2 | 22 | 222 | 2 | 32 | 322 | +| 4 | 14 | 144 | 4 | 24 | 244 | 4 | 34 | 344 | ++-----+-------+-------+-----+-------+-------+-----+-------+-------+ +(2 rows) + +!ok + +!if (use_old_decorr) { +EnumerableCalc(expr#0..8=[{inputs}], proj#0..8=[{exprs}]) + EnumerableHashJoin(condition=[AND(=($0, $6), OR(AND(>($1, 11), <=($7, 32)), AND(<($5, 255), >=($8, 344))))], joinType=[inner]) + EnumerableMergeJoin(condition=[AND(=($0, $3), OR(>($1, 11), <($5, 255)))], joinType=[inner]) + EnumerableValues(tuples=[[{ 1, 11, 111 }, { 2, 12, 122 }, { 3, 13, 133 }, { 4, 14, 144 }, { 5, 15, 155 }]]) + EnumerableValues(tuples=[[{ 1, 21, 211 }, { 2, 22, 222 }, { 3, 23, 233 }, { 4, 24, 244 }, { 5, 25, 255 }]]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[32], expr#4=[<=($t1, $t3)], expr#5=[344], expr#6=[>=($t2, $t5)], expr#7=[OR($t4, $t6)], proj#0..2=[{exprs}], $condition=[$t7]) + EnumerableValues(tuples=[[{ 1, 31, 311 }, { 2, 32, 322 }, { 3, 33, 333 }, { 4, 34, 344 }, { 5, 35, 355 }]]) +!plan +!} + +!if (use_new_decorr) { +EnumerableHashJoin(condition=[AND(=($0, $6), OR(AND(>($1, 11), <=($7, 32)), AND(<($5, 255), >=($8, 344))))], joinType=[inner]) + EnumerableMergeJoin(condition=[AND(=($0, $3), OR(>($1, 11), <($5, 255)))], joinType=[inner]) + EnumerableValues(tuples=[[{ 1, 11, 111 }, { 2, 12, 122 }, { 3, 13, 133 }, { 4, 14, 144 }, { 5, 15, 155 }]]) + EnumerableValues(tuples=[[{ 1, 21, 211 }, { 2, 22, 222 }, { 3, 23, 233 }, { 4, 24, 244 }, { 5, 25, 255 }]]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[32], expr#4=[<=($t1, $t3)], expr#5=[344], expr#6=[>=($t2, $t5)], expr#7=[OR($t4, $t6)], proj#0..2=[{exprs}], $condition=[$t7]) + EnumerableValues(tuples=[[{ 1, 31, 311 }, { 2, 32, 322 }, { 3, 33, 333 }, { 4, 34, 344 }, { 5, 35, 355 }]]) +!plan +!} +!set hep-rules original + # End hep.iq diff --git a/core/src/test/resources/sql/planner.iq b/core/src/test/resources/sql/planner.iq index 1147a534eff5..1c90347a216f 100644 --- a/core/src/test/resources/sql/planner.iq +++ b/core/src/test/resources/sql/planner.iq @@ -310,75 +310,6 @@ EnumerableMinus(all=[false]) EnumerableValues(tuples=[[{ 1.0 }, { 4.0 }, { null }]]) !plan -# Test predicate push down with/without expand disjunction. -with t1 (id1, col11, col12) as (values (1, 11, 111), (2, 12, 122), (3, 13, 133), (4, 14, 144), (5, 15, 155)), -t2 (id2, col21, col22) as (values (1, 21, 211), (2, 22, 222), (3, 23, 233), (4, 24, 244), (5, 25, 255)), -t3 (id3, col31, col32) as (values (1, 31, 311), (2, 32, 322), (3, 33, 333), (4, 34, 344), (5, 35, 355)) -select * from t1, t2, t3 where id1 = id2 and id1 = id3 and -( -(col11 > 11 and col31 <= 32) -or -(col22 < 255 and col32 >= 344) -); -+-----+-------+-------+-----+-------+-------+-----+-------+-------+ -| ID1 | COL11 | COL12 | ID2 | COL21 | COL22 | ID3 | COL31 | COL32 | -+-----+-------+-------+-----+-------+-------+-----+-------+-------+ -| 2 | 12 | 122 | 2 | 22 | 222 | 2 | 32 | 322 | -| 4 | 14 | 144 | 4 | 24 | 244 | 4 | 34 | 344 | -+-----+-------+-------+-----+-------+-------+-----+-------+-------+ -(2 rows) - -!ok - -EnumerableMergeJoin(condition=[AND(=($0, $6), OR(AND(>($1, 11), <=($7, 32)), AND(<($5, 255), >=($8, 344))))], joinType=[inner]) - EnumerableMergeJoin(condition=[=($0, $3)], joinType=[inner]) - EnumerableValues(tuples=[[{ 1, 11, 111 }, { 2, 12, 122 }, { 3, 13, 133 }, { 4, 14, 144 }, { 5, 15, 155 }]]) - EnumerableValues(tuples=[[{ 1, 21, 211 }, { 2, 22, 222 }, { 3, 23, 233 }, { 4, 24, 244 }, { 5, 25, 255 }]]) - EnumerableValues(tuples=[[{ 1, 31, 311 }, { 2, 32, 322 }, { 3, 33, 333 }, { 4, 34, 344 }, { 5, 35, 355 }]]) -!plan - -!set planner-rules " -+CoreRules.EXPAND_FILTER_DISJUNCTION_LOCAL" - -with t1 (id1, col11, col12) as (values (1, 11, 111), (2, 12, 122), (3, 13, 133), (4, 14, 144), (5, 15, 155)), -t2 (id2, col21, col22) as (values (1, 21, 211), (2, 22, 222), (3, 23, 233), (4, 24, 244), (5, 25, 255)), -t3 (id3, col31, col32) as (values (1, 31, 311), (2, 32, 322), (3, 33, 333), (4, 34, 344), (5, 35, 355)) -select * from t1, t2, t3 where id1 = id2 and id1 = id3 and -( -(col11 > 11 and col31 <= 32) -or -(col22 < 255 and col32 >= 344) -); -+-----+-------+-------+-----+-------+-------+-----+-------+-------+ -| ID1 | COL11 | COL12 | ID2 | COL21 | COL22 | ID3 | COL31 | COL32 | -+-----+-------+-------+-----+-------+-------+-----+-------+-------+ -| 2 | 12 | 122 | 2 | 22 | 222 | 2 | 32 | 322 | -| 4 | 14 | 144 | 4 | 24 | 244 | 4 | 34 | 344 | -+-----+-------+-------+-----+-------+-------+-----+-------+-------+ -(2 rows) - -!ok - -!if (use_old_decorr) { -EnumerableHashJoin(condition=[AND(=($0, $6), OR(AND(>($1, 11), <=($7, 32)), AND(<($5, 255), >=($8, 344))))], joinType=[inner]) - EnumerableMergeJoin(condition=[AND(=($0, $3), OR(>($1, 11), <($5, 255)))], joinType=[inner]) - EnumerableValues(tuples=[[{ 1, 11, 111 }, { 2, 12, 122 }, { 3, 13, 133 }, { 4, 14, 144 }, { 5, 15, 155 }]]) - EnumerableValues(tuples=[[{ 1, 21, 211 }, { 2, 22, 222 }, { 3, 23, 233 }, { 4, 24, 244 }, { 5, 25, 255 }]]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[32], expr#4=[<=($t1, $t3)], expr#5=[344], expr#6=[>=($t2, $t5)], expr#7=[OR($t4, $t6)], proj#0..2=[{exprs}], $condition=[$t7]) - EnumerableValues(tuples=[[{ 1, 31, 311 }, { 2, 32, 322 }, { 3, 33, 333 }, { 4, 34, 344 }, { 5, 35, 355 }]]) -!plan -!} - -!if (use_new_decorr) { -EnumerableMergeJoin(condition=[AND(=($0, $6), OR(AND(>($1, 11), <=($7, 32)), AND(<($5, 255), >=($8, 344))))], joinType=[inner]) - EnumerableMergeJoin(condition=[=($0, $3)], joinType=[inner]) - EnumerableValues(tuples=[[{ 1, 11, 111 }, { 2, 12, 122 }, { 3, 13, 133 }, { 4, 14, 144 }, { 5, 15, 155 }]]) - EnumerableValues(tuples=[[{ 1, 21, 211 }, { 2, 22, 222 }, { 3, 23, 233 }, { 4, 24, 244 }, { 5, 25, 255 }]]) - EnumerableValues(tuples=[[{ 1, 31, 311 }, { 2, 32, 322 }, { 3, 33, 333 }, { 4, 34, 344 }, { 5, 35, 355 }]]) -!plan -!} -!set planner-rules original - # [CALCITE-7086] Implement a rule that performs the inverse operation of AggregateCaseToFilterRule # Refer to RelToSqlConverterTest.testAggregateFilterToCase(). The following two SQL # represent the true filtered Aggregate and the case-style Aggregate converted by AggregateFilterToCaseRule. From ffff03ab164e92702e840e9b862538f07b7ba38a Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Wed, 31 Dec 2025 22:19:56 +0800 Subject: [PATCH 080/562] [CALCITE-7349] Upgrade the types of FETCH and OFFSET in SORT to BigDecimal --- .../adapter/enumerable/EnumerableMergeUnionRule.java | 4 ++-- .../org/apache/calcite/rel/metadata/RelMdUtil.java | 11 ++++++++--- .../org/apache/calcite/rel/rules/PruneEmptyRules.java | 4 ++-- .../calcite/rel/rules/SortJoinTransposeRule.java | 10 ++++++---- .../calcite/rel/rules/SortRemoveRedundantRule.java | 5 ++--- .../main/java/org/apache/calcite/rex/RexLiteral.java | 6 ++++++ .../java/org/apache/calcite/tools/RelBuilder.java | 4 ++-- 7 files changed, 28 insertions(+), 16 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeUnionRule.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeUnionRule.java index bfe0555b3fff..7d47e639b78e 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeUnionRule.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeUnionRule.java @@ -92,8 +92,8 @@ public EnumerableMergeUnionRule(Config config) { inputFetch = sort.fetch; } else if (sort.fetch instanceof RexLiteral && sort.offset instanceof RexLiteral) { inputFetch = - call.builder().literal(RexLiteral.longValue(sort.fetch) - + RexLiteral.longValue(sort.offset)); + call.builder().literal(RexLiteral.bigDecimalValue(sort.fetch) + .add(RexLiteral.bigDecimalValue(sort.offset))); } } diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdUtil.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdUtil.java index 3412e70eee20..1f6502243626 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdUtil.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdUtil.java @@ -1048,9 +1048,14 @@ private static boolean alreadySmaller(RelMetadataQuery mq, RelNode input, // Cannot be determined return false; } - final long offsetVal = offset == null ? 0 : RexLiteral.longValue(offset); - final long limit = RexLiteral.longValue(fetch); - return (double) offsetVal + (double) limit >= rowCount; + final BigDecimal offsetVal = offset == null + ? BigDecimal.ZERO + : RexLiteral.bigDecimalValue(offset); + final BigDecimal limit = RexLiteral.bigDecimalValue(fetch); + if (!Double.isFinite(rowCount)) { + return false; + } + return offsetVal.add(limit).compareTo(BigDecimal.valueOf(rowCount)) >= 0; } /** diff --git a/core/src/main/java/org/apache/calcite/rel/rules/PruneEmptyRules.java b/core/src/main/java/org/apache/calcite/rel/rules/PruneEmptyRules.java index 2001c8112883..221cfac09df0 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/PruneEmptyRules.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/PruneEmptyRules.java @@ -47,6 +47,7 @@ import org.immutables.value.Value; +import java.math.BigDecimal; import java.util.Collections; import java.util.List; import java.util.function.Predicate; @@ -500,9 +501,8 @@ public interface SortFetchZeroRuleConfig extends PruneEmptyRule.Config { Sort sort = call.rel(0); return sort.fetch != null && !(sort.fetch instanceof RexDynamicParam) - && RexLiteral.longValue(sort.fetch) == 0; + && RexLiteral.bigDecimalValue(sort.fetch).equals(BigDecimal.ZERO); } - }; } } diff --git a/core/src/main/java/org/apache/calcite/rel/rules/SortJoinTransposeRule.java b/core/src/main/java/org/apache/calcite/rel/rules/SortJoinTransposeRule.java index ec29fb92a439..4310d6d65576 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/SortJoinTransposeRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/SortJoinTransposeRule.java @@ -202,10 +202,12 @@ public SortJoinTransposeRule(Class sortClass, if (sort.fetch == null) { return null; } - final long outerFetch = RexLiteral.longValue(sort.fetch); - final long outerOffset = sort.offset != null ? RexLiteral.longValue(sort.offset) : 0; - final long totalFetch = outerOffset + outerFetch; - return rexBuilder.makeExactLiteral(BigDecimal.valueOf(totalFetch)); + final BigDecimal outerFetch = RexLiteral.bigDecimalValue(sort.fetch); + final BigDecimal outerOffset = sort.offset != null + ? RexLiteral.bigDecimalValue(sort.offset) + : BigDecimal.ZERO; + final BigDecimal totalFetch = outerOffset.add(outerFetch); + return rexBuilder.makeExactLiteral(totalFetch); } /** Rule configuration. */ diff --git a/core/src/main/java/org/apache/calcite/rel/rules/SortRemoveRedundantRule.java b/core/src/main/java/org/apache/calcite/rel/rules/SortRemoveRedundantRule.java index 9c950df5cb2a..9bcf026fc656 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/SortRemoveRedundantRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/SortRemoveRedundantRule.java @@ -125,7 +125,7 @@ protected SortRemoveRedundantRule(final SortRemoveRedundantRule.Config config) { // then we could remove the redundant sort. if (inputMaxRowCount != null && Double.isFinite(inputMaxRowCount) - && new BigDecimal(inputMaxRowCount).compareTo(rowCountThreshold.get()) <= 0) { + && BigDecimal.valueOf(inputMaxRowCount).compareTo(rowCountThreshold.get()) <= 0) { call.transformTo(sort.getInput()); } } @@ -133,10 +133,9 @@ && new BigDecimal(inputMaxRowCount).compareTo(rowCountThreshold.get()) <= 0) { private static Optional getRowCountThreshold(Sort sort) { if (RelOptUtil.isLimit(sort)) { assert sort.fetch != null; - final BigDecimal fetch = ((RexLiteral) sort.fetch).getValueAs(BigDecimal.class); + final BigDecimal fetch = RexLiteral.bigDecimalValue(sort.fetch); // We don't need to deal with fetch is 0. - assert fetch != null; if (fetch.equals(BigDecimal.ZERO)) { return Optional.empty(); } diff --git a/core/src/main/java/org/apache/calcite/rex/RexLiteral.java b/core/src/main/java/org/apache/calcite/rex/RexLiteral.java index ce7ac68abbf4..2dc824920889 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexLiteral.java +++ b/core/src/main/java/org/apache/calcite/rex/RexLiteral.java @@ -1290,6 +1290,12 @@ public static long longValue(RexNode node) { return number.longValue(); } + /** Returns the value of a literal, cast, or unary minus, as a BigDecimal; + * never null. */ + public static BigDecimal bigDecimalValue(RexNode node) { + return (BigDecimal) numberValue(node); + } + public static @Nullable String stringValue(RexNode node) { final Comparable value = findValue(node); return (value == null) ? null : ((NlsString) value).getValue(); diff --git a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java index 8f9e11797453..a1ab24a19dc5 100644 --- a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java +++ b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java @@ -117,6 +117,7 @@ import org.apache.calcite.util.ImmutableNullableList; import org.apache.calcite.util.Litmus; import org.apache.calcite.util.NlsString; +import org.apache.calcite.util.NumberUtil; import org.apache.calcite.util.Optionality; import org.apache.calcite.util.Pair; import org.apache.calcite.util.Util; @@ -494,8 +495,7 @@ public RexLiteral literal(@Nullable Object value) { return rexBuilder.makeApproxLiteral( ((Number) value).doubleValue(), getTypeFactory().createSqlType(SqlTypeName.DOUBLE)); } else if (value instanceof Number) { - return rexBuilder.makeExactLiteral( - BigDecimal.valueOf(((Number) value).longValue())); + return rexBuilder.makeExactLiteral(NumberUtil.toBigDecimal((Number) value)); } else if (value instanceof String) { return rexBuilder.makeLiteral((String) value); } else if (value instanceof Enum) { From 975000a7e95d6c14fdc7ab72f1230c17f2399028 Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Wed, 31 Dec 2025 17:54:41 +0800 Subject: [PATCH 081/562] [CALCITE-5733] Simplify "a = ARRAY[1,2] AND a = ARRAY[2,3]" to "false" --- .../org/apache/calcite/rex/RexAnalyzer.java | 4 + .../org/apache/calcite/rex/RexSimplify.java | 96 +++++++++-- .../apache/calcite/rex/RexProgramTest.java | 153 ++++++++++++++++++ 3 files changed, 239 insertions(+), 14 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rex/RexAnalyzer.java b/core/src/main/java/org/apache/calcite/rex/RexAnalyzer.java index 5dcad7aa9ebc..a124168137d8 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexAnalyzer.java +++ b/core/src/main/java/org/apache/calcite/rex/RexAnalyzer.java @@ -96,6 +96,10 @@ private static List getComparables(RexNode variable) { values.add(0); // 00:00:00.000 values.add(86_399_000); // 23:59:59.000 break; + case ARRAY: + case MAP: + case MULTISET: + break; default: throw new AssertionError("don't know values for " + variable + " of type " + variable.getType()); diff --git a/core/src/main/java/org/apache/calcite/rex/RexSimplify.java b/core/src/main/java/org/apache/calcite/rex/RexSimplify.java index 7065c25c5c7f..764100889ef8 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexSimplify.java +++ b/core/src/main/java/org/apache/calcite/rex/RexSimplify.java @@ -42,6 +42,7 @@ import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.BoundType; +import com.google.common.collect.HashMultiset; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableRangeSet; import com.google.common.collect.ImmutableSet; @@ -65,6 +66,7 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -1853,7 +1855,7 @@ private > RexNode simplifyAnd2ForUnknownAsFalse( ArrayListMultimap.create(); final Map, List>> rangeTerms = new HashMap<>(); - final Map equalityConstantTerms = new HashMap<>(); + final Map equalityConstantTerms = new HashMap<>(); final Set negatedTerms = new HashSet<>(); final Set nullOperands = new HashSet<>(); final Set notNullOperands = new LinkedHashSet<>(); @@ -1915,14 +1917,15 @@ private > RexNode simplifyAnd2ForUnknownAsFalse( // is equal to different constants, this condition cannot be satisfied, // and hence it can be evaluated to FALSE if (term.getKind() == SqlKind.EQUALS) { - if (comparison != null) { - final RexLiteral literal = comparison.literal; - final RexLiteral prevLiteral = - equalityConstantTerms.put(comparison.ref, literal); - - if (prevLiteral != null - && literal.getType().equals(prevLiteral.getType()) - && !literal.equals(prevLiteral)) { + final Pair constantEquality = constantEquality(call); + if (constantEquality != null) { + final RexNode constant = constantEquality.right; + final RexNode prevConstant = + equalityConstantTerms.put(constantEquality.left, constant); + + if (prevConstant != null + && constant.getType().equals(prevConstant.getType()) + && !constantsEquivalent(constant, prevConstant)) { return rexBuilder.makeLiteral(false); } } else if (RexUtil.isReferenceOrAccess(left, true) @@ -1983,17 +1986,18 @@ private > RexNode simplifyAnd2ForUnknownAsFalse( // Example #1. x=5 AND y=5 AND x=y : x=5 AND y=5 // Example #2. x=5 AND y=6 AND x=y - not satisfiable for (RexNode ref1 : equalityTerms.keySet()) { - final RexLiteral literal1 = equalityConstantTerms.get(ref1); - if (literal1 == null) { + final RexNode constant1 = equalityConstantTerms.get(ref1); + if (constant1 == null) { continue; } Collection> references = equalityTerms.get(ref1); for (Pair ref2 : references) { - final RexLiteral literal2 = equalityConstantTerms.get(ref2.left); - if (literal2 == null) { + final RexNode constant2 = equalityConstantTerms.get(ref2.left); + if (constant2 == null) { continue; } - if (literal1.getType().equals(literal2.getType()) && !literal1.equals(literal2)) { + if (constant1.getType().equals(constant2.getType()) + && !constantsEquivalent(constant1, constant2)) { // If an expression is equal to two different constants, // it is not satisfiable return rexBuilder.makeLiteral(false); @@ -3034,6 +3038,70 @@ private static class VariableCollector extends RexVisitorImpl { } } + private static final Set CONSTANT_VALUE_CONSTRUCTOR_KINDS = + EnumSet.of( + SqlKind.ARRAY_VALUE_CONSTRUCTOR, + SqlKind.MULTISET_VALUE_CONSTRUCTOR); + + private static @Nullable Pair constantEquality(RexCall call) { + final RexNode o0 = call.getOperands().get(0); + final RexNode o1 = call.getOperands().get(1); + if (RexUtil.isReferenceOrAccess(o0, true) && isConstant(o1)) { + return Pair.of(o0, o1); + } + if (RexUtil.isReferenceOrAccess(o1, true) && isConstant(o0)) { + return Pair.of(o1, o0); + } + return null; + } + + private static boolean constantsEquivalent(RexNode node1, RexNode node2) { + if (Objects.equals(node1, node2)) { + return true; + } + if (!(node1 instanceof RexCall) || !(node2 instanceof RexCall)) { + return false; + } + final RexCall call1 = (RexCall) node1; + final RexCall call2 = (RexCall) node2; + if (call1.getKind() != call2.getKind()) { + return false; + } + switch (call1.getKind()) { + case MULTISET_VALUE_CONSTRUCTOR: + return multisetLiteralEquals(call1, call2); + default: + return false; + } + } + + private static boolean multisetLiteralEquals(RexCall left, RexCall right) { + return canonicalMultisetLiteral(left).equals(canonicalMultisetLiteral(right)); + } + + private static HashMultiset canonicalMultisetLiteral(RexCall call) { + final HashMultiset canonical = HashMultiset.create(); + for (RexNode operand : call.getOperands()) { + canonical.add(canonicalMultisetOperand(operand)); + } + return canonical; + } + + private static Object canonicalMultisetOperand(RexNode operand) { + if (operand instanceof RexCall + && operand.getKind() == SqlKind.MULTISET_VALUE_CONSTRUCTOR) { + return canonicalMultisetLiteral((RexCall) operand); + } + return operand; + } + + private static boolean isConstant(RexNode node) { + return node instanceof RexLiteral + || (node instanceof RexCall + && CONSTANT_VALUE_CONSTRUCTOR_KINDS.contains(node.getKind()) + && RexUtil.isConstant(node)); + } + /** Represents a simple Comparison. * *

    Left hand side is a {@link RexNode}, right hand side is a literal. diff --git a/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java b/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java index 3daf4b896226..3c6620d1b78d 100644 --- a/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java +++ b/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java @@ -1876,6 +1876,159 @@ private void checkExponentialCnf(int n) { checkSimplifyUnchanged(rexBuilder.makeCall(SqlStdOperatorTable.SOME_GT, operand1, operand2)); } + /** Unit test for + * [CALCITE-5733] + * Simplify 'a = ARRAY[1,2] AND a = ARRAY[2,3]' to 'false'. */ + @Test void testSimplifyArrayEquality() { + final RelDataType arrayType = tArray(tInt()); + final RexNode aRef = input(arrayType, 0); + final RexNode array12 = + rexBuilder.makeCall(arrayType, SqlStdOperatorTable.ARRAY_VALUE_CONSTRUCTOR, + ImmutableList.of(literal(1), literal(2))); + final RexNode array21 = + rexBuilder.makeCall(arrayType, SqlStdOperatorTable.ARRAY_VALUE_CONSTRUCTOR, + ImmutableList.of(literal(2), literal(1))); + final RexNode array23 = + rexBuilder.makeCall(arrayType, SqlStdOperatorTable.ARRAY_VALUE_CONSTRUCTOR, + ImmutableList.of(literal(2), literal(3))); + final RexNode array2Null = + rexBuilder.makeCall(arrayType, SqlStdOperatorTable.ARRAY_VALUE_CONSTRUCTOR, + ImmutableList.of(literal(2), nullInt)); + final RexNode arrayDoubleNull = + rexBuilder.makeCall(arrayType, SqlStdOperatorTable.ARRAY_VALUE_CONSTRUCTOR, + ImmutableList.of(nullInt, nullInt)); + + // a = ARRAY[1,2] AND a = ARRAY[2,3] + final RexNode condition = and(eq(aRef, array12), eq(aRef, array23)); + checkSimplifyFilter(condition, "false"); + + // a = ARRAY[1,2] AND a = ARRAY[2,null] + final RexNode condition2 = and(eq(aRef, array12), eq(aRef, array2Null)); + checkSimplifyFilter(condition2, "false"); + + // a = ARRAY[1,2] AND a = ARRAY[2,null] + final RexNode condition3 = and(eq(aRef, array12), eq(aRef, arrayDoubleNull)); + checkSimplifyFilter(condition3, "false"); + + // a = ARRAY[2,null] AND a = ARRAY[2,null] + final RexNode condition4 = and(eq(aRef, arrayDoubleNull), eq(aRef, arrayDoubleNull)); + checkSimplifyFilter(condition4, "=($0, ARRAY(null:INTEGER, null:INTEGER))"); + + // a = ARRAY[1,2] AND a = ARRAY[2,1] + final RexNode condition5 = and(eq(aRef, array12), eq(aRef, array21)); + checkSimplifyFilter(condition5, "false"); + + // Nested type for Array + final RelDataType nestedArrayType = tArray(arrayType); + final RexNode nestedRef = input(nestedArrayType, 1); + + final RexNode nestedArray1212 = + rexBuilder.makeCall(nestedArrayType, SqlStdOperatorTable.ARRAY_VALUE_CONSTRUCTOR, + ImmutableList.of(array12, array12)); + final RexNode nestedArray1221 = + rexBuilder.makeCall(nestedArrayType, SqlStdOperatorTable.ARRAY_VALUE_CONSTRUCTOR, + ImmutableList.of(array12, array21)); + final RexNode nestedArray232Null = + rexBuilder.makeCall(nestedArrayType, SqlStdOperatorTable.ARRAY_VALUE_CONSTRUCTOR, + ImmutableList.of(array23, array2Null)); + final RexNode nestedArrayNulls = + rexBuilder.makeCall(nestedArrayType, SqlStdOperatorTable.ARRAY_VALUE_CONSTRUCTOR, + ImmutableList.of(arrayDoubleNull, arrayDoubleNull)); + + // a = ARRAY[ARRAY[1,2], ARRAY[1,2]] and a = ARRAY[ARRAY[1,2], ARRAY[2,1]] + final RexNode nestedCondition = + and(eq(nestedRef, nestedArray1212), eq(nestedRef, nestedArray1221)); + checkSimplifyFilter(nestedCondition, "false"); + + // a = ARRAY[ARRAY[1,2], ARRAY[1,2]] and a = ARRAY[ARRAY[2,3], ARRAY[2,null]] + final RexNode nestedCondition2 = + and(eq(nestedRef, nestedArray1212), eq(nestedRef, nestedArray232Null)); + checkSimplifyFilter(nestedCondition2, "false"); + + // a = ARRAY[ARRAY[1,2], ARRAY[1,2]] and a = ARRAY[ARRAY[null,null], ARRAY[null,null]] + final RexNode nestedCondition3 = + and(eq(nestedRef, nestedArray1212), eq(nestedRef, nestedArrayNulls)); + checkSimplifyFilter(nestedCondition3, "false"); + } + + /** Unit test for + * [CALCITE-5733] + * Simplify 'a = ARRAY[1,2] AND a = ARRAY[2,3]' to 'false'. */ + @Test void testSimplifyMultisetEquality() { + final RelDataType multisetType = typeFactory.createMultisetType(tInt(), -1); + final RexNode aRef = input(multisetType, 0); + final RexNode multiset12 = + rexBuilder.makeCall(multisetType, SqlStdOperatorTable.MULTISET_VALUE, + ImmutableList.of(literal(1), literal(2))); + final RexNode multiset21 = + rexBuilder.makeCall(multisetType, SqlStdOperatorTable.MULTISET_VALUE, + ImmutableList.of(literal(2), literal(1))); + final RexNode multiset23 = + rexBuilder.makeCall(multisetType, SqlStdOperatorTable.MULTISET_VALUE, + ImmutableList.of(literal(2), literal(3))); + final RexNode multiset2Null = + rexBuilder.makeCall(multisetType, SqlStdOperatorTable.MULTISET_VALUE, + ImmutableList.of(literal(2), nullInt)); + final RexNode multisetDoubleNull = + rexBuilder.makeCall(multisetType, SqlStdOperatorTable.MULTISET_VALUE, + ImmutableList.of(nullInt, nullInt)); + + // a = MULTISET[1,2] AND a = MULTISET[2,3] + final RexNode condition = and(eq(aRef, multiset12), eq(aRef, multiset23)); + checkSimplifyFilter(condition, "false"); + + // a = MULTISET[1,2] AND a = MULTISET[2,null] + final RexNode condition2 = and(eq(aRef, multiset12), eq(aRef, multiset2Null)); + checkSimplifyFilter(condition2, "false"); + + // a = MULTISET[1,2] AND a = MULTISET[2,null] + final RexNode condition3 = and(eq(aRef, multiset12), eq(aRef, multisetDoubleNull)); + checkSimplifyFilter(condition3, "false"); + + // a = MULTISET[2,null] AND a = MULTISET[2,null] + final RexNode condition4 = and(eq(aRef, multisetDoubleNull), eq(aRef, multisetDoubleNull)); + checkSimplifyFilter(condition4, "=($0, MULTISET(null:INTEGER, null:INTEGER))"); + + // a = MULTISET[1,2] AND a = MULTISET[2,1] + final RexNode condition5 = and(eq(aRef, multiset12), eq(aRef, multiset21)); + checkSimplifyFilter(condition5, "AND(=($0, MULTISET(1, 2)), =($0, MULTISET(2, 1)))"); + + // Nested type for Multiset + final RelDataType nestedMultisetType = typeFactory.createMultisetType(multisetType, -1); + final RexNode nestedRef = input(nestedMultisetType, 1); + + final RexNode nestedMultiset1212 = + rexBuilder.makeCall(nestedMultisetType, SqlStdOperatorTable.MULTISET_VALUE, + ImmutableList.of(multiset12, multiset12)); + final RexNode nestedMultiset1221 = + rexBuilder.makeCall(nestedMultisetType, SqlStdOperatorTable.MULTISET_VALUE, + ImmutableList.of(multiset12, multiset21)); + final RexNode nestedMultiset232Null = + rexBuilder.makeCall(nestedMultisetType, SqlStdOperatorTable.MULTISET_VALUE, + ImmutableList.of(multiset23, multiset2Null)); + final RexNode nestedMultisetNulls = + rexBuilder.makeCall(nestedMultisetType, SqlStdOperatorTable.MULTISET_VALUE, + ImmutableList.of(multisetDoubleNull, multisetDoubleNull)); + + // a = MULTISET[MULTISET[1,2], MULTISET[1,2]] and a = MULTISET[MULTISET[1,2], MULTISET[2,1]] + final RexNode nestedCondition = + and(eq(nestedRef, nestedMultiset1212), eq(nestedRef, nestedMultiset1221)); + checkSimplifyFilter(nestedCondition, + "AND(=($1, MULTISET(MULTISET(1, 2), MULTISET(1, 2)))," + + " =($1, MULTISET(MULTISET(1, 2), MULTISET(2, 1))))"); + + // a = MULTISET[MULTISET[1,2], MULTISET[1,2]] and a = MULTISET[MULTISET[2,3], MULTISET[2,null]] + final RexNode nestedCondition2 = + and(eq(nestedRef, nestedMultiset1212), eq(nestedRef, nestedMultiset232Null)); + checkSimplifyFilter(nestedCondition2, "false"); + + // a = MULTISET[MULTISET[1,2], MULTISET[1,2]] + // and a = MULTISET[MULTISET[null,null], MULTISET[null,null]] + final RexNode nestedCondition3 = + and(eq(nestedRef, nestedMultiset1212), eq(nestedRef, nestedMultisetNulls)); + checkSimplifyFilter(nestedCondition3, "false"); + } + @Test void testSimplifyRange() { final RexNode aRef = input(tInt(), 0); // ((0 < a and a <= 10) or a >= 15) and a <> 6 and a <> 12 From 1099b6aad8afbaa5abb2e435ccee9ad72110555e Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Sat, 3 Jan 2026 16:36:58 +0800 Subject: [PATCH 082/562] [CALCITE-7350] Missing allowEmptyOutputFromRewrite parameter in TopDownGeneralDecorrelator.unnestInternal --- .../sql2rel/TopDownGeneralDecorrelator.java | 2 +- .../apache/calcite/test/RelOptRulesTest.java | 18 ++++++++ .../apache/calcite/test/RelOptRulesTest.xml | 41 +++++++++++++++++++ 3 files changed, 60 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java b/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java index cea5e5d5a461..d3ccaa334dbc 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java @@ -786,7 +786,7 @@ public RelNode unnestInternal(SetOp setOp, boolean allowEmptyOutputFromRewrite) return newSetOp; } - public RelNode unnestInternal(RelNode other) { + public RelNode unnestInternal(RelNode other, boolean allowEmptyOutputFromRewrite) { throw new UnsupportedOperationException("Top-down general decorrelator does not support: " + other.getClass().getSimpleName()); } diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index 959cdada0bc9..1afbae451906 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -10212,6 +10212,24 @@ public interface Config extends RelRule.Config { .check(); } + /** Test case for + * [CALCITE-7350] + * Missing allowEmptyOutputFromRewrite parameter + * in TopDownGeneralDecorrelator.unnestInternal. */ + @Test void testUnnestInternalMissingParameter() { + final String sql = "SELECT empno FROM emp e" + + " WHERE sal > some(SELECT avg(sal) over (partition by deptno) from emp_b b" + + " where b.deptno = e.deptno)"; + + sql(sql) + .withRule( + CoreRules.FILTER_SUB_QUERY_TO_MARK_CORRELATE, + CoreRules.PROJECT_TO_LOGICAL_PROJECT_AND_WINDOW) + .withLateDecorrelate(true) + .withTopDownGeneralDecorrelate(true) + .check(); + } + /** * Test case for * [CALCITE-6824] diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index 7075571d9550..3887c4fe50da 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -21563,6 +21563,47 @@ LogicalProject(DEPTNO=[$7]) + + + + + some(SELECT avg(sal) over (partition by deptno) from emp_b b where b.deptno = e.deptno)]]> + + + SOME($5, { +LogicalProject(EXPR$0=[CAST(/(SUM($5) OVER (PARTITION BY $7), COUNT($5) OVER (PARTITION BY $7))):INTEGER NOT NULL]) + LogicalFilter(condition=[=($7, $cor0.DEPTNO)]) + LogicalTableScan(table=[[CATALOG, SALES, EMP_B]]) +})], variablesSet=[[$cor0]]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +]]> + + + ($5, $9)]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) + LogicalProject(EXPR$0=[CAST(/($10, $11)):INTEGER NOT NULL]) + LogicalWindow(window#0=[window(partition {7} aggs [SUM($5), COUNT($5)])]) + LogicalFilter(condition=[=($7, $cor0.DEPTNO)]) + LogicalTableScan(table=[[CATALOG, SALES, EMP_B]]) +]]> + + + ($5, $9)]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) + LogicalProject(EXPR$0=[CAST(/($10, $11)):INTEGER NOT NULL]) + LogicalWindow(window#0=[window(partition {7} aggs [SUM($5), COUNT($5)])]) + LogicalFilter(condition=[=($7, $cor0.DEPTNO)]) + LogicalTableScan(table=[[CATALOG, SALES, EMP_B]]) ]]> From 7337c49fb9a2361511ae6bada658415d796eee2a Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Sun, 4 Jan 2026 09:32:11 +0800 Subject: [PATCH 083/562] [CALCITE-7352] Incorrect SqlLibrary enum value used in ClickHouse SQL test --- .../java/org/apache/calcite/sql/fun/SqlLibraryOperators.java | 5 +++-- .../apache/calcite/rel/rel2sql/RelToSqlConverterTest.java | 4 ++-- site/_docs/reference.md | 4 ++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java index f8f4e9273b8f..f7f7029ae4c7 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java @@ -64,6 +64,7 @@ import static org.apache.calcite.sql.fun.SqlLibrary.ALL; import static org.apache.calcite.sql.fun.SqlLibrary.BIG_QUERY; import static org.apache.calcite.sql.fun.SqlLibrary.CALCITE; +import static org.apache.calcite.sql.fun.SqlLibrary.CLICKHOUSE; import static org.apache.calcite.sql.fun.SqlLibrary.HIVE; import static org.apache.calcite.sql.fun.SqlLibrary.MSSQL; import static org.apache.calcite.sql.fun.SqlLibrary.MYSQL; @@ -424,7 +425,7 @@ static RelDataType deriveTypeSplit(SqlOperatorBinding operatorBinding, OperandTypes.STRING_SAME_SAME); /** The "ENDSWITH(value1, value2)" function (Snowflake). */ - @LibraryOperator(libraries = {SNOWFLAKE, SPARK}) + @LibraryOperator(libraries = {SNOWFLAKE, SPARK, CLICKHOUSE}) public static final SqlFunction ENDSWITH = ENDS_WITH.withName("ENDSWITH"); /** The "STARTS_WITH(value1, value2)" function (BigQuery, PostgreSQL). */ @@ -434,7 +435,7 @@ static RelDataType deriveTypeSplit(SqlOperatorBinding operatorBinding, OperandTypes.STRING_SAME_SAME); /** The "STARTSWITH(value1, value2)" function (Snowflake). */ - @LibraryOperator(libraries = {SNOWFLAKE, SPARK}) + @LibraryOperator(libraries = {SNOWFLAKE, SPARK, CLICKHOUSE}) public static final SqlFunction STARTSWITH = STARTS_WITH.withName("STARTSWITH"); /** BigQuery's "SUBSTR(string, position [, substringLength ])" function. */ diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java index 43e855592422..27e9c23fecc3 100644 --- a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java +++ b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java @@ -8414,7 +8414,7 @@ private void checkLiteral2(String expression, String expected) { sql(query).withLibrary(SqlLibrary.SNOWFLAKE).withPostgresql().ok(expectedPostgres); sql(query).withLibrary(SqlLibrary.SNOWFLAKE).withSnowflake().ok(expectedSnowflake); sql(query).withLibrary(SqlLibrary.SPARK).withSpark().ok(expectedSpark); - sql(query).withLibrary(SqlLibrary.SPARK).withClickHouse().ok(expectedClickHouse); + sql(query).withLibrary(SqlLibrary.CLICKHOUSE).withClickHouse().ok(expectedClickHouse); } /** Test case for @@ -8463,7 +8463,7 @@ private void checkLiteral2(String expression, String expected) { sql(query).withLibrary(SqlLibrary.SNOWFLAKE).withPostgresql().ok(expectedPostgres); sql(query).withLibrary(SqlLibrary.SNOWFLAKE).withSnowflake().ok(expectedSnowflake); sql(query).withLibrary(SqlLibrary.SPARK).withSpark().ok(expectedSpark); - sql(query).withLibrary(SqlLibrary.SPARK).withClickHouse().ok(expectedClickHouse); + sql(query).withLibrary(SqlLibrary.CLICKHOUSE).withClickHouse().ok(expectedClickHouse); } /** Test case for diff --git a/site/_docs/reference.md b/site/_docs/reference.md index 617ce1a83bec..643cb69f18e9 100644 --- a/site/_docs/reference.md +++ b/site/_docs/reference.md @@ -2920,7 +2920,7 @@ In the following: | b | DATE_TRUNC(date, timeUnit) | Truncates *date* to the granularity of *timeUnit*, rounding to the beginning of the unit | o r s h | DECODE(value, value1, result1 [, valueN, resultN ]* [, default ]) | Compares *value* to each *valueN* value one by one; if *value* is equal to a *valueN*, returns the corresponding *resultN*, else returns *default*, or NULL if *default* is not specified | p r | DIFFERENCE(string, string) | Returns a measure of the similarity of two strings, namely the number of character positions that their `SOUNDEX` values have in common: 4 if the `SOUNDEX` values are same and 0 if the `SOUNDEX` values are totally different -| f s | ENDSWITH(string1, string2) | Returns whether *string2* is a suffix of *string1* +| f s i | ENDSWITH(string1, string2) | Returns whether *string2* is a suffix of *string1* | b | ENDS_WITH(string1, string2) | Equivalent to `ENDSWITH(string1, string2)` | s | EXISTS(array, func) | Returns whether a predicate *func* holds for one or more elements in the *array* | o | EXISTSNODE(xml, xpath, [, namespaces ]) | Determines whether traversal of a XML document using a specified xpath results in any nodes. Returns 0 if no nodes remain after applying the XPath traversal on the document fragment of the element or elements matched by the XPath expression. Returns 1 if any nodes remain. The optional namespace value that specifies a default mapping or namespace mapping for prefixes, which is used when evaluating the XPath expression. @@ -3041,7 +3041,7 @@ In the following: | m s h | SPACE(integer) | Returns a string of *integer* spaces; returns an empty string if *integer* is less than 1 | b | SPLIT(string [, delimiter ]) | Returns the string array of *string* split at *delimiter* (if omitted, default is comma). If the *string* is empty it returns an empty array, otherwise, if the *delimiter* is empty, it returns an array containing the original *string*. | p | SPLIT_PART(string, delimiter, n) | Returns the *n*th field in *string* using *delimiter*; returns empty string if *n* is less than 1 or greater than the number of fields, and the n can be negative to count from the end. -| f s | STARTSWITH(string1, string2) | Returns whether *string2* is a prefix of *string1* +| f s i | STARTSWITH(string1, string2) | Returns whether *string2* is a prefix of *string1* | b p | STARTS_WITH(string1, string2) | Equivalent to `STARTSWITH(string1, string2)` | m | STRCMP(string, string) | Returns 0 if both of the strings are same and returns -1 when the first argument is smaller than the second and 1 when the second one is smaller than the first one | b r p | STRPOS(string, substring) | Equivalent to `POSITION(substring IN string)` From d73a5c903ddf5a5050666712cae06b652879a734 Mon Sep 17 00:00:00 2001 From: Thomas Rebele Date: Sat, 27 Dec 2025 18:31:59 +0100 Subject: [PATCH 084/562] Site: Add Thomas Rebele as committer --- site/_data/contributors.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/site/_data/contributors.yml b/site/_data/contributors.yml index 14d83ab3ac1f..cc43c5be9eeb 100644 --- a/site/_data/contributors.yml +++ b/site/_data/contributors.yml @@ -348,6 +348,12 @@ githubId: tdunning org: MapR role: PMC +- name: Thomas Rebele + apacheId: thomasrebele + githubId: thomasrebele + pronouns: he/him + org: Cloudera + role: Committer - name: TJ Banghart apacheId: tjbanghart githubId: tjbanghart From 5ffb6961aa3b2af72387a155982112b92d294aec Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Sun, 4 Jan 2026 17:15:49 +0800 Subject: [PATCH 085/562] [CALCITE-6066] Add HYPOT function (enabled in Spark library) --- .../adapter/enumerable/RexImpTable.java | 2 + .../apache/calcite/runtime/SqlFunctions.java | 12 +++++ .../calcite/sql/fun/SqlLibraryOperators.java | 9 ++++ .../apache/calcite/util/BuiltInMethod.java | 1 + site/_docs/reference.md | 1 + .../apache/calcite/test/SqlOperatorTest.java | 54 +++++++++++++++++++ 6 files changed, 79 insertions(+) diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java index ca8f93feb9fe..bbe2743ac7cd 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java @@ -225,6 +225,7 @@ import static org.apache.calcite.sql.fun.SqlLibraryOperators.FROM_HEX; import static org.apache.calcite.sql.fun.SqlLibraryOperators.GETBIT; import static org.apache.calcite.sql.fun.SqlLibraryOperators.HEX; +import static org.apache.calcite.sql.fun.SqlLibraryOperators.HYPOT; import static org.apache.calcite.sql.fun.SqlLibraryOperators.ILIKE; import static org.apache.calcite.sql.fun.SqlLibraryOperators.IS_INF; import static org.apache.calcite.sql.fun.SqlLibraryOperators.IS_NAN; @@ -880,6 +881,7 @@ void populate1() { defineMethod(CSCH, BuiltInMethod.CSCH.method, NullPolicy.STRICT); defineMethod(DEGREES, BuiltInMethod.DEGREES.method, NullPolicy.STRICT); defineMethod(FACTORIAL, BuiltInMethod.FACTORIAL.method, NullPolicy.STRICT); + defineMethod(HYPOT, BuiltInMethod.HYPOT.method, NullPolicy.STRICT); defineMethod(IS_INF, BuiltInMethod.IS_INF.method, NullPolicy.STRICT); defineMethod(IS_NAN, BuiltInMethod.IS_NAN.method, NullPolicy.STRICT); defineMethod(POW, BuiltInMethod.POWER.method, NullPolicy.STRICT); diff --git a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java index dc4c6c258784..52ea3bbef5c4 100644 --- a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java +++ b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java @@ -4319,6 +4319,18 @@ public static double degrees(double b0) { return CombinatoricsUtils.factorial(b0); } + /** SQL HYPOT operator applied to double values. */ + public static double hypot(double a, double b) { + return Math.hypot(a, b); + } + + /** SQL HYPOT operator applied to general numeric values. */ + public static double hypot(Object a, Object b) { + final Number left = (Number) a; + final Number right = (Number) b; + return hypot(left.doubleValue(), right.doubleValue()); + } + /** SQL IS_INF operator applied to BigDecimal values. */ public static boolean isInf(BigDecimal b0) { return Double.isInfinite(b0.doubleValue()); diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java index f7f7029ae4c7..83f8787fd578 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java @@ -2562,6 +2562,15 @@ private static RelDataType deriveTypeMapFromEntries(SqlOperatorBinding opBinding OperandTypes.NUMERIC, SqlFunctionCategory.STRING); + /** The {@code HYPOT(numeric1, numeric2)} function; returns + * sqrt(numeric1^2 + numeric2^2) without intermediate overflow or underflow. */ + @LibraryOperator(libraries = {SPARK, CLICKHOUSE}) + public static final SqlFunction HYPOT = + SqlBasicFunction.create("HYPOT", + ReturnTypes.DOUBLE_NULLABLE, + OperandTypes.NUMERIC_NUMERIC, + SqlFunctionCategory.NUMERIC); + @LibraryOperator(libraries = {BIG_QUERY, MYSQL, POSTGRESQL, SPARK, HIVE}) public static final SqlFunction MD5 = SqlBasicFunction.create("MD5", diff --git a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java index 38344096404a..cfeb4fe77c42 100644 --- a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java +++ b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java @@ -598,6 +598,7 @@ public enum BuiltInMethod { TAND(SqlFunctions.class, "tand", double.class), TANH(SqlFunctions.class, "tanh", long.class), SINH(SqlFunctions.class, "sinh", long.class), + HYPOT(SqlFunctions.class, "hypot", double.class, double.class), TRUNCATE(SqlFunctions.class, "truncate", String.class, int.class), TRUNCATE_OR_PAD(SqlFunctions.class, "truncateOrPad", String.class, int.class), TRIM(SqlFunctions.class, "trim", boolean.class, boolean.class, String.class, diff --git a/site/_docs/reference.md b/site/_docs/reference.md index 643cb69f18e9..b922253fe0ab 100644 --- a/site/_docs/reference.md +++ b/site/_docs/reference.md @@ -2936,6 +2936,7 @@ In the following: | b | FORMAT_TIME(string, time) | Formats *time* according to the specified format *string* | b | FORMAT_TIMESTAMP(string timestamp) | Formats *timestamp* according to the specified format *string* | s | GETBIT(value, position) | Equivalent to `BIT_GET(value, position)` +| s i | HYPOT(numeric1, numeric2) | Returns sqrt(*numeric1*^2 + *numeric2*^2) without intermediate overflow or underflow | b o p r s h | GREATEST(expr [, expr ]*) | Returns the greatest of the expressions | b h s | IF(condition, value1, value2) | Returns *value1* if *condition* is TRUE, *value2* otherwise | b s | IFNULL(value1, value2) | Equivalent to `NVL(value1, value2)` diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java index 90839a1daa7c..b27c188620a4 100644 --- a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java +++ b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java @@ -10018,6 +10018,60 @@ void checkArrayReverseFunc(SqlOperatorFixture f0, SqlFunction function, f0.forEachLibrary(list(SqlLibrary.BIG_QUERY, SqlLibrary.SPARK), consumer); } + /** Test case for + * [CALCITE-6066] Add HYPOT function (enabled in Spark library). */ + @Test void testHypotFunc() { + final SqlOperatorFixture f0 = fixture().setFor(SqlLibraryOperators.HYPOT); + f0.checkFails("^hypot(3, 4)^", + "No match found for function signature HYPOT\\(, \\)", + false); + final Consumer consumer = f -> { + f.checkScalarApprox("hypot(3, 4)", "DOUBLE NOT NULL", + isWithin(5.0000d, 0.0001d)); + f.checkScalarApprox("hypot(3.0, cast(4 as bigint))", "DOUBLE NOT NULL", + isWithin(5.0000d, 0.0001d)); + f.checkScalarApprox("hypot(cast(-2 as bigint), cast(-4 as bigint))", + "DOUBLE NOT NULL", + isWithin(4.4721d, 0.0001d)); + f.checkScalarApprox("hypot(cast(3.0 as double), cast(4.0 as double))", + "DOUBLE NOT NULL", + isWithin(5.0000d, 0.0001d)); + f.checkScalarApprox("hypot(-2.5, cast(-4.5 as double))", "DOUBLE NOT NULL", + isWithin(5.1478d, 0.0001d)); + f.checkScalarApprox("hypot(-2.5, -4.5)", "DOUBLE NOT NULL", + isWithin(5.1478d, 0.0001d)); + f.checkScalarApprox("hypot(cast(3 as float), cast(4 as real))", "DOUBLE NOT NULL", + isWithin(5.0000d, 0.0001d)); + f.checkType("hypot(cast(null as bigint), 1)", "DOUBLE"); + f.checkNull("hypot(cast(null as bigint), 1)"); + f.checkNull("hypot(1, cast(null as bigint))"); + f.checkNull("hypot(cast(null as bigint), cast(null as bigint))"); + f.checkNull("hypot(cast(null as double), cast(null as double))"); + f.checkNull("hypot(cast(null as decimal), cast(null as decimal))"); + + // unsigned type + f.checkScalarApprox("hypot(cast(3 as integer unsigned), cast(4 as integer unsigned))", + "DOUBLE NOT NULL", isWithin(5.0000d, 0.0001d)); + f.checkScalarApprox("hypot(cast(3 as bigint unsigned), cast(4 as integer unsigned))", + "DOUBLE NOT NULL", isWithin(5.0000d, 0.0001d)); + f.checkScalarApprox("hypot(cast(3 as bigint unsigned), cast(4 as bigint unsigned))", + "DOUBLE NOT NULL", isWithin(5.0000d, 0.0001d)); + f.checkScalarApprox("hypot(cast(3 as smallint unsigned), cast(4 as smallint unsigned))", + "DOUBLE NOT NULL", isWithin(5.0000d, 0.0001d)); + f.checkScalarApprox("hypot(cast(3 as tinyint unsigned), cast(4 as tinyint unsigned))", + "DOUBLE NOT NULL", isWithin(5.0000d, 0.0001d)); + + // mixed type + f.checkScalarApprox("hypot(cast(3 as tinyint), cast(4 as tinyint unsigned))", + "DOUBLE NOT NULL", isWithin(5.0000d, 0.0001d)); + f.checkScalarApprox("hypot(cast(3 as bigint), cast(4 as tinyint unsigned))", + "DOUBLE NOT NULL", isWithin(5.0000d, 0.0001d)); + f.checkScalarApprox("hypot(cast(3 as double), cast(4 as tinyint unsigned))", + "DOUBLE NOT NULL", isWithin(5.0000d, 0.0001d)); + }; + f0.forEachLibrary(list(SqlLibrary.SPARK, SqlLibrary.CLICKHOUSE), consumer); + } + @Test void testInfinity() { final SqlOperatorFixture f = fixture(); f.checkScalar("cast('Infinity' as double)", "Infinity", From 528c93b6144b1586e53f094d51f514de3457f1de Mon Sep 17 00:00:00 2001 From: nobigo Date: Tue, 30 Dec 2025 16:33:25 +0800 Subject: [PATCH 086/562] [CALCITE-7348] Remove redundant extraction correlation variables when Trim Project Fields --- .../calcite/sql2rel/RelFieldTrimmer.java | 26 +----- .../calcite/sql2rel/RelFieldTrimmerTest.java | 31 ++++--- core/src/test/resources/sql/scalar.iq | 93 +++++++++++++++++++ 3 files changed, 116 insertions(+), 34 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java b/core/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java index 0c9c761b333d..864ee9d6a0f1 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java @@ -548,27 +548,11 @@ public TrimResult trimFields( } } - // Collect all the SubQueries in the projection list. - List subQueries = RexUtil.SubQueryCollector.collect(project); - // Get all the correlationIds present in the SubQueries - Set correlationIds = RelOptUtil.getVariablesUsed(subQueries); - ImmutableBitSet requiredColumns = ImmutableBitSet.of(); - if (!correlationIds.isEmpty()) { - assert correlationIds.size() == 1; - // Correlation columns are also needed by SubQueries, so add them to inputFieldsUsed. - requiredColumns = RelOptUtil.correlationColumns(correlationIds.iterator().next(), project); - } - ImmutableBitSet finderFields = inputFinder.build(); - ImmutableBitSet inputFieldsUsed = ImmutableBitSet.builder() - .addAll(requiredColumns) - .addAll(finderFields) - .build(); - // Create input with trimmed columns. TrimResult trimResult = - trimChild(project, input, inputFieldsUsed, inputExtraFields); + trimChild(project, input, finderFields, inputExtraFields); RelNode newInput = trimResult.left; final Mapping inputMapping = trimResult.right; @@ -589,14 +573,14 @@ public TrimResult trimFields( final List newProjects = new ArrayList<>(); final RexVisitor shuttle; - if (!correlationIds.isEmpty()) { - assert correlationIds.size() == 1; + if (!project.getVariablesSet().isEmpty()) { shuttle = new RexPermuteInputsShuttle(inputMapping, newInput) { @Override public RexNode visitSubQuery(RexSubQuery subQuery) { subQuery = (RexSubQuery) super.visitSubQuery(subQuery); return RelOptUtil.remapCorrelatesInSuqQuery(relBuilder.getRexBuilder(), - subQuery, correlationIds.iterator().next(), newInput.getRowType(), inputMapping); + subQuery, project.getVariablesSet().iterator().next(), + newInput.getRowType(), inputMapping); } }; } else { @@ -621,7 +605,7 @@ public TrimResult trimFields( mapping); relBuilder.push(newInput); - relBuilder.project(newProjects, newRowType.getFieldNames(), false, correlationIds); + relBuilder.project(newProjects, newRowType.getFieldNames(), false, project.getVariablesSet()); final RelNode newProject = relBuilder.build(); return result(newProject, mapping, project); } diff --git a/core/src/test/java/org/apache/calcite/sql2rel/RelFieldTrimmerTest.java b/core/src/test/java/org/apache/calcite/sql2rel/RelFieldTrimmerTest.java index 77e6d89091a9..c23185c8c64e 100644 --- a/core/src/test/java/org/apache/calcite/sql2rel/RelFieldTrimmerTest.java +++ b/core/src/test/java/org/apache/calcite/sql2rel/RelFieldTrimmerTest.java @@ -33,6 +33,7 @@ import org.apache.calcite.rel.hint.RelHint; import org.apache.calcite.rel.rules.CoreRules; import org.apache.calcite.rex.RexCorrelVariable; +import org.apache.calcite.rex.RexNode; import org.apache.calcite.schema.SchemaPlus; import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.parser.SqlParser; @@ -42,6 +43,7 @@ import org.apache.calcite.tools.RelBuilder; import org.apache.calcite.util.Holder; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import org.checkerframework.checker.nullness.qual.Nullable; @@ -680,24 +682,27 @@ public static Frameworks.ConfigBuilder config() { @Test void testTrimCorrelatedSubquery() { final RelBuilder builder = RelBuilder.create(config().build()); final Holder<@Nullable RexCorrelVariable> v = Holder.empty(); - RelNode root = builder.scan("EMP") + builder.scan("EMP") .variable(v::set) .filter( builder.call(SqlStdOperatorTable.GREATER_THAN, builder.field(5), - builder.literal(10))) - .project( - builder.field(0), - builder.scalarQuery( - b2 -> builder.scan("EMP").filter( - builder.call(SqlStdOperatorTable.LESS_THAN, - builder.field(3), builder.field(v.get(), "MGR"))) - .project(builder.field(0)) - .aggregate(builder.groupKey(), builder.countStar("c")) - .build())) - .build(); + builder.literal(10))); + final ImmutableList.Builder projectsNode = ImmutableList.builder(); + projectsNode.add(builder.field(0)); + projectsNode.add( + builder.scalarQuery( + b2 -> builder.scan("EMP").filter( + builder.call(SqlStdOperatorTable.LESS_THAN, + builder.field(3), builder.field(v.get(), "MGR"))) + .project(builder.field(0)) + .aggregate(builder.groupKey(), builder.countStar("c")) + .build())); + RelNode root = + builder.project(projectsNode.build(), + ImmutableList.of(), false, ImmutableList.of(v.get().id)).build(); String origTree = "" - + "LogicalProject(EMPNO=[$0], $f1=[$SCALAR_QUERY({\n" + + "LogicalProject(variablesSet=[[$cor0]], EMPNO=[$0], $f1=[$SCALAR_QUERY({\n" + "LogicalAggregate(group=[{}], c=[COUNT()])\n" + " LogicalFilter(condition=[<($3, $cor0.MGR)])\n" + " LogicalTableScan(table=[[scott, EMP]])\n})])\n" diff --git a/core/src/test/resources/sql/scalar.iq b/core/src/test/resources/sql/scalar.iq index 82a1eb9223ef..4b5e186cc99f 100644 --- a/core/src/test/resources/sql/scalar.iq +++ b/core/src/test/resources/sql/scalar.iq @@ -310,4 +310,97 @@ select !ok +# [CALCITE-7348] Remove redundant extraction correlation variables when Trim Project Fields + +!set trimfields true + +SELECT empno, (SELECT COUNT(*) AS c +FROM "scott".emp +WHERE mgr < "t".mgr) AS "$f1" +FROM "scott".emp as "t" +WHERE sal > 10; ++-------+-----+ +| EMPNO | $f1 | ++-------+-----+ +| 7369 | 12 | +| 7499 | 2 | +| 7521 | 2 | +| 7566 | 9 | +| 7654 | 2 | +| 7698 | 9 | +| 7782 | 9 | +| 7788 | 0 | +| 7839 | 0 | +| 7844 | 2 | +| 7876 | 8 | +| 7900 | 2 | +| 7902 | 0 | +| 7934 | 7 | ++-------+-----+ +(14 rows) + +!ok +EnumerableCalc(expr#0..3=[{inputs}], expr#4=[IS NULL($t3)], expr#5=[0:BIGINT], expr#6=[CASE($t4, $t5, $t3)], EMPNO=[$t0], $f1=[$t6]) + EnumerableHashJoin(condition=[IS NOT DISTINCT FROM($1, $2)], joinType=[left]) + EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t5):DECIMAL(12, 2)], expr#9=[10.00:DECIMAL(12, 2)], expr#10=[>($t8, $t9)], EMPNO=[$t0], MGR=[$t3], $condition=[$t10]) + EnumerableTableScan(table=[[scott, EMP]]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[IS NOT NULL($t2)], expr#4=[0], expr#5=[CASE($t3, $t2, $t4)], MGR0=[$t0], C=[$t5]) + EnumerableHashJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left]) + EnumerableAggregate(group=[{3}]) + EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t5):DECIMAL(12, 2)], expr#9=[10.00:DECIMAL(12, 2)], expr#10=[>($t8, $t9)], proj#0..7=[{exprs}], $condition=[$t10]) + EnumerableTableScan(table=[[scott, EMP]]) + EnumerableAggregate(group=[{2}], C=[COUNT()]) + EnumerableNestedLoopJoin(condition=[<($1, $2)], joinType=[inner]) + EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], MGR=[$t3]) + EnumerableTableScan(table=[[scott, EMP]]) + EnumerableAggregate(group=[{3}]) + EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t5):DECIMAL(12, 2)], expr#9=[10.00:DECIMAL(12, 2)], expr#10=[>($t8, $t9)], proj#0..7=[{exprs}], $condition=[$t10]) + EnumerableTableScan(table=[[scott, EMP]]) +!plan + +!set trimfields false + +SELECT empno, (SELECT COUNT(*) AS c +FROM "scott".emp +WHERE mgr < "t".mgr) AS "$f1" +FROM "scott".emp as "t" +WHERE sal > 10; ++-------+-----+ +| EMPNO | $f1 | ++-------+-----+ +| 7369 | 12 | +| 7499 | 2 | +| 7521 | 2 | +| 7566 | 9 | +| 7654 | 2 | +| 7698 | 9 | +| 7782 | 9 | +| 7788 | 0 | +| 7839 | 0 | +| 7844 | 2 | +| 7876 | 8 | +| 7900 | 2 | +| 7902 | 0 | +| 7934 | 7 | ++-------+-----+ +(14 rows) + +!ok +EnumerableCalc(expr#0..9=[{inputs}], expr#10=[IS NULL($t9)], expr#11=[0:BIGINT], expr#12=[CASE($t10, $t11, $t9)], EMPNO=[$t0], $f1=[$t12]) + EnumerableHashJoin(condition=[IS NOT DISTINCT FROM($3, $8)], joinType=[left]) + EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t5):DECIMAL(12, 2)], expr#9=[10.00:DECIMAL(12, 2)], expr#10=[>($t8, $t9)], proj#0..7=[{exprs}], $condition=[$t10]) + EnumerableTableScan(table=[[scott, EMP]]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[IS NOT NULL($t2)], expr#4=[0], expr#5=[CASE($t3, $t2, $t4)], MGR0=[$t0], C=[$t5]) + EnumerableHashJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left]) + EnumerableAggregate(group=[{3}]) + EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t5):DECIMAL(12, 2)], expr#9=[10.00:DECIMAL(12, 2)], expr#10=[>($t8, $t9)], proj#0..7=[{exprs}], $condition=[$t10]) + EnumerableTableScan(table=[[scott, EMP]]) + EnumerableAggregate(group=[{8}], C=[COUNT()]) + EnumerableNestedLoopJoin(condition=[<($3, $8)], joinType=[inner]) + EnumerableTableScan(table=[[scott, EMP]]) + EnumerableAggregate(group=[{3}]) + EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t5):DECIMAL(12, 2)], expr#9=[10.00:DECIMAL(12, 2)], expr#10=[>($t8, $t9)], proj#0..7=[{exprs}], $condition=[$t10]) + EnumerableTableScan(table=[[scott, EMP]]) +!plan + # End scalar.iq From 4a7421d73f2b1aa9aaff0cd7e54ec85aab389238 Mon Sep 17 00:00:00 2001 From: Terran Date: Mon, 29 Dec 2025 12:00:30 +0800 Subject: [PATCH 087/562] [CALCITE-7337] Add age function (enabled in PostgreSQL library) --- .../org/apache/calcite/test/BabelTest.java | 44 +++++++ .../adapter/enumerable/RexImpTable.java | 3 + .../apache/calcite/runtime/SqlFunctions.java | 111 ++++++++++++++++++ .../calcite/sql/fun/SqlLibraryOperators.java | 17 +++ .../apache/calcite/util/BuiltInMethod.java | 3 +- site/_docs/reference.md | 1 + .../apache/calcite/test/SqlOperatorTest.java | 108 +++++++++++++++++ 7 files changed, 286 insertions(+), 1 deletion(-) diff --git a/babel/src/test/java/org/apache/calcite/test/BabelTest.java b/babel/src/test/java/org/apache/calcite/test/BabelTest.java index 09da75550854..769975c4a894 100644 --- a/babel/src/test/java/org/apache/calcite/test/BabelTest.java +++ b/babel/src/test/java/org/apache/calcite/test/BabelTest.java @@ -490,4 +490,48 @@ private void checkSqlResult(String funLibrary, String query, String result) { .query(query) .returns(result); } + + /** Test case for + * [CALCITE-7337] + * Add age function (enabled in PostgreSQL library). */ + @Test void testAgeFunction() { + checkSqlResult("postgresql", + "SELECT AGE(timestamp '2023-12-25', timestamp '2020-01-01') FROM (VALUES (1)) t", + "EXPR$0=3 years 11 mons 24 days\n"); + + checkSqlResult("postgresql", + "SELECT AGE(timestamp '2023-01-01', timestamp '2023-01-01') FROM (VALUES (1)) t", + "EXPR$0=00:00:00\n"); + + checkSqlResult("postgresql", + "SELECT AGE(timestamp '2020-01-01', timestamp '2023-12-25') FROM (VALUES (1)) t", + "EXPR$0=-3 years -11 mons -24 days\n"); + + checkSqlResult("postgresql", + "SELECT AGE(timestamp '2023-02-01', timestamp '2023-01-31') FROM (VALUES (1)) t", + "EXPR$0=1 day\n"); + + checkSqlResult("postgresql", + "SELECT AGE(timestamp '2023-12-26 14:30:00', timestamp '2023-12-25 14:30:00') FROM (VALUES (1)) t", + "EXPR$0=1 day\n"); + + checkSqlResult("postgresql", + "SELECT AGE(timestamp '2023-12-25 00:00:00', timestamp '2020-01-01 23:59:59') FROM (VALUES (1)) t", + "EXPR$0=3 years 11 mons 23 days 00:00:01\n"); + + checkSqlResult("postgresql", + "SELECT AGE(timestamp '2023-12-25 00:00:00.101', timestamp '2020-01-01 23:59:59.202') FROM (VALUES (1)) t", + "EXPR$0=3 years 11 mons 23 days 00:00:00.899\n"); + + checkSqlResult("postgresql", + "SELECT AGE(timestamp '2023-12-25 12:00:00.500', timestamp '2023-12-25 12:00:00.000') FROM (VALUES (1)) t", + "EXPR$0=00:00:00.5\n"); + + CalciteAssert.that() + .with(CalciteConnectionProperty.PARSER_FACTORY, + SqlBabelParserImpl.class.getName() + "#FACTORY") + .with(CalciteConnectionProperty.FUN, "postgresql") + .query("SELECT AGE(timestamp '2023-12-25') FROM (VALUES (1)) t") + .runs(); + } } diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java index bbe2743ac7cd..103e91919f19 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java @@ -134,6 +134,7 @@ import static org.apache.calcite.sql.fun.SqlInternalOperators.THROW_UNLESS; import static org.apache.calcite.sql.fun.SqlLibraryOperators.ACOSD; import static org.apache.calcite.sql.fun.SqlLibraryOperators.ACOSH; +import static org.apache.calcite.sql.fun.SqlLibraryOperators.AGE; import static org.apache.calcite.sql.fun.SqlLibraryOperators.ARRAY; import static org.apache.calcite.sql.fun.SqlLibraryOperators.ARRAYS_OVERLAP; import static org.apache.calcite.sql.fun.SqlLibraryOperators.ARRAYS_ZIP; @@ -1029,6 +1030,8 @@ void populate2() { define(FORMAT_TIME, datetimeFormatImpl); define(FORMAT_TIMESTAMP, datetimeFormatImpl); + defineMethod(AGE, BuiltInMethod.AGE.method, NullPolicy.STRICT); + // Boolean operators define(IS_NULL, new IsNullImplementor()); define(IS_NOT_NULL, new IsNotNullImplementor()); diff --git a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java index 52ea3bbef5c4..50530aef675d 100644 --- a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java +++ b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java @@ -60,6 +60,7 @@ import org.apache.commons.codec.binary.Hex; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.codec.language.Soundex; +import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.math3.util.CombinatoricsUtils; import org.apache.commons.text.StringEscapeUtils; import org.apache.commons.text.similarity.LevenshteinDistance; @@ -99,11 +100,13 @@ import java.text.Normalizer; import java.text.ParsePosition; import java.text.SimpleDateFormat; +import java.time.Duration; import java.time.Instant; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.LocalTime; import java.time.OffsetDateTime; +import java.time.Period; import java.time.ZoneId; import java.time.ZoneOffset; import java.time.ZonedDateTime; @@ -7503,4 +7506,112 @@ private enum PartToExtract { AUTHORITY, USERINFO; } + + /** SQL {@code AGE(timestamp1, timestamp2)} function. */ + private static String age(long timestamp1, long timestamp2) { + // Convert timestamps to ZonedDateTime objects using UTC to avoid timezone issues + Instant instant1 = Instant.ofEpochMilli(timestamp1); + Instant instant2 = Instant.ofEpochMilli(timestamp2); + + ZonedDateTime dateTime1 = ZonedDateTime.ofInstant(instant1, ZoneOffset.UTC); + ZonedDateTime dateTime2 = ZonedDateTime.ofInstant(instant2, ZoneOffset.UTC); + + // Check if the original timestamps are in the correct order + boolean isNegative = timestamp1 < timestamp2; + + // Ensure dateTime1 is later than dateTime2 for consistent calculation + if (dateTime1.isBefore(dateTime2)) { + ZonedDateTime temp = dateTime1; + dateTime1 = dateTime2; + dateTime2 = temp; + } + + // Calculate period (years, months, days) + Period period = Period.between(dateTime2.toLocalDate(), dateTime1.toLocalDate()); + + // Calculate duration (hours, minutes, seconds, milliseconds) + Duration duration = Duration.between(dateTime2, dateTime1); + + // Adjust for possible day overflow when time part is negative + if (dateTime1.toLocalTime().isBefore(dateTime2.toLocalTime())) { + period = period.minusDays(1); + duration = duration.plusDays(1); + } + + // Extract components + int years = period.getYears(); + int months = period.getMonths(); + int days = period.getDays(); + + long hours = duration.toHours() % 24; + long minutes = duration.toMinutes() % 60; + long seconds = duration.getSeconds() % 60; + long millis = duration.toMillis() % 1000; + + // Apply negative sign if needed + if (isNegative) { + years = -years; + months = -months; + days = -days; + } + + StringBuilder sb = new StringBuilder(); + if (years != 0) { + sb = + Math.abs(years) > 1 ? sb.append(years).append(" years ") + : sb.append(years).append(" year "); + } + if (months != 0) { + sb = + Math.abs(months) > 1 ? sb.append(months).append(" mons ") + : sb.append(months).append(" mon "); + } + if (days != 0) { + sb = + Math.abs(days) > 1 ? sb.append(days).append(" days ") + : sb.append(days).append(" day "); + } + + + // Add negative sign if needed for time part + if (isNegative && (hours != 0 || minutes != 0 || seconds != 0)) { + sb.append("-"); + } + if (millis != 0) { + String millisString = BigDecimal.valueOf(millis) + .divide(BigDecimal.valueOf(1000)) + .stripTrailingZeros() + .toPlainString().substring(2); + sb.append( + String.format(Locale.ROOT, "%02d:%02d:%02d.%s", hours, minutes, seconds, + millisString)); + } else if (ObjectUtils.isNotEmpty(sb) + && hours == 0 && minutes == 0 && seconds == 0 && millis == 0) { + return sb.toString().trim(); + } else { + sb.append(String.format(Locale.ROOT, "%02d:%02d:%02d", hours, minutes, seconds)); + } + return sb.toString().trim(); + } + + /** SQL {@code AGE(timestamp1, timestamp2)} function. Supports 1 or 2 timestamp arguments. */ + public static String age(long... timestamps) { + if (timestamps.length == 0) { + throw new IllegalArgumentException("AGE function requires at least one timestamp argument"); + } + + if (timestamps.length == 1) { + // Single parameter version: calculate age relative to current time + long timestamp = timestamps[0]; + // Use the actual current timestamp (including time component) in UTC + long currentTimestamp = Instant.now().toEpochMilli(); + // Call the two-parameter version with current timestamp and input timestamp + return age(currentTimestamp, timestamp); + } else if (timestamps.length == 2) { + // Two parameter version: calculate age between two timestamps + return age(timestamps[0], timestamps[1]); + } else { + throw new IllegalArgumentException("AGE function supports only 1 or 2 timestamp arguments"); + } + } } diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java index 83f8787fd578..947223b682a8 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java @@ -2784,4 +2784,21 @@ private static RelDataType deriveTypeMapFromEntries(SqlOperatorBinding opBinding public static final SqlFunction RANDOM = SqlStdOperatorTable.RAND .withName("RANDOM") .withOperandTypeChecker(OperandTypes.NILADIC); + + /** + * AGE function for PostgreSQL. + * Returns a human-readable VARCHAR describing the interval between + * one or two timestamps (for example, + * "3 years 11 mons 24 days 0 hours 0 mins 0.0 secs"). + * + * @see PostgreSQL AGE + */ + @LibraryOperator(libraries = {POSTGRESQL}, exceptLibraries = {REDSHIFT}) + public static final SqlBasicFunction AGE = + SqlBasicFunction.create("AGE", + ReturnTypes.VARCHAR_NULLABLE, + OperandTypes.or( + OperandTypes.family(SqlTypeFamily.TIMESTAMP), + OperandTypes.family(SqlTypeFamily.TIMESTAMP, SqlTypeFamily.TIMESTAMP)), + SqlFunctionCategory.TIMEDATE); } diff --git a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java index cfeb4fe77c42..221ff5f71371 100644 --- a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java +++ b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java @@ -981,7 +981,8 @@ public enum BuiltInMethod { ImmutableBitSet.class), FUNCTIONAL_DEPENDENCY_DETERMINANTS(FunctionalDependency.class, "determinants", ImmutableBitSet.class), - FUNCTIONAL_DEPENDENCY_GET_FDS(FunctionalDependency.class, "getFDs"); + FUNCTIONAL_DEPENDENCY_GET_FDS(FunctionalDependency.class, "getFDs"), + AGE(SqlFunctions.class, "age", long[].class); @SuppressWarnings("ImmutableEnumChecker") public final Method method; diff --git a/site/_docs/reference.md b/site/_docs/reference.md index b922253fe0ab..2668cbf83c40 100644 --- a/site/_docs/reference.md +++ b/site/_docs/reference.md @@ -3076,6 +3076,7 @@ In the following: | b | TO_CODE_POINTS(string) | Converts *string* to an array of integers that represent code points or extended ASCII character values | o p r h | TO_DATE(string, format) | Converts *string* to a date using the format *format* | o p r | TO_TIMESTAMP(string, format) | Converts *string* to a timestamp using the format *format* +| p | AGE(timestamp1 [, timestamp2 ]) | Returns a formatted string representing the difference between timestamps (for example, "3 years 11 mons 24 days 0 hours 0 mins 0.0 secs"), not an interval type. With one argument, returns the difference between the current timestamp at midnight UTC and the specified timestamp. With two arguments, returns the difference between *timestamp1* and *timestamp2* | b o p r s | TRANSLATE(expr, fromString, toString) | Returns *expr* with all occurrences of each character in *fromString* replaced by its corresponding character in *toString*. Characters in *expr* that are not in *fromString* are not replaced | b | TRUNC(numeric1 [, integer2 ]) | Truncates *numeric1* to optionally *integer2* (if not specified 0) places right to the decimal point | q | TRY_CAST(value AS type) | Converts *value* to *type*, returning NULL if conversion fails diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java index b27c188620a4..5530349604f0 100644 --- a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java +++ b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java @@ -6948,6 +6948,114 @@ void checkRegexpExtract(SqlOperatorFixture f0, FunctionAlias functionAlias) { f.checkNull("json_storage_size(cast(null as varchar))"); } + /** Test case for + * [CALCITE-7337] + * Add age function (enabled in PostgreSQL library). */ + @Test void testAgePg() { + final SqlOperatorFixture f0 = fixture(); + f0.checkFails("^age(timestamp '2023-12-25', timestamp '2020-01-01')^", + "No match found for function signature AGE\\(, \\)", false); + + final SqlOperatorFixture f = f0.withLibrary(SqlLibrary.POSTGRESQL); + + // Test illegal timestamp argument + f.checkFails("age(^timestamp 'abc'^, timestamp '2023-12-25')", + "Illegal TIMESTAMP literal 'abc': not in format 'yyyy-MM-dd HH:mm:ss'", + false); + f.checkFails("age(timestamp '2023-12-25', ^timestamp 'invalid-date'^)", + "Illegal TIMESTAMP literal 'invalid-date': not in format 'yyyy-MM-dd HH:mm:ss'", + false); + f.checkFails("age(^timestamp '2023-12-25 25:61:61'^, timestamp '2023-12-25')", + "Illegal TIMESTAMP literal '2023-12-25 25:61:61': not in format 'yyyy-MM-dd HH:mm:ss'", + false); + f.checkFails("age(^timestamp '2023-02-30'^, timestamp '2023-12-25')", + "Illegal TIMESTAMP literal '2023-02-30': not in format 'yyyy-MM-dd HH:mm:ss'", + false); + f.checkFails("age(^timestamp ''^)", + "Illegal TIMESTAMP literal '': not in format 'yyyy-MM-dd HH:mm:ss'", + false); + f.checkFails("age(^timestamp '2023-13-25 12:00:00'^)", + "Illegal TIMESTAMP literal '2023-13-25 12:00:00': not in format 'yyyy-MM-dd HH:mm:ss'", + false); + + // Test two timestamp arguments + f.checkScalar("age(timestamp '2023-12-25', timestamp '2020-01-01')", + "3 years 11 mons 24 days", + "VARCHAR NOT NULL"); + f.checkScalar("age(timestamp '2023-01-01', timestamp '2023-01-01')", + "00:00:00", + "VARCHAR NOT NULL"); + f.checkScalar("age(timestamp '2020-01-01', timestamp '2023-12-25')", + "-3 years -11 mons -24 days", + "VARCHAR NOT NULL"); + f.checkScalar("age(timestamp '2023-02-01', timestamp '2023-01-31')", + "1 day", + "VARCHAR NOT NULL"); + f.checkScalar("age(timestamp '2023-12-26 14:30:00', timestamp '2023-12-25 14:30:00')", + "1 day", + "VARCHAR NOT NULL"); + f.checkScalar("age(timestamp '2023-12-25 00:00:00', timestamp '2020-01-01 23:59:59')", + "3 years 11 mons 23 days 00:00:01", + "VARCHAR NOT NULL"); + + // Test single timestamp argument (relative to current time) + f.checkType("age(timestamp '2023-12-25')", "VARCHAR NOT NULL"); + + // NULL value tests + f.checkNull("age(null, timestamp '2023-12-25')"); + f.checkNull("age(timestamp '2023-12-25', null)"); + f.checkNull("age(null, null)"); + f.checkNull("age(null)"); + + // Boundary date tests (Unix epoch time) + f.checkScalar("age(timestamp '1970-01-01', timestamp '1970-01-01')", + "00:00:00", + "VARCHAR NOT NULL"); + f.checkScalar("age(timestamp '1970-01-02', timestamp '1970-01-01')", + "1 day", + "VARCHAR NOT NULL"); + + // Time boundary tests (start and end of day) + f.checkScalar("age(timestamp '2023-12-25 23:59:59', timestamp '2023-12-25 00:00:00')", + "23:59:59", + "VARCHAR NOT NULL"); + f.checkScalar("age(timestamp '2023-12-26 00:00:00', timestamp '2023-12-25 23:59:59')", + "00:00:01", + "VARCHAR NOT NULL"); + + // Leap year tests + f.checkScalar("age(timestamp '2024-02-29', timestamp '2023-02-28')", + "1 year 1 day", + "VARCHAR NOT NULL"); + f.checkScalar("age(timestamp '2024-03-01', timestamp '2023-02-28')", + "1 year 2 days", + "VARCHAR NOT NULL"); + + // Month boundary tests (across months) + f.checkScalar("age(timestamp '2023-03-01', timestamp '2023-02-01')", + "1 mon", + "VARCHAR NOT NULL"); + f.checkScalar("age(timestamp '2023-03-31', timestamp '2023-02-28')", + "1 mon 3 days", + "VARCHAR NOT NULL"); + + // Year boundary tests + f.checkScalar("age(timestamp '2024-01-01', timestamp '2023-01-01')", + "1 year", + "VARCHAR NOT NULL"); + f.checkScalar("age(timestamp '2024-01-01', timestamp '2023-12-31')", + "1 day", + "VARCHAR NOT NULL"); + + // Actual execution test for single parameter version + f.checkType("age(timestamp '1970-01-01')", "VARCHAR NOT NULL"); + + // Millisecond precision tests + f.checkScalar("age(timestamp '2023-12-25 12:00:00.500', timestamp '2023-12-25 12:00:00.000')", + "00:00:00.5", + "VARCHAR NOT NULL"); + } + @Test void testJsonType() { final SqlOperatorFixture f = fixture(); f.setFor(SqlLibraryOperators.JSON_TYPE, VmName.EXPAND); From 6d428f6413e52120949b33622a66da0c6bcee1e9 Mon Sep 17 00:00:00 2001 From: Alessandro Solimando Date: Fri, 2 Jan 2026 19:18:22 +0100 Subject: [PATCH 088/562] [CALCITE-7351] Make getMaxNumericScale() and getMaxNumericPrecision() final Replace them with getMaxPrecision(SqlTypeName.DECIMAL) and getMaxScale(SqlTypeName.DECIMAL), respectively. --- .../calcite/rel/type/RelDataTypeSystem.java | 52 +++++-------------- .../rel/type/RelDataTypeSystemImpl.java | 17 +++--- .../sql/dialect/ClickHouseSqlDialect.java | 4 -- .../calcite/sql/dialect/DuckDBSqlDialect.java | 4 -- .../sql/dialect/PhoenixSqlDialect.java | 4 -- .../calcite/sql/dialect/PrestoSqlDialect.java | 4 -- .../sql/dialect/RedshiftSqlDialect.java | 8 --- .../rel/rel2sql/RelToSqlConverterTest.java | 8 --- .../sql/type/RelDataTypeSystemTest.java | 20 +++---- .../apache/calcite/tools/FrameworksTest.java | 10 ---- 10 files changed, 27 insertions(+), 104 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeSystem.java b/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeSystem.java index 5becafc2b74b..c9d6186747be 100644 --- a/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeSystem.java +++ b/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeSystem.java @@ -96,53 +96,29 @@ public interface RelDataTypeSystem { * * @deprecated Replaced by {@link #getMaxScale}(DECIMAL). * - *

    From Calcite release 1.38 onwards, instead of calling this method, you - * should call {@code getMaxScale(DECIMAL)}. + *

    Instead of calling this method, you should call + * {@code getMaxScale(DECIMAL)}. * - *

    In Calcite release 1.38, if you wish to change the maximum - * scale of {@link SqlTypeName#DECIMAL} values, you should do two things: - * - *

      - *
    • Override the {@link #getMaxScale(SqlTypeName)} method, - * changing its behavior for {@code DECIMAL}; - *
    • Make sure that the implementation of your - * {@code #getMaxNumericScale} method calls - * {@code getMaxScale(DECIMAL)}. - *
    - * - *

    In Calcite release 1.39, Calcite will cease calling this method, - * and will remove the override of the method in - * {@link RelDataTypeSystemImpl}. You should remove all calls to - * and overrides of this method. */ - @Deprecated // calcite will cease calling in 1.39, and removed before 2.0 + *

    If you wish to change the maximum scale of {@link SqlTypeName#DECIMAL} + * values, override the {@link #getMaxScale(SqlTypeName)} method, + * changing its behavior for {@code DECIMAL}. */ + @Deprecated // to be removed before 2.0 default int getMaxNumericScale() { - return 19; + return getMaxScale(SqlTypeName.DECIMAL); } /** Returns the maximum precision of a NUMERIC or DECIMAL type. * Default value is 19. * - * @deprecated Replaced by {@link #getMaxScale}(DECIMAL). + * @deprecated Replaced by {@link #getMaxPrecision}(DECIMAL). * - *

    From Calcite release 1.38 onwards, instead of calling this method, you - * should call {@code getMaxPrecision(DECIMAL)}. - * - *

    In Calcite release 1.38, if you wish to change the maximum - * precision of {@link SqlTypeName#DECIMAL} values, you should do two things: - * - *

      - *
    • Override the {@link #getMaxPrecision(SqlTypeName)} method, - * changing its behavior for {@code DECIMAL}; - *
    • Make sure that the implementation of your - * {@code #getMaxNumericPrecision} method calls - * {@code getMaxPrecision(DECIMAL)}. - *
    + *

    Instead of calling this method, you should call + * {@code getMaxPrecision(DECIMAL)}. * - *

    In Calcite release 1.39, Calcite will cease calling this method, - * and will remove the override of the method in - * {@link RelDataTypeSystemImpl}. You should remove all calls to - * and overrides of this method. */ - @Deprecated // calcite will cease calling in 1.39, and removed before 2.0 + *

    If you wish to change the maximum precision of {@link SqlTypeName#DECIMAL} + * values, override the {@link #getMaxPrecision(SqlTypeName)} method, + * changing its behavior for {@code DECIMAL}. */ + @Deprecated // to be removed before 2.0 default int getMaxNumericPrecision() { return getMaxPrecision(SqlTypeName.DECIMAL); } diff --git a/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeSystemImpl.java b/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeSystemImpl.java index b5eceedccefd..50d66f5e5f9c 100644 --- a/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeSystemImpl.java +++ b/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeSystemImpl.java @@ -46,8 +46,7 @@ public abstract class RelDataTypeSystemImpl implements RelDataTypeSystem { @Override public int getMaxScale(SqlTypeName typeName) { switch (typeName) { case DECIMAL: - // from 1.39, this will be 'return 19;' - return getMaxNumericScale(); + return 19; case INTERVAL_YEAR: case INTERVAL_YEAR_MONTH: case INTERVAL_MONTH: @@ -107,8 +106,7 @@ public abstract class RelDataTypeSystemImpl implements RelDataTypeSystem { case VARBINARY: return RelDataType.PRECISION_NOT_SPECIFIED; case DECIMAL: - // from 1.39, this will be 'return getMaxPrecision(typeName);' - return getMaxNumericPrecision(); + return getMaxPrecision(typeName); case INTERVAL_YEAR: case INTERVAL_YEAR_MONTH: case INTERVAL_MONTH: @@ -187,8 +185,7 @@ public abstract class RelDataTypeSystemImpl implements RelDataTypeSystem { @Override public int getMaxPrecision(SqlTypeName typeName) { switch (typeName) { case DECIMAL: - // from 1.39, this will be 'return 19;' - return getMaxNumericPrecision(); + return 19; case VARCHAR: case CHAR: return 65536; @@ -262,13 +259,13 @@ public abstract class RelDataTypeSystemImpl implements RelDataTypeSystem { } @SuppressWarnings("deprecation") - @Override public int getMaxNumericScale() { - return 19; + @Override public final int getMaxNumericScale() { + return getMaxScale(SqlTypeName.DECIMAL); } @SuppressWarnings("deprecation") - @Override public int getMaxNumericPrecision() { - return 19; + @Override public final int getMaxNumericPrecision() { + return getMaxPrecision(SqlTypeName.DECIMAL); } @Override public RoundingMode roundingMode() { diff --git a/core/src/main/java/org/apache/calcite/sql/dialect/ClickHouseSqlDialect.java b/core/src/main/java/org/apache/calcite/sql/dialect/ClickHouseSqlDialect.java index 884bb81261f8..f33e49bb51d8 100644 --- a/core/src/main/java/org/apache/calcite/sql/dialect/ClickHouseSqlDialect.java +++ b/core/src/main/java/org/apache/calcite/sql/dialect/ClickHouseSqlDialect.java @@ -69,10 +69,6 @@ public class ClickHouseSqlDialect extends SqlDialect { return super.getMaxScale(typeName); } } - - @Override public int getMaxNumericScale() { - return getMaxScale(SqlTypeName.DECIMAL); - } }; public static final SqlDialect.Context DEFAULT_CONTEXT = SqlDialect.EMPTY_CONTEXT diff --git a/core/src/main/java/org/apache/calcite/sql/dialect/DuckDBSqlDialect.java b/core/src/main/java/org/apache/calcite/sql/dialect/DuckDBSqlDialect.java index e7d3333ba3c0..75b4dcc65388 100644 --- a/core/src/main/java/org/apache/calcite/sql/dialect/DuckDBSqlDialect.java +++ b/core/src/main/java/org/apache/calcite/sql/dialect/DuckDBSqlDialect.java @@ -54,10 +54,6 @@ public class DuckDBSqlDialect extends SqlDialect { return super.getMaxScale(typeName); } } - - @Override public int getMaxNumericScale() { - return getMaxScale(SqlTypeName.DECIMAL); - } }; public static final SqlDialect.Context DEFAULT_CONTEXT = SqlDialect.EMPTY_CONTEXT diff --git a/core/src/main/java/org/apache/calcite/sql/dialect/PhoenixSqlDialect.java b/core/src/main/java/org/apache/calcite/sql/dialect/PhoenixSqlDialect.java index 85bc64a89e9d..c3cebb92ae42 100644 --- a/core/src/main/java/org/apache/calcite/sql/dialect/PhoenixSqlDialect.java +++ b/core/src/main/java/org/apache/calcite/sql/dialect/PhoenixSqlDialect.java @@ -55,10 +55,6 @@ public class PhoenixSqlDialect extends SqlDialect { return super.getMaxScale(typeName); } } - - @Override public int getMaxNumericScale() { - return getMaxScale(SqlTypeName.DECIMAL); - } }; public static final SqlDialect.Context DEFAULT_CONTEXT = SqlDialect.EMPTY_CONTEXT diff --git a/core/src/main/java/org/apache/calcite/sql/dialect/PrestoSqlDialect.java b/core/src/main/java/org/apache/calcite/sql/dialect/PrestoSqlDialect.java index cd1aa5f5ec33..1d8479aa5811 100644 --- a/core/src/main/java/org/apache/calcite/sql/dialect/PrestoSqlDialect.java +++ b/core/src/main/java/org/apache/calcite/sql/dialect/PrestoSqlDialect.java @@ -84,10 +84,6 @@ public class PrestoSqlDialect extends SqlDialect { return super.getMaxScale(typeName); } } - - @Override public int getMaxNumericScale() { - return getMaxScale(SqlTypeName.DECIMAL); - } }; public static final Context DEFAULT_CONTEXT = SqlDialect.EMPTY_CONTEXT diff --git a/core/src/main/java/org/apache/calcite/sql/dialect/RedshiftSqlDialect.java b/core/src/main/java/org/apache/calcite/sql/dialect/RedshiftSqlDialect.java index 0875826a253a..7caf2ab2be24 100644 --- a/core/src/main/java/org/apache/calcite/sql/dialect/RedshiftSqlDialect.java +++ b/core/src/main/java/org/apache/calcite/sql/dialect/RedshiftSqlDialect.java @@ -49,10 +49,6 @@ public class RedshiftSqlDialect extends SqlDialect { } } - @Override public int getMaxNumericPrecision() { - return getMaxPrecision(SqlTypeName.DECIMAL); - } - @Override public int getMaxScale(SqlTypeName typeName) { switch (typeName) { case DECIMAL: @@ -61,10 +57,6 @@ public class RedshiftSqlDialect extends SqlDialect { return super.getMaxScale(typeName); } } - - @Override public int getMaxNumericScale() { - return getMaxScale(SqlTypeName.DECIMAL); - } }; public static final SqlDialect.Context DEFAULT_CONTEXT = SqlDialect.EMPTY_CONTEXT diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java index 27e9c23fecc3..2c34707258a1 100644 --- a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java +++ b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java @@ -11380,10 +11380,6 @@ Sql withPostgresqlModifiedDecimalTypeSystem() { new PostgresqlSqlDialect(PostgresqlSqlDialect.DEFAULT_CONTEXT .withDataTypeSystem( new RelDataTypeSystemImpl() { - @Override public int getMaxNumericScale() { - return getMaxScale(SqlTypeName.DECIMAL); - } - @Override public int getMaxScale(SqlTypeName typeName) { switch (typeName) { case DECIMAL: @@ -11393,10 +11389,6 @@ Sql withPostgresqlModifiedDecimalTypeSystem() { } } - @Override public int getMaxNumericPrecision() { - return getMaxPrecision(SqlTypeName.DECIMAL); - } - @Override public int getMaxPrecision(SqlTypeName typeName) { switch (typeName) { case DECIMAL: diff --git a/core/src/test/java/org/apache/calcite/sql/type/RelDataTypeSystemTest.java b/core/src/test/java/org/apache/calcite/sql/type/RelDataTypeSystemTest.java index f21779f2c10a..2a96292eade8 100644 --- a/core/src/test/java/org/apache/calcite/sql/type/RelDataTypeSystemTest.java +++ b/core/src/test/java/org/apache/calcite/sql/type/RelDataTypeSystemTest.java @@ -127,15 +127,15 @@ private static final class CustomTypeSystem extends RelDataTypeSystemImpl { return type1; } - @Override public int getMaxNumericPrecision() { - return 38; - } - @Override public int getMaxPrecision(SqlTypeName typeName) { - if (typeName == SqlTypeName.TIMESTAMP) { + switch (typeName) { + case DECIMAL: + return 38; + case TIMESTAMP: return CUSTOM_MAX_TIMESTAMP_PRECISION; + default: + return super.getMaxPrecision(typeName); } - return super.getMaxPrecision(typeName); } } @@ -209,10 +209,6 @@ static class Fixture extends SqlTypeFixture { * Custom type system class that overrides the default max precision and max scale. */ final class CustomTypeSystem extends RelDataTypeSystemImpl { - @Override public int getMaxNumericPrecision() { - return getMaxPrecision(SqlTypeName.DECIMAL); - } - @Override public int getMaxPrecision(SqlTypeName typeName) { switch (typeName) { case DECIMAL: @@ -222,10 +218,6 @@ final class CustomTypeSystem extends RelDataTypeSystemImpl { } } - @Override public int getMaxNumericScale() { - return getMaxScale(SqlTypeName.DECIMAL); - } - @Override public int getMaxScale(SqlTypeName typeName) { switch (typeName) { case DECIMAL: diff --git a/core/src/test/java/org/apache/calcite/tools/FrameworksTest.java b/core/src/test/java/org/apache/calcite/tools/FrameworksTest.java index ed93b27c3da9..b413cc8b0125 100644 --- a/core/src/test/java/org/apache/calcite/tools/FrameworksTest.java +++ b/core/src/test/java/org/apache/calcite/tools/FrameworksTest.java @@ -530,11 +530,6 @@ public static class HiveLikeTypeSystem extends RelDataTypeSystemImpl { private HiveLikeTypeSystem() {} - @Override public int getMaxNumericPrecision() { - assert super.getMaxNumericPrecision() == 19; - return getMaxPrecision(SqlTypeName.DECIMAL); - } - @Override public int getMaxPrecision(SqlTypeName typeName) { switch (typeName) { case DECIMAL: @@ -550,11 +545,6 @@ private HiveLikeTypeSystem() {} public static class HiveLikeTypeSystem2 extends RelDataTypeSystemImpl { public HiveLikeTypeSystem2() {} - @Override public int getMaxNumericPrecision() { - assert super.getMaxNumericPrecision() == 19; - return getMaxPrecision(SqlTypeName.DECIMAL); - } - @Override public int getMaxPrecision(SqlTypeName typeName) { switch (typeName) { case DECIMAL: From 8195f614bbf9667af76c2455eb1dc38f05fc7214 Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Tue, 6 Jan 2026 08:47:13 +0800 Subject: [PATCH 089/562] [CALCITE-5093] Quantified comparison operators (e.g. ANY) should support ARRAY arguments --- .../calcite/test/SqlToRelConverterTest.java | 8 ++++++++ .../calcite/test/SqlToRelConverterTest.xml | 10 ++++++++++ core/src/test/resources/sql/some.iq | 18 ++++++++++++++++++ 3 files changed, 36 insertions(+) diff --git a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java index b62b9c247ff4..3e7d178ee605 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java @@ -5705,6 +5705,14 @@ void checkUserDefinedOrderByOver(NullCollation nullCollation) { expr(expr).ok(); } + /** Test case for + * [CALCITE-5093] + * Quantified comparison operators (e.g. ANY) should support ARRAY arguments. */ + @Test void testArrayType() { + String expr = "1 = any(array[1, 2, 3])"; + expr(expr).withExpand(false).ok(); + } + @Test void testFunctionExprInOver() { String sql = "select ename, row_number() over(partition by char_length(ename)\n" + " order by deptno desc) as rn\n" diff --git a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml index 0c8ce2192814..23be5c89da2e 100644 --- a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml @@ -616,6 +616,16 @@ LogicalSort(sort0=[$0], dir0=[ASC]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) })]) LogicalValues(tuples=[[{ 0 }]]) +]]> + + + + + + + + diff --git a/core/src/test/resources/sql/some.iq b/core/src/test/resources/sql/some.iq index 8b7639008194..5c7f6b469f30 100644 --- a/core/src/test/resources/sql/some.iq +++ b/core/src/test/resources/sql/some.iq @@ -968,5 +968,23 @@ where sal > any (4000, 2000); (6 rows) !ok + +# [CALCITE-5093] Quantified comparison operators (e.g. ANY) should support ARRAY arguments +select 1 = any(Array [1, 2, 3]), + 4 = any(Array [1, 2, 3]), + 1 > any(Array [1, 2, 3]), + 1 < any(Array [1, 2, 3]); ++--------+--------+--------+--------+ +| EXPR$0 | EXPR$1 | EXPR$2 | EXPR$3 | ++--------+--------+--------+--------+ +| true | false | false | true | ++--------+--------+--------+--------+ +(1 row) + +!ok +EnumerableCalc(expr#0=[{inputs}], expr#1=[1], expr#2=[2], expr#3=[3], expr#4=[ARRAY($t1, $t2, $t3)], expr#5=[= SOME($t1, $t4)], expr#6=[4], expr#7=[= SOME($t6, $t4)], expr#8=[> SOME($t1, $t4)], expr#9=[< SOME($t1, $t4)], EXPR$0=[$t5], EXPR$1=[$t7], EXPR$2=[$t8], EXPR$3=[$t9]) + EnumerableValues(tuples=[[{ 0 }]]) +!plan + # End some.iq From 15bf12e3997fc81894412a7ce014ab7fd9075744 Mon Sep 17 00:00:00 2001 From: nobigo Date: Tue, 6 Jan 2026 16:15:41 +0800 Subject: [PATCH 090/562] [CALCITE-7355] RelToSqlConverter throws exception when the join condition contains a correlated subquery --- .../rel/rel2sql/RelToSqlConverter.java | 15 ++++++++++++ .../rel/rel2sql/RelToSqlConverterTest.java | 23 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java b/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java index 54b60c63d72f..47834ea9d0e5 100644 --- a/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java +++ b/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java @@ -236,6 +236,7 @@ public Result visit(Join e) { final Result rightResult = visitInput(e, 1).resetAlias(); final Context leftContext = leftResult.qualifiedContext(); final Context rightContext = rightResult.qualifiedContext(); + parseCorrelTable(e, leftContext.implementor().joinContext(leftContext, rightContext)); final SqlNode sqlCondition; final JoinConditionType condType; JoinType joinType = joinType(e.getJoinType()); @@ -1437,6 +1438,20 @@ private void parseCorrelTable(RelNode relNode, Result x) { } } + /** + * Populate correlation table information and stores it in the correlation table map. + * Iterates through all variables in the relational node and maps each {@link CorrelationId} + * to its corresponding {@link Context} information. + * + * @param relNode The relational node containing the variable set to be parsed + * @param context The current context information used to establish mapping with correlation IDs + */ + private void parseCorrelTable(RelNode relNode, Context context) { + for (CorrelationId id : relNode.getVariablesSet()) { + correlTableMap.put(id, context); + } + } + /** Stack frame. */ private static class Frame { private final RelNode parent; diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java index 2c34707258a1..fb975386c0d8 100644 --- a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java +++ b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java @@ -11175,6 +11175,29 @@ private void checkLiteral2(String expression, String expected) { sql(sql).ok(expected); } + /** Test case of + * [CALCITE-7355] + * RelToSqlConverter throws exception + * when the join condition contains a correlated subquery. */ + @Test void testCorrelateInJoinCondition() { + final String sql = "SELECT E.EMPNO\n" + + "FROM EMP E\n" + + "JOIN DEPT D ON E.DEPTNO = D.DEPTNO\n" + + "AND D.DEPTNO = (\n" + + " SELECT MIN(D_INNER.DEPTNO)\n" + + " FROM DEPT D_INNER\n" + + " WHERE D_INNER.DEPTNO = E.DEPTNO)"; + final String expected = "SELECT \"EMP\".\"EMPNO\"\n" + + "FROM \"SCOTT\".\"EMP\"\n" + + "INNER JOIN \"SCOTT\".\"DEPT\" ON \"EMP\".\"DEPTNO\" = \"DEPT\".\"DEPTNO\"" + + " AND \"DEPT\".\"DEPTNO\" = (SELECT MIN(\"DEPTNO\")\n" + + "FROM \"SCOTT\".\"DEPT\"\nWHERE \"DEPTNO\" = \"EMP\".\"DEPTNO\")"; + sql(sql) + .schema(CalciteAssert.SchemaSpec.JDBC_SCOTT) + .withCalcite() + .ok(expected); + } + /** Fluid interface to run tests. */ static class Sql { private final CalciteAssert.SchemaSpec schemaSpec; From 989263ef97f66b913ecc287d697b84525bfaffa8 Mon Sep 17 00:00:00 2001 From: dssysolyatin Date: Wed, 19 Nov 2025 14:25:12 +0200 Subject: [PATCH 091/562] [CALCITE-7301] Add common test for SqlNode unparse after deep copy and missing createCall in operators --- .../sql/babel/SqlBabelCreateTable.java | 36 +++++++++++++++++-- .../apache/calcite/test/BabelParserTest.java | 8 ++++- .../calcite/test/BabelUnParserTest.java | 30 ++++++++++++++++ .../java/org/apache/calcite/sql/SqlMerge.java | 13 ++++++- .../org/apache/calcite/sql/SqlSetOption.java | 2 +- .../org/apache/calcite/sql/SqlUnpivot.java | 16 +++++++-- .../calcite/sql/ddl/SqlCreateTable.java | 18 ++++++---- .../apache/calcite/sql/ddl/SqlCreateView.java | 4 +-- .../calcite/sql/fun/SqlBasicOperator.java | 15 +++++--- site/_docs/history.md | 6 ++++ .../calcite/sql/parser/SqlParserTest.java | 22 ++++++++++++ 11 files changed, 149 insertions(+), 21 deletions(-) create mode 100644 babel/src/test/java/org/apache/calcite/test/BabelUnParserTest.java diff --git a/babel/src/main/java/org/apache/calcite/sql/babel/SqlBabelCreateTable.java b/babel/src/main/java/org/apache/calcite/sql/babel/SqlBabelCreateTable.java index 511bef36872b..652f0a92e372 100644 --- a/babel/src/main/java/org/apache/calcite/sql/babel/SqlBabelCreateTable.java +++ b/babel/src/main/java/org/apache/calcite/sql/babel/SqlBabelCreateTable.java @@ -17,17 +17,39 @@ package org.apache.calcite.sql.babel; import org.apache.calcite.sql.SqlIdentifier; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlLiteral; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlNodeList; +import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.SqlWriter; import org.apache.calcite.sql.ddl.SqlCreateTable; +import org.apache.calcite.sql.fun.SqlBasicOperator; import org.apache.calcite.sql.parser.SqlParserPos; +import org.apache.calcite.util.ImmutableNullableList; + +import org.checkerframework.checker.nullness.qual.Nullable; + +import java.util.List; + +import static java.util.Objects.requireNonNull; /** * Parse tree for {@code CREATE TABLE} statement, with extensions for particular * SQL dialects supported by Babel. */ public class SqlBabelCreateTable extends SqlCreateTable { + private static final SqlOperator OPERATOR = + SqlBasicOperator.create("CREATE TABLE", SqlKind.CREATE_TABLE).withCallFactory( + (operator, functionQualifier, pos, operands) -> + new SqlBabelCreateTable(pos, + requireNonNull((SqlLiteral) operands[0]).booleanValue(), + requireNonNull((SqlLiteral) operands[1]).symbolValue(TableCollectionType.class), + requireNonNull((SqlLiteral) operands[2]).booleanValue(), + requireNonNull((SqlLiteral) operands[3]).booleanValue(), + (SqlIdentifier) requireNonNull(operands[4]), + (SqlNodeList) operands[5], operands[6])); + private final TableCollectionType tableCollectionType; // CHECKSTYLE: IGNORE 2; can't use 'volatile' because it is a Java keyword // but checkstyle does not like trailing '_'. @@ -36,13 +58,21 @@ public class SqlBabelCreateTable extends SqlCreateTable { /** Creates a SqlBabelCreateTable. */ public SqlBabelCreateTable(SqlParserPos pos, boolean replace, TableCollectionType tableCollectionType, boolean volatile_, - boolean ifNotExists, SqlIdentifier name, SqlNodeList columnList, - SqlNode query) { - super(pos, replace, ifNotExists, name, columnList, query); + boolean ifNotExists, SqlIdentifier name, @Nullable SqlNodeList columnList, + @Nullable SqlNode query) { + super(OPERATOR, pos, replace, ifNotExists, name, columnList, query); this.tableCollectionType = tableCollectionType; this.volatile_ = volatile_; } + @SuppressWarnings("nullness") + @Override public List getOperandList() { + return ImmutableNullableList.of(SqlLiteral.createBoolean(getReplace(), pos), + SqlLiteral.createSymbol(tableCollectionType, pos), + SqlLiteral.createBoolean(volatile_, pos), + SqlLiteral.createBoolean(ifNotExists, pos), name, columnList, query); + } + @Override public void unparse(SqlWriter writer, int leftPrec, int rightPrec) { writer.keyword("CREATE"); switch (tableCollectionType) { diff --git a/babel/src/test/java/org/apache/calcite/test/BabelParserTest.java b/babel/src/test/java/org/apache/calcite/test/BabelParserTest.java index e3d49d2bb6e6..f458a174da6c 100644 --- a/babel/src/test/java/org/apache/calcite/test/BabelParserTest.java +++ b/babel/src/test/java/org/apache/calcite/test/BabelParserTest.java @@ -40,6 +40,7 @@ import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.hasToString; +import static org.junit.jupiter.api.Assumptions.assumeFalse; /** * Tests the "Babel" SQL parser, that understands all dialects of SQL. @@ -415,7 +416,12 @@ private void checkParseInfixCast(String sqlType) { } @Test void testPostgresSqlSetOption() { - SqlParserFixture f = fixture().withDialect(PostgresqlSqlDialect.DEFAULT); + // UnparsingTesterImpl has a check where it unparses a SqlNode into a SQL string + // using the calcite dialect, and then parses it back into a SqlNode. + // But the SQL string produced by the calcite dialect for `SET` cannot always be parsed back. + assumeFalse(fixture().tester.isUnparserTest()); + SqlParserFixture f = fixture() + .withDialect(PostgresqlSqlDialect.DEFAULT); f.sql("SET SESSION autovacuum = true") .ok("SET \"autovacuum\" = TRUE"); f.sql("SET SESSION autovacuum = DEFAULT") diff --git a/babel/src/test/java/org/apache/calcite/test/BabelUnParserTest.java b/babel/src/test/java/org/apache/calcite/test/BabelUnParserTest.java new file mode 100644 index 000000000000..99d6d2286e52 --- /dev/null +++ b/babel/src/test/java/org/apache/calcite/test/BabelUnParserTest.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.test; + +import org.apache.calcite.sql.parser.SqlParserFixture; + +/** + * Extension to {@link BabelParserTest} that ensures that every expression can + * un-parse successfully. + */ +public class BabelUnParserTest extends BabelParserTest { + @Override public SqlParserFixture fixture() { + return super.fixture() + .withTester(new UnparsingTesterImpl()); + } +} diff --git a/core/src/main/java/org/apache/calcite/sql/SqlMerge.java b/core/src/main/java/org/apache/calcite/sql/SqlMerge.java index cbed8e40a8de..ddadf32959e1 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlMerge.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlMerge.java @@ -28,13 +28,24 @@ import java.util.List; +import static java.util.Objects.requireNonNull; + /** * A SqlMerge is a node of a parse tree which represents a MERGE * statement. */ public class SqlMerge extends SqlCall { public static final SqlSpecialOperator OPERATOR = - new SqlSpecialOperator("MERGE", SqlKind.MERGE); + new SqlSpecialOperator("MERGE", SqlKind.MERGE) { + @Override public SqlCall createCall(final @Nullable SqlLiteral functionQualifier, + final SqlParserPos pos, + final @Nullable SqlNode... operands) { + return new SqlMerge(pos, requireNonNull(operands[0]), requireNonNull(operands[1]), + requireNonNull(operands[2]), + (SqlUpdate) operands[3], (SqlInsert) operands[4], + (SqlSelect) operands[5], (SqlIdentifier) operands[6]); + } + }; SqlNode targetTable; SqlNode condition; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlSetOption.java b/core/src/main/java/org/apache/calcite/sql/SqlSetOption.java index 3a7206eb2d99..689ba20642ba 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlSetOption.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlSetOption.java @@ -143,7 +143,7 @@ public SqlSetOption(SqlParserPos pos, @Nullable String scope, SqlIdentifier name } else { operandList.add(new SqlIdentifier(scope, SqlParserPos.ZERO)); } - operandList.add(name); + operandList.add(nameAsSqlNode); operandList.add(value); return ImmutableNullableList.copyOf(operandList); } diff --git a/core/src/main/java/org/apache/calcite/sql/SqlUnpivot.java b/core/src/main/java/org/apache/calcite/sql/SqlUnpivot.java index 528b9ee57db5..732e6d4e6a19 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlUnpivot.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlUnpivot.java @@ -67,10 +67,10 @@ public SqlUnpivot(SqlParserPos pos, SqlNode query, boolean includeNulls, SqlNodeList measureList, SqlNodeList axisList, SqlNodeList inList) { super(pos); this.query = requireNonNull(query, "query"); - this.includeNulls = includeNulls; this.measureList = requireNonNull(measureList, "measureList"); this.axisList = requireNonNull(axisList, "axisList"); this.inList = requireNonNull(inList, "inList"); + this.includeNulls = includeNulls; } //~ Methods ---------------------------------------------------------------- @@ -80,7 +80,8 @@ public SqlUnpivot(SqlParserPos pos, SqlNode query, boolean includeNulls, } @Override public List getOperandList() { - return ImmutableNullableList.of(query, measureList, axisList, inList); + return ImmutableNullableList.of(query, measureList, axisList, inList, + SqlLiteral.createBoolean(includeNulls, SqlParserPos.ZERO)); } @SuppressWarnings("nullness") @@ -176,5 +177,16 @@ static class Operator extends SqlSpecialOperator { Operator(SqlKind kind) { super(kind.name(), kind); } + + @Override public SqlCall createCall( + @Nullable SqlLiteral functionQualifier, + SqlParserPos pos, + @Nullable SqlNode... operands) { + return new SqlUnpivot(pos, requireNonNull(operands[0]), + requireNonNull((SqlLiteral) operands[4]).booleanValue(), + requireNonNull((SqlNodeList) operands[1]), + requireNonNull((SqlNodeList) operands[2]), + requireNonNull((SqlNodeList) operands[3])); + } } } diff --git a/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateTable.java b/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateTable.java index 1bf283bf0cd6..51d4469dcf42 100644 --- a/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateTable.java +++ b/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateTable.java @@ -47,24 +47,30 @@ public class SqlCreateTable extends SqlCreate { new SqlSpecialOperator("CREATE TABLE", SqlKind.CREATE_TABLE) { @Override public SqlCall createCall(@Nullable SqlLiteral functionQualifier, SqlParserPos pos, @Nullable SqlNode... operands) { - return new SqlCreateTable(pos, + return new SqlCreateTable(OPERATOR, pos, ((SqlLiteral) requireNonNull(operands[0], "replace")).booleanValue(), ((SqlLiteral) requireNonNull(operands[1], "ifNotExists")).booleanValue(), (SqlIdentifier) requireNonNull(operands[2], "name"), - (SqlNodeList) requireNonNull(operands[3], "columnList"), - operands[4]); + (SqlNodeList) operands[3], operands[4]); } }; /** Creates a SqlCreateTable. */ - protected SqlCreateTable(SqlParserPos pos, boolean replace, boolean ifNotExists, - SqlIdentifier name, @Nullable SqlNodeList columnList, @Nullable SqlNode query) { - super(OPERATOR, pos, replace, ifNotExists); + protected SqlCreateTable(SqlOperator operator, SqlParserPos pos, boolean replace, + boolean ifNotExists, SqlIdentifier name, @Nullable SqlNodeList columnList, + @Nullable SqlNode query) { + super(operator, pos, replace, ifNotExists); this.name = requireNonNull(name, "name"); this.columnList = columnList; // may be null this.query = query; // for "CREATE TABLE ... AS query"; may be null } + /** Creates a SqlCreateTable. */ + protected SqlCreateTable(SqlParserPos pos, boolean replace, boolean ifNotExists, + SqlIdentifier name, @Nullable SqlNodeList columnList, @Nullable SqlNode query) { + this(OPERATOR, pos, replace, ifNotExists, name, columnList, query); + } + @SuppressWarnings("nullness") @Override public List getOperandList() { return ImmutableNullableList.of( diff --git a/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateView.java b/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateView.java index 38297f83e6a3..53a688acfe2c 100644 --- a/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateView.java +++ b/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateView.java @@ -50,8 +50,8 @@ public class SqlCreateView extends SqlCreate { return new SqlCreateView(pos, ((SqlLiteral) requireNonNull(operands[0], "replace")).booleanValue(), (SqlIdentifier) requireNonNull(operands[1], "name"), - (SqlNodeList) operands[3], - requireNonNull(operands[4], "query")); + (SqlNodeList) operands[2], + requireNonNull(operands[3], "query")); } }; diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlBasicOperator.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlBasicOperator.java index e84042b5cdcf..a2dc815ff3e8 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlBasicOperator.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlBasicOperator.java @@ -40,24 +40,29 @@ public final class SqlBasicOperator extends SqlOperator { } /** Private constructor. Use {@link #create}. */ - private SqlBasicOperator(String name, int leftPrecedence, int rightPrecedence, + private SqlBasicOperator(String name, SqlKind kind, int leftPrecedence, int rightPrecedence, SqlCallFactory callFactory) { - super(name, SqlKind.OTHER, leftPrecedence, rightPrecedence, + super(name, kind, leftPrecedence, rightPrecedence, ReturnTypes.BOOLEAN, InferTypes.RETURN_TYPE, OperandTypes.ANY, callFactory); } public static SqlBasicOperator create(String name) { - return new SqlBasicOperator(name, 0, 0, + return new SqlBasicOperator(name, SqlKind.OTHER, 0, 0, + SqlCallFactories.SQL_BASIC_CALL_FACTORY); + } + + public static SqlBasicOperator create(String name, SqlKind kind) { + return new SqlBasicOperator(name, kind, 0, 0, SqlCallFactories.SQL_BASIC_CALL_FACTORY); } public SqlBasicOperator withPrecedence(int prec, boolean leftAssoc) { - return new SqlBasicOperator(getName(), leftPrec(prec, leftAssoc), + return new SqlBasicOperator(getName(), getKind(), leftPrec(prec, leftAssoc), rightPrec(prec, leftAssoc), getSqlCallFactory()); } public SqlBasicOperator withCallFactory(SqlCallFactory sqlCallFactory) { - return new SqlBasicOperator(getName(), getLeftPrec(), + return new SqlBasicOperator(getName(), getKind(), getLeftPrec(), getRightPrec(), sqlCallFactory); } } diff --git a/site/_docs/history.md b/site/_docs/history.md index ea6fd1372674..89d91bd2664c 100644 --- a/site/_docs/history.md +++ b/site/_docs/history.md @@ -49,6 +49,12 @@ other software versions as specified in gradle.properties. #### Breaking Changes {: #breaking-1-42-0} +* [CALCITE-7301] +Prior to this change, most `SqlNode`s in the `org.apache.calcite.sql.ddl` package could not be unparsed +when created with `SqlOperator#createCall`. To fix this, those `SqlNode`s now implement their own `SqlOperator`. +`SqlNode#getOperandList()` now returns all operands required by these operators; the number and order may differ from before. +The same applies to `SqlBabelCreateTable` and `SqlUnpivot`. + * [CALCITE-6942] Rename the method `decorrelateFetchOneSort` to `decorrelateSortWithRowNumber`. diff --git a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java index 613fe9b7de14..cfea5b6ce0e9 100644 --- a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java +++ b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java @@ -9886,6 +9886,17 @@ static void checkList(SqlNodeList sqlNodeList, } } + static SqlNode deepCopy(SqlNode sqlNode) { + return sqlNode.accept(new SqlShuttle() { + @Override public @Nullable SqlNode visit(final SqlCall call) { + // Handler always creates a new copy of 'call' + CallCopyingArgHandler argHandler = new CallCopyingArgHandler(call, true); + call.getOperator().acceptCall(this, call, false, argHandler); + return argHandler.result(); + } + }); + } + @Override public void checkList(SqlTestFactory factory, StringAndPos sap, @Nullable SqlDialect dialect, UnaryOperator converter, List expected) { @@ -9915,6 +9926,12 @@ static void checkList(SqlNodeList sqlNodeList, final Random random = new Random(); final String sql3 = toSqlString(sqlNodeList, randomize(random)); assertThat(sql3, notNullValue()); + + // Make a deep copy of the SqlNodeList, unparse it. + final SqlNodeList sqlNodeList3 = (SqlNodeList) deepCopy(sqlNodeList); + final String sql4 = toSqlString(sqlNodeList3, simple()); + // Should be the same as we started with. + assertThat(sql4, is(sql1)); } @Override public void check(SqlTestFactory factory, StringAndPos sap, @@ -9959,6 +9976,11 @@ static void checkList(SqlNodeList sqlNodeList, parseStmtAndHandleEx(factory2, sql1, parser -> { }); final String sql4 = sqlNode4.toSqlString(simple()).getSql(); assertThat(sql4, is(sql1)); + + // Make a deep copy of the original SqlNode, unparse it. + final SqlNode sqlNode5 = deepCopy(sqlNode); + final String actual5 = sqlNode5.toSqlString(writerTransform).getSql(); + assertThat(converter.apply(actual5), is(expected)); } @Override public void checkExp(SqlTestFactory factory, StringAndPos sap, From 9d07a156521a1de80e2d30c153b4b032d8a2a77b Mon Sep 17 00:00:00 2001 From: nobigo Date: Mon, 5 Jan 2026 17:01:34 +0800 Subject: [PATCH 092/562] [CALCITE-7336] RelFieldTrimmer generates an incorrect plan when handling correlated sub-query within Filter or Join condition --- .../calcite/sql2rel/RelFieldTrimmerTest.java | 101 ++++++++++++++++++ 1 file changed, 101 insertions(+) diff --git a/core/src/test/java/org/apache/calcite/sql2rel/RelFieldTrimmerTest.java b/core/src/test/java/org/apache/calcite/sql2rel/RelFieldTrimmerTest.java index c23185c8c64e..86623ccb0aa8 100644 --- a/core/src/test/java/org/apache/calcite/sql2rel/RelFieldTrimmerTest.java +++ b/core/src/test/java/org/apache/calcite/sql2rel/RelFieldTrimmerTest.java @@ -44,6 +44,7 @@ import org.apache.calcite.util.Holder; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import org.checkerframework.checker.nullness.qual.Nullable; @@ -725,4 +726,104 @@ public static Frameworks.ConfigBuilder config() { assertThat(trimmed, hasTree(expected)); } + /** + * Test case for + * [CALCITE-7336] + * RelFieldTrimmer generates an incorrect plan + * when handling correlated sub-query within Filter or Join condition. + */ + @Test void testTrimCorrelatedSubqueryInFilterCondition() { + final RelBuilder builder = RelBuilder.create(config().build()); + final Holder<@Nullable RexCorrelVariable> v = Holder.empty(); + RelNode original = builder.scan("EMP") + .variable(v::set) + .filter(ImmutableList.of(v.get().id), + builder.call(SqlStdOperatorTable.GREATER_THAN, builder.field(5), + builder.scalarQuery( + b2 -> builder.scan("EMP").filter( + builder.call(SqlStdOperatorTable.LESS_THAN, + builder.field(3), builder.field(v.get(), "MGR"))) + .project(builder.field(0)) + .aggregate(builder.groupKey(), builder.countStar("c")) + .build()))) + .project(builder.field(0)) + .build(); + + String origTree = "" + + "LogicalProject(EMPNO=[$0])\n" + + " LogicalFilter(condition=[>($5, $SCALAR_QUERY({\n" + + "LogicalAggregate(group=[{}], c=[COUNT()])\n" + + " LogicalFilter(condition=[<($3, $cor0.MGR)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + "}))], variablesSet=[[$cor0]])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + assertThat(original, hasTree(origTree)); + + final RelFieldTrimmer fieldTrimmer = new RelFieldTrimmer(null, builder); + final RelNode trimmed = fieldTrimmer.trim(original); + final String expected = "" + + "LogicalProject(EMPNO=[$0])\n" + + " LogicalFilter(condition=[>($2, $SCALAR_QUERY({\n" + + "LogicalAggregate(group=[{}], c=[COUNT()])\n" + + " LogicalFilter(condition=[<($3, $cor0.MGR)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + "}))], variablesSet=[[$cor0]])\n" + + " LogicalProject(EMPNO=[$0], MGR=[$3], SAL=[$5])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + assertThat(trimmed, hasTree(expected)); + } + + @Test void testTrimCorrelatedSubqueryInJoinCondition() { + final RelBuilder builder = RelBuilder.create(config().build()); + final Holder<@Nullable RexCorrelVariable> v = Holder.empty(); + final RelNode original = + builder.scan("EMP") + .variable(v::set) + .scan("DEPT") + .join(JoinRelType.INNER, + builder.and( + builder.equals( + builder.field(2, 0, "DEPTNO"), + builder.field(2, 1, "DEPTNO")), + builder.call(SqlStdOperatorTable.GREATER_THAN, builder.field(1), + builder.scalarQuery( + b2 -> builder.scan("EMP").filter( + builder.call(SqlStdOperatorTable.LESS_THAN, + builder.field(3), builder.field(v.get(), "MGR"))) + .project(builder.field(0)) + .aggregate(builder.groupKey(), builder.countStar("c")) + .build()))), ImmutableSet.of(v.get().id)) + .project( + builder.field("ENAME"), + builder.field("DNAME")) + .build(); + + String origTree = "" + + "LogicalProject(ENAME=[$1], DNAME=[$9])\n" + + " LogicalJoin(condition=[AND(=($7, $8), >($1, $SCALAR_QUERY({\n" + + "LogicalAggregate(group=[{}], c=[COUNT()])\n" + + " LogicalFilter(condition=[<($3, $cor0.MGR)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + "})))], joinType=[inner], variablesSet=[[$cor0]])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n"; + assertThat(original, hasTree(origTree)); + + final RelFieldTrimmer fieldTrimmer = new RelFieldTrimmer(null, builder); + final RelNode trimmed = fieldTrimmer.trim(original); + + final String expected = "" + + "LogicalProject(ENAME=[$1], DNAME=[$5])\n" + + " LogicalJoin(condition=[AND(=($3, $4), >($1, $SCALAR_QUERY({\n" + + "LogicalAggregate(group=[{}], c=[COUNT()])\n" + + " LogicalFilter(condition=[<($3, $cor0.MGR)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + "})))], joinType=[inner], variablesSet=[[$cor0]])\n" + + " LogicalProject(EMPNO=[$0], ENAME=[$1], MGR=[$3], DEPTNO=[$7])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalProject(DEPTNO=[$0], DNAME=[$1])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n"; + assertThat(trimmed, hasTree(expected)); + } + } From d4483d88ca4eb4dacf805367239ac76424ff0bcb Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Thu, 8 Jan 2026 06:51:48 +0800 Subject: [PATCH 093/562] [CALCITE-7357] Introduce the implementation of rex operator IS DISTINCT FROM --- .../adapter/enumerable/RexImpTable.java | 59 ++++++++++++++----- .../test/enumerable/EnumerableCalcTest.java | 22 +++++++ 2 files changed, 66 insertions(+), 15 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java index 103e91919f19..42afb720fce6 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java @@ -429,6 +429,7 @@ import static org.apache.calcite.sql.fun.SqlStdOperatorTable.INITCAP; import static org.apache.calcite.sql.fun.SqlStdOperatorTable.INTERSECTION; import static org.apache.calcite.sql.fun.SqlStdOperatorTable.IS_A_SET; +import static org.apache.calcite.sql.fun.SqlStdOperatorTable.IS_DISTINCT_FROM; import static org.apache.calcite.sql.fun.SqlStdOperatorTable.IS_EMPTY; import static org.apache.calcite.sql.fun.SqlStdOperatorTable.IS_FALSE; import static org.apache.calcite.sql.fun.SqlStdOperatorTable.IS_JSON_ARRAY; @@ -1040,6 +1041,7 @@ void populate2() { define(IS_FALSE, new IsFalseImplementor()); define(IS_NOT_FALSE, new IsNotFalseImplementor()); define(IS_NOT_DISTINCT_FROM, new IsNotDistinctFromImplementor()); + define(IS_DISTINCT_FROM, new IsDistinctFromImplementor()); // LIKE, ILIKE, RLIKE and SIMILAR defineReflective(LIKE, BuiltInMethod.LIKE.method, @@ -4796,10 +4798,11 @@ private static class IsNullImplementor extends AbstractRexCallImplementor { } } - /** Implementor for the {@code IS NOT DISTINCT FROM} SQL operator. */ - private static class IsNotDistinctFromImplementor extends AbstractRexCallImplementor { - IsNotDistinctFromImplementor() { - super("is_not_distinct_from", NullPolicy.NONE, false); + /** Base implementation class for the {@code IS DISTINCT FROM} + * and {@code IS NOT DISTINCT FROM} operators. */ + private abstract static class DistinctFromImplementor extends AbstractRexCallImplementor { + DistinctFromImplementor(String variableName, NullPolicy nullPolicy, boolean harmonize) { + super(variableName, nullPolicy, harmonize); } @Override public RexToLixTranslator.Result implement(final RexToLixTranslator translator, @@ -4807,17 +4810,7 @@ private static class IsNotDistinctFromImplementor extends AbstractRexCallImpleme final RexToLixTranslator.Result left = arguments.get(0); final RexToLixTranslator.Result right = arguments.get(1); - // Generated expression: - // left IS NULL ? - // (right IS NULL ? TRUE : FALSE) : -> when left is null - // (right IS NULL ? FALSE : -> when left is not null - // left.equals(right)) -> when both are not null, compare values - final Expression valueExpression = - Expressions.condition(left.isNullVariable, - Expressions.condition(right.isNullVariable, BOXED_TRUE_EXPR, BOXED_FALSE_EXPR), - Expressions.condition(right.isNullVariable, BOXED_FALSE_EXPR, - Expressions.call(BuiltInMethod.OBJECTS_EQUAL.method, - left.valueVariable, right.valueVariable))); + final Expression valueExpression = valueExpression(left, right); BlockBuilder builder = translator.getBlockBuilder(); final ParameterExpression valueVariable = @@ -4835,6 +4828,22 @@ private static class IsNotDistinctFromImplementor extends AbstractRexCallImpleme return new RexToLixTranslator.Result(isNullVariable, valueVariable); } + protected Expression valueExpression(RexToLixTranslator.Result left, + RexToLixTranslator.Result right) { + // Generated expression: + // left IS NULL ? + // (right IS NULL ? TRUE : FALSE) : -> when left is null + // (right IS NULL ? FALSE : -> when left is not null + // left.equals(right)) -> when both are not null, compare values + return Expressions.condition(left.isNullVariable, + Expressions.condition(right.isNullVariable, BOXED_TRUE_EXPR, BOXED_FALSE_EXPR), + Expressions.condition(right.isNullVariable, BOXED_FALSE_EXPR, + Expressions.condition( + Expressions.call(BuiltInMethod.OBJECTS_EQUAL.method, + left.valueVariable, right.valueVariable), + BOXED_TRUE_EXPR, BOXED_FALSE_EXPR))); + } + @Override Expression implementSafe(final RexToLixTranslator translator, final RexCall call, final List argValueList) { throw new IllegalStateException("This implementSafe should not be called," @@ -4842,6 +4851,26 @@ private static class IsNotDistinctFromImplementor extends AbstractRexCallImpleme } } + /** Implementor for the {@code IS NOT DISTINCT FROM} SQL operator. */ + private static class IsNotDistinctFromImplementor extends DistinctFromImplementor { + IsNotDistinctFromImplementor() { + super("is_not_distinct_from", NullPolicy.NONE, false); + } + } + + /** Implementor for the {@code IS DISTINCT FROM} SQL operator. */ + private static class IsDistinctFromImplementor extends DistinctFromImplementor { + IsDistinctFromImplementor() { + super("is_distinct_from", NullPolicy.NONE, false); + } + + @Override protected Expression valueExpression(RexToLixTranslator.Result left, + RexToLixTranslator.Result right) { + return Expressions.condition(super.valueExpression(left, right), + BOXED_FALSE_EXPR, BOXED_TRUE_EXPR); + } + } + /** Implementor for the {@code IS TRUE} SQL operator. */ private static class IsTrueImplementor extends AbstractRexCallImplementor { IsTrueImplementor() { diff --git a/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableCalcTest.java b/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableCalcTest.java index 708f07766294..4b545b677f96 100644 --- a/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableCalcTest.java +++ b/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableCalcTest.java @@ -129,4 +129,26 @@ private CalciteAssert.AssertQuery checkPosixRegex( .planContains("input_value != null && input_value.isEmpty()") .returnsUnordered("$f0=false", "$f0=true"); } + + /** Test case for [CALCITE-7357] + * Introduce the implementation of rex operator IS DISTINCT FROM. */ + @Test public void testIsDistinctFromImplementationDirectly() { + CalciteAssert.that() + .withSchema("s", new ReflectiveSchema(new HrSchema())) + .withRel( + builder -> builder + .scan("s", "emps") + .project( + builder.field("commission"), + builder.call( + SqlStdOperatorTable.IS_DISTINCT_FROM, + builder.field("commission"), + builder.literal(null))) + .build()) + .returnsUnordered( + "commission=1000; $f1=true", + "commission=250; $f1=true", + "commission=500; $f1=true", + "commission=null; $f1=false"); + } } From 3c0afebb0879977b733fcaa276aa76c1ff410ee0 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Fri, 26 Dec 2025 16:45:13 -0800 Subject: [PATCH 094/562] [CALCITE-7196] Create an optimization pass which can convert some cases of Correlate + Unnest to Unnest Signed-off-by: Mihai Budiu --- .../apache/calcite/rel/rules/CoreRules.java | 12 ++ .../rel/rules/UnnestDecorrelateRule.java | 182 ++++++++++++++++++ .../apache/calcite/test/RelOptRulesTest.java | 68 +++++++ .../apache/calcite/test/RelOptRulesTest.xml | 127 ++++++++++++ 4 files changed, 389 insertions(+) create mode 100644 core/src/main/java/org/apache/calcite/rel/rules/UnnestDecorrelateRule.java diff --git a/core/src/main/java/org/apache/calcite/rel/rules/CoreRules.java b/core/src/main/java/org/apache/calcite/rel/rules/CoreRules.java index c604f713580f..fbca06e760fe 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/CoreRules.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/CoreRules.java @@ -17,6 +17,7 @@ package org.apache.calcite.rel.rules; import org.apache.calcite.linq4j.function.Experimental; +import org.apache.calcite.plan.RelOptRule; import org.apache.calcite.plan.RelOptUtil.Exists; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Aggregate; @@ -32,6 +33,7 @@ import org.apache.calcite.rel.core.Sort; import org.apache.calcite.rel.core.TableFunctionScan; import org.apache.calcite.rel.core.TableScan; +import org.apache.calcite.rel.core.Uncollect; import org.apache.calcite.rel.core.Union; import org.apache.calcite.rel.core.Values; import org.apache.calcite.rel.logical.LogicalAggregate; @@ -959,4 +961,14 @@ private CoreRules() {} * into equivalent {@link Union} ALL of GROUP BY operations. */ public static final AggregateGroupingSetsToUnionRule AGGREGATE_GROUPING_SETS_TO_UNION = AggregateGroupingSetsToUnionRule.Config.DEFAULT.toRule(); + + /** Rule that converts a {@link Correlate} after an {@link Uncollect} into a simple + * Uncollect, if possible. */ + public static final RelOptRule UNNEST_DECORRELATE = + UnnestDecorrelateRule.Config.DEFAULT.toRule(); + + /** Rule that converts a {@link Correlate} after an {@link Project} of an + * {@link Uncollect} into a simple Uncollect, if possible. */ + public static final RelOptRule UNNEST_PROJECT_DECORRELATE = + UnnestDecorrelateRule.Config.WITH_PROJECT.toRule(); } diff --git a/core/src/main/java/org/apache/calcite/rel/rules/UnnestDecorrelateRule.java b/core/src/main/java/org/apache/calcite/rel/rules/UnnestDecorrelateRule.java new file mode 100644 index 000000000000..71005b52fd45 --- /dev/null +++ b/core/src/main/java/org/apache/calcite/rel/rules/UnnestDecorrelateRule.java @@ -0,0 +1,182 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.rel.rules; + +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptUtil; +import org.apache.calcite.plan.RelRule; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.Correlate; +import org.apache.calcite.rel.core.CorrelationId; +import org.apache.calcite.rel.core.Project; +import org.apache.calcite.rel.core.Uncollect; +import org.apache.calcite.rel.logical.LogicalValues; +import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.rex.RexCorrelVariable; +import org.apache.calcite.rex.RexFieldAccess; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexUtil; +import org.apache.calcite.tools.RelBuilder; +import org.apache.calcite.util.ImmutableBitSet; + +import org.immutables.value.Value; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static java.util.Objects.requireNonNull; + +/** Convert representations of a projected Unnest that use LogicalCorrelate into + * simple Unnest representations. + * + *

    Original plan: + * LogicalProject // only uses rightmost columns of correlate, outerProject + * LogicalCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{...}]) + * LeftSubquery + * LogicalProject (optional; innerProject) + * Uncollect + * LogicalProject(COL=[$cor0.ARRAY]) + * LogicalValues(tuples=[[{ 0 }]]) + * + *

    is converted to + * + *

    Resulting plan: + * LogicalProject + * LogicalProject (optional) + * Uncollect + * LogicalProject + * LeftSubquery + */ +@Value.Enclosing +public class UnnestDecorrelateRule extends RelRule + implements TransformationRule { + + protected UnnestDecorrelateRule(UnnestDecorrelateRule.Config config) { + super(config); + } + + /** Given an expression and a correlationId, find whether the expression is a + * sequence of field accesses that starts in the correlationId, i.e., it + * has the form corId.field1.field2. + * + * @param expr Expression to analyze + * @param corId Correlation id to search for + * @param fieldsAccessed On successful return, contains the list of fields accessed + * in reverse order, e.g., (field2, field1) + * @return True if {@code expr} has the expected shape, false otherwise. + */ + private boolean extractFieldReferences( + RexNode expr, CorrelationId corId, List fieldsAccessed) { + if (expr instanceof RexCorrelVariable) { + RexCorrelVariable cv = (RexCorrelVariable) expr; + return cv.id == corId; + } else if (expr instanceof RexFieldAccess) { + RexFieldAccess fieldAccess = (RexFieldAccess) expr; + fieldsAccessed.add(fieldAccess.getField()); + return extractFieldReferences(fieldAccess.getReferenceExpr(), corId, fieldsAccessed); + } else { + return false; + } + } + + @Override public void onMatch(RelOptRuleCall call) { + Project outerProject = call.rel(0); + Correlate cor = call.rel(1); + CorrelationId corId = cor.getCorrelationId(); + + RelNode left = call.rel(2); + int leftCount = left.getRowType().getFieldCount(); + ImmutableBitSet used = RelOptUtil.InputFinder.bits(outerProject.getProjects(), null); + int firstUsed = used.nextSetBit(0); + if (firstUsed != -1 && firstUsed < leftCount) { + return; + } + + int uncollectIndex = 3; + Project innerProject = null; + if (call.rel(uncollectIndex) instanceof Project) { + innerProject = call.rel(3); + uncollectIndex = 4; + } + + Uncollect uncollect = call.rel(uncollectIndex); + Project project = call.rel(uncollectIndex + 1); + + List projects = project.getProjects(); + if (projects.size() != 1) { + return; + } + + final RexNode projected = projects.get(0); + final ArrayList fieldsAccessed = new ArrayList<>(); + if (!extractFieldReferences(projected, corId, fieldsAccessed)) { + return; + } + + final RelBuilder builder = call.builder(); + builder.push(left); + + // Last field constructed by builder + RexNode field = null; + // Fields are in reverse order + Collections.reverse(fieldsAccessed); + for (RelDataTypeField index : fieldsAccessed) { + if (field != null) { + field = builder.field(field, index.getName()); + } else { + field = builder.field(index.getName()); + } + } + builder.project(requireNonNull(field, "field")) + .uncollect(uncollect.getItemAliases(), uncollect.withOrdinality); + if (innerProject != null) { + builder.project(innerProject.getProjects()); + } + final List shifted = RexUtil.shift(outerProject.getProjects(), -leftCount); + builder.project(shifted); + RelNode result = builder.build(); + call.transformTo(result); + } + + /** Rule configuration. */ + @Value.Immutable + public interface Config extends RelRule.Config { + UnnestDecorrelateRule.Config BASE = ImmutableUnnestDecorrelateRule.Config.of(); + + RelRule.Config DEFAULT = BASE + .withOperandSupplier(b0 -> b0.operand(Project.class) + .oneInput(b1 -> b1.operand(Correlate.class) + .inputs(b2 -> b2.operand(RelNode.class).anyInputs(), + b3 -> b3.operand(Uncollect.class) + .oneInput(b4 -> b4.operand(Project.class) + .oneInput(b5 -> b5.operand(LogicalValues.class).anyInputs()))))); + + RelRule.Config WITH_PROJECT = BASE + .withOperandSupplier(b0 -> b0.operand(Project.class) + .oneInput(b1 -> b1.operand(Correlate.class) + .inputs(b2 -> b2.operand(RelNode.class).anyInputs(), + b3 -> b3.operand(Project.class) + .oneInput(b4 -> b4.operand(Uncollect.class) + .oneInput(b5 -> b5.operand(Project.class) + .oneInput(b6 -> b6.operand(LogicalValues.class).anyInputs())))))); + + @Override default UnnestDecorrelateRule toRule() { + return new UnnestDecorrelateRule(this); + } + } +} diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index 1afbae451906..a83905434972 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -673,6 +673,74 @@ private HepProgram createHypergraphProgram() { .checkUnchanged(); } + /** Test case for + * [CALCITE-7196] + * Create an optimization pass which can convert some cases of Correlate + Unnest + * to Unnest. */ + @Test void testUnnestDecorrelate() { + final String sql = "WITH t1 AS (SELECT ARRAY[1, 2, 3] as arr)\n" + + "SELECT array_element.id\n" + + "FROM t1, UNNEST(t1.arr) AS array_element(id)"; + sql(sql) + .withRule(CoreRules.UNNEST_PROJECT_DECORRELATE) + .check(); + } + + /** Test case for + * [CALCITE-7196] + * Create an optimization pass which can convert some cases of Correlate + Unnest + * to Unnest. */ + @Test void testUnnestDecorrelate2() { + final String sql = "WITH t1 AS (SELECT ARRAY[1, 2, 3] as arr)\n" + + "SELECT array_element.id, array_element.ord\n" + + "FROM t1, UNNEST(t1.arr) WITH ORDINALITY AS array_element(id, ord)"; + sql(sql) + .withRule(CoreRules.UNNEST_PROJECT_DECORRELATE) + .check(); + } + + /** Test case for + * [CALCITE-7196] + * Create an optimization pass which can convert some cases of Correlate + Unnest + * to Unnest. */ + @Test void testUnnestDecorrelate3() { + final String sql = "WITH t1 AS (SELECT ARRAY[1, 2, 3] as arr)\n" + + "SELECT array_element.id\n" + + "FROM t1, UNNEST(t1.arr) AS array_element(id)"; + sql(sql) + .withPreRule(CoreRules.PROJECT_REMOVE) + .withRule(CoreRules.UNNEST_DECORRELATE) + .check(); + } + + /** Test case for + * [CALCITE-7196] + * Create an optimization pass which can convert some cases of Correlate + Unnest + * to Unnest. */ + @Test void testUnnestDecorrelate4() { + final String sql = "select t2.ename\n" + + "from DEPT_NESTED as t1,\n" + + "unnest(t1.employees) as t2"; + sql(sql) + .withPreRule(CoreRules.PROJECT_REMOVE) + .withRule(CoreRules.UNNEST_DECORRELATE) + .check(); + } + + /** Test case for + * [CALCITE-7196] + * Create an optimization pass which can convert some cases of Correlate + Unnest + * to Unnest. */ + @Test void testUnnestDecorrelate5() { + final String sql = "WITH t1 AS (SELECT ROW(ARRAY[1, 2, 3]) as struct_with_array_field)\n" + + "SELECT array_element.id\n" + + "FROM t1, UNNEST(t1.struct_with_array_field[1]) AS array_element(id)"; + sql(sql) + .withPreRule(CoreRules.PROJECT_REMOVE) + .withRule(CoreRules.UNNEST_DECORRELATE) + .check(); + } + @Test void testFilterProjectTransposeRule3() { final String sql = "select * from (select deptno from emp) as d\n" + "where NOT EXISTS (\n" diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index 3887c4fe50da..465ce0e3e1eb 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -21563,6 +21563,133 @@ LogicalProject(DEPTNO=[$7]) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From cf550476075d1a62b86bcbe0ddbf844d5d840c64 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Thu, 8 Jan 2026 11:37:38 -0800 Subject: [PATCH 095/562] [CALCITE-7363] Improve error message for ASOF JOIN Signed-off-by: Mihai Budiu --- .../apache/calcite/runtime/CalciteResource.java | 2 +- .../calcite/runtime/CalciteResource.properties | 2 +- .../apache/calcite/test/SqlValidatorTest.java | 12 +++++++----- server/src/test/resources/sql/type.iq | 16 ++++++++++++++++ 4 files changed, 25 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java b/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java index e3bc081496ea..124c5d520376 100644 --- a/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java +++ b/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java @@ -1151,7 +1151,7 @@ ExInst multipleCapturingGroupsForRegexpFunctions(String value, @BaseMessage("ASOF JOIN MATCH_CONDITION must be a comparison between columns from the two inputs") ExInst asofMatchMustBeComparison(); - @BaseMessage("ASOF JOIN condition must be a conjunction of equality comparisons") + @BaseMessage("ASOF JOIN condition must be a conjunction of equality comparisons of columns from both sides") ExInst asofConditionMustBeComparison(); @BaseMessage("ASOF JOIN does not support correlated subqueries") diff --git a/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties b/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties index cf600b98cf77..3ffb88a55203 100644 --- a/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties +++ b/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties @@ -376,7 +376,7 @@ WindowInHavingNotAllowed=Window expressions are not permitted in the HAVING clau MatchConditionRequiresAsof=MATCH_CONDITION only allowed with ASOF JOIN AsofRequiresMatchCondition=ASOF JOIN missing MATCH_CONDITION AsofMatchMustBeComparison=ASOF JOIN MATCH_CONDITION must be a comparison between columns from the two inputs -AsofConditionMustBeComparison=ASOF JOIN condition must be a conjunction of equality comparisons +AsofConditionMustBeComparison=ASOF JOIN condition must be a conjunction of equality comparisons of columns from both sides AsofCannotBeCorrelated=ASOF JOIN does not support correlated subqueries UnknownRowField=ROW type does not have a field named ''{0}'': {1} IllegalRowIndexValue=ROW type does not have a field with index {0,number}; legal range is 1 to {1,number} diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index 98c7ea665ec4..40cf5bddf871 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -3606,6 +3606,8 @@ void testWinPartClause() { + "on emp.ename = dept.name") .fails( "ASOF JOIN MATCH_CONDITION must be a comparison between columns from the two inputs"); + final String failMessage = "ASOF JOIN condition must be a conjunction of equality comparisons " + + "of columns from both sides"; // match condition does not compare columns from both tables sql("select emp.empno from emp asof join dept\n" + "match_condition ^emp.deptno < 12^\n" @@ -3616,29 +3618,29 @@ void testWinPartClause() { sql("select emp.empno from emp asof join dept\n" + "match_condition emp.deptno < dept.deptno\n" + "on ^emp.ename < 'foo'^") - .fails("ASOF JOIN condition must be a conjunction of equality comparisons"); + .fails(failMessage); // comparison contains an equality test that does not check both tables joined sql("select emp.empno from emp asof join dept\n" + "match_condition emp.deptno < dept.deptno\n" + "on ^emp.ename = 'foo'^") - .fails("ASOF JOIN condition must be a conjunction of equality comparisons"); + .fails(failMessage); // comparison contains is not a conjunction sql("select emp.empno from emp asof join dept\n" + "match_condition emp.deptno < dept.deptno\n" + "on ^emp.ename = dept.name OR emp.deptno = dept.deptno^") - .fails("ASOF JOIN condition must be a conjunction of equality comparisons"); + .fails(failMessage); // comparison is not a conjunction sql("select * from (VALUES(true, false)) AS T0(b0, b1)\n" + "asof join (VALUES(false, false)) AS T1(b0, b1)\n" + "match_condition T0.b0 < T1.b0\n" + "on ^T0.b1 AND T1.b1^") - .fails("ASOF JOIN condition must be a conjunction of equality comparisons"); + .fails(failMessage); // Condition contains a cast that is not applied to a column sql("select * from (VALUES(true, false)) AS T0(b0, b1)\n" + "asof join (VALUES(false, 1)) AS T1(b0, b1)\n" + "match_condition T0.b0 < T1.b0\n" + "on ^T0.b1 = CAST(T1.b1 + 1 AS BOOLEAN)^") - .fails("ASOF JOIN condition must be a conjunction of equality comparisons"); + .fails(failMessage); } /** Test case for diff --git a/server/src/test/resources/sql/type.iq b/server/src/test/resources/sql/type.iq index 9aa2648efc08..a63b8c58c188 100644 --- a/server/src/test/resources/sql/type.iq +++ b/server/src/test/resources/sql/type.iq @@ -18,6 +18,22 @@ !use server !set outputformat mysql +CREATE TABLE asof_tbl(intt INT, arr VARCHAR ARRAY ); +(0 rows modified) + +!update + +SELECT * FROM asof_tbl t1 +LEFT ASOF JOIN asof_tbl AS t2 +MATCH_CONDITION (t1.intt >= t2.intt) +ON t1.arr[2] = t2.arr[2]; +java.sql.SQLException: Error while executing SQL "SELECT * FROM asof_tbl t1 +LEFT ASOF JOIN asof_tbl AS t2 +MATCH_CONDITION (t1.intt >= t2.intt) +ON t1.arr[2] = t2.arr[2]": From line 4, column 4 to line 4, column 24: ASOF JOIN condition must be a conjunction of equality comparisons of columns from both sides + +!error + create type myint1 as int; (0 rows modified) From 539086315106e45c9eb6364825a385c85bd59058 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Thu, 8 Jan 2026 17:42:47 -0800 Subject: [PATCH 096/562] [CALCITE-7358] Casts involving MAP and ROW types cause compile-time exceptions Signed-off-by: Mihai Budiu --- .../calcite/sql/fun/SqlItemOperator.java | 20 +++++++++++ .../calcite/sql/type/SqlTypeCoercionRule.java | 2 ++ .../apache/calcite/sql/type/SqlTypeUtil.java | 30 ++++++++++++++++ .../calcite/sql/type/SqlTypeFactoryTest.java | 3 +- server/src/test/resources/sql/type.iq | 35 +++++++++++++++++++ .../org/apache/calcite/test/QuidemTest.java | 22 ++++++++++++ 6 files changed, 111 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlItemOperator.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlItemOperator.java index 2285b0804aab..5b21c8560e1a 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlItemOperator.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlItemOperator.java @@ -24,6 +24,7 @@ import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlOperandCountRange; +import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.SqlOperatorBinding; import org.apache.calcite.sql.SqlSpecialOperator; import org.apache.calcite.sql.SqlWriter; @@ -163,6 +164,25 @@ private static SqlSingleOperandTypeChecker getChecker(SqlCallBinding callBinding if (sqlTypeName == SqlTypeName.VARIANT) { // Allow any key type to be used when the map keys have a VARIANT type return OperandTypes.family(SqlTypeFamily.ANY); + } else if (sqlTypeName == SqlTypeName.ROW) { + // Check that the type of the argument is exactly the key type + return new SqlSingleOperandTypeChecker() { + @Override public boolean checkSingleOperandType( + SqlCallBinding callBinding, SqlNode operand, + int iFormalOperand, boolean throwOnFailure) { + // operand 0 of ITEM is the indexed object, operand 1 is the key value + RelDataType operandType = callBinding.getOperandType(1); + boolean match = operandType.equals(keyType); + if (!match && throwOnFailure) { + throw callBinding.newValidationSignatureError(); + } + return match; + } + + @Override public String getAllowedSignatures(SqlOperator op, String opName) { + return "[" + keyType.getSqlTypeName() + "]"; + } + }; } return OperandTypes.family( requireNonNull(sqlTypeName.getFamily(), diff --git a/core/src/main/java/org/apache/calcite/sql/type/SqlTypeCoercionRule.java b/core/src/main/java/org/apache/calcite/sql/type/SqlTypeCoercionRule.java index c3ba413bf33a..59fb82fd193f 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/SqlTypeCoercionRule.java +++ b/core/src/main/java/org/apache/calcite/sql/type/SqlTypeCoercionRule.java @@ -173,6 +173,7 @@ private SqlTypeCoercionRule(Map> map) { .add(SqlTypeName.VARBINARY) .addAll(SqlTypeName.CHAR_TYPES) .add(SqlTypeName.UUID) + .addAll(SqlTypeName.INT_TYPES) .build()); // VARBINARY is castable from BINARY, CHARACTERS. @@ -181,6 +182,7 @@ private SqlTypeCoercionRule(Map> map) { .add(SqlTypeName.BINARY) .addAll(SqlTypeName.CHAR_TYPES) .add(SqlTypeName.UUID) + .addAll(SqlTypeName.INT_TYPES) .build()); // VARCHAR is castable from BOOLEAN, DATE, TIME, TIMESTAMP, numeric types, binary, uuid, and diff --git a/core/src/main/java/org/apache/calcite/sql/type/SqlTypeUtil.java b/core/src/main/java/org/apache/calcite/sql/type/SqlTypeUtil.java index 2808c89514cf..1be36fa18b84 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/SqlTypeUtil.java +++ b/core/src/main/java/org/apache/calcite/sql/type/SqlTypeUtil.java @@ -1120,6 +1120,36 @@ public static boolean canCastFrom( || fromType.getSqlTypeName() == SqlTypeName.UUID || fromType.getFamily() == SqlTypeFamily.CHARACTER || fromType.getFamily() == SqlTypeFamily.BINARY; + } else if (toType.getSqlTypeName() == SqlTypeName.ARRAY) { + if (fromType.getSqlTypeName() == SqlTypeName.ARRAY + || fromType.getSqlTypeName() == SqlTypeName.MULTISET) { + return canCastFrom( + requireNonNull(toType.getComponentType(), "componentType"), + requireNonNull(fromType.getComponentType(), "componentType"), + typeMappingRule); + } else if (fromType.getFamily() == SqlTypeFamily.CHARACTER + || fromType.getSqlTypeName() == SqlTypeName.NULL) { + // Cast from NULL or string to array is legal + return true; + } + return false; + } else if (toType.getSqlTypeName() == SqlTypeName.MAP) { + if (fromType.getSqlTypeName() == SqlTypeName.MAP) { + // It is not clear whether this is sufficient, but it is clearly necessary + return canCastFrom( + requireNonNull(toType.getKeyType(), "keyType"), + requireNonNull(fromType.getKeyType(), "keyType"), + typeMappingRule) + && canCastFrom( + requireNonNull(toType.getValueType(), "valueType"), + requireNonNull(fromType.getValueType(), "valueType"), + typeMappingRule); + } else if (fromType.getFamily() == SqlTypeFamily.CHARACTER + || fromType.getSqlTypeName() == SqlTypeName.NULL) { + // Cast from NULL or string to map is legal + return true; + } + return false; } if (toType.isStruct() || fromType.isStruct()) { if (toTypeName == SqlTypeName.DISTINCT) { diff --git a/core/src/test/java/org/apache/calcite/sql/type/SqlTypeFactoryTest.java b/core/src/test/java/org/apache/calcite/sql/type/SqlTypeFactoryTest.java index 09731bf3a574..8f5e5e4018db 100644 --- a/core/src/test/java/org/apache/calcite/sql/type/SqlTypeFactoryTest.java +++ b/core/src/test/java/org/apache/calcite/sql/type/SqlTypeFactoryTest.java @@ -99,7 +99,8 @@ class SqlTypeFactoryTest { RelDataType leastRestrictive = f.typeFactory.leastRestrictive( Lists.newArrayList(f.arraySqlChar10, f.sqlChar)); - assertNull(leastRestrictive); + // Some SQL dialects, like Postgres, allow casts between strings and arrays + assertThat(leastRestrictive, is(f.arraySqlChar10)); } @Test void testLeastRestrictiveForArrays() { diff --git a/server/src/test/resources/sql/type.iq b/server/src/test/resources/sql/type.iq index a63b8c58c188..160f5769a5b5 100644 --- a/server/src/test/resources/sql/type.iq +++ b/server/src/test/resources/sql/type.iq @@ -18,6 +18,7 @@ !use server !set outputformat mysql +# Test case for [CALCITE-7363] Improve error message for ASOF JOIN CREATE TABLE asof_tbl(intt INT, arr VARCHAR ARRAY ); (0 rows modified) @@ -34,6 +35,40 @@ ON t1.arr[2] = t2.arr[2]": From line 4, column 4 to line 4, column 24: ASOF JOIN !error +# Test case for [CALCITE-7358] Casts involving MAP and ROW types cause compile-time exceptions +CREATE TYPE user_def AS(i1 INT, v1 VARCHAR NULL); +(0 rows modified) + +!update + +CREATE TABLE tbl(mapp1 MAP); +(0 rows modified) + +!update + +# Index in a map with a user-defined type +SELECT mapp1[user_def(1, 'a')] as field FROM tbl; ++-------+ +| FIELD | ++-------+ ++-------+ +(0 rows) + +!ok + +# Test case for [CALCITE-7358] Casts involving MAP and ROW types cause compile-time exceptions +SELECT CAST(mapp1[user_def(1, 'a')] AS INT) FROM tbl; +java.sql.SQLException: Error while executing SQL "SELECT CAST(mapp1[user_def(1, 'a')] AS INT) FROM tbl": From line 1, column 8 to line 1, column 43: Cast function cannot convert value of type RecordType(VARCHAR V) to type INTEGER NOT NULL + +!error + +SELECT CAST(mapp1 AS MAP) AS to_map +FROM tbl; +java.sql.SQLException: Error while executing SQL "SELECT CAST(mapp1 AS MAP) AS to_map +FROM tbl": From line 1, column 8 to line 1, column 39: Cast function cannot convert value of type (RecordType(INTEGER I1, VARCHAR V1) NOT NULL, RecordType(VARCHAR V)) MAP to type (VARCHAR NOT NULL, INTEGER) MAP NOT NULL + +!error + create type myint1 as int; (0 rows modified) diff --git a/testkit/src/main/java/org/apache/calcite/test/QuidemTest.java b/testkit/src/main/java/org/apache/calcite/test/QuidemTest.java index 67b43ff462d8..632d24cfeaa7 100644 --- a/testkit/src/main/java/org/apache/calcite/test/QuidemTest.java +++ b/testkit/src/main/java/org/apache/calcite/test/QuidemTest.java @@ -83,6 +83,7 @@ import java.util.function.Function; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; import static org.junit.jupiter.api.Assertions.fail; @@ -213,6 +214,27 @@ protected static Collection data(String first) { return paths; } + /** Debugging helper which returns only a subset of the files produced by {@code data(first)}. + * + * @param first File path indicating where IQ files are searched. + * @param substring Only files that contain this substring are returned. + * @return The list of IQ files produced by data(first) which match the restriction. + * + *

    I find that often when I debug quidem tests it is handy to only run the currently modified + * file. By replacing the call to data(first) with a call to data(first, restricted) one + * can easily just run the new tests. But do not forget to undo this change when submitting the + * final PR! */ + protected static Collection data(String first, String substring) { + List result = data(first) + .stream() + .filter(s -> s.contains(substring)) + .collect(Collectors.toList()); + if (result.isEmpty()) { + throw new RuntimeException("Filter is too strict, result is empty"); + } + return result; + } + protected void checkRun(String path) throws Exception { final File inFile; final File outFile; From 781578c2e4f519d1cb34c9d92b3c9a28da8708cb Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Fri, 9 Jan 2026 15:09:44 -0800 Subject: [PATCH 097/562] [CALCITE-7366] RexLiteral.valueMatchesType throws for a MAP type Signed-off-by: Mihai Budiu --- .../main/java/org/apache/calcite/rex/RexLiteral.java | 2 ++ .../apache/calcite/sql/type/SqlTypeCoercionRule.java | 6 ++++-- .../apache/calcite/test/SqlToRelConverterTest.java | 7 +++++++ .../org/apache/calcite/test/SqlToRelConverterTest.xml | 11 +++++++++++ 4 files changed, 24 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rex/RexLiteral.java b/core/src/main/java/org/apache/calcite/rex/RexLiteral.java index 2dc824920889..7fd05901e74f 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexLiteral.java +++ b/core/src/main/java/org/apache/calcite/rex/RexLiteral.java @@ -397,6 +397,8 @@ public static boolean valueMatchesType( return value instanceof List; case GEOMETRY: return value instanceof Geometry; + case MAP: + return value instanceof Map; case ANY: // Literal of type ANY is not legal. "CAST(2 AS ANY)" remains // an integer literal surrounded by a cast function. diff --git a/core/src/main/java/org/apache/calcite/sql/type/SqlTypeCoercionRule.java b/core/src/main/java/org/apache/calcite/sql/type/SqlTypeCoercionRule.java index 59fb82fd193f..91eea17b4d7f 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/SqlTypeCoercionRule.java +++ b/core/src/main/java/org/apache/calcite/sql/type/SqlTypeCoercionRule.java @@ -167,22 +167,24 @@ private SqlTypeCoercionRule(Map> map) { .build()); } - // BINARY is castable from VARBINARY, CHARACTERS. + // BINARY is castable from VARBINARY, CHARACTERS, INTEGERS coerceRules.add(SqlTypeName.BINARY, coerceRules.copyValues(SqlTypeName.BINARY) .add(SqlTypeName.VARBINARY) .addAll(SqlTypeName.CHAR_TYPES) .add(SqlTypeName.UUID) .addAll(SqlTypeName.INT_TYPES) + .addAll(SqlTypeName.UNSIGNED_TYPES) .build()); - // VARBINARY is castable from BINARY, CHARACTERS. + // VARBINARY is castable from BINARY, CHARACTERS, INTEGERS coerceRules.add(SqlTypeName.VARBINARY, coerceRules.copyValues(SqlTypeName.VARBINARY) .add(SqlTypeName.BINARY) .addAll(SqlTypeName.CHAR_TYPES) .add(SqlTypeName.UUID) .addAll(SqlTypeName.INT_TYPES) + .addAll(SqlTypeName.UNSIGNED_TYPES) .build()); // VARCHAR is castable from BOOLEAN, DATE, TIME, TIMESTAMP, numeric types, binary, uuid, and diff --git a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java index 3e7d178ee605..8a485421cf95 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java @@ -882,6 +882,13 @@ public static void checkActualAndReferenceFiles() { sql(sql).ok(); } + /** Test case for [CALCITE-7366] + * RexLiteral.valueMatchesType throws for a MAP type. */ + @Test void testStringToMapCast() { + final String sql = "SELECT CAST('a' AS MAP)"; + sql(sql).ok(); + } + @Test void testGroupBug281b() { // Try to confuse it with spurious columns. final String sql = "select name, foo from (\n" diff --git a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml index 23be5c89da2e..57dc035f9cae 100644 --- a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml @@ -8138,6 +8138,17 @@ from orders]]> LogicalDelta LogicalProject(ROWTIME=[$0], PRODUCTID=[$1], ORDERID=[$2], C=[COUNT() OVER (PARTITION BY $1 ORDER BY $0 RANGE 1000:INTERVAL SECOND PRECEDING)]) LogicalTableScan(table=[[CATALOG, SALES, ORDERS]]) +]]> + + + + + )]]> + + + From 40dc4408d7d79aecad9a362c54a2809175846696 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Sun, 11 Jan 2026 10:04:51 -0800 Subject: [PATCH 098/562] [CALCITE-7367] NULLS FIRST throws ClassCastException when sorting arrays Signed-off-by: Mihai Budiu --- core/src/test/resources/sql/sort.iq | 43 +++++++++++++++++++ .../calcite/linq4j/function/Functions.java | 16 +++++-- 2 files changed, 55 insertions(+), 4 deletions(-) diff --git a/core/src/test/resources/sql/sort.iq b/core/src/test/resources/sql/sort.iq index ce34c82d09a7..fb69970e41b8 100644 --- a/core/src/test/resources/sql/sort.iq +++ b/core/src/test/resources/sql/sort.iq @@ -422,4 +422,47 @@ select * from "hr"."emps" limit 3000000000 offset 2500000000; java.lang.ArithmeticException: Integer overflow: 2500000000 is out of range for INT !error +# [CALCITE-7367] NULLS FIRST throws ClassCastException when sorting arrays +select * from +(values + (2, array[null, 3]), + (3, array[3, 4]), + (1, array[1, 2]), + (4, array[4, 5]), + (5, cast(null as integer array))) as t(id, arr) +order by arr nulls first; ++----+-----------+ +| ID | ARR | ++----+-----------+ +| 5 | | +| 1 | [1, 2] | +| 3 | [3, 4] | +| 4 | [4, 5] | +| 2 | [null, 3] | ++----+-----------+ +(5 rows) + +!ok + +select * from +(values + (2, array[null, 3]), + (3, array[3, 4]), + (1, array[1, 2]), + (4, array[4, 5]), + (5, cast(null as integer array))) as t(id, arr) +order by arr desc nulls first; ++----+-----------+ +| ID | ARR | ++----+-----------+ +| 5 | | +| 2 | [null, 3] | +| 4 | [4, 5] | +| 3 | [3, 4] | +| 1 | [1, 2] | ++----+-----------+ +(5 rows) + +!ok + # End sort.iq diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/function/Functions.java b/linq4j/src/main/java/org/apache/calcite/linq4j/function/Functions.java index 5f7aa258d4c3..73a06f42bd8c 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/function/Functions.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/function/Functions.java @@ -537,8 +537,8 @@ private static final class SelectorEqualityComparer /** Nulls first comparator. */ private static class NullsFirstComparator - implements Comparator, Serializable { - @Override public int compare(Comparable o1, Comparable o2) { + implements Comparator, Serializable { + @Override public int compare(@Nullable Object o1, @Nullable Object o2) { if (o1 == o2) { return 0; } @@ -548,8 +548,16 @@ private static class NullsFirstComparator if (o2 == null) { return 1; } - //noinspection unchecked - return o1.compareTo(o2); + if (o1 instanceof Comparable && o2 instanceof Comparable) { + //noinspection unchecked + return ((Comparable) o1).compareTo(o2); + } else if (o1 instanceof List && o2 instanceof List) { + return compareLists((List) o1, (List) o2); + } else if (o1 instanceof Object[] && o2 instanceof Object[]) { + return compareObjectArrays((Object[]) o1, (Object[]) o2); + } else { + throw new IllegalArgumentException(); + } } } From 627dce9fe4876df6b7ca465c43731b23240b75d4 Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Fri, 9 Jan 2026 17:20:25 +0800 Subject: [PATCH 099/562] [CALCITE-6829] MSSQL dialect incorrectly translates of SELECT TRUE --- .../java/org/apache/calcite/sql/SqlWriter.java | 9 +++++++++ .../calcite/sql/dialect/MssqlSqlDialect.java | 17 ++++++++++++++++- .../calcite/sql/pretty/SqlPrettyWriter.java | 8 ++++++++ .../rel/rel2sql/RelToSqlConverterTest.java | 13 +++++++++++++ 4 files changed, 46 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/calcite/sql/SqlWriter.java b/core/src/main/java/org/apache/calcite/sql/SqlWriter.java index 57982fb2ab0a..9682f9c99f33 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlWriter.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlWriter.java @@ -548,6 +548,11 @@ SqlWriter list(FrameTypeEnum frameType, SqlBinaryOperator sepOp, */ boolean inQuery(); + /** Returns the current frame. */ + default @Nullable Frame getCurrentFrame() { + return null; + } + //~ Inner Interfaces ------------------------------------------------------- /** @@ -568,6 +573,10 @@ SqlWriter list(FrameTypeEnum frameType, SqlBinaryOperator sepOp, * the sub-frame is put onto a stack. */ interface Frame { + /** Returns the type of this frame. */ + default @Nullable FrameType getFrameType() { + return null; + } } /** Frame type. */ diff --git a/core/src/main/java/org/apache/calcite/sql/dialect/MssqlSqlDialect.java b/core/src/main/java/org/apache/calcite/sql/dialect/MssqlSqlDialect.java index 7f9907b081a0..2387ed54d9d4 100644 --- a/core/src/main/java/org/apache/calcite/sql/dialect/MssqlSqlDialect.java +++ b/core/src/main/java/org/apache/calcite/sql/dialect/MssqlSqlDialect.java @@ -197,7 +197,22 @@ public MssqlSqlDialect(Context context) { if (value == null) { return; } - unparseBoolLiteralToCondition(writer, value); + + SqlWriter.Frame frame = writer.getCurrentFrame(); + boolean expressionAllowed = true; + if (frame != null) { + SqlWriter.FrameType frameType = frame.getFrameType(); + if (frameType == SqlWriter.FrameTypeEnum.SELECT_LIST + || frameType == SqlWriter.FrameTypeEnum.VALUES) { + expressionAllowed = false; + } + } + + if (expressionAllowed) { + unparseBoolLiteralToCondition(writer, value); + } else { + writer.literal(value ? "1" : "0"); + } } @Override public boolean supportsApproxCountDistinct() { diff --git a/core/src/main/java/org/apache/calcite/sql/pretty/SqlPrettyWriter.java b/core/src/main/java/org/apache/calcite/sql/pretty/SqlPrettyWriter.java index 2269ecec49ae..95fdfd27c221 100644 --- a/core/src/main/java/org/apache/calcite/sql/pretty/SqlPrettyWriter.java +++ b/core/src/main/java/org/apache/calcite/sql/pretty/SqlPrettyWriter.java @@ -930,6 +930,10 @@ public String format(SqlNode node) { return dialect; } + @Override public @Nullable Frame getCurrentFrame() { + return listStack.peek(); + } + @Override public void literal(String s) { print(s); setNeedWhitespace(true); @@ -1135,6 +1139,10 @@ protected class FrameImpl implements Frame { final String open; final String close; + @Override public @Nullable FrameType getFrameType() { + return frameType; + } + private final int left; /** * Indent of sub-frame with respect to this one. diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java index fb975386c0d8..5a9b278c3571 100644 --- a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java +++ b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java @@ -11098,6 +11098,19 @@ private void checkLiteral2(String expression, String expected) { .ok(expected); } + /** Test case of + * [CALCITE-6829] + * MSSQL dialect incorrectly translates of SELECT TRUE. */ + @Test void testMssqlSelectTrue() { + final String query = "SELECT TRUE"; + final String expected = "SELECT *\nFROM (VALUES (1)) AS [t] ([EXPR$0])"; + sql(query).withMssql().ok(expected); + + final String query2 = "SELECT * FROM (VALUES (TRUE))"; + final String expected2 = "SELECT *\nFROM (VALUES (1)) AS [t] ([EXPR$0])"; + sql(query2).withMssql().ok(expected2); + } + /** Test case of * [CALCITE-7319] * FILTER_INTO_JOIN rule loses correlation variable context in HepPlanner. */ From 928893788834b09a5f0914962a363c4dc607d6ca Mon Sep 17 00:00:00 2001 From: Alessandro Solimando Date: Fri, 9 Jan 2026 19:44:28 +0100 Subject: [PATCH 100/562] [CALCITE-7365] RelMdRowCount ignores estimateRowCount() overrides in SingleRel's subclasses Change the SingleRel handler to delegate to estimateRowCount() so that custom SingleRel subclasses can provide accurate row count estimates --- .../calcite/rel/metadata/RelMdRowCount.java | 5 ++-- .../apache/calcite/test/RelMetadataTest.java | 29 +++++++++++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdRowCount.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdRowCount.java index f853fd647a05..e83f4c1da9f4 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdRowCount.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdRowCount.java @@ -191,9 +191,10 @@ public Double getRowCount(Calc rel, RelMetadataQuery mq) { return sampleRate * inputRowCount; } - // Covers Converter, Interpreter + // Covers Converter, Interpreter, and custom SingleRel subclasses + // Delegates to estimateRowCount() to allow subclasses to provide custom estimates public @Nullable Double getRowCount(SingleRel rel, RelMetadataQuery mq) { - return mq.getRowCount(rel.getInput()); + return rel.estimateRowCount(mq); } public @Nullable Double getRowCount(Join rel, RelMetadataQuery mq) { diff --git a/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java b/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java index acb04f176282..c5bdfcedd2cf 100644 --- a/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java @@ -1433,6 +1433,18 @@ void testColumnOriginsUnion() { .assertThatRowCount(is(EMP_SIZE * 0.2), is(0D), is(Double.POSITIVE_INFINITY)); } + /** Test case for + * [CALCITE-7365] + * RelMdRowCount ignores estimateRowCount() overrides in SingleRel's subclasses. */ + @Test void testRowCountCustomSingleRel() { + final RelNode scan = sql("select * from emp").toRel(); + final ExpandingRel expanding = + new ExpandingRel(scan.getCluster(), scan.getTraitSet(), scan); + final RelMetadataQuery mq = scan.getCluster().getMetadataQuery(); + final double rowCount = mq.getRowCount(expanding); + assertThat(rowCount, is(140D)); + } + @Test void testRowCountAggregate() { final String sql = "select deptno from emp group by deptno"; sql(sql).assertThatRowCount(is(1.4D), is(0D), is(Double.POSITIVE_INFINITY)); @@ -5409,6 +5421,23 @@ private static class DummyRelNode extends SingleRel { } } + /** + * A custom SingleRel that expands row count by a factor of 10. + * Used to test that estimateRowCount() overrides are respected + * by the metadata system. + */ + private static class ExpandingRel extends SingleRel { + private static final double EXPANSION_FACTOR = 10.0; + + ExpandingRel(RelOptCluster cluster, RelTraitSet traits, RelNode input) { + super(cluster, traits, input); + } + + @Override public double estimateRowCount(RelMetadataQuery mq) { + return mq.getRowCount(input) * EXPANSION_FACTOR; + } + } + /** Mock catalog reader for registering a table with composite keys. */ private static class CompositeKeysCatalogReader extends MockCatalogReaderSimple { From fc6eea6228fe5f95cf00bb5bdc32de11520a6c0c Mon Sep 17 00:00:00 2001 From: xuzifu666 Date: Mon, 12 Jan 2026 10:11:57 +0800 Subject: [PATCH 101/562] Update copyright NOTICE year to 2026 --- NOTICE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NOTICE b/NOTICE index b590a1150e2e..7787a7d4f839 100644 --- a/NOTICE +++ b/NOTICE @@ -1,5 +1,5 @@ Apache Calcite -Copyright 2012-2025 The Apache Software Foundation +Copyright 2012-2026 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). From ca155b6e659be0f6f367352aa274be045e9200a3 Mon Sep 17 00:00:00 2001 From: Silun Date: Tue, 6 Jan 2026 09:13:38 +0800 Subject: [PATCH 102/562] [CALCITE-7315] Support LEFT_MARK type for hash join in enumerable convention --- .../calcite/adapter/enumerable/EnumUtils.java | 69 +++- .../enumerable/EnumerableHashJoin.java | 100 +++++ .../enumerable/EnumerableJoinRule.java | 8 +- .../enumerable/RexToLixTranslator.java | 9 +- .../rel/core/ConditionalCorrelate.java | 10 + .../apache/calcite/rel/core/Correlate.java | 1 - .../org/apache/calcite/rel/core/Join.java | 11 + .../sql/validate/SqlValidatorUtil.java | 45 ++- .../apache/calcite/util/BuiltInMethod.java | 12 + .../enumerable/EnumerableHashJoinTest.java | 141 +++++++ core/src/test/resources/sql/blank.iq | 16 +- .../calcite/linq4j/DefaultEnumerable.java | 19 + .../calcite/linq4j/EnumerableDefaults.java | 372 +++++++++++++++++- .../calcite/linq4j/ExtendedEnumerable.java | 40 ++ .../org/apache/calcite/linq4j/JoinType.java | 27 +- .../linq4j/function/NullablePredicate2.java | 27 ++ 16 files changed, 857 insertions(+), 50 deletions(-) create mode 100644 linq4j/src/main/java/org/apache/calcite/linq4j/function/NullablePredicate2.java diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java index 93db65e2289d..8787e682cc3e 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java @@ -26,6 +26,7 @@ import org.apache.calcite.linq4j.Ord; import org.apache.calcite.linq4j.function.Function1; import org.apache.calcite.linq4j.function.Function2; +import org.apache.calcite.linq4j.function.NullablePredicate2; import org.apache.calcite.linq4j.function.Predicate2; import org.apache.calcite.linq4j.tree.BlockBuilder; import org.apache.calcite.linq4j.tree.BlockStatement; @@ -875,6 +876,8 @@ static JoinType toLinq4jJoinType(JoinRelType joinRelType) { return JoinType.ASOF; case LEFT_ASOF: return JoinType.LEFT_ASOF; + case LEFT_MARK: + return JoinType.LEFT_MARK; default: break; } @@ -882,7 +885,47 @@ static JoinType toLinq4jJoinType(JoinRelType joinRelType) { "Unable to convert " + joinRelType + " to Linq4j JoinType"); } - /** Returns a predicate expression based on a join condition. */ + /** + * Return the result selector of a mark join. It is a Expression that will generate a Function2 in + * runtime, the Function2 will concat the left/right side row and the marker. + * + *

    For example: + * + * 

    +   * new Function2<Object[], Boolean, Object[]>() {
+   *    public Object[] apply(Object[] input, Boolean marker) {
+   *        return new Object[] {
+   *          input[0], input[1], ..., input[n], marker
+   *        };
+   *    }
+   * }
    + * + * @param resultPhysType Physical type of result + * @param inputPhysType Physical type of lhs/rhs + * @return the result selector of a mark join + */ + static Expression markJoinSelector(PhysType resultPhysType, PhysType inputPhysType) { + final List parameters = new ArrayList<>(); + final ParameterExpression inputParameter = + Expressions.parameter(Primitive.box(inputPhysType.getJavaRowType()), "input"); + final ParameterExpression markerParameter + = Expressions.parameter(Boolean.class, "marker"); + parameters.add(inputParameter); + parameters.add(markerParameter); + + final List expressions = new ArrayList<>(); + final int inputFieldCount = inputPhysType.getRowType().getFieldCount(); + for (int i = 0; i < inputFieldCount; i++) { + Expression expression = inputPhysType.fieldReference(inputParameter, i); + expressions.add(expression); + } + expressions.add(markerParameter); + return Expressions.lambda( + Function2.class, + resultPhysType.record(expressions), + parameters); + } + static Expression generatePredicate( EnumerableRelImplementor implementor, RexBuilder rexBuilder, @@ -891,6 +934,24 @@ static Expression generatePredicate( PhysType leftPhysType, PhysType rightPhysType, RexNode condition) { + return generatePredicate(implementor, rexBuilder, left, right, + leftPhysType, rightPhysType, condition, false); + } + + /** + * Returns a predicate expression based on a join condition. If one of the arguments of the + * expression is NULL, when nullable is TRUE, the expression will return NULL value; + * when nullable is FALSE, it will return FALSE. + */ + static Expression generatePredicate( + EnumerableRelImplementor implementor, + RexBuilder rexBuilder, + RelNode left, + RelNode right, + PhysType leftPhysType, + PhysType rightPhysType, + RexNode condition, + boolean nullable) { final BlockBuilder builder = new BlockBuilder(); final ParameterExpression left_ = Expressions.parameter(leftPhysType.getJavaRowType(), "left"); @@ -913,8 +974,10 @@ static Expression generatePredicate( ImmutableMap.of(left_, leftPhysType, right_, rightPhysType)), implementor.allCorrelateVariables, - implementor.getConformance()))); - return Expressions.lambda(Predicate2.class, builder.toBlock(), left_, right_); + implementor.getConformance(), + nullable))); + Class clazz = nullable ? NullablePredicate2.class : Predicate2.class; + return Expressions.lambda(clazz, builder.toBlock(), left_, right_); } /** diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableHashJoin.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableHashJoin.java index e37173137aef..6fc9a9ba3d07 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableHashJoin.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableHashJoin.java @@ -45,6 +45,7 @@ import org.checkerframework.checker.nullness.qual.Nullable; import java.lang.reflect.Method; +import java.util.ArrayList; import java.util.List; import java.util.Set; @@ -173,11 +174,110 @@ public static EnumerableHashJoin create( case SEMI: case ANTI: return implementHashSemiJoin(implementor, pref); + case LEFT_MARK: + return implementHashMarkJoin(implementor, pref); default: return implementHashJoin(implementor, pref); } } + private Result implementHashMarkJoin(EnumerableRelImplementor implementor, Prefer pref) { + assert joinType == JoinRelType.LEFT_MARK; + BlockBuilder builder = new BlockBuilder(); + final Result leftResult = + implementor.visitChild(this, 0, (EnumerableRel) left, pref); + Expression leftExpression = + builder.append( + "left", leftResult.block); + final Result rightResult = + implementor.visitChild(this, 1, (EnumerableRel) right, pref); + Expression rightExpression = + builder.append( + "right", rightResult.block); + final PhysType physType = + PhysTypeImpl.of( + implementor.getTypeFactory(), getRowType(), pref.preferArray()); + + // convert equi and non-equi conditions to Expression + Expression nonEquiPredicate = Expressions.constant(null); + if (!joinInfo.nonEquiConditions.isEmpty()) { + RexNode nonEquiCondition = + RexUtil.composeConjunction(getCluster().getRexBuilder(), + joinInfo.nonEquiConditions, true); + if (nonEquiCondition != null) { + // need three-valued boolean logic + nonEquiPredicate = + EnumUtils.generatePredicate(implementor, + getCluster().getRexBuilder(), left, right, leftResult.physType, + rightResult.physType, nonEquiCondition, true); + } + } + RexNode equiCondition = joinInfo.getEquiCondition(left, right, getCluster().getRexBuilder()); + // need three-valued boolean logic + final Expression equiPredicate = + EnumUtils.generatePredicate(implementor, + getCluster().getRexBuilder(), left, right, leftResult.physType, + rightResult.physType, equiCondition, true); + + // create key selector and null-safe key selector + final Expression leftKeySelector = + leftResult.physType.generateNullAwareAccessor( + joinInfo.leftKeys, joinInfo.nullExclusionFlags); + final Expression rightKeySelector = + rightResult.physType.generateNullAwareAccessor( + joinInfo.rightKeys, joinInfo.nullExclusionFlags); + + int notNullSafeKeyCount = 0; + List leftNullSafeKeys = new ArrayList<>(); + List rightNullSafeKeys = new ArrayList<>(); + for (int i = 0; i < joinInfo.nullExclusionFlags.size(); i++) { + if (joinInfo.nullExclusionFlags.get(i)) { + notNullSafeKeyCount++; + } else { + leftNullSafeKeys.add(joinInfo.leftKeys.get(i)); + rightNullSafeKeys.add(joinInfo.rightKeys.get(i)); + } + } + final Expression leftNullSafeKeySelector = + leftNullSafeKeys.isEmpty() + ? Expressions.constant(null) + : leftResult.physType.generateAccessor(ImmutableIntList.copyOf(leftNullSafeKeys)); + final Expression rightNullSafeKeySelector = + rightNullSafeKeys.isEmpty() + ? Expressions.constant(null) + : rightResult.physType.generateAccessor(ImmutableIntList.copyOf(rightNullSafeKeys)); + final boolean atMostOneNotNullSafeKey = notNullSafeKeyCount <= 1; + + // create key comparator and null-safe key comparator + final PhysType nullSafeKeyPhysType = + leftResult.physType.project(leftNullSafeKeys, JavaRowFormat.LIST); + final Expression nullSafeKeyComparator = + Util.first(nullSafeKeyPhysType.comparer(), Expressions.constant(null)); + final PhysType keyPhysType = + leftResult.physType.project(joinInfo.leftKeys, JavaRowFormat.LIST); + final Expression keyComparator = + Util.first(keyPhysType.comparer(), Expressions.constant(null)); + + return implementor.result(physType, + builder.append( + Expressions.call( + leftExpression, + BuiltInMethod.LEFT_MARK_HASH_JOIN.method, + Expressions.list( + rightExpression, + leftKeySelector, + rightKeySelector, + leftNullSafeKeySelector, + rightNullSafeKeySelector, + Expressions.constant(atMostOneNotNullSafeKey), + EnumUtils.markJoinSelector(physType, leftResult.physType), + keyComparator, + nullSafeKeyComparator, + nonEquiPredicate, + equiPredicate))) + .toBlock()); + } + private Result implementHashSemiJoin(EnumerableRelImplementor implementor, Prefer pref) { assert joinType == JoinRelType.SEMI || joinType == JoinRelType.ANTI; final Method method = joinType == JoinRelType.SEMI diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableJoinRule.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableJoinRule.java index fe40d17dc451..08a301da5aa9 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableJoinRule.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableJoinRule.java @@ -54,10 +54,6 @@ protected EnumerableJoinRule(Config config) { @Override public @Nullable RelNode convert(RelNode rel) { Join join = (Join) rel; - if (!Bug.TODO_FIXED && join.getJoinType() == JoinRelType.LEFT_MARK) { - // TODO implement LEFT MARK join - return null; - } List newInputs = new ArrayList<>(); for (RelNode input : join.getInputs()) { if (!(input.getConvention() instanceof EnumerableConvention)) { @@ -100,6 +96,10 @@ protected EnumerableJoinRule(Config config) { join.getVariablesSet(), join.getJoinType()); } + if (!Bug.TODO_FIXED && join.getJoinType() == JoinRelType.LEFT_MARK) { + // TODO Support LEFT MARK type for nested loop join + return null; + } return EnumerableNestedLoopJoin.create( left, right, diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java index 7e5f1948ab41..e7b8657bf542 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java @@ -1211,6 +1211,13 @@ private Expression translateTableFunction(RexCall rexCall, Expression inputEnume public static Expression translateCondition(RexProgram program, JavaTypeFactory typeFactory, BlockBuilder list, InputGetter inputGetter, Function1 correlates, SqlConformance conformance) { + return translateCondition(program, typeFactory, list, inputGetter, + correlates, conformance, false); + } + + public static Expression translateCondition(RexProgram program, + JavaTypeFactory typeFactory, BlockBuilder list, InputGetter inputGetter, + Function1 correlates, SqlConformance conformance, boolean nullable) { RexLocalRef condition = program.getCondition(); if (condition == null) { return RexImpTable.TRUE_EXPR; @@ -1222,7 +1229,7 @@ public static Expression translateCondition(RexProgram program, translator = translator.setCorrelates(correlates); return translator.translate( condition, - RexImpTable.NullAs.FALSE); + nullable ? RexImpTable.NullAs.NULL : RexImpTable.NullAs.FALSE); } /** Returns whether an expression is nullable. diff --git a/core/src/main/java/org/apache/calcite/rel/core/ConditionalCorrelate.java b/core/src/main/java/org/apache/calcite/rel/core/ConditionalCorrelate.java index a6527f23fdc1..af203429efe9 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/ConditionalCorrelate.java +++ b/core/src/main/java/org/apache/calcite/rel/core/ConditionalCorrelate.java @@ -22,9 +22,13 @@ import org.apache.calcite.rel.RelWriter; import org.apache.calcite.rel.hint.RelHint; import org.apache.calcite.rel.rules.CoreRules; +import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.validate.SqlValidatorUtil; import org.apache.calcite.util.ImmutableBitSet; +import com.google.common.collect.ImmutableList; + import java.util.List; /** @@ -70,6 +74,12 @@ public abstract ConditionalCorrelate copy(RelTraitSet traitSet, RelNode left, Re .itemIf("condition", condition, !condition.isAlwaysTrue()); } + @Override protected RelDataType deriveRowType() { + assert joinType == JoinRelType.LEFT_MARK; + return SqlValidatorUtil.createMarkJoinType(getCluster().getTypeFactory(), left.getRowType(), + condition.getType(), ImmutableList.of()); + } + @Override public RexNode getCondition() { return condition; } diff --git a/core/src/main/java/org/apache/calcite/rel/core/Correlate.java b/core/src/main/java/org/apache/calcite/rel/core/Correlate.java index e9d2adbccdd4..c6a994e922eb 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/Correlate.java +++ b/core/src/main/java/org/apache/calcite/rel/core/Correlate.java @@ -170,7 +170,6 @@ public JoinRelType getJoinType() { switch (joinType) { case LEFT: case INNER: - case LEFT_MARK: return SqlValidatorUtil.deriveJoinRowType(left.getRowType(), right.getRowType(), joinType, getCluster().getTypeFactory(), null, diff --git a/core/src/main/java/org/apache/calcite/rel/core/Join.java b/core/src/main/java/org/apache/calcite/rel/core/Join.java index 1b81717aedde..999c4639f9c7 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/Join.java +++ b/core/src/main/java/org/apache/calcite/rel/core/Join.java @@ -190,6 +190,13 @@ public JoinRelType getJoinType() { + " failures in condition " + condition); } } + if (joinType == JoinRelType.LEFT_MARK + && joinInfo.nullExclusionFlags.contains(true) + && !joinInfo.nonEquiConditions.isEmpty()) { + return litmus.fail("Left mark join is produced by rewriting IN/SOME/EXISTS " + + "subqueries, it will never contain both not null-safe join keys and non-equi " + + "predicates."); + } return litmus.succeed(); } @@ -260,6 +267,10 @@ protected int deepHashCode0() { } @Override protected RelDataType deriveRowType() { + if (joinType == JoinRelType.LEFT_MARK) { + return SqlValidatorUtil.createMarkJoinType(getCluster().getTypeFactory(), left.getRowType(), + condition.getType(), getSystemFieldList()); + } return SqlValidatorUtil.deriveJoinRowType(left.getRowType(), right.getRowType(), joinType, getCluster().getTypeFactory(), null, getSystemFieldList()); diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorUtil.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorUtil.java index a01367057a32..f05c58a8c7f3 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorUtil.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorUtil.java @@ -57,7 +57,6 @@ import org.apache.calcite.sql.SqlUtil; import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.parser.SqlParserPos; -import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.sql.type.SqlTypeUtil; import org.apache.calcite.util.ImmutableBitSet; import org.apache.calcite.util.Litmus; @@ -557,15 +556,6 @@ public static RelDataType deriveJoinRowType( case ANTI: rightType = null; break; - case LEFT_MARK: - final String markColName = - SqlValidatorUtil.uniquify("markCol", Sets.newHashSet(leftType.getFieldNames()), - SqlValidatorUtil.EXPR_SUGGESTER); - rightType = - typeFactory.createStructType( - ImmutableList.of(typeFactory.createSqlType(SqlTypeName.BOOLEAN)), - ImmutableList.of(markColName)); - break; default: break; } @@ -627,6 +617,41 @@ public static RelDataType createJoinType( return typeFactory.createStructType(typeList, nameList); } + /** + * Returns the type of the result collection produced by a mark join. Taking LEFT_MARK join as an + * example, its output is all rows from the left side and creates a new attribute to mark a tuple + * as having join partners from right side or not. + * + * @param typeFactory Type factory + * @param inputType Type of lhs/rhs of the mark join + * @param joinConditionType Type of the join condition + * @param systemFieldList List of system fields that will be prefixed to output row type; + * typically empty but must not be null + * @return mark join type + */ + public static RelDataType createMarkJoinType( + RelDataTypeFactory typeFactory, + RelDataType inputType, + RelDataType joinConditionType, + List systemFieldList) { + final String markerName = + SqlValidatorUtil.uniquify("markCol", Sets.newHashSet(inputType.getFieldNames()), + SqlValidatorUtil.EXPR_SUGGESTER); + // conceptually the type of marker is a three-valued boolean, but it can be simplified to a + // two-valued boolean in specific cases (e.g., rewriting from an EXISTS subquery). Simple + // defining the marker type as nullable boolean might cause type mismatch errors after rewriting + // some subqueries (such as EXISTS subquery). + // When deriving the type of LEFT_MARK join, we no longer know which subquery it was + // rewritten from, but that information is implicit in the join condition. For example, after + // rewriting and decorrelating an EXISTS (correlated) subquery, the condition will only contain + // IS NOT DISTINCT FROM. Therefore, we derive the marker type from the condition. + final RelDataType markerType = + typeFactory.createStructType( + ImmutableList.of(joinConditionType), + ImmutableList.of(markerName)); + return createJoinType(typeFactory, inputType, markerType, null, systemFieldList); + } + private static void addFields(List fieldList, List typeList, List nameList, Set uniqueNames) { diff --git a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java index 221ff5f71371..04a816245005 100644 --- a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java +++ b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java @@ -47,6 +47,7 @@ import org.apache.calcite.linq4j.function.Function1; import org.apache.calcite.linq4j.function.Function2; import org.apache.calcite.linq4j.function.Functions; +import org.apache.calcite.linq4j.function.NullablePredicate2; import org.apache.calcite.linq4j.function.Predicate1; import org.apache.calcite.linq4j.function.Predicate2; import org.apache.calcite.linq4j.tree.FunctionExpression; @@ -212,6 +213,17 @@ public enum BuiltInMethod { Function1.class, Function1.class, Function2.class, EqualityComparer.class, boolean.class, boolean.class, Predicate2.class), + LEFT_MARK_HASH_JOIN(ExtendedEnumerable.class, "leftMarkHashJoin", Enumerable.class, + Function1.class, // outer key null aware selector + Function1.class, // inner key null aware selector + Function1.class, // outer null-safe key selector + Function1.class, // inner null-safe key selector + boolean.class, // whether there is at most one not null-safe key + Function2.class, // result selector + EqualityComparer.class, // join keys comparator + EqualityComparer.class, // null-safe join keys comparator + NullablePredicate2.class, // non-equi predicate that can return NULL + NullablePredicate2.class), // equi predicate that can return NULL ASOF_JOIN(ExtendedEnumerable.class, "asofJoin", Enumerable.class, Function1.class, // outer key selector Function1.class, // inner key selector diff --git a/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableHashJoinTest.java b/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableHashJoinTest.java index c589f6eea5e5..fce70ff8f589 100644 --- a/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableHashJoinTest.java +++ b/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableHashJoinTest.java @@ -19,15 +19,30 @@ import org.apache.calcite.adapter.enumerable.EnumerableRules; import org.apache.calcite.config.CalciteConnectionProperty; import org.apache.calcite.config.Lex; +import org.apache.calcite.plan.RelOptLattice; +import org.apache.calcite.plan.RelOptMaterialization; import org.apache.calcite.plan.RelOptPlanner; +import org.apache.calcite.plan.RelTraitSet; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.RelFactories; +import org.apache.calcite.rel.metadata.DefaultRelMetadataProvider; +import org.apache.calcite.rel.rules.CoreRules; import org.apache.calcite.runtime.Hook; import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql2rel.TopDownGeneralDecorrelator; import org.apache.calcite.test.CalciteAssert; import org.apache.calcite.test.ReflectiveSchemaWithoutRowCount; import org.apache.calcite.test.schemata.hr.HrSchema; +import org.apache.calcite.tools.Program; +import org.apache.calcite.tools.Programs; +import org.apache.calcite.tools.RelBuilder; +import org.apache.calcite.util.Holder; + +import com.google.common.collect.ImmutableList; import org.junit.jupiter.api.Test; +import java.util.List; import java.util.function.Consumer; /** @@ -451,6 +466,132 @@ class EnumerableHashJoinTest { "id1=2; sal1=null"); } + /** Test case for + * [CALCITE-7315] + * Support LEFT_MARK type for hash join in enumerable convention. */ + @Test void testLeftMarkJoin() { + Program subQuery = + Programs.hep( + ImmutableList.of(CoreRules.PROJECT_SUB_QUERY_TO_MARK_CORRELATE), + true, + DefaultRelMetadataProvider.INSTANCE); + Program toCalc = + Programs.hep( + ImmutableList.of(CoreRules.PROJECT_TO_CALC, CoreRules.FILTER_TO_CALC, + CoreRules.CALC_MERGE), + true, + DefaultRelMetadataProvider.INSTANCE); + Program topDownDecorrelator = new Program() { + @Override public RelNode run(RelOptPlanner planner, RelNode rel, + RelTraitSet requiredOutputTraits, List materializations, + List lattices) { + final RelBuilder relBuilder = + RelFactories.LOGICAL_BUILDER.create(rel.getCluster(), null); + return TopDownGeneralDecorrelator.decorrelateQuery(rel, relBuilder); + } + }; + Program enumerableImpl = Programs.ofRules(EnumerableRules.ENUMERABLE_RULES); + + // case1: left mark join from uncorrelated IN subquery (0 null-safe key, 1 not null-safe key) + tester(false, new HrSchema()) + .query( + "WITH t1(id) as (VALUES (1), (2), (NULL)), t2(id) as (VALUES (2), (3)) " + + "select id, id in (select id from t2) as marker from t1") + .withHook(Hook.PROGRAM, (Consumer>) program -> { + program.set(Programs.sequence(subQuery, toCalc, enumerableImpl)); + }) + .explainHookMatches( + "EnumerableHashJoin(condition=[=($0, $1)], joinType=[left_mark])\n" + + " EnumerableValues(tuples=[[{ 1 }, { 2 }, { null }]])\n" + + " EnumerableCalc(expr#0=[{inputs}], id=[$t0])\n" + + " EnumerableValues(tuples=[[{ 2 }, { 3 }]])\n") + .returnsUnordered( + "id=1; marker=false", + "id=2; marker=true", + "id=null; marker=null"); + + // case2: left mark join from uncorrelated IN subquery (0 null-safe key, 1 not null-safe key) + tester(false, new HrSchema()) + .query( + "WITH t1(id) as (VALUES (1), (2), (3)), t2(id) as (VALUES (2), (NULL)) " + + "select id, id in (select id from t2) as marker from t1") + .withHook(Hook.PROGRAM, (Consumer>) program -> { + program.set(Programs.sequence(subQuery, toCalc, enumerableImpl)); + }) + .explainHookMatches( + "EnumerableHashJoin(condition=[=($0, $1)], joinType=[left_mark])\n" + + " EnumerableValues(tuples=[[{ 1 }, { 2 }, { 3 }]])\n" + + " EnumerableCalc(expr#0=[{inputs}], id=[$t0])\n" + + " EnumerableValues(tuples=[[{ 2 }, { null }]])\n") + .returnsUnordered( + "id=1; marker=null", + "id=2; marker=true", + "id=3; marker=null"); + + // case3: left mark join from uncorrelated IN subquery (0 null-safe key, 2 not null-safe key) + tester(false, new HrSchema()) + .query( + "WITH t1(id, sal) as (VALUES (1, 10), (2, NULL), (3, NULL)), " + + "t2(id, sal) as (VALUES (1, 10), (2, NULL)) " + + "select id, sal, (id, sal) in (select id, sal from t2) as marker from t1") + .withHook(Hook.PROGRAM, (Consumer>) program -> { + program.set(Programs.sequence(subQuery, toCalc, enumerableImpl)); + }) + .explainHookMatches( + "EnumerableHashJoin(condition=[AND(=($0, $2), =($1, $3))], joinType=[left_mark])\n" + + " EnumerableValues(tuples=[[{ 1, 10 }, { 2, null }, { 3, null }]])\n" + + " EnumerableCalc(expr#0..1=[{inputs}], proj#0..1=[{exprs}])\n" + + " EnumerableValues(tuples=[[{ 1, 10 }, { 2, null }]])\n") + .returnsUnordered( + "id=1; sal=10; marker=true", + "id=2; sal=null; marker=null", + "id=3; sal=null; marker=false"); + + // case4: left mark join from correlated IN subquery (1 null-safe key, 1 not null-safe key) + tester(false, new HrSchema()) + .query( + "WITH t1(id, sal) as (VALUES (1, 10), (2, 20), (3, NULL)), " + + "t2(id, sal) as (VALUES (1, 10), (2, NULL)) " + + "select id, sal in (select sal from t2 where t1.id = t2.id) as marker from t1") + .withHook(Hook.PROGRAM, (Consumer>) program -> { + program.set(Programs.sequence(subQuery, topDownDecorrelator, toCalc, enumerableImpl)); + }) + .explainHookMatches( + "EnumerableCalc(expr#0..2=[{inputs}], id=[$t0], marker=[$t2])\n" + + " EnumerableHashJoin(condition=[AND(=($1, $2), IS NOT DISTINCT FROM($0, $3))], joinType=[left_mark])\n" + + " EnumerableValues(tuples=[[{ 1, 10 }, { 2, 20 }, { 3, null }]])\n" + + " EnumerableCalc(expr#0..1=[{inputs}], EXPR$1=[$t1], EXPR$0=[$t0])\n" + + " EnumerableValues(tuples=[[{ 1, 10 }, { 2, null }]])\n") + .returnsUnordered( + "id=1; marker=true", + "id=2; marker=null", + "id=3; marker=false"); + + // case5: left mark join from correlated SOME subquery (1 null-safe key, and non-equi predicate) + tester(false, new HrSchema()) + .query( + "WITH t1(id, sal) as (VALUES (1, 10), (2, 20), (NULL, 30)), " + + "t2(id, sal) as (VALUES (1, 9), (2, NULL), (NULL, 31)) " + + "select id, sal < SOME(select sal from t2 where t1.id = t2.id or t1.id is null) " + + "as marker from t1") + .withHook(Hook.PROGRAM, (Consumer>) program -> { + program.set(Programs.sequence(subQuery, topDownDecorrelator, toCalc, enumerableImpl)); + }) + .explainHookMatches( + "EnumerableCalc(expr#0..2=[{inputs}], id=[$t0], marker=[$t2])\n" + + " EnumerableHashJoin(condition=[AND(IS NOT DISTINCT FROM($0, $3), <($1, $2))], joinType=[left_mark])\n" + + " EnumerableValues(tuples=[[{ 1, 10 }, { 2, 20 }, { null, 30 }]])\n" + + " EnumerableCalc(expr#0..2=[{inputs}], EXPR$1=[$t1], EXPR$00=[$t2])\n" + + " EnumerableNestedLoopJoin(condition=[OR(=($2, $0), IS NULL($2))], joinType=[inner])\n" + + " EnumerableValues(tuples=[[{ 1, 9 }, { 2, null }, { null, 31 }]])\n" + + " EnumerableCalc(expr#0..1=[{inputs}], EXPR$0=[$t0])\n" + + " EnumerableValues(tuples=[[{ 1, 10 }, { 2, 20 }, { null, 30 }]])\n") + .returnsUnordered( + "id=1; marker=false", + "id=2; marker=null", + "id=null; marker=true"); + } + private CalciteAssert.AssertThat tester(boolean forceDecorrelate, Object schema) { return CalciteAssert.that() diff --git a/core/src/test/resources/sql/blank.iq b/core/src/test/resources/sql/blank.iq index 4053cbb26b87..206707287ec3 100644 --- a/core/src/test/resources/sql/blank.iq +++ b/core/src/test/resources/sql/blank.iq @@ -107,6 +107,7 @@ EnumerableCalc(expr#0..7=[{inputs}], expr#8=[0], expr#9=[=($t3, $t8)], expr#10=[ EnumerableCalc(expr#0..1=[{inputs}], expr#2=[IS NOT NULL($t1)], expr#3=[IS NOT NULL($t0)], expr#4=[AND($t2, $t3)], proj#0..1=[{exprs}], $condition=[$t4]) EnumerableTableScan(table=[[BLANK, TABLE2]]) !plan +!} +---+---+ | I | J | +---+---+ @@ -114,15 +115,7 @@ EnumerableCalc(expr#0..7=[{inputs}], expr#8=[0], expr#9=[=($t3, $t8)], expr#10=[ (0 rows) !ok -!} - -# TODO: This error needs to be fixed -!if (use_new_decorr) { -Unable to convert LEFT_MARK to Linq4j JoinType -!error -!} -!if (use_old_decorr) { select * from table1 where j not in (select i from table2); +---+---+ | I | J | @@ -162,13 +155,6 @@ select * from table1 where j not in (select i from table2) or j = 3; (1 row) !ok -!} - -# TODO: This error needs to be fixed -!if (use_new_decorr) { -Unable to convert LEFT_MARK to Linq4j JoinType -!error -!} # [CALCITE-4813] ANY_VALUE assumes that arguments should be comparable select any_value(r) over(), s from(select array[f, s] r, s from (select 1 as f, 2 as s) t) t; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/DefaultEnumerable.java b/linq4j/src/main/java/org/apache/calcite/linq4j/DefaultEnumerable.java index 4ea1b4e7e739..f90c9f112d0e 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/DefaultEnumerable.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/DefaultEnumerable.java @@ -30,6 +30,7 @@ import org.apache.calcite.linq4j.function.NullableFloatFunction1; import org.apache.calcite.linq4j.function.NullableIntegerFunction1; import org.apache.calcite.linq4j.function.NullableLongFunction1; +import org.apache.calcite.linq4j.function.NullablePredicate2; import org.apache.calcite.linq4j.function.Predicate1; import org.apache.calcite.linq4j.function.Predicate2; @@ -429,6 +430,24 @@ protected OrderedQueryable asOrderedQueryable() { generateNullsOnRight, predicate); } + @Override public Enumerable leftMarkHashJoin( + Enumerable inner, + Function1 outerKeyNullAwareSelector, + Function1 innerKeyNullAwareSelector, + @Nullable Function1 outerNullSafeKeySelector, + @Nullable Function1 innerNullSafeKeySelector, + boolean atMostOneNotNullSafeKey, + Function2 resultSelector, + @Nullable EqualityComparer comparer, + @Nullable EqualityComparer nullSafeComparer, + @Nullable NullablePredicate2 nonEquiPredicate, + NullablePredicate2 equiPredicate) { + return EnumerableDefaults.leftMarkHashJoin(getThis(), inner, outerKeyNullAwareSelector, + innerKeyNullAwareSelector, outerNullSafeKeySelector, innerNullSafeKeySelector, + atMostOneNotNullSafeKey, resultSelector, comparer, nullSafeComparer, + nonEquiPredicate, equiPredicate); + } + @Override public Enumerable correlateJoin( JoinType joinType, Function1> inner, Function2 resultSelector) { diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java b/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java index 14309bdfd652..96828708f7f0 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java @@ -31,6 +31,7 @@ import org.apache.calcite.linq4j.function.NullableFloatFunction1; import org.apache.calcite.linq4j.function.NullableIntegerFunction1; import org.apache.calcite.linq4j.function.NullableLongFunction1; +import org.apache.calcite.linq4j.function.NullablePredicate2; import org.apache.calcite.linq4j.function.Predicate1; import org.apache.calcite.linq4j.function.Predicate2; @@ -1685,6 +1686,285 @@ private static Enumerable hashJoinWith }; } + /** + * Left mark join implementation based on hash. It will keep all rows from the left side and + * creates a new attribute to mark the rows from left input as having join partners from right + * side or not. Refer to + * The Complete Story of Joins (in HyPer). The implementation in the paper targets the case + * of a single join key that is not null‑safe, which is a representative scenario. We need to + * consider the more general case of multiple join keys, including both the null‑safe and not + * null‑safe. The key point is whether the marker should be FALSE or NULL when the hash table + * match fails. + * + *

    Left mark join is produced by rewriting IN/SOME/EXISTS subqueries, so left mark join will + * never contain both not null-safe join keys and non-equi predicates. + * + * @param outer Left input + * @param inner Right input + * @param outerKeyNullAwareSelector Function that extracts keys from the row of left input + * (return NULL when a not null-safe key has a NULL value) + * @param innerKeyNullAwareSelector Function that extracts keys from the row of right input + * (return NULL when a not null-safe key has a NULL value) + * @param outerNullSafeKeySelector Function that extracts the null-safe keys from the row of + * left input + * @param innerNullSafeKeySelector Function that extracts the null-safe keys from the row of + * right input + * @param atMostOneNotNullSafeKey True when there is at most one not null-safe key in join + * keys + * @param resultSelector Function that concats the row of left input and marker + * @param comparer Function that compares the keys + * @param nullSafeComparer Function that compares the null-safe keys + * @param nonEquiPredicate Non-equi predicate that can return NULL + * @param equiPredicate Equi predicate that can return NULL + */ + public static Enumerable leftMarkHashJoin( + final Enumerable outer, final Enumerable inner, + final Function1 outerKeyNullAwareSelector, + final Function1 innerKeyNullAwareSelector, + final @Nullable Function1 outerNullSafeKeySelector, + final @Nullable Function1 innerNullSafeKeySelector, + final boolean atMostOneNotNullSafeKey, + final Function2 resultSelector, + final @Nullable EqualityComparer comparer, + final @Nullable EqualityComparer nullSafeComparer, + final @Nullable NullablePredicate2 nonEquiPredicate, + final NullablePredicate2 equiPredicate) { + if (atMostOneNotNullSafeKey) { + return leftMarkHashJoinOptimized(outer, inner, outerKeyNullAwareSelector, + innerKeyNullAwareSelector, outerNullSafeKeySelector, innerNullSafeKeySelector, + resultSelector, comparer, nullSafeComparer, nonEquiPredicate); + } + return leftMarkHashJoinGeneral(outer, inner, outerKeyNullAwareSelector, + innerKeyNullAwareSelector, outerNullSafeKeySelector, innerNullSafeKeySelector, + resultSelector, comparer, nullSafeComparer, nonEquiPredicate, equiPredicate); + } + + /** + * For other join types (especially INNER join), the hash table can be used to quickly determine + * which right-side rows should be joined with a given left-side row. But left mark join is more + * complicated. The marker indicates whether a left row has a join partner on the + * right side, it is a three-valued boolean, meaning we need to know the actual result (TRUE, + * FALSE, or NULL) of the join condition. + * + *

    Join key comes in two categories: + *

      + *
    • null-safe key (IS NOT DISTINCT FROM): it only produces TRUE/FALSE.
      • + *
    • not null-safe key (EQUALS): it produces a three-valued boolean.
      • + *
    + * + *

    If all join keys are null-safe, we can get the comparison result (TRUE or FALSE) based on + * hash table matching. + * + *

    If there are multiple not null-safe join keys, the hash table matching alone is + * insufficient to determine the comparison result. However, if there is only one not null-safe + * key, we can record whether this key has any NULL values when constructing the hash table. + * During probing, if no match is found in the hash table and there are any NULL values on this + * unique not null-safe key, we can know that the marker is NULL. + */ + static Enumerable leftMarkHashJoinOptimized( + final Enumerable outer, final Enumerable inner, + final Function1 outerKeyNullAwareSelector, + final Function1 innerKeyNullAwareSelector, + final @Nullable Function1 outerNullSafeKeySelector, + final @Nullable Function1 innerNullSafeKeySelector, + final Function2 resultSelector, + final @Nullable EqualityComparer comparer, + final @Nullable EqualityComparer nullSafeComparer, + final @Nullable NullablePredicate2 nonEquiPredicate) { + return new AbstractEnumerable() { + @Override public Enumerator enumerator() { + HashTableWithNullSafeKeySet ht = + HashTableWithNullSafeKeySet.build(inner, innerKeyNullAwareSelector, + innerNullSafeKeySelector, comparer, nullSafeComparer); + + return new Enumerator() { + Enumerator outers = outer.enumerator(); + @Nullable Boolean marker = false; + + @Override public TResult current() { + return resultSelector.apply(outers.current(), marker); + } + + @Override public boolean moveNext() { + if (!outers.moveNext()) { + return false; + } + marker = false; + final TSource outerRow = outers.current(); + final TKey outerKey = outerKeyNullAwareSelector.apply(outerRow); + if (outerNullSafeKeySelector != null + && !ht.containsNullSafeKey(outerNullSafeKeySelector.apply(outerRow))) { + // there are null-safe keys, but there is no match in the hash table of null-safe + // keys. The marker is FALSE + return true; + } + + if (outerKey == null) { + // outerRow has a NULL value on the unique not null-safe key. The marker is NULL + marker = null; + } else { + Enumerable innerEnumerable = ht.lookup.get(outerKey); + if (innerEnumerable == null) { + // no match found in the hash table. If there are any NULL values on the unique + // not null-safe key, the marker is NULL. + marker = ht.lookup.containsKey(null) ? null : false; + } else { + if (nonEquiPredicate == null) { + marker = true; + } else { + try (Enumerator innerEnumerator = innerEnumerable.enumerator()) { + while (innerEnumerator.moveNext()) { + final TInner innerRow = innerEnumerator.current(); + Boolean predicateMatched = nonEquiPredicate.apply(outerRow, innerRow); + if (predicateMatched == null) { + marker = null; + } else if (predicateMatched) { + marker = true; + break; + } + } + } + } + } + } + // if the inner is empty set, convert the NULL marker to FALSE + if (marker == null && ht.buildSideIsEmpty) { + marker = false; + } + return true; + } + + @Override public void reset() { + outers.reset(); + } + + @Override public void close() { + outers.close(); + } + }; + } + }; + } + + /** + * As described in {@link #leftMarkHashJoinOptimized}, if there are multiple not null-safe join + * keys, the hash table matching alone is insufficient to determine the comparison result. For + * left rows that fail to match in the hash table, we need to apply the equi-predicate against + * the right-side rows one by one to determine whether the marker should be + * FALSE or NULL. + */ + static Enumerable leftMarkHashJoinGeneral( + final Enumerable outer, final Enumerable inner, + final Function1 outerKeyNullAwareSelector, + final Function1 innerKeyNullAwareSelector, + final @Nullable Function1 outerNullSafeKeySelector, + final @Nullable Function1 innerNullSafeKeySelector, + final Function2 resultSelector, + final @Nullable EqualityComparer comparer, + final @Nullable EqualityComparer nullSafeComparer, + final @Nullable NullablePredicate2 nonEquiPredicate, + final NullablePredicate2 equiPredicate) { + return new AbstractEnumerable() { + @Override public Enumerator enumerator() { + HashTableWithNullSafeKeySet ht = + HashTableWithNullSafeKeySet.build(inner, innerKeyNullAwareSelector, + innerNullSafeKeySelector, comparer, nullSafeComparer); + + return new Enumerator() { + Enumerator outers = outer.enumerator(); + @Nullable Boolean marker = false; + + @Override public TResult current() { + return resultSelector.apply(outers.current(), marker); + } + + @Override public boolean moveNext() { + if (!outers.moveNext()) { + return false; + } + marker = false; + final TSource outerRow = outers.current(); + final TKey outerKey = outerKeyNullAwareSelector.apply(outerRow); + if (outerNullSafeKeySelector != null + && !ht.containsNullSafeKey(outerNullSafeKeySelector.apply(outerRow))) { + // there are null-safe keys, but there is no match in the hash table of null-safe + // keys. The marker is FALSE + return true; + } + + if (outerKey == null) { + // outerRow has NULL values on at least one not null-safe key. Need to apply the + // equi-predicate against all right-side rows to determine the marker is FALSE or NULL + flag: + for (Enumerable eachInnerEnumerable : ht.lookup.values()) { + try (Enumerator eachInnerEnumerator = eachInnerEnumerable.enumerator()) { + while (eachInnerEnumerator.moveNext()) { + TInner eachInnerRow = eachInnerEnumerator.current(); + Boolean equiPredicateMatched = equiPredicate.apply(outerRow, eachInnerRow); + if (equiPredicateMatched == null) { + marker = null; + break flag; + } + } + } + } + } else { + Enumerable innerEnumerable = ht.lookup.get(outerKey); + if (innerEnumerable == null) { + // no match found in the hash table. If there are any NULL values on not + // null-safe keys, need to apply the equi-predicate against those rows to determine + // the marker is FALSE or NULL. + Enumerable nullValueOnNotNullSafeKey = ht.lookup.get(null); + if (nullValueOnNotNullSafeKey != null) { + try (Enumerator enumerator = nullValueOnNotNullSafeKey.enumerator()) { + while (enumerator.moveNext()) { + TInner nullValueOnNotNullSafeKeyRow = enumerator.current(); + Boolean equiPredicateMatched = + equiPredicate.apply(outerRow, nullValueOnNotNullSafeKeyRow); + if (equiPredicateMatched == null) { + marker = null; + break; + } + } + } + } + } else { + if (nonEquiPredicate == null) { + marker = true; + } else { + try (Enumerator innerEnumerator = innerEnumerable.enumerator()) { + while (innerEnumerator.moveNext()) { + final TInner innerRow = innerEnumerator.current(); + Boolean predicateMatched = nonEquiPredicate.apply(outerRow, innerRow); + if (predicateMatched == null) { + marker = null; + } else if (predicateMatched) { + marker = true; + break; + } + } + } + } + } + } + if (marker == null && ht.buildSideIsEmpty) { + marker = false; + } + return true; + } + + @Override public void reset() { + outers.reset(); + } + + @Override public void close() { + outers.close(); + } + }; + } + }; + } + /** * For each row of the {@code outer} enumerable returns the correlated rows * from the {@code inner} enumerable. @@ -3801,21 +4081,8 @@ static LookupImpl toLookup_( while (os.moveNext()) { TSource o = os.current(); final TKey key = keySelector.apply(o); - @SuppressWarnings("nullness") - List list = map.get(key); - if (list == null) { - // for first entry, use a singleton list to save space - list = Collections.singletonList(elementSelector.apply(o)); - } else { - if (list.size() == 1) { - // when we go from 1 to 2 elements, switch to array list - TElement element = list.get(0); - list = new ArrayList<>(); - list.add(element); - } - list.add(elementSelector.apply(o)); - } - map.put(key, list); + final TElement data = elementSelector.apply(o); + appendDataForKey(map, key, data); } } return new LookupImpl<>(map); @@ -3840,6 +4107,24 @@ public static Lookup toLookup( elementSelector); } + private static void appendDataForKey( + Map> map, TKey key, TElement row) { + List list = map.get(key); + if (list == null) { + // for first entry, use a singleton list to save space + list = Collections.singletonList(row); + } else { + if (list.size() == 1) { + // when we go from 1 to 2 elements, switch to array list + TElement element = list.get(0); + list = new ArrayList<>(); + list.add(element); + } + list.add(row); + } + map.put(key, list); + } + /** * Produces the set union of two sequences by using * the default equality comparer. @@ -4024,6 +4309,63 @@ public static > C remove( return sink; } + /** + * Hash table with null-safe key set. + * + * @param key type + * @param null-safe key type + * @param build side row type + */ + static class HashTableWithNullSafeKeySet { + final Lookup lookup; + final Set> nullSafeKeySet; + final EqualityComparer nullSafeComparer; + // whether the build side is empty set + final boolean buildSideIsEmpty; + + private HashTableWithNullSafeKeySet( + Lookup lookup, + Set> nullSafeKeySet, + EqualityComparer nullSafeComparer) { + this.lookup = lookup; + this.nullSafeKeySet = nullSafeKeySet; + this.nullSafeComparer = nullSafeComparer; + this.buildSideIsEmpty = lookup.isEmpty(); + } + + static HashTableWithNullSafeKeySet build( + Enumerable data, + Function1 keyNullAwareSelector, + @Nullable Function1 nullSafeKeySelector, + @Nullable EqualityComparer comparer, + @Nullable EqualityComparer nullSafeComparer) { + Map> map = + comparer == null + ? new HashMap<>() + : new WrapMap<>(() -> new HashMap, List>(), comparer); + Set> nullSafeKeySet = new HashSet<>(); + nullSafeComparer = nullSafeComparer == null ? Functions.identityComparer() : nullSafeComparer; + + try (Enumerator enumerator = data.enumerator()) { + while (enumerator.moveNext()) { + TData row = enumerator.current(); + TKey nullAwareKey = keyNullAwareSelector.apply(row); + appendDataForKey(map, nullAwareKey, row); + if (nullSafeKeySelector != null) { + TNsKey nullSafeKey = nullSafeKeySelector.apply(row); + nullSafeKeySet.add(Wrapped.upAs(nullSafeComparer, nullSafeKey)); + } + } + } + Lookup nullAwareLookup = new LookupImpl<>(map); + return new HashTableWithNullSafeKeySet<>(nullAwareLookup, nullSafeKeySet, nullSafeComparer); + } + + public boolean containsNullSafeKey(TNsKey key) { + return nullSafeKeySet.contains(Wrapped.upAs(nullSafeComparer, key)); + } + } + /** Enumerable that implements take-while. * * @param element type */ diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedEnumerable.java b/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedEnumerable.java index 9c11f83dd769..67887e7d087b 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedEnumerable.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedEnumerable.java @@ -30,6 +30,7 @@ import org.apache.calcite.linq4j.function.NullableFloatFunction1; import org.apache.calcite.linq4j.function.NullableIntegerFunction1; import org.apache.calcite.linq4j.function.NullableLongFunction1; +import org.apache.calcite.linq4j.function.NullablePredicate2; import org.apache.calcite.linq4j.function.Predicate1; import org.apache.calcite.linq4j.function.Predicate2; @@ -646,6 +647,45 @@ Enumerable hashJoin(Enumerable inner, boolean generateNullsOnLeft, boolean generateNullsOnRight, Predicate2 predicate); + /** + * Mark each row of the current enumerable to see if it has a join partner in the + * inner. Whether a join partner exists depends on: + * - matching keys + * - non-equi predicate (if provided) + * + *

    Refer to + * The Complete Story of Joins (in HyPer). + * + * @param inner Inner enumerable + * @param outerKeyNullAwareSelector Function that extracts keys from the current enumerable + * (return NULL when a not null-safe key has a NULL value) + * @param innerKeyNullAwareSelector Function that extracts keys from the inner enumerable + * (return NULL when a not null-safe key has a NULL value) + * @param outerNullSafeKeySelector Function that extracts null-safe keys from the current + * enumerable + * @param innerNullSafeKeySelector Function that extracts null-safe keys from the inner + * enumerable + * @param atMostOneNotNullSafeKey True when there is at most one not null-safe key in join + * keys + * @param resultSelector Function that concat the row of the current enumerable and + * marker + * @param comparer Function that compares the keys + * @param nullSafeComparer Function that compares the null-safe keys + * @param nonEquiPredicate Non-equi predicate that can return NULL + * @param equiPredicate Equi predicate that can return NULL + */ + Enumerable leftMarkHashJoin(Enumerable inner, + Function1 outerKeyNullAwareSelector, + Function1 innerKeyNullAwareSelector, + Function1 outerNullSafeKeySelector, + Function1 innerNullSafeKeySelector, + boolean atMostOneNotNullSafeKey, + Function2 resultSelector, + EqualityComparer comparer, + EqualityComparer nullSafeComparer, + NullablePredicate2 nonEquiPredicate, + NullablePredicate2 equiPredicate); + /** * For each row of the current enumerable returns the correlated rows * from the {@code inner} enumerable (nested loops join). diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/JoinType.java b/linq4j/src/main/java/org/apache/calcite/linq4j/JoinType.java index 4b38f145ea6c..d5c15ed2240b 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/JoinType.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/JoinType.java @@ -87,7 +87,32 @@ public enum JoinType { /** * The left version of an ASOF join, where each row from the left table is part of the output. */ - LEFT_ASOF; + LEFT_ASOF, + + /** + * An LEFT MARK JOIN will keep all rows from the left side and creates a new attribute to mark a + * tuple as having join partners from right side or not. Refer to + * + * The Complete Story of Joins (in HyPer). + * + *

    Example: + * 

    +   * SELECT EMPNO FROM EMP
+   * WHERE EXISTS (SELECT 1 FROM DEPT
+   *     WHERE DEPT.DEPTNO = EMP.DEPTNO)
+   *     OR EMPNO > 1
+   *
+   * LogicalProject(EMPNO=[$0])
+   *   LogicalFilter(condition=[OR($9, >($0, 1))])
+   *     LogicalJoin(condition=[IS NOT DISTINCT FROM($7, $9)], joinType=[left_mark])
+   *       LogicalTableScan(table=[[CATALOG, SALES, EMP]])
+   *       LogicalTableScan(table=[[CATALOG, SALES, DEPT]])
+   *
    + * + *

    If the marker is used on only conjunctive predicates the optimizer will try to translate + * the mark join into semi or anti join. + */ + LEFT_MARK; /** * Returns whether a join of this type may generate NULL values on the diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/function/NullablePredicate2.java b/linq4j/src/main/java/org/apache/calcite/linq4j/function/NullablePredicate2.java new file mode 100644 index 000000000000..f413db463559 --- /dev/null +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/function/NullablePredicate2.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.linq4j.function; + +/** + * Function with two parameters returning a {@link Boolean} value that may be null. + * + * @param Type of argument #0 + * @param Type of argument #1 + */ +public interface NullablePredicate2 extends Function2 { + @Override Boolean apply(T0 v0, T1 v1); +} From 444baa412e5c9a16d88641d4d4a29b38dadbaedc Mon Sep 17 00:00:00 2001 From: "lincoln.lil" Date: Mon, 12 Jan 2026 14:33:22 +0800 Subject: [PATCH 103/562] [CALCITE-7369] ProjectToWindowRule loses column alias when optimizing OVER window queries --- .../calcite/rel/logical/LogicalWindow.java | 3 +- .../calcite/rel/rules/CalcRelSplitter.java | 63 ++++++++++++++--- .../rel/logical/ToLogicalConverterTest.java | 8 +-- .../rel/rel2sql/RelToSqlConverterTest.java | 32 ++++----- .../apache/calcite/test/RelOptRulesTest.java | 33 +++++++++ .../apache/calcite/test/RelOptRulesTest.xml | 67 ++++++++++++++++--- core/src/test/resources/sql/sub-query.iq | 4 +- core/src/test/resources/sql/winagg.iq | 2 +- .../org/apache/calcite/test/PigRelOpTest.java | 4 +- 9 files changed, 170 insertions(+), 46 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rel/logical/LogicalWindow.java b/core/src/main/java/org/apache/calcite/rel/logical/LogicalWindow.java index 815b8f8ca596..16b247974b3d 100644 --- a/core/src/main/java/org/apache/calcite/rel/logical/LogicalWindow.java +++ b/core/src/main/java/org/apache/calcite/rel/logical/LogicalWindow.java @@ -244,11 +244,12 @@ public static RelNode create(RelOptCluster cluster, } } + int callIndex = 0; for (Ord window : Ord.zip(groups)) { for (Ord over : Ord.zip(window.e.aggCalls)) { // Add the k-th over expression of // the i-th window to the output of the program. - String name = fieldNames.get(over.i); + String name = fieldNames.get(callIndex++); if (name == null || name.startsWith("$")) { name = "w" + window.i + "$o" + over.i; } diff --git a/core/src/main/java/org/apache/calcite/rel/rules/CalcRelSplitter.java b/core/src/main/java/org/apache/calcite/rel/rules/CalcRelSplitter.java index 67bbbae24626..17e6333dc223 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/CalcRelSplitter.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/CalcRelSplitter.java @@ -605,7 +605,7 @@ private RexProgram createProgramForLevel( projectRefs.add(new RexLocalRef(index, expr.getType())); // Inherit meaningful field name if possible. - fieldNames.add(deriveFieldName(expr, i)); + fieldNames.add(deriveFieldName(expr, projectExprOrdinal, i)); } RexLocalRef conditionRef; if (conditionExprOrdinal >= 0) { @@ -627,18 +627,61 @@ private RexProgram createProgramForLevel( outputRowType); } - private String deriveFieldName(RexNode expr, int ordinal) { + /** + * Derives a field name for a projected expression. + * + *

    If {@code expr} is a {@link RexInputRef}, returns the corresponding + * input field name. Otherwise, attempts to retrieve the name from the + * original program's projections. If no meaningful name is found, or if + * the name looks like an auto-generated name such as {@code $n} (but not + * starting with {@code $EXPR}), returns a synthesized name {@code "$" + ordinal}. + * + * @param expr Expression to derive the name for + * @param exprIndex Index of the expression in the program's expression list + * @param ordinal Position in the projection (used to generate a fallback name) + * @return Derived or synthesized field name + */ + private String deriveFieldName(RexNode expr, int exprIndex, int ordinal) { + String fieldName = null; if (expr instanceof RexInputRef) { - int inputIndex = ((RexInputRef) expr).getIndex(); - String fieldName = - child.getRowType().getFieldList().get(inputIndex).getName(); - // Don't inherit field names like '$3' from child: that's - // confusing. - if (!fieldName.startsWith("$") || fieldName.startsWith("$EXPR")) { - return fieldName; + fieldName = getInputRefName((RexInputRef) expr); + } else { + fieldName = findProjectedFieldName(exprIndex); + } + return normalizeFieldName(fieldName, ordinal); + } + + private String getInputRefName(RexInputRef ref) { + int inputIndex = ref.getIndex(); + return child.getRowType().getFieldList().get(inputIndex).getName(); + } + + /** + * Return the output field name corresponding to the given expression index {@code exprIndex}, + * or {@code null} if the expression is not part of the program's projection. + * + * @param exprIndex Index of the expression in the program's expression list + * @return the output field name for the given expression index, or {@code null} if not projected + */ + private @Nullable String findProjectedFieldName(int exprIndex) { + List projects = program.getProjectList(); + List fieldNames = program.getOutputRowType().getFieldNames(); + for (int i = 0; i < projects.size(); i++) { + // If the project entry refers to the given expression, return its name. + if (projects.get(i).getIndex() == exprIndex) { + return fieldNames.get(i); } } - return "$" + ordinal; + return null; + } + + private String normalizeFieldName(@Nullable String fieldName, int ordinal) { + // Don't inherit field names like '$3' from child: that's confusing. + if (fieldName == null + || (fieldName.startsWith("$") && !fieldName.startsWith("$EXPR"))) { + return "$" + ordinal; + } + return fieldName; } /** diff --git a/core/src/test/java/org/apache/calcite/rel/logical/ToLogicalConverterTest.java b/core/src/test/java/org/apache/calcite/rel/logical/ToLogicalConverterTest.java index 062bea6d50ae..fb7deb5b8a30 100644 --- a/core/src/test/java/org/apache/calcite/rel/logical/ToLogicalConverterTest.java +++ b/core/src/test/java/org/apache/calcite/rel/logical/ToLogicalConverterTest.java @@ -477,12 +477,12 @@ private void verify(RelNode rel, String expectedPhysical, String expectedLogical @Test void testWindow() { String sql = "SELECT rank() over (order by \"hire_date\") FROM \"employee\""; String expectedPhysical = "" - + "EnumerableProject($0=[$17])\n" + + "EnumerableProject(EXPR$0=[$17])\n" + " EnumerableWindow(window#0=[window(order by [9] aggs [RANK()])])\n" + " JdbcToEnumerableConverter\n" + " JdbcTableScan(table=[[foodmart, employee]])\n"; String expectedLogical = "" - + "LogicalProject($0=[$17])\n" + + "LogicalProject(EXPR$0=[$17])\n" + " LogicalWindow(window#0=[window(order by [9] aggs [RANK()])])\n" + " LogicalTableScan(table=[[foodmart, employee]])\n"; verify(rel(sql), expectedPhysical, expectedLogical); @@ -493,13 +493,13 @@ void testWindowExcludeImp(String excludeClause, String expectedExcludeString) { String sql = String.format(Locale.ROOT, "SELECT sum(\"salary\") over (order by \"hire_date\" " + "rows between unbounded preceding and current row %s) FROM \"employee\"", excludeClause); String expectedPhysical = - String.format(Locale.ROOT, "EnumerableProject($0=[$17])\n" + String.format(Locale.ROOT, "EnumerableProject(EXPR$0=[$17])\n" + " EnumerableWindow(window#0=[window(order by [9] rows between" + " UNBOUNDED PRECEDING and CURRENT ROW %saggs [SUM($11)])])\n" + " JdbcToEnumerableConverter\n" + " JdbcTableScan(table=[[foodmart, employee]])\n", expectedExcludeString); String expectedLogical = - String.format(Locale.ROOT, "LogicalProject($0=[$17])\n" + String.format(Locale.ROOT, "LogicalProject(EXPR$0=[$17])\n" + " LogicalWindow(window#0=[window(order by [9] rows between" + " UNBOUNDED PRECEDING and CURRENT ROW %saggs [SUM($11)])])\n" + " LogicalTableScan(table=[[foodmart, employee]])\n", expectedExcludeString); diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java index 5a9b278c3571..a44d05ce648f 100644 --- a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java +++ b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java @@ -5566,11 +5566,11 @@ private void checkLiteral2(String expression, String expected) { * Support Window in RelToSqlConverter. */ @Test void testConvertWindowToSql() { String query0 = "SELECT row_number() over (order by \"hire_date\") FROM \"employee\""; - String expected0 = "SELECT ROW_NUMBER() OVER (ORDER BY \"hire_date\") AS \"$0\"\n" + String expected0 = "SELECT ROW_NUMBER() OVER (ORDER BY \"hire_date\")\n" + "FROM \"foodmart\".\"employee\""; String query1 = "SELECT rank() over (order by \"hire_date\") FROM \"employee\""; - String expected1 = "SELECT RANK() OVER (ORDER BY \"hire_date\") AS \"$0\"\n" + String expected1 = "SELECT RANK() OVER (ORDER BY \"hire_date\")\n" + "FROM \"foodmart\".\"employee\""; String query2 = "SELECT lead(\"employee_id\",1,'NA') over " @@ -5578,14 +5578,14 @@ private void checkLiteral2(String expression, String expected) { + "FROM \"employee\""; String expected2 = "SELECT LEAD(\"employee_id\", 1, 'NA') OVER " + "(PARTITION BY \"hire_date\" " - + "ORDER BY \"employee_id\") AS \"$0\"\n" + + "ORDER BY \"employee_id\")\n" + "FROM \"foodmart\".\"employee\""; String query3 = "SELECT lag(\"employee_id\",1,'NA') over " + "(partition by \"hire_date\" order by \"employee_id\")\n" + "FROM \"employee\""; String expected3 = "SELECT LAG(\"employee_id\", 1, 'NA') OVER " - + "(PARTITION BY \"hire_date\" ORDER BY \"employee_id\") AS \"$0\"\n" + + "(PARTITION BY \"hire_date\" ORDER BY \"employee_id\")\n" + "FROM \"foodmart\".\"employee\""; String query4 = "SELECT lag(\"employee_id\",1,'NA') " @@ -5596,13 +5596,13 @@ private void checkLiteral2(String expression, String expected) { + "count(*) over (partition by \"birth_date\" order by \"employee_id\") as count2\n" + "FROM \"employee\""; String expected4 = "SELECT LAG(\"employee_id\", 1, 'NA') OVER " - + "(PARTITION BY \"hire_date\" ORDER BY \"employee_id\") AS \"$0\", " + + "(PARTITION BY \"hire_date\" ORDER BY \"employee_id\") AS \"LAG1\", " + "LAG(\"employee_id\", 1, 'NA') OVER " - + "(PARTITION BY \"birth_date\" ORDER BY \"employee_id\") AS \"$1\", " + + "(PARTITION BY \"birth_date\" ORDER BY \"employee_id\") AS \"LAG2\", " + "COUNT(*) OVER (PARTITION BY \"hire_date\" ORDER BY \"employee_id\" " - + "RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS \"$2\", " + + "RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS \"COUNT1\", " + "COUNT(*) OVER (PARTITION BY \"birth_date\" ORDER BY \"employee_id\" " - + "RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS \"$3\"\n" + + "RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS \"COUNT2\"\n" + "FROM \"foodmart\".\"employee\""; String query5 = "SELECT lag(\"employee_id\",1,'NA') " @@ -5613,13 +5613,13 @@ private void checkLiteral2(String expression, String expected) { + "max(sum(\"employee_id\")) over (partition by \"birth_date\" order by \"employee_id\") as count2\n" + "FROM \"employee\" group by \"employee_id\", \"hire_date\", \"birth_date\""; String expected5 = "SELECT LAG(\"employee_id\", 1, 'NA') OVER " - + "(PARTITION BY \"hire_date\" ORDER BY \"employee_id\") AS \"$0\", " + + "(PARTITION BY \"hire_date\" ORDER BY \"employee_id\") AS \"LAG1\", " + "LAG(\"employee_id\", 1, 'NA') OVER " - + "(PARTITION BY \"birth_date\" ORDER BY \"employee_id\") AS \"$1\", " + + "(PARTITION BY \"birth_date\" ORDER BY \"employee_id\") AS \"LAG2\", " + "MAX(SUM(\"employee_id\")) OVER (PARTITION BY \"hire_date\" ORDER BY \"employee_id\" " - + "RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS \"$2\", " + + "RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS \"COUNT1\", " + "MAX(SUM(\"employee_id\")) OVER (PARTITION BY \"birth_date\" ORDER BY \"employee_id\" " - + "RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS \"$3\"\n" + + "RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS \"COUNT2\"\n" + "FROM \"foodmart\".\"employee\"\n" + "GROUP BY \"employee_id\", \"hire_date\", \"birth_date\""; @@ -5635,7 +5635,7 @@ private void checkLiteral2(String expression, String expected) { + "count(distinct \"employee_id\") over (order by \"hire_date\") FROM \"employee\""; String expected7 = "SELECT " + "COUNT(DISTINCT \"employee_id\") OVER (ORDER BY \"hire_date\"" - + " RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS \"$0\"\n" + + " RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)\n" + "FROM \"foodmart\".\"employee\""; String query8 = "SELECT " @@ -5823,9 +5823,9 @@ private void checkLiteral2(String expression, String expected) { + "FROM ( SELECT \"product_name\", " + "SUM(\"product_id\") OVER (PARTITION BY \"product_name\") AS \"daily_sales\" " + "FROM \"product\" ) subquery"; - String expected00 = "SELECT RANK() OVER (ORDER BY \"$1\" DESC) AS \"$0\"\n" + String expected00 = "SELECT RANK() OVER (ORDER BY \"daily_sales\" DESC) AS \"rank1\"\n" + "FROM (SELECT \"product_name\", SUM(\"product_id\") OVER (PARTITION BY \"product_name\" " - + "RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS \"$1\"\n" + + "RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS \"daily_sales\"\n" + "FROM \"foodmart\".\"product\") AS \"t0\""; String expected01 = "SELECT RANK() OVER (ORDER BY \"daily_sales\" DESC) AS \"rank1\"\n" + "FROM (SELECT \"product_name\", SUM(\"product_id\") OVER (PARTITION BY \"product_name\"" @@ -5840,7 +5840,7 @@ private void checkLiteral2(String expression, String expected) { + "RANK() OVER (ORDER BY \"product_name\" DESC) AS \"rank1\" " + "FROM (SELECT \"product_id\", \"product_name\" FROM \"product\") a"; String expected10 = "SELECT \"product_id\"," - + " RANK() OVER (ORDER BY \"product_name\" DESC) AS \"$1\"\n" + + " RANK() OVER (ORDER BY \"product_name\" DESC) AS \"rank1\"\n" + "FROM \"foodmart\".\"product\""; String expected11 = "SELECT \"product_id\"," + " RANK() OVER (ORDER BY \"product_name\" DESC) AS \"rank1\"\n" diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index a83905434972..7de683e9ac66 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -12059,4 +12059,37 @@ private void checkLoptOptimizeJoinRule(LoptOptimizeJoinRule rule) { .check(); } + /** Test case of + * [CALCITE-7369] + * ProjectToWindowRule loses column alias when optimizing OVER window queries. */ + @Test void testProjectToWindowRuleForNestedOver() { + HepProgramBuilder builder = new HepProgramBuilder(); + builder.addRuleClass(ProjectToWindowRule.class); + HepPlanner hepPlanner = new HepPlanner(builder.build()); + hepPlanner.addRule(CoreRules.PROJECT_TO_LOGICAL_PROJECT_AND_WINDOW); + + final String sql = + "select deptno, f1, f2 from (select *, last_value(deptno) over (order by empno) f2\n" + + "from (select *, first_value(deptno) over (order by empno) f1 from emp))\n"; + sql(sql).withPlanner(hepPlanner) + .check(); + } + + /** Test case of + * [CALCITE-7369] + * ProjectToWindowRule loses column alias when optimizing OVER window queries. */ + @Test void testProjectToWindowRuleForMultiOver() { + HepProgramBuilder builder = new HepProgramBuilder(); + builder.addRuleClass(ProjectToWindowRule.class); + HepPlanner hepPlanner = new HepPlanner(builder.build()); + hepPlanner.addRule(CoreRules.PROJECT_TO_LOGICAL_PROJECT_AND_WINDOW); + + final String sql = "select * from (" + + "select empno, deptno, last_value(deptno) over (order by empno) f1,\n" + + "first_value(deptno) over (order by empno desc) f3,\n" + + "count(deptno) over (order by empno) f2\n" + + "from emp) where f2 > 10"; + sql(sql).withPlanner(hepPlanner) + .check(); + } } diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index 465ce0e3e1eb..7c56417de634 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -5439,7 +5439,7 @@ from emp]]> @@ -11833,6 +11833,31 @@ LogicalProject(DNAME=[$1]) LogicalAggregate(group=[{0}]) LogicalProject(DEPTNO=[$7]) LogicalTableScan(table=[[scott, EMP]]) +]]> + + + + + 10]]> + + + ($4, 10)]) + LogicalProject(EMPNO=[$0], DEPTNO=[$7], F1=[LAST_VALUE($7) OVER (ORDER BY $0)], F3=[FIRST_VALUE($7) OVER (ORDER BY $0 DESC)], F2=[COUNT($7) OVER (ORDER BY $0)]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +]]> + + + ($4, 10)]) + LogicalProject(EMPNO=[$0], DEPTNO=[$7], F1=[$9], F3=[$11], F2=[$10]) + LogicalWindow(window#0=[window(order by [0] aggs [LAST_VALUE($7), COUNT($7)])], window#1=[window(order by [0 DESC] aggs [FIRST_VALUE($7)])]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) ]]> @@ -11847,7 +11872,7 @@ from emp]]> @@ -11856,6 +11881,28 @@ LogicalProject($0=[$9], $1=[$11], $2=[$10], $3=[$12]) + + + + + + + + + + + @@ -11889,7 +11936,7 @@ from ( ($t3, $t4)], expr#6=[null:TINYINT], expr#7=[CASE($t5, $t0, $t6)], expr#8=[IS NOT NULL($t7)], expr#9=[CAST($t7):INTEGER], expr#10=[Sarg[10; NULL AS TRUE]], expr#11=[SEARCH($t9, $t10)], DEPTNO=[$t0], $1=[$t8], $condition=[$t11]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[CAST($t0):INTEGER NOT NULL], expr#4=[0], expr#5=[>($t3, $t4)], expr#6=[null:TINYINT], expr#7=[CASE($t5, $t0, $t6)], expr#8=[IS NOT NULL($t7)], expr#9=[CAST($t7):INTEGER], expr#10=[Sarg[10; NULL AS TRUE]], expr#11=[SEARCH($t9, $t10)], DEPTNO=[$t0], cs=[$t8], $condition=[$t11]) EnumerableTableScan(table=[[scott, DEPT]]) !plan diff --git a/core/src/test/resources/sql/winagg.iq b/core/src/test/resources/sql/winagg.iq index feb4dc695107..beb90d9fd4df 100644 --- a/core/src/test/resources/sql/winagg.iq +++ b/core/src/test/resources/sql/winagg.iq @@ -1003,7 +1003,7 @@ select gender, count(*) over(partition by gender order by ename) as count1 from # Get the plan and result which push filter past window select gender, count(*) over(partition by gender order by ename) as count1 from emp where gender = 'F'; -EnumerableCalc(expr#0..3=[{inputs}], GENDER=[$t2], $1=[$t3]) +EnumerableCalc(expr#0..3=[{inputs}], GENDER=[$t2], COUNT1=[$t3]) EnumerableWindow(window#0=[window(partition {2} order by [0] aggs [COUNT()])]) EnumerableCalc(expr#0..2=[{inputs}], expr#3=['F'], expr#4=[=($t2, $t3)], proj#0..2=[{exprs}], $condition=[$t4]) EnumerableValues(tuples=[[{ 'Jane ', 10, 'F' }, { 'Bob ', 10, 'M' }, { 'Eric ', 20, 'M' }, { 'Susan', 30, 'F' }, { 'Alice', 30, 'F' }, { 'Adam ', 50, 'M' }, { 'Eve ', 50, 'F' }, { 'Grace', 60, 'F' }, { 'Wilma', null, 'F' }]]) diff --git a/piglet/src/test/java/org/apache/calcite/test/PigRelOpTest.java b/piglet/src/test/java/org/apache/calcite/test/PigRelOpTest.java index 5067503b89c1..e1c60c10cb31 100644 --- a/piglet/src/test/java/org/apache/calcite/test/PigRelOpTest.java +++ b/piglet/src/test/java/org/apache/calcite/test/PigRelOpTest.java @@ -1665,10 +1665,10 @@ private Fluent pig(String script) { + " name=[$1], age=[$2], city=[$3])\n" + " LogicalTableScan(table=[[emp1]])\n"; - final String sql = "SELECT w0$o0 AS rank_A, id, name, age, city\n" + final String sql = "SELECT rank_A, id, name, age, city\n" + "FROM (SELECT id, name, age, city, RANK() OVER ()\n" + " FROM emp1) AS t\n" - + "WHERE w0$o0 > 1"; + + "WHERE rank_A > 1"; pig(script).assertRel(hasTree(plan)) .assertSql(is(sql)); } From 2794697fab9bae929fde30e3ce265f123e73ab81 Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Tue, 13 Jan 2026 19:53:42 +0800 Subject: [PATCH 104/562] Opened some iq files in CoreQuidemTest2 --- .../test/java/org/apache/calcite/test/CoreQuidemTest2.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/core/src/test/java/org/apache/calcite/test/CoreQuidemTest2.java b/core/src/test/java/org/apache/calcite/test/CoreQuidemTest2.java index 406bd3bc8f34..5ad60a3221b6 100644 --- a/core/src/test/java/org/apache/calcite/test/CoreQuidemTest2.java +++ b/core/src/test/java/org/apache/calcite/test/CoreQuidemTest2.java @@ -45,15 +45,11 @@ public static void main(String[] args) throws Exception { // once the new decorrelator can adapt to all scenarios. // TODO: The following files involves UNNEST and LEFT_MARK JOIN - paths.remove("sql/agg.iq"); paths.remove("sql/measure.iq"); paths.remove("sql/unnest.iq"); - paths.remove("sql/lateral.iq"); paths.remove("sql/some.iq"); paths.remove("sql/sub-query.iq"); paths.remove("sql/scalar.iq"); - paths.remove("sql/join.iq"); - paths.remove("sql/spatial.iq"); paths.remove("sql/measure-paper.iq"); paths.remove("sql/misc.iq"); return paths; From 545db9e7da1236bf6a04343e49fb8ad9644d004b Mon Sep 17 00:00:00 2001 From: ehds Date: Tue, 13 Jan 2026 17:53:59 +0800 Subject: [PATCH 105/562] [CALCITE-7370] Trailing dot is not removed when normalizing timestamp strings --- .../main/java/org/apache/calcite/util/TimestampString.java | 4 ++++ core/src/test/java/org/apache/calcite/rex/RexBuilderTest.java | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/core/src/main/java/org/apache/calcite/util/TimestampString.java b/core/src/main/java/org/apache/calcite/util/TimestampString.java index c6c7c18adeb4..a4b16b2896d9 100644 --- a/core/src/main/java/org/apache/calcite/util/TimestampString.java +++ b/core/src/main/java/org/apache/calcite/util/TimestampString.java @@ -126,6 +126,10 @@ private static String normalize(String v) { while (v.endsWith("0")) { v = v.substring(0, v.length() - 1); } + // Remove trailing dot + if (v.endsWith(".")) { + v = v.substring(0, v.length() - 1); + } } checkArgument(PATTERN.matcher(v).matches(), v); return v; diff --git a/core/src/test/java/org/apache/calcite/rex/RexBuilderTest.java b/core/src/test/java/org/apache/calcite/rex/RexBuilderTest.java index 4b714e8632d2..7653d5370c76 100644 --- a/core/src/test/java/org/apache/calcite/rex/RexBuilderTest.java +++ b/core/src/test/java/org/apache/calcite/rex/RexBuilderTest.java @@ -357,6 +357,10 @@ private static class MySqlTypeFactoryImpl extends SqlTypeFactoryImpl { final TimestampString ts10 = TimestampString.fromCalendarFields(c); assertThat(ts10, hasToString("1969-02-26 19:06:00.987")); assertThat(ts10.getMillisSinceEpoch(), is(c.getTimeInMillis())); + + // TimestampString with all zeros fraction + final TimestampString ts11 = new TimestampString("2016-02-26 19:06:00.000"); + assertThat(ts11, hasToString("2016-02-26 19:06:00")); } @Test void testTimeString() { From 9705ff0e691832c5bca2e3d5413dfa05ef4a61a5 Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Wed, 14 Jan 2026 09:02:27 +0800 Subject: [PATCH 106/562] [CALCITE-5597] SELECT DISTINCT query with ORDER BY column will get error result --- .../calcite/sql2rel/SqlToRelConverter.java | 192 +++++++++++++----- .../calcite/test/SqlToRelConverterTest.java | 8 + .../calcite/test/SqlToRelConverterTest.xml | 15 ++ 3 files changed, 165 insertions(+), 50 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java index 1ad87eaed62b..8fc83104c1d1 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java @@ -842,8 +842,14 @@ protected void convertSelectImpl( } /** - * Having translated 'SELECT ... FROM ... [GROUP BY ...] [HAVING ...]', adds - * a relational expression to make the results unique. + * The translation of 'SELECT ... FROM ... [GROUP BY ...] [HAVING ...]' uses + * an {@link org.apache.calcite.rel.core.Aggregate}. + * + *

    For example, {@code SELECT DISTINCT x FROM t ORDER BY y} is converted + * to an {@code Aggregate} on {@code (x, y)} if {@code y} is deterministic. + * If {@code y} is non-deterministic (e.g. {@code RAND()}), it is converted + * to an {@code Aggregate} on {@code x}, and {@code y} is applied over the + * result. * *

    If the SELECT clause contains duplicate expressions, adds * {@link org.apache.calcite.rel.logical.LogicalProject}s so that we are @@ -864,69 +870,155 @@ private void distinctify( throw new IllegalArgumentException("rel must not be null"); } final RelNode rel = bb.root; + int groupCount = rel.getRowType().getFieldCount(); + if (bb.scope != null && bb.scope.getNode() instanceof SqlSelect) { + groupCount = validator().getValidatedNodeType(bb.scope.getNode()).getFieldCount(); + } + distinctify(bb, checkForDupExprs, groupCount); + } + + /** + * The translation of 'SELECT ... FROM ... [GROUP BY ...] [HAVING ...]' uses + * an {@link org.apache.calcite.rel.core.Aggregate}. + * + *

    For example, {@code SELECT DISTINCT x FROM t ORDER BY y} is converted + * to an {@code Aggregate} on {@code (x, y)} if {@code y} is deterministic. + * If {@code y} is non-deterministic (e.g. {@code RAND()}), it is converted + * to an {@code Aggregate} on {@code x}, and {@code y} is applied over the + * result. + * + *

    If the SELECT clause contains duplicate expressions, adds + * {@link org.apache.calcite.rel.logical.LogicalProject}s so that we are + * grouping on the minimal set of keys. The performance gain isn't huge, but + * it is difficult to detect these duplicate expressions later. + * + * @param bb Blackboard + * @param checkForDupExprs Check for duplicate expressions + * @param groupCount Number of fields in the SELECT clause + */ + private void distinctify( + Blackboard bb, + boolean checkForDupExprs, + int groupCount) { + if (bb.root == null) { + throw new IllegalArgumentException("rel must not be null"); + } + RelNode rel = bb.root; + + // 1. Handle duplicate expressions in the Project if requested. if (checkForDupExprs && (rel instanceof LogicalProject)) { - LogicalProject project = (LogicalProject) rel; + final LogicalProject project = (LogicalProject) rel; final List projectExprs = project.getProjects(); final List origins = new ArrayList<>(); - int dupCount = 0; + final Map seen = new HashMap<>(); for (int i = 0; i < projectExprs.size(); i++) { - int x = projectExprs.indexOf(projectExprs.get(i)); - if (x >= 0 && x < i) { - origins.add(x); - ++dupCount; - } else { - origins.add(i); + Integer first = seen.putIfAbsent(projectExprs.get(i), i); + origins.add(first != null ? first : i); + } + + if (seen.size() < projectExprs.size()) { + final List fields = rel.getRowType().getFieldList(); + final PairList newProjects = PairList.of(); + final List mapping = new ArrayList<>(); + for (int i = 0; i < fields.size(); i++) { + if (origins.get(i) == i) { + mapping.add(newProjects.size()); + newProjects.add(projectExprs.get(i), fields.get(i).getName()); + } else { + mapping.add(-1); + } } - } - if (dupCount == 0) { - distinctify(bb, false); + bb.setRoot( + LogicalProject.create(project.getInput(), project.getHints(), + newProjects.leftList(), newProjects.rightList(), + project.getVariablesSet()), false); + + int newGroupCount = 0; + for (int i = 0; i < groupCount; i++) { + if (origins.get(i) == i) { + newGroupCount++; + } + } + distinctify(bb, false, newGroupCount); + + final RelNode distinctRel = bb.root(); + final PairList undoProjects = PairList.of(); + for (int i = 0; i < fields.size(); i++) { + final int origin = origins.get(i); + final int newIdx = mapping.get(origin); + undoProjects.add(rexBuilder.makeInputRef(distinctRel, newIdx), + fields.get(i).getName()); + } + + bb.setRoot( + LogicalProject.create(distinctRel, ImmutableList.of(), + undoProjects.leftList(), undoProjects.rightList(), + ImmutableSet.of()), + false); return; } + } - final Map squished = new HashMap<>(); - final List fields = rel.getRowType().getFieldList(); - final PairList newProjects = PairList.of(); - for (int i = 0; i < fields.size(); i++) { - if (origins.get(i) == i) { - squished.put(i, newProjects.size()); - RexInputRef.add2(newProjects, i, fields); - } - } - bb.root = - LogicalProject.create(rel, ImmutableList.of(), - newProjects.leftList(), newProjects.rightList(), - project.getVariablesSet()); - distinctify(bb, false); - final RelNode rel3 = bb.root(); - - // Create the expressions to reverse the mapping. - // Project($0, $1, $0, $2). - final PairList undoProjects = PairList.of(); - for (int i = 0; i < fields.size(); i++) { - final int origin = origins.get(i); - RelDataTypeField field = fields.get(i); - undoProjects.add( - new RexInputRef(castNonNull(squished.get(origin)), - field.getType()), - field.getName()); + // 2. Determine group set and mapping for non-deterministic columns. + final int totalCount = rel.getRowType().getFieldCount(); + final Project project = rel instanceof Project ? (Project) rel : null; + final ImmutableBitSet.Builder groupSetBuilder = ImmutableBitSet.builder(); + + for (int i = 0; i < totalCount; i++) { + if (i < groupCount + || project == null + || RexUtil.isDeterministic(project.getProjects().get(i))) { + groupSetBuilder.set(i); } + } + final ImmutableBitSet groupSet = groupSetBuilder.build(); + if (groupSet.cardinality() == totalCount) { bb.setRoot( - LogicalProject.create(rel3, ImmutableList.of(), - undoProjects.leftList(), undoProjects.rightList(), - ImmutableSet.of()), - false); - + createAggregate(bb, groupSet, ImmutableList.of(groupSet), + ImmutableList.of()), false); return; } - // Usual case: all expressions in the SELECT clause are different. - final ImmutableBitSet groupSet = - ImmutableBitSet.range(rel.getRowType().getFieldCount()); + // 3. Handle non-deterministic ORDER BY columns using the mapping. + final List bottomExprs = new ArrayList<>(); + final List bottomNames = new ArrayList<>(); + for (int i : groupSet) { + bottomExprs.add(castNonNull(project).getProjects().get(i)); + bottomNames.add(rel.getRowType().getFieldNames().get(i)); + } + bb.setRoot( - createAggregate(bb, groupSet, ImmutableList.of(groupSet), - ImmutableList.of()), - false); + LogicalProject.create(castNonNull(project).getInput(), project.getHints(), + bottomExprs, bottomNames, project.getVariablesSet()), false); + + final ImmutableBitSet aggGroupSet = ImmutableBitSet.range(groupSet.cardinality()); + bb.setRoot( + createAggregate(bb, aggGroupSet, ImmutableList.of(aggGroupSet), + ImmutableList.of()), false); + + final RelNode aggregate = bb.root(); + final RexShuttle shuttle = new RexShuttle() { + @Override public RexNode visitInputRef(RexInputRef ref) { + int idx = groupSet.indexOf(ref.getIndex()); + return idx >= 0 + ? rexBuilder.makeInputRef(aggregate, idx) + : super.visitInputRef(ref); + } + }; + + final List topExprs = new ArrayList<>(); + for (int i = 0; i < totalCount; i++) { + int idx = groupSet.indexOf(i); + if (idx >= 0) { + topExprs.add(rexBuilder.makeInputRef(aggregate, idx)); + } else { + topExprs.add(castNonNull(project).getProjects().get(i).accept(shuttle)); + } + } + bb.setRoot( + LogicalProject.create(aggregate, ImmutableList.of(), topExprs, + rel.getRowType().getFieldNames(), ImmutableSet.of()), false); } /** diff --git a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java index 8a485421cf95..28173513d8d1 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java @@ -6001,4 +6001,12 @@ void checkUserDefinedOrderByOver(NullCollation nullCollation) { + "and t1.ename in (select t3.ename from emp t3 )"; sql(sql).ok(); } + + /** Test case of + * [CALCITE-5597] + * SELECT DISTINCT query with ORDER BY column will get error result. */ + @Test void testDistinctOrderByRand() { + final String sql = "select distinct deptno, deptno, empno, 1, 'a' from emp order by rand(), 1"; + sql(sql).ok(); + } } diff --git a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml index 57dc035f9cae..bd829189cff3 100644 --- a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml @@ -1887,6 +1887,21 @@ LogicalTableModify(table=[[CATALOG, SALES, EMP]], operation=[DELETE], flattened= LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7], SLACKER=[$8]) LogicalFilter(condition=[=($7, 10)]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +]]> + + + + + + + + From 12ffa102e0841016c994eda564e236c750fb30d9 Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Wed, 14 Jan 2026 11:14:48 +0800 Subject: [PATCH 107/562] [CALCITE-7372] TopDownGeneralDecorrelator will throw an error when the JOIN condition has correlation --- .../sql2rel/TopDownGeneralDecorrelator.java | 15 +++++-- core/src/test/resources/sql/new-decorr.iq | 45 +++++++++++++++++++ 2 files changed, 57 insertions(+), 3 deletions(-) create mode 100644 core/src/test/resources/sql/new-decorr.iq diff --git a/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java b/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java index d3ccaa334dbc..10b9a411ff14 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java @@ -680,11 +680,13 @@ public RelNode unnestInternal(Join join, boolean allowEmptyOutputFromRewrite) { UnnestedQuery rightInfo; if (!leftHasCorrelation && !join.getJoinType().generatesNullsOnRight() - && join.getJoinType().projectsRight()) { - // there is no need to push down domain D to left side when both following conditions + && join.getJoinType().projectsRight() + && rightHasCorrelation) { + // there is no need to push down domain D to left side when all following conditions // are satisfied: // 1. there is no correlation on left side // 2. join type will not generate NULL values on right side and will project right + // 3. there is correlation on right side to carry domain D // In this case, the left side will start a decorrelation independently newLeft = decorrelateQuery(join.getLeft(), builder); Map leftOldToNewOutputs = new HashMap<>(); @@ -692,16 +694,23 @@ public RelNode unnestInternal(Join join, boolean allowEmptyOutputFromRewrite) { .forEach(i -> leftOldToNewOutputs.put(i, i)); leftInfo = new UnnestedQuery(join.getLeft(), newLeft, new TreeMap<>(), leftOldToNewOutputs); } else { + // when neither the left nor the right side has correlation, + // but the join condition has correlation, domain D is pushed down to the left by default. newLeft = unnest(join.getLeft(), allowEmptyOutputFromRewrite); pushDownToLeft = true; leftInfo = requireNonNull(mapRelToUnnestedQuery.get(join.getLeft())); } if (!rightHasCorrelation && !join.getJoinType().generatesNullsOnLeft()) { - // there is no need to push down domain D to right side when both following conditions + // there is no need to push down domain D to right side when all following conditions // are satisfied: // 1. there is no correlation on right side // 2. join type will not generate NULL values on left side + // 3. there is domain D pushed down to left side // In this case, the right side will start a decorrelation independently + + // either the left or the right side must carry domain D, + // and rightHasCorrelation is false, pushDownToLeft must be true here + assert pushDownToLeft; newRight = decorrelateQuery(join.getRight(), builder); Map rightOldToNewOutputs = new HashMap<>(); IntStream.range(0, newRight.getRowType().getFieldCount()) diff --git a/core/src/test/resources/sql/new-decorr.iq b/core/src/test/resources/sql/new-decorr.iq new file mode 100644 index 000000000000..148d4721acf2 --- /dev/null +++ b/core/src/test/resources/sql/new-decorr.iq @@ -0,0 +1,45 @@ +# new-decorr.iq +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# This is used to test for bugs associated with new decorrelator. +# Most of the test cases here come from the .iq file removed in CoreQuidemTest2. +# This file maybe can be deleted after these .iq files are made available. + +!use post +!set outputformat mysql + +# [CALCITE-7372] TopDownGeneralDecorrelator will throw an error when the JOIN condition has correlation +# This case comes from sub-query.iq [CALCITE-7257] +WITH t0(t0a, t0b) AS (VALUES (1, 1), (2, 0)), + t1(t1a, t1b, t1c) AS (VALUES (1, 1, 3)), + t2(t2a, t2b, t2c) AS (VALUES (1, 1, 5), (2, 2, 7)) +SELECT * FROM t0 WHERE t0a < +(SELECT sum(t1c) FROM + (SELECT t1c + FROM t1 JOIN t2 ON (t1a < t0a AND t2b >= t1b)) +); ++-----+-----+ +| T0A | T0B | ++-----+-----+ +| 2 | 0 | ++-----+-----+ +(1 row) + +!ok + +# End new-decorr.iq From 60ebcadf04fca623906c6d2e8ac1482b9918ff56 Mon Sep 17 00:00:00 2001 From: "lincoln.lil" Date: Wed, 14 Jan 2026 20:15:14 +0800 Subject: [PATCH 108/562] [CALCITE-7375] ProjectWindowTransposeRule does not correctly adjust column indices in window bounds --- .../rel/rules/ProjectWindowTransposeRule.java | 9 ++++- .../apache/calcite/test/RelOptRulesTest.java | 26 +++++++++++++ .../apache/calcite/test/RelOptRulesTest.xml | 37 +++++++++++++++++++ 3 files changed, 70 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rel/rules/ProjectWindowTransposeRule.java b/core/src/main/java/org/apache/calcite/rel/rules/ProjectWindowTransposeRule.java index e606fc8b726d..2517633615de 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/ProjectWindowTransposeRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/ProjectWindowTransposeRule.java @@ -31,6 +31,7 @@ import org.apache.calcite.rex.RexInputRef; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexShuttle; +import org.apache.calcite.rex.RexWindowBound; import org.apache.calcite.sql.SqlAggFunction; import org.apache.calcite.tools.RelBuilderFactory; import org.apache.calcite.util.BitSets; @@ -166,9 +167,13 @@ public ProjectWindowTransposeRule(RelBuilderFactory relBuilderFactory) { ++aggCallIndex; } + // Adjust Window Group RexWindowBound + RexWindowBound newLowerBound = group.lowerBound.accept(indexAdjustment); + RexWindowBound newUpperBound = group.upperBound.accept(indexAdjustment); + groups.add( - new Window.Group(keys.build(), group.isRows, group.lowerBound, - group.upperBound, group.exclude, RelCollations.of(orderKeys), aggCalls)); + new Window.Group(keys.build(), group.isRows, newLowerBound, + newUpperBound, group.exclude, RelCollations.of(orderKeys), aggCalls)); } final LogicalWindow newLogicalWindow = diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index 7de683e9ac66..899635c38c5a 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -502,6 +502,32 @@ private HepProgram createHypergraphProgram() { .checkUnchanged(); } + /** + * Test case for + * [CALCITE-7375] + * ProjectWindowTransposeRule does not correctly adjust column indices in window bounds. */ + @Test void testNestedConstantWindow() { + final String sql = "WITH t1 AS (\n" + + " SELECT *,\n" + + " FIRST_VALUE(deptno) OVER (\n" + + " ORDER BY empno\n" + + " ROWS BETWEEN 2 PRECEDING AND 1 FOLLOWING\n" + + " ) AS f1\n" + + " FROM emp\n" + + ")\n" + + "SELECT deptno,\n" + + " f1,\n" + + " LAST_VALUE(deptno) OVER (\n" + + " ORDER BY empno\n" + + " ROWS BETWEEN 2 PRECEDING AND 1 FOLLOWING\n" + + " ) AS f2\n" + + "FROM t1"; + sql(sql) + .withPreRule(CoreRules.PROJECT_TO_LOGICAL_PROJECT_AND_WINDOW) + .withRule(CoreRules.PROJECT_WINDOW_TRANSPOSE) + .check(); + } + /** * Test case for * [CALCITE-5813] diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index 7c56417de634..193b7c87dbce 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -10725,6 +10725,43 @@ LogicalProject(EXPR$0=[CAST(/($2, $3)):INTEGER NOT NULL]) LogicalAggregate(group=[{0}], agg#0=[SUM($1)], agg#1=[MIN($2)], agg#2=[AVG($2)]) LogicalProject(DEPTNO=[$7], SAL=[$5], EMPNO=[$0]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +]]> + + + + + + + + + + + From c9d783dcf0914e0c72b5c811586b2c34c78d3f3e Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Wed, 14 Jan 2026 20:48:56 -0800 Subject: [PATCH 109/562] [CALCITE-7377] Validator should reject a DESCRIPTOR in a table function when it is not an identifier Signed-off-by: Mihai Budiu --- .../org/apache/calcite/runtime/CalciteResource.java | 3 +++ .../apache/calcite/sql/SqlWindowTableFunction.java | 11 ++++++++--- .../calcite/runtime/CalciteResource.properties | 1 + .../org/apache/calcite/test/SqlValidatorTest.java | 13 +++++++++++++ 4 files changed, 25 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java b/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java index 124c5d520376..026715fbf148 100644 --- a/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java +++ b/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java @@ -1180,4 +1180,7 @@ ExInst multipleCapturingGroupsForRegexpFunctions(String value, @BaseMessage("SELECT BY cannot be used with ORDER BY") ExInst selectByCannotWithOrderBy(); + + @BaseMessage("The argument of DESCRIPTOR must be an identifier") + ExInst descriptorMustBeIdentifier(); } diff --git a/core/src/main/java/org/apache/calcite/sql/SqlWindowTableFunction.java b/core/src/main/java/org/apache/calcite/sql/SqlWindowTableFunction.java index 6decf5b11939..ca2fdd19a83f 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlWindowTableFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlWindowTableFunction.java @@ -233,11 +233,16 @@ boolean checkIntervalOperands(SqlCallBinding callBinding, int startPos) { void validateColumnNames(SqlValidator validator, List fieldNames, List columnNames) { final SqlNameMatcher matcher = validator.getCatalogReader().nameMatcher(); - Ord.forEach(SqlIdentifier.simpleNames(columnNames), (name, i) -> { - if (matcher.indexOf(fieldNames, name) < 0) { + Ord.forEach(columnNames, (name, i) -> { + if (!(name instanceof SqlIdentifier) || !((SqlIdentifier) name).isSimple()) { + throw SqlUtil.newContextException(name.getParserPosition(), + RESOURCE.descriptorMustBeIdentifier()); + } + String simpleName = ((SqlIdentifier) name).getSimple(); + if (matcher.indexOf(fieldNames, simpleName) < 0) { final SqlIdentifier columnName = (SqlIdentifier) columnNames.get(i); throw SqlUtil.newContextException(columnName.getParserPosition(), - RESOURCE.unknownIdentifier(name)); + RESOURCE.unknownIdentifier(simpleName)); } }); } diff --git a/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties b/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties index 3ffb88a55203..e0b1414a16ef 100644 --- a/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties +++ b/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties @@ -386,4 +386,5 @@ IllegalRowIndex=Index in ROW type does not have a constant integer or string val CannotInferReturnType=Cannot infer return type for {0}; operand types: {1} SelectByCannotWithGroupBy=SELECT BY cannot be used with GROUP BY SelectByCannotWithOrderBy=SELECT BY cannot be used with ORDER BY +DescriptorMustBeIdentifier=The argument of DESCRIPTOR must be an identifier # End CalciteResource.properties diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index 40cf5bddf871..dd66362819a7 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -11593,6 +11593,14 @@ private void checkCustomColumnResolving(String table) { sql("select * from table(tumble(table orders, descriptor(^column_not_exist^), " + "interval '2' hour))") .fails("Unknown identifier 'COLUMN_NOT_EXIST'"); + // Test case for [CALCITE-7377] Validator should reject a DESCRIPTOR in a table + // function when it is not an identifier + sql("select * from table(tumble(table orders, descriptor(^1+2^), " + + "interval '2' hour))") + .fails("The argument of DESCRIPTOR must be an identifier"); + sql("select * from table(tumble(table orders, descriptor(^orders.rowtime^), " + + "interval '2' hour))") + .fails("The argument of DESCRIPTOR must be an identifier"); } @Test void testTumbleTableFunction() { @@ -11758,6 +11766,11 @@ private void checkCustomColumnResolving(String table) { sql("select * from table(\n" + "hop(TABLE ^tabler_not_exist^, descriptor(rowtime), interval '2' hour, interval '1' hour))") .fails("Object 'TABLER_NOT_EXIST' not found"); + // Test case for [CALCITE-7377] Validator should reject a DESCRIPTOR in a table function + // when it is not an identifier + sql("select * from table(\n" + + "hop(table orders, descriptor(^1 + 2^), interval '2' hour, interval '1' hour))") + .fails("The argument of DESCRIPTOR must be an identifier"); } @Test void testSessionTableFunction() { From 188f52693efa9ac1bd1743d538bd1b0ef4cdcc43 Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Wed, 14 Jan 2026 13:54:41 +0800 Subject: [PATCH 110/562] [CALCITE-7331] Support the alias form SELECT * EXCEPT() for SELECT * EXCLUDE() --- .../org/apache/calcite/test/BabelTest.java | 36 +++++++++++-- babel/src/test/resources/sql/select.iq | 52 +++++++++++++++++++ core/src/main/codegen/templates/Parser.jj | 2 +- .../calcite/runtime/CalciteResource.java | 6 +-- .../runtime/CalciteResource.properties | 6 +-- site/_docs/reference.md | 2 +- 6 files changed, 93 insertions(+), 11 deletions(-) diff --git a/babel/src/test/java/org/apache/calcite/test/BabelTest.java b/babel/src/test/java/org/apache/calcite/test/BabelTest.java index 769975c4a894..6970280ed436 100644 --- a/babel/src/test/java/org/apache/calcite/test/BabelTest.java +++ b/babel/src/test/java/org/apache/calcite/test/BabelTest.java @@ -195,7 +195,21 @@ names, is( }); fixture.withSql("select * exclude (empno, ^foo^) from emp") - .fails("SELECT \\* EXCLUDE list contains unknown column\\(s\\): FOO"); + .fails("SELECT \\* EXCLUDE/EXCEPT list contains unknown column\\(s\\): FOO"); + + // Alias form: EXCEPT behaves the same as EXCLUDE + fixture.withSql("select * except(empno, deptno) from emp") + .type(type -> { + final List names = type.getFieldList().stream() + .map(RelDataTypeField::getName) + .collect(Collectors.toList()); + assertThat( + names, is( + ImmutableList.of("ENAME", "JOB", "MGR", "HIREDATE", "SAL", "COMM", "SLACKER"))); + }); + + fixture.withSql("select * except (empno, ^foo^) from emp") + .fails("SELECT \\* EXCLUDE/EXCEPT list contains unknown column\\(s\\): FOO"); fixture.withSql("select e.* exclude(e.empno, e.ename, e.job, e.mgr)" + " from emp e join dept d on e.deptno = d.deptno") @@ -210,7 +224,23 @@ names, is( fixture.withSql("select e.* exclude(e.empno, e.ename, e.job, e.mgr, ^d.deptno^)" + " from emp e join dept d on e.deptno = d.deptno") - .fails("SELECT \\* EXCLUDE list contains unknown column\\(s\\): D.DEPTNO"); + .fails("SELECT \\* EXCLUDE/EXCEPT list contains unknown column\\(s\\): D.DEPTNO"); + + // Alias form: EXCEPT for table-qualified star + fixture.withSql("select e.* except(e.empno, e.ename, e.job, e.mgr)" + + " from emp e join dept d on e.deptno = d.deptno") + .type(type -> { + final List names = type.getFieldList().stream() + .map(RelDataTypeField::getName) + .collect(Collectors.toList()); + assertThat( + names, is( + ImmutableList.of("HIREDATE", "SAL", "COMM", "DEPTNO", "SLACKER"))); + }); + + fixture.withSql("select e.* except(e.empno, e.ename, e.job, e.mgr, ^d.deptno^)" + + " from emp e join dept d on e.deptno = d.deptno") + .fails("SELECT \\* EXCLUDE/EXCEPT list contains unknown column\\(s\\): D.DEPTNO"); fixture.withSql("select e.* exclude(e.empno, e.ename, e.job, e.mgr), d.* exclude(d.name)" + " from emp e join dept d on e.deptno = d.deptno") @@ -242,7 +272,7 @@ names, is( // To verify that the exclude list contains all columns in the table fixture.withSql("select ^*^ exclude(deptno, name) from dept") - .fails("SELECT \\* EXCLUDE list cannot exclude all columns"); + .fails("SELECT \\* EXCLUDE/EXCEPT list cannot exclude all columns"); } /** Tests that DATEADD, DATEDIFF, DATEPART, DATE_PART allow custom time diff --git a/babel/src/test/resources/sql/select.iq b/babel/src/test/resources/sql/select.iq index 073daf8cd2b1..c969ad76559e 100755 --- a/babel/src/test/resources/sql/select.iq +++ b/babel/src/test/resources/sql/select.iq @@ -234,4 +234,56 @@ WHERE d.loc = 'CHICAGO'; !ok +# [CALCITE-7331] Support the alias form SELECT * EXCEPT() for SELECT * EXCLUDE() +select 1 as x, 2 as y except (select 3 as a, 4 as b); ++---+---+ +| X | Y | ++---+---+ +| 1 | 2 | ++---+---+ +(1 row) + +!ok + +with t(x, y) as (values(1, 2)) +select 1 as x, 2 as y except (y) from t; +Non-query expression encountered in illegal context +!error + +with t(x, y) as (values(1, 2)) +select x except (x) from t; +EXCLUDE/EXCEPT clause must follow a STAR expression +!error + +with t(x, y) as (values(1, 2)) +select * except (x) from t; ++---+ +| Y | ++---+ +| 2 | ++---+ +(1 row) + +!ok + +select 1 as x, e.* except(e.empno, e.ename, e.job, e.mgr), d.* except(d.dname), 2 as y +from emp e join dept d on e.deptno = d.deptno limit 1; ++---+------------+---------+------+--------+---------+----------+---+ +| X | HIREDATE | SAL | COMM | DEPTNO | DEPTNO0 | LOC | Y | ++---+------------+---------+------+--------+---------+----------+---+ +| 1 | 1981-06-09 | 2450.00 | | 10 | 10 | NEW YORK | 2 | ++---+------------+---------+------+--------+---------+----------+---+ +(1 row) + +!ok + +select d1.* except(d1.dname) from dept d1 except(select d2.* except(d2.dname) from dept d2); ++--------+-----+ +| DEPTNO | LOC | ++--------+-----+ ++--------+-----+ +(0 rows) + +!ok + # End select.iq diff --git a/core/src/main/codegen/templates/Parser.jj b/core/src/main/codegen/templates/Parser.jj index 2069882cc223..d87339e6602a 100644 --- a/core/src/main/codegen/templates/Parser.jj +++ b/core/src/main/codegen/templates/Parser.jj @@ -2049,7 +2049,7 @@ SqlNodeList StarExcludeList() : SqlIdentifier id; } { - { s = span(); } + ( | ) { s = span(); } id = CompoundIdentifier() { list.add(id); } diff --git a/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java b/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java index 026715fbf148..baf5fca10e04 100644 --- a/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java +++ b/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java @@ -807,13 +807,13 @@ ExInst illegalArgumentForTableFunctionCall(String a0, @BaseMessage("SELECT * requires a FROM clause") ExInst selectStarRequiresFrom(); - @BaseMessage("EXCLUDE clause must follow a STAR expression") + @BaseMessage("EXCLUDE/EXCEPT clause must follow a STAR expression") ExInst selectExcludeRequiresStar(); - @BaseMessage("SELECT * EXCLUDE list contains unknown column(s): {0}") + @BaseMessage("SELECT * EXCLUDE/EXCEPT list contains unknown column(s): {0}") ExInst selectStarExcludeListContainsUnknownColumns(String columns); - @BaseMessage("SELECT * EXCLUDE list cannot exclude all columns") + @BaseMessage("SELECT * EXCLUDE/EXCEPT list cannot exclude all columns") ExInst selectStarExcludeCannotExcludeAllColumns(); @BaseMessage("Group function ''{0}'' can only appear in GROUP BY clause") diff --git a/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties b/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties index e0b1414a16ef..ac137d058e70 100644 --- a/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties +++ b/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties @@ -266,9 +266,9 @@ CannotStreamResultsForNonStreamingInputs=Cannot stream results of a query with n MinusNotAllowed=MINUS is not allowed under the current SQL conformance level SelectMissingFrom=SELECT must have a FROM clause SelectStarRequiresFrom=SELECT * requires a FROM clause -SelectExcludeRequiresStar=EXCLUDE clause must follow a STAR expression -SelectStarExcludeListContainsUnknownColumns=SELECT * EXCLUDE list contains unknown column(s): {0} -SelectStarExcludeCannotExcludeAllColumns=SELECT * EXCLUDE list cannot exclude all columns +SelectExcludeRequiresStar=EXCLUDE/EXCEPT clause must follow a STAR expression +SelectStarExcludeListContainsUnknownColumns=SELECT * EXCLUDE/EXCEPT list contains unknown column(s): {0} +SelectStarExcludeCannotExcludeAllColumns=SELECT * EXCLUDE/EXCEPT list cannot exclude all columns GroupFunctionMustAppearInGroupByClause=Group function ''{0}'' can only appear in GROUP BY clause AuxiliaryWithoutMatchingGroupCall=Call to auxiliary group function ''{0}'' must have matching call to group function ''{1}'' in GROUP BY clause PivotAggMalformed=Measure expression in PIVOT must use aggregate function diff --git a/site/_docs/reference.md b/site/_docs/reference.md index 2668cbf83c40..e1ea8e720697 100644 --- a/site/_docs/reference.md +++ b/site/_docs/reference.md @@ -246,7 +246,7 @@ starWithExclude: Note: -* `SELECT * EXCLUDE (...)` is recognized only when the Babel parser is enabled. It sets the generated parser configuration flag `includeStarExclude` to `true` (the standard parser leaves that flag `false`), which allows a `STAR` token followed by `EXCLUDE` and a parenthesized identifier list to be parsed into a `SqlStarExclude` node and ensures validators respect the exclusion list when expanding the projection. Reusing the same parser configuration elsewhere enables the same syntax for other components that need it. +* `SELECT * EXCLUDE (...)` is recognized only when the Babel parser is enabled. It sets the generated parser configuration flag `includeStarExclude` to `true` (the standard parser leaves that flag `false`), which allows a `STAR` token followed by `EXCLUDE` (or the alias `EXCEPT`) and a parenthesized identifier list to be parsed into a `SqlStarExclude` node and ensures validators respect the exclusion list when expanding the projection. Reusing the same parser configuration elsewhere enables the same syntax for other components that need it. projectItem: expression [ [ AS ] columnAlias ] From f52c49e46cd6e8f872e5801848b4766e29b63d82 Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Thu, 15 Jan 2026 23:14:10 +0800 Subject: [PATCH 111/562] [CALCITE-7374] NULLS LAST throws ClassCastException when sorting arrays --- .../enumerable/EnumerableMergeUnion.java | 21 +++++---- .../org/apache/calcite/plan/RelTraitSet.java | 16 +++++++ core/src/test/resources/sql/sort.iq | 43 +++++++++++++++++++ .../calcite/linq4j/function/Functions.java | 28 ++++++++---- 4 files changed, 92 insertions(+), 16 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeUnion.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeUnion.java index 637a454c2137..d06fa31661a1 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeUnion.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeUnion.java @@ -22,8 +22,10 @@ import org.apache.calcite.linq4j.tree.Expressions; import org.apache.calcite.linq4j.tree.ParameterExpression; import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelTrait; import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.RelCollation; +import org.apache.calcite.rel.RelCollationTraitDef; import org.apache.calcite.rel.RelNode; import org.apache.calcite.util.BuiltInMethod; import org.apache.calcite.util.Pair; @@ -42,17 +44,20 @@ public class EnumerableMergeUnion extends EnumerableUnion { protected EnumerableMergeUnion(RelOptCluster cluster, RelTraitSet traitSet, List inputs, boolean all) { super(cluster, traitSet, inputs, all); - final RelCollation collation = traitSet.getCollation(); - if (collation == null || collation.getFieldCollations().isEmpty()) { + final List collations = traitSet.getCollations(); + if (collations.isEmpty() || collations.get(0).getFieldCollations().isEmpty()) { throw new IllegalArgumentException("EnumerableMergeUnion with no collation"); } for (RelNode input : inputs) { - final RelCollation inputCollation = input.getTraitSet().getCollation(); - if (inputCollation == null || !inputCollation.satisfies(collation)) { - throw new IllegalArgumentException("EnumerableMergeUnion input does " - + "not satisfy collation. EnumerableMergeUnion collation: " - + collation + ". Input collation: " + inputCollation + ". Input: " - + input); + final RelTrait inputCollationTrait = + input.getTraitSet().getTrait(RelCollationTraitDef.INSTANCE); + for (RelCollation collation : collations) { + if (inputCollationTrait == null || !inputCollationTrait.satisfies(collation)) { + throw new IllegalArgumentException("EnumerableMergeUnion input does " + + "not satisfy collation. EnumerableMergeUnion collation: " + + collation + ". Input collation: " + inputCollationTrait + ". Input: " + + input); + } } } } diff --git a/core/src/main/java/org/apache/calcite/plan/RelTraitSet.java b/core/src/main/java/org/apache/calcite/plan/RelTraitSet.java index b91b660752e6..65747426062c 100644 --- a/core/src/main/java/org/apache/calcite/plan/RelTraitSet.java +++ b/core/src/main/java/org/apache/calcite/plan/RelTraitSet.java @@ -392,6 +392,22 @@ public RelTraitSet getDefaultSansConvention() { return (@Nullable T) getTrait(RelCollationTraitDef.INSTANCE); } + /** + * Returns {@link RelCollation} traits defined by + * {@link RelCollationTraitDef#INSTANCE}. + */ + @SuppressWarnings("unchecked") + public List getCollations() { + RelCollation trait = getTrait(RelCollationTraitDef.INSTANCE); + if (trait == null) { + return ImmutableList.of(); + } + if (trait instanceof RelCompositeTrait) { + return ((RelCompositeTrait) trait).traitList(); + } + return ImmutableList.of(trait); + } + /** * Returns the size of the RelTraitSet. * diff --git a/core/src/test/resources/sql/sort.iq b/core/src/test/resources/sql/sort.iq index fb69970e41b8..b9e0412ff989 100644 --- a/core/src/test/resources/sql/sort.iq +++ b/core/src/test/resources/sql/sort.iq @@ -465,4 +465,47 @@ order by arr desc nulls first; !ok +# [CALCITE-7374] NULLS LAST throws ClassCastException when sorting arrays +select * from +(values + (2, array[null, 3]), + (3, array[3, 4]), + (1, array[1, 2]), + (4, array[4, 5]), + (5, cast(null as integer array))) as t(id, arr) +order by arr nulls last; ++----+-----------+ +| ID | ARR | ++----+-----------+ +| 1 | [1, 2] | +| 3 | [3, 4] | +| 4 | [4, 5] | +| 2 | [null, 3] | +| 5 | | ++----+-----------+ +(5 rows) + +!ok + +select * from +(values + (2, array[null, 3]), + (3, array[3, 4]), + (1, array[1, 2]), + (4, array[4, 5]), + (5, cast(null as integer array))) as t(id, arr) +order by arr desc nulls last; ++----+-----------+ +| ID | ARR | ++----+-----------+ +| 2 | [null, 3] | +| 4 | [4, 5] | +| 3 | [3, 4] | +| 1 | [1, 2] | +| 5 | | ++----+-----------+ +(5 rows) + +!ok + # End sort.iq diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/function/Functions.java b/linq4j/src/main/java/org/apache/calcite/linq4j/function/Functions.java index 73a06f42bd8c..6e9a186344c5 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/function/Functions.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/function/Functions.java @@ -556,7 +556,8 @@ private static class NullsFirstComparator } else if (o1 instanceof Object[] && o2 instanceof Object[]) { return compareObjectArrays((Object[]) o1, (Object[]) o2); } else { - throw new IllegalArgumentException(); + throw new IllegalArgumentException("Item types do not match: " + + o1.getClass() + " vs " + o2.getClass()); } } } @@ -582,7 +583,8 @@ private static class NullsLastComparator } else if (o1 instanceof Object[] && o2 instanceof Object[]) { return compareObjectArrays((Object[]) o1, (Object[]) o2); } else { - throw new IllegalArgumentException(); + throw new IllegalArgumentException("Item types do not match: " + + o1.getClass() + " vs " + o2.getClass()); } } } @@ -590,7 +592,7 @@ private static class NullsLastComparator /** Nulls first reverse comparator. */ private static class NullsFirstReverseComparator implements Comparator, Serializable { - @Override public int compare(Object o1, Object o2) { + @Override public int compare(@Nullable Object o1, @Nullable Object o2) { if (o1 == o2) { return 0; } @@ -608,7 +610,8 @@ private static class NullsFirstReverseComparator } else if (o1 instanceof Object[] && o2 instanceof Object[]) { return -compareObjectArrays((Object[]) o1, (Object[]) o2); } else { - throw new IllegalArgumentException(); + throw new IllegalArgumentException("Item types do not match: " + + o1.getClass() + " vs " + o2.getClass()); } } } @@ -707,8 +710,8 @@ public static int compareObjectArrays(@Nullable Object @Nullable [] b0, /** Nulls last reverse comparator. */ private static class NullsLastReverseComparator - implements Comparator, Serializable { - @Override public int compare(Comparable o1, Comparable o2) { + implements Comparator, Serializable { + @Override public int compare(@Nullable Object o1, @Nullable Object o2) { if (o1 == o2) { return 0; } @@ -718,8 +721,17 @@ private static class NullsLastReverseComparator if (o2 == null) { return -1; } - //noinspection unchecked - return -o1.compareTo(o2); + if (o1 instanceof Comparable && o2 instanceof Comparable) { + //noinspection unchecked + return -((Comparable) o1).compareTo(o2); + } else if (o1 instanceof List && o2 instanceof List) { + return -compareLists((List) o1, (List) o2); + } else if (o1 instanceof Object[] && o2 instanceof Object[]) { + return -compareObjectArrays((Object[]) o1, (Object[]) o2); + } else { + throw new IllegalArgumentException("Item types do not match: " + + o1.getClass() + " vs " + o2.getClass()); + } } } From 3e3edd66e265ffae035a70e67c4799c17eb48b18 Mon Sep 17 00:00:00 2001 From: nobigo Date: Thu, 15 Jan 2026 19:24:32 +0800 Subject: [PATCH 112/562] [CALCITE-7373] FILTER_INTO_JOIN should not push Filter into a join when the Filter contains non-deterministic function --- .../calcite/rel/rules/FilterJoinRule.java | 9 ++++ .../apache/calcite/test/RelOptRulesTest.java | 27 ++++++++++++ .../apache/calcite/test/RelOptRulesTest.xml | 44 +++++++++++++++++++ 3 files changed, 80 insertions(+) diff --git a/core/src/main/java/org/apache/calcite/rel/rules/FilterJoinRule.java b/core/src/main/java/org/apache/calcite/rel/rules/FilterJoinRule.java index 1f80126aa869..9c6b560bfa54 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/FilterJoinRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/FilterJoinRule.java @@ -72,6 +72,15 @@ protected FilterJoinRule(C config) { protected void perform(RelOptRuleCall call, @Nullable Filter filter, Join join) { + // Skip non-deterministic filter condition + if (filter != null && !RexUtil.isDeterministic(filter.getCondition())) { + return; + } + // Skip non-deterministic join condition + if (!RexUtil.isDeterministic(join.getCondition())) { + return; + } + List joinFilters = RelOptUtil.conjunctions(join.getCondition()); final List origJoinFilters = ImmutableList.copyOf(joinFilters); diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index 899635c38c5a..f46030dd8b3b 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -2187,6 +2187,33 @@ private void checkSemiOrAntiJoinProjectTranspose(JoinRelType type) { assertTrue(join.getHints().contains(noHashJoinHint)); } + /** Test case for + * [CALCITE-7373] + * FILTER_INTO_JOIN should not push Filter into a join + * when the Filter contains non-deterministic function. */ + @Test void testPushFilterThroughJoinWithNonDeterministic() { + final String sql = "select * from (\n" + + " select * from dept inner join\n" + + " emp on dept.deptno = emp.deptno) R\n" + + "where 0.9 <= rand()"; + sql(sql) + .withRule(CoreRules.FILTER_PROJECT_TRANSPOSE, + CoreRules.FILTER_INTO_JOIN, + CoreRules.JOIN_CONDITION_PUSH) + .check(); + } + + @Test void testPushJoinConditionWithNonDeterministic() { + final String sql = "select * from (\n" + + " select * from dept inner join\n" + + " emp on dept.deptno = emp.deptno and emp.mgr <= rand()) R\n"; + sql(sql) + .withRule(CoreRules.FILTER_PROJECT_TRANSPOSE, + CoreRules.FILTER_INTO_JOIN, + CoreRules.JOIN_CONDITION_PUSH) + .checkUnchanged(); + } + /** Test case for * [CALCITE-438] * Push predicates through SemiJoin. */ diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index 193b7c87dbce..c7c39f82a8ef 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -14228,6 +14228,34 @@ LogicalProject(DNAME=[$1]) LogicalTableScan(table=[[scott, DEPT]]) LogicalFilter(condition=[=($5, 100)]) LogicalTableScan(table=[[scott, EMP]]) +]]> + + + + + + + + + + + @@ -14337,6 +14365,22 @@ LogicalProject(DEPTNO=[$0], DEPTNO0=[$9]) LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7], SLACKER=[$8], $f9=[*($7, 2)]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +]]> + + + + + + + + From 6c72c91c8f1773d49bada0e83870fdb141fe2b87 Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Sat, 17 Jan 2026 17:47:20 +0800 Subject: [PATCH 113/562] [CALCITE-7359] Incorrect result for array comparison with ANY operator --- .../adapter/enumerable/EnumerableCollect.java | 14 +- .../calcite/rel/metadata/RelMdPredicates.java | 33 ++-- .../apache/calcite/runtime/SqlFunctions.java | 174 +++++++++++------- core/src/test/resources/sql/some.iq | 34 +++- .../calcite/linq4j/function/Functions.java | 88 +++------ 5 files changed, 203 insertions(+), 140 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableCollect.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableCollect.java index d794dc6c4ec7..0d2a521851fd 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableCollect.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableCollect.java @@ -109,15 +109,21 @@ public static Collect create(RelNode input, RelDataType rowType) { RelDataType childRecordType = result.physType.getRowType().getFieldList().get(0).getType(); if (!SqlTypeUtil.sameNamedType(collectionComponentType, childRecordType)) { - // In the internal representation of multisets , every element must be a record. In case the - // result above is a scalar type we have to wrap it around a physical type capable of - // representing records. For this reason the following conversion is necessary. + // In the internal representation of multisets, every element must be a record. + // In case the result above is a scalar type we have to wrap it around a + // physical type capable of representing records. + // For ARRAY type with a single field, we use SCALAR format to avoid + // unnecessary wrapping, which allows correct comparison semantics. // REVIEW zabetak January 7, 2019: If we can ensure that the input to this operator // has the correct physical type (e.g., respecting the Prefer.ARRAY above) // then this conversion can be removed. + JavaRowFormat targetFormat = + collectionType == SqlTypeName.ARRAY && child.getRowType().getFieldCount() == 1 + ? JavaRowFormat.SCALAR + : JavaRowFormat.ARRAY; conv_ = builder.append( - "converted", result.physType.convertTo(child_, JavaRowFormat.ARRAY)); + "converted", result.physType.convertTo(child_, targetFormat)); } collectionExpr = diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdPredicates.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdPredicates.java index 11a670515891..e04e2098ff78 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdPredicates.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdPredicates.java @@ -40,6 +40,7 @@ import org.apache.calcite.rel.core.TableScan; import org.apache.calcite.rel.core.Union; import org.apache.calcite.rel.core.Values; +import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexExecutor; @@ -54,6 +55,7 @@ import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.fun.SqlInternalOperators; import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.type.SqlTypeUtil; import org.apache.calcite.util.BitSets; import org.apache.calcite.util.Bug; import org.apache.calcite.util.ImmutableBitSet; @@ -599,21 +601,26 @@ public RelOptPredicateList getPredicates(Values values, RelMetadataQuery mq) { eqConstant(values, rexBuilder, i, rexLiteral)); } } else { - RexUnknownAs rexUnknownAs = RexUnknownAs.UNKNOWN; - RangeSet rangeSet = TreeRangeSet.create(); - for (RexLiteral rexLiteral : rexLiteralSet) { - if (RexUtil.isNull(rexLiteral)) { - rexUnknownAs = RexUnknownAs.TRUE; - continue; + final RelDataType type = values.getRowType().getFieldList().get(i).getType(); + if (!type.isStruct() + && !SqlTypeUtil.isCollection(type) + && !SqlTypeUtil.isMap(type)) { + RexUnknownAs rexUnknownAs = RexUnknownAs.UNKNOWN; + RangeSet rangeSet = TreeRangeSet.create(); + for (RexLiteral rexLiteral : rexLiteralSet) { + if (RexUtil.isNull(rexLiteral)) { + rexUnknownAs = RexUnknownAs.TRUE; + continue; + } + rangeSet.add( + Range.singleton(requireNonNull(rexLiteral.getValueAs(Comparable.class)))); } - rangeSet.add(Range.singleton(requireNonNull(rexLiteral.getValueAs(Comparable.class)))); + final Sarg sarg = Sarg.of(rexUnknownAs, rangeSet); + predicates.add( + i, rexBuilder.makeCall(SqlStdOperatorTable.SEARCH, + rexBuilder.makeInputRef(values, i), + rexBuilder.makeSearchArgumentLiteral(sarg, type))); } - final Sarg sarg = Sarg.of(rexUnknownAs, rangeSet); - predicates.add( - i, rexBuilder.makeCall(SqlStdOperatorTable.SEARCH, - rexBuilder.makeInputRef(values, i), - rexBuilder.makeSearchArgumentLiteral(sarg, - values.getRowType().getFieldList().get(i).getType()))); } } return RelOptPredicateList.of(rexBuilder, predicates); diff --git a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java index 50530aef675d..28b34513e1c0 100644 --- a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java +++ b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java @@ -2116,7 +2116,7 @@ public static boolean eq(BigDecimal b0, BigDecimal b1) { /** SQL = operator applied to Object[] values (neither may be * null). */ public static boolean eq(@Nullable Object @Nullable [] b0, @Nullable Object @Nullable [] b1) { - return Arrays.deepEquals(b0, b1); + return Functions.compareObjectArrays(b0, b1) == 0; } /** SQL = operator applied to Object values (including String; @@ -2125,6 +2125,16 @@ public static boolean eq(Object b0, Object b1) { return b0.equals(b1); } + /** SQL = operator applied to List values. */ + public static boolean eq(List b0, List b1) { + return eqNullable(b0, b1); + } + + /** SQL = operator applied to Map values. */ + public static boolean eq(Map b0, Map b1) { + return eqNullable(b0, b1); + } + /** SQL = operator applied to String values with a certain Comparator. */ public static boolean eq(String s0, String s1, Comparator comparator) { return comparator.compare(s0, s1) == 0; @@ -2133,22 +2143,7 @@ public static boolean eq(String s0, String s1, Comparator comparator) { /** SQL = operator applied to Object values (at least one operand * has ANY type; neither may be null). */ public static boolean eqAny(Object b0, Object b1) { - if (b0.getClass().equals(b1.getClass())) { - // The result of SqlFunctions.eq(BigDecimal, BigDecimal) makes more sense - // than BigDecimal.equals(BigDecimal). So if both of types are BigDecimal, - // we just use SqlFunctions.eq(BigDecimal, BigDecimal). - if (BigDecimal.class.isInstance(b0)) { - return eq((BigDecimal) b0, (BigDecimal) b1); - } else { - return b0.equals(b1); - } - } else if (allAssignable(Number.class, b0, b1)) { - return eq(toBigDecimal((Number) b0), toBigDecimal((Number) b1)); - } - // We shouldn't rely on implementation even though overridden equals can - // handle other types which may create worse result: for example, - // a.equals(b) != b.equals(a) - return false; + return eqNullable(b0, b1); } /** Returns whether two objects can both be assigned to a given class. */ @@ -2169,6 +2164,21 @@ public static boolean ne(Object b0, Object b1) { return !eq(b0, b1); } + /** SQL <gt; operator applied to List values. */ + public static boolean ne(List b0, List b1) { + return !eqNullable(b0, b1); + } + + /** SQL <gt; operator applied to Map values. */ + public static boolean ne(Map b0, Map b1) { + return !eqNullable(b0, b1); + } + + /** SQL <gt; operator applied to Object[] values. */ + public static boolean ne(@Nullable Object @Nullable [] b0, @Nullable Object @Nullable [] b1) { + return !eqNullable(b0, b1); + } + /** SQL <gt; operator applied to OString values with a certain Comparator. */ public static boolean ne(String s0, String s1, Comparator comparator) { return !eq(s0, s1, comparator); @@ -2180,6 +2190,66 @@ public static boolean neAny(Object b0, Object b1) { return !eqAny(b0, b1); } + private static boolean eqNullable(@Nullable Object b0, @Nullable Object b1) { + if (b0 == b1) { + return true; + } + if (b0 == null || b1 == null) { + return false; + } + if (b0 instanceof List && b1 instanceof List) { + return Functions.compareLists((List) b0, (List) b1) == 0; + } + if (b0 instanceof Map && b1 instanceof Map) { + return Functions.compareMaps((Map) b0, (Map) b1) == 0; + } + if (b0 instanceof Object[] && b1 instanceof Object[]) { + return Functions.compareObjectArrays((Object[]) b0, (Object[]) b1) == 0; + } + if (b0.getClass().equals(b1.getClass())) { + if (b0 instanceof BigDecimal) { + return eq((BigDecimal) b0, (BigDecimal) b1); + } + return b0.equals(b1); + } + if (b0 instanceof Number && b1 instanceof Number) { + return eq(toBigDecimal((Number) b0), toBigDecimal((Number) b1)); + } + return false; + } + + /** Compares two nullable objects recursively if they are collections. + * Nulls are treated as larger than non-null values. The op is used to record + * the type of comparison operation for user-friendly display in error messages. */ + private static int compareNullable(@Nullable Object b0, @Nullable Object b1, String op) { + if (b0 == b1) { + return 0; + } + if (b0 == null) { + return 1; + } + if (b1 == null) { + return -1; + } + if (b0 instanceof List && b1 instanceof List) { + return Functions.compareLists((List) b0, (List) b1); + } + if (b0 instanceof Map && b1 instanceof Map) { + return Functions.compareMaps((Map) b0, (Map) b1); + } + if (b0 instanceof Object[] && b1 instanceof Object[]) { + return Functions.compareObjectArrays((Object[]) b0, (Object[]) b1); + } + if (b0.getClass().equals(b1.getClass()) && b0 instanceof Comparable) { + //noinspection unchecked + return ((Comparable) b0).compareTo(b1); + } + if (b0 instanceof Number && b1 instanceof Number) { + return toBigDecimal((Number) b0).compareTo(toBigDecimal((Number) b1)); + } + throw notComparable(op, b0, b1); + } + // < /** SQL < operator applied to boolean values. */ @@ -2242,28 +2312,20 @@ public static boolean lt(double b0, double b1) { } public static boolean lt(List b0, List b1) { - return Functions.compareLists(b0, b1) < 0; + return compareNullable(b0, b1, "<") < 0; } public static boolean lt(Map b0, Map b1) { - return Functions.compareMaps(b0, b1) < 0; + return compareNullable(b0, b1, "<") < 0; } public static boolean lt(@Nullable Object @Nullable [] b0, @Nullable Object @Nullable [] b1) { - return Functions.compareObjectArrays(b0, b1) < 0; + return compareNullable(b0, b1, "<") < 0; } /** SQL < operator applied to Object values. */ public static boolean ltAny(Object b0, Object b1) { - if (b0.getClass().equals(b1.getClass()) - && b0 instanceof Comparable) { - //noinspection unchecked - return ((Comparable) b0).compareTo(b1) < 0; - } else if (allAssignable(Number.class, b0, b1)) { - return lt(toBigDecimal((Number) b0), toBigDecimal((Number) b1)); - } - - throw notComparable("<", b0, b1); + return compareNullable(b0, b1, "<") < 0; } // <= @@ -2295,26 +2357,23 @@ public static boolean le(BigDecimal b0, BigDecimal b1) { /** SQL ≤ operator applied to List values. */ public static boolean le(List b0, List b1) { - return Functions.compareLists(b0, b1) <= 0; + return compareNullable(b0, b1, "<=") <= 0; + } + + /** SQL ≤ operator applied to Map values. */ + public static boolean le(Map b0, Map b1) { + return compareNullable(b0, b1, "<=") <= 0; } /** SQL ≤ operator applied to Object[] values. */ public static boolean le(@Nullable Object @Nullable [] b0, @Nullable Object @Nullable [] b1) { - return Functions.compareObjectArrays(b0, b1) <= 0; + return compareNullable(b0, b1, "<=") <= 0; } /** SQL ≤ operator applied to Object values (at least one * operand has ANY type; neither may be null). */ public static boolean leAny(Object b0, Object b1) { - if (b0.getClass().equals(b1.getClass()) - && b0 instanceof Comparable) { - //noinspection unchecked - return ((Comparable) b0).compareTo(b1) <= 0; - } else if (allAssignable(Number.class, b0, b1)) { - return le(toBigDecimal((Number) b0), toBigDecimal((Number) b1)); - } - - throw notComparable("<=", b0, b1); + return compareNullable(b0, b1, "<=") <= 0; } // > @@ -2379,29 +2438,21 @@ public static boolean gt(double b0, double b1) { } public static boolean gt(List b0, List b1) { - return Functions.compareLists(b0, b1) > 0; + return compareNullable(b0, b1, ">") > 0; } public static boolean gt(Map b0, Map b1) { - return Functions.compareMaps(b0, b1) > 0; + return compareNullable(b0, b1, ">") > 0; } public static boolean gt(@Nullable Object @Nullable [] b0, @Nullable Object @Nullable [] b1) { - return Functions.compareObjectArrays(b0, b1) > 0; + return compareNullable(b0, b1, ">") > 0; } /** SQL > operator applied to Object values (at least one * operand has ANY type; neither may be null). */ public static boolean gtAny(Object b0, Object b1) { - if (b0.getClass().equals(b1.getClass()) - && b0 instanceof Comparable) { - //noinspection unchecked - return ((Comparable) b0).compareTo(b1) > 0; - } else if (allAssignable(Number.class, b0, b1)) { - return gt(toBigDecimal((Number) b0), toBigDecimal((Number) b1)); - } - - throw notComparable(">", b0, b1); + return compareNullable(b0, b1, ">") > 0; } // >= @@ -2433,26 +2484,23 @@ public static boolean ge(BigDecimal b0, BigDecimal b1) { /** SQL ≥ operator applied to List values. */ public static boolean ge(List b0, List b1) { - return Functions.compareLists(b0, b1) >= 0; + return compareNullable(b0, b1, ">=") >= 0; + } + + /** SQL ≥ operator applied to Map values. */ + public static boolean ge(Map b0, Map b1) { + return compareNullable(b0, b1, ">=") >= 0; } /** SQL ≥ operator applied to Object[] values. */ public static boolean ge(@Nullable Object @Nullable [] b0, @Nullable Object @Nullable [] b1) { - return Functions.compareObjectArrays(b0, b1) >= 0; + return compareNullable(b0, b1, ">=") >= 0; } /** SQL ≥ operator applied to Object values (at least one * operand has ANY type; neither may be null). */ public static boolean geAny(Object b0, Object b1) { - if (b0.getClass().equals(b1.getClass()) - && b0 instanceof Comparable) { - //noinspection unchecked - return ((Comparable) b0).compareTo(b1) >= 0; - } else if (allAssignable(Number.class, b0, b1)) { - return ge(toBigDecimal((Number) b0), toBigDecimal((Number) b1)); - } - - throw notComparable(">=", b0, b1); + return compareNullable(b0, b1, ">=") >= 0; } // + diff --git a/core/src/test/resources/sql/some.iq b/core/src/test/resources/sql/some.iq index 5c7f6b469f30..acc61d9c92a3 100644 --- a/core/src/test/resources/sql/some.iq +++ b/core/src/test/resources/sql/some.iq @@ -986,5 +986,37 @@ EnumerableCalc(expr#0=[{inputs}], expr#1=[1], expr#2=[2], expr#3=[3], expr#4=[AR EnumerableValues(tuples=[[{ 0 }]]) !plan -# End some.iq +# [CALCITE-7359] Support various nested type comparisons in ANY/ALL +# Test with different operators and deeper nesting +select + -- Equality for nested arrays + array[array[1,1],array[2,2]] = any(array[array[1,1],array[2,2]], array[array[3,3],array[4,4]]) as arr_arr_t, + -- Array of Row + array[row(1, 'a'), row(2, 'b')] = any(array[row(1, 'a'), row(2, 'b')], array[row(3, 'c')]) as arr_row_t, + -- Non-equality for nested arrays + array[1, 2] <> any(array[1, 2], array[1, 3]) as arr_ne_t, + -- Inequalities for nested arrays + array[1, 2] < any(array[1, 1], array[1, 3]) as arr_lt_t, + array[1, 2] <= any(array[0, 9], array[1, 2]) as arr_le_t, + array[1, 2] > any(array[1, 1], array[2, 0]) as arr_gt_t, + array[1, 2] >= any(array[1, 2], array[1, 3]) as arr_ge_t, + + -- Nested structures with NULL values + array[1, cast(null as integer)] = any(array[1, cast(null as integer)], array[1, 2]) as arr_null_t, + array[1, cast(null as integer)] = any(array[1, 2], array[1, 3]) as arr_null_f, + + -- Deeper nesting: Array of Row of Array + array[row(1, array[10, 20]), row(2, array[30])] = + any(array[row(1, array[10, 20]), row(2, array[30])], array[row(1, array[0])]) as deep_t, + array[row(1, array[10, 20]), row(2, array[30])] = + any(array[row(1, array[10, 21]), row(2, array[30])], array[row(1, array[0])]) as deep_f; ++-----------+-----------+----------+----------+----------+----------+----------+------------+------------+--------+--------+ +| ARR_ARR_T | ARR_ROW_T | ARR_NE_T | ARR_LT_T | ARR_LE_T | ARR_GT_T | ARR_GE_T | ARR_NULL_T | ARR_NULL_F | DEEP_T | DEEP_F | ++-----------+-----------+----------+----------+----------+----------+----------+------------+------------+--------+--------+ +| true | true | true | true | true | true | true | true | false | true | false | ++-----------+-----------+----------+----------+----------+----------+----------+------------+------------+--------+--------+ +(1 row) +!ok + +# End some.iq diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/function/Functions.java b/linq4j/src/main/java/org/apache/calcite/linq4j/function/Functions.java index 6e9a186344c5..6c0a41297b3c 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/function/Functions.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/function/Functions.java @@ -23,6 +23,7 @@ import java.io.Serializable; import java.lang.reflect.Type; import java.math.BigDecimal; +import java.math.BigInteger; import java.util.AbstractList; import java.util.ArrayList; import java.util.Arrays; @@ -548,17 +549,7 @@ private static class NullsFirstComparator if (o2 == null) { return 1; } - if (o1 instanceof Comparable && o2 instanceof Comparable) { - //noinspection unchecked - return ((Comparable) o1).compareTo(o2); - } else if (o1 instanceof List && o2 instanceof List) { - return compareLists((List) o1, (List) o2); - } else if (o1 instanceof Object[] && o2 instanceof Object[]) { - return compareObjectArrays((Object[]) o1, (Object[]) o2); - } else { - throw new IllegalArgumentException("Item types do not match: " - + o1.getClass() + " vs " + o2.getClass()); - } + return compareListItems(o1, o2); } } @@ -566,26 +557,7 @@ private static class NullsFirstComparator private static class NullsLastComparator implements Comparator, Serializable { @Override public int compare(@Nullable Object o1, @Nullable Object o2) { - if (o1 == o2) { - return 0; - } - if (o1 == null) { - return 1; - } - if (o2 == null) { - return -1; - } - if (o1 instanceof Comparable && o2 instanceof Comparable) { - //noinspection unchecked - return ((Comparable) o1).compareTo(o2); - } else if (o1 instanceof List && o2 instanceof List) { - return compareLists((List) o1, (List) o2); - } else if (o1 instanceof Object[] && o2 instanceof Object[]) { - return compareObjectArrays((Object[]) o1, (Object[]) o2); - } else { - throw new IllegalArgumentException("Item types do not match: " - + o1.getClass() + " vs " + o2.getClass()); - } + return compareListItems(o1, o2); } } @@ -602,17 +574,7 @@ private static class NullsFirstReverseComparator if (o2 == null) { return 1; } - if (o1 instanceof Comparable && o2 instanceof Comparable) { - //noinspection unchecked - return -((Comparable) o1).compareTo(o2); - } else if (o1 instanceof List && o2 instanceof List) { - return -compareLists((List) o1, (List) o2); - } else if (o1 instanceof Object[] && o2 instanceof Object[]) { - return -compareObjectArrays((Object[]) o1, (Object[]) o2); - } else { - throw new IllegalArgumentException("Item types do not match: " - + o1.getClass() + " vs " + o2.getClass()); - } + return -compareListItems(o1, o2); } } @@ -667,6 +629,13 @@ public static int compareMaps(Map b0, Map b1) { return 0; } + private static BigDecimal toBigDecimal(Number number) { + return number instanceof BigDecimal ? (BigDecimal) number + : number instanceof BigInteger ? new BigDecimal((BigInteger) number) + : number instanceof Long ? new BigDecimal(number.longValue()) + : new BigDecimal(number.doubleValue()); + } + private static int compareListItems(@Nullable Object item0, @Nullable Object item1) { if (item0 == item1) { return 0; @@ -684,13 +653,24 @@ private static int compareListItems(@Nullable Object item0, @Nullable Object ite return compareMaps((Map) item0, (Map) item1); } else if (item0 instanceof Object[] && item1 instanceof Object[]) { return compareObjectArrays((Object[]) item0, (Object[]) item1); - } else if (item0.getClass().equals(item1.getClass()) && item0 instanceof Comparable) { - final Comparable b0Comparable = (Comparable) item0; - final Comparable b1Comparable = (Comparable) item1; - return b0Comparable.compareTo(b1Comparable); + } else if (item0 instanceof Number && item1 instanceof Number) { + final BigDecimal d0 = toBigDecimal((Number) item0); + final BigDecimal d1 = toBigDecimal((Number) item1); + return d0.compareTo(d1); + } else if (item0.getClass().equals(item1.getClass())) { + if (item0 instanceof Comparable) { + final Comparable b0Comparable = (Comparable) item0; + final Comparable b1Comparable = (Comparable) item1; + //noinspection unchecked + return b0Comparable.compareTo(b1Comparable); + } + return Objects.equals(item0, item1) + ? 0 + : Integer.compare(System.identityHashCode(item0), System.identityHashCode(item1)); } else { - throw new IllegalArgumentException("Item types do not match: " - + item0.getClass() + " vs " + item1.getClass()); + // comparison between objects with different types are possible, + // and they always return false + return item0.getClass().getName().compareTo(item1.getClass().getName()); } } @@ -721,17 +701,7 @@ private static class NullsLastReverseComparator if (o2 == null) { return -1; } - if (o1 instanceof Comparable && o2 instanceof Comparable) { - //noinspection unchecked - return -((Comparable) o1).compareTo(o2); - } else if (o1 instanceof List && o2 instanceof List) { - return -compareLists((List) o1, (List) o2); - } else if (o1 instanceof Object[] && o2 instanceof Object[]) { - return -compareObjectArrays((Object[]) o1, (Object[]) o2); - } else { - throw new IllegalArgumentException("Item types do not match: " - + o1.getClass() + " vs " + o2.getClass()); - } + return -compareListItems(o1, o2); } } From f12dd09a1f64c97ebe32cc30c4208d6718dcd9cb Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Sun, 18 Jan 2026 21:04:28 +0800 Subject: [PATCH 114/562] [CALCITE-7381] Parameters modified by !set must be restored to their default values in Quidem test --- .../apache/calcite/runtime/SqlFunctions.java | 5 + .../apache/calcite/test/JdbcAdapterTest.java | 38 +- core/src/test/resources/sql/agg.iq | 3 + core/src/test/resources/sql/misc.iq | 151 +- core/src/test/resources/sql/planner.iq | 65 + core/src/test/resources/sql/scalar.iq | 3 + core/src/test/resources/sql/set-op.iq | 68 - core/src/test/resources/sql/some.iq | 5 +- core/src/test/resources/sql/sub-query.iq | 1303 +++++++++-------- .../org/apache/calcite/test/QuidemTest.java | 11 + 10 files changed, 942 insertions(+), 710 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java index 28b34513e1c0..f9a74cec9a58 100644 --- a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java +++ b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java @@ -232,6 +232,11 @@ public class SqlFunctions { private static final TryThreadLocal> THREAD_SEQUENCES = TryThreadLocal.withInitial(HashMap::new); + /** Resets the sequences in the current thread. */ + public static void resetThreadSequences() { + THREAD_SEQUENCES.get().clear(); + } + /** A byte string consisting of a single byte that is the ASCII space * character (0x20). */ private static final ByteString SINGLE_SPACE_BYTE_STRING = diff --git a/core/src/test/java/org/apache/calcite/test/JdbcAdapterTest.java b/core/src/test/java/org/apache/calcite/test/JdbcAdapterTest.java index a94cad903bb6..989aac3604e3 100644 --- a/core/src/test/java/org/apache/calcite/test/JdbcAdapterTest.java +++ b/core/src/test/java/org/apache/calcite/test/JdbcAdapterTest.java @@ -21,6 +21,7 @@ import org.apache.calcite.config.CalciteConnectionProperty; import org.apache.calcite.config.Lex; import org.apache.calcite.plan.RelOptPlanner; +import org.apache.calcite.prepare.Prepare; import org.apache.calcite.runtime.Hook; import org.apache.calcite.test.CalciteAssert.AssertThat; import org.apache.calcite.test.CalciteAssert.DatabaseInstance; @@ -28,6 +29,7 @@ import org.apache.calcite.test.schemata.hr.HrSchema; import org.apache.calcite.util.Smalls; import org.apache.calcite.util.TestUtil; +import org.apache.calcite.util.TryThreadLocal; import org.hsqldb.jdbcDriver; import org.junit.jupiter.api.Test; @@ -181,23 +183,25 @@ class JdbcAdapterTest { } @Test void testInPlan() { - CalciteAssert.model(FoodmartSchema.FOODMART_MODEL) - .query("select \"store_id\", \"store_name\" from \"store\"\n" - + "where \"store_name\" in ('Store 1', 'Store 10', 'Store 11', 'Store 15', 'Store 16', 'Store 24', 'Store 3', 'Store 7')") - .runs() - .enable(CalciteAssert.DB == CalciteAssert.DatabaseInstance.HSQLDB) - .planHasSql("SELECT \"store_id\", \"store_name\"\n" - + "FROM \"foodmart\".\"store\"\n" - + "WHERE \"store_name\" IN ('Store 1', 'Store 10', 'Store 11'," - + " 'Store 15', 'Store 16', 'Store 24', 'Store 3', 'Store 7')") - .returns("store_id=1; store_name=Store 1\n" - + "store_id=3; store_name=Store 3\n" - + "store_id=7; store_name=Store 7\n" - + "store_id=10; store_name=Store 10\n" - + "store_id=11; store_name=Store 11\n" - + "store_id=15; store_name=Store 15\n" - + "store_id=16; store_name=Store 16\n" - + "store_id=24; store_name=Store 24\n"); + try (TryThreadLocal.Memo ignore = Prepare.THREAD_INSUBQUERY_THRESHOLD.push(20)) { + CalciteAssert.model(FoodmartSchema.FOODMART_MODEL) + .query("select \"store_id\", \"store_name\" from \"store\"\n" + + "where \"store_name\" in ('Store 1', 'Store 10', 'Store 11', 'Store 15', 'Store 16', 'Store 24', 'Store 3', 'Store 7')") + .runs() + .enable(CalciteAssert.DB == CalciteAssert.DatabaseInstance.HSQLDB) + .planHasSql("SELECT \"store_id\", \"store_name\"\n" + + "FROM \"foodmart\".\"store\"\n" + + "WHERE \"store_name\" IN ('Store 1', 'Store 10', 'Store 11'," + + " 'Store 15', 'Store 16', 'Store 24', 'Store 3', 'Store 7')") + .returns("store_id=1; store_name=Store 1\n" + + "store_id=3; store_name=Store 3\n" + + "store_id=7; store_name=Store 7\n" + + "store_id=10; store_name=Store 10\n" + + "store_id=11; store_name=Store 11\n" + + "store_id=15; store_name=Store 15\n" + + "store_id=16; store_name=Store 16\n" + + "store_id=24; store_name=Store 24\n"); + } } @Test void testEquiJoinPlan() { diff --git a/core/src/test/resources/sql/agg.iq b/core/src/test/resources/sql/agg.iq index 51e6221f1b78..5478f6bb6e2c 100644 --- a/core/src/test/resources/sql/agg.iq +++ b/core/src/test/resources/sql/agg.iq @@ -3710,6 +3710,9 @@ group by !ok +# Reset to default value 20 +!set insubquerythreshold 20 + select deptno, case when deptno in (10) then 1 else 2 end as col1, diff --git a/core/src/test/resources/sql/misc.iq b/core/src/test/resources/sql/misc.iq index 6db6e0bd450d..d4d8dd93d7d9 100644 --- a/core/src/test/resources/sql/misc.iq +++ b/core/src/test/resources/sql/misc.iq @@ -2267,40 +2267,68 @@ values atan2(0.5, 2); !ok -!set outputformat csv - # [CALCITE-1167] OVERLAPS should match even if operands are in (high, low) order values ((date '1999-12-01', date '2001-12-31') overlaps (date '2001-01-01' , date '2002-11-11')); -EXPR$0 -true ++--------+ +| EXPR$0 | ++--------+ +| true | ++--------+ +(1 row) + !ok values ((date '2001-12-31', date '1999-12-01') overlaps (date '2001-01-01' , date '2002-11-11')); -EXPR$0 -true ++--------+ +| EXPR$0 | ++--------+ +| true | ++--------+ +(1 row) + !ok values ((date '2001-12-31', date '1999-12-01') overlaps (date '2002-11-11', date '2001-01-01')); -EXPR$0 -true ++--------+ +| EXPR$0 | ++--------+ +| true | ++--------+ +(1 row) + !ok values ((date '2001-12-31', date '1999-12-01') overlaps (date '2002-01-01', date '2002-11-11')); -EXPR$0 -false ++--------+ +| EXPR$0 | ++--------+ +| false | ++--------+ +(1 row) + !ok # Sub-query returns a MAP, column is renamed, and enclosing query references the map. select mycol['b'] as x from (select map['a', false, 'b', true] from (values (2))) as t(mycol); -X -true ++------+ +| X | ++------+ +| true | ++------+ +(1 row) + !ok # JSON values json_exists('{"foo":"bar"}', 'strict $.foo' false on error); -EXPR$0 -true ++--------+ +| EXPR$0 | ++--------+ +| true | ++--------+ +(1 row) + !ok # [CALCITE-2908] Implement SQL LAST_DAY function @@ -2313,22 +2341,32 @@ with data(c_date, c_timestamp) as (select * from (values (DATE'2019-06-28', TIMESTAMP '2019-06-28 17:32:01'), (DATE'2019-12-12', TIMESTAMP '2019-12-12 12:12:01'))) select last_day(c_date), last_day(c_timestamp) from data; -EXPR$0, EXPR$1 -1965-01-31, 1965-01-31 -2019-01-31, 2019-01-31 -2019-02-28, 2019-02-28 -2019-02-28, 2019-02-28 -2019-03-31, 2019-03-31 -2019-06-30, 2019-06-30 -2019-12-31, 2019-12-31 ++------------+------------+ +| EXPR$0 | EXPR$1 | ++------------+------------+ +| 1965-01-31 | 1965-01-31 | +| 2019-01-31 | 2019-01-31 | +| 2019-02-28 | 2019-02-28 | +| 2019-02-28 | 2019-02-28 | +| 2019-03-31 | 2019-03-31 | +| 2019-06-30 | 2019-06-30 | +| 2019-12-31 | 2019-12-31 | ++------------+------------+ +(7 rows) + !ok # [CALCITE-3142] An NPE when rounding a nullable numeric SELECT ROUND(CAST((X/Y) AS NUMERIC), 2) FROM (VALUES (1, 2), (NULLIF(5, 5), NULLIF(5, 5))) A(X, Y); -EXPR$0 -0.00 -null ++--------+ +| EXPR$0 | ++--------+ +| 0.00 | +| | ++--------+ +(2 rows) + !ok # [CALCITE-3143]Dividing NULLIF clause may cause Division by zero error @@ -2337,17 +2375,27 @@ FROM ( SELECT SUM("X") / NULLIF(SUM(0),0) AS Z FROM (VALUES (1.1, 2.5), (4.51, 32.5)) A(X, Y) GROUP BY "Y"); -EXPR$0 -88 -88 ++--------+ +| EXPR$0 | ++--------+ +| 88 | +| 88 | ++--------+ +(2 rows) + !ok # [CALCITE-3150] NPE in UPPER when repeated and combine with LIKE SELECT "NAME" FROM (VALUES ('Bill'), NULLIF('x', 'x'), ('Eric')) A(NAME) WHERE UPPER("NAME") LIKE 'B%' AND UPPER("NAME") LIKE '%L'; -NAME -Bill ++------+ +| NAME | ++------+ +| Bill | ++------+ +(1 row) + !ok # [CALCITE-3717] Query fails with "division by zero" exception @@ -2364,32 +2412,45 @@ FROM (VALUES (0, 2, 4, 8), (1, 2, 4, 0), (0, 0, 0, 0), (1, 2, 4, 8), - (CAST(null as int), CAST(null as int), CAST(null as int), CAST(null as int))) AS T(A,B,C,D); -V -13.00000000 -9.50000000 -1.75000000 -1.87500000 -null -0E-8 -14.00000000 + (CAST(null as int), CAST(null as int), CAST(null as int), CAST(null as int))) AS T(A,B,C,D) order by V; ++-------------+ +| V | ++-------------+ +| 0E-8 | +| 1.75000000 | +| 1.87500000 | +| 9.50000000 | +| 13.00000000 | +| 14.00000000 | +| | ++-------------+ +(7 rows) + !ok # TIMESTAMP literals without a time part are OK. SELECT TIMESTAMP '1969-07-20' AS ts; -TS -1969-07-20 00:00:00 ++---------------------+ +| TS | ++---------------------+ +| 1969-07-20 00:00:00 | ++---------------------+ +(1 row) + !ok # Short TIMESTAMP literals are equivalent to long TIMESTAMP literals SELECT TIMESTAMP '1969-07-20' + i AS ts FROM (VALUES (INTERVAL '1' DAY)) AS t (i) GROUP BY TIMESTAMP '1969-07-20 00:00:00' + i; -TS -1969-07-21 00:00:00 -!ok ++---------------------+ +| TS | ++---------------------+ +| 1969-07-21 00:00:00 | ++---------------------+ +(1 row) -!set outputformat mysql +!ok # [CALCITE-5870] Allow literals like DECIMAL '12.3' (consistent with Postgres) # Test a decimal value between decimal logic for range checking. diff --git a/core/src/test/resources/sql/planner.iq b/core/src/test/resources/sql/planner.iq index 1c90347a216f..96aaaf5eedc1 100644 --- a/core/src/test/resources/sql/planner.iq +++ b/core/src/test/resources/sql/planner.iq @@ -541,4 +541,69 @@ EnumerableCalc(expr#0..2=[{inputs}], proj#0..1=[{exprs}]) !plan !set planner-rules original +# [CALCITE-7125] Impossible to get a plan with partial aggregate push-down via IntersectToDistinctRule +!set planner-rules " +-EnumerableRules.ENUMERABLE_INTERSECT_RULE, +-AGGREGATE_REMOVE" +# AGGREGATE_REMOVE is disabled as it would remove the inner COUNTs +# as the grouping columns are unique + +# Intersect rewrite as aggregation + union with partial aggregation pushdown +select empno, ename from emp where deptno = 10 +intersect +select empno, ename from emp where empno >= 150; + +EnumerableCalc(expr#0..2=[{inputs}], expr#3=[2:BIGINT], expr#4=[=($t2, $t3)], proj#0..1=[{exprs}], $condition=[$t4]) + EnumerableAggregate(group=[{0, 1}], agg#0=[COUNT()]) + EnumerableUnion(all=[true]) + EnumerableAggregate(group=[{0, 1}], agg#0=[COUNT()]) + EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t7):INTEGER], expr#9=[10], expr#10=[=($t8, $t9)], proj#0..7=[{exprs}], $condition=[$t10]) + EnumerableTableScan(table=[[scott, EMP]]) + EnumerableAggregate(group=[{0, 1}], agg#0=[COUNT()]) + EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t0):INTEGER NOT NULL], expr#9=[150], expr#10=[>=($t8, $t9)], proj#0..7=[{exprs}], $condition=[$t10]) + EnumerableTableScan(table=[[scott, EMP]]) +!plan ++-------+--------+ +| EMPNO | ENAME | ++-------+--------+ +| 7782 | CLARK | +| 7839 | KING | +| 7934 | MILLER | ++-------+--------+ +(3 rows) + +!ok +!set planner-rules original + +!set planner-rules " +-CoreRules.INTERSECT_TO_DISTINCT, +-EnumerableRules.ENUMERABLE_INTERSECT_RULE, ++CoreRules.INTERSECT_TO_DISTINCT_NO_AGGREGATE_PUSHDOWN" + +# Intersect rewrite as aggregation + union without partial aggregation pushdown +select empno, ename from emp where deptno = 10 +intersect +select empno, ename from emp where empno >= 150; + +EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[>($t2, $t4)], expr#6=[>($t3, $t4)], expr#7=[AND($t5, $t6)], proj#0..1=[{exprs}], $condition=[$t7]) + EnumerableAggregate(group=[{0, 1}], count_i0=[COUNT() FILTER $2], count_i1=[COUNT() FILTER $3]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[1], expr#6=[=($t2, $t5)], proj#0..1=[{exprs}], $f3=[$t4], $f4=[$t6]) + EnumerableUnion(all=[true]) + EnumerableCalc(expr#0..7=[{inputs}], expr#8=[0], expr#9=[CAST($t7):INTEGER], expr#10=[10], expr#11=[=($t9, $t10)], proj#0..1=[{exprs}], i=[$t8], $condition=[$t11]) + EnumerableTableScan(table=[[scott, EMP]]) + EnumerableCalc(expr#0..7=[{inputs}], expr#8=[1], expr#9=[CAST($t0):INTEGER NOT NULL], expr#10=[150], expr#11=[>=($t9, $t10)], proj#0..1=[{exprs}], i=[$t8], $condition=[$t11]) + EnumerableTableScan(table=[[scott, EMP]]) +!plan ++-------+--------+ +| EMPNO | ENAME | ++-------+--------+ +| 7782 | CLARK | +| 7839 | KING | +| 7934 | MILLER | ++-------+--------+ +(3 rows) + +!ok +!set planner-rules original + # End planner.iq diff --git a/core/src/test/resources/sql/scalar.iq b/core/src/test/resources/sql/scalar.iq index 4b5e186cc99f..91cd88279ae7 100644 --- a/core/src/test/resources/sql/scalar.iq +++ b/core/src/test/resources/sql/scalar.iq @@ -403,4 +403,7 @@ EnumerableCalc(expr#0..9=[{inputs}], expr#10=[IS NULL($t9)], expr#11=[0:BIGINT], EnumerableTableScan(table=[[scott, EMP]]) !plan +# Reset to default value true +!set trimfields true + # End scalar.iq diff --git a/core/src/test/resources/sql/set-op.iq b/core/src/test/resources/sql/set-op.iq index b52debdf3e8c..4b3917ad309d 100644 --- a/core/src/test/resources/sql/set-op.iq +++ b/core/src/test/resources/sql/set-op.iq @@ -47,74 +47,6 @@ intersect !ok -!use scott - -!set planner-rules " --EnumerableRules.ENUMERABLE_INTERSECT_RULE, --AGGREGATE_REMOVE" -# AGGREGATE_REMOVE is disabled as it would remove the inner COUNTs -# as the grouping columns are unique - -# Intersect rewrite as aggregation + union with partial aggregation pushdown -select empno, ename from emp where deptno = 10 -intersect -select empno, ename from emp where empno >= 150; - -EnumerableCalc(expr#0..2=[{inputs}], expr#3=[2:BIGINT], expr#4=[=($t2, $t3)], proj#0..1=[{exprs}], $condition=[$t4]) - EnumerableAggregate(group=[{0, 1}], agg#0=[COUNT()]) - EnumerableUnion(all=[true]) - EnumerableAggregate(group=[{0, 1}], agg#0=[COUNT()]) - EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t7):INTEGER], expr#9=[10], expr#10=[=($t8, $t9)], proj#0..7=[{exprs}], $condition=[$t10]) - EnumerableTableScan(table=[[scott, EMP]]) - EnumerableAggregate(group=[{0, 1}], agg#0=[COUNT()]) - EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t0):INTEGER NOT NULL], expr#9=[150], expr#10=[>=($t8, $t9)], proj#0..7=[{exprs}], $condition=[$t10]) - EnumerableTableScan(table=[[scott, EMP]]) -!plan -+-------+--------+ -| EMPNO | ENAME | -+-------+--------+ -| 7782 | CLARK | -| 7839 | KING | -| 7934 | MILLER | -+-------+--------+ -(3 rows) - -!ok -!set planner-rules original - -!use scott - -!set planner-rules " --CoreRules.INTERSECT_TO_DISTINCT, --EnumerableRules.ENUMERABLE_INTERSECT_RULE, -+CoreRules.INTERSECT_TO_DISTINCT_NO_AGGREGATE_PUSHDOWN" - -# Intersect rewrite as aggregation + union without partial aggregation pushdown -select empno, ename from emp where deptno = 10 -intersect -select empno, ename from emp where empno >= 150; - -EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[>($t2, $t4)], expr#6=[>($t3, $t4)], expr#7=[AND($t5, $t6)], proj#0..1=[{exprs}], $condition=[$t7]) - EnumerableAggregate(group=[{0, 1}], count_i0=[COUNT() FILTER $2], count_i1=[COUNT() FILTER $3]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[1], expr#6=[=($t2, $t5)], proj#0..1=[{exprs}], $f3=[$t4], $f4=[$t6]) - EnumerableUnion(all=[true]) - EnumerableCalc(expr#0..7=[{inputs}], expr#8=[0], expr#9=[CAST($t7):INTEGER], expr#10=[10], expr#11=[=($t9, $t10)], proj#0..1=[{exprs}], i=[$t8], $condition=[$t11]) - EnumerableTableScan(table=[[scott, EMP]]) - EnumerableCalc(expr#0..7=[{inputs}], expr#8=[1], expr#9=[CAST($t0):INTEGER NOT NULL], expr#10=[150], expr#11=[>=($t9, $t10)], proj#0..1=[{exprs}], i=[$t8], $condition=[$t11]) - EnumerableTableScan(table=[[scott, EMP]]) -!plan -+-------+--------+ -| EMPNO | ENAME | -+-------+--------+ -| 7782 | CLARK | -| 7839 | KING | -| 7934 | MILLER | -+-------+--------+ -(3 rows) - -!ok -!set planner-rules original - # Intersect all with null value rows select * from (select x, y from (values (cast(NULL as int), cast(NULL as varchar(1))), diff --git a/core/src/test/resources/sql/some.iq b/core/src/test/resources/sql/some.iq index acc61d9c92a3..02e771241f34 100644 --- a/core/src/test/resources/sql/some.iq +++ b/core/src/test/resources/sql/some.iq @@ -895,7 +895,10 @@ where sal > some (4000, 2000); !ok -# CALCITE-6786: ANY/SOME operator yields multiple rows in correlated queries +# Reset to default value 20 +!set insubquerythreshold 20 + +# [CALCITE-6786] ANY/SOME operator yields multiple rows in correlated queries WITH tb as (select array(SELECT * FROM (VALUES (TRUE), (NULL)) as x(a)) as a) SELECT TRUE IN (SELECT b FROM UNNEST(a) AS x1(b)) AS test FROM tb; diff --git a/core/src/test/resources/sql/sub-query.iq b/core/src/test/resources/sql/sub-query.iq index 404520f0dc07..e981cc15d3a3 100644 --- a/core/src/test/resources/sql/sub-query.iq +++ b/core/src/test/resources/sql/sub-query.iq @@ -16,7 +16,7 @@ # limitations under the License. # !use post -!set outputformat psql +!set outputformat mysql # [CALCITE-373] # the following should return no rows, because the IN list has a null. @@ -27,8 +27,10 @@ t2(x) as (select * from (values 1,case when 1 = 1 then null else 3 end)) select * from t1 where t1.x not in (select t2.x from t2); - X ---- ++---+ +| X | ++---+ ++---+ (0 rows) !ok @@ -50,16 +52,20 @@ t2(x) as (select * from (values (1),(case when 1 = 1 then null else 3 end)) as select * from t1 where t1.x not in (select t2.x from t2); - X ---- ++---+ +| X | ++---+ ++---+ (0 rows) !ok # RHS has a mixture of NULL and NOT NULL keys select * from dept where deptno not in (select deptno from emp); - DEPTNO | DNAME ---------+------- ++--------+-------+ +| DEPTNO | DNAME | ++--------+-------+ ++--------+-------+ (0 rows) !ok @@ -70,12 +76,14 @@ SELECT "hr"."emps"."empid", "hr"."emps"."deptno", FROM "hr"."emps" WHERE "hr"."emps"."empid"<"hr"."emps"."salary" ORDER BY 1,2,3; - empid | deptno | EXPR$2 --------+--------+-------- - 100 | 10 | 0 - 110 | 10 | 0 - 150 | 10 | 0 - 200 | 20 | 2 ++-------+--------+--------+ +| empid | deptno | EXPR$2 | ++-------+--------+--------+ +| 100 | 10 | 0 | +| 110 | 10 | 0 | +| 150 | 10 | 0 | +| 200 | 20 | 2 | ++-------+--------+--------+ (4 rows) !ok @@ -83,63 +91,73 @@ FROM "hr"."emps" # [CALCITE-5638] Assertion Failure during planning correlated query SELECT t1.deptno FROM dept AS t0 JOIN emp AS t1 ON (t1.deptno = (SELECT inner_t1.deptno FROM emp AS inner_t1 WHERE inner_t1.ENAME = t0.DNAME)); - DEPTNO --------- ++--------+ +| DEPTNO | ++--------+ ++--------+ (0 rows) !ok select deptno, deptno in (select deptno from emp) from dept; - DEPTNO | EXPR$1 ---------+-------- - 10 | true - 20 | true - 30 | true - 40 | null ++--------+--------+ +| DEPTNO | EXPR$1 | ++--------+--------+ +| 10 | true | +| 20 | true | +| 30 | true | +| 40 | | ++--------+--------+ (4 rows) !ok select deptno, deptno not in (select deptno from emp) from dept; - DEPTNO | EXPR$1 ---------+-------- - 10 | false - 20 | false - 30 | false - 40 | null ++--------+--------+ +| DEPTNO | EXPR$1 | ++--------+--------+ +| 10 | false | +| 20 | false | +| 30 | false | +| 40 | | ++--------+--------+ (4 rows) !ok # RHS has only NULL keys select * from dept where deptno not in (select deptno from emp where deptno is null); - DEPTNO | DNAME ---------+------- ++--------+-------+ +| DEPTNO | DNAME | ++--------+-------+ ++--------+-------+ (0 rows) !ok select deptno, deptno in (select deptno from emp where deptno is null) from dept; - DEPTNO | EXPR$1 ---------+-------- - 10 | null - 20 | null - 30 | null - 40 | null ++--------+--------+ +| DEPTNO | EXPR$1 | ++--------+--------+ +| 10 | | +| 20 | | +| 30 | | +| 40 | | ++--------+--------+ (4 rows) !ok select deptno, deptno not in (select deptno from emp where deptno is null) from dept; - DEPTNO | EXPR$1 ---------+-------- - 10 | null - 20 | null - 30 | null - 40 | null ++--------+--------+ +| DEPTNO | EXPR$1 | ++--------+--------+ +| 10 | | +| 20 | | +| 30 | | +| 40 | | ++--------+--------+ (4 rows) !ok -!set outputformat mysql - # RHS has only NOT NULL keys select * from dept where deptno not in (select deptno from emp where deptno is not null); +--------+-------------+ @@ -1314,8 +1332,6 @@ and empno in (7876, 7698, 7900); !ok -!set outputformat psql - # [CALCITE-2329] Enhance SubQueryRemoveRule to rewrite IN operator with the constant from the left side more optimally # Test project null IN null select sal, @@ -1323,22 +1339,24 @@ select sal, select cast(null as int) from "scott".dept) from "scott".emp; - SAL | EXPR$1 ----------+-------- - 1100.00 | null - 1250.00 | null - 1250.00 | null - 1300.00 | null - 1500.00 | null - 1600.00 | null - 2450.00 | null - 2850.00 | null - 2975.00 | null - 3000.00 | null - 3000.00 | null - 5000.00 | null - 800.00 | null - 950.00 | null ++---------+--------+ +| SAL | EXPR$1 | ++---------+--------+ +| 1100.00 | | +| 1250.00 | | +| 1250.00 | | +| 1300.00 | | +| 1500.00 | | +| 1600.00 | | +| 2450.00 | | +| 2850.00 | | +| 2975.00 | | +| 3000.00 | | +| 3000.00 | | +| 5000.00 | | +| 800.00 | | +| 950.00 | | ++---------+--------+ (14 rows) !ok @@ -1359,22 +1377,24 @@ select sal, select cast(null as int) from "scott".dept) from "scott".emp; - SAL | EXPR$1 ----------+-------- - 1100.00 | null - 1250.00 | null - 1250.00 | null - 1300.00 | null - 1500.00 | null - 1600.00 | null - 2450.00 | null - 2850.00 | null - 2975.00 | null - 3000.00 | null - 3000.00 | null - 5000.00 | null - 800.00 | null - 950.00 | null ++---------+--------+ +| SAL | EXPR$1 | ++---------+--------+ +| 1100.00 | | +| 1250.00 | | +| 1250.00 | | +| 1300.00 | | +| 1500.00 | | +| 1600.00 | | +| 2450.00 | | +| 2850.00 | | +| 2975.00 | | +| 3000.00 | | +| 3000.00 | | +| 5000.00 | | +| 800.00 | | +| 950.00 | | ++---------+--------+ (14 rows) !ok @@ -1395,22 +1415,24 @@ select sal, select 1 from "scott".dept) from "scott".emp; - SAL | EXPR$1 ----------+-------- - 1100.00 | null - 1250.00 | null - 1250.00 | null - 1300.00 | null - 1500.00 | null - 1600.00 | null - 2450.00 | null - 2850.00 | null - 2975.00 | null - 3000.00 | null - 3000.00 | null - 5000.00 | null - 800.00 | null - 950.00 | null ++---------+--------+ +| SAL | EXPR$1 | ++---------+--------+ +| 1100.00 | | +| 1250.00 | | +| 1250.00 | | +| 1300.00 | | +| 1500.00 | | +| 1600.00 | | +| 2450.00 | | +| 2850.00 | | +| 2975.00 | | +| 3000.00 | | +| 3000.00 | | +| 5000.00 | | +| 800.00 | | +| 950.00 | | ++---------+--------+ (14 rows) !ok @@ -1431,22 +1453,24 @@ select sal, select deptno from "scott".dept) from "scott".emp; - SAL | EXPR$1 ----------+-------- - 1100.00 | null - 1250.00 | null - 1250.00 | null - 1300.00 | null - 1500.00 | null - 1600.00 | null - 2450.00 | null - 2850.00 | null - 2975.00 | null - 3000.00 | null - 3000.00 | null - 5000.00 | null - 800.00 | null - 950.00 | null ++---------+--------+ +| SAL | EXPR$1 | ++---------+--------+ +| 1100.00 | | +| 1250.00 | | +| 1250.00 | | +| 1300.00 | | +| 1500.00 | | +| 1600.00 | | +| 2450.00 | | +| 2850.00 | | +| 2975.00 | | +| 3000.00 | | +| 3000.00 | | +| 5000.00 | | +| 800.00 | | +| 950.00 | | ++---------+--------+ (14 rows) !ok @@ -1467,22 +1491,24 @@ select sal, select case when deptno > 0 then deptno else null end from "scott".dept) from "scott".emp; - SAL | EXPR$1 ----------+-------- - 1100.00 | null - 1250.00 | null - 1250.00 | null - 1300.00 | null - 1500.00 | null - 1600.00 | null - 2450.00 | null - 2850.00 | null - 2975.00 | null - 3000.00 | null - 3000.00 | null - 5000.00 | null - 800.00 | null - 950.00 | null ++---------+--------+ +| SAL | EXPR$1 | ++---------+--------+ +| 1100.00 | | +| 1250.00 | | +| 1250.00 | | +| 1300.00 | | +| 1500.00 | | +| 1600.00 | | +| 2450.00 | | +| 2850.00 | | +| 2975.00 | | +| 3000.00 | | +| 3000.00 | | +| 5000.00 | | +| 800.00 | | +| 950.00 | | ++---------+--------+ (14 rows) !ok @@ -1503,22 +1529,24 @@ select sal, select deptno from "scott".dept) from "scott".emp; - SAL | EXPR$1 ----------+-------- - 1100.00 | true - 1250.00 | true - 1250.00 | true - 1300.00 | true - 1500.00 | true - 1600.00 | true - 2450.00 | true - 2850.00 | true - 2975.00 | true - 3000.00 | true - 3000.00 | true - 5000.00 | true - 800.00 | true - 950.00 | true ++---------+--------+ +| SAL | EXPR$1 | ++---------+--------+ +| 1100.00 | true | +| 1250.00 | true | +| 1250.00 | true | +| 1300.00 | true | +| 1500.00 | true | +| 1600.00 | true | +| 2450.00 | true | +| 2850.00 | true | +| 2975.00 | true | +| 3000.00 | true | +| 3000.00 | true | +| 5000.00 | true | +| 800.00 | true | +| 950.00 | true | ++---------+--------+ (14 rows) !ok @@ -1537,22 +1565,24 @@ select sal, select case when deptno > 0 then deptno else null end from "scott".dept) from "scott".emp; - SAL | EXPR$1 ----------+-------- - 1100.00 | true - 1250.00 | true - 1250.00 | true - 1300.00 | true - 1500.00 | true - 1600.00 | true - 2450.00 | true - 2850.00 | true - 2975.00 | true - 3000.00 | true - 3000.00 | true - 5000.00 | true - 800.00 | true - 950.00 | true ++---------+--------+ +| SAL | EXPR$1 | ++---------+--------+ +| 1100.00 | true | +| 1250.00 | true | +| 1250.00 | true | +| 1300.00 | true | +| 1500.00 | true | +| 1600.00 | true | +| 2450.00 | true | +| 2850.00 | true | +| 2975.00 | true | +| 3000.00 | true | +| 3000.00 | true | +| 5000.00 | true | +| 800.00 | true | +| 950.00 | true | ++---------+--------+ (14 rows) !ok @@ -1573,22 +1603,24 @@ select sal, select cast(null as int) from "scott".dept) from "scott".emp; - SAL | EXPR$1 ----------+-------- - 1100.00 | null - 1250.00 | null - 1250.00 | null - 1300.00 | null - 1500.00 | null - 1600.00 | null - 2450.00 | null - 2850.00 | null - 2975.00 | null - 3000.00 | null - 3000.00 | null - 5000.00 | null - 800.00 | null - 950.00 | null ++---------+--------+ +| SAL | EXPR$1 | ++---------+--------+ +| 1100.00 | | +| 1250.00 | | +| 1250.00 | | +| 1300.00 | | +| 1500.00 | | +| 1600.00 | | +| 2450.00 | | +| 2850.00 | | +| 2975.00 | | +| 3000.00 | | +| 3000.00 | | +| 5000.00 | | +| 800.00 | | +| 950.00 | | ++---------+--------+ (14 rows) !ok @@ -1609,22 +1641,24 @@ select sal, select cast(null as int) from "scott".dept) from "scott".emp; - SAL | EXPR$1 ----------+-------- - 1100.00 | null - 1250.00 | null - 1250.00 | null - 1300.00 | null - 1500.00 | null - 1600.00 | null - 2450.00 | null - 2850.00 | null - 2975.00 | null - 3000.00 | null - 3000.00 | null - 5000.00 | null - 800.00 | null - 950.00 | null ++---------+--------+ +| SAL | EXPR$1 | ++---------+--------+ +| 1100.00 | | +| 1250.00 | | +| 1250.00 | | +| 1300.00 | | +| 1500.00 | | +| 1600.00 | | +| 2450.00 | | +| 2850.00 | | +| 2975.00 | | +| 3000.00 | | +| 3000.00 | | +| 5000.00 | | +| 800.00 | | +| 950.00 | | ++---------+--------+ (14 rows) !ok @@ -1645,22 +1679,24 @@ select sal, select 1 from "scott".dept) from "scott".emp; - SAL | EXPR$1 ----------+-------- - 1100.00 | null - 1250.00 | null - 1250.00 | null - 1300.00 | null - 1500.00 | null - 1600.00 | null - 2450.00 | null - 2850.00 | null - 2975.00 | null - 3000.00 | null - 3000.00 | null - 5000.00 | null - 800.00 | null - 950.00 | null ++---------+--------+ +| SAL | EXPR$1 | ++---------+--------+ +| 1100.00 | | +| 1250.00 | | +| 1250.00 | | +| 1300.00 | | +| 1500.00 | | +| 1600.00 | | +| 2450.00 | | +| 2850.00 | | +| 2975.00 | | +| 3000.00 | | +| 3000.00 | | +| 5000.00 | | +| 800.00 | | +| 950.00 | | ++---------+--------+ (14 rows) !ok @@ -1681,22 +1717,24 @@ select sal, select deptno from "scott".dept) from "scott".emp; - SAL | EXPR$1 ----------+-------- - 1100.00 | null - 1250.00 | null - 1250.00 | null - 1300.00 | null - 1500.00 | null - 1600.00 | null - 2450.00 | null - 2850.00 | null - 2975.00 | null - 3000.00 | null - 3000.00 | null - 5000.00 | null - 800.00 | null - 950.00 | null ++---------+--------+ +| SAL | EXPR$1 | ++---------+--------+ +| 1100.00 | | +| 1250.00 | | +| 1250.00 | | +| 1300.00 | | +| 1500.00 | | +| 1600.00 | | +| 2450.00 | | +| 2850.00 | | +| 2975.00 | | +| 3000.00 | | +| 3000.00 | | +| 5000.00 | | +| 800.00 | | +| 950.00 | | ++---------+--------+ (14 rows) !ok @@ -1717,22 +1755,24 @@ select sal, select case when deptno > 0 then deptno else null end from "scott".dept) from "scott".emp; - SAL | EXPR$1 ----------+-------- - 1100.00 | null - 1250.00 | null - 1250.00 | null - 1300.00 | null - 1500.00 | null - 1600.00 | null - 2450.00 | null - 2850.00 | null - 2975.00 | null - 3000.00 | null - 3000.00 | null - 5000.00 | null - 800.00 | null - 950.00 | null ++---------+--------+ +| SAL | EXPR$1 | ++---------+--------+ +| 1100.00 | | +| 1250.00 | | +| 1250.00 | | +| 1300.00 | | +| 1500.00 | | +| 1600.00 | | +| 2450.00 | | +| 2850.00 | | +| 2975.00 | | +| 3000.00 | | +| 3000.00 | | +| 5000.00 | | +| 800.00 | | +| 950.00 | | ++---------+--------+ (14 rows) !ok @@ -1753,22 +1793,24 @@ select sal, select deptno from "scott".dept) from "scott".emp; - SAL | EXPR$1 ----------+-------- - 1100.00 | false - 1250.00 | false - 1250.00 | false - 1300.00 | false - 1500.00 | false - 1600.00 | false - 2450.00 | false - 2850.00 | false - 2975.00 | false - 3000.00 | false - 3000.00 | false - 5000.00 | false - 800.00 | false - 950.00 | false ++---------+--------+ +| SAL | EXPR$1 | ++---------+--------+ +| 1100.00 | false | +| 1250.00 | false | +| 1250.00 | false | +| 1300.00 | false | +| 1500.00 | false | +| 1600.00 | false | +| 2450.00 | false | +| 2850.00 | false | +| 2975.00 | false | +| 3000.00 | false | +| 3000.00 | false | +| 5000.00 | false | +| 800.00 | false | +| 950.00 | false | ++---------+--------+ (14 rows) !ok @@ -1787,22 +1829,24 @@ select sal, select case when deptno > 0 then deptno else null end from "scott".dept) from "scott".emp; - SAL | EXPR$1 ----------+-------- - 1100.00 | false - 1250.00 | false - 1250.00 | false - 1300.00 | false - 1500.00 | false - 1600.00 | false - 2450.00 | false - 2850.00 | false - 2975.00 | false - 3000.00 | false - 3000.00 | false - 5000.00 | false - 800.00 | false - 950.00 | false ++---------+--------+ +| SAL | EXPR$1 | ++---------+--------+ +| 1100.00 | false | +| 1250.00 | false | +| 1250.00 | false | +| 1300.00 | false | +| 1500.00 | false | +| 1600.00 | false | +| 2450.00 | false | +| 2850.00 | false | +| 2975.00 | false | +| 3000.00 | false | +| 3000.00 | false | +| 5000.00 | false | +| 800.00 | false | +| 950.00 | false | ++---------+--------+ (14 rows) !ok @@ -1823,22 +1867,24 @@ select sal, select deptno from "scott".dept) is unknown from "scott".emp; - SAL | EXPR$1 ----------+-------- - 1100.00 | true - 1250.00 | true - 1250.00 | true - 1300.00 | true - 1500.00 | true - 1600.00 | true - 2450.00 | true - 2850.00 | true - 2975.00 | true - 3000.00 | true - 3000.00 | true - 5000.00 | true - 800.00 | true - 950.00 | true ++---------+--------+ +| SAL | EXPR$1 | ++---------+--------+ +| 1100.00 | true | +| 1250.00 | true | +| 1250.00 | true | +| 1300.00 | true | +| 1500.00 | true | +| 1600.00 | true | +| 2450.00 | true | +| 2850.00 | true | +| 2975.00 | true | +| 3000.00 | true | +| 3000.00 | true | +| 5000.00 | true | +| 800.00 | true | +| 950.00 | true | ++---------+--------+ (14 rows) !ok @@ -1858,8 +1904,10 @@ select sal from "scott".emp where cast(null as int) IN ( select cast(null as int) from "scott".dept); - SAL ------ ++-----+ +| SAL | ++-----+ ++-----+ (0 rows) !ok @@ -1871,8 +1919,10 @@ select sal from "scott".emp where 123 IN ( select cast(null as int) from "scott".dept); - SAL ------ ++-----+ +| SAL | ++-----+ ++-----+ (0 rows) !ok @@ -1884,8 +1934,10 @@ select sal from "scott".emp where cast(null as int) IN ( select 1 from "scott".dept); - SAL ------ ++-----+ +| SAL | ++-----+ ++-----+ (0 rows) !ok @@ -1897,8 +1949,10 @@ select sal from "scott".emp where cast(null as int) IN ( select deptno from "scott".dept); - SAL ------ ++-----+ +| SAL | ++-----+ ++-----+ (0 rows) !ok @@ -1910,8 +1964,10 @@ select sal from "scott".emp where cast(null as int) IN ( select case when deptno > 0 then deptno else null end from "scott".dept); - SAL ------ ++-----+ +| SAL | ++-----+ ++-----+ (0 rows) !ok @@ -1923,22 +1979,24 @@ select sal from "scott".emp where 10 IN ( select deptno from "scott".dept); - SAL ---------- - 1100.00 - 1250.00 - 1250.00 - 1300.00 - 1500.00 - 1600.00 - 2450.00 - 2850.00 - 2975.00 - 3000.00 - 3000.00 - 5000.00 - 800.00 - 950.00 ++---------+ +| SAL | ++---------+ +| 1100.00 | +| 1250.00 | +| 1250.00 | +| 1300.00 | +| 1500.00 | +| 1600.00 | +| 2450.00 | +| 2850.00 | +| 2975.00 | +| 3000.00 | +| 3000.00 | +| 5000.00 | +| 800.00 | +| 950.00 | ++---------+ (14 rows) !ok @@ -1956,22 +2014,24 @@ select sal from "scott".emp where 10 IN ( select case when deptno > 0 then deptno else null end from "scott".dept); - SAL ---------- - 1100.00 - 1250.00 - 1250.00 - 1300.00 - 1500.00 - 1600.00 - 2450.00 - 2850.00 - 2975.00 - 3000.00 - 3000.00 - 5000.00 - 800.00 - 950.00 ++---------+ +| SAL | ++---------+ +| 1100.00 | +| 1250.00 | +| 1250.00 | +| 1300.00 | +| 1500.00 | +| 1600.00 | +| 2450.00 | +| 2850.00 | +| 2975.00 | +| 3000.00 | +| 3000.00 | +| 5000.00 | +| 800.00 | +| 950.00 | ++---------+ (14 rows) !ok @@ -1989,8 +2049,10 @@ select sal from "scott".emp where cast(null as int) NOT IN ( select cast(null as int) from "scott".dept); - SAL ------ ++-----+ +| SAL | ++-----+ ++-----+ (0 rows) !ok @@ -2010,8 +2072,10 @@ select sal from "scott".emp where 123 NOT IN ( select cast(null as int) from "scott".dept); - SAL ------ ++-----+ +| SAL | ++-----+ ++-----+ (0 rows) !ok @@ -2031,8 +2095,10 @@ select sal from "scott".emp where cast(null as int) NOT IN ( select 1 from "scott".dept); - SAL ------ ++-----+ +| SAL | ++-----+ ++-----+ (0 rows) !ok @@ -2052,8 +2118,10 @@ select sal from "scott".emp where cast(null as int) NOT IN ( select deptno from "scott".dept); - SAL ------ ++-----+ +| SAL | ++-----+ ++-----+ (0 rows) !ok @@ -2073,8 +2141,10 @@ select sal from "scott".emp where cast(null as int) NOT IN ( select case when deptno > 0 then deptno else null end from "scott".dept); - SAL ------ ++-----+ +| SAL | ++-----+ ++-----+ (0 rows) !ok @@ -2094,8 +2164,10 @@ select sal from "scott".emp where 10 NOT IN ( select deptno from "scott".dept); - SAL ------ ++-----+ +| SAL | ++-----+ ++-----+ (0 rows) !ok @@ -2115,8 +2187,10 @@ select sal from "scott".emp where 10 NOT IN ( select case when deptno > 0 then deptno else null end from "scott".dept); - SAL ------ ++-----+ +| SAL | ++-----+ ++-----+ (0 rows) !ok @@ -2136,22 +2210,24 @@ select sal from "scott".emp where cast(null as int) IN ( select deptno from "scott".dept) is unknown; - SAL ---------- - 1100.00 - 1250.00 - 1250.00 - 1300.00 - 1500.00 - 1600.00 - 2450.00 - 2850.00 - 2975.00 - 3000.00 - 3000.00 - 5000.00 - 800.00 - 950.00 ++---------+ +| SAL | ++---------+ +| 1100.00 | +| 1250.00 | +| 1250.00 | +| 1300.00 | +| 1500.00 | +| 1600.00 | +| 2450.00 | +| 2850.00 | +| 2975.00 | +| 3000.00 | +| 3000.00 | +| 5000.00 | +| 800.00 | +| 950.00 | ++---------+ (14 rows) !ok @@ -2173,8 +2249,10 @@ select sal from "scott".emp e where cast(null as int) IN ( select cast(null as int) from "scott".dept d where e.deptno=d.deptno); - SAL ------ ++-----+ +| SAL | ++-----+ ++-----+ (0 rows) !ok @@ -2186,8 +2264,10 @@ select sal from "scott".emp e where 123 IN ( select cast(null as int) from "scott".dept d where e.deptno=d.deptno); - SAL ------ ++-----+ +| SAL | ++-----+ ++-----+ (0 rows) !ok @@ -2199,8 +2279,10 @@ select sal from "scott".emp e where cast(null as int) IN ( select 1 from "scott".dept d where e.deptno=d.deptno); - SAL ------ ++-----+ +| SAL | ++-----+ ++-----+ (0 rows) !ok @@ -2212,8 +2294,10 @@ select sal from "scott".emp e where cast(null as int) IN ( select deptno from "scott".dept d where e.deptno=d.deptno); - SAL ------ ++-----+ +| SAL | ++-----+ ++-----+ (0 rows) !ok @@ -2225,8 +2309,10 @@ select sal from "scott".emp e where mod(cast(rand() as int), 2) = 3 OR 123 IN ( select cast(null as int) from "scott".dept d where d.deptno = e.deptno); - SAL ------ ++-----+ +| SAL | ++-----+ ++-----+ (0 rows) !ok @@ -2247,8 +2333,10 @@ select sal from "scott".emp e where cast(null as int) IN ( select case when deptno > 0 then deptno else null end from "scott".dept d where e.deptno=d.deptno); - SAL ------ ++-----+ +| SAL | ++-----+ ++-----+ (0 rows) !ok @@ -2260,11 +2348,13 @@ select sal from "scott".emp e where 10 IN ( select deptno from "scott".dept d where e.deptno=d.deptno); - SAL ---------- - 1300.00 - 2450.00 - 5000.00 ++---------+ +| SAL | ++---------+ +| 1300.00 | +| 2450.00 | +| 5000.00 | ++---------+ (3 rows) !ok @@ -2281,11 +2371,13 @@ select sal from "scott".emp e where 10 IN ( select case when deptno > 0 then deptno else null end from "scott".dept d where e.deptno=d.deptno); - SAL ---------- - 1300.00 - 2450.00 - 5000.00 ++---------+ +| SAL | ++---------+ +| 1300.00 | +| 2450.00 | +| 5000.00 | ++---------+ (3 rows) !ok @@ -2302,8 +2394,10 @@ select sal from "scott".emp e where cast(null as int) NOT IN ( select cast(null as int) from "scott".dept d where e.deptno=d.deptno); - SAL ------ ++-----+ +| SAL | ++-----+ ++-----+ (0 rows) !ok @@ -2315,8 +2409,10 @@ select sal from "scott".emp e where 123 NOT IN ( select cast(null as int) from "scott".dept d where e.deptno=d.deptno); - SAL ------ ++-----+ +| SAL | ++-----+ ++-----+ (0 rows) !ok @@ -2337,8 +2433,10 @@ select sal from "scott".emp e where cast(null as int) NOT IN ( select 1 from "scott".dept d where e.deptno=d.deptno); - SAL ------ ++-----+ +| SAL | ++-----+ ++-----+ (0 rows) !ok @@ -2350,8 +2448,10 @@ select sal from "scott".emp e where cast(null as int) NOT IN ( select deptno from "scott".dept d where e.deptno=d.deptno); - SAL ------ ++-----+ +| SAL | ++-----+ ++-----+ (0 rows) !ok @@ -2363,8 +2463,10 @@ select sal from "scott".emp e where cast(null as int) NOT IN ( select case when deptno > 0 then deptno else null end from "scott".dept d where e.deptno=d.deptno); - SAL ------ ++-----+ +| SAL | ++-----+ ++-----+ (0 rows) !ok @@ -2376,19 +2478,21 @@ select sal from "scott".emp e where 10 NOT IN ( select deptno from "scott".dept d where e.deptno=d.deptno); - SAL ---------- - 1100.00 - 1250.00 - 1250.00 - 1500.00 - 1600.00 - 2850.00 - 2975.00 - 3000.00 - 3000.00 - 800.00 - 950.00 ++---------+ +| SAL | ++---------+ +| 1100.00 | +| 1250.00 | +| 1250.00 | +| 1500.00 | +| 1600.00 | +| 2850.00 | +| 2975.00 | +| 3000.00 | +| 3000.00 | +| 800.00 | +| 950.00 | ++---------+ (11 rows) !ok @@ -2410,19 +2514,21 @@ select sal from "scott".emp e where 10 NOT IN ( select case when deptno > 0 then deptno else null end from "scott".dept d where e.deptno=d.deptno); - SAL ---------- - 1100.00 - 1250.00 - 1250.00 - 1500.00 - 1600.00 - 2850.00 - 2975.00 - 3000.00 - 3000.00 - 800.00 - 950.00 ++---------+ +| SAL | ++---------+ +| 1100.00 | +| 1250.00 | +| 1250.00 | +| 1500.00 | +| 1600.00 | +| 2850.00 | +| 2975.00 | +| 3000.00 | +| 3000.00 | +| 800.00 | +| 950.00 | ++---------+ (11 rows) !ok @@ -2444,22 +2550,24 @@ select sal from "scott".emp e where cast(null as int) IN ( select deptno from "scott".dept d where e.deptno=d.deptno) is unknown; - SAL ---------- - 1100.00 - 1250.00 - 1250.00 - 1300.00 - 1500.00 - 1600.00 - 2450.00 - 2850.00 - 2975.00 - 3000.00 - 3000.00 - 5000.00 - 800.00 - 950.00 ++---------+ +| SAL | ++---------+ +| 1100.00 | +| 1250.00 | +| 1250.00 | +| 1300.00 | +| 1500.00 | +| 1600.00 | +| 2450.00 | +| 2850.00 | +| 2975.00 | +| 3000.00 | +| 3000.00 | +| 5000.00 | +| 800.00 | +| 950.00 | ++---------+ (14 rows) !ok @@ -2474,22 +2582,24 @@ select sal, select case when deptno > 10 then deptno else null end from "scott".dept) from "scott".emp; - SAL | EXPR$1 ----------+-------- - 1100.00 | true - 1250.00 | true - 1250.00 | true - 1300.00 | true - 1500.00 | true - 1600.00 | true - 2450.00 | true - 2850.00 | true - 2975.00 | true - 3000.00 | true - 3000.00 | true - 5000.00 | true - 800.00 | true - 950.00 | true ++---------+--------+ +| SAL | EXPR$1 | ++---------+--------+ +| 1100.00 | true | +| 1250.00 | true | +| 1250.00 | true | +| 1300.00 | true | +| 1500.00 | true | +| 1600.00 | true | +| 2450.00 | true | +| 2850.00 | true | +| 2975.00 | true | +| 3000.00 | true | +| 3000.00 | true | +| 5000.00 | true | +| 800.00 | true | +| 950.00 | true | ++---------+--------+ (14 rows) !ok @@ -2501,22 +2611,24 @@ select sal, from "scott".dept where deptno < 0) from "scott".emp; - SAL | EXPR$1 ----------+-------- - 1100.00 | false - 1250.00 | false - 1250.00 | false - 1300.00 | false - 1500.00 | false - 1600.00 | false - 2450.00 | false - 2850.00 | false - 2975.00 | false - 3000.00 | false - 3000.00 | false - 5000.00 | false - 800.00 | false - 950.00 | false ++---------+--------+ +| SAL | EXPR$1 | ++---------+--------+ +| 1100.00 | false | +| 1250.00 | false | +| 1250.00 | false | +| 1300.00 | false | +| 1500.00 | false | +| 1600.00 | false | +| 2450.00 | false | +| 2850.00 | false | +| 2975.00 | false | +| 3000.00 | false | +| 3000.00 | false | +| 5000.00 | false | +| 800.00 | false | +| 950.00 | false | ++---------+--------+ (14 rows) !ok @@ -2528,36 +2640,40 @@ select sal, from "scott".dept where deptno < 0) from "scott".emp; - SAL | EXPR$1 ----------+-------- - 1100.00 | false - 1250.00 | false - 1250.00 | false - 1300.00 | false - 1500.00 | false - 1600.00 | false - 2450.00 | false - 2850.00 | false - 2975.00 | false - 3000.00 | false - 3000.00 | false - 5000.00 | false - 800.00 | false - 950.00 | false ++---------+--------+ +| SAL | EXPR$1 | ++---------+--------+ +| 1100.00 | false | +| 1250.00 | false | +| 1250.00 | false | +| 1300.00 | false | +| 1500.00 | false | +| 1600.00 | false | +| 2450.00 | false | +| 2850.00 | false | +| 2975.00 | false | +| 3000.00 | false | +| 3000.00 | false | +| 5000.00 | false | +| 800.00 | false | +| 950.00 | false | ++---------+--------+ (14 rows) !ok # Test nested sub-query in PROJECT within FILTER select * from emp where deptno IN (select (select max(deptno) from "scott".emp t1) from "scott".emp t2); - EMPNO | ENAME | JOB | MGR | HIREDATE | SAL | COMM | DEPTNO --------+--------+----------+------+------------+---------+---------+-------- - 7499 | ALLEN | SALESMAN | 7698 | 1981-02-20 | 1600.00 | 300.00 | 30 - 7521 | WARD | SALESMAN | 7698 | 1981-02-22 | 1250.00 | 500.00 | 30 - 7654 | MARTIN | SALESMAN | 7698 | 1981-09-28 | 1250.00 | 1400.00 | 30 - 7698 | BLAKE | MANAGER | 7839 | 1981-01-05 | 2850.00 | | 30 - 7844 | TURNER | SALESMAN | 7698 | 1981-09-08 | 1500.00 | 0.00 | 30 - 7900 | JAMES | CLERK | 7698 | 1981-12-03 | 950.00 | | 30 ++-------+--------+----------+------+------------+---------+---------+--------+ +| EMPNO | ENAME | JOB | MGR | HIREDATE | SAL | COMM | DEPTNO | ++-------+--------+----------+------+------------+---------+---------+--------+ +| 7499 | ALLEN | SALESMAN | 7698 | 1981-02-20 | 1600.00 | 300.00 | 30 | +| 7521 | WARD | SALESMAN | 7698 | 1981-02-22 | 1250.00 | 500.00 | 30 | +| 7654 | MARTIN | SALESMAN | 7698 | 1981-09-28 | 1250.00 | 1400.00 | 30 | +| 7698 | BLAKE | MANAGER | 7839 | 1981-01-05 | 2850.00 | | 30 | +| 7844 | TURNER | SALESMAN | 7698 | 1981-09-08 | 1500.00 | 0.00 | 30 | +| 7900 | JAMES | CLERK | 7698 | 1981-12-03 | 950.00 | | 30 | ++-------+--------+----------+------+------------+---------+---------+--------+ (6 rows) !ok @@ -2572,22 +2688,24 @@ EnumerableHashJoin(condition=[=($7, $9)], joinType=[semi]) # Test nested sub-query in FILTER within PROJECT select (select max(deptno) from "scott".emp where deptno IN (select deptno from "scott".emp)) from emp ; - EXPR$0 --------- - 30 - 30 - 30 - 30 - 30 - 30 - 30 - 30 - 30 - 30 - 30 - 30 - 30 - 30 ++--------+ +| EXPR$0 | ++--------+ +| 30 | +| 30 | +| 30 | +| 30 | +| 30 | +| 30 | +| 30 | +| 30 | +| 30 | +| 30 | +| 30 | +| 30 | +| 30 | +| 30 | ++--------+ (14 rows) !ok @@ -2606,38 +2724,44 @@ EnumerableCalc(expr#0..1=[{inputs}], EXPR$0=[$t1]) # [CALCITE-7317] SubQueryRemoveRule should skip NULL-safety checks for IN subqueries when both keys and subquery columns are NOT NULL select * from emp as e1 where empno in (select empno from emp e2); - EMPNO | ENAME | JOB | MGR | HIREDATE | SAL | COMM | DEPTNO --------+--------+-----------+------+------------+---------+---------+-------- - 7369 | SMITH | CLERK | 7902 | 1980-12-17 | 800.00 | | 20 - 7499 | ALLEN | SALESMAN | 7698 | 1981-02-20 | 1600.00 | 300.00 | 30 - 7521 | WARD | SALESMAN | 7698 | 1981-02-22 | 1250.00 | 500.00 | 30 - 7566 | JONES | MANAGER | 7839 | 1981-02-04 | 2975.00 | | 20 - 7654 | MARTIN | SALESMAN | 7698 | 1981-09-28 | 1250.00 | 1400.00 | 30 - 7698 | BLAKE | MANAGER | 7839 | 1981-01-05 | 2850.00 | | 30 - 7782 | CLARK | MANAGER | 7839 | 1981-06-09 | 2450.00 | | 10 - 7788 | SCOTT | ANALYST | 7566 | 1987-04-19 | 3000.00 | | 20 - 7839 | KING | PRESIDENT | | 1981-11-17 | 5000.00 | | 10 - 7844 | TURNER | SALESMAN | 7698 | 1981-09-08 | 1500.00 | 0.00 | 30 - 7876 | ADAMS | CLERK | 7788 | 1987-05-23 | 1100.00 | | 20 - 7900 | JAMES | CLERK | 7698 | 1981-12-03 | 950.00 | | 30 - 7902 | FORD | ANALYST | 7566 | 1981-12-03 | 3000.00 | | 20 - 7934 | MILLER | CLERK | 7782 | 1982-01-23 | 1300.00 | | 10 ++-------+--------+-----------+------+------------+---------+---------+--------+ +| EMPNO | ENAME | JOB | MGR | HIREDATE | SAL | COMM | DEPTNO | ++-------+--------+-----------+------+------------+---------+---------+--------+ +| 7369 | SMITH | CLERK | 7902 | 1980-12-17 | 800.00 | | 20 | +| 7499 | ALLEN | SALESMAN | 7698 | 1981-02-20 | 1600.00 | 300.00 | 30 | +| 7521 | WARD | SALESMAN | 7698 | 1981-02-22 | 1250.00 | 500.00 | 30 | +| 7566 | JONES | MANAGER | 7839 | 1981-02-04 | 2975.00 | | 20 | +| 7654 | MARTIN | SALESMAN | 7698 | 1981-09-28 | 1250.00 | 1400.00 | 30 | +| 7698 | BLAKE | MANAGER | 7839 | 1981-01-05 | 2850.00 | | 30 | +| 7782 | CLARK | MANAGER | 7839 | 1981-06-09 | 2450.00 | | 10 | +| 7788 | SCOTT | ANALYST | 7566 | 1987-04-19 | 3000.00 | | 20 | +| 7839 | KING | PRESIDENT | | 1981-11-17 | 5000.00 | | 10 | +| 7844 | TURNER | SALESMAN | 7698 | 1981-09-08 | 1500.00 | 0.00 | 30 | +| 7876 | ADAMS | CLERK | 7788 | 1987-05-23 | 1100.00 | | 20 | +| 7900 | JAMES | CLERK | 7698 | 1981-12-03 | 950.00 | | 30 | +| 7902 | FORD | ANALYST | 7566 | 1981-12-03 | 3000.00 | | 20 | +| 7934 | MILLER | CLERK | 7782 | 1982-01-23 | 1300.00 | | 10 | ++-------+--------+-----------+------+------------+---------+---------+--------+ (14 rows) !ok # [CALCITE-7317] SubQueryRemoveRule should skip NULL-safety checks for IN subqueries when both keys and subquery columns are NOT NULL select * from emp as e1 where coalesce(deptno, 0) not in (select deptno from emp e2); - EMPNO | ENAME | JOB | MGR | HIREDATE | SAL | COMM | DEPTNO --------+-------+-----+-----+----------+-----+------+-------- ++-------+-------+-----+-----+----------+-----+------+--------+ +| EMPNO | ENAME | JOB | MGR | HIREDATE | SAL | COMM | DEPTNO | ++-------+-------+-----+-----+----------+-----+------+--------+ ++-------+-------+-----+-----+----------+-----+------+--------+ (0 rows) !ok # [CALCITE-7317] SubQueryRemoveRule should skip NULL-safety checks for IN subqueries when both keys and subquery columns are NOT NULL select * from emp as e1 where deptno not in (select coalesce(deptno, 0) from emp e2); - EMPNO | ENAME | JOB | MGR | HIREDATE | SAL | COMM | DEPTNO --------+-------+-----+-----+----------+-----+------+-------- ++-------+-------+-----+-----+----------+-----+------+--------+ +| EMPNO | ENAME | JOB | MGR | HIREDATE | SAL | COMM | DEPTNO | ++-------+-------+-----+-----+----------+-----+------+--------+ ++-------+-------+-----+-----+----------+-----+------+--------+ (0 rows) !ok @@ -2649,9 +2773,11 @@ where sal + 100 not in ( select deptno from dept where dname = e.ename); - C ----- - 14 ++----+ +| C | ++----+ +| 14 | ++----+ (1 row) !ok @@ -2690,22 +2816,24 @@ EnumerableCalc(expr#0..6=[{inputs}], EMPNO=[$t5]) EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], DEPTNO=[$t7]) EnumerableTableScan(table=[[scott, EMP]]) !plan - EMPNO -------- - 7369 - 7499 - 7521 - 7566 - 7654 - 7698 - 7782 - 7788 - 7839 - 7844 - 7876 - 7900 - 7902 - 7934 ++-------+ +| EMPNO | ++-------+ +| 7369 | +| 7499 | +| 7521 | +| 7566 | +| 7654 | +| 7698 | +| 7782 | +| 7788 | +| 7839 | +| 7844 | +| 7876 | +| 7900 | +| 7902 | +| 7934 | ++-------+ (14 rows) !ok @@ -2727,22 +2855,24 @@ EnumerableCalc(expr#0..6=[{inputs}], expr#7=[>($t1, $t2)], expr#8=[IS TRUE($t7)] EnumerableCalc(expr#0..2=[{inputs}], expr#3=[CAST($t0):SMALLINT NOT NULL], expr#4=[2], DEPTNO0=[$t3], EXPR$0=[$t4]) EnumerableTableScan(table=[[scott, DEPT]]) !plan - EMPNO | EXPR$1 --------+-------- - 7369 | false - 7499 | false - 7521 | false - 7566 | false - 7654 | false - 7698 | false - 7782 | false - 7788 | false - 7839 | false - 7844 | false - 7876 | false - 7900 | false - 7902 | false - 7934 | false ++-------+--------+ +| EMPNO | EXPR$1 | ++-------+--------+ +| 7369 | false | +| 7499 | false | +| 7521 | false | +| 7566 | false | +| 7654 | false | +| 7698 | false | +| 7782 | false | +| 7788 | false | +| 7839 | false | +| 7844 | false | +| 7876 | false | +| 7900 | false | +| 7902 | false | +| 7934 | false | ++-------+--------+ (14 rows) !ok @@ -2764,27 +2894,28 @@ EnumerableCalc(expr#0..2=[{inputs}], ENAME=[$t1]) EnumerableCalc(expr#0..7=[{inputs}], expr#8=[IS NOT NULL($t3)], expr#9=[CAST($t3):INTEGER NOT NULL], expr#10=[0], expr#11=[CASE($t8, $t9, $t10)], $f8=[$t11]) EnumerableTableScan(table=[[scott, EMP]]) !plan - ENAME --------- - ADAMS - ALLEN - BLAKE - CLARK - FORD - JAMES - JONES - KING - MARTIN - MILLER - SCOTT - SMITH - TURNER - WARD ++--------+ +| ENAME | ++--------+ +| ADAMS | +| ALLEN | +| BLAKE | +| CLARK | +| FORD | +| JAMES | +| JONES | +| KING | +| MARTIN | +| MILLER | +| SCOTT | +| SMITH | +| TURNER | +| WARD | ++--------+ (14 rows) !ok -!set outputformat mysql # Correlated SOME sub-query with not equality # Both sides Not NUll. select empno @@ -3063,7 +3194,6 @@ FROM tb; # [CALCITE-4486] UNIQUE predicate !use scott -!set outputformat mysql # singleton keys have unique value which excludes fully or partially null rows. select deptno @@ -3941,8 +4071,11 @@ EnumerableCalc(expr#0..7=[{inputs}], expr#8=[7782], expr#9=[CAST($t0):INTEGER NO EnumerableTableScan(table=[[scott, EMP]]) !plan -# [CALCITE-4846] IN-list that includes NULL converted to Values throws exception +# Reset to default value 20 +!set insubquerythreshold 20 +# [CALCITE-4846] IN-list that includes NULL converted to Values throws exception +!set insubquerythreshold 0 select * from "scott".emp where empno not in (null, 7782); +-------+-------+-----+-----+----------+-----+------+--------+ | EMPNO | ENAME | JOB | MGR | HIREDATE | SAL | COMM | DEPTNO | @@ -4069,6 +4202,9 @@ EnumerableCalc(expr#0..14=[{inputs}], expr#15=[0], expr#16=[=($t8, $t15)], expr# EnumerableValues(tuples=[[{ 7369, 20 }, { 7499, 30 }]]) !plan +# Reset to default value 20 +!set insubquerythreshold 20 + # [CALCITE-5117] Optimize the EXISTS sub-query by Metadata RowCount # Test case about sub-query is guaranteed to produce at least one row @@ -4740,7 +4876,7 @@ WHERE s1.total > (SELECT avg(total) FROM agg_sal s2 WHERE s1.deptno = s2.deptno) !ok # [CALCITE-6506] Type inference for IN list is incorrect - +!set insubquerythreshold 0 # Test LHS is not nullable and RHS is not nullable select empno, empno in (7369, 7499, 7521) from emp; +-------+--------+ @@ -4852,8 +4988,11 @@ EnumerableCalc(expr#0..6=[{inputs}], expr#7=[IS NULL($t1)], expr#8=[null:BOOLEAN EnumerableValues(tuples=[[{ 500.00 }, { 300.00 }, { 0.00 }, { null }]]) !plan -# Test LHS is (not nullable, not nullable) and RHS is (not nullable, not nullable) +# Reset to default value 20 +!set insubquerythreshold 20 +# Test LHS is (not nullable, not nullable) and RHS is (not nullable, not nullable) +!set insubquerythreshold 0 select empno, (empno, empno) in ((7369, 7369), (7499, 7499), (7521, 7521)) from emp; +-------+--------+ | EMPNO | EXPR$1 | @@ -4965,6 +5104,9 @@ EnumerableCalc(expr#0..8=[{inputs}], expr#9=[IS NULL($t1)], expr#10=[null:BOOLEA EnumerableValues(tuples=[[{ 500.00, 500.00 }, { 300.00, 300.00 }, { 0.00, 0.00 }, { null, null }]]) !plan +# Reset to default value 20 +!set insubquerythreshold 20 + # [CALCITE-5156] Support implicit integer types cast for IN Sub-query # Test case about the IN sub-query left operand type is INTEGER and right operand type is TINYINT @@ -7649,4 +7791,7 @@ EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0]) !ok +# Reset to default value 20 +!set trimfields true + # End sub-query.iq diff --git a/testkit/src/main/java/org/apache/calcite/test/QuidemTest.java b/testkit/src/main/java/org/apache/calcite/test/QuidemTest.java index 632d24cfeaa7..e9d18930d408 100644 --- a/testkit/src/main/java/org/apache/calcite/test/QuidemTest.java +++ b/testkit/src/main/java/org/apache/calcite/test/QuidemTest.java @@ -85,6 +85,9 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; +import static org.apache.calcite.runtime.SqlFunctions.resetThreadSequences; +import static org.apache.calcite.sql2rel.SqlToRelConverter.DEFAULT_IN_SUB_QUERY_THRESHOLD; + import static org.junit.jupiter.api.Assertions.fail; import static java.util.Objects.requireNonNull; @@ -511,9 +514,17 @@ private static String n2u(String s) { : s; } + private void resetThreadConfig() { + resetThreadSequences(); + Prepare.THREAD_INSUBQUERY_THRESHOLD.push(DEFAULT_IN_SUB_QUERY_THRESHOLD); + Prepare.THREAD_EXPAND.push(false); + Prepare.THREAD_TRIM.push(true); + } + @ParameterizedTest @MethodSource("getPath") public void test(String path) throws Exception { + resetThreadConfig(); final Method method = findMethod(path); if (method != null) { try { From ab285afc9c507adc967cc1cdee83d117e0d10bd7 Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Mon, 19 Jan 2026 21:08:06 +0800 Subject: [PATCH 115/562] [CALCITE-5787] The RelMdInputFieldsUsed is introduced to track the usage of input fields --- .../calcite/rel/metadata/BuiltInMetadata.java | 42 ++++++ .../metadata/DefaultRelMetadataProvider.java | 1 + .../rel/metadata/RelMdInputFieldsUsed.java | 124 ++++++++++++++++++ .../rel/metadata/RelMetadataQuery.java | 17 +++ .../apache/calcite/util/BuiltInMethod.java | 2 + .../apache/calcite/test/RelMetadataTest.java | 86 ++++++++++++ 6 files changed, 272 insertions(+) create mode 100644 core/src/main/java/org/apache/calcite/rel/metadata/RelMdInputFieldsUsed.java diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/BuiltInMetadata.java b/core/src/main/java/org/apache/calcite/rel/metadata/BuiltInMetadata.java index b0bc96b39c92..0f297a95a56c 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/BuiltInMetadata.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/BuiltInMetadata.java @@ -76,6 +76,48 @@ interface Handler extends MetadataHandler { } } + /** + * Metadata that identifies, per input, which fields of each + * input are referenced by a relational expression ({@link RelNode}). + * Here, "referenced" means the input field is used by the parent + * RelNode. Operators such as Filter, while not inherently consuming + * all input fields, must preserve them since parent RelNodes may depend on + * these fields. Thus, Filter is regarded as utilizing all fields. + * + *

    For a relational expression with N inputs, this returns an + * {@link ImmutableList} of length N. Each element is an + * {@link ImmutableBitSet} with bits set for zero-based field ordinals of + * that input which are referenced by the expression. + * + *

    Returns empty {@link ImmutableList} if information cannot be determined. + */ + public interface InputFieldsUsed extends Metadata { + MetadataDef DEF = + MetadataDef.of(InputFieldsUsed.class, InputFieldsUsed.Handler.class, + BuiltInMethod.INPUT_FIELDS_USED.method); + + /** + * Returns, for each input of this relational expression, a bit set of the + * referenced field ordinals. + * + * @return an {@link ImmutableList} of {@link ImmutableBitSet} of length N + * where N is the number of inputs, or empty {@link ImmutableList} + * if the information is not available + */ + ImmutableList getInputFieldsUsed(); + + /** Handler API. */ + @FunctionalInterface + interface Handler extends MetadataHandler { + ImmutableList getInputFieldsUsed(RelNode r, + RelMetadataQuery mq); + + @Override default MetadataDef getDef() { + return DEF; + } + } + } + /** Metadata about which combinations of columns are unique identifiers. */ public interface UniqueKeys extends Metadata { MetadataDef DEF = diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/DefaultRelMetadataProvider.java b/core/src/main/java/org/apache/calcite/rel/metadata/DefaultRelMetadataProvider.java index d47e5aac23d4..c4b62b83f482 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/DefaultRelMetadataProvider.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/DefaultRelMetadataProvider.java @@ -62,6 +62,7 @@ protected DefaultRelMetadataProvider() { RelMdSelectivity.SOURCE, RelMdExplainVisibility.SOURCE, RelMdPredicates.SOURCE, + RelMdInputFieldsUsed.SOURCE, RelMdAllPredicates.SOURCE, RelMdCollation.SOURCE, RelMdFunctionalDependency.SOURCE)); diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdInputFieldsUsed.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdInputFieldsUsed.java new file mode 100644 index 000000000000..771ed7e5fc80 --- /dev/null +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdInputFieldsUsed.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.rel.metadata; + +import org.apache.calcite.plan.RelOptUtil; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.Aggregate; +import org.apache.calcite.rel.core.Calc; +import org.apache.calcite.rel.core.Filter; +import org.apache.calcite.rel.core.Join; +import org.apache.calcite.rel.core.JoinRelType; +import org.apache.calcite.rel.core.Project; +import org.apache.calcite.rel.core.SetOp; +import org.apache.calcite.rel.core.TableScan; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexProgram; +import org.apache.calcite.util.ImmutableBitSet; + +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Set; + +/** + * Metadata provider to determine which input fields are used by a RelNode. + */ +public class RelMdInputFieldsUsed + implements MetadataHandler { + public static final RelMetadataProvider SOURCE = + ReflectiveRelMetadataProvider.reflectiveSource( + new RelMdInputFieldsUsed(), BuiltInMetadata.InputFieldsUsed.Handler.class); + + @Override public MetadataDef getDef() { + return BuiltInMetadata.InputFieldsUsed.DEF; + } + + public ImmutableList getInputFieldsUsed(RelNode rel, + RelMetadataQuery mq) { + ImmutableList.Builder builder = ImmutableList.builder(); + rel.getInputs().forEach(input -> { + builder.addAll(mq.getInputFieldsUsed(input)); + }); + return builder.build(); + } + + public ImmutableList getInputFieldsUsed(TableScan scan, + RelMetadataQuery mq) { + final BuiltInMetadata.InputFieldsUsed.Handler handler = + scan.getTable().unwrap(BuiltInMetadata.InputFieldsUsed.Handler.class); + if (handler != null) { + return handler.getInputFieldsUsed(scan, mq); + } + final int fieldCount = scan.getRowType().getFieldCount(); + return ImmutableList.of(ImmutableBitSet.range(fieldCount)); + } + + public ImmutableList getInputFieldsUsed(Project project, + RelMetadataQuery mq) { + final ImmutableBitSet bits = RelOptUtil.InputFinder.bits(project.getProjects(), null); + return ImmutableList.of(bits); + } + + public ImmutableList getInputFieldsUsed(Filter filter, + RelMetadataQuery mq) { + return mq.getInputFieldsUsed(filter.getInput()); + } + + public ImmutableList getInputFieldsUsed(Calc calc, + RelMetadataQuery mq) { + final RexProgram program = calc.getProgram(); + final List expandedProjects = program.expandList(program.getProjectList()); + final RexNode cond = program.getCondition() == null + ? null + : program.expandLocalRef(program.getCondition()); + final ImmutableBitSet bits = RelOptUtil.InputFinder.bits(expandedProjects, cond); + return ImmutableList.of(bits); + } + + public ImmutableList getInputFieldsUsed(Join join, + RelMetadataQuery mq) { + List leftInputFieldsUsed = mq.getInputFieldsUsed(join.getLeft()); + List rightInputFieldsUsed = mq.getInputFieldsUsed(join.getRight()); + assert leftInputFieldsUsed.size() == 1 && rightInputFieldsUsed.size() == 1; + + ImmutableBitSet rightUsedBits = rightInputFieldsUsed.get(0); + if (join.getJoinType() == JoinRelType.SEMI + || join.getJoinType() == JoinRelType.ANTI) { + rightUsedBits = ImmutableBitSet.of(); + } + + return ImmutableList.of(leftInputFieldsUsed.get(0), rightUsedBits); + } + + public ImmutableList getInputFieldsUsed(SetOp setOp, + RelMetadataQuery mq) { + final ImmutableList.Builder builder = ImmutableList.builder(); + for (RelNode input : setOp.getInputs()) { + ImmutableList inputFieldsBits = mq.getInputFieldsUsed(input); + assert inputFieldsBits.size() == 1; + builder.add(inputFieldsBits.get(0)); + } + return builder.build(); + } + + public ImmutableList getInputFieldsUsed(Aggregate agg, + RelMetadataQuery mq) { + Set fields = RelOptUtil.getAllFields(agg); + return ImmutableList.of(ImmutableBitSet.of(fields)); + } +} diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMetadataQuery.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMetadataQuery.java index 804484625457..f86f1de868fd 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMetadataQuery.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMetadataQuery.java @@ -110,6 +110,7 @@ public class RelMetadataQuery extends RelMetadataQueryBase { private BuiltInMetadata.UniqueKeys.Handler uniqueKeysHandler; private BuiltInMetadata.LowerBoundCost.Handler lowerBoundCostHandler; private BuiltInMetadata.FunctionalDependency.Handler functionalDependencyHandler; + private BuiltInMetadata.InputFieldsUsed.Handler inputFieldsUsedHandler; /** * Creates the instance with {@link JaninoRelMetadataProvider} instance @@ -158,6 +159,7 @@ public RelMetadataQuery(MetadataHandlerProvider provider) { this.lowerBoundCostHandler = provider.handler(BuiltInMetadata.LowerBoundCost.Handler.class); this.functionalDependencyHandler = provider.handler(BuiltInMetadata.FunctionalDependency.Handler.class); + this.inputFieldsUsedHandler = provider.handler(BuiltInMetadata.InputFieldsUsed.Handler.class); } /** Creates and initializes the instance that will serve as a prototype for @@ -193,6 +195,7 @@ private RelMetadataQuery(@SuppressWarnings("unused") boolean dummy) { this.lowerBoundCostHandler = initialHandler(BuiltInMetadata.LowerBoundCost.Handler.class); this.functionalDependencyHandler = initialHandler(BuiltInMetadata.FunctionalDependency.Handler.class); + this.inputFieldsUsedHandler = initialHandler(BuiltInMetadata.InputFieldsUsed.Handler.class); } private RelMetadataQuery( @@ -225,6 +228,7 @@ private RelMetadataQuery( this.uniqueKeysHandler = prototype.uniqueKeysHandler; this.lowerBoundCostHandler = prototype.lowerBoundCostHandler; this.functionalDependencyHandler = prototype.functionalDependencyHandler; + this.inputFieldsUsedHandler = prototype.inputFieldsUsedHandler; } //~ Methods ---------------------------------------------------------------- @@ -1058,4 +1062,17 @@ public ArrowSet getFDs(RelNode rel) { } } } + + /** + * Returns the input fields are used by a RelNode. + */ + public ImmutableList getInputFieldsUsed(RelNode rel) { + for (;;) { + try { + return inputFieldsUsedHandler.getInputFieldsUsed(rel, this); + } catch (MetadataHandlerProvider.NoHandler e) { + inputFieldsUsedHandler = revise(BuiltInMetadata.InputFieldsUsed.Handler.class); + } + } + } } diff --git a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java index 04a816245005..8b96db847aa8 100644 --- a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java +++ b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java @@ -65,6 +65,7 @@ import org.apache.calcite.rel.metadata.BuiltInMetadata.ExplainVisibility; import org.apache.calcite.rel.metadata.BuiltInMetadata.ExpressionLineage; import org.apache.calcite.rel.metadata.BuiltInMetadata.FunctionalDependency; +import org.apache.calcite.rel.metadata.BuiltInMetadata.InputFieldsUsed; import org.apache.calcite.rel.metadata.BuiltInMetadata.LowerBoundCost; import org.apache.calcite.rel.metadata.BuiltInMetadata.MaxRowCount; import org.apache.calcite.rel.metadata.BuiltInMetadata.Measure; @@ -911,6 +912,7 @@ public enum BuiltInMethod { STR_TO_MAP(SqlFunctions.class, "strToMap", String.class, String.class, String.class), SUBSTRING_INDEX(SqlFunctions.class, "substringIndex", String.class, String.class, int.class), SELECTIVITY(Selectivity.class, "getSelectivity", RexNode.class), + INPUT_FIELDS_USED(InputFieldsUsed.class, "getInputFieldsUsed"), UNIQUE_KEYS(UniqueKeys.class, "getUniqueKeys", boolean.class), AVERAGE_ROW_SIZE(Size.class, "averageRowSize"), AVERAGE_COLUMN_SIZES(Size.class, "averageColumnSizes"), diff --git a/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java b/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java index c5bdfcedd2cf..110c0d0f51fb 100644 --- a/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java @@ -43,6 +43,7 @@ import org.apache.calcite.rel.SingleRel; import org.apache.calcite.rel.core.Aggregate; import org.apache.calcite.rel.core.AggregateCall; +import org.apache.calcite.rel.core.Calc; import org.apache.calcite.rel.core.Correlate; import org.apache.calcite.rel.core.Exchange; import org.apache.calcite.rel.core.Filter; @@ -52,6 +53,7 @@ import org.apache.calcite.rel.core.Minus; import org.apache.calcite.rel.core.Project; import org.apache.calcite.rel.core.Sample; +import org.apache.calcite.rel.core.SetOp; import org.apache.calcite.rel.core.Sort; import org.apache.calcite.rel.core.TableModify; import org.apache.calcite.rel.core.TableScan; @@ -931,6 +933,90 @@ final RelMetadataFixture sql(String sql) { assertThat(fd2, sameInstance(fd1)); } + // ---------------------------------------------------------------------- + // Tests for InputFieldsUsed metadata in RelMdInputFieldsUsed + // ---------------------------------------------------------------------- + + @Test void testInputFieldsUsedSemiJoin() { + final RelBuilder relBuilder = RelBuilderTest.createBuilder(); + relBuilder.scan("EMP"); + relBuilder.scan("DEPT"); + // Build semi-join on DEPTNO + relBuilder.semiJoin( + relBuilder.equals(relBuilder.field(2, 0, "DEPTNO"), + relBuilder.field(2, 1, "DEPTNO"))); + final Join join = (Join) relBuilder.build(); + final RelMetadataQuery mq = join.getCluster().getMetadataQuery(); + final List inputFields = mq.getInputFieldsUsed(join); + + // For SEMI join expect left input fields to be all columns of left input + // and right input fields to be empty (semi-join does not require right output). + final int leftCount = join.getLeft().getRowType().getFieldCount(); + assertThat(inputFields, hasSize(2)); + assertThat(inputFields.get(0), equalTo(ImmutableBitSet.range(leftCount))); + assertThat(inputFields.get(1).isEmpty(), is(true)); + } + + @Test void testInputFieldsUsedUnionSetOp() { + final RelBuilder builder = RelBuilderTest.createBuilder(); + builder.scan("DEPT").project(builder.field(1)); // name + builder.scan("EMP").project(builder.field(2)); // job + builder.union(true); + final SetOp setOp = (SetOp) builder.build(); + final RelMetadataQuery mq = setOp.getCluster().getMetadataQuery(); + final List inputFields = mq.getInputFieldsUsed(setOp); + assertThat( + inputFields, equalTo( + ImmutableList.of(ImmutableBitSet.of(1), ImmutableBitSet.of(2)))); + } + + @Test void testInputFieldsUsedProject() { + final RelBuilder builder = RelBuilderTest.createBuilder(); + final RelNode project = builder + .scan("EMP") + .project(builder.field(0), builder.field(2)) + .build(); + final RelMetadataQuery mq = project.getCluster().getMetadataQuery(); + final java.util.List inputFields = mq.getInputFieldsUsed(project); + + assertThat(inputFields, hasSize(1)); + assertThat(inputFields.get(0), equalTo(ImmutableBitSet.of(0, 2))); + } + + @Test void testInputFieldsUsedFilter() { + final RelBuilder builder = RelBuilderTest.createBuilder(); + final RelNode filter = builder + .scan("EMP") + .filter(builder.equals(builder.field(2), builder.literal(10))) + .build(); + final RelMetadataQuery mq = filter.getCluster().getMetadataQuery(); + final List inputFields = mq.getInputFieldsUsed(filter); + + final int fieldCount = filter.getInput(0).getRowType().getFieldCount(); + assertThat(inputFields, hasSize(1)); + assertThat(inputFields.get(0), equalTo(ImmutableBitSet.range(fieldCount))); + } + + @Test void testInputFieldsUsedCalc() { + final RelBuilder builder = RelBuilderTest.createBuilder(); + final RelNode proj = builder + .scan("EMP") + .project(builder.field(0), builder.field(2)) + .build(); + final HepProgram program = new HepProgramBuilder() + .addRuleInstance(CoreRules.PROJECT_TO_CALC) + .build(); + final HepPlanner planner = new HepPlanner(program); + planner.setRoot(proj); + final RelNode calc = planner.findBestExp(); + assertThat(calc, instanceOf(Calc.class)); + + final RelMetadataQuery mq = calc.getCluster().getMetadataQuery(); + final List inputFields = mq.getInputFieldsUsed(calc); + assertThat(inputFields, hasSize(1)); + assertThat(inputFields.get(0), equalTo(ImmutableBitSet.of(0, 2))); + } + // ---------------------------------------------------------------------- // Tests for getColumnOrigins // ---------------------------------------------------------------------- From e782980cdcc6ba8205bd9355098caf858c7914a6 Mon Sep 17 00:00:00 2001 From: zzwqqq Date: Thu, 15 Jan 2026 21:22:54 +0800 Subject: [PATCH 116/562] [CALCITE-7378] Potential incorrect column attribution in RelToSqlConverter due to implicit table alias handling --- .../calcite/rel/rel2sql/SqlImplementor.java | 28 +++++- .../rel/rel2sql/RelToSqlConverterTest.java | 21 ++++- core/src/test/resources/sql/sub-query.iq | 86 +++++++++++++++++++ 3 files changed, 133 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java b/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java index ece87d0b5c2b..6d60d52aa4e0 100644 --- a/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java +++ b/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java @@ -139,6 +139,7 @@ import java.util.function.Function; import java.util.function.IntFunction; import java.util.function.Predicate; +import java.util.stream.Collectors; import static com.google.common.collect.ImmutableList.toImmutableList; @@ -2242,12 +2243,37 @@ public SqlSelect asSelect() { if (node instanceof SqlSelect) { return (SqlSelect) node; } - if (!dialect.hasImplicitTableAlias()) { + if (!dialect.hasImplicitTableAlias() || hasConflictTableAlias(node)) { return wrapSelect(asFrom()); } return wrapSelect(node); } + private boolean hasConflictTableAlias(SqlNode node) { + if (!(node instanceof SqlIdentifier)) { + return false; + } + if (correlTableMap.isEmpty()) { + return false; + } + if (neededAlias == null) { + return false; + } + SqlIdentifier identifier = (SqlIdentifier) node; + List aliasContexts = + correlTableMap.values().stream() + .filter(context -> context instanceof AliasContext) + .map(context -> (AliasContext) context) + .collect(Collectors.toList()); + + for (AliasContext aliasContext : aliasContexts) { + if (aliasContext.aliases.containsKey(Util.last(identifier.names))) { + return true; + } + } + return false; + } + public void stripTrivialAliases(SqlNode node) { switch (node.getKind()) { case SELECT: diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java index a44d05ce648f..aba4ee7eaaf3 100644 --- a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java +++ b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java @@ -6426,6 +6426,25 @@ private void checkLiteral2(String expression, String expected) { sql(query).withConfig(c -> c.withExpand(false)).ok(expected); } + /** Test cases of + * [CALCITE-7378] + * Potential incorrect column attribution in RelToSqlConverter due to implicit + * table alias handling. */ + @Test void testColumnAttributionWithImplicitAlias() { + String query = "select \"product_name\" from \"product\" t1 " + + "where \"product_id\" not in (select \"product_id\" " + + "from \"product\" t2 " + + "where t2.\"product_id\" = t1.\"product_id\" " + + "and t1.\"product_id\" = 2 and t2.\"product_id\" = 1)"; + String expected = "SELECT \"product_name\"\n" + + "FROM \"foodmart\".\"product\"\n" + + "WHERE \"product_id\" NOT IN (SELECT \"product_id\"\n" + + "FROM \"foodmart\".\"product\" AS \"product0\"\n" + + "WHERE \"product_id\" = \"product\".\"product_id\" " + + "AND \"product\".\"product_id\" = 2 AND \"product_id\" = 1)"; + sql(query).withConfig(c -> c.withExpand(false)).ok(expected); + } + /** Test case for * [CALCITE-5711] * Implement the SINGLE_VALUE aggregation in PostgreSQL Dialect @@ -11126,7 +11145,7 @@ private void checkLiteral2(String expression, String expected) { + "FROM \"SCOTT\".\"EMP\"\n" + "INNER JOIN \"SCOTT\".\"DEPT\" ON \"EMP\".\"DEPTNO\" = \"DEPT\".\"DEPTNO\"\n" + "WHERE \"DEPT\".\"DEPTNO\" = (SELECT MIN(\"DEPTNO\")\n" - + "FROM \"SCOTT\".\"DEPT\"\n" + + "FROM \"SCOTT\".\"DEPT\" AS \"DEPT0\"\n" + "WHERE \"DEPTNO\" = \"EMP\".\"DEPTNO\")"; HepProgramBuilder builder = new HepProgramBuilder(); diff --git a/core/src/test/resources/sql/sub-query.iq b/core/src/test/resources/sql/sub-query.iq index e981cc15d3a3..73b8c939b417 100644 --- a/core/src/test/resources/sql/sub-query.iq +++ b/core/src/test/resources/sql/sub-query.iq @@ -7791,6 +7791,92 @@ EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0]) !ok +# [CALCITE-7378] Potential incorrect column attribution in RelToSqlConverter due to implicit table alias handling +!use blank +CREATE TABLE a ( + deptno INTEGER NOT NULL, + ename VARCHAR(10) +); +(0 rows modified) + +!update + +CREATE TABLE b ( + deptno INTEGER NOT NULL, + ename VARCHAR(10) +); +(0 rows modified) + +!update + +INSERT INTO a VALUES + (10, 'ALLEN'), + (20, 'WARD'), + (30, 'WARD'), + (40, 'SMITH'); +(4 rows modified) + +!update + +INSERT INTO b VALUES + (20, 'WARD'), + (30, 'WARD'), + (30, 'ALLEN'), + (10, 'KING'); +(4 rows modified) + +!update + +SELECT deptno +FROM b as b1 +WHERE deptno NOT IN ( + SELECT deptno + FROM a + WHERE deptno = b1.deptno AND ename = 'WARD' AND b1.ename = 'WARD' +); ++--------+ +| DEPTNO | ++--------+ +| 10 | +| 30 | ++--------+ +(2 rows) + +!ok + +SELECT deptno +FROM b as a +WHERE deptno NOT IN ( + SELECT deptno + FROM a + WHERE deptno = a.deptno AND ename = 'WARD' AND a.ename = 'WARD' +); ++--------+ +| DEPTNO | ++--------+ +| 10 | ++--------+ +(1 row) + +!ok + +SELECT deptno +FROM b as a +WHERE deptno NOT IN ( + SELECT deptno + FROM a as a2 + WHERE deptno = a.deptno AND ename = 'WARD' AND a.ename = 'WARD' +); ++--------+ +| DEPTNO | ++--------+ +| 10 | +| 30 | ++--------+ +(2 rows) + +!ok + # Reset to default value 20 !set trimfields true From b337b7a4385a5ba8a95b3b64e07fce2471500f4e Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Tue, 20 Jan 2026 06:48:23 +0800 Subject: [PATCH 117/562] [CALCITE-7356] The MARK JOIN generated by TopDownGeneralDecorrelator needs to be adapted to RelFieldTrimmer --- .../calcite/sql2rel/RelFieldTrimmer.java | 22 +++++++- core/src/test/resources/sql/new-decorr.iq | 56 +++++++++++++++++++ 2 files changed, 75 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java b/core/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java index 864ee9d6a0f1..48cb1f052139 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java @@ -836,6 +836,11 @@ public TrimResult trimFields( Join join, ImmutableBitSet fieldsUsed, Set extraFields) { + // If the column "mark" is included, it needs to be excluded first. + if (join.getJoinType() == JoinRelType.LEFT_MARK) { + int markIndex = join.getRowType().getFieldCount() - 1; + fieldsUsed = fieldsUsed.except(ImmutableBitSet.of(markIndex)); + } final int fieldCount = join.getSystemFieldList().size() + join.getLeft().getRowType().getFieldCount() + join.getRight().getRowType().getFieldCount(); @@ -957,17 +962,25 @@ public TrimResult trimFields( switch (join.getJoinType()) { case SEMI: case ANTI: - // For SemiJoins and AntiJoins only map fields from the left-side + case LEFT_MARK: + // For SemiJoins, AntiJoins and LeftMarkJoins only map fields from the left-side. + // For LeftMarkJoins, the mark column is also mapped. if (join.getJoinType() == JoinRelType.SEMI) { relBuilder.semiJoin(newConditionExpr); - } else { + } else if (join.getJoinType() == JoinRelType.ANTI) { relBuilder.antiJoin(newConditionExpr); + } else { + relBuilder.join(join.getJoinType(), newConditionExpr, join.getVariablesSet()); } Mapping inputMapping = inputMappings.get(0); + int targetCount = newSystemFieldCount + inputMapping.getTargetCount(); + if (join.getJoinType() == JoinRelType.LEFT_MARK) { + targetCount++; + } mapping = Mappings.create(MappingType.INVERSE_SURJECTION, join.getRowType().getFieldCount(), - newSystemFieldCount + inputMapping.getTargetCount()); + targetCount); for (int i = 0; i < newSystemFieldCount; ++i) { mapping.set(i, i); } @@ -976,6 +989,9 @@ public TrimResult trimFields( for (IntPair pair : inputMapping) { mapping.set(pair.source + offset, pair.target + newOffset); } + if (join.getJoinType() == JoinRelType.LEFT_MARK) { + mapping.set(join.getRowType().getFieldCount() - 1, targetCount - 1); + } break; case ASOF: case LEFT_ASOF: diff --git a/core/src/test/resources/sql/new-decorr.iq b/core/src/test/resources/sql/new-decorr.iq index 148d4721acf2..d329c160ed07 100644 --- a/core/src/test/resources/sql/new-decorr.iq +++ b/core/src/test/resources/sql/new-decorr.iq @@ -42,4 +42,60 @@ SELECT * FROM t0 WHERE t0a < !ok +# [CALCITE-7356] The MARK JOIN generated by TopDownGeneralDecorrelator needs to be adapted to RelFieldTrimmer +!use blank +CREATE TABLE emps ( + empid INTEGER NOT NULL, + deptno INTEGER NOT NULL, + name VARCHAR(10) NOT NULL, + salary DECIMAL(10, 2) NOT NULL, + commission INTEGER); +(0 rows modified) + +!update + +INSERT INTO emps (empid, deptno, name, salary, commission) VALUES +(100, 10, 'Bill', 10000.00, 1000), +(200, 20, 'Eric', 8000.00, 500), +(150, 10, 'Sebastian', 7000.00, NULL), +(110, 10, 'Theodore', 11500.00, 250), +(170, 30, 'Theodore', 11500.00, 250), +(140, 10, 'Sebastian', 7000.00, NULL); +(6 rows modified) + +!update + +SELECT empid, EXISTS(select * from ( + SELECT e2.deptno FROM emps e2 where e1.commission = e2.commission) as table3 + where table3.deptno <> e1.deptno) +from emps e1 order by empid; ++-------+--------+ +| EMPID | EXPR$1 | ++-------+--------+ +| 100 | false | +| 110 | true | +| 140 | false | +| 150 | false | +| 170 | true | +| 200 | false | ++-------+--------+ +(6 rows) + +!ok + +!if (use_new_decorr) { +EnumerableSort(sort0=[$0], dir0=[ASC]) + EnumerableCalc(expr#0..3=[{inputs}], EMPID=[$t0], EXPR$1=[$t3]) + EnumerableHashJoin(condition=[AND(IS NOT DISTINCT FROM($1, $3), IS NOT DISTINCT FROM($2, $4))], joinType=[left_mark]) + EnumerableCalc(expr#0..4=[{inputs}], proj#0..1=[{exprs}], COMMISSION=[$t4]) + EnumerableTableScan(table=[[BLANK, EMPS]]) + EnumerableCalc(expr#0..3=[{inputs}], proj#0..1=[{exprs}]) + EnumerableHashJoin(condition=[AND(=($1, $3), <>($2, $0))], joinType=[inner]) + EnumerableAggregate(group=[{1, 4}]) + EnumerableTableScan(table=[[BLANK, EMPS]]) + EnumerableCalc(expr#0..4=[{inputs}], DEPTNO=[$t1], COMMISSION=[$t4]) + EnumerableTableScan(table=[[BLANK, EMPS]]) +!plan +!} + # End new-decorr.iq From e61412b9abeacd24bf4b891d4c0d16674cbfa6c5 Mon Sep 17 00:00:00 2001 From: Silun Dong Date: Sun, 18 Jan 2026 17:07:53 +0800 Subject: [PATCH 118/562] [CALCITE-7382] The TopDownGeneralDecorrelator returns an error result when a subquery contains a LIMIT 1 --- .../sql2rel/TopDownGeneralDecorrelator.java | 85 +++++++++---------- core/src/test/resources/sql/new-decorr.iq | 26 ++++++ 2 files changed, 68 insertions(+), 43 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java b/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java index 10b9a411ff14..72401e342691 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java @@ -43,6 +43,7 @@ import org.apache.calcite.rex.RexShuttle; import org.apache.calcite.rex.RexUtil; import org.apache.calcite.rex.RexWindow; +import org.apache.calcite.rex.RexWindowBounds; import org.apache.calcite.sql.SqlAggFunction; import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.fun.SqlCountAggFunction; @@ -598,50 +599,48 @@ public RelNode unnestInternal(Sort sort, boolean allowEmptyOutputFromRewrite) { RelCollation shiftCollation = sort.getCollation().apply(targetMapping); builder.push(newInput); - if (!sort.collation.getFieldCollations().isEmpty() - && (sort.offset != null || sort.fetch != null)) { - // the Sort with ORDER BY and LIMIT or OFFSET have to be changed during rewriting because - // now the limit has to be enforced per value of the outer bindings instead of globally. - // It can be rewritten using ROW_NUMBER() window function and filtering on it, - // see section 4.4 in paper Improving Unnesting of Complex Queries - List partitionKeys = new ArrayList<>(); - for (CorDef corDef : corDefs) { - int partitionKeyIndex = requireNonNull(inputInfo.corDefOutputs.get(corDef)); - partitionKeys.add(builder.field(partitionKeyIndex)); - } - RexNode rowNumber = builder.aggregateCall(SqlStdOperatorTable.ROW_NUMBER) - .over() - .partitionBy(partitionKeys) - .orderBy(builder.fields(shiftCollation)) - .toRex(); - List projectsWithRowNumber = new ArrayList<>(builder.fields()); - projectsWithRowNumber.add(rowNumber); - builder.project(projectsWithRowNumber); - - List conditions = new ArrayList<>(); - if (sort.offset != null) { - RexNode greaterThenLowerBound = - builder.call( - SqlStdOperatorTable.GREATER_THAN, - builder.field(projectsWithRowNumber.size() - 1), - sort.offset); - conditions.add(greaterThenLowerBound); - } - if (sort.fetch != null) { - RexNode upperBound = sort.offset == null - ? sort.fetch - : builder.call(SqlStdOperatorTable.PLUS, sort.offset, sort.fetch); - RexNode lessThenOrEqualUpperBound = - builder.call( - SqlStdOperatorTable.LESS_THAN_OR_EQUAL, - builder.field(projectsWithRowNumber.size() - 1), - upperBound); - conditions.add(lessThenOrEqualUpperBound); - } - builder.filter(conditions); - } else { - builder.sortLimit(sort.offset, sort.fetch, builder.fields(shiftCollation)); + // the Sort have to be changed during rewriting because now the order/limit/offset has to be + // enforced per value of the outer bindings instead of globally. It can be rewritten using + // ROW_NUMBER() window function and filtering on it, see section 4.4 in paper + // Improving Unnesting of Complex Queries + List partitionKeys = new ArrayList<>(); + for (CorDef corDef : corDefs) { + int partitionKeyIndex = requireNonNull(inputInfo.corDefOutputs.get(corDef)); + partitionKeys.add(builder.field(partitionKeyIndex)); + } + RexNode rowNumber = builder.aggregateCall(SqlStdOperatorTable.ROW_NUMBER) + .over() + .partitionBy(partitionKeys) + .orderBy(builder.fields(shiftCollation)) + .rowsFrom(RexWindowBounds.UNBOUNDED_PRECEDING) + .rowsTo(RexWindowBounds.CURRENT_ROW) + .toRex(); + List projectsWithRowNumber = new ArrayList<>(builder.fields()); + projectsWithRowNumber.add(rowNumber); + builder.project(projectsWithRowNumber); + + List conditions = new ArrayList<>(); + if (sort.offset != null) { + RexNode greaterThenLowerBound = + builder.call( + SqlStdOperatorTable.GREATER_THAN, + builder.field(projectsWithRowNumber.size() - 1), + sort.offset); + conditions.add(greaterThenLowerBound); } + if (sort.fetch != null) { + RexNode upperBound = sort.offset == null + ? sort.fetch + : builder.call(SqlStdOperatorTable.PLUS, sort.offset, sort.fetch); + RexNode lessThenOrEqualUpperBound = + builder.call( + SqlStdOperatorTable.LESS_THAN_OR_EQUAL, + builder.field(projectsWithRowNumber.size() - 1), + upperBound); + conditions.add(lessThenOrEqualUpperBound); + } + builder.filter(conditions); + RelNode newSort = builder.build(); UnnestedQuery unnestedQuery = new UnnestedQuery(sort, newSort, inputInfo.corDefOutputs, inputInfo.oldToNewOutputs); diff --git a/core/src/test/resources/sql/new-decorr.iq b/core/src/test/resources/sql/new-decorr.iq index d329c160ed07..4e804687daf2 100644 --- a/core/src/test/resources/sql/new-decorr.iq +++ b/core/src/test/resources/sql/new-decorr.iq @@ -98,4 +98,30 @@ EnumerableSort(sort0=[$0], dir0=[ASC]) !plan !} +# [CALCITE-7382] The TopDownGeneralDecorrelator returns an error result when a subquery contains a LIMIT 1 +# This case comes from sub-query.iq [CALCITE-6652] +!use scott +SELECT dname, (SELECT emp.comm FROM "scott".emp WHERE dept.deptno = emp.deptno ORDER BY emp.comm LIMIT 1) FROM "scott".dept; ++------------+--------+ +| DNAME | EXPR$1 | ++------------+--------+ +| ACCOUNTING | | +| OPERATIONS | | +| RESEARCH | | +| SALES | 0.00 | ++------------+--------+ +(4 rows) + +!ok + +SELECT dname, (SELECT empno FROM emp WHERE dept.deptno = emp.deptno LIMIT 1) FROM dept WHERE deptno = 10; ++------------+--------+ +| DNAME | EXPR$1 | ++------------+--------+ +| ACCOUNTING | 7782 | ++------------+--------+ +(1 row) + +!ok + # End new-decorr.iq From c034019825048a7bcab5124320efd0c9cbf4cba5 Mon Sep 17 00:00:00 2001 From: Sergey Nuyanzin Date: Sat, 17 Jan 2026 19:33:56 +0100 Subject: [PATCH 119/562] Make SqlValidatorImpl#maybeCast protected to allow using it by child classes It will allow downstream projects to reuse it in extended validators --- .../java/org/apache/calcite/sql/validate/SqlValidatorImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java index 0fc14cf6dc2a..ae5c8ee792d0 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java @@ -905,7 +905,7 @@ private void throwIfExcludeEliminatesAllColumns(List excludeIdent } } - private SqlNode maybeCast(SqlNode node, RelDataType currentType, + protected SqlNode maybeCast(SqlNode node, RelDataType currentType, RelDataType desiredType) { return SqlTypeUtil.equalSansNullability(typeFactory, currentType, desiredType) ? node From 37f347c2fcb8f32995bf4931df4373e83cdac947 Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Wed, 21 Jan 2026 14:09:22 +0800 Subject: [PATCH 120/562] [CALCITE-5740] Support for AggToSemiJoinRule --- .../apache/calcite/rel/rules/CoreRules.java | 6 ++ .../calcite/rel/rules/SemiJoinRule.java | 76 +++++++++++++++++-- .../apache/calcite/test/RelOptRulesTest.java | 13 ++++ .../apache/calcite/test/RelOptRulesTest.xml | 23 ++++++ core/src/test/resources/sql/hep.iq | 30 ++++++++ 5 files changed, 140 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rel/rules/CoreRules.java b/core/src/main/java/org/apache/calcite/rel/rules/CoreRules.java index fbca06e760fe..21d5c971d641 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/CoreRules.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/CoreRules.java @@ -162,6 +162,12 @@ private CoreRules() {} public static final AggregateJoinTransposeRule AGGREGATE_JOIN_TRANSPOSE_EXTENDED = AggregateJoinTransposeRule.Config.EXTENDED.toRule(); + /** Rule that creates a {@link Join#isSemiJoin semi-join} from a + * {@link Aggregate} on top of a {@link Join} with an {@link Aggregate} as its + * right input. */ + public static final SemiJoinRule.AggregateToSemiJoinRule AGGREGATE_TO_SEMI_JOIN = + SemiJoinRule.AggregateToSemiJoinRule.AggregateToSemiJoinRuleConfig.DEFAULT.toRule(); + /** Rule that pushes an {@link Aggregate} * past a non-distinct {@link Union}. */ public static final AggregateUnionTransposeRule AGGREGATE_UNION_TRANSPOSE = diff --git a/core/src/main/java/org/apache/calcite/rel/rules/SemiJoinRule.java b/core/src/main/java/org/apache/calcite/rel/rules/SemiJoinRule.java index 4a10ec533a20..427ea979bd38 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/SemiJoinRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/SemiJoinRule.java @@ -67,13 +67,15 @@ protected SemiJoinRule(Config config) { super(config); } - protected void perform(RelOptRuleCall call, @Nullable Project project, + protected void perform(RelOptRuleCall call, @Nullable RelNode topRel, Join join, RelNode left, Aggregate aggregate) { final RelOptCluster cluster = join.getCluster(); final RexBuilder rexBuilder = cluster.getRexBuilder(); - if (project != null) { - final ImmutableBitSet bits = - RelOptUtil.InputFinder.bits(project.getProjects(), null); + if (topRel != null) { + final ImmutableBitSet bits = getUsedFields(topRel); + if (bits.isEmpty()) { + return; + } final ImmutableBitSet rightBits = ImmutableBitSet.range(left.getRowType().getFieldCount(), join.getRowType().getFieldCount()); @@ -123,13 +125,72 @@ protected void perform(RelOptRuleCall call, @Nullable Project project, default: throw new AssertionError(join.getJoinType()); } - if (project != null) { - relBuilder.project(project.getProjects(), project.getRowType().getFieldNames()); + if (topRel != null) { + if (topRel instanceof Project) { + Project topProject = (Project) topRel; + relBuilder.project(topProject.getProjects(), topProject.getRowType().getFieldNames()); + } else if (topRel instanceof Aggregate) { + Aggregate topAgg = (Aggregate) topRel; + relBuilder.aggregate( + relBuilder.groupKey(topAgg.getGroupSet(), topAgg.getGroupSets()), + topAgg.getAggCallList()); + } } final RelNode relNode = relBuilder.build(); call.transformTo(relNode); } + /** Returns a bit set of the input fields used by a relational expression. */ + private static ImmutableBitSet getUsedFields(RelNode rel) { + final RelMetadataQuery mq = rel.getCluster().getMetadataQuery(); + return ImmutableBitSet.union(mq.getInputFieldsUsed(rel)); + } + + /** SemiJoinRule that matches a Aggregate on top of a Join with an Aggregate + * as its right child. + * + * @see CoreRules#AGGREGATE_TO_SEMI_JOIN */ + public static class AggregateToSemiJoinRule extends SemiJoinRule { + /** Creates a AggregateToSemiJoinRule. */ + protected AggregateToSemiJoinRule(AggregateToSemiJoinRuleConfig config) { + super(config); + } + + @Override public void onMatch(RelOptRuleCall call) { + final Aggregate topAgg = call.rel(0); + final Join join = call.rel(1); + final RelNode left = call.rel(2); + final Aggregate rightAgg = call.rel(3); + perform(call, topAgg, join, left, rightAgg); + } + + /** Rule configuration. */ + @Value.Immutable + public interface AggregateToSemiJoinRuleConfig extends SemiJoinRule.Config { + AggregateToSemiJoinRuleConfig DEFAULT = ImmutableAggregateToSemiJoinRuleConfig.of() + .withDescription("SemiJoinRule:aggregate") + .withOperandFor(Aggregate.class, Join.class, Aggregate.class); + + @Override default AggregateToSemiJoinRule toRule() { + return new AggregateToSemiJoinRule(this); + } + + /** Defines an operand tree for the given classes. */ + default AggregateToSemiJoinRuleConfig withOperandFor( + Class topAggClass, + Class joinClass, + Class rightAggClass) { + return withOperandSupplier(b -> + b.operand(topAggClass).oneInput(b2 -> + b2.operand(joinClass) + .predicate(SemiJoinRule::isJoinTypeSupported).inputs( + b3 -> b3.operand(RelNode.class).anyInputs(), + b4 -> b4.operand(rightAggClass).anyInputs()))) + .as(AggregateToSemiJoinRuleConfig.class); + } + } + } + /** SemiJoinRule that matches a Project on top of a Join with an Aggregate * as its right child. * @@ -251,8 +312,7 @@ protected JoinOnUniqueToSemiJoinRule(JoinOnUniqueToSemiJoinRuleConfig config) { final Join join = call.rel(1); final RelNode left = call.rel(2); - final ImmutableBitSet bits = - RelOptUtil.InputFinder.bits(project.getProjects(), null); + final ImmutableBitSet bits = getUsedFields(project); final ImmutableBitSet rightBits = ImmutableBitSet.range(left.getRowType().getFieldCount(), join.getRowType().getFieldCount()); diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index f46030dd8b3b..ff0e5ac9d2f7 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -2098,6 +2098,19 @@ private void checkSemiOrAntiJoinProjectTranspose(JoinRelType type) { .check(); } + /** Test case for + * [CALCITE-5740] + * Support for AggToSemiJoinRule . */ + @Test void testAggregateToSemiJoinRule() { + final String sql = "select distinct emp.deptno from emp\n" + + "join (select distinct mgr from emp) d on emp.deptno = d.mgr"; + sql(sql) + .withDecorrelate(true) + .withPreRule(CoreRules.AGGREGATE_PROJECT_MERGE) + .withRule(CoreRules.AGGREGATE_TO_SEMI_JOIN) + .check(); + } + /** Test case for * [CALCITE-1495] * SemiJoinRule should not apply to RIGHT and FULL JOIN. */ diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index c7c39f82a8ef..d33273b466c2 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -1130,6 +1130,29 @@ LogicalProject(MGR=[$0], SUM_SAL=[$2]) LogicalAggregate(group=[{0, 1}], SUM_SAL=[SUM($2)]) LogicalProject(MGR=[$3], DEPTNO=[$7], SAL=[$5]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +]]> + + + + + + + + + + + diff --git a/core/src/test/resources/sql/hep.iq b/core/src/test/resources/sql/hep.iq index 8dd530234735..6e1146c90d48 100644 --- a/core/src/test/resources/sql/hep.iq +++ b/core/src/test/resources/sql/hep.iq @@ -238,4 +238,34 @@ EnumerableHashJoin(condition=[AND(=($0, $6), OR(AND(>($1, 11), <=($7, 32)), AND( !} !set hep-rules original +# [CALCITE-5740] Support for AggToSemiJoinRule +!set hep-rules " ++CoreRules.AGGREGATE_PROJECT_MERGE, ++CoreRules.AGGREGATE_TO_SEMI_JOIN" + +select dept.deptno, count(*) +from dept join ( + select distinct deptno from emp + where sal > 100) using (deptno) +group by dept.deptno; ++--------+--------+ +| DEPTNO | EXPR$1 | ++--------+--------+ +| 10 | 1 | +| 20 | 1 | +| 30 | 1 | ++--------+--------+ +(3 rows) + +!ok +EnumerableAggregate(group=[{0}], EXPR$1=[COUNT()]) + EnumerableHashJoin(condition=[=($0, $3)], joinType=[semi]) + EnumerableCalc(expr#0..2=[{inputs}], DEPTNO=[$t0]) + EnumerableTableScan(table=[[scott, DEPT]]) + EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t5):DECIMAL(12, 2)], expr#9=[100.00:DECIMAL(12, 2)], expr#10=[>($t8, $t9)], EMPNO=[$t0], SAL=[$t5], DEPTNO=[$t7], $condition=[$t10]) + EnumerableTableScan(table=[[scott, EMP]]) +!plan + +!set hep-rules original + # End hep.iq From df82fcbd194791bdb0537d5fa40eaf44905a09e3 Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Wed, 21 Jan 2026 21:54:11 +0800 Subject: [PATCH 121/562] [CALCITE-7389] PruneJoinSingleValue rule causes type mismatch in EXISTS --- .../rules/SingleValuesOptimizationRules.java | 67 ++++++++++--------- .../apache/calcite/test/RelOptRulesTest.java | 29 ++++++++ .../apache/calcite/test/RelOptRulesTest.xml | 35 ++++++++++ core/src/test/resources/sql/new-decorr.iq | 45 +++++++++++++ 4 files changed, 143 insertions(+), 33 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rel/rules/SingleValuesOptimizationRules.java b/core/src/main/java/org/apache/calcite/rel/rules/SingleValuesOptimizationRules.java index c10504b31eb3..3212e98686da 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/SingleValuesOptimizationRules.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/SingleValuesOptimizationRules.java @@ -34,6 +34,8 @@ import org.apache.calcite.tools.RelBuilder; import org.apache.calcite.util.ImmutableBitSet; +import com.google.common.collect.ImmutableList; + import org.checkerframework.checker.nullness.qual.Nullable; import org.immutables.value.Value; @@ -121,41 +123,38 @@ protected SingleValuesRelTransformer( if (!transformable.test(join)) { return null; } - int end = valuesAsLeftChild - ? join.getLeft().getRowType().getFieldCount() - : join.getRowType().getFieldCount(); - - int start = valuesAsLeftChild - ? 0 - : join.getLeft().getRowType().getFieldCount(); - ImmutableBitSet bitSet = ImmutableBitSet.range(start, end); - RexNode trueNode = relBuilder.getRexBuilder().makeLiteral(true); - final RexNode filterCondition = - new RexNodeReplacer(bitSet, - literals, - (valuesAsLeftChild ? 0 : -1) * join.getLeft().getRowType().getFieldCount()) + final int leftCount = join.getLeft().getRowType().getFieldCount(); + final int rightCount = join.getRight().getRowType().getFieldCount(); + final int start = valuesAsLeftChild ? 0 : leftCount; + final int end = start + (valuesAsLeftChild ? leftCount : rightCount); + final int offset = valuesAsLeftChild ? 0 : -leftCount; + + final ImmutableBitSet bitSet = ImmutableBitSet.range(start, end); + RexNode condition = + new RexNodeReplacer(bitSet, literals, offset) .go(join.getCondition()); - RexNode fixedCondition = - valuesAsLeftChild - ? RexUtil.shift(filterCondition, - -1 * join.getLeft().getRowType().getFieldCount()) - : filterCondition; - - List rexLiterals = litTransformer.apply(fixedCondition, literals); - relBuilder.push(relNode) - .filter(join.getJoinType().isOuterJoin() ? trueNode : fixedCondition); - - List rexNodes = relNode - .getRowType() - .getFieldList() - .stream() - .map(fld -> relBuilder.field(fld.getIndex())) - .collect(Collectors.toList()); - - List projects = new ArrayList<>(); - projects.addAll(valuesAsLeftChild ? rexLiterals : rexNodes); - projects.addAll(valuesAsLeftChild ? rexNodes : rexLiterals); + if (valuesAsLeftChild) { + condition = RexUtil.shift(condition, -leftCount); + } + + relBuilder.push(relNode); + if (!join.getJoinType().isOuterJoin() + && join.getJoinType() != JoinRelType.LEFT_MARK) { + relBuilder.filter(condition); + } + + final List otherNodes = relBuilder.fields(); + final List valuesNodes = litTransformer.apply(condition, literals); + + final List joinLeftNodes = valuesAsLeftChild ? valuesNodes : otherNodes; + final List joinRightNodes = valuesAsLeftChild ? otherNodes : valuesNodes; + + final List projects = new ArrayList<>(joinLeftNodes); + if (join.getJoinType().projectsRight() + || join.getJoinType() == JoinRelType.LEFT_MARK) { + projects.addAll(joinRightNodes); + } return relBuilder.project(projects).build(); } } @@ -216,6 +215,8 @@ protected PruneSingleValueRule(PruneSingleValueRule.Config config) { return (condition, rexLiterals) -> rexLiterals.stream().map(lit -> rexBuilder.makeCall(SqlStdOperatorTable.CASE, condition, lit, rexBuilder.makeNullLiteral(lit.getType()))).collect(Collectors.toList()); + case LEFT_MARK: + return (condition, rexLiterals) -> ImmutableList.of(condition); default: return (condition, rexLiterals) -> rexLiterals; } diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index ff0e5ac9d2f7..2c8c5eab4372 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -5341,6 +5341,35 @@ RelOptFixture checkDynamicFunctions(boolean treatDynamicCallsAsConstant) { .check(); } + /** Test case for + * [CALCITE-7389] + * PruneJoinSingleValue rule causes type mismatch in EXISTS. */ + @Test void testExistsDecorrelationProducesSingleValues() { + final String sql = "select empno, deptno in (select 10) from emp"; + sql(sql) + .withPreRule( + CoreRules.PROJECT_SUB_QUERY_TO_MARK_CORRELATE, + CoreRules.PROJECT_MERGE, + CoreRules.PROJECT_REMOVE) + .withTopDownGeneralDecorrelate(true) + .withRule(SingleValuesOptimizationRules.JOIN_RIGHT_INSTANCE) + .check(); + } + + /** Test case for + * [CALCITE-7389] + * PruneJoinSingleValue rule causes type mismatch in EXISTS. */ + @Test void testLeftMarkJoinWithSingleValues() { + relFn(builder -> builder + .scan("EMP") + .values(new String[]{"val"}, 1) + .join(JoinRelType.LEFT_MARK, + builder.equals(builder.field(2, 0, 0), builder.field(2, 1, 0))) + .build()) + .withRule(SingleValuesOptimizationRules.JOIN_RIGHT_INSTANCE) + .check(); + } + @Test void testInnerJoinWithTimeStampSingleRowOnRight() { final String sql = "select e.empno, e.ename, c.t" + " from emp e inner join (select 7934 as ono, current_timestamp as t) c on e.empno=c.ono"; diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index d33273b466c2..5fa2ddc3ebaf 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -4407,6 +4407,26 @@ LogicalSortExchange(distribution=[hash[1]], collation=[[1]]) LogicalExchange(distribution=[single]) LogicalFilter(condition=[=($0, 10)]) LogicalTableScan(table=[[scott, EMP]]) +]]> + + + + + + + + + + + @@ -9181,6 +9201,21 @@ LogicalProject(EMPNO=[$0], ENAME=[$1], T=[$10]) LogicalProject(EMPNO=[$0], ENAME=[$1], T=[$10]) LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7], SLACKER=[$8], $f9=[CASE(=($0, 7934), 7934, null:INTEGER)], $f10=[CASE(=($0, 7934), CURRENT_TIMESTAMP, null:TIMESTAMP(0))]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +]]> + + + + + + + + diff --git a/core/src/test/resources/sql/new-decorr.iq b/core/src/test/resources/sql/new-decorr.iq index 4e804687daf2..1c60b58a1d12 100644 --- a/core/src/test/resources/sql/new-decorr.iq +++ b/core/src/test/resources/sql/new-decorr.iq @@ -124,4 +124,49 @@ SELECT dname, (SELECT empno FROM emp WHERE dept.deptno = emp.deptno LIMIT 1) FRO !ok +# [CALCITE-7389] PruneJoinSingleValue rule causes type mismatch in EXISTS +!use scott +select count(*) as c from "scott".dept where exists (select 1); ++---+ +| C | ++---+ +| 4 | ++---+ +(1 row) + +!ok + +EnumerableAggregate(group=[{}], C=[COUNT()]) + EnumerableTableScan(table=[[scott, DEPT]]) +!plan + +select empno, deptno in (select 10) from emp; ++-------+--------+ +| EMPNO | EXPR$1 | ++-------+--------+ +| 7782 | true | +| 7839 | true | +| 7934 | true | +| 7369 | false | +| 7499 | false | +| 7521 | false | +| 7566 | false | +| 7654 | false | +| 7698 | false | +| 7788 | false | +| 7844 | false | +| 7876 | false | +| 7900 | false | +| 7902 | false | ++-------+--------+ +(14 rows) + +!ok + +!if (use_new_decorr) { +EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t7):INTEGER], expr#9=[10], expr#10=[=($t8, $t9)], EMPNO=[$t0], EXPR$1=[$t10]) + EnumerableTableScan(table=[[scott, EMP]]) +!plan +!} + # End new-decorr.iq From 49d6fb9cc158db8e090b13fc2a0cf2eecb897736 Mon Sep 17 00:00:00 2001 From: krooswu Date: Wed, 17 Dec 2025 22:55:10 +0800 Subject: [PATCH 122/562] [CALCITE-7279] ClickHouse dialect should wrap nested JOINs with explicit column aliases --- .../rel/rel2sql/RelToSqlConverter.java | 88 +++++++++++++- .../org/apache/calcite/sql/SqlDialect.java | 18 +++ .../sql/dialect/ClickHouseSqlDialect.java | 53 +++++++++ .../rel/rel2sql/RelToSqlConverterTest.java | 111 ++++++++++++++++++ 4 files changed, 269 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java b/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java index 47834ea9d0e5..592df51a0b1b 100644 --- a/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java +++ b/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java @@ -223,6 +223,74 @@ private static class AliasReplacementShuttle extends SqlShuttle { } } + /** + * Wraps a nested join into a subquery with explicit column aliases. + * + *

    This is specifically required for dialects like ClickHouse where the + * identifier resolver cannot resolve columns with internal table qualifiers + * (e.g., 'd1.loc') when they are wrapped in a subquery alias. By forcing + * an explicit 'AS' projection for every column, we ensure the identifiers + * are flattened and visible to the outer query block. + * + * @param input The Result of the nested join branch + * @param inputRel The RelNode representing the join branch to extract row types + * @param outerAlias The alias to be assigned to the wrapped subquery + * @return A new Result containing the wrapped SQL with explicit aliases + */ + protected Result wrapNestedJoin(Result input, RelNode inputRel, String outerAlias) { + final SqlParserPos pos = SqlParserPos.ZERO; + + // Obtain a SqlSelect representation of the input. + // We manually rewrite the SelectList to enforce explicit column aliases, + // preventing ClickHouse scoping issues while avoiding redundant sub-query nesting. + final SqlSelect innerSelect = input.asSelect(); + final SqlNodeList originalSelectList = innerSelect.getSelectList(); + final List fieldNames = inputRel.getRowType().getFieldNames(); + + final List newSelectList = new ArrayList<>(); + + // Iterate through the fields to build explicit projections. + // Example: transforms 'd1.deptno' into 'd1.deptno AS deptno'. + for (int i = 0; i < fieldNames.size(); i++) { + SqlNode expr = originalSelectList.get(i); + String targetName = fieldNames.get(i); + + // If the expression is already aliased, strip the AS to get the raw expression. + if (expr.getKind() == SqlKind.AS) { + expr = ((SqlCall) expr).operand(0); + } + + // Force an explicit alias to mask internal table qualifiers. + // This ensures the outer JOIN can resolve the column name + // without being confused by nested scope identifiers. + newSelectList.add( + SqlStdOperatorTable.AS.createCall( + pos, + expr, + new SqlIdentifier(targetName, pos))); + } + + // Update the select list of the inner query with flattened aliases. + innerSelect.setSelectList(new SqlNodeList(newSelectList, pos)); + + // Wrap the modified Select node with the outer alias (e.g., AS j). + SqlNode wrappedNode = + SqlStdOperatorTable.AS.createCall(pos, + innerSelect, + new SqlIdentifier(outerAlias, pos)); + + // Return a new Result with empty clause lists. This "finalizes" the + // current sub-query and ensures the Implementor won't add + // redundant SELECT wrappers in subsequent steps. + return new Result( + wrappedNode, + Collections.emptyList(), + outerAlias, + inputRel.getRowType(), + ImmutableMap.of(outerAlias, inputRel.getRowType())); + } + + /** Visits a Join; called by {@link #dispatch} via reflection. */ public Result visit(Join e) { switch (e.getJoinType()) { @@ -233,7 +301,25 @@ public Result visit(Join e) { break; } final Result leftResult = visitInput(e, 0).resetAlias(); - final Result rightResult = visitInput(e, 1).resetAlias(); + Result rightResult = visitInput(e, 1).resetAlias(); + + if (dialect.shouldWrapNestedJoin(e)) { + + Set usedNames = new HashSet<>(e.getRowType().getFieldNames()); + // Add both left and right aliases + if (leftResult.neededAlias != null) { + usedNames.add(leftResult.neededAlias); + } + if (rightResult.neededAlias != null) { + usedNames.add(rightResult.neededAlias); + } + + String safeAlias = + SqlValidatorUtil.uniquify("t", usedNames, SqlValidatorUtil.EXPR_SUGGESTER); + + rightResult = wrapNestedJoin(rightResult, e.getRight(), safeAlias); + } + final Context leftContext = leftResult.qualifiedContext(); final Context rightContext = rightResult.qualifiedContext(); parseCorrelTable(e, leftContext.implementor().joinContext(leftContext, rightContext)); diff --git a/core/src/main/java/org/apache/calcite/sql/SqlDialect.java b/core/src/main/java/org/apache/calcite/sql/SqlDialect.java index a148e6ee6c64..869976f0d42a 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlDialect.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlDialect.java @@ -1825,4 +1825,22 @@ private ContextImpl(DatabaseProduct databaseProduct, conformance, nullCollation, dataTypeSystem, jethroInfo); } } + + /** + * Returns whether this dialect requires wrapping a nested JOIN in a subquery with mandatory + * aliases for both the table and its projected columns. + * + *

    Example for ClickHouse (returns true): + * 

    {@code
+   * // Before: ... LEFT JOIN (dept d1 INNER JOIN dept d2 ON ...) AS j ON ...
+   * // After:  ... LEFT JOIN (
+   * //           SELECT d1.deptno AS deptno, d2.loc AS loc ...
+   * //           FROM dept d1 INNER JOIN dept d2 ON ...
+   * //         ) AS t0 ON ...
+   * }
    + */ + public boolean shouldWrapNestedJoin(RelNode relNode) { + return false; + } + } diff --git a/core/src/main/java/org/apache/calcite/sql/dialect/ClickHouseSqlDialect.java b/core/src/main/java/org/apache/calcite/sql/dialect/ClickHouseSqlDialect.java index f33e49bb51d8..cd9e736f242b 100644 --- a/core/src/main/java/org/apache/calcite/sql/dialect/ClickHouseSqlDialect.java +++ b/core/src/main/java/org/apache/calcite/sql/dialect/ClickHouseSqlDialect.java @@ -18,6 +18,12 @@ import org.apache.calcite.avatica.util.TimeUnitRange; import org.apache.calcite.config.NullCollation; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.Correlate; +import org.apache.calcite.rel.core.Filter; +import org.apache.calcite.rel.core.Join; +import org.apache.calcite.rel.core.Project; +import org.apache.calcite.rel.core.Sort; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeSystem; import org.apache.calcite.rel.type.RelDataTypeSystemImpl; @@ -400,4 +406,51 @@ private static void unparseFloor(SqlWriter writer, SqlCall call) { call.operand(0).unparse(writer, 0, 0); writer.endList(frame); } + + @Override public boolean shouldWrapNestedJoin(RelNode rel) { + if (!(rel instanceof Join)) { + return false; + } + Join join = (Join) rel; + + // ClickHouse requires wrapping the right-side input if it's a JOIN + // to ensure that internal table qualifiers are flattened into explicit aliases. + // This solves the Code 47 UNKNOWN_IDENTIFIER error. + RelNode right = join.getRight(); + + // If the right side is a Join or a Project containing a Join, it needs aliasing protection + return right instanceof Join || containsJoinRecursive(right); + } + + /** + * Checks whether the given RelNode contains a JOIN that is directly exposed + * to the outer scope, which could lead to "Unknown Identifier" errors in ClickHouse. + * + *

    ClickHouse (v25.x+) has strict scoping rules: when a JOIN appears on the + * right side of another JOIN, internal table qualifiers (e.g., 'd2.loc') are + * stripped and become invisible to the outer query unless they are explicitly + * aliased within a subquery. + * + *

    We only check for JOINs wrapped by transparent single-input operators + * (Project, Filter, Sort) because these operators are typically collapsed + * into the same SELECT block, exposing the problematic JOIN structure to + * the outer boundary. + */ + private static boolean containsJoinRecursive(RelNode rel) { + if (rel instanceof Join || rel instanceof Correlate) { + return true; + } + + // Look through transparent single-input operators. + // We exclude Aggregate here because it naturally triggers a subquery + // boundary in RelToSqlConverter, which already provides the necessary isolation. + if (rel instanceof Project + || rel instanceof Filter + || rel instanceof Sort) { + return rel.getInputs().size() == 1 && containsJoinRecursive(rel.getInput(0)); + } + + return false; + } + } diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java index aba4ee7eaaf3..0c8e009c6e7d 100644 --- a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java +++ b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java @@ -11597,4 +11597,115 @@ public Sql schema(CalciteAssert.SchemaSpec schemaSpec) { relFn, transforms); } } + + /** Test case for + * [CALCITE-7279] + * ClickHouse dialect should wrap nested JOINs with explicit aliasing. */ + @Test void testClickHouseNestedJoin() { + final String query = "SELECT e.empno, j.dname, j.loc\n" + + "FROM emp e\n" + + "LEFT JOIN (\n" + + " SELECT d1.deptno, d1.dname, d2.loc\n" + + " FROM dept d1\n" + + " INNER JOIN dept d2 ON d1.deptno = d2.deptno\n" + + ") AS j ON e.deptno = j.deptno"; + + final String expected = "SELECT `EMP`.`EMPNO`, `t0`.`DNAME`, `t0`.`LOC`\n" + + "FROM `SCOTT`.`EMP`\n" + + "LEFT JOIN (SELECT `DEPT`.`DEPTNO` AS `DEPTNO`, `DEPT`.`DNAME` AS `DNAME`," + + " `DEPT0`.`LOC` AS `LOC`\n" + + "FROM `SCOTT`.`DEPT`\n" + + "INNER JOIN `SCOTT`.`DEPT` AS `DEPT0` ON `DEPT`.`DEPTNO` = `DEPT0`.`DEPTNO`) " + + "AS `t0` ON `EMP`.`DEPTNO` = `t0`.`DEPTNO`"; + + sql(query) + .schema(CalciteAssert.SchemaSpec.JDBC_SCOTT) + .withClickHouse() + .ok(expected); + } + + /** Test that simple JOINs without nesting are not wrapped unnecessarily. */ + @Test void testClickHouseSimpleJoinNotWrapped() { + final String query = "SELECT e.empno, d.dname\n" + + "FROM emp e\n" + + "LEFT JOIN dept d ON e.deptno = d.deptno"; + + // Simple joins should remain flat as standard SQL + final String expected = "SELECT `EMP`.`EMPNO`, `DEPT`.`DNAME`\n" + + "FROM `SCOTT`.`EMP`\n" + + "LEFT JOIN `SCOTT`.`DEPT` ON `EMP`.`DEPTNO` = `DEPT`.`DEPTNO`"; + + sql(query) + .schema(CalciteAssert.SchemaSpec.JDBC_SCOTT) + .withClickHouse() + .ok(expected); + } + + /** Test nested JOIN with aggregation to ensure expressions like COUNT(*) are aliased. */ + @Test void testClickHouseNestedJoinWithAggregation() { + final String query = "SELECT e.empno, j.\"EXPR$1\"\n" + + "FROM emp e\n" + + "LEFT JOIN (\n" + + " SELECT d1.deptno, COUNT(*)\n" + + " FROM dept d1\n" + + " INNER JOIN dept d2 ON d1.deptno = d2.deptno\n" + + " GROUP BY d1.deptno\n" + + ") AS j ON e.deptno = j.deptno"; + + final String expected = "SELECT `EMP`.`EMPNO`, `t0`.`EXPR$1`\n" + + "FROM `SCOTT`.`EMP`\n" + + "LEFT JOIN (SELECT `DEPT`.`DEPTNO`, COUNT(*) AS `EXPR$1`\n" + + "FROM `SCOTT`.`DEPT`\n" + + "INNER JOIN `SCOTT`.`DEPT` AS `DEPT0` ON `DEPT`.`DEPTNO` = `DEPT0`.`DEPTNO`\n" + + "GROUP BY `DEPT`.`DEPTNO`) AS `t0` ON `EMP`.`DEPTNO` = `t0`.`DEPTNO`"; + + sql(query) + .schema(CalciteAssert.SchemaSpec.JDBC_SCOTT) + .withClickHouse() + .ok(expected); + } + + /** Test three-way JOIN to ensure the converter handles linear joins normally. */ + @Test void testClickHouseThreeWayJoin() { + final String query = "SELECT e.empno, d1.dname, d2.loc\n" + + "FROM emp e\n" + + "INNER JOIN dept d1 ON e.deptno = d1.deptno\n" + + "INNER JOIN dept d2 ON d1.deptno = d2.deptno"; + + // Standard multi-way joins shouldn't be forced into subqueries unless nested on the right + final String expected = "SELECT `EMP`.`EMPNO`, `DEPT`.`DNAME`, `DEPT0`.`LOC`\n" + + "FROM `SCOTT`.`EMP`\n" + + "INNER JOIN `SCOTT`.`DEPT` ON `EMP`.`DEPTNO` = `DEPT`.`DEPTNO`\n" + + "INNER JOIN `SCOTT`.`DEPT` AS `DEPT0` ON `DEPT`.`DEPTNO` = `DEPT0`.`DEPTNO`"; + + sql(query) + .schema(CalciteAssert.SchemaSpec.JDBC_SCOTT) + .withClickHouse() + .ok(expected); + } + + /** Regression test: Ensure MySQL dialect remains unaffected by ClickHouse-specific fix. */ + @Test void testMysqlNestedJoinNotWrapped() { + final String query = "SELECT e.empno, j.dname\n" + + "FROM emp e\n" + + "LEFT JOIN (\n" + + " SELECT d1.deptno, d1.dname\n" + + " FROM dept d1\n" + + " INNER JOIN dept d2 ON d1.deptno = d2.deptno\n" + + ") AS j ON e.deptno = j.deptno"; + + // MySQL supports nested join syntax; no additional wrapping select should be added by our fix. + final String expected = "SELECT `EMP`.`EMPNO`, `t`.`DNAME`\n" + + "FROM `SCOTT`.`EMP`\n" + + "LEFT JOIN (SELECT `DEPT`.`DEPTNO`, `DEPT`.`DNAME`\n" + + "FROM `SCOTT`.`DEPT`\n" + + "INNER JOIN `SCOTT`.`DEPT` AS `DEPT0` ON `DEPT`.`DEPTNO` = `DEPT0`.`DEPTNO`) AS `t` " + + "ON `EMP`.`DEPTNO` = `t`.`DEPTNO`"; + + sql(query) + .schema(CalciteAssert.SchemaSpec.JDBC_SCOTT) + .withMysql() + .ok(expected); + } + } From 5f8bbd360d271216b1fb6b13dbe54528e43bbdc8 Mon Sep 17 00:00:00 2001 From: ehds Date: Thu, 22 Jan 2026 10:43:50 +0800 Subject: [PATCH 123/562] [CALCITE-7391] FILTER_REDUCE_EXPRESSIONS crashes on expression where 123 in (SELECT NULL FROM emps) --- .../rel/rules/FilterProjectTransposeRule.java | 9 ++++++ .../apache/calcite/test/RelOptRulesTest.java | 11 ++++++++ .../apache/calcite/test/RelOptRulesTest.xml | 28 +++++++++++++++++++ 3 files changed, 48 insertions(+) diff --git a/core/src/main/java/org/apache/calcite/rel/rules/FilterProjectTransposeRule.java b/core/src/main/java/org/apache/calcite/rel/rules/FilterProjectTransposeRule.java index 3e3cf46a1117..3c365222cd66 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/FilterProjectTransposeRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/FilterProjectTransposeRule.java @@ -28,6 +28,7 @@ import org.apache.calcite.rel.core.Project; import org.apache.calcite.rel.core.RelFactories; import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexSimplify; import org.apache.calcite.rex.RexUtil; import org.apache.calcite.tools.RelBuilder; import org.apache.calcite.tools.RelBuilderFactory; @@ -35,7 +36,9 @@ import org.immutables.value.Value; import java.util.Collections; +import java.util.List; import java.util.function.Predicate; +import java.util.stream.Collectors; /** * Planner rule that pushes @@ -170,6 +173,12 @@ protected FilterProjectTransposeRule( } final RelBuilder relBuilder = call.builder(); + List conjuncts = RelOptUtil.conjunctions(newCondition); + List simplified = conjuncts.stream() + .map(e -> RexSimplify.simplifyComparisonWithNull(e, relBuilder.getRexBuilder())) + .collect(Collectors.toList()); + newCondition = RexUtil.composeConjunction(relBuilder.getRexBuilder(), simplified); + RelNode newFilterRel; if (config.isCopyFilter()) { final RelNode input = project.getInput(); diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index 2c8c5eab4372..94bb87651f1d 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -9593,6 +9593,17 @@ private void checkSemiJoinRuleOnAntiJoin(RelOptRule rule) { sql(sql).withRule(subQueryFilterRule).withLateDecorrelate(true).check(); } + /** Test case for [CALCITE-7391] + * FILTER_REDUCE_EXPRESSIONS crashes on expression where 123 in (SELECT NULL FROM emps). */ + @Test void testNullSelect2() { + final String sql = "SELECT * from emp where 123 in (select null from dept)"; + sql(sql) + .withRule(CoreRules.FILTER_SUB_QUERY_TO_CORRELATE, + CoreRules.FILTER_PROJECT_TRANSPOSE, + CoreRules.PROJECT_REDUCE_EXPRESSIONS) + .check(); + } + /** Test case for * [CALCITE-6652] * RelDecorrelator can't decorrelate query with limit 1. diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index 5fa2ddc3ebaf..67e86d45b6b0 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -11062,6 +11062,34 @@ LogicalProject(EXPR$0=[1]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) LogicalProject(cs=[true]) LogicalValues(tuples=[[]]) +]]> + + + + + + + + + + + From 86c44173e1b2fc47bb255a63476f01301efa87ac Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Wed, 21 Jan 2026 23:02:06 +0800 Subject: [PATCH 124/562] Opened misc and scalar iq files in CoreQuidemTest2 --- .../apache/calcite/test/CoreQuidemTest2.java | 2 - core/src/test/resources/sql/misc.iq | 80 +++++++++++++++++++ core/src/test/resources/sql/new-decorr.iq | 36 +++++++++ core/src/test/resources/sql/scalar.iq | 47 ++++++++++- 4 files changed, 162 insertions(+), 3 deletions(-) diff --git a/core/src/test/java/org/apache/calcite/test/CoreQuidemTest2.java b/core/src/test/java/org/apache/calcite/test/CoreQuidemTest2.java index 5ad60a3221b6..f0887b71891e 100644 --- a/core/src/test/java/org/apache/calcite/test/CoreQuidemTest2.java +++ b/core/src/test/java/org/apache/calcite/test/CoreQuidemTest2.java @@ -49,9 +49,7 @@ public static void main(String[] args) throws Exception { paths.remove("sql/unnest.iq"); paths.remove("sql/some.iq"); paths.remove("sql/sub-query.iq"); - paths.remove("sql/scalar.iq"); paths.remove("sql/measure-paper.iq"); - paths.remove("sql/misc.iq"); return paths; } diff --git a/core/src/test/resources/sql/misc.iq b/core/src/test/resources/sql/misc.iq index d4d8dd93d7d9..aa73ca9a08dd 100644 --- a/core/src/test/resources/sql/misc.iq +++ b/core/src/test/resources/sql/misc.iq @@ -531,6 +531,7 @@ where exists (select 1 from "hr"."emps"); (3 rows) !ok +!if (use_old_decorr) { EnumerableCalc(expr#0..1=[{inputs}], deptno=[$t0]) EnumerableNestedLoopJoin(condition=[true], joinType=[inner]) EnumerableCalc(expr#0..3=[{inputs}], deptno=[$t0]) @@ -539,6 +540,15 @@ EnumerableCalc(expr#0..1=[{inputs}], deptno=[$t0]) EnumerableCalc(expr#0..4=[{inputs}], expr#5=[true], i=[$t5]) EnumerableTableScan(table=[[hr, emps]]) !plan +!} +!if (use_new_decorr) { +EnumerableNestedLoopJoin(condition=[true], joinType=[semi]) + EnumerableCalc(expr#0..3=[{inputs}], deptno=[$t0]) + EnumerableTableScan(table=[[hr, depts]]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[0], DUMMY=[$t5]) + EnumerableTableScan(table=[[hr, emps]]) +!plan +!} # Un-correlated NOT EXISTS select "deptno" from "hr"."depts" @@ -550,6 +560,7 @@ where not exists (select 1 from "hr"."emps"); (0 rows) !ok +!if (use_old_decorr) { EnumerableCalc(expr#0..1=[{inputs}], expr#2=[IS NULL($t1)], deptno=[$t0], $condition=[$t2]) EnumerableNestedLoopJoin(condition=[true], joinType=[left]) EnumerableCalc(expr#0..3=[{inputs}], deptno=[$t0]) @@ -558,6 +569,15 @@ EnumerableCalc(expr#0..1=[{inputs}], expr#2=[IS NULL($t1)], deptno=[$t0], $condi EnumerableCalc(expr#0..4=[{inputs}], expr#5=[true], i=[$t5]) EnumerableTableScan(table=[[hr, emps]]) !plan +!} +!if (use_new_decorr) { +EnumerableNestedLoopJoin(condition=[true], joinType=[anti]) + EnumerableCalc(expr#0..3=[{inputs}], deptno=[$t0]) + EnumerableTableScan(table=[[hr, depts]]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[0], DUMMY=[$t5]) + EnumerableTableScan(table=[[hr, emps]]) +!plan +!} # Un-correlated EXISTS (table empty) select "deptno" from "hr"."depts" @@ -569,6 +589,7 @@ where exists (select 1 from "hr"."emps" where "empid" < 0); (0 rows) !ok +!if (use_old_decorr) { EnumerableCalc(expr#0..1=[{inputs}], deptno=[$t0]) EnumerableNestedLoopJoin(condition=[true], joinType=[inner]) EnumerableCalc(expr#0..3=[{inputs}], deptno=[$t0]) @@ -577,6 +598,15 @@ EnumerableCalc(expr#0..1=[{inputs}], deptno=[$t0]) EnumerableCalc(expr#0..4=[{inputs}], expr#5=[true], expr#6=[CAST($t0):INTEGER NOT NULL], expr#7=[0], expr#8=[<($t6, $t7)], i=[$t5], $condition=[$t8]) EnumerableTableScan(table=[[hr, emps]]) !plan +!} +!if (use_new_decorr) { +EnumerableNestedLoopJoin(condition=[true], joinType=[semi]) + EnumerableCalc(expr#0..3=[{inputs}], deptno=[$t0]) + EnumerableTableScan(table=[[hr, depts]]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[CAST($t0):INTEGER NOT NULL], expr#6=[0], expr#7=[<($t5, $t6)], empid=[$t0], $condition=[$t7]) + EnumerableTableScan(table=[[hr, emps]]) +!plan +!} # Un-correlated NOT EXISTS (table empty) select "deptno" from "hr"."depts" @@ -591,6 +621,7 @@ where not exists (select 1 from "hr"."emps" where "empid" < 0); (3 rows) !ok +!if (use_old_decorr) { EnumerableCalc(expr#0..1=[{inputs}], expr#2=[IS NULL($t1)], deptno=[$t0], $condition=[$t2]) EnumerableNestedLoopJoin(condition=[true], joinType=[left]) EnumerableCalc(expr#0..3=[{inputs}], deptno=[$t0]) @@ -599,6 +630,15 @@ EnumerableCalc(expr#0..1=[{inputs}], expr#2=[IS NULL($t1)], deptno=[$t0], $condi EnumerableCalc(expr#0..4=[{inputs}], expr#5=[true], expr#6=[CAST($t0):INTEGER NOT NULL], expr#7=[0], expr#8=[<($t6, $t7)], i=[$t5], $condition=[$t8]) EnumerableTableScan(table=[[hr, emps]]) !plan +!} +!if (use_new_decorr) { +EnumerableNestedLoopJoin(condition=[true], joinType=[anti]) + EnumerableCalc(expr#0..3=[{inputs}], deptno=[$t0]) + EnumerableTableScan(table=[[hr, depts]]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[CAST($t0):INTEGER NOT NULL], expr#6=[0], expr#7=[<($t5, $t6)], empid=[$t0], $condition=[$t7]) + EnumerableTableScan(table=[[hr, emps]]) +!plan +!} # EXISTS select * from "hr"."emps" @@ -614,10 +654,19 @@ where exists ( (3 rows) !ok +!if (use_old_decorr) { EnumerableHashJoin(condition=[=($1, $5)], joinType=[semi]) EnumerableTableScan(table=[[hr, emps]]) EnumerableTableScan(table=[[hr, depts]]) !plan +!} +!if (use_new_decorr) { +EnumerableHashJoin(condition=[IS NOT DISTINCT FROM($1, $5)], joinType=[semi]) + EnumerableTableScan(table=[[hr, emps]]) + EnumerableCalc(expr#0..3=[{inputs}], deptno=[$t0]) + EnumerableTableScan(table=[[hr, depts]]) +!plan +!} # NOT EXISTS # Right results, but it would be better if the plan used EnumerableCorrelateRel; see [CALCITE-374] @@ -632,6 +681,7 @@ where not exists ( (1 row) !ok +!if (use_old_decorr) { EnumerableCalc(expr#0..6=[{inputs}], expr#7=[IS NULL($t6)], proj#0..4=[{exprs}], $condition=[$t7]) EnumerableMergeJoin(condition=[=($1, $5)], joinType=[left]) EnumerableSort(sort0=[$1], dir0=[ASC]) @@ -641,6 +691,14 @@ EnumerableCalc(expr#0..6=[{inputs}], expr#7=[IS NULL($t6)], proj#0..4=[{exprs}], EnumerableAggregate(group=[{0}]) EnumerableTableScan(table=[[hr, depts]]) !plan +!} +!if (use_new_decorr) { +EnumerableHashJoin(condition=[IS NOT DISTINCT FROM($1, $5)], joinType=[anti]) + EnumerableTableScan(table=[[hr, emps]]) + EnumerableCalc(expr#0..3=[{inputs}], deptno=[$t0]) + EnumerableTableScan(table=[[hr, depts]]) +!plan +!} # NOT EXISTS .. OR NOT EXISTS # Right results, but it would be better if the plan used EnumerableCorrelateRel; see [CALCITE-374] @@ -659,6 +717,7 @@ or not exists ( (3 rows) !ok +!if (use_old_decorr) { EnumerableCalc(expr#0..8=[{inputs}], expr#9=[IS NULL($t5)], expr#10=[IS NULL($t8)], expr#11=[OR($t9, $t10)], proj#0..4=[{exprs}], $condition=[$t11]) EnumerableMergeJoin(condition=[=($6, $7)], joinType=[left]) EnumerableSort(sort0=[$6], dir0=[ASC]) @@ -676,6 +735,27 @@ EnumerableCalc(expr#0..8=[{inputs}], expr#9=[IS NULL($t5)], expr#10=[IS NULL($t8 EnumerableCalc(expr#0..3=[{inputs}], expr#4=[90], expr#5=[+($t0, $t4)], $f4=[$t5]) EnumerableTableScan(table=[[hr, depts]]) !plan +!} +!if (use_new_decorr) { +EnumerableCalc(expr#0..6=[{inputs}], expr#7=[NOT($t5)], expr#8=[NOT($t6)], expr#9=[OR($t7, $t8)], proj#0..4=[{exprs}], $condition=[$t9]) + EnumerableHashJoin(condition=[IS NOT DISTINCT FROM($0, $7)], joinType=[left_mark]) + EnumerableHashJoin(condition=[IS NOT DISTINCT FROM($1, $5)], joinType=[left_mark]) + EnumerableTableScan(table=[[hr, emps]]) + EnumerableCalc(expr#0..3=[{inputs}], deptno=[$t0]) + EnumerableTableScan(table=[[hr, depts]]) + EnumerableCalc(expr#0..3=[{inputs}], deptno=[$t2], empid=[$t0]) + EnumerableHashJoin(condition=[=($1, $3)], joinType=[inner]) + EnumerableCalc(expr#0=[{inputs}], expr#1=[CAST($t0):INTEGER NOT NULL], proj#0..1=[{exprs}]) + EnumerableAggregate(group=[{0}]) + EnumerableHashJoin(condition=[IS NOT DISTINCT FROM($1, $2)], joinType=[left_mark]) + EnumerableCalc(expr#0..4=[{inputs}], proj#0..1=[{exprs}]) + EnumerableTableScan(table=[[hr, emps]]) + EnumerableCalc(expr#0..3=[{inputs}], deptno=[$t0]) + EnumerableTableScan(table=[[hr, depts]]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[90], expr#5=[+($t0, $t4)], deptno=[$t0], $f1=[$t5]) + EnumerableTableScan(table=[[hr, depts]]) +!plan +!} # Left join to a relation with one row is recognized as a trivial semi-join # and eliminated. diff --git a/core/src/test/resources/sql/new-decorr.iq b/core/src/test/resources/sql/new-decorr.iq index 1c60b58a1d12..8c1bc0b23d35 100644 --- a/core/src/test/resources/sql/new-decorr.iq +++ b/core/src/test/resources/sql/new-decorr.iq @@ -169,4 +169,40 @@ EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t7):INTEGER], expr#9=[10], ex !plan !} +# # This case comes from scalar.iq [CALCITE-709] +# Aggregate functions do not support type promotion, so a cast is added to pass the test. +select deptno, (select sum(cast(empno as bigint)) from "scott".emp where deptno = dept.deptno limit 0) as x from "scott".dept; ++--------+---+ +| DEPTNO | X | ++--------+---+ +| 10 | | +| 20 | | +| 30 | | +| 40 | | ++--------+---+ +(4 rows) + +!ok + +!if (use_old_decorr) { +EnumerableCorrelate(correlation=[$cor0], joinType=[left], requiredColumns=[{0}]) + EnumerableCalc(expr#0..2=[{inputs}], DEPTNO=[$t0]) + EnumerableTableScan(table=[[scott, DEPT]]) + EnumerableValues(tuples=[[]]) +!plan +!} + +!if (use_new_decorr) { +EnumerableCalc(expr#0..3=[{inputs}], DEPTNO=[$t0], EXPR$0=[$t2]) + EnumerableHashJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left]) + EnumerableCalc(expr#0..2=[{inputs}], DEPTNO=[$t0]) + EnumerableTableScan(table=[[scott, DEPT]]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[<=($t2, $t3)], proj#0..2=[{exprs}], $condition=[$t4]) + EnumerableWindow(window#0=[window(partition {0} rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + EnumerableAggregate(group=[{1}], EXPR$0=[$SUM0($0)]) + EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t0):BIGINT NOT NULL], expr#9=[IS NOT NULL($t7)], EMPNO=[$t8], DEPTNO=[$t7], $condition=[$t9]) + EnumerableTableScan(table=[[scott, EMP]]) +!plan +!} + # End new-decorr.iq diff --git a/core/src/test/resources/sql/scalar.iq b/core/src/test/resources/sql/scalar.iq index 91cd88279ae7..6eeb2167c211 100644 --- a/core/src/test/resources/sql/scalar.iq +++ b/core/src/test/resources/sql/scalar.iq @@ -164,7 +164,7 @@ select deptno, (select sum(cast(empno as bigint)) from "scott".emp where deptno !ok # [CALCITE-709] Errors with LIMIT inside scalar sub-query -select deptno, (select sum(empno) from "scott".emp where deptno = dept.deptno limit 0) as x from "scott".dept; +select deptno, (select sum(cast(empno as bigint)) from "scott".emp where deptno = dept.deptno limit 0) as x from "scott".dept; +--------+---+ | DEPTNO | X | +--------+---+ @@ -340,6 +340,8 @@ WHERE sal > 10; (14 rows) !ok + +!if (use_old_decorr) { EnumerableCalc(expr#0..3=[{inputs}], expr#4=[IS NULL($t3)], expr#5=[0:BIGINT], expr#6=[CASE($t4, $t5, $t3)], EMPNO=[$t0], $f1=[$t6]) EnumerableHashJoin(condition=[IS NOT DISTINCT FROM($1, $2)], joinType=[left]) EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t5):DECIMAL(12, 2)], expr#9=[10.00:DECIMAL(12, 2)], expr#10=[>($t8, $t9)], EMPNO=[$t0], MGR=[$t3], $condition=[$t10]) @@ -357,6 +359,27 @@ EnumerableCalc(expr#0..3=[{inputs}], expr#4=[IS NULL($t3)], expr#5=[0:BIGINT], e EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t5):DECIMAL(12, 2)], expr#9=[10.00:DECIMAL(12, 2)], expr#10=[>($t8, $t9)], proj#0..7=[{exprs}], $condition=[$t10]) EnumerableTableScan(table=[[scott, EMP]]) !plan +!} + +!if (use_new_decorr) { +EnumerableCalc(expr#0..3=[{inputs}], EMPNO=[$t0], $f1=[$t3]) + EnumerableHashJoin(condition=[IS NOT DISTINCT FROM($1, $2)], joinType=[left]) + EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t5):DECIMAL(12, 2)], expr#9=[10.00:DECIMAL(12, 2)], expr#10=[>($t8, $t9)], EMPNO=[$t0], MGR=[$t3], $condition=[$t10]) + EnumerableTableScan(table=[[scott, EMP]]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[IS NOT NULL($t2)], expr#4=[0], expr#5=[CASE($t3, $t2, $t4)], MGR=[$t0], $f2=[$t5]) + EnumerableHashJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left]) + EnumerableAggregate(group=[{3}]) + EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t5):DECIMAL(12, 2)], expr#9=[10.00:DECIMAL(12, 2)], expr#10=[>($t8, $t9)], proj#0..7=[{exprs}], $condition=[$t10]) + EnumerableTableScan(table=[[scott, EMP]]) + EnumerableAggregate(group=[{2}], C=[COUNT()]) + EnumerableNestedLoopJoin(condition=[<($1, $2)], joinType=[inner]) + EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], MGR=[$t3]) + EnumerableTableScan(table=[[scott, EMP]]) + EnumerableAggregate(group=[{3}]) + EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t5):DECIMAL(12, 2)], expr#9=[10.00:DECIMAL(12, 2)], expr#10=[>($t8, $t9)], proj#0..7=[{exprs}], $condition=[$t10]) + EnumerableTableScan(table=[[scott, EMP]]) +!plan +!} !set trimfields false @@ -386,6 +409,8 @@ WHERE sal > 10; (14 rows) !ok + +!if (use_old_decorr) { EnumerableCalc(expr#0..9=[{inputs}], expr#10=[IS NULL($t9)], expr#11=[0:BIGINT], expr#12=[CASE($t10, $t11, $t9)], EMPNO=[$t0], $f1=[$t12]) EnumerableHashJoin(condition=[IS NOT DISTINCT FROM($3, $8)], joinType=[left]) EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t5):DECIMAL(12, 2)], expr#9=[10.00:DECIMAL(12, 2)], expr#10=[>($t8, $t9)], proj#0..7=[{exprs}], $condition=[$t10]) @@ -402,6 +427,26 @@ EnumerableCalc(expr#0..9=[{inputs}], expr#10=[IS NULL($t9)], expr#11=[0:BIGINT], EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t5):DECIMAL(12, 2)], expr#9=[10.00:DECIMAL(12, 2)], expr#10=[>($t8, $t9)], proj#0..7=[{exprs}], $condition=[$t10]) EnumerableTableScan(table=[[scott, EMP]]) !plan +!} + +!if (use_new_decorr) { +EnumerableCalc(expr#0..10=[{inputs}], EMPNO=[$t0], $f1=[$t10]) + EnumerableHashJoin(condition=[IS NOT DISTINCT FROM($3, $8)], joinType=[left]) + EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t5):DECIMAL(12, 2)], expr#9=[10.00:DECIMAL(12, 2)], expr#10=[>($t8, $t9)], proj#0..7=[{exprs}], $condition=[$t10]) + EnumerableTableScan(table=[[scott, EMP]]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[IS NOT NULL($t2)], expr#4=[0], expr#5=[CASE($t3, $t2, $t4)], proj#0..1=[{exprs}], $f2=[$t5]) + EnumerableHashJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left]) + EnumerableAggregate(group=[{3}]) + EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t5):DECIMAL(12, 2)], expr#9=[10.00:DECIMAL(12, 2)], expr#10=[>($t8, $t9)], proj#0..7=[{exprs}], $condition=[$t10]) + EnumerableTableScan(table=[[scott, EMP]]) + EnumerableAggregate(group=[{8}], C=[COUNT()]) + EnumerableNestedLoopJoin(condition=[<($3, $8)], joinType=[inner]) + EnumerableTableScan(table=[[scott, EMP]]) + EnumerableAggregate(group=[{3}]) + EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t5):DECIMAL(12, 2)], expr#9=[10.00:DECIMAL(12, 2)], expr#10=[>($t8, $t9)], proj#0..7=[{exprs}], $condition=[$t10]) + EnumerableTableScan(table=[[scott, EMP]]) +!plan +!} # Reset to default value true !set trimfields true From 92ca537473525ef52e86b8446689fb0c943dcfec Mon Sep 17 00:00:00 2001 From: Terran Date: Tue, 20 Jan 2026 14:51:05 +0800 Subject: [PATCH 125/562] [CALCITE-7388] Redis Adapter operand config should not support empty string --- .../calcite/adapter/redis/RedisSchema.java | 27 ++- .../redis/RedisAdapterConfigCaseBase.java | 200 ++++++++++++++++++ 2 files changed, 219 insertions(+), 8 deletions(-) create mode 100644 redis/src/test/java/org/apache/calcite/adapter/redis/RedisAdapterConfigCaseBase.java diff --git a/redis/src/main/java/org/apache/calcite/adapter/redis/RedisSchema.java b/redis/src/main/java/org/apache/calcite/adapter/redis/RedisSchema.java index 8115d93f6ccd..1aad9ae459d5 100644 --- a/redis/src/main/java/org/apache/calcite/adapter/redis/RedisSchema.java +++ b/redis/src/main/java/org/apache/calcite/adapter/redis/RedisSchema.java @@ -20,6 +20,7 @@ import org.apache.calcite.schema.Table; import org.apache.calcite.schema.impl.AbstractSchema; +import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import com.google.common.cache.CacheBuilder; @@ -42,6 +43,11 @@ * is an HTML table on a URL. */ class RedisSchema extends AbstractSchema { + private static final String DATA_FORMAT = "dataFormat"; + private static final String FIELDS = "fields"; + private static final String KEY_DELIMITER = "keyDelimiter"; + private static final String OPERAND = "operand"; + public final String host; public final int port; public final int database; @@ -85,17 +91,22 @@ public RedisTableFieldInfo getTableFieldInfo(String tableName) { for (JsonCustomTable jsonCustomTable : jsonCustomTables) { if (jsonCustomTable.name.equals(tableName)) { Map map = - requireNonNull(jsonCustomTable.operand, "operand"); - if (map.get("dataFormat") == null) { - throw new RuntimeException("dataFormat is null"); + requireNonNull(jsonCustomTable.operand, OPERAND); + if (ObjectUtils.isEmpty(map.get(DATA_FORMAT))) { + throw new RuntimeException("dataFormat is invalid, it must be raw, csv or json"); + } + RedisDataFormat dataFormatEnum = + RedisDataFormat.fromTypeName(map.get(DATA_FORMAT).toString()); + if (dataFormatEnum == null) { + throw new RuntimeException("dataFormat is invalid, it must be raw, csv or json"); } - if (map.get("fields") == null) { + if (ObjectUtils.isEmpty(map.get(FIELDS))) { throw new RuntimeException("fields is null"); } - dataFormat = map.get("dataFormat").toString(); - fields = (List>) map.get("fields"); - if (map.get("keyDelimiter") != null) { - keyDelimiter = map.get("keyDelimiter").toString(); + dataFormat = map.get(DATA_FORMAT).toString(); + fields = (List>) map.get(FIELDS); + if (map.get(KEY_DELIMITER) != null) { + keyDelimiter = map.get(KEY_DELIMITER).toString(); } break; } diff --git a/redis/src/test/java/org/apache/calcite/adapter/redis/RedisAdapterConfigCaseBase.java b/redis/src/test/java/org/apache/calcite/adapter/redis/RedisAdapterConfigCaseBase.java new file mode 100644 index 000000000000..c1affc2bfb4b --- /dev/null +++ b/redis/src/test/java/org/apache/calcite/adapter/redis/RedisAdapterConfigCaseBase.java @@ -0,0 +1,200 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.adapter.redis; + +import org.apache.calcite.config.CalciteSystemProperty; +import org.apache.calcite.test.CalciteAssert; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +import org.junit.jupiter.api.Test; + +import redis.clients.jedis.Protocol; + +import static org.junit.jupiter.api.Assertions.assertNotNull; + +/** + * Tests for the {@code org.apache.calcite.adapter.redis} package. + */ +public class RedisAdapterConfigCaseBase extends RedisAdapterCaseBase { + + private String model; + + /** Test case of + * [CALCITE-7388] + * Redis Adapter operand config should not support empty string. */ + protected void readModelFromJsonString(String jsonString) { + String strResult = null; + try { + ObjectMapper objMapper = new ObjectMapper(); + objMapper.configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, true) + .configure(JsonParser.Feature.ALLOW_SINGLE_QUOTES, true) + .configure(JsonParser.Feature.ALLOW_COMMENTS, true); + JsonNode rootNode = objMapper.readTree(jsonString); + strResult = + rootNode.toString().replace(Integer.toString(Protocol.DEFAULT_PORT), + Integer.toString(getRedisServerPort())); + } catch (Exception e) { + throw new RuntimeException("Failed to read model from json string", e); + } + model = strResult; + } + + /** Test case of + * [CALCITE-7388] + * Redis Adapter operand config should not support empty string. */ + @Test void testDataFormatEmptyException() { + String jsonString = "{" + + " \"version\": \"1.0\"," + + " \"defaultSchema\": \"redis\"," + + " \"schemas\": [" + + " {" + + " \"type\": \"custom\"," + + " \"name\": \"foodmart\"," + + " \"factory\": \"org.apache.calcite.adapter.redis.RedisSchemaFactory\"," + + " \"operand\": {" + + " \"host\": \"localhost\"," + + " \"port\": 6379 ," + + " \"database\": 0," + + " \"password\": \"\"" + + " }," + + " \"tables\": [" + + " {" + + " \"name\": \"csv_05\"," + + " \"type\": \"custom\"," + + " \"factory\": \"org.apache.calcite.adapter.redis.RedisTableFactory\"," + + " \"operand\": {" + + " \"dataFormat\": \"\"," + + " \"keyDelimiter\": \":\"," + + " \"fields\": [\n" + + " {\n" + + " \"name\": \"DEPTNO\",\n" + + " \"type\": \"varchar\",\n" + + " \"mapping\": 0\n" + + " },\n" + + " {\n" + + " \"name\": \"NAME\",\n" + + " \"type\": \"varchar\",\n" + + " \"mapping\": 1\n" + + " }\n" + + " ]\n" + + " }" + + " }" + + " ]" + + " }" + + " ]" + + "}"; + readModelFromJsonString(jsonString); + assertNotNull(model, "model cannot be null!"); + CalciteAssert.model(model) + .enable(CalciteSystemProperty.TEST_REDIS.value()) + .connectThrows("dataFormat is invalid, it must be raw, csv or json"); + } + + /** Test case of + * [CALCITE-7388] + * Redis Adapter operand config should not support empty string. */ + @Test void testDataFieldsEmptyException() { + String jsonString = "{" + + " \"version\": \"1.0\"," + + " \"defaultSchema\": \"redis\"," + + " \"schemas\": [" + + " {" + + " \"type\": \"custom\"," + + " \"name\": \"foodmart\"," + + " \"factory\": \"org.apache.calcite.adapter.redis.RedisSchemaFactory\"," + + " \"operand\": {" + + " \"host\": \"localhost\"," + + " \"port\": 6379 ," + + " \"database\": 0," + + " \"password\": \"\"" + + " }," + + " \"tables\": [" + + " {" + + " \"name\": \"csv_05\"," + + " \"type\": \"custom\"," + + " \"factory\": \"org.apache.calcite.adapter.redis.RedisTableFactory\"," + + " \"operand\": {" + + " \"dataFormat\": \"csv\"," + + " \"keyDelimiter\": \":\"," + + " \"fields\": []\n" + + " }" + + " }" + + " ]" + + " }" + + " ]" + + "}"; + readModelFromJsonString(jsonString); + assertNotNull(model, "model cannot be null!"); + CalciteAssert.model(model) + .enable(CalciteSystemProperty.TEST_REDIS.value()) + .connectThrows("fields is null"); + } + + /** Test case of + * [CALCITE-7388] + * Redis Adapter operand config should not support empty string. */ + @Test void testDataFormatInvalidException() { + String jsonString = "{" + + " \"version\": \"1.0\"," + + " \"defaultSchema\": \"redis\"," + + " \"schemas\": [" + + " {" + + " \"type\": \"custom\"," + + " \"name\": \"foodmart\"," + + " \"factory\": \"org.apache.calcite.adapter.redis.RedisSchemaFactory\"," + + " \"operand\": {" + + " \"host\": \"localhost\"," + + " \"port\": 6379 ," + + " \"database\": 0," + + " \"password\": \"\"" + + " }," + + " \"tables\": [" + + " {" + + " \"name\": \"csv_05\"," + + " \"type\": \"custom\"," + + " \"factory\": \"org.apache.calcite.adapter.redis.RedisTableFactory\"," + + " \"operand\": {" + + " \"dataFormat\": \"CSV\"," + + " \"keyDelimiter\": \":\"," + + " \"fields\": [\n" + + " {\n" + + " \"name\": \"DEPTNO\",\n" + + " \"type\": \"varchar\",\n" + + " \"mapping\": 0\n" + + " },\n" + + " {\n" + + " \"name\": \"NAME\",\n" + + " \"type\": \"varchar\",\n" + + " \"mapping\": 1\n" + + " }\n" + + " ]\n" + + " }" + + " }" + + " ]" + + " }" + + " ]" + + "}"; + readModelFromJsonString(jsonString); + assertNotNull(model, "model cannot be null!"); + CalciteAssert.model(model) + .enable(CalciteSystemProperty.TEST_REDIS.value()) + .connectThrows("dataFormat is invalid, it must be raw, csv or json"); + } +} From 312b2d73dba60e2a1729cab217a8fca5fae19bd3 Mon Sep 17 00:00:00 2001 From: iwanttobepowerful <745778074@qq.com> Date: Fri, 16 Jan 2026 11:46:45 +0800 Subject: [PATCH 126/562] [CALCITE-7379] LHS correlated variables are shadowed by nullable RHS outputs in LEFT JOIN --- .../calcite/sql2rel/RelDecorrelator.java | 286 +++++++-- .../calcite/sql2rel/RelDecorrelatorTest.java | 166 +++++ core/src/test/resources/sql/sub-query.iq | 588 ++++++++++++++++++ 3 files changed, 990 insertions(+), 50 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java b/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java index 80f1f95335b9..9813ef27d1fb 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java @@ -959,18 +959,19 @@ private RelNode rewriteScalarAggregate(Aggregate oldRel, // Build join conditions final Map newProjectMap = new HashMap<>(); - final List conditions = new ArrayList<>(); for (Map.Entry corDefOutput : corDefOutputs.entrySet()) { final CorDef corDef = corDefOutput.getKey(); final int leftPos = requireNonNull(valueGenCorDefOutputs.get(corDef)); final int rightPos = corDefOutput.getValue(); final RelDataType leftType = valueGen.getRowType().getFieldList().get(leftPos).getType(); - final RelDataType rightType = newRel.getRowType().getFieldList().get(rightPos).getType(); final RexNode leftRef = new RexInputRef(leftPos, leftType); - final RexNode rightRef = new RexInputRef(valueGenFieldCount + rightPos, rightType); - conditions.add(relBuilder.isNotDistinctFrom(leftRef, rightRef)); newProjectMap.put(valueGenFieldCount + rightPos, leftRef); } + + final List conditions = + buildCorDefJoinConditions(valueGenCorDefOutputs, corDefOutputs, + valueGen, newRel, relBuilder); + final RexNode joinCond = RexUtil.composeConjunction(relBuilder.getRexBuilder(), conditions); // Build [08] LogicalJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left]) @@ -1277,21 +1278,13 @@ private static void shiftMapping(Map mapping, int startIndex, // Build join conditions: for each CorDef of this branch that belongs // to the current outFrameCorrId, equate valueGen(col) with branch(col). - final List conditions = new ArrayList<>(); - for (Map.Entry e : frame.corDefOutputs.entrySet()) { - final CorDef corDef = e.getKey(); - final int leftPos = requireNonNull(valueGenCorDefOutputs.get(corDef)); - final int rightPos = e.getValue(); - final RelDataType leftType = valueGen.getRowType().getFieldList().get(leftPos).getType(); - final RelDataType rightType = frame.r.getRowType().getFieldList().get(rightPos).getType(); - final RexNode leftRef = new RexInputRef(leftPos, leftType); - final RexNode rightRef = new RexInputRef(valueGenFieldCount + rightPos, rightType); - conditions.add(relBuilder.isNotDistinctFrom(leftRef, rightRef)); - } + final List conditions = + buildCorDefJoinConditions(valueGenCorDefOutputs, frame.corDefOutputs, + valueGen, frame.r, relBuilder); final RexNode joinCondition = RexUtil.composeConjunction(relBuilder.getRexBuilder(), conditions); RelNode join = relBuilder.push(valueGen).push(frame.r) - .join(JoinRelType.INNER, joinCondition).build(); + .join(JoinRelType.INNER, joinCondition).build(); final List joinFields = join.getRowType().getFieldList(); @@ -1935,10 +1928,16 @@ private static boolean isWidening(RelDataType type, RelDataType type1) { return decorrelateRel((RelNode) rel, isCorVarDefined, parentPropagatesNullValues); } // - // Rewrite logic: + // For other join types (INNER, LEFT, RIGHT, FULL): // - // 1. rewrite join condition. - // 2. map output positions and produce corVars if any. + // 1. Decorrelates the left and right inputs recursively. + // 2. Ensures that required correlated variables are present in the inputs, adding + // value generators if necessary (e.g., for the nullable side of an outer join). + // 3. Constructs a new join condition that includes the original condition and + // equality conditions for the correlated variables. + // 4. For {@link JoinRelType#FULL}, adds a projection on top of the join to coalesce + // correlated variables that might be null on one side due to the join nature. + // 5. Updates the output mapping to reflect the new join structure. // final RelNode oldLeft = rel.getInput(0); @@ -1952,53 +1951,178 @@ private static boolean isWidening(RelDataType type, RelDataType type1) { return null; } + // 1. Collect all CorRefs involved + final CorelMap localCorelMap = new CorelMapBuilder().build(rel); + final List corVarList = new ArrayList<>(localCorelMap.mapRefRelToCorRef.values()); + Collections.sort(corVarList); + + // 2. Ensure CorVars are present in inputs (adding ValueGenerators if needed) Frame newLeftFrame = leftFrame; - boolean joinConditionContainsFieldAccess = RexUtil.containsFieldAccess(rel.getCondition()); - if (joinConditionContainsFieldAccess && isCorVarDefined) { - final CorelMap localCorelMap = new CorelMapBuilder().build(rel); - final List corVarList = new ArrayList<>(localCorelMap.mapRefRelToCorRef.values()); - Collections.sort(corVarList); + Frame newRightFrame = rightFrame; + final NavigableMap leftCorDefOutputs = new TreeMap<>(); + final NavigableMap rightCorDefOutputs = new TreeMap<>(); + boolean generatesNullsOnRight = rel.getJoinType().generatesNullsOnRight(); + boolean generatesNullsOnLeft = rel.getJoinType().generatesNullsOnLeft(); + + if (isCorVarDefined) { + // ensure CorVars are present in left input + if (generatesNullsOnRight || RexUtil.containsFieldAccess(rel.getCondition())) { + newLeftFrame = supplyMissingCorVars(oldLeft, leftFrame, corVarList, leftCorDefOutputs); + rightCorDefOutputs.putAll(rightFrame.corDefOutputs); + } + // ensure CorVars are present in right input + if (generatesNullsOnLeft) { + newRightFrame = supplyMissingCorVars(oldRight, rightFrame, corVarList, rightCorDefOutputs); + leftCorDefOutputs.putAll(leftFrame.corDefOutputs); + } + } else { + leftCorDefOutputs.putAll(leftFrame.corDefOutputs); + rightCorDefOutputs.putAll(rightFrame.corDefOutputs); + } + + // 3. Build Join Conditions + final List joinConditions = new ArrayList<>(); + RexNode originalCond = decorrelateExpr(castNonNull(currentRel), map, cm, rel.getCondition()); + if (!originalCond.isAlwaysTrue()) { + joinConditions.add(originalCond); + } - final NavigableMap corDefOutputs = new TreeMap<>(); - newLeftFrame = createFrameWithValueGenerator(oldLeft, leftFrame, corVarList, corDefOutputs); + if (generatesNullsOnLeft || generatesNullsOnRight) { + List conds = + buildCorDefJoinConditions(leftCorDefOutputs, rightCorDefOutputs, + newLeftFrame.r, newRightFrame.r, relBuilder); + joinConditions.addAll(conds); } + RexNode finalCondition = joinConditions.isEmpty() + ? relBuilder.literal(true) + : RexUtil.composeConjunction(relBuilder.getRexBuilder(), joinConditions); + RelNode newJoin = relBuilder .push(newLeftFrame.r) - .push(rightFrame.r) - .join(rel.getJoinType(), - decorrelateExpr(castNonNull(currentRel), map, cm, rel.getCondition()), - ImmutableSet.of()) + .push(newRightFrame.r) + .join(rel.getJoinType(), finalCondition, ImmutableSet.of()) .build(); - // Create the mapping between the output of the old correlation rel - // and the new join rel - Map mapOldToNewOutputs = new HashMap<>(); - - int oldLeftFieldCount = oldLeft.getRowType().getFieldCount(); + // 4. Handle Full Join Projections (Coalesce) + NavigableMap corDefOutputs = new TreeMap<>(newLeftFrame.corDefOutputs); int newLeftFieldCount = newLeftFrame.r.getRowType().getFieldCount(); + if (rel.getJoinType() == JoinRelType.FULL && isCorVarDefined) { + // + // SELECT + // d.dname, + // ( + // SELECT COUNT(sub.empno) + // FROM ( + // SELECT * FROM emp e2 WHERE e2.deptno = d.deptno + // ) sub + // FULL JOIN emp e + // ON sub.mgr = e.mgr + // ) as matched_subordinate_count + // FROM dept d + // order by d.dname; + // + // LogicalJoin(condition=[=($3, $11)], joinType=[full]) + // LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], ...) + // LogicalFilter(condition=[=($7, $cor0.DEPTNO)]) + // LogicalTableScan(table=[[scott, EMP]]) + // LogicalTableScan(table=[[scott, EMP]]) + // + // convert to: + // + // LogicalProject(_cor_$cor0_0=[COALESCE($8, $17)], EMPNO=[$0]) + // LogicalJoin(condition=[AND(=($3, $12), IS NOT DISTINCT FROM($8, $17))], joinType=[full]) + // LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], ...) + // LogicalFilter(condition=[IS NOT NULL($7)]) + // LogicalTableScan(table=[[scott, EMP]]) + // LogicalJoin(condition=[true], joinType=[inner]) + // LogicalTableScan(table=[[scott, EMP]]) + // LogicalProject(DEPTNO=[$0]) + // LogicalTableScan(table=[[scott, DEPT]]) + List joinFields = newJoin.getRowType().getFieldList(); + + // 4.1. Pass through existing fields + final PairList projects = PairList.of(); + for (int i = 0; i < joinFields.size(); i++) { + RexInputRef.add2(projects, i, joinFields); + } + + // 4.2. Build Coalesced CorVars + NavigableMap mergedCorDefOutputs = new TreeMap<>(corDefOutputs); + int projectedIndex = joinFields.size(); + boolean appended = false; + + for (CorRef corRef : corVarList) { + CorDef corDef = corRef.def(); + + Integer leftPos = leftCorDefOutputs.get(corDef); + Integer rightPos = rightCorDefOutputs.get(corDef); + + // If missing on both sides, nothing to coalesce or project + if (leftPos == null && rightPos == null) { + continue; + } + + // Create references + RexNode leftRef = null; + if (leftPos != null) { + leftRef = new RexInputRef(leftPos, joinFields.get(leftPos).getType()); + } + + RexNode rightRef = null; + if (rightPos != null) { + // Right side indices are offset by the left field count in the join + int actualRightIndex = rightPos + newLeftFieldCount; + rightRef = new RexInputRef(actualRightIndex, joinFields.get(actualRightIndex).getType()); + } + + // Determine the expression + RexNode expr; + if (leftRef == null) { + expr = rightRef; + } else if (rightRef == null) { + expr = leftRef; + } else { + // Both exist, create COALESCE + expr = relBuilder.call(SqlStdOperatorTable.COALESCE, leftRef, rightRef); + } + String name = "_cor_" + corDef.corr.getName() + "_" + corDef.field; + projects.add(requireNonNull(expr, "expr"), name); + mergedCorDefOutputs.put(corDef, projectedIndex++); + appended = true; + } + + if (appended) { + newJoin = relBuilder.push(newJoin) + .projectNamed(projects.leftList(), projects.rightList(), true) + .build(); + corDefOutputs.clear(); + corDefOutputs.putAll(mergedCorDefOutputs); + } + } else { + // Standard output mapping for non-Full Join (or Full Join without CorVars) + // Right input positions are shifted. + for (Map.Entry entry : newRightFrame.corDefOutputs.entrySet()) { + final int shifted = entry.getValue() + newLeftFieldCount; + if (rel.getJoinType().generatesNullsOnRight()) { + corDefOutputs.putIfAbsent(entry.getKey(), shifted); + } else { + corDefOutputs.put(entry.getKey(), shifted); + } + } + } + + // 5. Output Mapping + int oldLeftFieldCount = oldLeft.getRowType().getFieldCount(); int oldRightFieldCount = oldRight.getRowType().getFieldCount(); - //noinspection AssertWithSideEffects - assert rel.getRowType().getFieldCount() - == oldLeftFieldCount + oldRightFieldCount; - // Left input positions are not changed. - mapOldToNewOutputs.putAll(newLeftFrame.oldToNewOutputs); - // Right input positions are shifted by newLeftFieldCount. + Map mapOldToNewOutputs = new HashMap<>(newLeftFrame.oldToNewOutputs); for (int i = 0; i < oldRightFieldCount; i++) { mapOldToNewOutputs.put(i + oldLeftFieldCount, - requireNonNull(rightFrame.oldToNewOutputs.get(i)) + newLeftFieldCount); + requireNonNull(newRightFrame.oldToNewOutputs.get(i)) + newLeftFieldCount); } - final NavigableMap corDefOutputs = - new TreeMap<>(newLeftFrame.corDefOutputs); - // Right input positions are shifted by newLeftFieldCount. - for (Map.Entry entry - : rightFrame.corDefOutputs.entrySet()) { - corDefOutputs.put(entry.getKey(), - entry.getValue() + newLeftFieldCount); - } return register(rel, newJoin, mapOldToNewOutputs, corDefOutputs); } @@ -3719,6 +3843,68 @@ private static boolean isFieldNotNullRecursive(RelNode rel, int index) { } } + /** + * Ensures that the correlated variables in {@code allCorDefs} are present + * in the output of the frame. + * If any are missing, it creates a value generator to produce them and joins it with the frame. + * + * @param oldInput The original input RelNode + * @param frame The current frame for the input + * @param corVarList List of all correlated variables + * @param corDefOutputs Map to populate with the output positions of the correlated variables + * @return A new Frame with all required correlated variables, or the original frame + * if all were present + */ + private Frame supplyMissingCorVars(RelNode oldInput, Frame frame, + List corVarList, NavigableMap corDefOutputs) { + final ImmutableSortedSet haves = frame.corDefOutputs.keySet(); + if (hasAll(corVarList, haves)) { + corDefOutputs.putAll(frame.corDefOutputs); + return frame; + } + + final List miss = new ArrayList<>(); + for (CorRef r : corVarList) { + if (!haves.contains(r.def())) { + miss.add(r); + } + } + + return createFrameWithValueGenerator(oldInput, frame, miss, corDefOutputs); + } + + /** + * Builds join conditions to equate correlated variables that are present in both left + * and right inputs. + * + * @param leftCorDefOutputs Map of CorDefs to output positions in the left input + * @param rightCorDefOutputs Map of CorDefs to output positions in the right input + * @param leftRel The left input RelNode + * @param rightRel The right input RelNode + * @param relBuilder RelBuilder for creating expressions + * @return A list of join conditions (IS NOT DISTINCT FROM) for matching correlated variables + */ + private List buildCorDefJoinConditions( + NavigableMap leftCorDefOutputs, + NavigableMap rightCorDefOutputs, + RelNode leftRel, RelNode rightRel, RelBuilder relBuilder) { + List joinConditions = new ArrayList<>(); + int leftFieldCount = leftRel.getRowType().getFieldCount(); + for (Map.Entry leftEntry : leftCorDefOutputs.entrySet()) { + CorDef corDef = leftEntry.getKey(); + if (rightCorDefOutputs.containsKey(corDef)) { + int leftPos = leftEntry.getValue(); + int rightPos = rightCorDefOutputs.get(corDef); + final RelDataType leftType = leftRel.getRowType().getFieldList().get(leftPos).getType(); + final RelDataType rightType = rightRel.getRowType().getFieldList().get(rightPos).getType(); + final RexNode leftRef = new RexInputRef(leftPos, leftType); + final RexNode rightRef = new RexInputRef(leftFieldCount + rightPos, rightType); + joinConditions.add(relBuilder.isNotDistinctFrom(leftRef, rightRef)); + } + } + return joinConditions; + } + // ------------------------------------------------------------------------- // Getter/Setter // ------------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java b/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java index 45ef07cc45ed..6d1d2a974be6 100644 --- a/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java +++ b/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java @@ -1272,4 +1272,170 @@ public static Frameworks.ConfigBuilder config() { + " LogicalTableScan(table=[[scott, BONUS]])\n"; assertThat(after, hasTree(planAfter)); } + + /** Test case for [CALCITE-7379] + * LHS correlated variables are shadowed by nullable RHS outputs in LEFT JOIN. */ + @Test void testDecorrelateLeftJoinCorVarShadowing() { + final FrameworkConfig frameworkConfig = config().build(); + final RelBuilder builder = RelBuilder.create(frameworkConfig); + final RelOptCluster cluster = builder.getCluster(); + final Planner planner = Frameworks.getPlanner(frameworkConfig); + final String sql = "" + + "WITH\n" + + " t1(a, b, c) AS (VALUES (2, 2, 2), (3, 3, 3), (4, 4, 4)),\n" + + " t2(a, b, c) AS (VALUES (1, 1, 1), (3, 3, 3), (4, 4, 4)),\n" + + " t3(a, b, c) AS (VALUES (1, 1, 1), (2, 2, 2), (4, 4, 4))\n" + + "SELECT * FROM t1 WHERE EXISTS (\n" + + "SELECT * FROM t2\n" + + "LEFT JOIN\n" + + "(SELECT * FROM t3 WHERE t3.a = t1.a) foo\n" + + "ON t2.a = foo.a)"; + final RelNode originalRel; + try { + final SqlNode parse = planner.parse(sql); + final SqlNode validate = planner.validate(parse); + originalRel = planner.rel(validate).rel; + } catch (Exception e) { + throw TestUtil.rethrow(e); + } + + final HepProgram hepProgram = HepProgram.builder() + .addRuleCollection( + ImmutableList.of( + // SubQuery program rules + CoreRules.FILTER_SUB_QUERY_TO_CORRELATE, + CoreRules.PROJECT_SUB_QUERY_TO_CORRELATE, + CoreRules.JOIN_SUB_QUERY_TO_CORRELATE)) + .build(); + final Program program = + Programs.of(hepProgram, true, + requireNonNull(cluster.getMetadataProvider())); + final RelNode before = + program.run(cluster.getPlanner(), originalRel, cluster.traitSet(), + Collections.emptyList(), Collections.emptyList()); + final String planBefore = "" + + "LogicalProject(A=[$0], B=[$1], C=[$2])\n" + + " LogicalProject(EXPR$0=[$0], EXPR$1=[$1], EXPR$2=[$2])\n" + + " LogicalCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{0}])\n" + + " LogicalValues(tuples=[[{ 2, 2, 2 }, { 3, 3, 3 }, { 4, 4, 4 }]])\n" + + " LogicalAggregate(group=[{0}])\n" + + " LogicalProject(i=[true])\n" + + " LogicalJoin(condition=[=($0, $3)], joinType=[left])\n" + + " LogicalValues(tuples=[[{ 1, 1, 1 }, { 3, 3, 3 }, { 4, 4, 4 }]])\n" + + " LogicalProject(A=[$0], B=[$1], C=[$2])\n" + + " LogicalFilter(condition=[=($0, $cor0.A)])\n" + + " LogicalValues(tuples=[[{ 1, 1, 1 }, { 2, 2, 2 }, { 4, 4, 4 }]])\n"; + assertThat(before, hasTree(planBefore)); + + // Decorrelate without any rules, just "purely" decorrelation algorithm on RelDecorrelator + final RelNode after = + RelDecorrelator.decorrelateQuery(before, builder, RuleSets.ofList(Collections.emptyList()), + RuleSets.ofList(Collections.emptyList())); + // The plan before fix: + // + // LogicalProject(A=[$0], B=[$1], C=[$2]) + // LogicalJoin(condition=[IS NOT DISTINCT FROM($0, $3)], joinType=[inner]) + // LogicalValues(tuples=[[{ 2, 2, 2 }, { 3, 3, 3 }, { 4, 4, 4 }]]) + // LogicalProject(EXPR$00=[$0], $f1=[true]) + // LogicalAggregate(group=[{0}]) + // LogicalProject(EXPR$00=[$6]) + // LogicalJoin(condition=[=($0, $3)], joinType=[left]) + // LogicalValues(tuples=[[{ 1, 1, 1 }, { 3, 3, 3 }, { 4, 4, 4 }]]) + // LogicalProject(A=[$0], B=[$1], C=[$2], EXPR$0=[$0]) + // LogicalValues(tuples=[[{ 1, 1, 1 }, { 2, 2, 2 }, { 4, 4, 4 }]]) + final String planAfter = "" + + "LogicalProject(A=[$0], B=[$1], C=[$2])\n" + + " LogicalJoin(condition=[=($0, $3)], joinType=[inner])\n" + + " LogicalValues(tuples=[[{ 2, 2, 2 }, { 3, 3, 3 }, { 4, 4, 4 }]])\n" + + " LogicalProject(EXPR$00=[$0], $f1=[true])\n" + + " LogicalAggregate(group=[{0}])\n" + + " LogicalProject(EXPR$00=[$3])\n" + + " LogicalJoin(condition=[AND(=($0, $4), IS NOT DISTINCT FROM($3, $7))], joinType=[left])\n" + + " LogicalJoin(condition=[true], joinType=[inner])\n" + + " LogicalValues(tuples=[[{ 1, 1, 1 }, { 3, 3, 3 }, { 4, 4, 4 }]])\n" + + " LogicalProject(EXPR$0=[$0])\n" + + " LogicalValues(tuples=[[{ 2, 2, 2 }, { 3, 3, 3 }, { 4, 4, 4 }]])\n" + + " LogicalProject(A=[$0], B=[$1], C=[$2], EXPR$0=[$0])\n" + + " LogicalValues(tuples=[[{ 1, 1, 1 }, { 2, 2, 2 }, { 4, 4, 4 }]])\n"; + assertThat(after, hasTree(planAfter)); + } + + /** Test case for [CALCITE-7379] + * LHS correlated variables are shadowed by nullable RHS outputs in LEFT JOIN. */ + @Test void testDecorrelateFullJoinCorVarShadowing() { + final FrameworkConfig frameworkConfig = config().build(); + final RelBuilder builder = RelBuilder.create(frameworkConfig); + final RelOptCluster cluster = builder.getCluster(); + final Planner planner = Frameworks.getPlanner(frameworkConfig); + final String sql = "" + + "SELECT\n" + + " d.dname,\n" + + " (SELECT COUNT(sub.empno)\n" + + " FROM (\n" + + " SELECT * FROM emp e2 WHERE e2.deptno = d.deptno\n" + + " ) sub\n" + + " FULL JOIN emp e\n" + + " ON sub.mgr = e.mgr\n" + + " ) as matched_subordinate_count\n" + + "FROM dept d"; + final RelNode originalRel; + try { + final SqlNode parse = planner.parse(sql); + final SqlNode validate = planner.validate(parse); + originalRel = planner.rel(validate).rel; + } catch (Exception e) { + throw TestUtil.rethrow(e); + } + + final HepProgram hepProgram = HepProgram.builder() + .addRuleCollection( + ImmutableList.of( + // SubQuery program rules + CoreRules.FILTER_SUB_QUERY_TO_CORRELATE, + CoreRules.PROJECT_SUB_QUERY_TO_CORRELATE, + CoreRules.JOIN_SUB_QUERY_TO_CORRELATE)) + .build(); + final Program program = + Programs.of(hepProgram, true, + requireNonNull(cluster.getMetadataProvider())); + final RelNode before = + program.run(cluster.getPlanner(), originalRel, cluster.traitSet(), + Collections.emptyList(), Collections.emptyList()); + final String planBefore = "" + + "LogicalProject(DNAME=[$1], MATCHED_SUBORDINATE_COUNT=[$3])\n" + + " LogicalCorrelate(correlation=[$cor0], joinType=[left], requiredColumns=[{0}])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n" + + " LogicalAggregate(group=[{}], EXPR$0=[COUNT($0)])\n" + + " LogicalProject(EMPNO=[$0])\n" + + " LogicalJoin(condition=[=($3, $11)], joinType=[full])\n" + + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7])\n" + + " LogicalFilter(condition=[=($7, $cor0.DEPTNO)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + assertThat(before, hasTree(planBefore)); + + // Decorrelate without any rules, just "purely" decorrelation algorithm on RelDecorrelator + final RelNode after = + RelDecorrelator.decorrelateQuery(before, builder, RuleSets.ofList(Collections.emptyList()), + RuleSets.ofList(Collections.emptyList())); + final String planAfter = "" + + "LogicalProject(DNAME=[$1], MATCHED_SUBORDINATE_COUNT=[$4])\n" + + " LogicalJoin(condition=[=($0, $3)], joinType=[left])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n" + + " LogicalProject(_cor_$cor0_0=[$0], EXPR$0=[CASE(IS NOT NULL($2), $2, 0)])\n" + + " LogicalJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left])\n" + + " LogicalProject(DEPTNO=[$0])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n" + + " LogicalAggregate(group=[{0}], EXPR$0=[COUNT($1)])\n" + + " LogicalProject(_cor_$cor0_0=[COALESCE($8, $17)], EMPNO=[$0])\n" + + " LogicalJoin(condition=[AND(=($3, $12), IS NOT DISTINCT FROM($8, $17))], joinType=[full])\n" + + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7], DEPTNO8=[$7])\n" + + " LogicalFilter(condition=[IS NOT NULL($7)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalJoin(condition=[true], joinType=[inner])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalProject(DEPTNO=[$0])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n"; + assertThat(after, hasTree(planAfter)); + } } diff --git a/core/src/test/resources/sql/sub-query.iq b/core/src/test/resources/sql/sub-query.iq index 73b8c939b417..9095ff305915 100644 --- a/core/src/test/resources/sql/sub-query.iq +++ b/core/src/test/resources/sql/sub-query.iq @@ -7880,4 +7880,592 @@ WHERE deptno NOT IN ( # Reset to default value 20 !set trimfields true +!use scott +# [CALCITE-7379] LHS correlated variables are shadowed by nullable RHS outputs in LEFT JOIN +# Correlated scalar subquery with LEFT JOIN. +# The correlation variable (d.deptno) is used in the RHS of the join. +SELECT + d.dname, + ( + SELECT COUNT(sub.empno) + FROM emp e + LEFT JOIN ( + SELECT * FROM emp e2 WHERE e2.deptno = d.deptno + ) sub + ON e.mgr = sub.mgr + ) as matched_subordinate_count +FROM dept d; ++------------+---------------------------+ +| DNAME | MATCHED_SUBORDINATE_COUNT | ++------------+---------------------------+ +| ACCOUNTING | 4 | +| OPERATIONS | 0 | +| RESEARCH | 9 | +| SALES | 28 | ++------------+---------------------------+ +(4 rows) + +!ok + +# Correlated scalar subquery with RIGHT JOIN. +# The correlation variable (d.deptno) is used in the LHS of the join. +SELECT + d.dname, + ( + SELECT COUNT(sub.empno) + FROM ( + SELECT * FROM emp e2 WHERE e2.deptno = d.deptno + ) sub + RIGHT JOIN emp e + ON sub.mgr = e.mgr + ) as matched_subordinate_count +FROM dept d; ++------------+---------------------------+ +| DNAME | MATCHED_SUBORDINATE_COUNT | ++------------+---------------------------+ +| ACCOUNTING | 4 | +| OPERATIONS | 0 | +| RESEARCH | 9 | +| SALES | 28 | ++------------+---------------------------+ +(4 rows) + +!ok + +# Correlated scalar subquery with FULL JOIN. +# The correlation variable (d.deptno) is used in the LHS of the join. +SELECT + d.dname, + ( + SELECT COUNT(sub.empno) + FROM ( + SELECT * FROM emp e2 WHERE e2.deptno = d.deptno + ) sub + FULL JOIN emp e + ON sub.mgr = e.mgr + ) as matched_subordinate_count +FROM dept d +order by d.dname; ++------------+---------------------------+ +| DNAME | MATCHED_SUBORDINATE_COUNT | ++------------+---------------------------+ +| ACCOUNTING | 5 | +| OPERATIONS | 0 | +| RESEARCH | 9 | +| SALES | 28 | ++------------+---------------------------+ +(4 rows) + +!ok + +# Correlated NOT EXISTS subquery with LEFT JOIN. +# The correlation variable (d.deptno) is used in the RHS of the join. +SELECT * FROM dept d +WHERE NOT EXISTS ( + SELECT 1 + FROM emp e + LEFT JOIN ( + SELECT * FROM emp e3 WHERE e3.deptno = d.deptno + ) foo + ON e.empno = foo.mgr +); ++--------+-------+-----+ +| DEPTNO | DNAME | LOC | ++--------+-------+-----+ ++--------+-------+-----+ +(0 rows) + +!ok + +# Correlated NOT EXISTS subquery with RIGHT JOIN. +# The correlation variable (d.deptno) is used in the LHS of the join. +SELECT * FROM dept d +WHERE NOT EXISTS ( + SELECT 1 + FROM ( + SELECT * FROM emp e3 WHERE e3.deptno = d.deptno + ) foo + RIGHT JOIN emp e + ON foo.mgr = e.empno +); ++--------+-------+-----+ +| DEPTNO | DNAME | LOC | ++--------+-------+-----+ ++--------+-------+-----+ +(0 rows) + +!ok + +# Correlated NOT EXISTS subquery with FULL JOIN. +# The correlation variable (d.deptno) is used in the LHS of the join. +SELECT * FROM dept d +WHERE NOT EXISTS ( + SELECT 1 + FROM ( + SELECT * FROM emp e3 WHERE e3.deptno = d.deptno + ) foo + FULL JOIN emp e + ON foo.mgr = e.empno +); ++--------+-------+-----+ +| DEPTNO | DNAME | LOC | ++--------+-------+-----+ ++--------+-------+-----+ +(0 rows) + +!ok + +# Correlated EXISTS subquery with LEFT JOIN involving 'bonus' table. +# The correlation variable (e.ename) is used in the RHS of the join. +SELECT e.ename, e.job, e.sal +FROM emp e +WHERE EXISTS ( + SELECT 1 + FROM dept d + LEFT JOIN ( + SELECT * FROM bonus b WHERE b.ename = e.ename + ) foo + ON d.loc = foo.job +); ++--------+-----------+---------+ +| ENAME | JOB | SAL | ++--------+-----------+---------+ +| ADAMS | CLERK | 1100.00 | +| ALLEN | SALESMAN | 1600.00 | +| BLAKE | MANAGER | 2850.00 | +| CLARK | MANAGER | 2450.00 | +| FORD | ANALYST | 3000.00 | +| JAMES | CLERK | 950.00 | +| JONES | MANAGER | 2975.00 | +| KING | PRESIDENT | 5000.00 | +| MARTIN | SALESMAN | 1250.00 | +| MILLER | CLERK | 1300.00 | +| SCOTT | ANALYST | 3000.00 | +| SMITH | CLERK | 800.00 | +| TURNER | SALESMAN | 1500.00 | +| WARD | SALESMAN | 1250.00 | ++--------+-----------+---------+ +(14 rows) + +!ok + +# Correlated EXISTS subquery with RIGHT JOIN involving 'bonus' table. +# The correlation variable (e.ename) is used in the LHS of the join. +SELECT e.ename, e.job, e.sal +FROM emp e +WHERE EXISTS ( + SELECT 1 + FROM ( + SELECT * FROM bonus b WHERE b.ename = e.ename + ) foo + RIGHT JOIN dept d + ON foo.job = d.loc +); ++--------+-----------+---------+ +| ENAME | JOB | SAL | ++--------+-----------+---------+ +| ADAMS | CLERK | 1100.00 | +| ALLEN | SALESMAN | 1600.00 | +| BLAKE | MANAGER | 2850.00 | +| CLARK | MANAGER | 2450.00 | +| FORD | ANALYST | 3000.00 | +| JAMES | CLERK | 950.00 | +| JONES | MANAGER | 2975.00 | +| KING | PRESIDENT | 5000.00 | +| MARTIN | SALESMAN | 1250.00 | +| MILLER | CLERK | 1300.00 | +| SCOTT | ANALYST | 3000.00 | +| SMITH | CLERK | 800.00 | +| TURNER | SALESMAN | 1500.00 | +| WARD | SALESMAN | 1250.00 | ++--------+-----------+---------+ +(14 rows) + +!ok + +# Correlated EXISTS subquery with FULL JOIN involving 'bonus' table. +# The correlation variable (e.ename) is used in the LHS of the join. +SELECT e.ename, e.job, e.sal +FROM emp e +WHERE EXISTS ( + SELECT 1 + FROM ( + SELECT * FROM bonus b WHERE b.ename = e.ename + ) foo + FULL JOIN dept d + ON foo.job = d.loc +); ++--------+-----------+---------+ +| ENAME | JOB | SAL | ++--------+-----------+---------+ +| ADAMS | CLERK | 1100.00 | +| ALLEN | SALESMAN | 1600.00 | +| BLAKE | MANAGER | 2850.00 | +| CLARK | MANAGER | 2450.00 | +| FORD | ANALYST | 3000.00 | +| JAMES | CLERK | 950.00 | +| JONES | MANAGER | 2975.00 | +| KING | PRESIDENT | 5000.00 | +| MARTIN | SALESMAN | 1250.00 | +| MILLER | CLERK | 1300.00 | +| SCOTT | ANALYST | 3000.00 | +| SMITH | CLERK | 800.00 | +| TURNER | SALESMAN | 1500.00 | +| WARD | SALESMAN | 1250.00 | ++--------+-----------+---------+ +(14 rows) + +!ok + +# Correlated EXISTS subquery with LEFT JOIN and complex correlation condition. +# The correlation variable (e1.sal, e1.comm) is used in the RHS of the join. +SELECT empno FROM emp e1 +WHERE EXISTS ( + SELECT 1 + FROM dept d + LEFT JOIN ( + SELECT * FROM emp e2 + WHERE e2.sal > (e1.sal + COALESCE(e1.comm, 0)) + ) foo + ON d.deptno = foo.deptno +); ++-------+ +| EMPNO | ++-------+ +| 7369 | +| 7499 | +| 7521 | +| 7566 | +| 7654 | +| 7698 | +| 7782 | +| 7788 | +| 7839 | +| 7844 | +| 7876 | +| 7900 | +| 7902 | +| 7934 | ++-------+ +(14 rows) + +!ok + +# Correlated EXISTS subquery with RIGHT JOIN and complex correlation condition. +# The correlation variable (e1.sal, e1.comm) is used in the LHS of the join. +SELECT empno FROM emp e1 +WHERE EXISTS ( + SELECT 1 + FROM ( + SELECT * FROM emp e2 + WHERE e2.sal > (e1.sal + COALESCE(e1.comm, 0)) + ) foo + RIGHT JOIN dept d + ON foo.deptno = d.deptno +); ++-------+ +| EMPNO | ++-------+ +| 7369 | +| 7499 | +| 7521 | +| 7566 | +| 7654 | +| 7698 | +| 7782 | +| 7788 | +| 7839 | +| 7844 | +| 7876 | +| 7900 | +| 7902 | +| 7934 | ++-------+ +(14 rows) + +!ok + +# Correlated EXISTS subquery with FULL JOIN and complex correlation condition. +# The correlation variable (e1.sal, e1.comm) is used in the LHS of the join. +SELECT empno FROM emp e1 +WHERE EXISTS ( + SELECT 1 + FROM ( + SELECT * FROM emp e2 + WHERE e2.sal > (e1.sal + COALESCE(e1.comm, 0)) + ) foo + FULL JOIN dept d + ON foo.deptno = d.deptno +); ++-------+ +| EMPNO | ++-------+ +| 7369 | +| 7499 | +| 7521 | +| 7566 | +| 7654 | +| 7698 | +| 7782 | +| 7788 | +| 7839 | +| 7844 | +| 7876 | +| 7900 | +| 7902 | +| 7934 | ++-------+ +(14 rows) + +!ok + +# Correlated EXISTS subquery with LEFT JOIN on TRUE condition. +# The correlation variable (d.deptno) is used in the RHS of the join. +SELECT d.deptno +FROM dept d +WHERE EXISTS ( + SELECT 1 + FROM emp e + LEFT JOIN ( + SELECT deptno FROM emp WHERE deptno = d.deptno + ) foo + ON TRUE + WHERE foo.deptno IS NOT DISTINCT FROM d.deptno +); ++--------+ +| DEPTNO | ++--------+ +| 10 | +| 20 | +| 30 | ++--------+ +(3 rows) + +!ok + +# Correlated EXISTS subquery with RIGHT JOIN on TRUE condition. +# The correlation variable (d.deptno) is used in the LHS of the join. +SELECT d.deptno +FROM dept d +WHERE EXISTS ( + SELECT 1 + FROM ( + SELECT deptno FROM emp WHERE deptno = d.deptno + ) foo + RIGHT JOIN emp e + ON TRUE + WHERE foo.deptno IS NOT DISTINCT FROM d.deptno +); ++--------+ +| DEPTNO | ++--------+ +| 10 | +| 20 | +| 30 | ++--------+ +(3 rows) + +!ok + +# Correlated EXISTS subquery with FULL JOIN on TRUE condition. +# The correlation variable (d.deptno) is used in the LHS of the join. +SELECT d.deptno +FROM dept d +WHERE EXISTS ( + SELECT 1 + FROM ( + SELECT deptno FROM emp WHERE deptno = d.deptno + ) foo + FULL JOIN emp e + ON TRUE + WHERE foo.deptno IS NOT DISTINCT FROM d.deptno +); ++--------+ +| DEPTNO | ++--------+ +| 10 | +| 20 | +| 30 | ++--------+ +(3 rows) + +!ok + +# Correlated EXISTS subquery with LEFT JOIN. +# The correlation variable (dept.deptno) is used in the RHS of the join. +SELECT * FROM dept +WHERE EXISTS ( + SELECT * FROM emp + LEFT JOIN ( + SELECT * FROM emp e_sub + WHERE e_sub.deptno = dept.deptno + ) foo + ON emp.deptno = foo.deptno +); ++--------+------------+----------+ +| DEPTNO | DNAME | LOC | ++--------+------------+----------+ +| 10 | ACCOUNTING | NEW YORK | +| 20 | RESEARCH | DALLAS | +| 30 | SALES | CHICAGO | +| 40 | OPERATIONS | BOSTON | ++--------+------------+----------+ +(4 rows) + +!ok + +# Correlated EXISTS subquery with RIGHT JOIN. +# The correlation variable (dept.deptno) is used in the LHS of the join. +SELECT * FROM dept +WHERE EXISTS ( + SELECT * FROM ( + SELECT * FROM emp e_sub + WHERE e_sub.deptno = dept.deptno + ) foo + RIGHT JOIN emp + ON foo.deptno = emp.deptno +); ++--------+------------+----------+ +| DEPTNO | DNAME | LOC | ++--------+------------+----------+ +| 10 | ACCOUNTING | NEW YORK | +| 20 | RESEARCH | DALLAS | +| 30 | SALES | CHICAGO | +| 40 | OPERATIONS | BOSTON | ++--------+------------+----------+ +(4 rows) + +!ok + +# Correlated EXISTS subquery with FULL JOIN. +# The correlation variable (dept.deptno) is used in the LHS of the join. +SELECT * FROM dept +WHERE EXISTS ( + SELECT * FROM ( + SELECT * FROM emp e_sub + WHERE e_sub.deptno = dept.deptno + ) foo + FULL JOIN emp + ON foo.deptno = emp.deptno +); ++--------+------------+----------+ +| DEPTNO | DNAME | LOC | ++--------+------------+----------+ +| 10 | ACCOUNTING | NEW YORK | +| 20 | RESEARCH | DALLAS | +| 30 | SALES | CHICAGO | +| 40 | OPERATIONS | BOSTON | ++--------+------------+----------+ +(4 rows) + +!ok + +# Correlated EXISTS subquery with LEFT JOIN using VALUES clause. +# The correlation variable (t1.a) is used in the RHS of the join. +WITH + t1(a, b, c) AS (VALUES (2, 2, 2), (3, 3, 3), (4, 4, 4)), + t2(a, b, c) AS (VALUES (1, 1, 1), (3, 3, 3), (4, 4, 4)), + t3(a, b, c) AS (VALUES (1, 1, 1), (2, 2, 2), (4, 4, 4)) +SELECT * FROM t1 WHERE EXISTS ( +SELECT * FROM t2 +LEFT JOIN +(SELECT * FROM t3 WHERE t3.a = t1.a) foo +ON t2.a = foo.a +); ++---+---+---+ +| A | B | C | ++---+---+---+ +| 2 | 2 | 2 | +| 3 | 3 | 3 | +| 4 | 4 | 4 | ++---+---+---+ +(3 rows) + +!ok + +# Correlated EXISTS subquery with LEFT JOIN using VALUES clause. +# The correlation variable (t1.a) is used in the LHS of the join. +WITH + t1(a, b, c) AS (VALUES (2, 2, 2), (3, 3, 3), (4, 4, 4)), + t2(a, b, c) AS (VALUES (1, 1, 1), (3, 3, 3), (4, 4, 4)), + t3(a, b, c) AS (VALUES (1, 1, 1), (2, 2, 2), (4, 4, 4)) +SELECT * FROM t1 WHERE EXISTS ( +SELECT * FROM (SELECT * FROM t3 WHERE t3.a = t1.a) foo +LEFT JOIN t2 +ON foo.a = t2.a +); ++---+---+---+ +| A | B | C | ++---+---+---+ +| 2 | 2 | 2 | +| 4 | 4 | 4 | ++---+---+---+ +(2 rows) + +!ok + +# Correlated EXISTS subquery with RIGHT JOIN using VALUES clause. +# The correlation variable (t1.a) is used in the LHS of the join. +WITH + t1(a, b, c) AS (VALUES (2, 2, 2), (3, 3, 3), (4, 4, 4)), + t2(a, b, c) AS (VALUES (1, 1, 1), (3, 3, 3), (4, 4, 4)), + t3(a, b, c) AS (VALUES (1, 1, 1), (2, 2, 2), (4, 4, 4)) +SELECT * FROM t1 WHERE EXISTS ( +SELECT * FROM (SELECT * FROM t3 WHERE t3.a = t1.a) foo +RIGHT JOIN t2 +ON foo.a = t2.a +); ++---+---+---+ +| A | B | C | ++---+---+---+ +| 2 | 2 | 2 | +| 3 | 3 | 3 | +| 4 | 4 | 4 | ++---+---+---+ +(3 rows) + +!ok + +# Correlated EXISTS subquery with FULL JOIN using VALUES clause. +# The correlation variable (t1.a) is used in the LHS of the join. +WITH + t1(a, b, c) AS (VALUES (2, 2, 2), (3, 3, 3), (4, 4, 4)), + t2(a, b, c) AS (VALUES (1, 1, 1), (3, 3, 3), (4, 4, 4)), + t3(a, b, c) AS (VALUES (1, 1, 1), (2, 2, 2), (4, 4, 4)) +SELECT * FROM t1 WHERE EXISTS ( +SELECT * FROM (SELECT * FROM t3 WHERE t3.a = t1.a) foo +FULL JOIN t2 +ON foo.a = t2.a +); ++---+---+---+ +| A | B | C | ++---+---+---+ +| 2 | 2 | 2 | +| 3 | 3 | 3 | +| 4 | 4 | 4 | ++---+---+---+ +(3 rows) + +!ok + +# Correlated EXISTS subquery with INNER JOIN using VALUES clause. +# The correlation variable (t1.a) is used in the RHS of the join. +WITH + t1(a, b, c) AS (VALUES (2, 2, 2), (3, 3, 3), (4, 4, 4)), + t2(a, b, c) AS (VALUES (1, 1, 1), (3, 3, 3), (4, 4, 4)), + t3(a, b, c) AS (VALUES (1, 1, 1), (2, 2, 2), (4, 4, 4)) +SELECT * FROM t1 WHERE EXISTS ( +SELECT * FROM t2 +INNER JOIN +(SELECT * FROM t3 WHERE t3.a = t1.a) foo +ON t2.a = foo.a +); ++---+---+---+ +| A | B | C | ++---+---+---+ +| 4 | 4 | 4 | ++---+---+---+ +(1 row) + +!ok # End sub-query.iq From 8c3bf34a69e2f16917ddd1d9dc086e1066395886 Mon Sep 17 00:00:00 2001 From: nobigo Date: Fri, 23 Jan 2026 09:11:51 +0800 Subject: [PATCH 127/562] Add debug log for query plan after decorrelation completion --- .../java/org/apache/calcite/sql2rel/RelDecorrelator.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java b/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java index 9813ef27d1fb..226159104a97 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java @@ -275,6 +275,12 @@ public static RelNode decorrelateQuery(RelNode rootRel, "Decorrelation produced a relation with a different type; before: " + rootRel.getRowType() + " after: " + newRootRel.getRowType()); + if (SQL2REL_LOGGER.isDebugEnabled()) { + SQL2REL_LOGGER.debug( + RelOptUtil.dumpPlan("Plan after decorrelation", newRootRel, + SqlExplainFormat.TEXT, SqlExplainLevel.EXPPLAN_ATTRIBUTES)); + } + // Re-propagate the hints. newRootRel = RelOptUtil.propagateRelHints(newRootRel, true); return newRootRel; From 41d66a84cb339959ae94912a09a8c70ae4ae70e1 Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Sat, 24 Jan 2026 20:25:27 +0800 Subject: [PATCH 128/562] [CALCITE-7395] ProjectMergeRule incorrectly merges PROJECTs with correlation variables --- .../calcite/rel/rules/ProjectMergeRule.java | 7 +++++++ .../apache/calcite/test/CoreQuidemTest2.java | 1 - .../apache/calcite/test/RelOptRulesTest.java | 12 ++++++++++++ .../apache/calcite/test/RelOptRulesTest.xml | 18 ++++++++++++++++++ 4 files changed, 37 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/calcite/rel/rules/ProjectMergeRule.java b/core/src/main/java/org/apache/calcite/rel/rules/ProjectMergeRule.java index 4608a8c8bc59..8441699ad4a8 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/ProjectMergeRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/ProjectMergeRule.java @@ -92,6 +92,13 @@ public ProjectMergeRule(boolean force, ProjectFactory projectFactory) { final Project bottomProject = call.rel(1); final RelBuilder relBuilder = call.builder(); + // Do not merge projects if any of them has correlation variables. + // Merging would lose the correlation context needed for proper query execution. + if (!topProject.getVariablesSet().isEmpty() + || !bottomProject.getVariablesSet().isEmpty()) { + return; + } + // If one or both projects are permutations, short-circuit the complex logic // of building a RexProgram. final Permutation topPermutation = topProject.getPermutation(); diff --git a/core/src/test/java/org/apache/calcite/test/CoreQuidemTest2.java b/core/src/test/java/org/apache/calcite/test/CoreQuidemTest2.java index f0887b71891e..865aec400ae8 100644 --- a/core/src/test/java/org/apache/calcite/test/CoreQuidemTest2.java +++ b/core/src/test/java/org/apache/calcite/test/CoreQuidemTest2.java @@ -46,7 +46,6 @@ public static void main(String[] args) throws Exception { // TODO: The following files involves UNNEST and LEFT_MARK JOIN paths.remove("sql/measure.iq"); - paths.remove("sql/unnest.iq"); paths.remove("sql/some.iq"); paths.remove("sql/sub-query.iq"); paths.remove("sql/measure-paper.iq"); diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index 94bb87651f1d..f9cedb9d81ea 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -12165,6 +12165,18 @@ private void checkLoptOptimizeJoinRule(LoptOptimizeJoinRule rule) { .check(); } + /** Test case of + * [CALCITE-7395] + * ProjectMergeRule incorrectly merges PROJECTs with correlation variables. */ + @Test void testProjectMergeRuleWithCorrelation() { + final String sql = "SELECT ARRAY(SELECT y + 1 FROM UNNEST(s.x) y)\n" + + "FROM (SELECT ARRAY[1,2,3] as x) s"; + + sql(sql) + .withRule(CoreRules.PROJECT_MERGE) + .checkUnchanged(); + } + /** Test case of * [CALCITE-7369] * ProjectToWindowRule loses column alias when optimizing OVER window queries. */ diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index 67e86d45b6b0..785bce783988 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -11821,6 +11821,24 @@ LogicalProject(DEPTNO=[$0]) LogicalAggregate(group=[{}], DUMMY=[COUNT()]) LogicalProject(EMPNO=[$0]) LogicalTableScan(table=[[scott, EMP]]) +]]> + + + + + + + + From 85cd5c163993e6de16f7ffb601438babdd3ba955 Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Sun, 25 Jan 2026 22:07:29 +0800 Subject: [PATCH 129/562] [CALCITE-7396] PruneEmptyRules does not support LEFT_MARK JOIN --- .../calcite/rel/rules/PruneEmptyRules.java | 14 +++++++++++ .../apache/calcite/test/RelOptRulesTest.java | 16 +++++++++++++ .../apache/calcite/test/RelOptRulesTest.xml | 24 +++++++++++++++++++ core/src/test/resources/sql/new-decorr.iq | 20 +++++++++++++++- 4 files changed, 73 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/calcite/rel/rules/PruneEmptyRules.java b/core/src/main/java/org/apache/calcite/rel/rules/PruneEmptyRules.java index 221cfac09df0..0ee85558aa67 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/PruneEmptyRules.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/PruneEmptyRules.java @@ -42,12 +42,14 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexDynamicParam; import org.apache.calcite.rex.RexLiteral; +import org.apache.calcite.rex.RexNode; import org.apache.calcite.tools.RelBuilder; import org.apache.calcite.tools.RelBuilderFactory; import org.immutables.value.Value; import java.math.BigDecimal; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.function.Predicate; @@ -260,6 +262,7 @@ private static boolean isEmpty(RelNode node) { *

  • Join(Scan(Emp), Empty, RIGHT) becomes Empty *
  • Join(Scan(Emp), Empty, SEMI) becomes Empty *
  • Join(Scan(Emp), Empty, ANTI) becomes Scan(Emp) + *
  • Join(Scan(Emp), Empty, LEFT_MARK) becomes Project(Scan(Emp), FALSE) * */ public static final RelOptRule JOIN_RIGHT_INSTANCE = @@ -566,6 +569,17 @@ public interface JoinRightEmptyRuleConfig extends PruneEmptyRule.Config { call.transformTo(join.getLeft()); return; } + if (join.getJoinType() == JoinRelType.LEFT_MARK) { + // In case of left mark join with empty right: Join(X, Empty, LEFT_MARK) + // The mark column is always FALSE when right is empty + relBuilder.push(left); + List projects = new ArrayList<>(relBuilder.fields()); + projects.add(relBuilder.literal(false)); + relBuilder.project(projects) + .convert(join.getRowType(), true); + call.transformTo(relBuilder.build()); + return; + } call.transformTo(relBuilder.push(join).empty().build()); } }; diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index f9cedb9d81ea..a7dd7dd614e1 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -12165,6 +12165,22 @@ private void checkLoptOptimizeJoinRule(LoptOptimizeJoinRule rule) { .check(); } + /** Test case of + * [CALCITE-7396] + * PruneEmptyRules does not support LEFT_MARK JOIN. */ + @Test void testPruneEmptyRuleForLeftMarkJoin() { + final String sql = "select * from dept" + + " where deptno not in (select deptno from emp where false)"; + + sql(sql) + .withPreRule( + CoreRules.FILTER_SUB_QUERY_TO_MARK_CORRELATE, + CoreRules.FILTER_REDUCE_EXPRESSIONS, + PruneEmptyRules.PROJECT_INSTANCE) + .withRule(PruneEmptyRules.JOIN_RIGHT_INSTANCE) + .check(); + } + /** Test case of * [CALCITE-7395] * ProjectMergeRule incorrectly merges PROJECTs with correlation variables. */ diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index 785bce783988..977367cf1905 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -12087,6 +12087,30 @@ LogicalProject(COL1=[$2], COL2=[$3]) + + + + + + + + + + + diff --git a/core/src/test/resources/sql/new-decorr.iq b/core/src/test/resources/sql/new-decorr.iq index 8c1bc0b23d35..9257f05495d8 100644 --- a/core/src/test/resources/sql/new-decorr.iq +++ b/core/src/test/resources/sql/new-decorr.iq @@ -169,7 +169,7 @@ EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t7):INTEGER], expr#9=[10], ex !plan !} -# # This case comes from scalar.iq [CALCITE-709] +# This case comes from scalar.iq [CALCITE-709] # Aggregate functions do not support type promotion, so a cast is added to pass the test. select deptno, (select sum(cast(empno as bigint)) from "scott".emp where deptno = dept.deptno limit 0) as x from "scott".dept; +--------+---+ @@ -205,4 +205,22 @@ EnumerableCalc(expr#0..3=[{inputs}], DEPTNO=[$t0], EXPR$0=[$t2]) !plan !} +# [CALCITE-7396] PruneEmptyRules does not support LEFT_MARK JOIN +# This case comes from sub-query.iq +!use post +select * from dept where deptno not in (select deptno from emp where false); ++--------+-------------+ +| DEPTNO | DNAME | ++--------+-------------+ +| 10 | Sales | +| 20 | Marketing | +| 30 | Engineering | +| 40 | Empty | ++--------+-------------+ +(4 rows) + +!ok +EnumerableValues(tuples=[[{ 10, 'Sales ' }, { 20, 'Marketing ' }, { 30, 'Engineering' }, { 40, 'Empty ' }]]) +!plan + # End new-decorr.iq From 6490a31c30d00fc88fa0e6419bef56318363ae99 Mon Sep 17 00:00:00 2001 From: nobigo Date: Fri, 23 Jan 2026 19:54:26 +0800 Subject: [PATCH 130/562] [CALCITE-5578] RelOptRulesTest testAggregateCaseToFilter optimized plan not semantically equivalent to the original one after conversion --- .../rel/rel2sql/RelToSqlConverterTest.java | 40 ++++++++++++ core/src/test/resources/sql/blank.iq | 61 +++++++++++++++++++ 2 files changed, 101 insertions(+) diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java index 0c8e009c6e7d..d161f774b5e7 100644 --- a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java +++ b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java @@ -11003,6 +11003,46 @@ private void checkLiteral2(String expression, String expected) { .ok(expected); } + + /** Test case of + * [CALCITE-5578] + * RelOptRulesTest testAggregateCaseToFilter optimized plan not semantically + * equivalent to the original one after conversion. */ + @Test void testAggregateCaseToFilter() { + final String sql = "select\n" + + " sum(sal) as sum_sal,\n" + + " count(distinct case\n" + + " when job = 'CLERK'\n" + + " then deptno else null end) as count_distinct_clerk,\n" + + " sum(case when deptno = 10 then sal end) as sum_sal_d10,\n" + + " sum(case when deptno = 20 then sal else 0 end) as sum_sal_d20,\n" + + " sum(case when deptno = 30 then 1 else 0 end) as count_d30,\n" + + " count(case when deptno = 40 then 'x' end) as count_d40,\n" + + " sum(case when deptno = 45 then 1 end) as count_d45,\n" + + " sum(case when deptno = 50 then 1 else null end) as count_d50,\n" + + " sum(case when deptno = 60 then null end) as sum_null_d60,\n" + + " sum(case when deptno = 70 then null else 1 end) as sum_null_d70,\n" + + " count(case when deptno = 20 then 1 end) as count_d20\n" + + "from emp"; + final String expected = "SELECT" + + " SUM(\"SAL\") AS \"SUM_SAL\"," + + " COUNT(DISTINCT \"DEPTNO\") FILTER (WHERE \"JOB\" = 'CLERK' IS TRUE) AS \"COUNT_DISTINCT_CLERK\"," + + " SUM(\"SAL\") FILTER (WHERE CAST(\"DEPTNO\" AS INTEGER) = 10 IS TRUE) AS \"SUM_SAL_D10\"," + + " SUM(CASE WHEN CAST(\"DEPTNO\" AS INTEGER) = 20 THEN CAST(\"SAL\" AS DECIMAL(12, 2)) ELSE 0.00 END) AS \"SUM_SAL_D20\"," + + " SUM(CASE WHEN CAST(\"DEPTNO\" AS INTEGER) = 30 THEN 1 ELSE 0 END) AS \"COUNT_D30\"," + + " COUNT(*) FILTER (WHERE CAST(\"DEPTNO\" AS INTEGER) = 40 IS TRUE) AS \"COUNT_D40\"," + + " SUM(1) FILTER (WHERE CAST(\"DEPTNO\" AS INTEGER) = 45 IS TRUE) AS \"COUNT_D45\"," + + " SUM(1) FILTER (WHERE CAST(\"DEPTNO\" AS INTEGER) = 50 IS TRUE) AS \"COUNT_D50\"," + + " SUM(CAST(NULL AS DECIMAL(19, 9))) AS \"SUM_NULL_D60\"," + + " SUM(1) FILTER (WHERE CAST(\"DEPTNO\" AS INTEGER) = 70 IS NOT TRUE) AS \"SUM_NULL_D70\"," + + " COUNT(*) FILTER (WHERE CAST(\"DEPTNO\" AS INTEGER) = 20 IS TRUE) AS \"COUNT_D20\"\n" + + "FROM \"scott\".\"EMP\""; + sql(sql) + .schema(CalciteAssert.SchemaSpec.SCOTT) + .optimize(RuleSets.ofList(CoreRules.AGGREGATE_CASE_TO_FILTER), null) + .ok(expected); + } + @Test void testAggregateFilterToCase() { final String query = "select\n" + " sum(sal) filter(where deptno = 10) as sum_match,\n" diff --git a/core/src/test/resources/sql/blank.iq b/core/src/test/resources/sql/blank.iq index 206707287ec3..c92dc0f095a3 100644 --- a/core/src/test/resources/sql/blank.iq +++ b/core/src/test/resources/sql/blank.iq @@ -233,4 +233,65 @@ from complex_t; !ok +# Test case for [CALCITE-5578] RelOptRulesTest testAggregateCaseToFilter optimized plan not semantically equivalent to the original one after conversion + +CREATE TABLE EMP ( + EMPNO INTEGER, + DEPTNO INTEGER, + ENAME VARCHAR(20), + JOB VARCHAR(20), + MGR INTEGER, + HIREDATE DATE, + SAL INTEGER, + COMM INTEGER, + SLACKER INTEGER +); + +(0 rows modified) + +!update + +INSERT INTO EMP VALUES (0, 70, '-2147483649', '-6721455509335307966', 0, '1970-01-01', 0, 0, 1); + +(1 row modified) + +!update + +select + sum(sal) as sum_sal, + count(distinct case + when job = 'CLERK' + then deptno else null end) as count_distinct_clerk, + sum(case when deptno = 10 then sal end) as sum_sal_d10, + sum(case when deptno = 20 then sal else 0 end) as sum_sal_d20, + sum(case when deptno = 30 then 1 else 0 end) as count_d30, + count(case when deptno = 40 then 'x' end) as count_d40, + sum(case when deptno = 45 then 1 end) as count_d45, + sum(case when deptno = 50 then 1 else null end) as count_d50, + sum(case when deptno = 60 then null end) as sum_null_d60, + sum(case when deptno = 70 then null else 1 end) as sum_null_d70, + count(case when deptno = 20 then 1 end) as count_d20 +from emp; ++---------+----------------------+-------------+-------------+-----------+-----------+-----------+-----------+--------------+--------------+-----------+ +| SUM_SAL | COUNT_DISTINCT_CLERK | SUM_SAL_D10 | SUM_SAL_D20 | COUNT_D30 | COUNT_D40 | COUNT_D45 | COUNT_D50 | SUM_NULL_D60 | SUM_NULL_D70 | COUNT_D20 | ++---------+----------------------+-------------+-------------+-----------+-----------+-----------+-----------+--------------+--------------+-----------+ +| 0 | 0 | | 0 | 0 | 0 | | | | | 0 | ++---------+----------------------+-------------+-------------+-----------+-----------+-----------+-----------+--------------+--------------+-----------+ +(1 row) + +!ok + +# Test same sql after apply AGGREGATE_CASE_TO_FILTER + +SELECT SUM("SAL") AS "SUM_SAL", COUNT(DISTINCT "DEPTNO") FILTER (WHERE "JOB" = 'CLERK' IS TRUE) AS "COUNT_DISTINCT_CLERK", SUM("SAL") FILTER (WHERE CAST("DEPTNO" AS INTEGER) = 10 IS TRUE) AS "SUM_SAL_D10", SUM(CASE WHEN CAST("DEPTNO" AS INTEGER) = 20 THEN CAST("SAL" AS DECIMAL(12, 2)) ELSE 0.00 END) AS "SUM_SAL_D20", SUM(CASE WHEN CAST("DEPTNO" AS INTEGER) = 30 THEN 1 ELSE 0 END) AS "COUNT_D30", COUNT(*) FILTER (WHERE CAST("DEPTNO" AS INTEGER) = 40 IS TRUE) AS "COUNT_D40", SUM(1) FILTER (WHERE CAST("DEPTNO" AS INTEGER) = 45 IS TRUE) AS "COUNT_D45", SUM(1) FILTER (WHERE CAST("DEPTNO" AS INTEGER) = 50 IS TRUE) AS "COUNT_D50", SUM(CAST(NULL AS DECIMAL(19, 9))) AS "SUM_NULL_D60", SUM(1) FILTER (WHERE CAST("DEPTNO" AS INTEGER) = 70 IS NOT TRUE) AS "SUM_NULL_D70", COUNT(*) FILTER (WHERE CAST("DEPTNO" AS INTEGER) = 20 IS TRUE) AS "COUNT_D20" +FROM "EMP"; ++---------+----------------------+-------------+-------------+-----------+-----------+-----------+-----------+--------------+--------------+-----------+ +| SUM_SAL | COUNT_DISTINCT_CLERK | SUM_SAL_D10 | SUM_SAL_D20 | COUNT_D30 | COUNT_D40 | COUNT_D45 | COUNT_D50 | SUM_NULL_D60 | SUM_NULL_D70 | COUNT_D20 | ++---------+----------------------+-------------+-------------+-----------+-----------+-----------+-----------+--------------+--------------+-----------+ +| 0 | 0 | | 0.00 | 0 | 0 | | | | | 0 | ++---------+----------------------+-------------+-------------+-----------+-----------+-----------+-----------+--------------+--------------+-----------+ +(1 row) + +!ok + # End blank.iq From 2374f9574a2164dff1d923052f05fcf5a27839c0 Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Mon, 26 Jan 2026 21:47:45 +0800 Subject: [PATCH 131/562] [CALCITE-7397] Error in simplifying join condition when creating LEFT MARK JOIN --- .../org/apache/calcite/tools/RelBuilder.java | 8 ++++- .../apache/calcite/test/RelOptRulesTest.java | 17 +++++++++ .../apache/calcite/test/RelOptRulesTest.xml | 36 +++++++++++++++++++ 3 files changed, 60 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java index a1ab24a19dc5..9551c45395fa 100644 --- a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java +++ b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java @@ -84,6 +84,7 @@ import org.apache.calcite.rex.RexShuttle; import org.apache.calcite.rex.RexSimplify; import org.apache.calcite.rex.RexSubQuery; +import org.apache.calcite.rex.RexUnknownAs; import org.apache.calcite.rex.RexUtil; import org.apache.calcite.rex.RexWindowBound; import org.apache.calcite.rex.RexWindowBounds; @@ -3315,7 +3316,12 @@ public RelBuilder join(JoinRelType joinType, RexNode condition, RelOptUtil.collapseExpandedIsNotDistinctFromExpr((RexCall) condition, getRexBuilder()); } - condition = simplifier.simplifyUnknownAsFalse(condition); + + condition = + simplifier.simplifyUnknownAs(condition, + joinType == JoinRelType.LEFT_MARK + ? RexUnknownAs.UNKNOWN + : RexUnknownAs.FALSE); } if (correlate) { final CorrelationId id = Iterables.getOnlyElement(variablesSet); diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index a7dd7dd614e1..81f07a6f6a52 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -12193,6 +12193,23 @@ private void checkLoptOptimizeJoinRule(LoptOptimizeJoinRule rule) { .checkUnchanged(); } + /** Test case of + * [CALCITE-7397] + * Error in simplifying join condition when creating LEFT MARK JOIN. */ + @Test void testMarkJoinMarkerColumnTypeNullableMismatch() { + final String sql = "select sal,\n" + + " cast(null as int) IN (\n" + + " select cast(null as int)\n" + + " from dept)\n" + + "from emp"; + + sql(sql) + .withRule(CoreRules.PROJECT_SUB_QUERY_TO_MARK_CORRELATE) + .withLateDecorrelate(true) + .withTopDownGeneralDecorrelate(true) + .check(); + } + /** Test case of * [CALCITE-7369] * ProjectToWindowRule loses column alias when optimizing OVER window queries. */ diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index 977367cf1905..103744e3118c 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -9705,6 +9705,42 @@ LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$ MultiJoin(joinFilter=[true], isFullOuterJoin=[false], joinTypes=[[INNER, LEFT]], outerJoinConditions=[[NULL, =($7, $9)]], projFields=[[{0, 1, 2, 3, 4, 5, 6, 7, 8}, {0, 1}]], postJoinFilter=[AND(LIKE($1, 'bar'), >($9, 3))]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) +]]> + + + + + + + + + + + + + + From 15c12d5698f838e9bd3d8c65b1c88a1feb21cdc2 Mon Sep 17 00:00:00 2001 From: iwanttobepowerful <745778074@qq.com> Date: Fri, 23 Jan 2026 10:37:07 +0800 Subject: [PATCH 132/562] [CALCITE-7320] AggregateProjectMergeRule throws AssertionError when Project maps multiple grouping keys to the same field --- .../rel/rules/AggregateProjectMergeRule.java | 8 ++ .../calcite/sql2rel/RelDecorrelator.java | 2 +- .../calcite/sql2rel/RelDecorrelatorTest.java | 73 +++++++++++++++++++ core/src/test/resources/sql/sub-query.iq | 22 ++++++ 4 files changed, 104 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/calcite/rel/rules/AggregateProjectMergeRule.java b/core/src/main/java/org/apache/calcite/rel/rules/AggregateProjectMergeRule.java index 7ad0d57b8472..2068b0e40c83 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/AggregateProjectMergeRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/AggregateProjectMergeRule.java @@ -110,6 +110,14 @@ public AggregateProjectMergeRule( newGroupingSets = ImmutableBitSet.ORDERING.immutableSortedCopy( ImmutableBitSet.permute(aggregate.getGroupSets(), map)); + for (int i = 0; i < newGroupingSets.size() - 1; i++) { + if (newGroupingSets.get(i).equals(newGroupingSets.get(i + 1))) { + // If the project merges two columns that are both in the grouping sets, + // we might get duplicate grouping sets. Aggregate does not allow + // duplicate grouping sets, so we abort the rule. + return null; + } + } } final ImmutableList.Builder aggCalls = diff --git a/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java b/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java index 226159104a97..f0b7ad4098db 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java @@ -992,7 +992,7 @@ private RelNode rewriteScalarAggregate(Aggregate oldRel, for (int i1 = 0; i1 < oldRel.getAggCallList().size(); i1++) { AggregateCall aggCall = oldRel.getAggCallList().get(i1); if (aggCall.getAggregation() instanceof SqlCountAggFunction) { - int index = requireNonNull(outputMap.get(i1 + oldRel.getGroupSet().size())); + int index = requireNonNull(outputMap.get(i1 + oldRel.getGroupCount())); final RexInputRef ref = RexInputRef.of(index + valueGenFieldCount, joinRowType); ImmutableList exprs = ImmutableList.of(relBuilder.isNotNull(ref), ref, relBuilder.literal(0)); diff --git a/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java b/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java index 6d1d2a974be6..6ac711e40651 100644 --- a/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java +++ b/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java @@ -1438,4 +1438,77 @@ public static Frameworks.ConfigBuilder config() { + " LogicalTableScan(table=[[scott, DEPT]])\n"; assertThat(after, hasTree(planAfter)); } + + /** Test case for [CALCITE-7320] + * AggregateProjectMergeRule throws AssertionError when Project maps multiple grouping keys + * to the same field. */ + @Test void test7320() { + final FrameworkConfig frameworkConfig = config().build(); + final RelBuilder builder = RelBuilder.create(frameworkConfig); + final RelOptCluster cluster = builder.getCluster(); + final Planner planner = Frameworks.getPlanner(frameworkConfig); + final String sql = "" + + "SELECT deptno,\n" + + " (SELECT SUM(cnt)\n" + + " FROM (\n" + + " SELECT COUNT(*) AS cnt\n" + + " FROM emp\n" + + " WHERE emp.deptno = dept.deptno\n" + + " GROUP BY GROUPING SETS ((deptno), ())\n" + + "))\n" + + "FROM dept"; + final RelNode originalRel; + try { + final SqlNode parse = planner.parse(sql); + final SqlNode validate = planner.validate(parse); + originalRel = planner.rel(validate).rel; + } catch (Exception e) { + throw TestUtil.rethrow(e); + } + + final HepProgram hepProgram = HepProgram.builder() + .addRuleCollection( + ImmutableList.of( + // SubQuery program rules + CoreRules.FILTER_SUB_QUERY_TO_CORRELATE, + CoreRules.PROJECT_SUB_QUERY_TO_CORRELATE, + CoreRules.JOIN_SUB_QUERY_TO_CORRELATE)) + .build(); + final Program program = + Programs.of(hepProgram, true, + requireNonNull(cluster.getMetadataProvider())); + final RelNode before = + program.run(cluster.getPlanner(), originalRel, cluster.traitSet(), + Collections.emptyList(), Collections.emptyList()); + final String planBefore = "" + + "LogicalProject(DEPTNO=[$0], EXPR$1=[$3])\n" + + " LogicalCorrelate(correlation=[$cor0], joinType=[left], requiredColumns=[{0}])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n" + + " LogicalAggregate(group=[{}], EXPR$0=[SUM($0)])\n" + + " LogicalProject(CNT=[$1])\n" + + " LogicalAggregate(group=[{0}], groups=[[{0}, {}]], CNT=[COUNT()])\n" + + " LogicalProject(DEPTNO=[$7])\n" + + " LogicalFilter(condition=[=($7, $cor0.DEPTNO)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + assertThat(before, hasTree(planBefore)); + + // Decorrelate without any rules, just "purely" decorrelation algorithm on RelDecorrelator + final RelNode after = + RelDecorrelator.decorrelateQuery(before, builder, RuleSets.ofList(Collections.emptyList()), + RuleSets.ofList(Collections.emptyList())); + final String planAfter = "" + + "LogicalProject(DEPTNO=[$0], EXPR$1=[$4])\n" + + " LogicalJoin(condition=[=($0, $3)], joinType=[left])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n" + + " LogicalAggregate(group=[{0}], EXPR$0=[SUM($1)])\n" + + " LogicalProject(DEPTNO1=[$0], CNT=[CASE(IS NOT NULL($3), $3, 0)])\n" + + " LogicalJoin(condition=[IS NOT DISTINCT FROM($0, $2)], joinType=[left])\n" + + " LogicalProject(DEPTNO=[$0])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n" + + " LogicalAggregate(group=[{0, 1}], groups=[[{0, 1}, {1}]], CNT=[COUNT()])\n" + + " LogicalProject(DEPTNO=[$7], DEPTNO1=[$7])\n" + + " LogicalFilter(condition=[IS NOT NULL($7)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + assertThat(after, hasTree(planAfter)); + } } diff --git a/core/src/test/resources/sql/sub-query.iq b/core/src/test/resources/sql/sub-query.iq index 9095ff305915..0d649f8558aa 100644 --- a/core/src/test/resources/sql/sub-query.iq +++ b/core/src/test/resources/sql/sub-query.iq @@ -8467,5 +8467,27 @@ ON t2.a = foo.a +---+---+---+ (1 row) +!ok + +# [CALCITE-7320] AggregateProjectMergeRule throws AssertionError when Project maps multiple grouping keys to the same field +SELECT deptno, + (SELECT SUM(cnt) + FROM ( + SELECT COUNT(*) AS cnt + FROM emp + WHERE emp.deptno = dept.deptno + GROUP BY GROUPING SETS ((deptno), ()) + )) +FROM dept; ++--------+--------+ +| DEPTNO | EXPR$1 | ++--------+--------+ +| 10 | 6 | +| 20 | 10 | +| 30 | 12 | +| 40 | 0 | ++--------+--------+ +(4 rows) + !ok # End sub-query.iq From e6b0b561a945a6f7661ce23f4b773680683d8a92 Mon Sep 17 00:00:00 2001 From: Heng Qian Date: Mon, 26 Jan 2026 16:44:34 +0800 Subject: [PATCH 133/562] [CALCITE-7398] Incorrect int cast in VariantNonNull#cast for BIGINT Signed-off-by: Heng Qian --- .../java/org/apache/calcite/runtime/variant/VariantNonNull.java | 2 +- .../src/main/java/org/apache/calcite/test/SqlOperatorTest.java | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/calcite/runtime/variant/VariantNonNull.java b/core/src/main/java/org/apache/calcite/runtime/variant/VariantNonNull.java index fa296ac70210..9c288e3058cc 100644 --- a/core/src/main/java/org/apache/calcite/runtime/variant/VariantNonNull.java +++ b/core/src/main/java/org/apache/calcite/runtime/variant/VariantNonNull.java @@ -286,7 +286,7 @@ public class VariantNonNull extends VariantSqlValue { break; } case BIGINT: { - long l = (int) value; + long l = (long) value; switch (type.getTypeName()) { case TINYINT: case SMALLINT: diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java index 5530349604f0..c7144c92fbdc 100644 --- a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java +++ b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java @@ -1868,6 +1868,8 @@ void testCastToBoolean(CastType castType, SqlOperatorFixture f) { "INTEGER ARRAY"); f.checkScalar("cast(cast('abc' as VARIANT) AS VARCHAR)", "abc", "VARCHAR"); f.checkScalar("cast(cast('abc' as VARIANT) AS CHAR(3))", "abc", "CHAR(3)"); + // Test for [CALCITE-7398] Incorrect int cast in VariantNonNull#cast for BIGINT + f.checkScalar("cast(cast(2147483648 as VARIANT) as DECIMAL)", "2147483648", "DECIMAL(19, 0)"); // Converting a variant to anything that does not match the runtime type returns null f.checkScalar("cast(cast(1 as VARIANT) as INTEGER)", "1", "INTEGER"); From da94c3522007694f0d4ca39bda6f6b8196eb36c1 Mon Sep 17 00:00:00 2001 From: Silun Dong Date: Mon, 26 Jan 2026 23:57:54 +0800 Subject: [PATCH 134/562] [CALCITE-7385] Support LEFT_MARK type for nested loop join in enumerable convention --- .../enumerable/EnumerableJoinRule.java | 10 +- .../enumerable/EnumerableNestedLoopJoin.java | 39 +++++ .../apache/calcite/util/BuiltInMethod.java | 4 + .../test/enumerable/EnumerableJoinTest.java | 146 ++++++++++++++++++ .../calcite/linq4j/DefaultEnumerable.java | 7 + .../calcite/linq4j/EnumerableDefaults.java | 55 +++++++ .../calcite/linq4j/ExtendedEnumerable.java | 13 ++ 7 files changed, 265 insertions(+), 9 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableJoinRule.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableJoinRule.java index 08a301da5aa9..57a6778e0d09 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableJoinRule.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableJoinRule.java @@ -21,14 +21,10 @@ import org.apache.calcite.rel.convert.ConverterRule; import org.apache.calcite.rel.core.Join; import org.apache.calcite.rel.core.JoinInfo; -import org.apache.calcite.rel.core.JoinRelType; import org.apache.calcite.rel.logical.LogicalJoin; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexUtil; -import org.apache.calcite.util.Bug; - -import org.checkerframework.checker.nullness.qual.Nullable; import java.util.ArrayList; import java.util.Arrays; @@ -52,7 +48,7 @@ protected EnumerableJoinRule(Config config) { super(config); } - @Override public @Nullable RelNode convert(RelNode rel) { + @Override public RelNode convert(RelNode rel) { Join join = (Join) rel; List newInputs = new ArrayList<>(); for (RelNode input : join.getInputs()) { @@ -96,10 +92,6 @@ protected EnumerableJoinRule(Config config) { join.getVariablesSet(), join.getJoinType()); } - if (!Bug.TODO_FIXED && join.getJoinType() == JoinRelType.LEFT_MARK) { - // TODO Support LEFT MARK type for nested loop join - return null; - } return EnumerableNestedLoopJoin.create( left, right, diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableNestedLoopJoin.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableNestedLoopJoin.java index de539ad574cf..545329726fb0 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableNestedLoopJoin.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableNestedLoopJoin.java @@ -150,6 +150,45 @@ public static EnumerableNestedLoopJoin create( } @Override public Result implement(EnumerableRelImplementor implementor, Prefer pref) { + switch (joinType) { + case LEFT_MARK: + return implementNLMarkJoin(implementor, pref); + default: + return implementNLJoin(implementor, pref); + } + } + + private Result implementNLMarkJoin(EnumerableRelImplementor implementor, Prefer pref) { + final BlockBuilder builder = new BlockBuilder(); + final Result leftResult = + implementor.visitChild(this, 0, (EnumerableRel) left, pref); + Expression leftExpression = + builder.append("left", leftResult.block); + final Result rightResult = + implementor.visitChild(this, 1, (EnumerableRel) right, pref); + Expression rightExpression = + builder.append("right", rightResult.block); + final PhysType physType = + PhysTypeImpl.of(implementor.getTypeFactory(), + getRowType(), + pref.preferArray()); + final Expression predicate = + EnumUtils.generatePredicate(implementor, getCluster().getRexBuilder(), left, right, + leftResult.physType, rightResult.physType, condition, true); + return implementor.result( + physType, + builder.append( + Expressions.call( + leftExpression, + BuiltInMethod.LEFT_MARK_NESTED_LOOP_JOIN.method, + Expressions.list( + rightExpression, + predicate, + EnumUtils.markJoinSelector(physType, leftResult.physType)))) + .toBlock()); + } + + private Result implementNLJoin(EnumerableRelImplementor implementor, Prefer pref) { final BlockBuilder builder = new BlockBuilder(); final Result leftResult = implementor.visitChild(this, 0, (EnumerableRel) left, pref); diff --git a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java index 8b96db847aa8..6adf1aa3946d 100644 --- a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java +++ b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java @@ -259,6 +259,10 @@ public enum BuiltInMethod { EqualityComparer.class, Predicate2.class), NESTED_LOOP_JOIN(EnumerableDefaults.class, "nestedLoopJoin", Enumerable.class, Enumerable.class, Predicate2.class, Function2.class, JoinType.class), + LEFT_MARK_NESTED_LOOP_JOIN(ExtendedEnumerable.class, "leftMarkNestedLoopJoin", + Enumerable.class, // inner enumerable + NullablePredicate2.class, // non-equi predicate that can return NULL + Function2.class), // result selector CORRELATE_JOIN(ExtendedEnumerable.class, "correlateJoin", JoinType.class, Function1.class, Function2.class), CORRELATE_BATCH_JOIN(EnumerableDefaults.class, "correlateBatchJoin", diff --git a/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableJoinTest.java b/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableJoinTest.java index a93bd9469232..56393e184a12 100644 --- a/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableJoinTest.java +++ b/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableJoinTest.java @@ -24,12 +24,19 @@ import org.apache.calcite.interpreter.Bindables; import org.apache.calcite.plan.RelOptPlanner; import org.apache.calcite.rel.core.JoinRelType; +import org.apache.calcite.rel.metadata.DefaultRelMetadataProvider; +import org.apache.calcite.rel.rules.CoreRules; import org.apache.calcite.runtime.Hook; import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.test.CalciteAssert; import org.apache.calcite.test.schemata.hr.HierarchySchema; import org.apache.calcite.test.schemata.hr.HrSchema; import org.apache.calcite.test.schemata.hr.HrSchemaBig; +import org.apache.calcite.tools.Program; +import org.apache.calcite.tools.Programs; +import org.apache.calcite.util.Holder; + +import com.google.common.collect.ImmutableList; import org.junit.jupiter.api.Test; @@ -440,6 +447,145 @@ private void checkMergeJoinWithCompositeKeyAndNullValues(boolean bigSchema, Join "empid=5; name=Emp5"); } + /** Test case for + * [CALCITE-7385] + * Support LEFT_MARK type for nested loop join in enumerable convention. */ + @Test void testLeftMarkJoinBasedNestedLoop() { + Program subQuery = + Programs.hep( + ImmutableList.of(CoreRules.PROJECT_SUB_QUERY_TO_MARK_CORRELATE), + true, + DefaultRelMetadataProvider.INSTANCE); + Program subQueryWithoutMarkJoin = + Programs.hep( + ImmutableList.of(CoreRules.PROJECT_SUB_QUERY_TO_CORRELATE), + true, + DefaultRelMetadataProvider.INSTANCE); + Program toCalc = + Programs.hep( + ImmutableList.of(CoreRules.PROJECT_TO_CALC, CoreRules.FILTER_TO_CALC, + CoreRules.CALC_MERGE), + true, + DefaultRelMetadataProvider.INSTANCE); + Program enumerableImpl = Programs.ofRules(EnumerableRules.ENUMERABLE_RULES); + + // case1: left mark join from uncorrelated SOME subquery + CalciteAssert.AssertQuery test1 = + tester(false, new HrSchema()).query( + "WITH t1(id) as (VALUES (1), (2), (NULL)), t2(id) as (VALUES (2), (3)) " + + "select id, id >= SOME(select id from t2) as marker from t1"); + // result of new subquery removal and decorrelation algorithms + test1 + .withHook(Hook.PROGRAM, (Consumer>) program -> { + program.set(Programs.sequence(subQuery, toCalc, enumerableImpl)); + }) + .explainHookMatches( + "EnumerableNestedLoopJoin(condition=[>=($0, $1)], joinType=[left_mark])\n" + + " EnumerableValues(tuples=[[{ 1 }, { 2 }, { null }]])\n" + + " EnumerableCalc(expr#0=[{inputs}], id=[$t0])\n" + + " EnumerableValues(tuples=[[{ 2 }, { 3 }]])\n") + .returnsUnordered( + "id=1; marker=false", + "id=2; marker=true", + "id=null; marker=null"); + // result of the old + test1 + .withHook(Hook.PROGRAM, (Consumer>) program -> { + program.set(Programs.sequence(subQueryWithoutMarkJoin, toCalc, enumerableImpl)); + }) + .returnsUnordered( + "id=1; marker=false", + "id=2; marker=true", + "id=null; marker=null"); + + // case2: left mark join whose condition is simplified to a NULL constant + CalciteAssert.AssertQuery test2 = + tester(false, new HrSchema()).query( + "WITH t1(id) as (VALUES (1), (2), (NULL)), t2(id) as (VALUES (2), (3), (null)) " + + "select id, cast(null as int) = SOME(select cast(id as int) as id from t2) " + + "as marker from t1"); + // result of new subquery removal and decorrelation algorithms + test2 + .withHook(Hook.PROGRAM, (Consumer>) program -> { + program.set(Programs.sequence(subQuery, toCalc, enumerableImpl)); + }) + .explainHookMatches( + "EnumerableNestedLoopJoin(condition=[null:BOOLEAN], joinType=[left_mark])\n" + + " EnumerableValues(tuples=[[{ 1 }, { 2 }, { null }]])\n" + + " EnumerableCalc(expr#0=[{inputs}], id=[$t0])\n" + + " EnumerableValues(tuples=[[{ 2 }, { 3 }, { null }]])\n") + .returnsUnordered( + "id=1; marker=null", + "id=2; marker=null", + "id=null; marker=null"); + // result of the old + test2 + .withHook(Hook.PROGRAM, (Consumer>) program -> { + program.set(Programs.sequence(subQueryWithoutMarkJoin, toCalc, enumerableImpl)); + }) + .returnsUnordered( + "id=1; marker=null", + "id=2; marker=null", + "id=null; marker=null"); + + // case3: left mark join from uncorrelated EXISTS subquery + CalciteAssert.AssertQuery test3 = + tester(false, new HrSchema()).query( + "WITH t1(id) as (VALUES (1), (2), (NULL)), t2(id) as (VALUES (2), (3), (NULL)) " + + "select id, EXISTS(select id from t2) as marker from t1"); + // result of new subquery removal and decorrelation algorithms + test3 + .withHook(Hook.PROGRAM, (Consumer>) program -> { + program.set(Programs.sequence(subQuery, toCalc, enumerableImpl)); + }) + .explainHookMatches( + "EnumerableNestedLoopJoin(condition=[true], joinType=[left_mark])\n" + + " EnumerableValues(tuples=[[{ 1 }, { 2 }, { null }]])\n" + + " EnumerableValues(tuples=[[{ 2 }, { 3 }, { null }]])\n") + .returnsUnordered( + "id=1; marker=true", + "id=2; marker=true", + "id=null; marker=true"); + // result of the old + test3 + .withHook(Hook.PROGRAM, (Consumer>) program -> { + program.set(Programs.sequence(subQueryWithoutMarkJoin, toCalc, enumerableImpl)); + }) + .returnsUnordered( + "id=1; marker=true", + "id=2; marker=true", + "id=null; marker=true"); + + // case4: left mark join from uncorrelated EXISTS subquery that is empty + CalciteAssert.AssertQuery test4 = + tester(false, new HrSchema()).query( + "WITH t1(id) as (VALUES (1), (2), (NULL)), t2(id) as (VALUES (2), (3), (NULL)) " + + "select id, EXISTS(select id from t2 where false) as marker from t1"); + // result of new subquery removal and decorrelation algorithms + test4 + .withHook(Hook.PROGRAM, (Consumer>) program -> { + program.set(Programs.sequence(subQuery, toCalc, enumerableImpl)); + }) + .explainHookMatches( + "EnumerableNestedLoopJoin(condition=[true], joinType=[left_mark])\n" + + " EnumerableValues(tuples=[[{ 1 }, { 2 }, { null }]])\n" + + " EnumerableCalc(expr#0=[{inputs}], expr#1=[false], EXPR$0=[$t0], $condition=[$t1])\n" + + " EnumerableValues(tuples=[[{ 2 }, { 3 }, { null }]])\n") + .returnsUnordered( + "id=1; marker=false", + "id=2; marker=false", + "id=null; marker=false"); + // result of the old + test4 + .withHook(Hook.PROGRAM, (Consumer>) program -> { + program.set(Programs.sequence(subQueryWithoutMarkJoin, toCalc, enumerableImpl)); + }) + .returnsUnordered( + "id=1; marker=false", + "id=2; marker=false", + "id=null; marker=false"); + } + private CalciteAssert.AssertThat tester(boolean forceDecorrelate, Object schema) { return CalciteAssert.that() diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/DefaultEnumerable.java b/linq4j/src/main/java/org/apache/calcite/linq4j/DefaultEnumerable.java index f90c9f112d0e..8a45548d3107 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/DefaultEnumerable.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/DefaultEnumerable.java @@ -448,6 +448,13 @@ protected OrderedQueryable asOrderedQueryable() { nonEquiPredicate, equiPredicate); } + @Override public Enumerable leftMarkNestedLoopJoin( + Enumerable inner, + NullablePredicate2 predicate, + Function2 resultSelector) { + return EnumerableDefaults.leftMarkNestedLoopJoin(getThis(), inner, predicate, resultSelector); + } + @Override public Enumerable correlateJoin( JoinType joinType, Function1> inner, Function2 resultSelector) { diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java b/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java index 96828708f7f0..28f4a1185e25 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java @@ -1965,6 +1965,61 @@ static Enumerable leftMarkHash }; } + /** + * The implementation of left mark join based on nested loop. + * + * @param outer Left input + * @param inner Right input + * @param predicate Non-equi predicate that can return NULL + * @param resultSelector Function that concats the row of left input and marker + */ + public static Enumerable leftMarkNestedLoopJoin( + final Enumerable outer, final Enumerable inner, + final NullablePredicate2 predicate, + final Function2 resultSelector) { + return new AbstractEnumerable() { + @Override public Enumerator enumerator() { + return new Enumerator() { + Enumerator outers = outer.enumerator(); + @Nullable Boolean marker = false; + + @Override public TResult current() { + return resultSelector.apply(outers.current(), marker); + } + + @Override public boolean moveNext() { + if (!outers.moveNext()) { + return false; + } + marker = false; + final TSource outerRow = outers.current(); + try (Enumerator inners = inner.enumerator()) { + while (inners.moveNext()) { + final TInner innerRow = inners.current(); + Boolean predicateMatched = predicate.apply(outerRow, innerRow); + if (predicateMatched == null) { + marker = null; + } else if (predicateMatched) { + marker = true; + break; + } + } + } + return true; + } + + @Override public void reset() { + outers.reset(); + } + + @Override public void close() { + outers.close(); + } + }; + } + }; + } + /** * For each row of the {@code outer} enumerable returns the correlated rows * from the {@code inner} enumerable. diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedEnumerable.java b/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedEnumerable.java index 67887e7d087b..982ab0ca85a9 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedEnumerable.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedEnumerable.java @@ -686,6 +686,19 @@ Enumerable leftMarkHashJoin(Enumerable< NullablePredicate2 nonEquiPredicate, NullablePredicate2 equiPredicate); + /** + * The implementation of left mark join based on nested loop. + * + * @param inner Inner enumerable + * @param predicate Non-equi predicate that can return NULL + * @param resultSelector Function that concat the row of the current enumerable and + * marker + * @see #leftMarkHashJoin + */ + Enumerable leftMarkNestedLoopJoin(Enumerable inner, + NullablePredicate2 predicate, + Function2 resultSelector); + /** * For each row of the current enumerable returns the correlated rows * from the {@code inner} enumerable (nested loops join). From 102db6c0ce5b091dc5e68ca191966f09262be169 Mon Sep 17 00:00:00 2001 From: iwanttobepowerful <745778074@qq.com> Date: Mon, 26 Jan 2026 19:37:15 +0800 Subject: [PATCH 135/562] [CALCITE-7394] Nested sub-query with multiple levels of correlation returns incorrect results --- .../calcite/sql2rel/RelDecorrelator.java | 35 ++-- .../calcite/sql2rel/RelDecorrelatorTest.java | 132 +++++++++++++ core/src/test/resources/sql/sub-query.iq | 175 ++++++++++++++++++ 3 files changed, 330 insertions(+), 12 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java b/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java index f0b7ad4098db..47c0040fc3e5 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java @@ -954,16 +954,12 @@ private RelNode rewriteScalarAggregate(Aggregate oldRel, RelNode newRel, Map outputMap, NavigableMap corDefOutputs) { - final CorelMap localCorelMap = new CorelMapBuilder().build(oldRel); - final List corVarList = new ArrayList<>(localCorelMap.mapRefRelToCorRef.values()); - Collections.sort(corVarList); - + final List corVarList = collectExternalCorVars(oldRel); final NavigableMap valueGenCorDefOutputs = new TreeMap<>(); final RelNode valueGen = requireNonNull(createValueGenerator(corVarList, 0, valueGenCorDefOutputs)); final int valueGenFieldCount = valueGen.getRowType().getFieldCount(); - // Build join conditions final Map newProjectMap = new HashMap<>(); for (Map.Entry corDefOutput : corDefOutputs.entrySet()) { final CorDef corDef = corDefOutput.getKey(); @@ -974,6 +970,7 @@ private RelNode rewriteScalarAggregate(Aggregate oldRel, newProjectMap.put(valueGenFieldCount + rightPos, leftRef); } + // Build join conditions final List conditions = buildCorDefJoinConditions(valueGenCorDefOutputs, corDefOutputs, valueGen, newRel, relBuilder); @@ -1260,10 +1257,7 @@ private static void shiftMapping(Map mapping, int startIndex, return decorrelateRel((RelNode) rel, false, parentPropagatesNullValues); } - final CorelMap localCorelMap = new CorelMapBuilder().build(rel); - final List corVarList = new ArrayList<>(localCorelMap.mapRefRelToCorRef.values()); - Collections.sort(corVarList); - + final List corVarList = collectExternalCorVars(rel); final NavigableMap valueGenCorDefOutputs = new TreeMap<>(); final RelNode valueGen = requireNonNull(createValueGenerator(corVarList, 0, valueGenCorDefOutputs)); @@ -1958,9 +1952,7 @@ private static boolean isWidening(RelDataType type, RelDataType type1) { } // 1. Collect all CorRefs involved - final CorelMap localCorelMap = new CorelMapBuilder().build(rel); - final List corVarList = new ArrayList<>(localCorelMap.mapRefRelToCorRef.values()); - Collections.sort(corVarList); + final List corVarList = collectExternalCorVars(rel); // 2. Ensure CorVars are present in inputs (adding ValueGenerators if needed) Frame newLeftFrame = leftFrame; @@ -3849,6 +3841,25 @@ private static boolean isFieldNotNullRecursive(RelNode rel, int index) { } } + /** + * Collects all correlated variables used in the given relational expression + * that are not defined within the expression itself. + * + * @param rel The relational expression to inspect + * @return A sorted list of external correlated variables + */ + private static List collectExternalCorVars(RelNode rel) { + final CorelMap localCorelMap = new CorelMapBuilder().build(rel); + final List corVarList = new ArrayList<>(); + for (CorRef corVar : localCorelMap.mapRefRelToCorRef.values()) { + if (!localCorelMap.mapCorToCorRel.containsKey(corVar.corr)) { + corVarList.add(corVar); + } + } + Collections.sort(corVarList); + return corVarList; + } + /** * Ensures that the correlated variables in {@code allCorDefs} are present * in the output of the frame. diff --git a/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java b/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java index 6ac711e40651..7ddaeaf42027 100644 --- a/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java +++ b/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java @@ -356,6 +356,138 @@ public static Frameworks.ConfigBuilder config() { assertThat(after, hasTree(planAfter)); } + /** Test case for [CALCITE-7394] + * Nested sub-query with multiple levels of correlation returns incorrect results. */ + @Test void testNestedSubQueryWithMultiLevelCorrelation() { + final FrameworkConfig frameworkConfig = config().build(); + final RelBuilder builder = RelBuilder.create(frameworkConfig); + final RelOptCluster cluster = builder.getCluster(); + final Planner planner = Frameworks.getPlanner(frameworkConfig); + final String sql = "" + + "select d.dname,\n" + + " (select count(*)\n" + + " from emp e\n" + + " where e.deptno = d.deptno\n" + + " and exists (\n" + + " select 1\n" + + " from (values (1000), (2000), (3000)) as v(sal)\n" + + " where e.sal > v.sal\n" + + " and d.deptno * 100 < v.sal\n" + + " )\n" + + " ) as c\n" + + "from dept d\n" + + "order by d.dname"; + final RelNode originalRel; + try { + final SqlNode parse = planner.parse(sql); + final SqlNode validate = planner.validate(parse); + originalRel = planner.rel(validate).rel; + } catch (Exception e) { + throw TestUtil.rethrow(e); + } + + final HepProgram hepProgram = HepProgram.builder() + .addRuleCollection( + ImmutableList.of( + // SubQuery program rules + CoreRules.FILTER_SUB_QUERY_TO_CORRELATE, + CoreRules.PROJECT_SUB_QUERY_TO_CORRELATE, + CoreRules.JOIN_SUB_QUERY_TO_CORRELATE)) + .build(); + final Program program = + Programs.of(hepProgram, true, + requireNonNull(cluster.getMetadataProvider())); + final RelNode before = + program.run(cluster.getPlanner(), originalRel, cluster.traitSet(), + Collections.emptyList(), Collections.emptyList()); + final String planBefore = "" + + "LogicalSort(sort0=[$0], dir0=[ASC])\n" + + " LogicalProject(DNAME=[$1], C=[$3])\n" + + " LogicalCorrelate(correlation=[$cor0], joinType=[left], requiredColumns=[{0}])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n" + + " LogicalAggregate(group=[{}], EXPR$0=[COUNT()])\n" + + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7])\n" + + " LogicalFilter(condition=[=($7, $cor0.DEPTNO)])\n" + + " LogicalCorrelate(correlation=[$cor1], joinType=[inner], requiredColumns=[{5}])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalAggregate(group=[{0}])\n" + + " LogicalProject(i=[true])\n" + + " LogicalFilter(condition=[AND(>(CAST($cor1.SAL):DECIMAL(12, 2), CAST($0):DECIMAL(12, 2) NOT NULL), <(*($cor0.DEPTNO, 100), $0))])\n" + + " LogicalValues(tuples=[[{ 1000 }, { 2000 }, { 3000 }]])\n"; + assertThat(before, hasTree(planBefore)); + + // Decorrelate without any rules, just "purely" decorrelation algorithm on RelDecorrelator + final RelNode after = + RelDecorrelator.decorrelateQuery(before, builder, RuleSets.ofList(Collections.emptyList()), + RuleSets.ofList(Collections.emptyList())); + // before fix: + // + // LogicalSort(sort0=[$0], dir0=[ASC]) + // LogicalProject(DNAME=[$1], C=[$7]) + // LogicalJoin(condition=[AND(=($0, $5), =($4, $6))], joinType=[left]) + // LogicalProject(DEPTNO=[$0], DNAME=[$1], LOC=[$2], DEPTNO0=[$0], $f4=[*($0, 100)]) + // LogicalTableScan(table=[[scott, DEPT]]) + // LogicalProject(DEPTNO8=[$0], $f4=[$1], EXPR$0=[CASE(IS NOT NULL($5), $5, 0)]) + // LogicalJoin(condition=[AND(IS NOT DISTINCT FROM($0, $3), + // IS NOT DISTINCT FROM($1, $4))], joinType=[left]) + // LogicalJoin(condition=[true], joinType=[inner]) // <---- error part + // LogicalProject(DEPTNO=[$0], $f4=[*($0, 100)]) + // LogicalTableScan(table=[[scott, DEPT]]) + // LogicalAggregate(group=[{0}]) // <---- error part + // LogicalProject(SAL0=[CAST($5):DECIMAL(12, 2)]) // <---- error part + // LogicalTableScan(table=[[scott, EMP]]) // <---- error part + // LogicalAggregate(group=[{0, 1}], EXPR$0=[COUNT()]) + // LogicalProject(DEPTNO8=[$7], $f4=[$9]) + // LogicalFilter(condition=[IS NOT NULL($7)]) + // LogicalProject(..., DEPTNO=[$7], i=[$11], $f4=[$9]) + // LogicalJoin(condition=[=($8, $10)], joinType=[inner]) + // LogicalProject(..., SAL0=[CAST($5):DECIMAL(12, 2)]) + // LogicalTableScan(table=[[scott, EMP]]) + // LogicalProject($f4=[$0], SAL0=[$1], $f2=[true]) + // LogicalAggregate(group=[{0, 1}]) + // LogicalProject($f4=[$1], SAL0=[$2]) + // LogicalJoin(condition=[AND(>($2, CAST($0):DECIMAL(12, 2) NOT NULL), + // <($1, $0))], joinType=[inner]) + // LogicalValues(tuples=[[{ 1000 }, { 2000 }, { 3000 }]]) + // LogicalJoin(condition=[true], joinType=[inner]) + // LogicalAggregate(group=[{0}]) + // LogicalProject($f4=[*($0, 100)]) + // LogicalTableScan(table=[[scott, DEPT]]) + // LogicalAggregate(group=[{0}]) + // LogicalProject(SAL0=[CAST($5):DECIMAL(12, 2)]) + // LogicalTableScan(table=[[scott, EMP]]) + final String planAfter = "" + + "LogicalSort(sort0=[$0], dir0=[ASC])\n" + + " LogicalProject(DNAME=[$1], C=[$7])\n" + + " LogicalJoin(condition=[AND(=($0, $5), =($4, $6))], joinType=[left])\n" + + " LogicalProject(DEPTNO=[$0], DNAME=[$1], LOC=[$2], DEPTNO0=[$0], $f4=[*($0, 100)])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n" + + " LogicalProject(DEPTNO8=[$0], $f4=[$1], EXPR$0=[CASE(IS NOT NULL($4), $4, 0)])\n" + + " LogicalJoin(condition=[AND(IS NOT DISTINCT FROM($0, $2), IS NOT DISTINCT FROM($1, $3))], joinType=[left])\n" + + " LogicalProject(DEPTNO=[$0], $f4=[*($0, 100)])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n" + + " LogicalAggregate(group=[{0, 1}], EXPR$0=[COUNT()])\n" + + " LogicalProject(DEPTNO8=[$7], $f4=[$9])\n" + + " LogicalFilter(condition=[IS NOT NULL($7)])\n" + + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7], i=[$11], $f4=[$9])\n" + + " LogicalJoin(condition=[=($8, $10)], joinType=[inner])\n" + + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7], SAL0=[CAST($5):DECIMAL(12, 2)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalProject($f4=[$0], SAL0=[$1], $f2=[true])\n" + + " LogicalAggregate(group=[{0, 1}])\n" + + " LogicalProject($f4=[$1], SAL0=[$2])\n" + + " LogicalJoin(condition=[AND(>($2, CAST($0):DECIMAL(12, 2) NOT NULL), <($1, $0))], joinType=[inner])\n" + + " LogicalValues(tuples=[[{ 1000 }, { 2000 }, { 3000 }]])\n" + + " LogicalJoin(condition=[true], joinType=[inner])\n" + + " LogicalAggregate(group=[{0}])\n" + + " LogicalProject($f4=[*($0, 100)])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n" + + " LogicalAggregate(group=[{0}])\n" + + " LogicalProject(SAL0=[CAST($5):DECIMAL(12, 2)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + assertThat(after, hasTree(planAfter)); + } + /** Test case for [CALCITE-7297] * The result is incorrect when the GROUP BY key in a subquery is a RexFieldAccess. */ @Test void testSkipsRedundantValueGenerator() { diff --git a/core/src/test/resources/sql/sub-query.iq b/core/src/test/resources/sql/sub-query.iq index 0d649f8558aa..a9440f6f43b9 100644 --- a/core/src/test/resources/sql/sub-query.iq +++ b/core/src/test/resources/sql/sub-query.iq @@ -5617,6 +5617,181 @@ ORDER BY deptno; !ok +# [CALCITE-7394] Nested sub-query with multiple levels of correlation returns incorrect results +select d.dname, + (select count(*) + from emp e + where e.deptno = d.deptno + and e.sal > ( + select min(s.losal) + from (VALUES (1, 700, 1200), (2, 1201, 1400), (3, 1401, 2000), (4, 2001, 3000), (5, 3001, 9999)) AS s(grade, losal, hisal) + where e.sal BETWEEN s.losal AND s.hisal + and s.hisal > d.deptno * 10 + ) + ) as high_paid_count +from dept d +order by d.dname; ++------------+-----------------+ +| DNAME | HIGH_PAID_COUNT | ++------------+-----------------+ +| ACCOUNTING | 3 | +| OPERATIONS | 0 | +| RESEARCH | 5 | +| SALES | 6 | ++------------+-----------------+ +(4 rows) + +!ok + +# [CALCITE-7394] Nested sub-query with multiple levels of correlation returns incorrect results +select e.ename +from emp e +where e.sal > ( + select avg(e2.sal) + from emp e2 + where e2.deptno = e.deptno + and exists ( + select 1 + from (values (7369, 20)) as b(empno, deptno) + where b.empno = e2.empno + and b.deptno = e.deptno + ) +) +and e.sal < 2000 +order by e.ename; ++-------+ +| ENAME | ++-------+ +| ADAMS | ++-------+ +(1 row) + +!ok + +# [CALCITE-7394] Nested sub-query with multiple levels of correlation returns incorrect results +select d.deptno +from dept d +where exists ( + select 1 + from emp e + where e.deptno = d.deptno + and exists ( + select 1 + from (VALUES (1, 700, 1200), (2, 1201, 1400), (3, 1401, 2000), (4, 2001, 3000), (5, 3001, 9999)) AS s(grade, losal, hisal) + where s.grade = 1 + and s.hisal >= e.sal + and s.losal <= d.deptno * 20 + ) +) +order by d.deptno; ++--------+ +| DEPTNO | ++--------+ ++--------+ +(0 rows) + +!ok + +# [CALCITE-7394] Nested sub-query with multiple levels of correlation returns incorrect results +select e.ename +from emp e +where e.deptno in ( + select d.deptno + from dept d + where d.deptno = e.deptno and d.deptno = 10 + union + select d.deptno + from dept d + where d.deptno = e.deptno + and exists ( + select 1 + from emp e2 + where e2.deptno = d.deptno + and e2.empno = e.empno + and e2.sal > 2000 + ) +) +order by e.ename; ++--------+ +| ENAME | ++--------+ +| BLAKE | +| CLARK | +| FORD | +| JONES | +| KING | +| MILLER | +| SCOTT | ++--------+ +(7 rows) + +!ok + +# [CALCITE-7394] Nested sub-query with multiple levels of correlation returns incorrect results +select e.ename +from emp e +where exists ( + select 1 + from dept d + join emp e2 on d.deptno = e2.deptno + where d.deptno = e.deptno + and exists ( + select 1 + from (values (10), (20), (30)) as v(deptno) + where v.deptno = e2.deptno + and v.deptno = e.deptno + ) + and e2.empno = e.empno +) +order by e.ename; ++--------+ +| ENAME | ++--------+ +| ADAMS | +| ALLEN | +| BLAKE | +| CLARK | +| FORD | +| JAMES | +| JONES | +| KING | +| MARTIN | +| MILLER | +| SCOTT | +| SMITH | +| TURNER | +| WARD | ++--------+ +(14 rows) + +!ok + +# [CALCITE-7394] Nested sub-query with multiple levels of correlation returns incorrect results +select d.dname, + (select count(*) + from emp e + where e.deptno = d.deptno + and exists ( + select 1 + from (values (1000), (2000), (3000)) as v(sal) + where e.sal > v.sal + and d.deptno * 100 < v.sal + ) + ) as c +from dept d +order by d.dname; ++------------+---+ +| DNAME | C | ++------------+---+ +| ACCOUNTING | 2 | +| OPERATIONS | 0 | +| RESEARCH | 0 | +| SALES | 0 | ++------------+---+ +(4 rows) + +!ok + # [CALCITE-7303] Subqueries cannot be decorrelated if filter condition have multi CorrelationId SELECT deptno FROM emp e From e6405a4872fd12bb79dfd8bd7bde210c8e96812f Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Tue, 27 Jan 2026 20:08:21 +0800 Subject: [PATCH 136/562] [CALCITE-7400] PruneJoinSingleValue rule causes type mismatch in IN --- .../rel/rules/SingleValuesOptimizationRules.java | 6 ++++-- core/src/test/resources/sql/new-decorr.iq | 12 ++++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rel/rules/SingleValuesOptimizationRules.java b/core/src/main/java/org/apache/calcite/rel/rules/SingleValuesOptimizationRules.java index 3212e98686da..d0e53be44d12 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/SingleValuesOptimizationRules.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/SingleValuesOptimizationRules.java @@ -284,9 +284,11 @@ static Predicate isJoinTransformable(boolean isLeft) { || jn.getJoinType() == JoinRelType.FULL; if (isLeft) { - return jn -> !(jn.getJoinType() == JoinRelType.LEFT || isFullOrAntiJoin.test(jn)); + return jn -> !(jn.getJoinType() == JoinRelType.LEFT + || jn.getJoinType() == JoinRelType.LEFT_MARK + || isFullOrAntiJoin.test(jn)); } else { - return jn -> !(jn.getJoinType() == JoinRelType.RIGHT || isFullOrAntiJoin.test(jn)); + return jn -> !(jn.getJoinType() == JoinRelType.RIGHT || isFullOrAntiJoin.test(jn)); } } diff --git a/core/src/test/resources/sql/new-decorr.iq b/core/src/test/resources/sql/new-decorr.iq index 9257f05495d8..529bca6d1bb8 100644 --- a/core/src/test/resources/sql/new-decorr.iq +++ b/core/src/test/resources/sql/new-decorr.iq @@ -223,4 +223,16 @@ select * from dept where deptno not in (select deptno from emp where false); EnumerableValues(tuples=[[{ 10, 'Sales ' }, { 20, 'Marketing ' }, { 30, 'Engineering' }, { 40, 'Empty ' }]]) !plan +# [CALCITE-7400] PruneJoinSingleValue rule causes type mismatch in IN +# This case comes from sub-query.iq [CALCITE-4756] +select 1 in (values(null), (null)); ++--------+ +| EXPR$0 | ++--------+ +| | ++--------+ +(1 row) + +!ok + # End new-decorr.iq From 4ac9b8599167d32d0f2cb75687a840e1998f3373 Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Thu, 29 Jan 2026 08:53:20 +0800 Subject: [PATCH 137/562] [CALCITE-5787] The RelMdInputFieldsUsed is introduced to track the usage of input fields --- .../calcite/rel/metadata/BuiltInMetadata.java | 28 +--- .../rel/metadata/RelMdInputFieldsUsed.java | 144 ++++++++++++----- .../rel/metadata/RelMetadataQuery.java | 4 +- .../calcite/rel/rules/SemiJoinRule.java | 2 +- .../apache/calcite/test/RelMetadataTest.java | 151 ++++++++++++------ 5 files changed, 212 insertions(+), 117 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/BuiltInMetadata.java b/core/src/main/java/org/apache/calcite/rel/metadata/BuiltInMetadata.java index 0f297a95a56c..9c571dfddcd3 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/BuiltInMetadata.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/BuiltInMetadata.java @@ -77,19 +77,8 @@ interface Handler extends MetadataHandler { } /** - * Metadata that identifies, per input, which fields of each - * input are referenced by a relational expression ({@link RelNode}). - * Here, "referenced" means the input field is used by the parent - * RelNode. Operators such as Filter, while not inherently consuming - * all input fields, must preserve them since parent RelNodes may depend on - * these fields. Thus, Filter is regarded as utilizing all fields. - * - *

    For a relational expression with N inputs, this returns an - * {@link ImmutableList} of length N. Each element is an - * {@link ImmutableBitSet} with bits set for zero-based field ordinals of - * that input which are referenced by the expression. - * - *

    Returns empty {@link ImmutableList} if information cannot be determined. + * Metadata that identifies which columns of its inputs are referenced by a + * relational expression. */ public interface InputFieldsUsed extends Metadata { MetadataDef DEF = @@ -97,19 +86,18 @@ public interface InputFieldsUsed extends Metadata { BuiltInMethod.INPUT_FIELDS_USED.method); /** - * Returns, for each input of this relational expression, a bit set of the - * referenced field ordinals. + * Returns which columns of its inputs are referenced by this relational + * expression. * - * @return an {@link ImmutableList} of {@link ImmutableBitSet} of length N - * where N is the number of inputs, or empty {@link ImmutableList} - * if the information is not available + * @return an {@link ImmutableBitSet} where bits correspond to input column + * ordinals from the first input to the last */ - ImmutableList getInputFieldsUsed(); + ImmutableBitSet getInputFieldsUsed(); /** Handler API. */ @FunctionalInterface interface Handler extends MetadataHandler { - ImmutableList getInputFieldsUsed(RelNode r, + ImmutableBitSet getInputFieldsUsed(RelNode r, RelMetadataQuery mq); @Override default MetadataDef getDef() { diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdInputFieldsUsed.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdInputFieldsUsed.java index 771ed7e5fc80..2d2d7d6a83b0 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdInputFieldsUsed.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdInputFieldsUsed.java @@ -20,23 +20,41 @@ import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Aggregate; import org.apache.calcite.rel.core.Calc; +import org.apache.calcite.rel.core.Correlate; import org.apache.calcite.rel.core.Filter; import org.apache.calcite.rel.core.Join; import org.apache.calcite.rel.core.JoinRelType; import org.apache.calcite.rel.core.Project; import org.apache.calcite.rel.core.SetOp; +import org.apache.calcite.rel.core.Sort; import org.apache.calcite.rel.core.TableScan; +import org.apache.calcite.rel.core.Window; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexProgram; import org.apache.calcite.util.ImmutableBitSet; -import com.google.common.collect.ImmutableList; - import java.util.List; import java.util.Set; /** * Metadata provider to determine which input fields are used by a RelNode. + * + *

    A field is considered "used" if it is referenced by the relational + * expression. The result is an {@link ImmutableBitSet} where bits correspond to + * input column ordinals. + * + *

    Examples: + *

      + *
    • For an {@link Aggregate}, "used" fields are those in the group set or + * referenced in aggregate functions. see {@link RelOptUtil#getAllFields}
      • + *
    • For a {@link Join}, it is the union of "used" fields from both inputs + * (shifted appropriately for the right input). For SEMI and ANTI joins, fields + * from the right input are not considered "used" as they are not projected to + * the output
      • + *
    + * + * @see BuiltInMetadata.InputFieldsUsed + * @see RelMetadataQuery#getInputFieldsUsed(RelNode) */ public class RelMdInputFieldsUsed implements MetadataHandler { @@ -48,77 +66,115 @@ public class RelMdInputFieldsUsed return BuiltInMetadata.InputFieldsUsed.DEF; } - public ImmutableList getInputFieldsUsed(RelNode rel, - RelMetadataQuery mq) { - ImmutableList.Builder builder = ImmutableList.builder(); - rel.getInputs().forEach(input -> { - builder.addAll(mq.getInputFieldsUsed(input)); - }); - return builder.build(); + /** Catch-all implementation for + * {@link BuiltInMetadata.InputFieldsUsed#getInputFieldsUsed()}, + * invoked using reflection. + * + * @see org.apache.calcite.rel.metadata.RelMetadataQuery#getInputFieldsUsed(RelNode) + */ + public ImmutableBitSet getInputFieldsUsed(RelNode rel, RelMetadataQuery mq) { + // By default, a RelNode uses all of its input fields. + return getAllInputFieldsUsed(rel); } - public ImmutableList getInputFieldsUsed(TableScan scan, - RelMetadataQuery mq) { + public ImmutableBitSet getInputFieldsUsed(TableScan scan, RelMetadataQuery mq) { final BuiltInMetadata.InputFieldsUsed.Handler handler = scan.getTable().unwrap(BuiltInMetadata.InputFieldsUsed.Handler.class); if (handler != null) { return handler.getInputFieldsUsed(scan, mq); } final int fieldCount = scan.getRowType().getFieldCount(); - return ImmutableList.of(ImmutableBitSet.range(fieldCount)); + return ImmutableBitSet.range(fieldCount); } - public ImmutableList getInputFieldsUsed(Project project, - RelMetadataQuery mq) { - final ImmutableBitSet bits = RelOptUtil.InputFinder.bits(project.getProjects(), null); - return ImmutableList.of(bits); + public ImmutableBitSet getInputFieldsUsed(Project project, RelMetadataQuery mq) { + // Project involves column trimming, returning only the columns that are used. + return RelOptUtil.InputFinder.bits(project.getProjects(), null); } - public ImmutableList getInputFieldsUsed(Filter filter, - RelMetadataQuery mq) { - return mq.getInputFieldsUsed(filter.getInput()); + public ImmutableBitSet getInputFieldsUsed(Filter filter, RelMetadataQuery mq) { + return getAllFieldsUsed(filter); } - public ImmutableList getInputFieldsUsed(Calc calc, - RelMetadataQuery mq) { + public ImmutableBitSet getInputFieldsUsed(Sort sort, RelMetadataQuery mq) { + return getAllFieldsUsed(sort); + } + + public ImmutableBitSet getInputFieldsUsed(Window window, RelMetadataQuery mq) { + return getAllFieldsUsed(window); + } + + public ImmutableBitSet getInputFieldsUsed(Calc calc, RelMetadataQuery mq) { final RexProgram program = calc.getProgram(); final List expandedProjects = program.expandList(program.getProjectList()); final RexNode cond = program.getCondition() == null ? null : program.expandLocalRef(program.getCondition()); - final ImmutableBitSet bits = RelOptUtil.InputFinder.bits(expandedProjects, cond); - return ImmutableList.of(bits); - } - public ImmutableList getInputFieldsUsed(Join join, - RelMetadataQuery mq) { - List leftInputFieldsUsed = mq.getInputFieldsUsed(join.getLeft()); - List rightInputFieldsUsed = mq.getInputFieldsUsed(join.getRight()); - assert leftInputFieldsUsed.size() == 1 && rightInputFieldsUsed.size() == 1; + // Same as Project. + return RelOptUtil.InputFinder.bits(expandedProjects, cond); + } - ImmutableBitSet rightUsedBits = rightInputFieldsUsed.get(0); + public ImmutableBitSet getInputFieldsUsed(Join join, RelMetadataQuery mq) { + // Computes the union of fields used by both inputs. For SEMI and ANTI joins, + // fields from the right input are excluded as they are not projected to the output. + final ImmutableBitSet leftInputFieldsUsed = getAllFieldsUsed(join.getLeft()); if (join.getJoinType() == JoinRelType.SEMI - || join.getJoinType() == JoinRelType.ANTI) { - rightUsedBits = ImmutableBitSet.of(); + || join.getJoinType() == JoinRelType.ANTI) { + return leftInputFieldsUsed; } - return ImmutableList.of(leftInputFieldsUsed.get(0), rightUsedBits); + final ImmutableBitSet rightInputFieldsUsedShifted = + getAllFieldsUsed(join.getRight(), + join.getLeft().getRowType().getFieldCount()); + return leftInputFieldsUsed.union(rightInputFieldsUsedShifted); } - public ImmutableList getInputFieldsUsed(SetOp setOp, - RelMetadataQuery mq) { - final ImmutableList.Builder builder = ImmutableList.builder(); - for (RelNode input : setOp.getInputs()) { - ImmutableList inputFieldsBits = mq.getInputFieldsUsed(input); - assert inputFieldsBits.size() == 1; - builder.add(inputFieldsBits.get(0)); + public ImmutableBitSet getInputFieldsUsed(SetOp setOp, RelMetadataQuery mq) { + return getAllInputFieldsUsed(setOp); + } + + public ImmutableBitSet getInputFieldsUsed(Aggregate agg, RelMetadataQuery mq) { + Set fields = RelOptUtil.getAllFields(agg); + return ImmutableBitSet.of(fields); + } + + public ImmutableBitSet getInputFieldsUsed(Correlate correlate, RelMetadataQuery mq) { + // Computes the union of fields referenced by both inputs. For SEMI and ANTI + // correlates, fields from the right input are excluded from the projection. + final ImmutableBitSet leftInputFieldsUsed = getAllFieldsUsed(correlate.getLeft()); + if (correlate.getJoinType() == JoinRelType.SEMI + || correlate.getJoinType() == JoinRelType.ANTI) { + return leftInputFieldsUsed; + } + + final ImmutableBitSet rightInputFieldsUsedShifted = + getAllFieldsUsed(correlate.getRight(), + correlate.getLeft().getRowType().getFieldCount()); + return leftInputFieldsUsed.union(rightInputFieldsUsedShifted); + } + + // ~ Private helper methods ------------------------------------------------ + + /** + * Returns a bitset of all fields used by all inputs of a {@link RelNode}, + * shifted by the cumulative field count of preceding inputs. + */ + private static ImmutableBitSet getAllInputFieldsUsed(RelNode rel) { + ImmutableBitSet.Builder builder = ImmutableBitSet.builder(); + int offset = 0; + for (RelNode input : rel.getInputs()) { + builder.addAll(getAllFieldsUsed(input, offset)); + offset += input.getRowType().getFieldCount(); } return builder.build(); } - public ImmutableList getInputFieldsUsed(Aggregate agg, - RelMetadataQuery mq) { - Set fields = RelOptUtil.getAllFields(agg); - return ImmutableList.of(ImmutableBitSet.of(fields)); + private static ImmutableBitSet getAllFieldsUsed(RelNode rel, int offset) { + return ImmutableBitSet.range(rel.getRowType().getFieldCount()).shift(offset); + } + + private static ImmutableBitSet getAllFieldsUsed(RelNode rel) { + return getAllFieldsUsed(rel, 0); } } diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMetadataQuery.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMetadataQuery.java index f86f1de868fd..aeeefa9763a3 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMetadataQuery.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMetadataQuery.java @@ -1064,9 +1064,9 @@ public ArrowSet getFDs(RelNode rel) { } /** - * Returns the input fields are used by a RelNode. + * Returns which columns of its inputs are referenced by a relational expression. */ - public ImmutableList getInputFieldsUsed(RelNode rel) { + public ImmutableBitSet getInputFieldsUsed(RelNode rel) { for (;;) { try { return inputFieldsUsedHandler.getInputFieldsUsed(rel, this); diff --git a/core/src/main/java/org/apache/calcite/rel/rules/SemiJoinRule.java b/core/src/main/java/org/apache/calcite/rel/rules/SemiJoinRule.java index 427ea979bd38..8b011ed08bf1 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/SemiJoinRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/SemiJoinRule.java @@ -143,7 +143,7 @@ protected void perform(RelOptRuleCall call, @Nullable RelNode topRel, /** Returns a bit set of the input fields used by a relational expression. */ private static ImmutableBitSet getUsedFields(RelNode rel) { final RelMetadataQuery mq = rel.getCluster().getMetadataQuery(); - return ImmutableBitSet.union(mq.getInputFieldsUsed(rel)); + return mq.getInputFieldsUsed(rel); } /** SemiJoinRule that matches a Aggregate on top of a Join with an Aggregate diff --git a/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java b/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java index 110c0d0f51fb..f438a0ad9423 100644 --- a/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java @@ -53,7 +53,6 @@ import org.apache.calcite.rel.core.Minus; import org.apache.calcite.rel.core.Project; import org.apache.calcite.rel.core.Sample; -import org.apache.calcite.rel.core.SetOp; import org.apache.calcite.rel.core.Sort; import org.apache.calcite.rel.core.TableModify; import org.apache.calcite.rel.core.TableScan; @@ -941,80 +940,132 @@ final RelMetadataFixture sql(String sql) { final RelBuilder relBuilder = RelBuilderTest.createBuilder(); relBuilder.scan("EMP"); relBuilder.scan("DEPT"); - // Build semi-join on DEPTNO relBuilder.semiJoin( relBuilder.equals(relBuilder.field(2, 0, "DEPTNO"), - relBuilder.field(2, 1, "DEPTNO"))); - final Join join = (Join) relBuilder.build(); - final RelMetadataQuery mq = join.getCluster().getMetadataQuery(); - final List inputFields = mq.getInputFieldsUsed(join); + relBuilder.field(2, 1, "DEPTNO"))); + final RelNode rel = relBuilder.build(); + assertThat(Util.toLinux(RelOptUtil.toString(rel)), + is("" + + "LogicalJoin(condition=[=($7, $8)], joinType=[semi])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n")); + + final RelMetadataQuery mq = rel.getCluster().getMetadataQuery(); + final ImmutableBitSet inputFields = mq.getInputFieldsUsed(rel); // For SEMI join expect left input fields to be all columns of left input - // and right input fields to be empty (semi-join does not require right output). - final int leftCount = join.getLeft().getRowType().getFieldCount(); - assertThat(inputFields, hasSize(2)); - assertThat(inputFields.get(0), equalTo(ImmutableBitSet.range(leftCount))); - assertThat(inputFields.get(1).isEmpty(), is(true)); + assertThat(inputFields, equalTo(ImmutableBitSet.range(8))); } - @Test void testInputFieldsUsedUnionSetOp() { - final RelBuilder builder = RelBuilderTest.createBuilder(); - builder.scan("DEPT").project(builder.field(1)); // name - builder.scan("EMP").project(builder.field(2)); // job - builder.union(true); - final SetOp setOp = (SetOp) builder.build(); - final RelMetadataQuery mq = setOp.getCluster().getMetadataQuery(); - final List inputFields = mq.getInputFieldsUsed(setOp); - assertThat( - inputFields, equalTo( - ImmutableList.of(ImmutableBitSet.of(1), ImmutableBitSet.of(2)))); + @Test void testInputFieldsUsedJoin() { + final RelBuilder relBuilder = RelBuilderTest.createBuilder(); + final RelNode rel = relBuilder + .scan("EMP") + .project(relBuilder.field(0), relBuilder.field(7)) + .scan("DEPT") + .project(relBuilder.field(0), relBuilder.field(1)) + .join(JoinRelType.INNER, + relBuilder.equals(relBuilder.field(2, 0, "DEPTNO"), + relBuilder.field(2, 1, "DEPTNO"))) + .build(); + + assertThat(Util.toLinux(RelOptUtil.toString(rel)), + is("LogicalJoin(condition=[=($1, $2)], joinType=[inner])\n" + + " LogicalProject(EMPNO=[$0], DEPTNO=[$7])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalProject(DEPTNO=[$0], DNAME=[$1])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n")); + + final RelMetadataQuery mq = rel.getCluster().getMetadataQuery(); + final ImmutableBitSet inputFields = mq.getInputFieldsUsed(rel); + + // For normal join expect all columns of both inputs to be used. + assertThat(inputFields, equalTo(ImmutableBitSet.range(4))); + } + + @Test void testInputFieldsUsedUnion() { + final String sql = "select deptno from dept union all select deptno from emp"; + final RelNode rel = sql(sql).toRel(); + assertThat(Util.toLinux(RelOptUtil.toString(rel)), + is("" + + "LogicalUnion(all=[true])\n" + + " LogicalProject(DEPTNO=[$0])\n" + + " LogicalTableScan(table=[[CATALOG, SALES, DEPT]])\n" + + " LogicalProject(DEPTNO=[$7])\n" + + " LogicalTableScan(table=[[CATALOG, SALES, EMP]])\n")); + + final RelMetadataQuery mq = rel.getCluster().getMetadataQuery(); + final ImmutableBitSet inputFields = mq.getInputFieldsUsed(rel); + + // Expected result columns: [0, 1] + // this representing the sole field from input 0 and the sole field from input 1. + assertThat(inputFields, equalTo(ImmutableBitSet.of(0, 1))); } @Test void testInputFieldsUsedProject() { - final RelBuilder builder = RelBuilderTest.createBuilder(); - final RelNode project = builder - .scan("EMP") - .project(builder.field(0), builder.field(2)) - .build(); - final RelMetadataQuery mq = project.getCluster().getMetadataQuery(); - final java.util.List inputFields = mq.getInputFieldsUsed(project); + final String sql = "select empno, job from emp"; + final RelNode rel = sql(sql).toRel(); + assertThat(Util.toLinux(RelOptUtil.toString(rel)), + is("" + + "LogicalProject(EMPNO=[$0], JOB=[$2])\n" + + " LogicalTableScan(table=[[CATALOG, SALES, EMP]])\n")); + + final RelMetadataQuery mq = rel.getCluster().getMetadataQuery(); + final ImmutableBitSet inputFields = mq.getInputFieldsUsed(rel); - assertThat(inputFields, hasSize(1)); - assertThat(inputFields.get(0), equalTo(ImmutableBitSet.of(0, 2))); + assertThat(inputFields, equalTo(ImmutableBitSet.of(0, 2))); } @Test void testInputFieldsUsedFilter() { - final RelBuilder builder = RelBuilderTest.createBuilder(); - final RelNode filter = builder - .scan("EMP") - .filter(builder.equals(builder.field(2), builder.literal(10))) - .build(); - final RelMetadataQuery mq = filter.getCluster().getMetadataQuery(); - final List inputFields = mq.getInputFieldsUsed(filter); + final String sql = "select * from emp where sal > 1000"; + final RelNode rel = sql(sql).toRel(); + assertThat(Util.toLinux(RelOptUtil.toString(rel)), + is("" + + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7], SLACKER=[$8])\n" + + " LogicalFilter(condition=[>($5, 1000)])\n" + + " LogicalTableScan(table=[[CATALOG, SALES, EMP]])\n")); - final int fieldCount = filter.getInput(0).getRowType().getFieldCount(); - assertThat(inputFields, hasSize(1)); - assertThat(inputFields.get(0), equalTo(ImmutableBitSet.range(fieldCount))); + final RelMetadataQuery mq = rel.getCluster().getMetadataQuery(); + final RelNode filter = rel.getInput(0); + final ImmutableBitSet inputFields = mq.getInputFieldsUsed(filter); + + assertThat(inputFields, equalTo(ImmutableBitSet.range(9))); } @Test void testInputFieldsUsedCalc() { - final RelBuilder builder = RelBuilderTest.createBuilder(); - final RelNode proj = builder - .scan("EMP") - .project(builder.field(0), builder.field(2)) - .build(); + final String sql = "select empno, job from emp"; + final RelNode rel = sql(sql).toRel(); final HepProgram program = new HepProgramBuilder() .addRuleInstance(CoreRules.PROJECT_TO_CALC) .build(); final HepPlanner planner = new HepPlanner(program); - planner.setRoot(proj); + planner.setRoot(rel); final RelNode calc = planner.findBestExp(); assertThat(calc, instanceOf(Calc.class)); + assertThat(Util.toLinux(RelOptUtil.toString(calc)), + is("" + + "LogicalCalc(expr#0..8=[{inputs}], EMPNO=[$t0], JOB=[$t2])\n" + + " LogicalTableScan(table=[[CATALOG, SALES, EMP]])\n")); final RelMetadataQuery mq = calc.getCluster().getMetadataQuery(); - final List inputFields = mq.getInputFieldsUsed(calc); - assertThat(inputFields, hasSize(1)); - assertThat(inputFields.get(0), equalTo(ImmutableBitSet.of(0, 2))); + final ImmutableBitSet inputFields = mq.getInputFieldsUsed(calc); + + assertThat(inputFields, equalTo(ImmutableBitSet.of(0, 2))); + } + + @Test void testInputFieldsUsedAggregate() { + final String sql = "select deptno, sum(sal) from emp group by deptno"; + final RelNode rel = sql(sql).toRel(); + assertThat(Util.toLinux(RelOptUtil.toString(rel)), + is("" + + "LogicalAggregate(group=[{0}], EXPR$1=[SUM($1)])\n" + + " LogicalProject(DEPTNO=[$7], SAL=[$5])\n" + + " LogicalTableScan(table=[[CATALOG, SALES, EMP]])\n")); + + final RelMetadataQuery mq = rel.getCluster().getMetadataQuery(); + final ImmutableBitSet inputFields = mq.getInputFieldsUsed(rel); + + assertThat(inputFields, equalTo(ImmutableBitSet.of(0, 1))); } // ---------------------------------------------------------------------- From 980af1d40411f6ac69f3187a5b9e3c516692ee10 Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Thu, 22 Jan 2026 09:42:11 +0800 Subject: [PATCH 138/562] [CALCITE-4765] Complex correlated EXISTS sub-queries used as scalar subqueries can return wrong results --- .../calcite/test/SqlToRelConverterTest.java | 23 ++++ .../calcite/test/SqlToRelConverterTest.xml | 44 ++++++++ core/src/test/resources/sql/blank.iq | 102 ++++++++++++++++++ 3 files changed, 169 insertions(+) diff --git a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java index 28173513d8d1..f0d068d5bb23 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java @@ -2048,6 +2048,29 @@ void checkCorrelatedMapSubQuery(boolean expand) { sql(sql).withDecorrelate(true).withExpand(false).ok(); } + /** Test case for [CALCITE-4765] + * Complex correlated EXISTS sub-queries used as scalar subqueries + * can return wrong results. */ + @Test void testExistsCorrelatedSubQuery() { + final String sql = "select * from emp e1 where exists (\n" + + " select * from (\n" + + " select e2.deptno from emp e2\n" + + " where e2.comm = e1.comm) as table3\n" + + " where table3.deptno <> e1.deptno)"; + sql(sql).withDecorrelate(false).ok(); + } + + /** Test case for [CALCITE-4765] + * Complex correlated EXISTS sub-queries used as scalar subqueries + * can return wrong results. */ + @Test void testExistsCorrelatedSubQuery2() { + final String sql = "SELECT *, EXISTS(select * from (\n" + + " SELECT e2.deptno FROM emp e2 where e1.comm = e2.comm) as table3\n" + + " where table3.deptno <> e1.deptno)\n" + + "from emp e1"; + sql(sql).withDecorrelate(false).ok(); + } + @Test void testExistsCorrelatedLimit() { final String sql = "select*from emp where exists (\n" + " select 1 from dept where emp.deptno=dept.deptno limit 1)"; diff --git a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml index bd829189cff3..c8c136c97f04 100644 --- a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml @@ -2241,6 +2241,50 @@ LogicalSort(fetch=[1]) LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) })], variablesSet=[[$cor0]]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +]]> + + + + + e1.deptno)]]> + + + ($0, $cor1.DEPTNO)]) + LogicalProject(DEPTNO=[$7]) + LogicalFilter(condition=[=($6, $cor1.COMM)]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +]]> + + + + + e1.deptno) +from emp e1]]> + + + ($0, $cor1.DEPTNO)]) + LogicalProject(DEPTNO=[$7]) + LogicalFilter(condition=[=($cor1.COMM, $6)]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) ]]> diff --git a/core/src/test/resources/sql/blank.iq b/core/src/test/resources/sql/blank.iq index c92dc0f095a3..a84952e7f933 100644 --- a/core/src/test/resources/sql/blank.iq +++ b/core/src/test/resources/sql/blank.iq @@ -294,4 +294,106 @@ FROM "EMP"; !ok +# [CALCITE-4765] Complex correlated EXISTS sub-queries used as scalar subqueries can return wrong results +CREATE TABLE tmp_emps ( + empid INTEGER NOT NULL, + deptno INTEGER NOT NULL, + name VARCHAR(10) NOT NULL, + salary DECIMAL(10, 2) NOT NULL, + commission INTEGER); +(0 rows modified) + +!update +INSERT INTO tmp_emps (empid, deptno, name, salary, commission) VALUES +(100, 10, 'Bill', 10000.00, 1000), +(200, 20, 'Eric', 8000.00, 500), +(150, 10, 'Sebastian', 7000.00, NULL), +(110, 10, 'Theodore', 11500.00, 250), +(170, 30, 'Theodore', 11500.00, 250), +(140, 10, 'Sebastian', 7000.00, NULL); +(6 rows modified) + +!update +select * from tmp_emps e1 where EXISTS(select * from ( + select e2.deptno from tmp_emps e2 + where e2.commission = e1.commission) as table3 +where table3.deptno <> e1.deptno); ++-------+--------+----------+----------+------------+ +| EMPID | DEPTNO | NAME | SALARY | COMMISSION | ++-------+--------+----------+----------+------------+ +| 110 | 10 | Theodore | 11500.00 | 250 | +| 170 | 30 | Theodore | 11500.00 | 250 | ++-------+--------+----------+----------+------------+ +(2 rows) + +!ok +!if (use_old_decorr) { +EnumerableHashJoin(condition=[AND(=($1, $5), =($4, $7))], joinType=[semi]) + EnumerableTableScan(table=[[BLANK, TMP_EMPS]]) + EnumerableNestedLoopJoin(condition=[<>($1, $0)], joinType=[inner]) + EnumerableAggregate(group=[{1}]) + EnumerableTableScan(table=[[BLANK, TMP_EMPS]]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NOT NULL($t4)], DEPTNO=[$t1], COMMISSION=[$t4], $condition=[$t5]) + EnumerableTableScan(table=[[BLANK, TMP_EMPS]]) +!plan +!} + +!if (use_new_decorr) { +EnumerableHashJoin(condition=[AND(IS NOT DISTINCT FROM($1, $5), IS NOT DISTINCT FROM($4, $6))], joinType=[semi]) + EnumerableTableScan(table=[[BLANK, TMP_EMPS]]) + EnumerableCalc(expr#0..3=[{inputs}], proj#0..1=[{exprs}]) + EnumerableHashJoin(condition=[AND(=($1, $3), <>($2, $0))], joinType=[inner]) + EnumerableAggregate(group=[{1, 4}]) + EnumerableTableScan(table=[[BLANK, TMP_EMPS]]) + EnumerableCalc(expr#0..4=[{inputs}], DEPTNO=[$t1], COMMISSION=[$t4]) + EnumerableTableScan(table=[[BLANK, TMP_EMPS]]) +!plan +!} + +SELECT *, EXISTS(select * from ( + SELECT e2.deptno FROM tmp_emps e2 where e1.commission = e2.commission) as table3 + where table3.deptno <> e1.deptno) +from tmp_emps e1; ++-------+--------+-----------+----------+------------+--------+ +| EMPID | DEPTNO | NAME | SALARY | COMMISSION | EXPR$5 | ++-------+--------+-----------+----------+------------+--------+ +| 100 | 10 | Bill | 10000.00 | 1000 | false | +| 110 | 10 | Theodore | 11500.00 | 250 | true | +| 140 | 10 | Sebastian | 7000.00 | | false | +| 150 | 10 | Sebastian | 7000.00 | | false | +| 170 | 30 | Theodore | 11500.00 | 250 | true | +| 200 | 20 | Eric | 8000.00 | 500 | false | ++-------+--------+-----------+----------+------------+--------+ +(6 rows) + +!ok + +!if (use_old_decorr) { +EnumerableCalc(expr#0..7=[{inputs}], expr#8=[IS NOT NULL($t7)], proj#0..4=[{exprs}], EXPR$5=[$t8]) + EnumerableMergeJoin(condition=[AND(=($1, $5), =($4, $6))], joinType=[left]) + EnumerableSort(sort0=[$1], sort1=[$4], dir0=[ASC], dir1=[ASC]) + EnumerableTableScan(table=[[BLANK, TMP_EMPS]]) + EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[true], DEPTNO0=[$t1], COMMISSION=[$t0], $f2=[$t2]) + EnumerableAggregate(group=[{1, 2}]) + EnumerableNestedLoopJoin(condition=[<>($0, $2)], joinType=[inner]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NOT NULL($t4)], DEPTNO=[$t1], COMMISSION=[$t4], $condition=[$t5]) + EnumerableTableScan(table=[[BLANK, TMP_EMPS]]) + EnumerableAggregate(group=[{1}]) + EnumerableTableScan(table=[[BLANK, TMP_EMPS]]) +!plan +!} + +!if (use_new_decorr) { +EnumerableHashJoin(condition=[AND(IS NOT DISTINCT FROM($1, $5), IS NOT DISTINCT FROM($4, $6))], joinType=[left_mark]) + EnumerableTableScan(table=[[BLANK, TMP_EMPS]]) + EnumerableCalc(expr#0..3=[{inputs}], proj#0..1=[{exprs}]) + EnumerableHashJoin(condition=[AND(=($1, $3), <>($2, $0))], joinType=[inner]) + EnumerableAggregate(group=[{1, 4}]) + EnumerableTableScan(table=[[BLANK, TMP_EMPS]]) + EnumerableCalc(expr#0..4=[{inputs}], DEPTNO=[$t1], COMMISSION=[$t4]) + EnumerableTableScan(table=[[BLANK, TMP_EMPS]]) +!plan +!} + # End blank.iq From 2a7b687667ac52383cd4bc545fcb8897c9782f28 Mon Sep 17 00:00:00 2001 From: Silun Dong Date: Wed, 28 Jan 2026 23:07:48 +0800 Subject: [PATCH 139/562] [CALCITE-7401] Multi-level correlated subqueries cause an out-of-range error in the TopDownGeneralDecorrelator --- .../sql2rel/TopDownGeneralDecorrelator.java | 23 +++++++++++++------ core/src/test/resources/sql/new-decorr.iq | 17 ++++++++++++++ 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java b/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java index 72401e342691..068ec441e557 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java @@ -17,6 +17,7 @@ package org.apache.calcite.sql2rel; import org.apache.calcite.linq4j.function.Experimental; +import org.apache.calcite.plan.RelOptCostImpl; import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.plan.Strong; import org.apache.calcite.plan.hep.HepPlanner; @@ -66,6 +67,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.IdentityHashMap; import java.util.List; import java.util.Map; import java.util.NavigableMap; @@ -122,10 +124,10 @@ public class TopDownGeneralDecorrelator implements ReflectiveVisitor { private final NavigableSet corDefs; // a map from RelNode to whether existing correlated expressions (according to corDefs). - private final Map hasCorrelatedExpressions; + private final IdentityHashMap hasCorrelatedExpressions; // a map from RelNode to its UnnestedQuery. - private final Map mapRelToUnnestedQuery; + private final IdentityHashMap mapRelToUnnestedQuery; private final boolean hasParent; @@ -155,8 +157,8 @@ private TopDownGeneralDecorrelator( RelBuilder builder, boolean hasParent, @Nullable Set parentCorDefs, - @Nullable Map parentHasCorrelatedExpressions, - @Nullable Map parentMapRelToUnnestedQuery) { + @Nullable IdentityHashMap parentHasCorrelatedExpressions, + @Nullable IdentityHashMap parentMapRelToUnnestedQuery) { this.builder = builder; this.hasParent = hasParent; this.corDefs = new TreeSet<>(); @@ -164,10 +166,10 @@ private TopDownGeneralDecorrelator( this.corDefs.addAll(parentCorDefs); } this.hasCorrelatedExpressions = parentHasCorrelatedExpressions == null - ? new HashMap<>() + ? new IdentityHashMap<>() : parentHasCorrelatedExpressions; this.mapRelToUnnestedQuery = parentMapRelToUnnestedQuery == null - ? new HashMap<>() + ? new IdentityHashMap<>() : parentMapRelToUnnestedQuery; } @@ -202,7 +204,14 @@ public static RelNode decorrelateQuery(RelNode rel, RelBuilder builder) { CoreRules.FILTER_INTO_JOIN, CoreRules.FILTER_CORRELATE)) .build(); - HepPlanner prePlanner = new HepPlanner(preProgram); + // In scenarios with nested correlations, equivalent nodes at different nesting levels bind + // to different outer variables and therefore have different decorrelation information. We + // need to avoid equivalent nodes in the plan sharing the same object (in the form of a DAG) + // to prevent corruption of entries in mapRelToUnnestedQuery and hasCorrelatedExpressions. + // So we set noDag config of HepPlanner to TRUE. + HepPlanner prePlanner = + new HepPlanner(preProgram, null, true, + null, RelOptCostImpl.FACTORY); prePlanner.setRoot(rel); RelNode preparedRel = prePlanner.findBestExp(); diff --git a/core/src/test/resources/sql/new-decorr.iq b/core/src/test/resources/sql/new-decorr.iq index 529bca6d1bb8..9677fae713bd 100644 --- a/core/src/test/resources/sql/new-decorr.iq +++ b/core/src/test/resources/sql/new-decorr.iq @@ -235,4 +235,21 @@ select 1 in (values(null), (null)); !ok +# [CALCITE-7401] Multi-level correlated subqueries cause an out-of-range error in the TopDownGeneralDecorrelator +# This case comes from sub-query.iq [CALCITE-5789] +select deptno from dept d1 where exists ( + select 1 from dept d2 where d2.deptno = d1.deptno and exists ( + select 1 from dept d3 where d3.deptno = d2.deptno and d3.dname = d1.dname)); ++--------+ +| DEPTNO | ++--------+ +| 10 | +| 20 | +| 30 | +| 40 | ++--------+ +(4 rows) + +!ok + # End new-decorr.iq From 0f0de51a611d0a267c6becff9a68c033cf8044e0 Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Thu, 29 Jan 2026 14:27:30 +0800 Subject: [PATCH 140/562] [CALCITE-7403] Missing ENUMERABLE Convention for LogicalConditionalCorrelate --- .../EnumerableConditionalCorrelate.java | 214 ++++++++++++++++++ .../EnumerableConditionalCorrelateRule.java | 57 +++++ .../adapter/enumerable/EnumerableRules.java | 8 + .../rel/core/ConditionalCorrelate.java | 2 +- .../org/apache/calcite/tools/Programs.java | 1 + .../apache/calcite/util/BuiltInMethod.java | 4 + .../enumerable/EnumerableCorrelateTest.java | 172 ++++++++++++++ core/src/test/resources/sql/new-decorr.iq | 24 ++ .../calcite/linq4j/DefaultEnumerable.java | 7 + .../calcite/linq4j/EnumerableDefaults.java | 59 ++++- .../calcite/linq4j/ExtendedEnumerable.java | 13 ++ 11 files changed, 548 insertions(+), 13 deletions(-) create mode 100644 core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableConditionalCorrelate.java create mode 100644 core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableConditionalCorrelateRule.java diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableConditionalCorrelate.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableConditionalCorrelate.java new file mode 100644 index 000000000000..02b51a624926 --- /dev/null +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableConditionalCorrelate.java @@ -0,0 +1,214 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.adapter.enumerable; + +import org.apache.calcite.linq4j.tree.BlockBuilder; +import org.apache.calcite.linq4j.tree.Expression; +import org.apache.calcite.linq4j.tree.Expressions; +import org.apache.calcite.linq4j.tree.ParameterExpression; +import org.apache.calcite.linq4j.tree.Primitive; +import org.apache.calcite.plan.DeriveMode; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelTraitSet; +import org.apache.calcite.rel.RelCollationTraitDef; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.ConditionalCorrelate; +import org.apache.calcite.rel.core.CorrelationId; +import org.apache.calcite.rel.core.JoinRelType; +import org.apache.calcite.rel.metadata.RelMdCollation; +import org.apache.calcite.rel.metadata.RelMetadataQuery; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.util.BuiltInMethod; +import org.apache.calcite.util.ImmutableBitSet; +import org.apache.calcite.util.Pair; + +import com.google.common.collect.ImmutableList; + +import org.checkerframework.checker.nullness.qual.Nullable; + +import java.lang.reflect.Modifier; +import java.lang.reflect.Type; +import java.util.List; + +/** Implementation of {@link org.apache.calcite.rel.core.ConditionalCorrelate} in + * {@link org.apache.calcite.adapter.enumerable.EnumerableConvention enumerable calling convention}. */ +public class EnumerableConditionalCorrelate extends ConditionalCorrelate + implements EnumerableRel { + + protected EnumerableConditionalCorrelate( + RelOptCluster cluster, + RelTraitSet traits, + RelNode left, + RelNode right, + CorrelationId correlationId, + ImmutableBitSet requiredColumns, + JoinRelType joinType, + RexNode condition) { + super(cluster, traits, ImmutableList.of(), left, right, correlationId, + requiredColumns, joinType, condition); + } + + /** Creates an EnumerableConditionalCorrelate. */ + public static EnumerableConditionalCorrelate create( + RelNode left, + RelNode right, + CorrelationId correlationId, + ImmutableBitSet requiredColumns, + JoinRelType joinType, + RexNode condition) { + final RelOptCluster cluster = left.getCluster(); + final RelMetadataQuery mq = cluster.getMetadataQuery(); + final RelTraitSet traitSet = + cluster.traitSetOf(EnumerableConvention.INSTANCE) + .replaceIfs(RelCollationTraitDef.INSTANCE, + () -> RelMdCollation.enumerableCorrelate(mq, left, right, joinType)); + return new EnumerableConditionalCorrelate( + cluster, + traitSet, + left, + right, + correlationId, + requiredColumns, + joinType, + condition); + } + + @Override public EnumerableConditionalCorrelate copy( + RelTraitSet traitSet, + RelNode left, + RelNode right, + CorrelationId correlationId, + ImmutableBitSet requiredColumns, + JoinRelType joinType, + RexNode condition) { + return new EnumerableConditionalCorrelate( + getCluster(), + traitSet, + left, + right, + correlationId, + requiredColumns, + joinType, + condition); + } + + @Override public EnumerableConditionalCorrelate copy( + RelTraitSet traitSet, + RelNode left, + RelNode right, + CorrelationId correlationId, + ImmutableBitSet requiredColumns, + JoinRelType joinType) { + // This method does not provide the condition as an argument, so it should never be called + throw new RuntimeException("This method should not be called"); + } + + @Override public @Nullable Pair> passThroughTraits( + final RelTraitSet required) { + // EnumerableConditionalCorrelate traits passdown shall only pass through + // collation to left input. This is because for EnumerableConditionalCorrelate + // always uses left input as the outer loop, thus only left input can preserve ordering. + return EnumerableTraitsUtils.passThroughTraitsForJoin( + required, joinType, left.getRowType().getFieldCount(), getTraitSet()); + } + + @Override public @Nullable Pair> deriveTraits( + final RelTraitSet childTraits, final int childId) { + // should only derive traits (limited to collation for now) from left input. + return EnumerableTraitsUtils.deriveTraitsForJoin( + childTraits, childId, joinType, traitSet, right.getTraitSet()); + } + + @Override public DeriveMode getDeriveMode() { + return DeriveMode.LEFT_FIRST; + } + + @Override public Result implement(EnumerableRelImplementor implementor, + Prefer pref) { + final BlockBuilder builder = new BlockBuilder(); + final Result leftResult = + implementor.visitChild(this, 0, (EnumerableRel) left, pref); + Expression leftExpression = + builder.append( + "left", leftResult.block); + + final BlockBuilder corrBlock = new BlockBuilder(); + Type corrVarType = leftResult.physType.getJavaRowType(); + ParameterExpression corrRef; // correlate to be used in inner loop + ParameterExpression corrArg; // argument to correlate lambda (must be boxed) + if (!Primitive.is(corrVarType)) { + corrArg = + Expressions.parameter(Modifier.FINAL, + corrVarType, getCorrelVariable()); + corrRef = corrArg; + } else { + corrArg = + Expressions.parameter(Modifier.FINAL, + Primitive.box(corrVarType), "$box" + getCorrelVariable()); + corrRef = + (ParameterExpression) corrBlock.append(getCorrelVariable(), + Expressions.unbox(corrArg)); + } + + implementor.registerCorrelVariable(getCorrelVariable(), corrRef, + corrBlock, leftResult.physType); + + final Result rightResult = + implementor.visitChild(this, 1, (EnumerableRel) right, pref); + + implementor.clearCorrelVariable(getCorrelVariable()); + + // Generate the condition predicate + final Expression predicate = + EnumUtils.generatePredicate( + implementor, + getCluster().getRexBuilder(), + left, + right, + leftResult.physType, + rightResult.physType, + getCondition(), + true); + + corrBlock.add(rightResult.block); + + final PhysType physType = + PhysTypeImpl.of( + implementor.getTypeFactory(), + getRowType(), + pref.prefer(JavaRowFormat.CUSTOM)); + + if (joinType == JoinRelType.LEFT_MARK) { + // For LEFT_MARK join, use CORRELATE_LEFT_MARK_JOIN with predicate + Expression selector = + EnumUtils.markJoinSelector(physType, leftResult.physType); + + builder.append( + Expressions.call(leftExpression, BuiltInMethod.CORRELATE_LEFT_MARK_JOIN.method, + Expressions.lambda(corrBlock.toBlock(), corrArg), + predicate, + selector)); + } else { + // TODO: Support other join types. Currently, ConditionalCorrelate is only created + // when rewriting correlated IN/SOME/EXISTS subqueries, and its type is always LEFT_MARK. + throw new UnsupportedOperationException( + "EnumerableConditionalCorrelate does not support join type: " + joinType); + } + + return implementor.result(physType, builder.toBlock()); + } +} diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableConditionalCorrelateRule.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableConditionalCorrelateRule.java new file mode 100644 index 000000000000..4d1402c342ca --- /dev/null +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableConditionalCorrelateRule.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.adapter.enumerable; + +import org.apache.calcite.plan.Convention; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.convert.ConverterRule; +import org.apache.calcite.rel.logical.LogicalConditionalCorrelate; + +import org.immutables.value.Value; + +/** + * Rule which converts a {@link LogicalConditionalCorrelate} into its enumerable implementation, + * implementing conditional correlates via nested loops over enumerable inputs. + * + * @see EnumerableRules#ENUMERABLE_CONDITIONAL_CORRELATE_RULE + */ +@Value.Enclosing +public class EnumerableConditionalCorrelateRule extends ConverterRule { + /** Default configuration. */ + public static final Config DEFAULT_CONFIG = Config.INSTANCE + .withConversion(LogicalConditionalCorrelate.class, r -> true, Convention.NONE, + EnumerableConvention.INSTANCE, "EnumerableConditionalCorrelateRule") + .withRuleFactory(EnumerableConditionalCorrelateRule::new); + + /** Creates an EnumerableConditionalCorrelateRule. */ + protected EnumerableConditionalCorrelateRule(Config config) { + super(config); + } + + @Override public RelNode convert(RelNode rel) { + final LogicalConditionalCorrelate c = (LogicalConditionalCorrelate) rel; + return EnumerableConditionalCorrelate.create( + convert(c.getLeft(), c.getLeft().getTraitSet() + .replace(EnumerableConvention.INSTANCE)), + convert(c.getRight(), c.getRight().getTraitSet() + .replace(EnumerableConvention.INSTANCE)), + c.getCorrelationId(), + c.getRequiredColumns(), + c.getJoinType(), + c.getCondition()); + } +} diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableRules.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableRules.java index f33997450ca0..a470d3b7cee0 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableRules.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableRules.java @@ -65,6 +65,13 @@ private EnumerableRules() { EnumerableCorrelateRule.DEFAULT_CONFIG .toRule(EnumerableCorrelateRule.class); + /** Rule that converts a + * {@link org.apache.calcite.rel.logical.LogicalConditionalCorrelate} to + * {@link EnumerableConvention enumerable calling convention}. */ + public static final RelOptRule ENUMERABLE_CONDITIONAL_CORRELATE_RULE = + EnumerableConditionalCorrelateRule.DEFAULT_CONFIG + .toRule(EnumerableConditionalCorrelateRule.class); + /** Rule that converts a * {@link org.apache.calcite.rel.logical.LogicalJoin} into an * {@link org.apache.calcite.adapter.enumerable.EnumerableBatchNestedLoopJoin}. */ @@ -219,6 +226,7 @@ private EnumerableRules() { EnumerableRules.ENUMERABLE_ASOFJOIN_RULE, EnumerableRules.ENUMERABLE_MERGE_JOIN_RULE, EnumerableRules.ENUMERABLE_CORRELATE_RULE, + EnumerableRules.ENUMERABLE_CONDITIONAL_CORRELATE_RULE, EnumerableRules.ENUMERABLE_PROJECT_RULE, EnumerableRules.ENUMERABLE_FILTER_RULE, EnumerableRules.ENUMERABLE_CALC_RULE, diff --git a/core/src/main/java/org/apache/calcite/rel/core/ConditionalCorrelate.java b/core/src/main/java/org/apache/calcite/rel/core/ConditionalCorrelate.java index af203429efe9..f5e1d38377d7 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/ConditionalCorrelate.java +++ b/core/src/main/java/org/apache/calcite/rel/core/ConditionalCorrelate.java @@ -42,7 +42,7 @@ */ public abstract class ConditionalCorrelate extends Correlate { - private final RexNode condition; + protected final RexNode condition; protected ConditionalCorrelate( RelOptCluster cluster, diff --git a/core/src/main/java/org/apache/calcite/tools/Programs.java b/core/src/main/java/org/apache/calcite/tools/Programs.java index 4974db3c20b5..83f6834d2850 100644 --- a/core/src/main/java/org/apache/calcite/tools/Programs.java +++ b/core/src/main/java/org/apache/calcite/tools/Programs.java @@ -82,6 +82,7 @@ public class Programs { EnumerableRules.ENUMERABLE_JOIN_RULE, EnumerableRules.ENUMERABLE_MERGE_JOIN_RULE, EnumerableRules.ENUMERABLE_CORRELATE_RULE, + EnumerableRules.ENUMERABLE_CONDITIONAL_CORRELATE_RULE, EnumerableRules.ENUMERABLE_PROJECT_RULE, EnumerableRules.ENUMERABLE_FILTER_RULE, EnumerableRules.ENUMERABLE_AGGREGATE_RULE, diff --git a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java index 6adf1aa3946d..465a89347e9d 100644 --- a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java +++ b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java @@ -263,6 +263,10 @@ public enum BuiltInMethod { Enumerable.class, // inner enumerable NullablePredicate2.class, // non-equi predicate that can return NULL Function2.class), // result selector + CORRELATE_LEFT_MARK_JOIN(ExtendedEnumerable.class, "correlateLeftMarkJoin", + Function1.class, // function to generate inner enumerable from correlate variable + NullablePredicate2.class, // non-equi predicate that can return NULL + Function2.class), // result selector CORRELATE_JOIN(ExtendedEnumerable.class, "correlateJoin", JoinType.class, Function1.class, Function2.class), CORRELATE_BATCH_JOIN(EnumerableDefaults.class, "correlateBatchJoin", diff --git a/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableCorrelateTest.java b/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableCorrelateTest.java index 9ea76b84e814..72455d5abbcf 100644 --- a/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableCorrelateTest.java +++ b/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableCorrelateTest.java @@ -21,15 +21,23 @@ import org.apache.calcite.config.CalciteConnectionProperty; import org.apache.calcite.config.Lex; import org.apache.calcite.plan.RelOptPlanner; +import org.apache.calcite.plan.RelOptRule; +import org.apache.calcite.rel.metadata.DefaultRelMetadataProvider; import org.apache.calcite.rel.rules.CoreRules; import org.apache.calcite.runtime.Hook; import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.test.CalciteAssert; import org.apache.calcite.test.ReflectiveSchemaWithoutRowCount; import org.apache.calcite.test.schemata.hr.HrSchema; +import org.apache.calcite.tools.Program; +import org.apache.calcite.tools.Programs; +import org.apache.calcite.util.Holder; + +import com.google.common.collect.ImmutableList; import org.junit.jupiter.api.Test; +import java.util.List; import java.util.function.Consumer; /** @@ -296,6 +304,170 @@ class EnumerableCorrelateTest { .returnsUnordered("empid=200; name=Eric"); } + private static Program getConditionalCorrelateProgram() { + Program subQuery = + Programs.hep( + ImmutableList.of(CoreRules.PROJECT_SUB_QUERY_TO_MARK_CORRELATE, + CoreRules.FILTER_SUB_QUERY_TO_MARK_CORRELATE), + true, + DefaultRelMetadataProvider.INSTANCE); + Program toCalc = + Programs.hep( + ImmutableList.of( + CoreRules.PROJECT_TO_CALC, + CoreRules.FILTER_TO_CALC, + CoreRules.CALC_MERGE), + true, + DefaultRelMetadataProvider.INSTANCE); + + final List enumerableRules = + ImmutableList.of( + EnumerableRules.ENUMERABLE_VALUES_RULE, + EnumerableRules.ENUMERABLE_CALC_RULE, + EnumerableRules.ENUMERABLE_UNCOLLECT_RULE, + EnumerableRules.ENUMERABLE_CONDITIONAL_CORRELATE_RULE); + Program enumerableImpl = Programs.ofRules(enumerableRules); + return Programs.sequence(subQuery, toCalc, enumerableImpl); + } + + /** Test case for + * [CALCITE-7403] + * Missing ENUMERABLE Convention for LogicalConditionalCorrelate. */ + @Test void testConditionalCorrelateForExists() { + // test for exists + tester(false, new HrSchema()) + .query( + "WITH t1(id, val) AS (\n" + + " VALUES (1, 10), (2, 20), (NULL, 30)\n" + + "),\n" + + "t2(id, val) AS (\n" + + " VALUES (2, 15), (3, 25)\n" + + ")\n" + + "SELECT\n" + + " t1.id,\n" + + " EXISTS (\n" + + " SELECT 1\n" + + " FROM t2\n" + + " WHERE t2.id = t1.id\n" + + " AND t2.val > 10\n" + + " ) AS marker\n" + + "FROM t1") + .withHook(Hook.PROGRAM, (Consumer>) program -> { + program.set(getConditionalCorrelateProgram()); + }) + .explainHookMatches("" + + "EnumerableCalc(expr#0..2=[{inputs}], id=[$t0], marker=[$t2])\n" + + " EnumerableConditionalCorrelate(correlation=[$cor0], joinType=[left_mark], requiredColumns=[{0}])\n" + + " EnumerableValues(tuples=[[{ 1, 10 }, { 2, 20 }, { null, 30 }]])\n" + + " EnumerableCalc(expr#0..1=[{inputs}], expr#2=[$cor0], expr#3=[$t2.id], expr#4=[=($t0, $t3)], expr#5=[10], expr#6=[>($t1, $t5)], expr#7=[AND($t4, $t6)], proj#0..1=[{exprs}], $condition=[$t7])\n" + + " EnumerableValues(tuples=[[{ 2, 15 }, { 3, 25 }]])\n") + .returnsUnordered( + "id=1; marker=false", + "id=2; marker=true", + "id=null; marker=false"); + + // test for not exists + tester(false, new HrSchema()) + .query( + "WITH t1(id, val) AS (\n" + + " VALUES (1, 10), (2, 20), (NULL, 30)\n" + + "),\n" + + "t2(id, val) AS (\n" + + " VALUES (2, 15), (3, 25)\n" + + ")\n" + + "SELECT\n" + + " t1.id,\n" + + " NOT EXISTS (\n" + + " SELECT 1\n" + + " FROM t2\n" + + " WHERE t2.id = t1.id\n" + + " AND t2.val > 10\n" + + " ) AS marker\n" + + "FROM t1") + .withHook(Hook.PROGRAM, (Consumer>) program -> { + program.set(getConditionalCorrelateProgram()); + }) + .explainHookMatches("" + + "EnumerableCalc(expr#0..2=[{inputs}], expr#3=[NOT($t2)], id=[$t0], marker=[$t3])\n" + + " EnumerableConditionalCorrelate(correlation=[$cor0], joinType=[left_mark], requiredColumns=[{0}])\n" + + " EnumerableValues(tuples=[[{ 1, 10 }, { 2, 20 }, { null, 30 }]])\n" + + " EnumerableCalc(expr#0..1=[{inputs}], expr#2=[$cor0], expr#3=[$t2.id], expr#4=[=($t0, $t3)], expr#5=[10], expr#6=[>($t1, $t5)], expr#7=[AND($t4, $t6)], proj#0..1=[{exprs}], $condition=[$t7])\n" + + " EnumerableValues(tuples=[[{ 2, 15 }, { 3, 25 }]])\n") + .returnsUnordered( + "id=1; marker=true", + "id=2; marker=false", + "id=null; marker=true"); + } + + /** Test case for + * [CALCITE-7403] + * Missing ENUMERABLE Convention for LogicalConditionalCorrelate. */ + @Test void testConditionalCorrelateForIn() { + // test in + tester(false, new HrSchema()) + .query( + "WITH t1(id, val) AS (\n" + + " VALUES (1, 10), (2, 20), (NULL, 30)\n" + + "),\n" + + "t2(id, val) AS (\n" + + " VALUES (2, 15), (3, 25)\n" + + ")\n" + + "SELECT\n" + + " t1.id,\n" + + " t1.id IN (\n" + + " SELECT t2.id\n" + + " FROM t2\n" + + " WHERE t2.id = t1.id\n" + + " AND t2.val > 10\n" + + " ) AS marker\n" + + "FROM t1") + .withHook(Hook.PROGRAM, (Consumer>) program -> { + program.set(getConditionalCorrelateProgram()); + }) + .explainHookMatches("" + + "EnumerableCalc(expr#0..2=[{inputs}], id=[$t0], marker=[$t2])\n" + + " EnumerableConditionalCorrelate(correlation=[$cor0], joinType=[left_mark], requiredColumns=[{0}], condition=[=($0, $2)])\n" + + " EnumerableValues(tuples=[[{ 1, 10 }, { 2, 20 }, { null, 30 }]])\n" + + " EnumerableCalc(expr#0..1=[{inputs}], expr#2=[$cor0], expr#3=[$t2.id], expr#4=[=($t0, $t3)], expr#5=[10], expr#6=[>($t1, $t5)], expr#7=[AND($t4, $t6)], id=[$t0], $condition=[$t7])\n" + + " EnumerableValues(tuples=[[{ 2, 15 }, { 3, 25 }]])\n") + .returnsUnordered( + "id=1; marker=false", + "id=2; marker=true", + "id=null; marker=false"); + + // test not in + tester(false, new HrSchema()) + .query( + "WITH t1(id, val) AS (\n" + + " VALUES (1, 10), (2, 20), (NULL, 30)\n" + + "),\n" + + "t2(id, val) AS (\n" + + " VALUES (2, 15), (3, 25)\n" + + ")\n" + + "SELECT\n" + + " t1.id,\n" + + " t1.id NOT IN (\n" + + " SELECT t2.id\n" + + " FROM t2\n" + + " WHERE t2.id = t1.id\n" + + " AND t2.val > 10\n" + + " ) AS marker\n" + + "FROM t1") + .withHook(Hook.PROGRAM, (Consumer>) program -> { + program.set(getConditionalCorrelateProgram()); + }) + .explainHookMatches("" + + "EnumerableCalc(expr#0..2=[{inputs}], expr#3=[NOT($t2)], id=[$t0], marker=[$t3])\n" + + " EnumerableConditionalCorrelate(correlation=[$cor0], joinType=[left_mark], requiredColumns=[{0}], condition=[=($0, $2)])\n" + + " EnumerableValues(tuples=[[{ 1, 10 }, { 2, 20 }, { null, 30 }]])\n" + + " EnumerableCalc(expr#0..1=[{inputs}], expr#2=[$cor0], expr#3=[$t2.id], expr#4=[=($t0, $t3)], expr#5=[10], expr#6=[>($t1, $t5)], expr#7=[AND($t4, $t6)], id=[$t0], $condition=[$t7])\n" + + " EnumerableValues(tuples=[[{ 2, 15 }, { 3, 25 }]])\n") + .returnsUnordered( + "id=1; marker=true", + "id=2; marker=false", + "id=null; marker=true"); + } + private CalciteAssert.AssertThat tester(boolean forceDecorrelate, Object schema) { return CalciteAssert.that() diff --git a/core/src/test/resources/sql/new-decorr.iq b/core/src/test/resources/sql/new-decorr.iq index 9677fae713bd..8f05bbac2a28 100644 --- a/core/src/test/resources/sql/new-decorr.iq +++ b/core/src/test/resources/sql/new-decorr.iq @@ -205,6 +205,30 @@ EnumerableCalc(expr#0..3=[{inputs}], DEPTNO=[$t0], EXPR$0=[$t2]) !plan !} +# [CALCITE-7403] Missing ENUMERABLE Convention for LogicalConditionalCorrelate +# This case comes from some.iq [CALCITE-6786] +WITH tb as (select array(SELECT * FROM (VALUES (TRUE), (NULL)) as x(a)) as a) +SELECT TRUE IN (SELECT b FROM UNNEST(a) AS x1(b)) AS test FROM tb; ++------+ +| TEST | ++------+ +| true | ++------+ +(1 row) + +!ok + +!if (use_new_decorr) { +EnumerableCalc(expr#0..1=[{inputs}], TEST=[$t1]) + EnumerableConditionalCorrelate(correlation=[$cor0], joinType=[left_mark], requiredColumns=[{0}], condition=[$1]) + EnumerableCollect(field=[x]) + EnumerableValues(tuples=[[{ true }, { null }]]) + EnumerableUncollect + EnumerableCalc(expr#0=[{inputs}], expr#1=[$cor0], expr#2=[$t1.A], A=[$t2]) + EnumerableValues(tuples=[[{ 0 }]]) +!plan +!} + # [CALCITE-7396] PruneEmptyRules does not support LEFT_MARK JOIN # This case comes from sub-query.iq !use post diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/DefaultEnumerable.java b/linq4j/src/main/java/org/apache/calcite/linq4j/DefaultEnumerable.java index 8a45548d3107..d859519b4178 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/DefaultEnumerable.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/DefaultEnumerable.java @@ -455,6 +455,13 @@ protected OrderedQueryable asOrderedQueryable() { return EnumerableDefaults.leftMarkNestedLoopJoin(getThis(), inner, predicate, resultSelector); } + @Override public Enumerable correlateLeftMarkJoin( + Function1> inner, + NullablePredicate2 predicate, + Function2 resultSelector) { + return EnumerableDefaults.correlateLeftMarkJoin(getThis(), inner, predicate, resultSelector); + } + @Override public Enumerable correlateJoin( JoinType joinType, Function1> inner, Function2 resultSelector) { diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java b/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java index 28f4a1185e25..988450df3643 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java @@ -1967,14 +1967,18 @@ static Enumerable leftMarkHash /** * The implementation of left mark join based on nested loop. + * This is a unified implementation supporting both correlated and non-correlated cases. * * @param outer Left input - * @param inner Right input + * @param innerProvider Function that provides inner enumerable for each outer row + * (for correlated join, pass outerRow -> correlatedInner) + * (for non-correlated join, pass outerRow -> staticInner) * @param predicate Non-equi predicate that can return NULL * @param resultSelector Function that concats the row of left input and marker */ - public static Enumerable leftMarkNestedLoopJoin( - final Enumerable outer, final Enumerable inner, + private static Enumerable leftMarkJoinInternal( + final Enumerable outer, + final Function1> innerProvider, final NullablePredicate2 predicate, final Function2 resultSelector) { return new AbstractEnumerable() { @@ -1993,15 +1997,18 @@ public static Enumerable leftMarkNestedLoopJ } marker = false; final TSource outerRow = outers.current(); - try (Enumerator inners = inner.enumerator()) { - while (inners.moveNext()) { - final TInner innerRow = inners.current(); - Boolean predicateMatched = predicate.apply(outerRow, innerRow); - if (predicateMatched == null) { - marker = null; - } else if (predicateMatched) { - marker = true; - break; + Enumerable innerEnumerable = innerProvider.apply(outerRow); + if (innerEnumerable != null) { + try (Enumerator inners = innerEnumerable.enumerator()) { + while (inners.moveNext()) { + final TInner innerRow = inners.current(); + Boolean predicateMatched = predicate.apply(outerRow, innerRow); + if (predicateMatched == null) { + marker = null; + } else if (predicateMatched) { + marker = true; + break; + } } } } @@ -2020,6 +2027,34 @@ public static Enumerable leftMarkNestedLoopJ }; } + /** + * For each row of the {@code outer} enumerable returns correlated rows + * from the inner enumerable generated for each outer row, filtered by a predicate + * (correlated LEFT_MARK join). + */ + public static Enumerable correlateLeftMarkJoin( + final Enumerable outer, + final Function1> inner, + final NullablePredicate2 predicate, + final Function2 resultSelector) { + return leftMarkJoinInternal(outer, inner, predicate, resultSelector); + } + + /** + * The implementation of left mark join based on nested loop. + * + * @param outer Left input + * @param inner Right input + * @param predicate Non-equi predicate that can return NULL + * @param resultSelector Function that concats the row of left input and marker + */ + public static Enumerable leftMarkNestedLoopJoin( + final Enumerable outer, final Enumerable inner, + final NullablePredicate2 predicate, + final Function2 resultSelector) { + return leftMarkJoinInternal(outer, ignored -> inner, predicate, resultSelector); + } + /** * For each row of the {@code outer} enumerable returns the correlated rows * from the {@code inner} enumerable. diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedEnumerable.java b/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedEnumerable.java index 982ab0ca85a9..160f2afa0b1e 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedEnumerable.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedEnumerable.java @@ -699,6 +699,19 @@ Enumerable leftMarkNestedLoopJoin(Enumerable NullablePredicate2 predicate, Function2 resultSelector); + /** + * For each row of the current enumerable returns correlated rows where each row + * from the inner enumerable satisfies the predicate (correlated LEFT_MARK join). + * + * @param inner function to generate inner enumerable from outer row + * @param predicate predicate that can return NULL + * @param resultSelector selector of the result, receives outer row and boolean marker + */ + Enumerable correlateLeftMarkJoin( + Function1> inner, + NullablePredicate2 predicate, + Function2 resultSelector); + /** * For each row of the current enumerable returns the correlated rows * from the {@code inner} enumerable (nested loops join). From 1698929e33892a993c33cb7876c2f2e6af8bc49e Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Fri, 30 Jan 2026 06:48:32 +0800 Subject: [PATCH 141/562] Included the `some` iq files in CoreQuidemTest2 --- .../org/apache/calcite/test/CoreQuidemTest2.java | 3 +-- core/src/test/resources/sql/some.iq | 12 ++++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/core/src/test/java/org/apache/calcite/test/CoreQuidemTest2.java b/core/src/test/java/org/apache/calcite/test/CoreQuidemTest2.java index 865aec400ae8..0164eeea6e61 100644 --- a/core/src/test/java/org/apache/calcite/test/CoreQuidemTest2.java +++ b/core/src/test/java/org/apache/calcite/test/CoreQuidemTest2.java @@ -44,9 +44,8 @@ public static void main(String[] args) throws Exception { // These remove operations are temporary and will be deleted // once the new decorrelator can adapt to all scenarios. - // TODO: The following files involves UNNEST and LEFT_MARK JOIN + // TODO: Support measure paths.remove("sql/measure.iq"); - paths.remove("sql/some.iq"); paths.remove("sql/sub-query.iq"); paths.remove("sql/measure-paper.iq"); return paths; diff --git a/core/src/test/resources/sql/some.iq b/core/src/test/resources/sql/some.iq index 02e771241f34..97414f845023 100644 --- a/core/src/test/resources/sql/some.iq +++ b/core/src/test/resources/sql/some.iq @@ -128,12 +128,24 @@ from "scott".emp; (14 rows) !ok + +!if (use_old_decorr) { EnumerableCalc(expr#0..10=[{inputs}], expr#11=[0], expr#12=[=($t9, $t11)], expr#13=[>($t9, $t10)], expr#14=[null:BOOLEAN], expr#15=[<=($t5, $t8)], expr#16=[IS NOT TRUE($t15)], expr#17=[AND($t13, $t14, $t16)], expr#18=[>($t5, $t8)], expr#19=[<=($t9, $t10)], expr#20=[AND($t18, $t16, $t19)], expr#21=[OR($t12, $t17, $t20)], proj#0..7=[{exprs}], X=[$t21]) EnumerableNestedLoopJoin(condition=[true], joinType=[inner]) EnumerableTableScan(table=[[scott, EMP]]) EnumerableAggregate(group=[{}], m=[MAX($6)], c=[COUNT()], d=[COUNT($6)]) EnumerableTableScan(table=[[scott, EMP]]) !plan +!} + +!if (use_new_decorr) { +EnumerableCalc(expr#0..8=[{inputs}], expr#9=[NOT($t8)], proj#0..7=[{exprs}], X=[$t9]) + EnumerableNestedLoopJoin(condition=[<=($5, $8)], joinType=[left_mark]) + EnumerableTableScan(table=[[scott, EMP]]) + EnumerableCalc(expr#0..7=[{inputs}], COMM=[$t6]) + EnumerableTableScan(table=[[scott, EMP]]) +!plan +!} # NOT SOME; left side NOT NULL, right side nullable; converse of previous query. select * from "scott".emp From f8115e6ff9c22cf75b1482811d6bfe5a6b178584 Mon Sep 17 00:00:00 2001 From: Silun Dong Date: Wed, 28 Jan 2026 23:59:12 +0800 Subject: [PATCH 142/562] [CALCITE-7402] Two-level nested correlated subquery causes TopDownGeneralDecorrelator type mismatch during translation --- .../sql2rel/TopDownGeneralDecorrelator.java | 3 ++- core/src/test/resources/sql/new-decorr.iq | 24 +++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java b/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java index 068ec441e557..6959d56354b8 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java @@ -931,7 +931,8 @@ static List rewrite( (RexCorrelVariable) fieldAccess.getReferenceExpr(); CorDef corDef = new CorDef(v.id, fieldAccess.getField().getIndex()); int newIndex = requireNonNull(unnestedQuery.corDefOutputs.get(corDef)); - return new RexInputRef(newIndex, fieldAccess.getType()); + return new RexInputRef( + newIndex, unnestedQuery.r.getRowType().getFieldList().get(newIndex).getType()); } return super.visitFieldAccess(fieldAccess); } diff --git a/core/src/test/resources/sql/new-decorr.iq b/core/src/test/resources/sql/new-decorr.iq index 8f05bbac2a28..4c2e5791af37 100644 --- a/core/src/test/resources/sql/new-decorr.iq +++ b/core/src/test/resources/sql/new-decorr.iq @@ -276,4 +276,28 @@ select deptno from dept d1 where exists ( !ok +# [CALCITE-7402] Two-level nested correlated subquery causes TopDownGeneralDecorrelator type mismatch during translation +# This case comes from sub-query.iq [CALCITE-5716] +!use scott +SELECT dept.deptno, ( + SELECT max(emp.empno) + FROM emp + WHERE empno = (SELECT max(empno) AS maxDept + FROM emp e2 + WHERE e2.deptno = dept.deptno) + AND emp.deptno = dept.deptno), + dept.dname +FROM dept; ++--------+--------+------------+ +| DEPTNO | EXPR$1 | DNAME | ++--------+--------+------------+ +| 10 | 7934 | ACCOUNTING | +| 20 | 7902 | RESEARCH | +| 30 | 7900 | SALES | +| 40 | | OPERATIONS | ++--------+--------+------------+ +(4 rows) + +!ok + # End new-decorr.iq From a993da007b12e93d65af2a6b889739692ecc4941 Mon Sep 17 00:00:00 2001 From: Terran Date: Sat, 31 Jan 2026 17:27:53 +0800 Subject: [PATCH 143/562] [CALCITE-7406] Add abs function (enabled in Mongodb library) --- .../calcite/adapter/mongodb/MongoRules.java | 1 + .../adapter/mongodb/MongoAdapterTest.java | 45 +++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoRules.java b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoRules.java index fa10035746c2..47ad046c142f 100644 --- a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoRules.java +++ b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoRules.java @@ -151,6 +151,7 @@ static class RexToMongoTranslator extends RexVisitorImpl { MONGO_OPERATORS.put(SqlStdOperatorTable.GREATER_THAN_OR_EQUAL, "$gte"); MONGO_OPERATORS.put(SqlStdOperatorTable.LESS_THAN, "$lt"); MONGO_OPERATORS.put(SqlStdOperatorTable.LESS_THAN_OR_EQUAL, "$lte"); + MONGO_OPERATORS.put(SqlStdOperatorTable.ABS, "$abs"); } protected RexToMongoTranslator(JavaTypeFactory typeFactory, diff --git a/mongodb/src/test/java/org/apache/calcite/adapter/mongodb/MongoAdapterTest.java b/mongodb/src/test/java/org/apache/calcite/adapter/mongodb/MongoAdapterTest.java index a7b483cc99e8..c562de6d9dfc 100644 --- a/mongodb/src/test/java/org/apache/calcite/adapter/mongodb/MongoAdapterTest.java +++ b/mongodb/src/test/java/org/apache/calcite/adapter/mongodb/MongoAdapterTest.java @@ -1001,4 +1001,49 @@ private static Consumer mongoChecker(final String... expected) { "{$sort: {CITY: 1}}")) .returnsOrdered(""); } + + /** Test case for + * [CALCITE-7406] + * Add abs function (enabled in Mongodb library). */ + @Test void testAbs() { + assertModel(MODEL) + .query("select abs(pop) from zips") + .runs() + .queryContains( + mongoChecker( + "{$project:{EXPR$0:{$abs:['$pop']}}}")); + } + + /** Test case for + * [CALCITE-7406] + * Add abs function (enabled in Mongodb library). */ + @Test void testAbsAlias() { + assertModel(MODEL) + .query("select abs(pop) as pop_result from zips" + + " order by pop") + .limit(3) + .runs() + .queryContains( + mongoChecker( + "{$project:{POP_RESULT:{$abs:['$pop']},POP:'$pop'}}", + "{$sort:{POP:1}}")) + .returnsOrdered( + "POP_RESULT=21", + "POP_RESULT=17522", + "POP_RESULT=22576"); + } + + /** Test case for + * [CALCITE-7406] + * Add abs function (enabled in Mongodb library). */ + @Test void testAbsMin() { + assertModel(MODEL) + .query("select abs(min(pop)) from zips") + .returnsOrdered("EXPR$0=21") + .queryContains( + mongoChecker( + "{$project:{POP:'$pop'}}", + "{$group:{_id:{},_0:{$min:'$POP'}}}", + "{$project:{EXPR$0:{$abs:['$_0']}}}")); + } } From f6516affd702a85040def7f8d284b1c7a8ee9131 Mon Sep 17 00:00:00 2001 From: Terran Date: Wed, 28 Jan 2026 18:36:08 +0800 Subject: [PATCH 144/562] [CALCITE-7404] Incorrect Field Alias in MongoDB project Stage --- .../calcite/adapter/mongodb/MongoProject.java | 4 +- .../adapter/mongodb/MongoAdapterTest.java | 47 +++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoProject.java b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoProject.java index ee0721fe4a92..a0d795aa0d8f 100644 --- a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoProject.java +++ b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoProject.java @@ -76,7 +76,9 @@ public MongoProject(RelOptCluster cluster, RelTraitSet traitSet, MongoRules.mongoFieldNames(getInput().getRowType())); final List items = new ArrayList<>(); for (Pair pair : getNamedProjects()) { - final String name = pair.right; + final String name = pair.right.startsWith("$f") + ? "_" + pair.right.substring(2) + : pair.right; final String expr = pair.left.accept(translator); items.add(expr.equals("'$" + name + "'") ? MongoRules.maybeQuote(name) + ": 1" diff --git a/mongodb/src/test/java/org/apache/calcite/adapter/mongodb/MongoAdapterTest.java b/mongodb/src/test/java/org/apache/calcite/adapter/mongodb/MongoAdapterTest.java index c562de6d9dfc..ce83c6e1d831 100644 --- a/mongodb/src/test/java/org/apache/calcite/adapter/mongodb/MongoAdapterTest.java +++ b/mongodb/src/test/java/org/apache/calcite/adapter/mongodb/MongoAdapterTest.java @@ -1046,4 +1046,51 @@ private static Consumer mongoChecker(final String... expected) { "{$group:{_id:{},_0:{$min:'$POP'}}}", "{$project:{EXPR$0:{$abs:['$_0']}}}")); } + + /** Test case for + * [CALCITE-7404] + * Incorrect Field Alias in MongoDB project Stage. */ + @Test void testAggFunctionMinFilter() { + assertModel(MODEL) + .query("select min(pop>5000) as pop_result from zips") + .queryContains( + mongoChecker( + "{$project:{_0:{$gt:['$pop',{$literal:5000}]}}}", + "{$group:{_id: {},POP_RESULT:{$min:'$_0'}}}")) + .returns("POP_RESULT=false\n"); + } + + /** Test case for + * [CALCITE-7404] + * Incorrect Field Alias in MongoDB project Stage. */ + @Test void testAliasNameOrderBy() { + assertModel(MODEL) + .query("select pop as pop_a from zips" + + " order by pop_a") + .limit(3) + .queryContains( + mongoChecker( + "{$project:{POP_A:'$pop'}}", + "{$sort: {POP_A: 1}}")) + .returnsOrdered("POP_A=21", + "POP_A=17522", + "POP_A=22576"); + } + + /** Test case for + * [CALCITE-7404] + * Incorrect Field Alias in MongoDB project Stage. */ + @Test void testNameOrderBy() { + assertModel(MODEL) + .query("select pop as pop_a from zips" + + " order by pop") + .limit(3) + .queryContains( + mongoChecker( + "{$project:{POP_A:'$pop'}}", + "{$sort: {POP_A: 1}}")) + .returnsOrdered("POP_A=21", + "POP_A=17522", + "POP_A=22576"); + } } From cd1613da31dde835d1fd0dac05af111950836e5a Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Thu, 5 Feb 2026 19:40:19 +0800 Subject: [PATCH 145/562] [CALCITE-7409] MERGE JOIN condition cannot contain IS NOT DISTINCT FROM --- .../enumerable/EnumerableMergeJoinRule.java | 8 ++++ core/src/test/resources/sql/sub-query.iq | 46 +++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeJoinRule.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeJoinRule.java index 624db0a60483..f78423edbde5 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeJoinRule.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeJoinRule.java @@ -31,6 +31,7 @@ import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexUtil; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.checkerframework.checker.nullness.qual.Nullable; @@ -60,6 +61,13 @@ protected EnumerableMergeJoinRule(Config config) { @Override public @Nullable RelNode convert(RelNode rel) { Join join = (Join) rel; + // TODO: support IS NOT DISTINCT FROM condition as join keys of MergeJoin. + // MergeJoin cannot handle IS NOT DISTINCT FROM because it stops at NULL values + // while IS NOT DISTINCT FROM treats NULL = NULL as true. + if (RexUtil.findOperatorCall(SqlStdOperatorTable.IS_NOT_DISTINCT_FROM, + join.getCondition()) != null) { + return null; + } // EnumerableMergeJoin cannot use IS NOT DISTINCT FROM condition as join keys. More details // in EnumerableMergeJoin.java. final JoinInfo info = diff --git a/core/src/test/resources/sql/sub-query.iq b/core/src/test/resources/sql/sub-query.iq index a9440f6f43b9..20c7f19472d1 100644 --- a/core/src/test/resources/sql/sub-query.iq +++ b/core/src/test/resources/sql/sub-query.iq @@ -8665,4 +8665,50 @@ FROM dept; (4 rows) !ok + +# [CALCITE-7409] MERGE JOIN condition cannot contain IS NOT DISTINCT FROM +select e.ename, + (select count(*) + from emp as f + where f.comm is not distinct from e.comm) as c +from emp as e; ++--------+----+ +| ENAME | C | ++--------+----+ +| ALLEN | 1 | +| WARD | 1 | +| MARTIN | 1 | +| TURNER | 1 | +| SMITH | 10 | +| JONES | 10 | +| BLAKE | 10 | +| CLARK | 10 | +| SCOTT | 10 | +| KING | 10 | +| ADAMS | 10 | +| JAMES | 10 | +| FORD | 10 | +| MILLER | 10 | ++--------+----+ +(14 rows) + +!ok +EnumerableCalc(expr#0..6=[{inputs}], expr#7=[IS NULL($t6)], expr#8=[0:BIGINT], expr#9=[CASE($t7, $t8, $t6)], ENAME=[$t1], C=[$t9]) + EnumerableHashJoin(condition=[AND(IS NOT DISTINCT FROM($2, $4), =($3, $5))], joinType=[left]) + EnumerableCalc(expr#0..7=[{inputs}], expr#8=[IS NULL($t6)], proj#0..1=[{exprs}], COMM=[$t6], $f3=[$t8]) + EnumerableTableScan(table=[[scott, EMP]]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NOT NULL($t4)], expr#6=[0], expr#7=[CASE($t5, $t4, $t6)], proj#0..1=[{exprs}], EXPR$0=[$t7]) + EnumerableHashJoin(condition=[AND(IS NOT DISTINCT FROM($0, $2), IS NOT DISTINCT FROM($1, $3))], joinType=[left]) + EnumerableAggregate(group=[{0, 1}]) + EnumerableCalc(expr#0..7=[{inputs}], expr#8=[IS NULL($t6)], COMM=[$t6], $f3=[$t8]) + EnumerableTableScan(table=[[scott, EMP]]) + EnumerableAggregate(group=[{0, 1}], EXPR$0=[COUNT()]) + EnumerableNestedLoopJoin(condition=[OR(AND(IS NULL($3), $1), =($3, $0))], joinType=[inner]) + EnumerableAggregate(group=[{0, 1}]) + EnumerableCalc(expr#0..7=[{inputs}], expr#8=[IS NULL($t6)], COMM=[$t6], $f3=[$t8]) + EnumerableTableScan(table=[[scott, EMP]]) + EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], COMM=[$t6]) + EnumerableTableScan(table=[[scott, EMP]]) +!plan + # End sub-query.iq From 769587000fc3f06da5bd18f34ea69023460ed2c0 Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Fri, 6 Feb 2026 09:00:01 +0800 Subject: [PATCH 146/562] Test case for [CALCITE-6452] Scalar sub-query that uses IS NOT DISTINCT FROM returns incorrect result --- core/src/test/resources/sql/sub-query.iq | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/core/src/test/resources/sql/sub-query.iq b/core/src/test/resources/sql/sub-query.iq index 20c7f19472d1..fbe51075cf24 100644 --- a/core/src/test/resources/sql/sub-query.iq +++ b/core/src/test/resources/sql/sub-query.iq @@ -8711,4 +8711,22 @@ EnumerableCalc(expr#0..6=[{inputs}], expr#7=[IS NULL($t6)], expr#8=[0:BIGINT], e EnumerableTableScan(table=[[scott, EMP]]) !plan +# Test case for [CALCITE-6452] Scalar sub-query that uses IS NOT DISTINCT FROM returns incorrect result +select e.ename, + (select count(*) + from emp as f + where f.comm is not distinct from e.comm) as c +from emp as e +where e.deptno = 10; ++--------+----+ +| ENAME | C | ++--------+----+ +| CLARK | 10 | +| KING | 10 | +| MILLER | 10 | ++--------+----+ +(3 rows) + +!ok + # End sub-query.iq From 4e7aa0fc68026291b4a5c6fedcdf8c0515c41990 Mon Sep 17 00:00:00 2001 From: iwanttobepowerful <745778074@qq.com> Date: Fri, 6 Feb 2026 21:32:31 +0800 Subject: [PATCH 147/562] Site: Add Weihua Zhang as committer --- site/_data/contributors.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/site/_data/contributors.yml b/site/_data/contributors.yml index cc43c5be9eeb..f8347e026a59 100644 --- a/site/_data/contributors.yml +++ b/site/_data/contributors.yml @@ -385,6 +385,11 @@ githubId: yanlin-Lynn org: Ant Financial role: Committer +- name: Weihua Zhang + apacheId: zwh + githubId: iwanttobepowerful + org: Tencent + role: Committer - name: Xiong Duan apacheId: xiong githubId: NobiGo From 409d59c9b682c4cc2f4541062453100b6dc718ce Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Sun, 8 Feb 2026 10:16:38 +0800 Subject: [PATCH 148/562] [CALCITE-7412] Redis test failed on higher versions of macOS --- redis/build.gradle.kts | 1 + .../java/redis/embedded/util/JarUtil.java | 50 +++++++++++++++++++ .../redis/embedded/util/package-info.java | 21 ++++++++ 3 files changed, 72 insertions(+) create mode 100644 redis/src/main/java/redis/embedded/util/JarUtil.java create mode 100644 redis/src/main/java/redis/embedded/util/package-info.java diff --git a/redis/build.gradle.kts b/redis/build.gradle.kts index 80e24755d953..0cbc43112fe4 100644 --- a/redis/build.gradle.kts +++ b/redis/build.gradle.kts @@ -22,6 +22,7 @@ dependencies { implementation("com.fasterxml.jackson.core:jackson-core") implementation("com.fasterxml.jackson.core:jackson-databind") implementation("com.google.guava:guava") + implementation("commons-io:commons-io") implementation("org.apache.calcite.avatica:avatica-core") implementation("org.apache.commons:commons-lang3") implementation("org.apache.commons:commons-pool2") diff --git a/redis/src/main/java/redis/embedded/util/JarUtil.java b/redis/src/main/java/redis/embedded/util/JarUtil.java new file mode 100644 index 000000000000..9e8896ee10d5 --- /dev/null +++ b/redis/src/main/java/redis/embedded/util/JarUtil.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package redis.embedded.util; + +import org.apache.commons.io.FileUtils; + +import com.google.common.io.Resources; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; + +/** Utility class for extracting executables from JAR files. */ +public class JarUtil { + + private JarUtil() { + // Utility class should not be instantiated + } + + public static File extractExecutableFromJar(String executable) throws IOException { + File tmpDir = Files.createTempDirectory("redis-").toFile(); + tmpDir.deleteOnExit(); + + // On newer versions of macOS, the temporary directory /var/folders does not + // support executing binary files with a .app extension, which prevents Redis + // from starting. To ensure Redis runs correctly, the redis-server binary is + // uniformly renamed to redis-server. + String executableName = "redis-server"; + File command = new File(tmpDir, executableName); + FileUtils.copyURLToFile(Resources.getResource(executable), command); + command.deleteOnExit(); + command.setExecutable(true); + + return command; + } +} diff --git a/redis/src/main/java/redis/embedded/util/package-info.java b/redis/src/main/java/redis/embedded/util/package-info.java new file mode 100644 index 000000000000..5e31ad3901b5 --- /dev/null +++ b/redis/src/main/java/redis/embedded/util/package-info.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Utility classes for embedded Redis. + */ +package redis.embedded.util; From 31790f3b0885660a803d2604ad06c9a1289e537b Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Mon, 9 Feb 2026 10:45:21 +0800 Subject: [PATCH 149/562] [CALCITE-7411] When a SCALAR_QUERY in PROJECT contains correlated variables execution fails using TopDownGeneralDecorrelator --- .../calcite/sql2rel/SqlToRelConverter.java | 21 ++++++- .../sql2rel/TopDownGeneralDecorrelator.java | 9 ++- .../apache/calcite/test/CoreQuidemTest2.java | 1 - .../apache/calcite/test/RelOptRulesTest.java | 24 ++++++++ .../apache/calcite/test/RelOptRulesTest.xml | 53 ++++++++++++++++++ core/src/test/resources/sql/measure-paper.iq | 55 +++++++++++++++++++ 6 files changed, 159 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java index 8fc83104c1d1..4622176a0733 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java @@ -3882,7 +3882,26 @@ private void createAggImpl(Blackboard bb, // implement the SELECT list relBuilder.project(projects.leftList(), projects.rightList()) .rename(projects.rightList()); - bb.setRoot(relBuilder.build(), false); + + RelNode tmpProject = relBuilder.build(); + + // Check for correlation variables that may be used in the SELECT list + final RelNode finalProject; + final CorrelationUse correlationUse = getCorrelationUse(bb, tmpProject); + if (correlationUse != null) { + assert correlationUse.r instanceof Project; + // correlation variables have been normalized in correlationUse.r, + // we should use expressions in correlationUse.r + Project project1 = (Project) correlationUse.r; + finalProject = relBuilder.push(tmpProject.getInput(0)) + .project(project1.getProjects(), project1.getRowType().getFieldNames(), true, + ImmutableSet.of(correlationUse.id)) + .build(); + } else { + finalProject = tmpProject; + } + + bb.setRoot(finalProject, false); // Tell bb which of group columns are sorted. bb.columnMonotonicities.clear(); diff --git a/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java b/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java index 6959d56354b8..50b200615458 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java @@ -919,10 +919,15 @@ static List rewrite( @Override public RexNode visitInputRef(RexInputRef inputRef) { int newIndex = requireNonNull(unnestedQuery.oldToNewOutputs.get(inputRef.getIndex())); - if (newIndex == inputRef.getIndex()) { + if (newIndex == inputRef.getIndex() + && inputRef.getType().equals( + unnestedQuery.r.getRowType().getFieldList().get(newIndex).getType())) { return inputRef; } - return new RexInputRef(newIndex, inputRef.getType()); + // Use the type from the new row type to handle nullability changes + // (e.g., after LEFT JOIN, right-side fields become nullable) + return new RexInputRef(newIndex, + unnestedQuery.r.getRowType().getFieldList().get(newIndex).getType()); } @Override public RexNode visitFieldAccess(RexFieldAccess fieldAccess) { diff --git a/core/src/test/java/org/apache/calcite/test/CoreQuidemTest2.java b/core/src/test/java/org/apache/calcite/test/CoreQuidemTest2.java index 0164eeea6e61..b8ef8562bc8c 100644 --- a/core/src/test/java/org/apache/calcite/test/CoreQuidemTest2.java +++ b/core/src/test/java/org/apache/calcite/test/CoreQuidemTest2.java @@ -47,7 +47,6 @@ public static void main(String[] args) throws Exception { // TODO: Support measure paths.remove("sql/measure.iq"); paths.remove("sql/sub-query.iq"); - paths.remove("sql/measure-paper.iq"); return paths; } diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index 81f07a6f6a52..98beae87c775 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -12243,4 +12243,28 @@ private void checkLoptOptimizeJoinRule(LoptOptimizeJoinRule rule) { sql(sql).withPlanner(hepPlanner) .check(); } + + /** Test case of + * [CALCITE-7411] + * When a SCALAR_QUERY in PROJECT contains correlated variables execution fails + * using TopDownGeneralDecorrelator. */ + @Test void testTopDownGeneralDecorrelateForMeasure() { + final String sql = "SELECT job,\n" + + " (SELECT\n" + + " CAST(SUM(i.sal) - SUM(COALESCE(i.comm, 0)) AS DECIMAL(10, 2)) / SUM(i.sal)\n" + + " FROM emp AS i\n" + + " WHERE i.job = e.job) AS profitMargin,\n" + + " COUNT(*) AS \"count\"\n" + + "FROM emp AS e\n" + + "GROUP BY job"; + + sql(sql) + .withRule( + CoreRules.PROJECT_SUB_QUERY_TO_MARK_CORRELATE, + CoreRules.PROJECT_MERGE, + CoreRules.PROJECT_REMOVE) + .withLateDecorrelate(true) + .withTopDownGeneralDecorrelate(true) + .check(); + } } diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index 103744e3118c..e2f83bf2c199 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -20169,6 +20169,59 @@ LogicalProject(EMPNO=[$0]) LogicalJoin(condition=[IS NOT DISTINCT FROM($7, $9)], joinType=[inner]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) +]]> + + + + + + + + + + + + + + diff --git a/core/src/test/resources/sql/measure-paper.iq b/core/src/test/resources/sql/measure-paper.iq index 190713d81129..c64150fdbafe 100644 --- a/core/src/test/resources/sql/measure-paper.iq +++ b/core/src/test/resources/sql/measure-paper.iq @@ -224,6 +224,7 @@ SELECT "prodName", COUNT(*) AS "count" FROM "Orders" AS o GROUP BY "prodName"; +!if (use_old_decorr) { +----------+--------------+-------+ | prodName | profitMargin | count | +----------+--------------+-------+ @@ -234,6 +235,20 @@ GROUP BY "prodName"; (3 rows) !ok +!} + +!if (use_new_decorr) { ++----------+--------------------+-------+ +| prodName | profitMargin | count | ++----------+--------------------+-------+ +| Acme | 0.60 | 1 | +| Happy | 0.4705882352941176 | 3 | +| Whizz | 0.6666666666666667 | 1 | ++----------+--------------------+-------+ +(3 rows) + +!ok +!} # Profit margin for 'Happy' orders for each customer SELECT "custName", @@ -505,4 +520,44 @@ FROM !ok !} +# [CALCITE-7411] When a SCALAR_QUERY in PROJECT contains correlated variables +# execution fails using TopDownGeneralDecorrelator +!use scott +SELECT job, + (SELECT + CAST(SUM(i.sal) - SUM(COALESCE(i.comm, 0)) AS DECIMAL(10, 2)) / SUM(i.sal) + FROM emp AS i + WHERE i.job = e.job) AS profitMargin, + COUNT(*) AS "count" +FROM emp AS e +GROUP BY job; +!if (use_old_decorr) { ++-----------+--------------+-------+ +| JOB | PROFITMARGIN | count | ++-----------+--------------+-------+ +| ANALYST | 1.000000 | 2 | +| CLERK | 1.000000 | 4 | +| MANAGER | 1.000000 | 3 | +| PRESIDENT | 1.000000 | 1 | +| SALESMAN | 0.607142 | 4 | ++-----------+--------------+-------+ +(5 rows) + +!ok +!} + +!if (use_new_decorr) { ++-----------+--------------------+-------+ +| JOB | PROFITMARGIN | count | ++-----------+--------------------+-------+ +| ANALYST | 1 | 2 | +| CLERK | 1 | 4 | +| MANAGER | 1 | 3 | +| PRESIDENT | 1 | 1 | +| SALESMAN | 0.6071428571428571 | 4 | ++-----------+--------------------+-------+ +(5 rows) + +!ok +!} # End measure-paper.iq From 69428f09f6c31fc511e001140cfe1eef2601e1fe Mon Sep 17 00:00:00 2001 From: xuzifu666 Date: Wed, 4 Feb 2026 14:38:43 +0800 Subject: [PATCH 150/562] [CALCITE-7408] URL_ENCODE/URL_DECODE is unparsed incorrectly for ClickHouseSqlDialect --- .../calcite/sql/dialect/ClickHouseSqlDialect.java | 15 +++++++++++++++ .../rel/rel2sql/RelToSqlConverterTest.java | 13 +++++++++++++ 2 files changed, 28 insertions(+) diff --git a/core/src/main/java/org/apache/calcite/sql/dialect/ClickHouseSqlDialect.java b/core/src/main/java/org/apache/calcite/sql/dialect/ClickHouseSqlDialect.java index cd9e736f242b..fd5a719e1a38 100644 --- a/core/src/main/java/org/apache/calcite/sql/dialect/ClickHouseSqlDialect.java +++ b/core/src/main/java/org/apache/calcite/sql/dialect/ClickHouseSqlDialect.java @@ -39,6 +39,7 @@ import org.apache.calcite.sql.SqlTimeLiteral; import org.apache.calcite.sql.SqlTimestampLiteral; import org.apache.calcite.sql.SqlWriter; +import org.apache.calcite.sql.fun.SqlLibraryOperators; import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.sql.type.SqlTypeName; @@ -228,6 +229,20 @@ private static SqlDataTypeSpec createSqlDataTypeSpecByName(String typeAlias, return; } + // refer to https://clickhouse.com/docs/sql-reference/functions/url-functions#encodeURLComponent + if (call.getOperator() == SqlLibraryOperators.URL_ENCODE) { + RelToSqlConverterUtil.specialOperatorByName("encodeURLComponent") + .unparse(writer, call, 0, 0); + return; + } + + // refer to https://clickhouse.com/docs/sql-reference/functions/url-functions#decodeURLComponent + if (call.getOperator() == SqlLibraryOperators.URL_DECODE) { + RelToSqlConverterUtil.specialOperatorByName("decodeURLComponent") + .unparse(writer, call, 0, 0); + return; + } + switch (call.getKind()) { case MAP_VALUE_CONSTRUCTOR: writer.print(call.getOperator().getName().toLowerCase(Locale.ROOT)); diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java index d161f774b5e7..a98f7fd1f8ff 100644 --- a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java +++ b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java @@ -3701,6 +3701,19 @@ private SqlDialect nonOrdinalDialect() { sql(query2).withSpark().ok(expected2); } + /** Test case for + * [CALCITE-7408] + * URL_ENCODE/URL_DECODE is unparsed incorrectly for ClickHouseSqlDialect. */ + @Test void testUrlencodeAndUrldecode() { + final String query = "SELECT URL_ENCODE(\"product_name\") from \"product\""; + final String expected = "SELECT encodeURLComponent(`product_name`)\nFROM `foodmart`.`product`"; + sql(query).withLibrary(SqlLibrary.SPARK).withClickHouse().ok(expected); + + final String query1 = "SELECT URL_DECODE(\"product_name\") from \"product\""; + final String expected1 = "SELECT decodeURLComponent(`product_name`)\nFROM `foodmart`.`product`"; + sql(query1).withLibrary(SqlLibrary.SPARK).withClickHouse().ok(expected1); + } + @Test void testInstrFunction4Operands() { final String query = "SELECT INSTR('ABC', 'A', 1, 1) from \"product\""; final String expectedBQ = "SELECT INSTR('ABC', 'A', 1, 1)\n" From e315698a1f083403f5fd1986bf50389b60207f26 Mon Sep 17 00:00:00 2001 From: iwanttobepowerful <745778074@qq.com> Date: Tue, 10 Feb 2026 10:17:00 +0800 Subject: [PATCH 151/562] [CALCITE-7057] NPE when decorrelating query containing nested correlated subqueries --- .../calcite/rel/rules/SubQueryRemoveRule.java | 5 + .../calcite/sql2rel/RelDecorrelatorTest.java | 98 +++++++++++++++++++ core/src/test/resources/sql/scalar.iq | 10 ++ 3 files changed, 113 insertions(+) diff --git a/core/src/main/java/org/apache/calcite/rel/rules/SubQueryRemoveRule.java b/core/src/main/java/org/apache/calcite/rel/rules/SubQueryRemoveRule.java index 37e92aa23aef..9fa5f612ead8 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/SubQueryRemoveRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/SubQueryRemoveRule.java @@ -884,6 +884,7 @@ private static List fields(RelBuilder builder, int fieldCount) { private static void matchProject(SubQueryRemoveRule rule, RelOptRuleCall call) { final Project project = call.rel(0); + final Set projectVariablesSet = project.getVariablesSet(); final RelBuilder builder = call.builder(); final RexSubQuery e = requireNonNull(RexUtil.SubQueryFinder.find(project.getProjects())); @@ -894,6 +895,10 @@ private static void matchProject(SubQueryRemoveRule rule, final int fieldCount = builder.peek().getRowType().getFieldCount(); final Set variablesSet = RelOptUtil.getVariablesUsed(e.rel); + if (!projectVariablesSet.isEmpty()) { + // Only consider the correlated variables which originated from this sub-query level. + variablesSet.retainAll(projectVariablesSet); + } final RexNode target = rule.apply(e, variablesSet, logic, builder, 1, fieldCount, 0); final RexShuttle shuttle = new ReplaceSubQueryShuttle(e, target); diff --git a/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java b/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java index 7ddaeaf42027..1fabc39d8cc7 100644 --- a/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java +++ b/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java @@ -208,6 +208,104 @@ public static Frameworks.ConfigBuilder config() { assertThat(after, hasTree(planAfter)); } + /** Test case for [CALCITE-7057] + * NPE when decorrelating query containing nested correlated subqueries. */ + @Test void test7057() { + final FrameworkConfig frameworkConfig = config().build(); + final RelBuilder builder = RelBuilder.create(frameworkConfig); + final RelOptCluster cluster = builder.getCluster(); + final Planner planner = Frameworks.getPlanner(frameworkConfig); + final String sql = "" + + "select\n" + + " (select ename || ' from dept '\n" + + " || (select dname from dept where deptno = emp.deptno and emp.empno = empnos.empno)\n" + + " from emp\n" + + " ) as ename_from_dept\n" + + "from (values (7369), (7499)) as empnos(empno) order by 1"; + final RelNode originalRel; + try { + final SqlNode parse = planner.parse(sql); + final SqlNode validate = planner.validate(parse); + originalRel = planner.rel(validate).rel; + } catch (Exception e) { + throw TestUtil.rethrow(e); + } + + final HepProgram hepProgram = HepProgram.builder() + .addRuleCollection( + ImmutableList.of( + // SubQuery program rules + CoreRules.FILTER_SUB_QUERY_TO_CORRELATE, + CoreRules.PROJECT_SUB_QUERY_TO_CORRELATE, + CoreRules.JOIN_SUB_QUERY_TO_CORRELATE)) + .build(); + final Program program = + Programs.of(hepProgram, true, + requireNonNull(cluster.getMetadataProvider())); + final RelNode before = + program.run(cluster.getPlanner(), originalRel, cluster.traitSet(), + Collections.emptyList(), Collections.emptyList()); + + // before fix: + // + // [01]LogicalSort(sort0=[$0], dir0=[ASC]) + // [02] LogicalProject(ENAME_FROM_DEPT=[$1]) + // [03] LogicalCorrelate(correlation=[$cor2], joinType=[left], requiredColumns=[{0}]) + // [04] LogicalValues(tuples=[[{ 7369 }, { 7499 }]]) + // [05] LogicalAggregate(group=[{}], agg#0=[SINGLE_VALUE($0)]) + // [06] LogicalProject(EXPR$0=[||(||($1, ' from dept '), $8)]) + // [07] LogicalJoin(condition=[true], joinType=[left], variablesSet=[[$cor0, $cor2]]) + // [08] LogicalTableScan(table=[[scott, EMP]]) + // [09] LogicalAggregate(group=[{}], agg#0=[SINGLE_VALUE($0)]) + // [10] LogicalProject(DNAME=[$1]) + // [11] LogicalFilter(condition=[AND(=($0, $cor0.DEPTNO), + // =(CAST($cor0.EMPNO):INTEGER NOT NULL, $cor2.EMPNO))]) + // [12] LogicalTableScan(table=[[scott, DEPT]]) + // + // diff is [07]LogicalJoin(condition=[true], joinType=[left], variablesSet=[[$cor0, $cor2]]) + // LogicalCorrelate(correlation=[$cor0], joinType=[left], requiredColumns=[{0, 7}]) + // + final String planBefore = "" + + "LogicalSort(sort0=[$0], dir0=[ASC])\n" + + " LogicalProject(ENAME_FROM_DEPT=[$1])\n" + + " LogicalCorrelate(correlation=[$cor2], joinType=[left], requiredColumns=[{0}])\n" + + " LogicalValues(tuples=[[{ 7369 }, { 7499 }]])\n" + + " LogicalAggregate(group=[{}], agg#0=[SINGLE_VALUE($0)])\n" + + " LogicalProject(EXPR$0=[||(||($1, ' from dept '), $8)])\n" + + " LogicalCorrelate(correlation=[$cor0], joinType=[left], requiredColumns=[{0, 7}])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalAggregate(group=[{}], agg#0=[SINGLE_VALUE($0)])\n" + + " LogicalProject(DNAME=[$1])\n" + + " LogicalFilter(condition=[AND(=($0, $cor0.DEPTNO), =(CAST($cor0.EMPNO):INTEGER NOT NULL, $cor2.EMPNO))])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n"; + assertThat(before, hasTree(planBefore)); + + // Decorrelate without any rules, just "purely" decorrelation algorithm on RelDecorrelator + final RelNode after = + RelDecorrelator.decorrelateQuery(before, builder, RuleSets.ofList(Collections.emptyList()), + RuleSets.ofList(Collections.emptyList())); + final String planAfter = "" + + "LogicalSort(sort0=[$0], dir0=[ASC])\n" + + " LogicalProject(ENAME_FROM_DEPT=[$2])\n" + + " LogicalJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left])\n" + + " LogicalValues(tuples=[[{ 7369 }, { 7499 }]])\n" + + " LogicalAggregate(group=[{0}], agg#0=[SINGLE_VALUE($1)])\n" + + " LogicalProject(EMPNO1=[$12], EXPR$0=[||(||($1, ' from dept '), $13)])\n" + + " LogicalJoin(condition=[AND(=($7, $10), =($9, $11))], joinType=[left])\n" + + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7], DEPTNO0=[$7], EMPNO0=[CAST($0):INTEGER NOT NULL])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalAggregate(group=[{0, 1, 2}], agg#0=[SINGLE_VALUE($3)])\n" + + " LogicalProject(DEPTNO0=[$3], EMPNO0=[$4], EMPNO=[$5], DNAME=[$1])\n" + + " LogicalJoin(condition=[=($0, $3)], joinType=[inner])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n" + + " LogicalJoin(condition=[=($1, $2)], joinType=[inner])\n" + + " LogicalAggregate(group=[{0, 1}])\n" + + " LogicalProject(DEPTNO=[$7], EMPNO0=[CAST($0):INTEGER NOT NULL])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalValues(tuples=[[{ 7369 }, { 7499 }]])\n"; + assertThat(after, hasTree(planAfter)); + } + @Test void testDecorrelateCountBug() { final FrameworkConfig frameworkConfig = config().build(); final RelBuilder builder = RelBuilder.create(frameworkConfig); diff --git a/core/src/test/resources/sql/scalar.iq b/core/src/test/resources/sql/scalar.iq index 6eeb2167c211..d82d69f58901 100644 --- a/core/src/test/resources/sql/scalar.iq +++ b/core/src/test/resources/sql/scalar.iq @@ -451,4 +451,14 @@ EnumerableCalc(expr#0..10=[{inputs}], EMPNO=[$t0], $f1=[$t10]) # Reset to default value true !set trimfields true +# [CALCITE-7057] NPE when decorrelating query containing nested correlated subqueries +# Nested scalar sub-queries +select + (select ename || ' from dept ' + || (select dname from dept where deptno = emp.deptno and emp.empno = empnos.empno) + from emp + ) as ename_from_dept +from (values (7369), (7499)) as empnos(empno) order by 1; +more than one value in agg SINGLE_VALUE +!error # End scalar.iq From 8d8853717462b54ce49b9b25d158b69608ee3499 Mon Sep 17 00:00:00 2001 From: Silun Dong Date: Wed, 11 Feb 2026 22:18:50 +0800 Subject: [PATCH 152/562] [CALCITE-7414] Incorrect mapping of CorDef after decorrelating a Join in TopDownGeneralDecorrelator --- .../sql2rel/TopDownGeneralDecorrelator.java | 110 +++++++++++++----- core/src/test/resources/sql/new-decorr.iq | 101 ++++++++++++++++ 2 files changed, 179 insertions(+), 32 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java b/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java index 50b200615458..82465799460f 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java @@ -53,7 +53,6 @@ import org.apache.calcite.sql2rel.RelDecorrelator.Frame; import org.apache.calcite.tools.RelBuilder; import org.apache.calcite.util.ImmutableBitSet; -import org.apache.calcite.util.Litmus; import org.apache.calcite.util.Pair; import org.apache.calcite.util.ReflectUtil; import org.apache.calcite.util.ReflectiveVisitor; @@ -317,14 +316,16 @@ private RelNode correlateElimination(RelNode rel, boolean allowEmptyOutputFromRe if (!hasParent) { // ensure that the fields are in the same order as in the original plan. - builder.push(unnestedRel); UnnestedQuery unnestedQuery = - UnnestedQuery.createJoinUnnestInfo( + UnnestedQuery.createJoinUnnestedQuery( leftInfo, rightInfo, correlate, unnestedRel, - correlate.getJoinType()); + correlate.getJoinType(), + builder, + corDefs); + builder.push(unnestedQuery.r); List projects = builder.fields(new ArrayList<>(unnestedQuery.oldToNewOutputs.values())); unnestedRel = builder.project(projects).build(); @@ -669,10 +670,10 @@ public RelNode unnestInternal(Correlate correlate, boolean allowEmptyOutputFromR UnnestedQuery rightInfo = requireNonNull(subDecorrelator.mapRelToUnnestedQuery.get(correlate.getRight())); UnnestedQuery unnestedQuery = - UnnestedQuery.createJoinUnnestInfo(leftInfo, rightInfo, correlate, - newJoin, correlate.getJoinType()); + UnnestedQuery.createJoinUnnestedQuery(leftInfo, rightInfo, correlate, + newJoin, correlate.getJoinType(), builder, corDefs); mapRelToUnnestedQuery.put(correlate, unnestedQuery); - return newJoin; + return unnestedQuery.r; } public RelNode unnestInternal(Join join, boolean allowEmptyOutputFromRewrite) { @@ -745,14 +746,16 @@ public RelNode unnestInternal(Join join, boolean allowEmptyOutputFromRewrite) { corDefs); RelNode newJoin = builder.join(join.getJoinType(), newJoinCondition).build(); UnnestedQuery unnestedQuery = - UnnestedQuery.createJoinUnnestInfo( + UnnestedQuery.createJoinUnnestedQuery( leftInfo, rightInfo, join, newJoin, - join.getJoinType()); + join.getJoinType(), + builder, + corDefs); mapRelToUnnestedQuery.put(join, unnestedQuery); - return newJoin; + return unnestedQuery.r; } public RelNode unnestInternal(SetOp setOp, boolean allowEmptyOutputFromRewrite) { @@ -989,23 +992,27 @@ static class UnnestedQuery extends Frame { /** * Create UnnestedQuery for Join/Correlate after decorrelating. * - * @param leftInfo UnnestedQuery of the left side - * @param rightInfo UnnestedQuery of the right side - * @param oriJoinNode original Join/Correlate node - * @param unnestedJoinNode new node after decorrelating - * @param joinRelType join type of original Join/Correlate + * @param leftUnnestedQuery UnnestedQuery of the left side + * @param rightUnnestedQuery UnnestedQuery of the right side + * @param oriJoinNode original Join/Correlate node + * @param unnestedJoinNode new node after decorrelating + * @param joinRelType join type of original Join/Correlate + * @param builder RelBuilder + * @param corDefs the CorDef in the current decorrelator context * @return UnnestedQuery */ - private static UnnestedQuery createJoinUnnestInfo( - UnnestedQuery leftInfo, - UnnestedQuery rightInfo, + private static UnnestedQuery createJoinUnnestedQuery( + UnnestedQuery leftUnnestedQuery, + UnnestedQuery rightUnnestedQuery, RelNode oriJoinNode, RelNode unnestedJoinNode, - JoinRelType joinRelType) { + JoinRelType joinRelType, + RelBuilder builder, + NavigableSet corDefs) { Map oldToNewOutputs = new HashMap<>(); - oldToNewOutputs.putAll(leftInfo.oldToNewOutputs); - int oriLeftFieldCount = leftInfo.oldRel.getRowType().getFieldCount(); - int newLeftFieldCount = leftInfo.r.getRowType().getFieldCount(); + oldToNewOutputs.putAll(leftUnnestedQuery.oldToNewOutputs); + int oriLeftFieldCount = leftUnnestedQuery.oldRel.getRowType().getFieldCount(); + int newLeftFieldCount = leftUnnestedQuery.r.getRowType().getFieldCount(); switch (joinRelType) { case SEMI: case ANTI: @@ -1014,24 +1021,63 @@ private static UnnestedQuery createJoinUnnestInfo( oldToNewOutputs.put(oriLeftFieldCount, newLeftFieldCount); break; default: - rightInfo.oldToNewOutputs.forEach((oriIndex, newIndex) -> + rightUnnestedQuery.oldToNewOutputs.forEach((oriIndex, newIndex) -> oldToNewOutputs.put( requireNonNull(oriIndex, "oriIndex") + oriLeftFieldCount, requireNonNull(newIndex, "newIndex") + newLeftFieldCount)); break; } + // we have to take the join type into account to decide which side of the join to use for + // mapping CorDef to output index. See section 3.3 in paper Improving Unnesting of Complex + // Queries TreeMap corDefOutputs = new TreeMap<>(); - if (!leftInfo.corDefOutputs.isEmpty()) { - corDefOutputs.putAll(leftInfo.corDefOutputs); - } else if (!rightInfo.corDefOutputs.isEmpty()) { - Litmus.THROW.check(joinRelType.projectsRight(), - "If the joinType doesn't project right, its left side must have UnnestInfo."); - rightInfo.corDefOutputs.forEach((corDef, index) -> + switch (joinRelType) { + case SEMI: + case ANTI: + case LEFT_MARK: + case LEFT: + // if output only includes the left, or the unmatched rows from the left, + // we use the left for mapping. + corDefOutputs.putAll(leftUnnestedQuery.corDefOutputs); + break; + case RIGHT: + // if the unmatched rows from the right, we use the right for mapping. + rightUnnestedQuery.corDefOutputs.forEach((corDef, index) -> corDefOutputs.put(corDef, index + newLeftFieldCount)); - } else { - throw new IllegalArgumentException("The UnnestInfo for both sides of Join/Correlate that " - + "has correlation should not all be empty."); + break; + case FULL: + // when full outer join, we must use COALESCE(left_cor_index, right_cor_index) to map the + // CorDef, so we need to add a Project on top of the Join. + builder.push(unnestedJoinNode); + List projects = new ArrayList<>(builder.fields()); + for (CorDef corDef : corDefs) { + int leftIndex = requireNonNull(leftUnnestedQuery.corDefOutputs.get(corDef)); + int rightIndex = requireNonNull(rightUnnestedQuery.corDefOutputs.get(corDef)); + RexNode coalesce = + builder.call( + SqlStdOperatorTable.COALESCE, + builder.field(leftIndex), + builder.field(rightIndex + newLeftFieldCount)); + projects.add(coalesce); + corDefOutputs.put(corDef, projects.size() - 1); + } + unnestedJoinNode = builder.project(projects).build(); + break; + case INNER: + // when inner join, we can use either side that contains D for mapping. + if (!leftUnnestedQuery.corDefOutputs.isEmpty()) { + corDefOutputs.putAll(leftUnnestedQuery.corDefOutputs); + } else if (!rightUnnestedQuery.corDefOutputs.isEmpty()) { + rightUnnestedQuery.corDefOutputs.forEach((corDef, index) -> + corDefOutputs.put(corDef, index + newLeftFieldCount)); + } else { + throw new IllegalArgumentException("The UnnestInfo for both sides of Join/Correlate that " + + "has correlation should not all be empty."); + } + break; + default: + throw new UnsupportedOperationException("Unsupported join type : " + joinRelType); } return new UnnestedQuery(oriJoinNode, unnestedJoinNode, corDefOutputs, oldToNewOutputs); } diff --git a/core/src/test/resources/sql/new-decorr.iq b/core/src/test/resources/sql/new-decorr.iq index 4c2e5791af37..73ed32f38648 100644 --- a/core/src/test/resources/sql/new-decorr.iq +++ b/core/src/test/resources/sql/new-decorr.iq @@ -300,4 +300,105 @@ FROM dept; !ok +# [CALCITE-7414] Incorrect mapping of CorDef after decorrelating a Join in TopDownGeneralDecorrelator +# These cases come from sub-query.iq [CALCITE-7379] +SELECT e.ename, e.job, e.sal +FROM emp e +WHERE EXISTS ( + SELECT 1 + FROM ( + SELECT * FROM bonus b WHERE b.ename = e.ename + ) foo + LEFT JOIN dept d + ON foo.job = d.loc +); ++-------+-----+-----+ +| ENAME | JOB | SAL | ++-------+-----+-----+ ++-------+-----+-----+ +(0 rows) + +!ok + +SELECT e.ename, e.job, e.sal +FROM emp e +WHERE EXISTS ( + SELECT 1 + FROM ( + SELECT * FROM bonus b WHERE b.ename = e.ename + ) foo + RIGHT JOIN dept d + ON foo.job = d.loc +); ++--------+-----------+---------+ +| ENAME | JOB | SAL | ++--------+-----------+---------+ +| ADAMS | CLERK | 1100.00 | +| ALLEN | SALESMAN | 1600.00 | +| BLAKE | MANAGER | 2850.00 | +| CLARK | MANAGER | 2450.00 | +| FORD | ANALYST | 3000.00 | +| JAMES | CLERK | 950.00 | +| JONES | MANAGER | 2975.00 | +| KING | PRESIDENT | 5000.00 | +| MARTIN | SALESMAN | 1250.00 | +| MILLER | CLERK | 1300.00 | +| SCOTT | ANALYST | 3000.00 | +| SMITH | CLERK | 800.00 | +| TURNER | SALESMAN | 1500.00 | +| WARD | SALESMAN | 1250.00 | ++--------+-----------+---------+ +(14 rows) + +!ok + +SELECT e.ename, e.job, e.sal +FROM emp e +WHERE EXISTS ( + SELECT 1 + FROM ( + SELECT * FROM bonus b WHERE b.ename = e.ename + ) foo + FULL JOIN dept d + ON foo.job = d.loc +); ++--------+-----------+---------+ +| ENAME | JOB | SAL | ++--------+-----------+---------+ +| ADAMS | CLERK | 1100.00 | +| ALLEN | SALESMAN | 1600.00 | +| BLAKE | MANAGER | 2850.00 | +| CLARK | MANAGER | 2450.00 | +| FORD | ANALYST | 3000.00 | +| JAMES | CLERK | 950.00 | +| JONES | MANAGER | 2975.00 | +| KING | PRESIDENT | 5000.00 | +| MARTIN | SALESMAN | 1250.00 | +| MILLER | CLERK | 1300.00 | +| SCOTT | ANALYST | 3000.00 | +| SMITH | CLERK | 800.00 | +| TURNER | SALESMAN | 1500.00 | +| WARD | SALESMAN | 1250.00 | ++--------+-----------+---------+ +(14 rows) + +!ok + +!if (use_new_decorr) { +EnumerableCalc(expr#0..3=[{inputs}], ENAME=[$t1], JOB=[$t2], SAL=[$t3]) + EnumerableHashJoin(condition=[IS NOT DISTINCT FROM($1, $4)], joinType=[semi]) + EnumerableCalc(expr#0..7=[{inputs}], proj#0..2=[{exprs}], SAL=[$t5]) + EnumerableTableScan(table=[[scott, EMP]]) + EnumerableCalc(expr#0..5=[{inputs}], expr#6=[COALESCE($t2, $t5)], $f10=[$t6]) + EnumerableHashJoin(condition=[AND(=($1, $4), IS NOT DISTINCT FROM($2, $5))], joinType=[full]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[CAST($t1):VARCHAR(13)], expr#5=[IS NOT NULL($t0)], ENAME=[$t0], JOB0=[$t4], ENAME0=[$t0], $condition=[$t5]) + EnumerableTableScan(table=[[scott, BONUS]]) + EnumerableNestedLoopJoin(condition=[true], joinType=[inner]) + EnumerableCalc(expr#0..2=[{inputs}], DEPTNO=[$t0], LOC=[$t2]) + EnumerableTableScan(table=[[scott, DEPT]]) + EnumerableCalc(expr#0..7=[{inputs}], ENAME=[$t1]) + EnumerableTableScan(table=[[scott, EMP]]) +!plan +!} + # End new-decorr.iq From 8718ee7633c1c711ae057fb75ffb8fc3452db400 Mon Sep 17 00:00:00 2001 From: Silun Date: Thu, 12 Feb 2026 17:08:40 +0800 Subject: [PATCH 153/562] Included the sub-query.iq in CoreQuidemTest2 --- .../apache/calcite/test/CoreQuidemTest2.java | 1 - core/src/test/resources/sql/sub-query.iq | 410 ++++++++++++++---- 2 files changed, 324 insertions(+), 87 deletions(-) diff --git a/core/src/test/java/org/apache/calcite/test/CoreQuidemTest2.java b/core/src/test/java/org/apache/calcite/test/CoreQuidemTest2.java index b8ef8562bc8c..0a55d7764b96 100644 --- a/core/src/test/java/org/apache/calcite/test/CoreQuidemTest2.java +++ b/core/src/test/java/org/apache/calcite/test/CoreQuidemTest2.java @@ -46,7 +46,6 @@ public static void main(String[] args) throws Exception { // TODO: Support measure paths.remove("sql/measure.iq"); - paths.remove("sql/sub-query.iq"); return paths; } diff --git a/core/src/test/resources/sql/sub-query.iq b/core/src/test/resources/sql/sub-query.iq index fbe51075cf24..5fb06553726a 100644 --- a/core/src/test/resources/sql/sub-query.iq +++ b/core/src/test/resources/sql/sub-query.iq @@ -34,6 +34,7 @@ where t1.x not in (select t2.x from t2); (0 rows) !ok +!if (use_old_decorr) { EnumerableCalc(expr#0..4=[{inputs}], expr#5=[0], expr#6=[=($t1, $t5)], expr#7=[IS NULL($t4)], expr#8=[>=($t2, $t1)], expr#9=[IS NOT NULL($t0)], expr#10=[AND($t7, $t8, $t9)], expr#11=[OR($t6, $t10)], X=[$t0], $condition=[$t11]) EnumerableMergeJoin(condition=[=($0, $3)], joinType=[left]) EnumerableNestedLoopJoin(condition=[true], joinType=[inner]) @@ -43,6 +44,7 @@ EnumerableCalc(expr#0..4=[{inputs}], expr#5=[0], expr#6=[=($t1, $t5)], expr#7=[I EnumerableCalc(expr#0=[{inputs}], expr#1=[true], proj#0..1=[{exprs}]) EnumerableValues(tuples=[[{ 1 }, { null }]]) !plan +!} # Use of case is to get around issue with directly specifying null in values # list. Postgres gives 0 rows. @@ -259,8 +261,10 @@ select * from dept where deptno not in (select deptno from emp where false); (4 rows) !ok +!if (use_old_decorr) { EnumerableValues(tuples=[[{ 10, 'Sales ' }, { 20, 'Marketing ' }, { 30, 'Engineering' }, { 40, 'Empty ' }]]) !plan +!} select deptno, deptno in (select deptno from emp where false) from dept; +--------+--------+ @@ -274,9 +278,11 @@ select deptno, deptno in (select deptno from emp where false) from dept; (4 rows) !ok +!if (use_old_decorr) { EnumerableCalc(expr#0..1=[{inputs}], expr#2=[false], expr#3=[CAST($t2):BOOLEAN], DEPTNO=[$t0], EXPR$1=[$t3]) EnumerableValues(tuples=[[{ 10, 'Sales ' }, { 20, 'Marketing ' }, { 30, 'Engineering' }, { 40, 'Empty ' }]]) !plan +!} select deptno, deptno not in (select deptno from emp where false) from dept; +--------+--------+ @@ -429,6 +435,7 @@ where e.job in ( (5 rows) !ok +!if (use_old_decorr) { EnumerableCalc(expr#0..4=[{inputs}], EMPNO=[$t0]) EnumerableHashJoin(condition=[=($2, $5)], joinType=[semi]) EnumerableCalc(expr#0..4=[{inputs}], EMPNO=[$t2], JOB=[$t3], DEPTNO=[$t4], JOB0=[$t0], DEPTNO0=[$t1]) @@ -448,6 +455,7 @@ EnumerableCalc(expr#0..4=[{inputs}], EMPNO=[$t0]) EnumerableCalc(expr#0..2=[{inputs}], DEPTNO=[$t0]) EnumerableTableScan(table=[[scott, DEPT]]) !plan +!} # [CALCITE-6824] Subquery in join conditions rewrite fails if referencing a column from the right-hand side table select empno from "scott".emp where (empno not in (select dept.deptno from dept)) @@ -459,6 +467,7 @@ in (select deptno = 0 from dept); (0 rows) !ok +!if (use_old_decorr) { EnumerableCalc(expr#0..3=[{inputs}], EMPNO=[$t0]) EnumerableNestedLoopJoin(condition=[=(IS NULL($2), $3)], joinType=[inner]) EnumerableMergeJoin(condition=[=($0, $1)], joinType=[left]) @@ -472,6 +481,7 @@ EnumerableCalc(expr#0..3=[{inputs}], EMPNO=[$t0]) EnumerableCalc(expr#0..2=[{inputs}], expr#3=[CAST($t0):INTEGER NOT NULL], expr#4=[0], expr#5=[=($t3, $t4)], EXPR$0=[$t5]) EnumerableTableScan(table=[[scott, DEPT]]) !plan +!} # [CALCITE-6824] Subquery in join conditions rewrite fails if referencing a column from the right-hand side table SELECT empno FROM emp JOIN dept on emp.deptno <= ALL(SELECT deptno FROM dept) and emp.deptno = dept.deptno; @@ -485,6 +495,7 @@ SELECT empno FROM emp JOIN dept on emp.deptno <= ALL(SELECT deptno FROM dept) an (3 rows) !ok +!if (use_old_decorr) { EnumerableCalc(expr#0..4=[{inputs}], EMPNO=[$t0]) EnumerableHashJoin(condition=[=($1, $5)], joinType=[semi]) EnumerableNestedLoopJoin(condition=[OR(=($3, 0), AND(<=($1, $2), IS NOT TRUE(OR(>($1, $2), >($3, $4)))))], joinType=[inner]) @@ -496,6 +507,7 @@ EnumerableCalc(expr#0..4=[{inputs}], EMPNO=[$t0]) EnumerableCalc(expr#0..2=[{inputs}], DEPTNO=[$t0]) EnumerableTableScan(table=[[scott, DEPT]]) !plan +!} # [CALCITE-6824] Subquery in join conditions rewrite fails if referencing a column from the right-hand side table SELECT empno FROM emp JOIN dept on emp.deptno = (SELECT min(deptno) FROM dept) and emp.deptno = dept.deptno; @@ -509,6 +521,7 @@ SELECT empno FROM emp JOIN dept on emp.deptno = (SELECT min(deptno) FROM dept) a (3 rows) !ok +!if (use_old_decorr) { EnumerableCalc(expr#0..2=[{inputs}], EMPNO=[$t0]) EnumerableHashJoin(condition=[=($2, $3)], joinType=[semi]) EnumerableCalc(expr#0..2=[{inputs}], EMPNO=[$t1], DEPTNO=[$t2], EXPR$0=[$t0]) @@ -520,6 +533,7 @@ EnumerableCalc(expr#0..2=[{inputs}], EMPNO=[$t0]) EnumerableCalc(expr#0..2=[{inputs}], DEPTNO=[$t0]) EnumerableTableScan(table=[[scott, DEPT]]) !plan +!} # Correlated NOT IN sub-query in WHERE clause of JOIN select empno from "scott".emp as e @@ -542,6 +556,7 @@ where e.job not in ( (9 rows) !ok +!if (use_old_decorr) { EnumerableCalc(expr#0..9=[{inputs}], expr#10=[0], expr#11=[=($t5, $t10)], expr#12=[IS NULL($t1)], expr#13=[IS NOT NULL($t9)], expr#14=[<($t6, $t5)], expr#15=[OR($t12, $t13, $t14)], expr#16=[IS NOT TRUE($t15)], expr#17=[OR($t11, $t16)], EMPNO=[$t0], $condition=[$t17]) EnumerableMergeJoin(condition=[AND(=($1, $7), =($2, $8))], joinType=[left]) EnumerableSort(sort0=[$1], sort1=[$2], dir0=[ASC], dir1=[ASC]) @@ -584,6 +599,7 @@ EnumerableCalc(expr#0..9=[{inputs}], expr#10=[0], expr#11=[=($t5, $t10)], expr#1 EnumerableCalc(expr#0..2=[{inputs}], DEPTNO=[$t0]) EnumerableTableScan(table=[[scott, DEPT]]) !plan +!} # Condition that returns a NULL key. # Tested on Oracle. @@ -603,7 +619,7 @@ where sal + 100 not in ( # [CALCITE-6652] RelDecorrelator can't decorrelate query with limit 1 SELECT dname FROM "scott".dept WHERE 2000 > (SELECT emp.sal FROM "scott".emp where dept.deptno = emp.deptno ORDER BY emp.sal limit 1); - +!if (use_old_decorr) { EnumerableCalc(expr#0..1=[{inputs}], DNAME=[$t1]) EnumerableHashJoin(condition=[=($0, $2)], joinType=[semi]) EnumerableCalc(expr#0..2=[{inputs}], proj#0..1=[{exprs}]) @@ -613,6 +629,7 @@ EnumerableCalc(expr#0..1=[{inputs}], DNAME=[$t1]) EnumerableCalc(expr#0..7=[{inputs}], expr#8=[IS NOT NULL($t7)], proj#0..7=[{exprs}], $condition=[$t8]) EnumerableTableScan(table=[[scott, EMP]]) !plan +!} +------------+ | DNAME | +------------+ @@ -626,7 +643,7 @@ EnumerableCalc(expr#0..1=[{inputs}], DNAME=[$t1]) # [CALCITE-6652] RelDecorrelator can't decorrelate query with limit 1 SELECT dname FROM "scott".dept WHERE 4000 > (SELECT emp.sal FROM "scott".emp where dept.deptno = emp.deptno ORDER BY emp.sal desc nulls last limit 1); - +!if (use_old_decorr) { EnumerableCalc(expr#0..1=[{inputs}], DNAME=[$t1]) EnumerableHashJoin(condition=[=($0, $2)], joinType=[semi]) EnumerableCalc(expr#0..2=[{inputs}], proj#0..1=[{exprs}]) @@ -636,6 +653,7 @@ EnumerableCalc(expr#0..1=[{inputs}], DNAME=[$t1]) EnumerableCalc(expr#0..7=[{inputs}], expr#8=[IS NOT NULL($t7)], proj#0..7=[{exprs}], $condition=[$t8]) EnumerableTableScan(table=[[scott, EMP]]) !plan +!} +----------+ | DNAME | +----------+ @@ -649,7 +667,7 @@ EnumerableCalc(expr#0..1=[{inputs}], DNAME=[$t1]) # [CALCITE-6652] RelDecorrelator can't decorrelate query with limit 1 # The case of the subquery that returns 0 rows SELECT dname FROM "scott".dept WHERE 2000 > (SELECT emp.sal FROM "scott".emp where dept.deptno = emp.deptno and mgr > 8000 ORDER BY emp.sal limit 1); - +!if (use_old_decorr) { EnumerableCalc(expr#0..1=[{inputs}], DNAME=[$t1]) EnumerableHashJoin(condition=[=($0, $2)], joinType=[semi]) EnumerableCalc(expr#0..2=[{inputs}], proj#0..1=[{exprs}]) @@ -659,6 +677,7 @@ EnumerableCalc(expr#0..1=[{inputs}], DNAME=[$t1]) EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t3):INTEGER], expr#9=[8000], expr#10=[>($t8, $t9)], expr#11=[IS NOT NULL($t7)], expr#12=[AND($t10, $t11)], proj#0..7=[{exprs}], $condition=[$t12]) EnumerableTableScan(table=[[scott, EMP]]) !plan +!} +-------+ | DNAME | +-------+ @@ -669,7 +688,7 @@ EnumerableCalc(expr#0..1=[{inputs}], DNAME=[$t1]) # [CALCITE-6652] RelDecorrelator can't decorrelate query with limit 1 SELECT dname FROM "scott".dept WHERE 2000 > (SELECT emp.sal FROM "scott".emp where dept.deptno = emp.deptno ORDER BY year(hiredate), emp.sal limit 1); - +!if (use_old_decorr) { EnumerableCalc(expr#0..3=[{inputs}], DNAME=[$t3]) EnumerableHashJoin(condition=[=($1, $2)], joinType=[inner]) EnumerableCalc(expr#0..3=[{inputs}], expr#4=[1], expr#5=[<=($t3, $t4)], expr#6=[2000.00:DECIMAL(12, 2)], expr#7=[CAST($t0):DECIMAL(12, 2)], expr#8=[>($t6, $t7)], expr#9=[AND($t5, $t8)], proj#0..1=[{exprs}], $condition=[$t9]) @@ -679,6 +698,7 @@ EnumerableCalc(expr#0..3=[{inputs}], DNAME=[$t3]) EnumerableCalc(expr#0..2=[{inputs}], proj#0..1=[{exprs}]) EnumerableTableScan(table=[[scott, DEPT]]) !plan +!} +----------+ | DNAME | +----------+ @@ -692,7 +712,7 @@ EnumerableCalc(expr#0..3=[{inputs}], DNAME=[$t3]) # [CALCITE-6652] RelDecorrelator can't decorrelate query with limit 1 # The case of the subquery that returns 0 rows SELECT dname FROM "scott".dept WHERE 2000 > (SELECT emp.sal FROM "scott".emp where dept.deptno = emp.deptno and mgr > 8000 ORDER BY year(hiredate), emp.sal limit 1); - +!if (use_old_decorr) { EnumerableCalc(expr#0..3=[{inputs}], DNAME=[$t3]) EnumerableHashJoin(condition=[=($1, $2)], joinType=[inner]) EnumerableCalc(expr#0..3=[{inputs}], expr#4=[1], expr#5=[<=($t3, $t4)], expr#6=[2000.00:DECIMAL(12, 2)], expr#7=[CAST($t0):DECIMAL(12, 2)], expr#8=[>($t6, $t7)], expr#9=[AND($t5, $t8)], proj#0..1=[{exprs}], $condition=[$t9]) @@ -702,6 +722,7 @@ EnumerableCalc(expr#0..3=[{inputs}], DNAME=[$t3]) EnumerableCalc(expr#0..2=[{inputs}], proj#0..1=[{exprs}]) EnumerableTableScan(table=[[scott, DEPT]]) !plan +!} +-------+ | DNAME | +-------+ @@ -712,7 +733,7 @@ EnumerableCalc(expr#0..3=[{inputs}], DNAME=[$t3]) # [CALCITE-6652] RelDecorrelator can't decorrelate query with limit 1 SELECT dname, (SELECT emp.sal FROM "scott".emp where dept.deptno = emp.deptno ORDER BY emp.sal desc nulls last limit 1) FROM "scott".dept; - +!if (use_old_decorr) { EnumerableCalc(expr#0..3=[{inputs}], DNAME=[$t1], EXPR$1=[$t3]) EnumerableMergeJoin(condition=[=($0, $2)], joinType=[left]) EnumerableCalc(expr#0..2=[{inputs}], proj#0..1=[{exprs}]) @@ -722,6 +743,7 @@ EnumerableCalc(expr#0..3=[{inputs}], DNAME=[$t1], EXPR$1=[$t3]) EnumerableCalc(expr#0..7=[{inputs}], expr#8=[IS NOT NULL($t7)], proj#0..7=[{exprs}], $condition=[$t8]) EnumerableTableScan(table=[[scott, EMP]]) !plan +!} +------------+---------+ | DNAME | EXPR$1 | +------------+---------+ @@ -737,7 +759,7 @@ EnumerableCalc(expr#0..3=[{inputs}], DNAME=[$t1], EXPR$1=[$t3]) # [CALCITE-6652] RelDecorrelator can't decorrelate query with limit 1 # subquery contains null SELECT dname, (SELECT emp.comm FROM "scott".emp where dept.deptno = emp.deptno ORDER BY emp.comm desc limit 1) FROM "scott".dept; - +!if (use_old_decorr) { EnumerableCalc(expr#0..4=[{inputs}], DNAME=[$t1], EXPR$1=[$t2]) EnumerableHashJoin(condition=[=($0, $3)], joinType=[left]) EnumerableCalc(expr#0..2=[{inputs}], proj#0..1=[{exprs}]) @@ -747,6 +769,7 @@ EnumerableCalc(expr#0..4=[{inputs}], DNAME=[$t1], EXPR$1=[$t2]) EnumerableCalc(expr#0..7=[{inputs}], expr#8=[IS NOT NULL($t7)], proj#0..7=[{exprs}], $condition=[$t8]) EnumerableTableScan(table=[[scott, EMP]]) !plan +!} +------------+--------+ | DNAME | EXPR$1 | +------------+--------+ @@ -762,7 +785,7 @@ EnumerableCalc(expr#0..4=[{inputs}], DNAME=[$t1], EXPR$1=[$t2]) # [CALCITE-6652] RelDecorrelator can't decorrelate query with limit 1 # subquery contains null SELECT dname, (SELECT emp.comm FROM "scott".emp where dept.deptno = emp.deptno ORDER BY emp.comm limit 1) FROM "scott".dept; - +!if (use_old_decorr) { EnumerableCalc(expr#0..3=[{inputs}], DNAME=[$t1], EXPR$1=[$t3]) EnumerableMergeJoin(condition=[=($0, $2)], joinType=[left]) EnumerableCalc(expr#0..2=[{inputs}], proj#0..1=[{exprs}]) @@ -772,6 +795,7 @@ EnumerableCalc(expr#0..3=[{inputs}], DNAME=[$t1], EXPR$1=[$t3]) EnumerableCalc(expr#0..7=[{inputs}], expr#8=[IS NOT NULL($t7)], proj#0..7=[{exprs}], $condition=[$t8]) EnumerableTableScan(table=[[scott, EMP]]) !plan +!} +------------+--------+ | DNAME | EXPR$1 | +------------+--------+ @@ -787,7 +811,7 @@ EnumerableCalc(expr#0..3=[{inputs}], DNAME=[$t1], EXPR$1=[$t3]) # [CALCITE-6652] RelDecorrelator can't decorrelate query with limit 1 # The case of the subquery that returns 0 rows SELECT dname, (SELECT emp.sal FROM "scott".emp where dept.deptno = emp.deptno and mgr > 8000 ORDER BY emp.sal limit 1) FROM "scott".dept; - +!if (use_old_decorr) { EnumerableCalc(expr#0..3=[{inputs}], DNAME=[$t1], EXPR$1=[$t3]) EnumerableMergeJoin(condition=[=($0, $2)], joinType=[left]) EnumerableCalc(expr#0..2=[{inputs}], proj#0..1=[{exprs}]) @@ -797,6 +821,7 @@ EnumerableCalc(expr#0..3=[{inputs}], DNAME=[$t1], EXPR$1=[$t3]) EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t3):INTEGER], expr#9=[8000], expr#10=[>($t8, $t9)], expr#11=[IS NOT NULL($t7)], expr#12=[AND($t10, $t11)], proj#0..7=[{exprs}], $condition=[$t12]) EnumerableTableScan(table=[[scott, EMP]]) !plan +!} +------------+--------+ | DNAME | EXPR$1 | +------------+--------+ @@ -811,7 +836,7 @@ EnumerableCalc(expr#0..3=[{inputs}], DNAME=[$t1], EXPR$1=[$t3]) # [CALCITE-6652] RelDecorrelator can't decorrelate query with limit 1 SELECT dname, (SELECT emp.sal FROM "scott".emp where dept.deptno = emp.deptno ORDER BY year(hiredate), emp.sal limit 1) FROM "scott".dept; - +!if (use_old_decorr) { EnumerableCalc(expr#0..3=[{inputs}], DNAME=[$t1], EXPR$1=[$t2]) EnumerableHashJoin(condition=[=($0, $3)], joinType=[left]) EnumerableCalc(expr#0..2=[{inputs}], proj#0..1=[{exprs}]) @@ -821,6 +846,7 @@ EnumerableCalc(expr#0..3=[{inputs}], DNAME=[$t1], EXPR$1=[$t2]) EnumerableCalc(expr#0..7=[{inputs}], expr#8=[FLAG(YEAR)], expr#9=[EXTRACT($t8, $t4)], expr#10=[IS NOT NULL($t7)], SAL=[$t5], DEPTNO=[$t7], $2=[$t9], $condition=[$t10]) EnumerableTableScan(table=[[scott, EMP]]) !plan +!} +------------+---------+ | DNAME | EXPR$1 | +------------+---------+ @@ -836,7 +862,7 @@ EnumerableCalc(expr#0..3=[{inputs}], DNAME=[$t1], EXPR$1=[$t2]) # [CALCITE-6652] RelDecorrelator can't decorrelate query with limit 1 # The case of the subquery that returns 0 rows SELECT dname, (SELECT emp.sal FROM "scott".emp where dept.deptno = emp.deptno and mgr > 8000 ORDER BY year(hiredate), emp.sal limit 1) FROM "scott".dept; - +!if (use_old_decorr) { EnumerableCalc(expr#0..3=[{inputs}], DNAME=[$t1], EXPR$1=[$t2]) EnumerableHashJoin(condition=[=($0, $3)], joinType=[left]) EnumerableCalc(expr#0..2=[{inputs}], proj#0..1=[{exprs}]) @@ -846,6 +872,7 @@ EnumerableCalc(expr#0..3=[{inputs}], DNAME=[$t1], EXPR$1=[$t2]) EnumerableCalc(expr#0..7=[{inputs}], expr#8=[FLAG(YEAR)], expr#9=[EXTRACT($t8, $t4)], expr#10=[CAST($t3):INTEGER], expr#11=[8000], expr#12=[>($t10, $t11)], expr#13=[IS NOT NULL($t7)], expr#14=[AND($t12, $t13)], SAL=[$t5], DEPTNO=[$t7], $2=[$t9], $condition=[$t14]) EnumerableTableScan(table=[[scott, EMP]]) !plan +!} +------------+--------+ | DNAME | EXPR$1 | +------------+--------+ @@ -911,6 +938,7 @@ where sal + 100 not in ( !} # [CALCITE-356] AssertionError while translating query with WITH and correlated sub-query +!if (use_old_decorr) { with t (a, b) as (select * from (values (1, 2))) select * from t where exists (select 1 from "scott".emp where deptno = t.a); EnumerableCalc(expr#0=[{inputs}], expr#1=[1], expr#2=[2], A=[$t1], B=[$t2]) @@ -918,8 +946,10 @@ EnumerableCalc(expr#0=[{inputs}], expr#1=[1], expr#2=[2], A=[$t1], B=[$t2]) EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t7):INTEGER], expr#9=[1], expr#10=[=($t9, $t8)], DEPTNO0=[$t8], $condition=[$t10]) EnumerableTableScan(table=[[scott, EMP]]) !plan +!} # Similar query, identical plan +!if (use_old_decorr) { with t as (select * from (values (1, 2)) as t(a, b)) select * from t where exists (select 1 from "scott".emp where deptno = t.a); EnumerableCalc(expr#0=[{inputs}], expr#1=[1], expr#2=[2], A=[$t1], B=[$t2]) @@ -927,15 +957,18 @@ EnumerableCalc(expr#0=[{inputs}], expr#1=[1], expr#2=[2], A=[$t1], B=[$t2]) EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t7):INTEGER], expr#9=[1], expr#10=[=($t9, $t8)], DEPTNO0=[$t8], $condition=[$t10]) EnumerableTableScan(table=[[scott, EMP]]) !plan +!} # Uncorrelated with t (a, b) as (select * from (values (60, 'b'))) select * from t where a in (select deptno from "scott".dept); +!if (use_old_decorr) { EnumerableCalc(expr#0=[{inputs}], expr#1=[60], expr#2=['b'], A=[$t1], B=[$t2]) EnumerableAggregate(group=[{0}]) EnumerableCalc(expr#0..2=[{inputs}], expr#3=[CAST($t0):INTEGER NOT NULL], expr#4=[60], expr#5=[=($t4, $t3)], DEPTNO=[$t3], $condition=[$t5]) EnumerableTableScan(table=[[scott, DEPT]]) !plan +!} +---+---+ | A | B | +---+---+ @@ -946,11 +979,13 @@ EnumerableCalc(expr#0=[{inputs}], expr#1=[60], expr#2=['b'], A=[$t1], B=[$t2]) with t (a, b) as (select * from (values (30, 'b'))) select * from t where a in (select deptno from "scott".dept); +!if (use_old_decorr) { EnumerableCalc(expr#0=[{inputs}], expr#1=[30], expr#2=['b'], A=[$t1], B=[$t2]) EnumerableAggregate(group=[{0}]) EnumerableCalc(expr#0..2=[{inputs}], expr#3=[CAST($t0):INTEGER NOT NULL], expr#4=[30], expr#5=[=($t4, $t3)], DEPTNO=[$t3], $condition=[$t5]) EnumerableTableScan(table=[[scott, DEPT]]) !plan +!} +----+---+ | A | B | +----+---+ @@ -1141,7 +1176,7 @@ FROM "scott".emp AS bosses; (14 rows) !ok - +!if (use_old_decorr) { EnumerableCalc(expr#0..3=[{inputs}], ENAME=[$t1], DEEP2SAL=[$t3]) EnumerableMergeJoin(condition=[=($0, $2)], joinType=[left]) EnumerableCalc(expr#0..7=[{inputs}], proj#0..1=[{exprs}]) @@ -1157,6 +1192,7 @@ EnumerableCalc(expr#0..3=[{inputs}], ENAME=[$t1], DEEP2SAL=[$t3]) EnumerableCalc(expr#0..7=[{inputs}], expr#8=[IS NOT NULL($t3)], proj#0..7=[{exprs}], $condition=[$t8]) EnumerableTableScan(table=[[scott, EMP]]) !plan +!} # [CALCITE-1494] Inefficient plan for correlated sub-queries # Plan must have only one scan each of emp and dept. @@ -1173,6 +1209,7 @@ where empno IN ( (0 rows) !ok +!if (use_old_decorr) { EnumerableCalc(expr#0..2=[{inputs}], SAL=[$t1]) EnumerableHashJoin(condition=[AND(=($2, $4), =($0, $3))], joinType=[semi]) EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t2):VARCHAR(14)], EMPNO=[$t0], SAL=[$t5], JOB0=[$t8]) @@ -1180,6 +1217,7 @@ EnumerableCalc(expr#0..2=[{inputs}], SAL=[$t1]) EnumerableCalc(expr#0..2=[{inputs}], expr#3=[CAST($t0):SMALLINT NOT NULL], expr#4=[IS NOT NULL($t1)], DEPTNO=[$t3], DNAME=[$t1], $condition=[$t4]) EnumerableTableScan(table=[[scott, DEPT]]) !plan +!} # As above, but for EXISTS select * @@ -1196,11 +1234,13 @@ where exists ( (1 row) !ok +!if (use_old_decorr) { EnumerableHashJoin(condition=[=($0, $3)], joinType=[semi]) EnumerableTableScan(table=[[scott, DEPT]]) EnumerableCalc(expr#0..7=[{inputs}], expr#8=['SMITH':VARCHAR(10)], expr#9=[=($t1, $t8)], expr#10=[IS NOT NULL($t7)], expr#11=[AND($t9, $t10)], DEPTNO=[$t7], $condition=[$t11]) EnumerableTableScan(table=[[scott, EMP]]) !plan +!} # [DRILL-5644] select TJOIN1.RNUM, TJOIN1.C1, @@ -1360,6 +1400,7 @@ from "scott".emp; (14 rows) !ok +!if (use_old_decorr) { EnumerableCalc(expr#0..3=[{inputs}], expr#4=[null:BOOLEAN], expr#5=[IS NOT NULL($t3)], expr#6=[AND($t4, $t5)], SAL=[$t1], EXPR$1=[$t6]) EnumerableNestedLoopJoin(condition=[true], joinType=[left]) EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], SAL=[$t5]) @@ -1370,6 +1411,7 @@ EnumerableCalc(expr#0..3=[{inputs}], expr#4=[null:BOOLEAN], expr#5=[IS NOT NULL( EnumerableCalc(expr#0..2=[{inputs}], expr#3=[false], cs=[$t3]) EnumerableTableScan(table=[[scott, DEPT]]) !plan +!} # Test project literal IN null non-correlated select sal, @@ -1398,6 +1440,7 @@ from "scott".emp; (14 rows) !ok +!if (use_old_decorr) { EnumerableCalc(expr#0..3=[{inputs}], expr#4=[IS FALSE($t2)], expr#5=[null:BOOLEAN], expr#6=[IS NOT NULL($t3)], expr#7=[AND($t4, $t5, $t6)], expr#8=[IS NOT NULL($t2)], expr#9=[IS NOT FALSE($t2)], expr#10=[AND($t8, $t6, $t9)], expr#11=[OR($t7, $t10)], SAL=[$t1], EXPR$1=[$t11]) EnumerableNestedLoopJoin(condition=[true], joinType=[left]) EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], SAL=[$t5]) @@ -1408,6 +1451,7 @@ EnumerableCalc(expr#0..3=[{inputs}], expr#4=[IS FALSE($t2)], expr#5=[null:BOOLEA EnumerableCalc(expr#0..2=[{inputs}], expr#3=[false], cs=[$t3]) EnumerableTableScan(table=[[scott, DEPT]]) !plan +!} # Test project null IN literal non-correlated select sal, @@ -1436,6 +1480,7 @@ from "scott".emp; (14 rows) !ok +!if (use_old_decorr) { EnumerableCalc(expr#0..3=[{inputs}], expr#4=[null:BOOLEAN], expr#5=[IS NOT NULL($t3)], expr#6=[AND($t4, $t5)], SAL=[$t1], EXPR$1=[$t6]) EnumerableNestedLoopJoin(condition=[true], joinType=[left]) EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], SAL=[$t5]) @@ -1446,6 +1491,7 @@ EnumerableCalc(expr#0..3=[{inputs}], expr#4=[null:BOOLEAN], expr#5=[IS NOT NULL( EnumerableCalc(expr#0..2=[{inputs}], expr#3=[true], cs=[$t3]) EnumerableTableScan(table=[[scott, DEPT]]) !plan +!} # Test project null IN required select sal, @@ -1474,6 +1520,7 @@ from "scott".emp; (14 rows) !ok +!if (use_old_decorr) { EnumerableCalc(expr#0..3=[{inputs}], expr#4=[null:BOOLEAN], expr#5=[IS NOT NULL($t3)], expr#6=[AND($t4, $t5)], SAL=[$t1], EXPR$1=[$t6]) EnumerableNestedLoopJoin(condition=[true], joinType=[left]) EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], SAL=[$t5]) @@ -1484,6 +1531,7 @@ EnumerableCalc(expr#0..3=[{inputs}], expr#4=[null:BOOLEAN], expr#5=[IS NOT NULL( EnumerableCalc(expr#0..2=[{inputs}], expr#3=[true], cs=[$t3]) EnumerableTableScan(table=[[scott, DEPT]]) !plan +!} # Test project null IN nullable select sal, @@ -1512,6 +1560,7 @@ from "scott".emp; (14 rows) !ok +!if (use_old_decorr) { EnumerableCalc(expr#0..3=[{inputs}], expr#4=[null:BOOLEAN], expr#5=[IS NOT NULL($t3)], expr#6=[AND($t4, $t5)], SAL=[$t1], EXPR$1=[$t6]) EnumerableNestedLoopJoin(condition=[true], joinType=[left]) EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], SAL=[$t5]) @@ -1522,6 +1571,7 @@ EnumerableCalc(expr#0..3=[{inputs}], expr#4=[null:BOOLEAN], expr#5=[IS NOT NULL( EnumerableCalc(expr#0..2=[{inputs}], expr#3=[CAST($t0):INTEGER NOT NULL], expr#4=[0], expr#5=[>($t3, $t4)], expr#6=[null:TINYINT], expr#7=[CASE($t5, $t0, $t6)], expr#8=[IS NOT NULL($t7)], cs=[$t8]) EnumerableTableScan(table=[[scott, DEPT]]) !plan +!} # Test project literal IN required select sal, @@ -1550,6 +1600,7 @@ from "scott".emp; (14 rows) !ok +!if (use_old_decorr) { EnumerableCalc(expr#0..2=[{inputs}], expr#3=[IS NOT NULL($t2)], SAL=[$t1], EXPR$1=[$t3]) EnumerableNestedLoopJoin(condition=[true], joinType=[left]) EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], SAL=[$t5]) @@ -1558,6 +1609,7 @@ EnumerableCalc(expr#0..2=[{inputs}], expr#3=[IS NOT NULL($t2)], SAL=[$t1], EXPR$ EnumerableCalc(expr#0..2=[{inputs}], expr#3=[true], expr#4=[10], expr#5=[CAST($t0):INTEGER NOT NULL], expr#6=[=($t4, $t5)], cs=[$t3], $condition=[$t6]) EnumerableTableScan(table=[[scott, DEPT]]) !plan +!} # Test project literal IN nullable select sal, @@ -1586,6 +1638,7 @@ from "scott".emp; (14 rows) !ok +!if (use_old_decorr) { EnumerableCalc(expr#0..3=[{inputs}], expr#4=[IS FALSE($t2)], expr#5=[null:BOOLEAN], expr#6=[IS NOT NULL($t3)], expr#7=[AND($t4, $t5, $t6)], expr#8=[IS NOT NULL($t2)], expr#9=[IS NOT FALSE($t2)], expr#10=[AND($t8, $t6, $t9)], expr#11=[OR($t7, $t10)], SAL=[$t1], EXPR$1=[$t11]) EnumerableNestedLoopJoin(condition=[true], joinType=[left]) EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], SAL=[$t5]) @@ -1596,6 +1649,7 @@ EnumerableCalc(expr#0..3=[{inputs}], expr#4=[IS FALSE($t2)], expr#5=[null:BOOLEA EnumerableCalc(expr#0..2=[{inputs}], expr#3=[CAST($t0):INTEGER NOT NULL], expr#4=[0], expr#5=[>($t3, $t4)], expr#6=[null:TINYINT], expr#7=[CASE($t5, $t0, $t6)], expr#8=[IS NOT NULL($t7)], expr#9=[CAST($t7):INTEGER], expr#10=[Sarg[10; NULL AS TRUE]], expr#11=[SEARCH($t9, $t10)], cs=[$t8], $condition=[$t11]) EnumerableTableScan(table=[[scott, DEPT]]) !plan +!} # Test project null NOT IN null non-correlated select sal, @@ -1624,6 +1678,7 @@ from "scott".emp; (14 rows) !ok +!if (use_old_decorr) { EnumerableCalc(expr#0..3=[{inputs}], expr#4=[IS NULL($t3)], expr#5=[null:BOOLEAN], expr#6=[OR($t4, $t5)], SAL=[$t1], EXPR$1=[$t6]) EnumerableNestedLoopJoin(condition=[true], joinType=[left]) EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], SAL=[$t5]) @@ -1634,6 +1689,7 @@ EnumerableCalc(expr#0..3=[{inputs}], expr#4=[IS NULL($t3)], expr#5=[null:BOOLEAN EnumerableCalc(expr#0..2=[{inputs}], expr#3=[false], cs=[$t3]) EnumerableTableScan(table=[[scott, DEPT]]) !plan +!} # Test project literal NOT IN null non-correlated select sal, @@ -1662,6 +1718,7 @@ from "scott".emp; (14 rows) !ok +!if (use_old_decorr) { EnumerableCalc(expr#0..3=[{inputs}], expr#4=[IS NULL($t3)], expr#5=[IS FALSE($t2)], expr#6=[null:BOOLEAN], expr#7=[AND($t5, $t6)], expr#8=[IS NOT FALSE($t2)], expr#9=[IS NULL($t2)], expr#10=[AND($t8, $t9)], expr#11=[OR($t4, $t7, $t10)], SAL=[$t1], EXPR$1=[$t11]) EnumerableNestedLoopJoin(condition=[true], joinType=[left]) EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], SAL=[$t5]) @@ -1672,6 +1729,7 @@ EnumerableCalc(expr#0..3=[{inputs}], expr#4=[IS NULL($t3)], expr#5=[IS FALSE($t2 EnumerableCalc(expr#0..2=[{inputs}], expr#3=[false], cs=[$t3]) EnumerableTableScan(table=[[scott, DEPT]]) !plan +!} # Test project null NOT IN literal non-correlated select sal, @@ -1700,6 +1758,7 @@ from "scott".emp; (14 rows) !ok +!if (use_old_decorr) { EnumerableCalc(expr#0..3=[{inputs}], expr#4=[IS NULL($t3)], expr#5=[null:BOOLEAN], expr#6=[OR($t4, $t5)], SAL=[$t1], EXPR$1=[$t6]) EnumerableNestedLoopJoin(condition=[true], joinType=[left]) EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], SAL=[$t5]) @@ -1710,6 +1769,7 @@ EnumerableCalc(expr#0..3=[{inputs}], expr#4=[IS NULL($t3)], expr#5=[null:BOOLEAN EnumerableCalc(expr#0..2=[{inputs}], expr#3=[true], cs=[$t3]) EnumerableTableScan(table=[[scott, DEPT]]) !plan +!} # Test project null NOT IN required select sal, @@ -1738,6 +1798,7 @@ from "scott".emp; (14 rows) !ok +!if (use_old_decorr) { EnumerableCalc(expr#0..3=[{inputs}], expr#4=[IS NULL($t3)], expr#5=[null:BOOLEAN], expr#6=[OR($t4, $t5)], SAL=[$t1], EXPR$1=[$t6]) EnumerableNestedLoopJoin(condition=[true], joinType=[left]) EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], SAL=[$t5]) @@ -1748,6 +1809,7 @@ EnumerableCalc(expr#0..3=[{inputs}], expr#4=[IS NULL($t3)], expr#5=[null:BOOLEAN EnumerableCalc(expr#0..2=[{inputs}], expr#3=[true], cs=[$t3]) EnumerableTableScan(table=[[scott, DEPT]]) !plan +!} # Test project null NOT IN nullable select sal, @@ -1776,6 +1838,7 @@ from "scott".emp; (14 rows) !ok +!if (use_old_decorr) { EnumerableCalc(expr#0..3=[{inputs}], expr#4=[IS NULL($t3)], expr#5=[null:BOOLEAN], expr#6=[OR($t4, $t5)], SAL=[$t1], EXPR$1=[$t6]) EnumerableNestedLoopJoin(condition=[true], joinType=[left]) EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], SAL=[$t5]) @@ -1786,6 +1849,7 @@ EnumerableCalc(expr#0..3=[{inputs}], expr#4=[IS NULL($t3)], expr#5=[null:BOOLEAN EnumerableCalc(expr#0..2=[{inputs}], expr#3=[CAST($t0):INTEGER NOT NULL], expr#4=[0], expr#5=[>($t3, $t4)], expr#6=[null:TINYINT], expr#7=[CASE($t5, $t0, $t6)], expr#8=[IS NOT NULL($t7)], cs=[$t8]) EnumerableTableScan(table=[[scott, DEPT]]) !plan +!} # Test project literal NOT IN required select sal, @@ -1814,6 +1878,7 @@ from "scott".emp; (14 rows) !ok +!if (use_old_decorr) { EnumerableCalc(expr#0..2=[{inputs}], expr#3=[IS NULL($t2)], SAL=[$t1], EXPR$1=[$t3]) EnumerableNestedLoopJoin(condition=[true], joinType=[left]) EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], SAL=[$t5]) @@ -1822,6 +1887,7 @@ EnumerableCalc(expr#0..2=[{inputs}], expr#3=[IS NULL($t2)], SAL=[$t1], EXPR$1=[$ EnumerableCalc(expr#0..2=[{inputs}], expr#3=[true], expr#4=[10], expr#5=[CAST($t0):INTEGER NOT NULL], expr#6=[=($t4, $t5)], cs=[$t3], $condition=[$t6]) EnumerableTableScan(table=[[scott, DEPT]]) !plan +!} # Test project literal NOT IN nullable select sal, @@ -1850,6 +1916,7 @@ from "scott".emp; (14 rows) !ok +!if (use_old_decorr) { EnumerableCalc(expr#0..3=[{inputs}], expr#4=[IS NULL($t3)], expr#5=[IS FALSE($t2)], expr#6=[null:BOOLEAN], expr#7=[AND($t5, $t6)], expr#8=[IS NOT FALSE($t2)], expr#9=[IS NULL($t2)], expr#10=[AND($t8, $t9)], expr#11=[OR($t4, $t7, $t10)], SAL=[$t1], EXPR$1=[$t11]) EnumerableNestedLoopJoin(condition=[true], joinType=[left]) EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], SAL=[$t5]) @@ -1860,6 +1927,7 @@ EnumerableCalc(expr#0..3=[{inputs}], expr#4=[IS NULL($t3)], expr#5=[IS FALSE($t2 EnumerableCalc(expr#0..2=[{inputs}], expr#3=[CAST($t0):INTEGER NOT NULL], expr#4=[0], expr#5=[>($t3, $t4)], expr#6=[null:TINYINT], expr#7=[CASE($t5, $t0, $t6)], expr#8=[IS NOT NULL($t7)], expr#9=[CAST($t7):INTEGER], expr#10=[Sarg[10; NULL AS TRUE]], expr#11=[SEARCH($t9, $t10)], cs=[$t8], $condition=[$t11]) EnumerableTableScan(table=[[scott, DEPT]]) !plan +!} # Test project null IN required is unknown select sal, @@ -1888,6 +1956,7 @@ from "scott".emp; (14 rows) !ok +!if (use_old_decorr) { EnumerableCalc(expr#0..3=[{inputs}], expr#4=[null:BOOLEAN], expr#5=[IS NOT NULL($t3)], expr#6=[AND($t4, $t5)], expr#7=[IS NULL($t6)], SAL=[$t1], EXPR$1=[$t7]) EnumerableNestedLoopJoin(condition=[true], joinType=[left]) EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], SAL=[$t5]) @@ -1898,6 +1967,7 @@ EnumerableCalc(expr#0..3=[{inputs}], expr#4=[null:BOOLEAN], expr#5=[IS NOT NULL( EnumerableCalc(expr#0..2=[{inputs}], expr#3=[true], cs=[$t3]) EnumerableTableScan(table=[[scott, DEPT]]) !plan +!} # Test filter null IN null select sal from "scott".emp @@ -1911,8 +1981,10 @@ select sal from "scott".emp (0 rows) !ok +!if (use_old_decorr) { EnumerableValues(tuples=[[]]) !plan +!} # Test filter literal IN null non-correlated select sal from "scott".emp @@ -1926,8 +1998,10 @@ select sal from "scott".emp (0 rows) !ok +!if (use_old_decorr) { EnumerableValues(tuples=[[]]) !plan +!} # Test filter null IN literal non-correlated select sal from "scott".emp @@ -1941,8 +2015,10 @@ select sal from "scott".emp (0 rows) !ok +!if (use_old_decorr) { EnumerableValues(tuples=[[]]) !plan +!} # Test filter null IN required select sal from "scott".emp @@ -1956,8 +2032,10 @@ select sal from "scott".emp (0 rows) !ok +!if (use_old_decorr) { EnumerableValues(tuples=[[]]) !plan +!} # Test filter null IN nullable select sal from "scott".emp @@ -1971,8 +2049,10 @@ select sal from "scott".emp (0 rows) !ok +!if (use_old_decorr) { EnumerableValues(tuples=[[]]) !plan +!} # Test filter literal IN required select sal from "scott".emp @@ -2000,6 +2080,7 @@ select sal from "scott".emp (14 rows) !ok +!if (use_old_decorr) { EnumerableCalc(expr#0..2=[{inputs}], SAL=[$t1]) EnumerableNestedLoopJoin(condition=[true], joinType=[inner]) EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], SAL=[$t5]) @@ -2008,6 +2089,7 @@ EnumerableCalc(expr#0..2=[{inputs}], SAL=[$t1]) EnumerableCalc(expr#0..2=[{inputs}], expr#3=[true], expr#4=[10], expr#5=[CAST($t0):INTEGER NOT NULL], expr#6=[=($t4, $t5)], cs=[$t3], $condition=[$t6]) EnumerableTableScan(table=[[scott, DEPT]]) !plan +!} # Test filter literal IN nullable select sal from "scott".emp @@ -2035,6 +2117,7 @@ select sal from "scott".emp (14 rows) !ok +!if (use_old_decorr) { EnumerableCalc(expr#0..2=[{inputs}], SAL=[$t1]) EnumerableNestedLoopJoin(condition=[true], joinType=[inner]) EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], SAL=[$t5]) @@ -2043,6 +2126,7 @@ EnumerableCalc(expr#0..2=[{inputs}], SAL=[$t1]) EnumerableCalc(expr#0..2=[{inputs}], expr#3=[true], expr#4=[10], expr#5=[CAST($t0):INTEGER NOT NULL], expr#6=[0], expr#7=[>($t5, $t6)], expr#8=[null:TINYINT], expr#9=[CASE($t7, $t0, $t8)], expr#10=[CAST($t9):INTEGER], expr#11=[=($t4, $t10)], cs=[$t3], $condition=[$t11]) EnumerableTableScan(table=[[scott, DEPT]]) !plan +!} # Test filter null NOT IN null non-correlated select sal from "scott".emp @@ -2056,6 +2140,7 @@ select sal from "scott".emp (0 rows) !ok +!if (use_old_decorr) { EnumerableCalc(expr#0..3=[{inputs}], expr#4=[IS NULL($t3)], SAL=[$t1], $condition=[$t4]) EnumerableNestedLoopJoin(condition=[true], joinType=[left]) EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], SAL=[$t5]) @@ -2066,6 +2151,7 @@ EnumerableCalc(expr#0..3=[{inputs}], expr#4=[IS NULL($t3)], SAL=[$t1], $conditio EnumerableCalc(expr#0..2=[{inputs}], expr#3=[false], cs=[$t3]) EnumerableTableScan(table=[[scott, DEPT]]) !plan +!} # Test filter literal NOT IN null non-correlated select sal from "scott".emp @@ -2079,6 +2165,7 @@ select sal from "scott".emp (0 rows) !ok +!if (use_old_decorr) { EnumerableCalc(expr#0..3=[{inputs}], expr#4=[NOT($t2)], expr#5=[IS NOT NULL($t2)], expr#6=[OR($t4, $t5)], expr#7=[IS NOT TRUE($t6)], expr#8=[IS NULL($t3)], expr#9=[OR($t7, $t8)], SAL=[$t1], $condition=[$t9]) EnumerableNestedLoopJoin(condition=[true], joinType=[left]) EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], SAL=[$t5]) @@ -2089,6 +2176,7 @@ EnumerableCalc(expr#0..3=[{inputs}], expr#4=[NOT($t2)], expr#5=[IS NOT NULL($t2) EnumerableCalc(expr#0..2=[{inputs}], expr#3=[false], cs=[$t3]) EnumerableTableScan(table=[[scott, DEPT]]) !plan +!} # Test filter null NOT IN literal non-correlated select sal from "scott".emp @@ -2102,6 +2190,7 @@ select sal from "scott".emp (0 rows) !ok +!if (use_old_decorr) { EnumerableCalc(expr#0..3=[{inputs}], expr#4=[IS NULL($t3)], SAL=[$t1], $condition=[$t4]) EnumerableNestedLoopJoin(condition=[true], joinType=[left]) EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], SAL=[$t5]) @@ -2112,6 +2201,7 @@ EnumerableCalc(expr#0..3=[{inputs}], expr#4=[IS NULL($t3)], SAL=[$t1], $conditio EnumerableCalc(expr#0..2=[{inputs}], expr#3=[true], cs=[$t3]) EnumerableTableScan(table=[[scott, DEPT]]) !plan +!} # Test filter null NOT IN required select sal from "scott".emp @@ -2125,6 +2215,7 @@ select sal from "scott".emp (0 rows) !ok +!if (use_old_decorr) { EnumerableCalc(expr#0..3=[{inputs}], expr#4=[IS NULL($t3)], SAL=[$t1], $condition=[$t4]) EnumerableNestedLoopJoin(condition=[true], joinType=[left]) EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], SAL=[$t5]) @@ -2135,6 +2226,7 @@ EnumerableCalc(expr#0..3=[{inputs}], expr#4=[IS NULL($t3)], SAL=[$t1], $conditio EnumerableCalc(expr#0..2=[{inputs}], expr#3=[true], cs=[$t3]) EnumerableTableScan(table=[[scott, DEPT]]) !plan +!} # Test filter null NOT IN nullable select sal from "scott".emp @@ -2148,6 +2240,7 @@ select sal from "scott".emp (0 rows) !ok +!if (use_old_decorr) { EnumerableCalc(expr#0..3=[{inputs}], expr#4=[IS NULL($t3)], SAL=[$t1], $condition=[$t4]) EnumerableNestedLoopJoin(condition=[true], joinType=[left]) EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], SAL=[$t5]) @@ -2158,6 +2251,7 @@ EnumerableCalc(expr#0..3=[{inputs}], expr#4=[IS NULL($t3)], SAL=[$t1], $conditio EnumerableCalc(expr#0..2=[{inputs}], expr#3=[CAST($t0):INTEGER NOT NULL], expr#4=[0], expr#5=[>($t3, $t4)], expr#6=[null:TINYINT], expr#7=[CASE($t5, $t0, $t6)], expr#8=[IS NOT NULL($t7)], cs=[$t8]) EnumerableTableScan(table=[[scott, DEPT]]) !plan +!} # Test filter literal NOT IN required select sal from "scott".emp @@ -2171,6 +2265,7 @@ select sal from "scott".emp (0 rows) !ok +!if (use_old_decorr) { EnumerableCalc(expr#0..3=[{inputs}], expr#4=[IS NULL($t3)], expr#5=[NOT($t2)], expr#6=[IS NOT NULL($t2)], expr#7=[OR($t5, $t6)], expr#8=[IS NOT TRUE($t7)], expr#9=[OR($t4, $t8)], SAL=[$t1], $condition=[$t9]) EnumerableNestedLoopJoin(condition=[true], joinType=[left]) EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], SAL=[$t5]) @@ -2181,6 +2276,7 @@ EnumerableCalc(expr#0..3=[{inputs}], expr#4=[IS NULL($t3)], expr#5=[NOT($t2)], e EnumerableCalc(expr#0..2=[{inputs}], expr#3=[true], expr#4=[10], expr#5=[CAST($t0):INTEGER NOT NULL], expr#6=[=($t4, $t5)], cs=[$t3], $condition=[$t6]) EnumerableTableScan(table=[[scott, DEPT]]) !plan +!} # Test filter literal NOT IN nullable select sal from "scott".emp @@ -2194,6 +2290,7 @@ select sal from "scott".emp (0 rows) !ok +!if (use_old_decorr) { EnumerableCalc(expr#0..3=[{inputs}], expr#4=[NOT($t2)], expr#5=[IS NOT NULL($t2)], expr#6=[OR($t4, $t5)], expr#7=[IS NOT TRUE($t6)], expr#8=[IS NULL($t3)], expr#9=[OR($t7, $t8)], SAL=[$t1], $condition=[$t9]) EnumerableNestedLoopJoin(condition=[true], joinType=[left]) EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], SAL=[$t5]) @@ -2204,6 +2301,7 @@ EnumerableCalc(expr#0..3=[{inputs}], expr#4=[NOT($t2)], expr#5=[IS NOT NULL($t2) EnumerableCalc(expr#0..2=[{inputs}], expr#3=[CAST($t0):INTEGER NOT NULL], expr#4=[0], expr#5=[>($t3, $t4)], expr#6=[null:TINYINT], expr#7=[CASE($t5, $t0, $t6)], expr#8=[IS NOT NULL($t7)], expr#9=[CAST($t7):INTEGER], expr#10=[Sarg[10; NULL AS TRUE]], expr#11=[SEARCH($t9, $t10)], cs=[$t8], $condition=[$t11]) EnumerableTableScan(table=[[scott, DEPT]]) !plan +!} # Test filter null IN required is unknown select sal from "scott".emp @@ -2231,6 +2329,7 @@ select sal from "scott".emp (14 rows) !ok +!if (use_old_decorr) { EnumerableCalc(expr#0..3=[{inputs}], expr#4=[null:BOOLEAN], expr#5=[IS NOT NULL($t3)], expr#6=[AND($t4, $t5)], expr#7=[IS NULL($t6)], SAL=[$t1], $condition=[$t7]) EnumerableNestedLoopJoin(condition=[true], joinType=[left]) EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], SAL=[$t5]) @@ -2241,6 +2340,7 @@ EnumerableCalc(expr#0..3=[{inputs}], expr#4=[null:BOOLEAN], expr#5=[IS NOT NULL( EnumerableCalc(expr#0..2=[{inputs}], expr#3=[true], cs=[$t3]) EnumerableTableScan(table=[[scott, DEPT]]) !plan +!} #------------------------------- @@ -2256,8 +2356,10 @@ select sal from "scott".emp e (0 rows) !ok +!if (use_old_decorr) { EnumerableValues(tuples=[[]]) !plan +!} # Test filter literal IN null correlated select sal from "scott".emp e @@ -2271,8 +2373,10 @@ select sal from "scott".emp e (0 rows) !ok +!if (use_old_decorr) { EnumerableValues(tuples=[[]]) !plan +!} # Test filter null IN literal correlated select sal from "scott".emp e @@ -2286,8 +2390,10 @@ select sal from "scott".emp e (0 rows) !ok +!if (use_old_decorr) { EnumerableValues(tuples=[[]]) !plan +!} # Test filter null IN required correlated select sal from "scott".emp e @@ -2301,8 +2407,10 @@ select sal from "scott".emp e (0 rows) !ok +!if (use_old_decorr) { EnumerableValues(tuples=[[]]) !plan +!} # Test filter literal IN null liter with query that can not be trivially simplified select sal from "scott".emp e @@ -2316,6 +2424,7 @@ select sal from "scott".emp e (0 rows) !ok +!if (use_old_decorr) { EnumerableCalc(expr#0..5=[{inputs}], expr#6=[RAND()], expr#7=[CAST($t6):INTEGER NOT NULL], expr#8=[2], expr#9=[MOD($t7, $t8)], expr#10=[3], expr#11=[=($t9, $t10)], expr#12=[OR($t11, $t3)], SAL=[$t1], $condition=[$t12]) EnumerableMergeJoin(condition=[=($2, $4)], joinType=[left]) EnumerableSort(sort0=[$2], dir0=[ASC]) @@ -2327,6 +2436,7 @@ EnumerableCalc(expr#0..5=[{inputs}], expr#6=[RAND()], expr#7=[CAST($t6):INTEGER EnumerableCalc(expr#0..2=[{inputs}], DEPTNO=[$t0]) EnumerableTableScan(table=[[scott, DEPT]]) !plan +!} # Test filter null IN nullable correlated select sal from "scott".emp e @@ -2340,8 +2450,10 @@ select sal from "scott".emp e (0 rows) !ok +!if (use_old_decorr) { EnumerableValues(tuples=[[]]) !plan +!} # Test filter literal IN required correlated select sal from "scott".emp e @@ -2358,6 +2470,7 @@ select sal from "scott".emp e (3 rows) !ok +!if (use_old_decorr) { EnumerableCalc(expr#0..2=[{inputs}], SAL=[$t1]) EnumerableHashJoin(condition=[=($2, $3)], joinType=[semi]) EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], SAL=[$t5], DEPTNO=[$t7]) @@ -2365,6 +2478,7 @@ EnumerableCalc(expr#0..2=[{inputs}], SAL=[$t1]) EnumerableCalc(expr#0..2=[{inputs}], expr#3=[10], expr#4=[CAST($t0):INTEGER NOT NULL], expr#5=[=($t3, $t4)], DEPTNO=[$t0], $condition=[$t5]) EnumerableTableScan(table=[[scott, DEPT]]) !plan +!} # Test filter literal IN nullable correlated select sal from "scott".emp e @@ -2381,6 +2495,7 @@ select sal from "scott".emp e (3 rows) !ok +!if (use_old_decorr) { EnumerableCalc(expr#0..2=[{inputs}], SAL=[$t1]) EnumerableHashJoin(condition=[=($2, $3)], joinType=[semi]) EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], SAL=[$t5], DEPTNO=[$t7]) @@ -2388,6 +2503,7 @@ EnumerableCalc(expr#0..2=[{inputs}], SAL=[$t1]) EnumerableCalc(expr#0..2=[{inputs}], expr#3=[10], expr#4=[CAST($t0):INTEGER NOT NULL], expr#5=[0], expr#6=[>($t4, $t5)], expr#7=[null:TINYINT], expr#8=[CASE($t6, $t0, $t7)], expr#9=[CAST($t8):INTEGER], expr#10=[=($t3, $t9)], DEPTNO=[$t0], $condition=[$t10]) EnumerableTableScan(table=[[scott, DEPT]]) !plan +!} # Test filter null NOT IN null correlated select sal from "scott".emp e @@ -2401,8 +2517,10 @@ select sal from "scott".emp e (0 rows) !ok +!if (use_old_decorr) { EnumerableValues(tuples=[[]]) !plan +!} # Test filter literal NOT IN null correlated select sal from "scott".emp e @@ -2416,6 +2534,7 @@ select sal from "scott".emp e (0 rows) !ok +!if (use_old_decorr) { EnumerableCalc(expr#0..5=[{inputs}], expr#6=[NOT($t3)], expr#7=[IS NOT NULL($t3)], expr#8=[OR($t6, $t7)], expr#9=[IS NOT TRUE($t8)], SAL=[$t1], $condition=[$t9]) EnumerableMergeJoin(condition=[=($2, $4)], joinType=[left]) EnumerableSort(sort0=[$2], dir0=[ASC]) @@ -2427,6 +2546,7 @@ EnumerableCalc(expr#0..5=[{inputs}], expr#6=[NOT($t3)], expr#7=[IS NOT NULL($t3) EnumerableCalc(expr#0..2=[{inputs}], DEPTNO=[$t0]) EnumerableTableScan(table=[[scott, DEPT]]) !plan +!} # Test filter null NOT IN literal correlated select sal from "scott".emp e @@ -2440,8 +2560,10 @@ select sal from "scott".emp e (0 rows) !ok +!if (use_old_decorr) { EnumerableValues(tuples=[[]]) !plan +!} # Test filter null NOT IN required correlated select sal from "scott".emp e @@ -2455,8 +2577,10 @@ select sal from "scott".emp e (0 rows) !ok +!if (use_old_decorr) { EnumerableValues(tuples=[[]]) !plan +!} # Test filter null NOT IN nullable correlated select sal from "scott".emp e @@ -2470,8 +2594,10 @@ select sal from "scott".emp e (0 rows) !ok +!if (use_old_decorr) { EnumerableValues(tuples=[[]]) !plan +!} # Test filter literal NOT IN required correlated select sal from "scott".emp e @@ -2496,6 +2622,7 @@ select sal from "scott".emp e (11 rows) !ok +!if (use_old_decorr) { EnumerableCalc(expr#0..5=[{inputs}], expr#6=[NOT($t3)], expr#7=[IS NOT NULL($t3)], expr#8=[OR($t6, $t7)], expr#9=[IS NOT TRUE($t8)], SAL=[$t1], $condition=[$t9]) EnumerableMergeJoin(condition=[=($2, $4)], joinType=[left]) EnumerableSort(sort0=[$2], dir0=[ASC]) @@ -2508,6 +2635,7 @@ EnumerableCalc(expr#0..5=[{inputs}], expr#6=[NOT($t3)], expr#7=[IS NOT NULL($t3) EnumerableCalc(expr#0..2=[{inputs}], expr#3=[10], expr#4=[CAST($t0):INTEGER NOT NULL], expr#5=[=($t3, $t4)], DEPTNO=[$t0], $condition=[$t5]) EnumerableTableScan(table=[[scott, DEPT]]) !plan +!} # Test filter literal NOT IN nullable correlated select sal from "scott".emp e @@ -2532,6 +2660,7 @@ select sal from "scott".emp e (11 rows) !ok +!if (use_old_decorr) { EnumerableCalc(expr#0..5=[{inputs}], expr#6=[NOT($t3)], expr#7=[IS NOT NULL($t3)], expr#8=[OR($t6, $t7)], expr#9=[IS NOT TRUE($t8)], SAL=[$t1], $condition=[$t9]) EnumerableMergeJoin(condition=[=($2, $4)], joinType=[left]) EnumerableSort(sort0=[$2], dir0=[ASC]) @@ -2544,6 +2673,7 @@ EnumerableCalc(expr#0..5=[{inputs}], expr#6=[NOT($t3)], expr#7=[IS NOT NULL($t3) EnumerableCalc(expr#0..2=[{inputs}], expr#3=[CAST($t0):INTEGER NOT NULL], expr#4=[0], expr#5=[>($t3, $t4)], expr#6=[null:TINYINT], expr#7=[CASE($t5, $t0, $t6)], expr#8=[IS NOT NULL($t7)], expr#9=[CAST($t7):INTEGER], expr#10=[Sarg[10; NULL AS TRUE]], expr#11=[SEARCH($t9, $t10)], DEPTNO=[$t0], cs=[$t8], $condition=[$t11]) EnumerableTableScan(table=[[scott, DEPT]]) !plan +!} # Test filter null IN required is unknown correlated select sal from "scott".emp e @@ -2571,9 +2701,11 @@ select sal from "scott".emp e (14 rows) !ok +!if (use_old_decorr) { EnumerableCalc(expr#0..7=[{inputs}], SAL=[$t5]) EnumerableTableScan(table=[[scott, EMP]]) !plan +!} # Test project constant IN an expression that is sometimes null @@ -2677,6 +2809,7 @@ select * from emp where deptno IN (select (select max(deptno) from "scott".emp t (6 rows) !ok +!if (use_old_decorr) { EnumerableHashJoin(condition=[=($7, $9)], joinType=[semi]) EnumerableTableScan(table=[[scott, EMP]]) EnumerableNestedLoopJoin(condition=[true], joinType=[left]) @@ -2685,6 +2818,7 @@ EnumerableHashJoin(condition=[=($7, $9)], joinType=[semi]) EnumerableAggregate(group=[{}], EXPR$0=[MAX($7)]) EnumerableTableScan(table=[[scott, EMP]]) !plan +!} # Test nested sub-query in FILTER within PROJECT select (select max(deptno) from "scott".emp where deptno IN (select deptno from "scott".emp)) from emp ; @@ -2709,6 +2843,7 @@ select (select max(deptno) from "scott".emp where deptno IN (select deptno from (14 rows) !ok +!if (use_old_decorr) { EnumerableCalc(expr#0..1=[{inputs}], EXPR$0=[$t1]) EnumerableNestedLoopJoin(condition=[true], joinType=[left]) EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0]) @@ -2719,6 +2854,7 @@ EnumerableCalc(expr#0..1=[{inputs}], EXPR$0=[$t1]) EnumerableTableScan(table=[[scott, EMP]]) EnumerableTableScan(table=[[scott, EMP]]) !plan +!} !use scott @@ -2781,6 +2917,7 @@ where sal + 100 not in ( (1 row) !ok +!if (use_old_decorr) { EnumerableAggregate(group=[{}], C=[COUNT()]) EnumerableCalc(expr#0..7=[{inputs}], expr#8=[0], expr#9=[=($t1, $t8)], expr#10=[IS NULL($t0)], expr#11=[IS NOT NULL($t7)], expr#12=[<($t2, $t1)], expr#13=[OR($t10, $t11, $t12)], expr#14=[IS NOT TRUE($t13)], expr#15=[OR($t9, $t14)], proj#0..7=[{exprs}], $condition=[$t15]) EnumerableMergeJoin(condition=[AND(=($3, $5), =($4, $6))], joinType=[left]) @@ -2800,11 +2937,13 @@ EnumerableAggregate(group=[{}], C=[COUNT()]) EnumerableCalc(expr#0..2=[{inputs}], expr#3=[CAST($t0):DECIMAL(13, 2) NOT NULL], expr#4=[true], expr#5=[IS NOT NULL($t1)], DEPTNO=[$t3], DNAME=[$t1], i=[$t4], $condition=[$t5]) EnumerableTableScan(table=[[scott, DEPT]]) !plan +!} # Correlated ANY sub-query select empno from "scott".emp as e where e.empno > ANY( select 2 from "scott".dept e2 where e2.deptno = e.deptno) ; +!if (use_old_decorr) { EnumerableCalc(expr#0..6=[{inputs}], EMPNO=[$t5]) EnumerableHashJoin(condition=[AND(IS NOT DISTINCT FROM($4, $6), OR(AND(>($5, $0), IS NOT TRUE(OR(IS NULL($3), =($1, 0)))), AND(>($5, $0), IS NOT TRUE(OR(IS NULL($3), =($1, 0))), IS NOT TRUE(>($5, $0)), <=($1, $2))))], joinType=[inner]) EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NOT NULL($t3)], expr#6=[0], expr#7=[CASE($t5, $t3, $t6)], m=[$t2], c=[$t7], d=[$t7], trueLiteral=[$t4], DEPTNO=[$t0]) @@ -2816,6 +2955,7 @@ EnumerableCalc(expr#0..6=[{inputs}], EMPNO=[$t5]) EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], DEPTNO=[$t7]) EnumerableTableScan(table=[[scott, EMP]]) !plan +!} +-------+ | EMPNO | +-------+ @@ -2842,7 +2982,7 @@ EnumerableCalc(expr#0..6=[{inputs}], EMPNO=[$t5]) select empno, e.deptno > ANY( select 2 from "scott".dept e2 where e2.deptno = e.empno) from "scott".emp as e; - +!if (use_old_decorr) { EnumerableCalc(expr#0..6=[{inputs}], expr#7=[>($t1, $t2)], expr#8=[IS TRUE($t7)], expr#9=[IS NULL($t5)], expr#10=[0], expr#11=[=($t3, $t10)], expr#12=[OR($t9, $t11)], expr#13=[IS NOT TRUE($t12)], expr#14=[AND($t8, $t13)], expr#15=[>($t3, $t4)], expr#16=[IS TRUE($t15)], expr#17=[null:BOOLEAN], expr#18=[IS NOT TRUE($t7)], expr#19=[AND($t16, $t17, $t13, $t18)], expr#20=[IS NOT TRUE($t15)], expr#21=[AND($t7, $t13, $t18, $t20)], expr#22=[OR($t14, $t19, $t21)], EMPNO=[$t0], EXPR$1=[$t22]) EnumerableHashJoin(condition=[=($0, $6)], joinType=[left]) EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], DEPTNO=[$t7]) @@ -2855,6 +2995,7 @@ EnumerableCalc(expr#0..6=[{inputs}], expr#7=[>($t1, $t2)], expr#8=[IS TRUE($t7)] EnumerableCalc(expr#0..2=[{inputs}], expr#3=[CAST($t0):SMALLINT NOT NULL], expr#4=[2], DEPTNO0=[$t3], EXPR$0=[$t4]) EnumerableTableScan(table=[[scott, DEPT]]) !plan +!} +-------+--------+ | EMPNO | EXPR$1 | +-------+--------+ @@ -2887,6 +3028,7 @@ where exists # The plan before the fix was wrong but also inefficient since it required the generation of # a value generator (see RelDecorrelator code). The value generator is not present in the # following plan (two scans of EMP table instead of three). +!if (use_old_decorr) { EnumerableCalc(expr#0..2=[{inputs}], ENAME=[$t1]) EnumerableHashJoin(condition=[=($2, $3)], joinType=[semi]) EnumerableCalc(expr#0..7=[{inputs}], expr#8=[IS NOT NULL($t3)], expr#9=[CAST($t3):INTEGER NOT NULL], expr#10=[0], expr#11=[CASE($t8, $t9, $t10)], proj#0..1=[{exprs}], $f3=[$t11]) @@ -2894,6 +3036,7 @@ EnumerableCalc(expr#0..2=[{inputs}], ENAME=[$t1]) EnumerableCalc(expr#0..7=[{inputs}], expr#8=[IS NOT NULL($t3)], expr#9=[CAST($t3):INTEGER NOT NULL], expr#10=[0], expr#11=[CASE($t8, $t9, $t10)], $f8=[$t11]) EnumerableTableScan(table=[[scott, EMP]]) !plan +!} +--------+ | ENAME | +--------+ @@ -2921,6 +3064,7 @@ EnumerableCalc(expr#0..2=[{inputs}], ENAME=[$t1]) select empno from "scott".emp emp1 where empno <> some (select emp2.empno from "scott".emp emp2 where emp2.empno = emp1.empno); +!if (use_old_decorr) { EnumerableCalc(expr#0..6=[{inputs}], expr#7=[<>($t2, $t1)], expr#8=[1], expr#9=[<=($t3, $t8)], expr#10=[<>($t0, $t4)], expr#11=[IS NULL($t5)], expr#12=[0], expr#13=[=($t1, $t12)], expr#14=[OR($t11, $t13)], expr#15=[IS NOT TRUE($t14)], expr#16=[AND($t7, $t9, $t10, $t15)], expr#17=[=($t3, $t8)], expr#18=[IS NOT NULL($t3)], expr#19=[AND($t7, $t18)], expr#20=[IS NOT TRUE($t19)], expr#21=[AND($t17, $t10, $t15, $t20)], expr#22=[AND($t7, $t9)], expr#23=[IS NOT TRUE($t22)], expr#24=[IS NOT TRUE($t17)], expr#25=[AND($t15, $t23, $t24)], expr#26=[OR($t16, $t21, $t25)], EMPNO=[$t0], $condition=[$t26]) EnumerableHashJoin(condition=[=($0, $6)], joinType=[left]) EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0]) @@ -2932,6 +3076,7 @@ EnumerableCalc(expr#0..6=[{inputs}], expr#7=[<>($t2, $t1)], expr#8=[1], expr#9=[ EnumerableCalc(expr#0..7=[{inputs}], expr#8=[1:BIGINT], expr#9=[true], EMPNO1=[$t0], $f1=[$t8], $f2=[$t8], EMPNO=[$t0], $f4=[$t9]) EnumerableTableScan(table=[[scott, EMP]]) !plan +!} +-------+ | EMPNO | +-------+ @@ -2969,6 +3114,7 @@ from "scott".emp emp1; select * from "scott".emp emp1 where empno <> some (select comm from "scott".emp where deptno = emp1.deptno); +!if (use_old_decorr) { EnumerableCalc(expr#0..13=[{inputs}], expr#14=[<>($t10, $t9)], expr#15=[1], expr#16=[<=($t11, $t15)], expr#17=[AND($t14, $t16)], expr#18=[=($t11, $t15)], expr#19=[OR($t17, $t18)], expr#20=[<>($t0, $t12)], expr#21=[IS NULL($t13)], expr#22=[0], expr#23=[=($t9, $t22)], expr#24=[OR($t21, $t23)], expr#25=[IS NOT TRUE($t24)], expr#26=[AND($t19, $t20, $t25)], expr#27=[IS NOT TRUE($t19)], expr#28=[AND($t25, $t27)], expr#29=[OR($t26, $t28)], proj#0..7=[{exprs}], $condition=[$t29]) EnumerableHashJoin(condition=[IS NOT DISTINCT FROM($7, $8)], joinType=[left]) EnumerableTableScan(table=[[scott, EMP]]) @@ -2983,6 +3129,7 @@ EnumerableCalc(expr#0..13=[{inputs}], expr#14=[<>($t10, $t9)], expr#15=[1], expr EnumerableCalc(expr#0..7=[{inputs}], expr#8=[IS NOT NULL($t7)], proj#0..7=[{exprs}], $condition=[$t8]) EnumerableTableScan(table=[[scott, EMP]]) !plan +!} +-------+--------+----------+------+------------+---------+---------+--------+ | EMPNO | ENAME | JOB | MGR | HIREDATE | SAL | COMM | DEPTNO | +-------+--------+----------+------+------------+---------+---------+--------+ @@ -3026,6 +3173,7 @@ from "scott".emp as emp1; select * from "scott".emp as emp1 where empno <> some (select 2 from "scott".dept dept1 where dept1.deptno = emp1.empno); +!if (use_old_decorr) { EnumerableCalc(expr#0..13=[{inputs}], expr#14=[<>($t9, $t8)], expr#15=[1], expr#16=[<=($t10, $t15)], expr#17=[<>($t0, $t11)], expr#18=[IS NULL($t12)], expr#19=[0], expr#20=[=($t8, $t19)], expr#21=[OR($t18, $t20)], expr#22=[IS NOT TRUE($t21)], expr#23=[AND($t14, $t16, $t17, $t22)], expr#24=[=($t10, $t15)], expr#25=[IS NOT NULL($t10)], expr#26=[AND($t14, $t25)], expr#27=[IS NOT TRUE($t26)], expr#28=[AND($t24, $t17, $t22, $t27)], expr#29=[AND($t14, $t16)], expr#30=[IS NOT TRUE($t29)], expr#31=[IS NOT TRUE($t24)], expr#32=[AND($t22, $t30, $t31)], expr#33=[OR($t23, $t28, $t32)], proj#0..7=[{exprs}], $condition=[$t33]) EnumerableHashJoin(condition=[=($0, $13)], joinType=[left]) EnumerableTableScan(table=[[scott, EMP]]) @@ -3040,6 +3188,7 @@ EnumerableCalc(expr#0..13=[{inputs}], expr#14=[<>($t9, $t8)], expr#15=[1], expr# EnumerableCalc(expr#0..2=[{inputs}], expr#3=[CAST($t0):SMALLINT NOT NULL], expr#4=[2], DEPTNO0=[$t3], EXPR$0=[$t4]) EnumerableTableScan(table=[[scott, DEPT]]) !plan +!} +-------+-------+-----+-----+----------+-----+------+--------+ | EMPNO | ENAME | JOB | MGR | HIREDATE | SAL | COMM | DEPTNO | +-------+-------+-----+-----+----------+-----+------+--------+ @@ -3077,6 +3226,7 @@ from "scott".emp as emp1; select * from "scott".emp as emp1 where comm <> some (select 2 from "scott".dept dept1 where dept1.deptno = emp1.empno); +!if (use_old_decorr) { EnumerableCalc(expr#0..13=[{inputs}], expr#14=[<>($t9, $t8)], expr#15=[1], expr#16=[<=($t10, $t15)], expr#17=[AND($t14, $t16)], expr#18=[=($t10, $t15)], expr#19=[OR($t17, $t18)], expr#20=[<>($t6, $t11)], expr#21=[IS NULL($t12)], expr#22=[IS NULL($t6)], expr#23=[0], expr#24=[=($t8, $t23)], expr#25=[OR($t21, $t22, $t24)], expr#26=[IS NOT TRUE($t25)], expr#27=[AND($t19, $t20, $t26)], expr#28=[IS NOT TRUE($t19)], expr#29=[AND($t26, $t28)], expr#30=[OR($t27, $t29)], proj#0..7=[{exprs}], $condition=[$t30]) EnumerableHashJoin(condition=[=($0, $13)], joinType=[left]) EnumerableTableScan(table=[[scott, EMP]]) @@ -3091,6 +3241,7 @@ EnumerableCalc(expr#0..13=[{inputs}], expr#14=[<>($t9, $t8)], expr#15=[1], expr# EnumerableCalc(expr#0..2=[{inputs}], expr#3=[CAST($t0):SMALLINT NOT NULL], expr#4=[2], DEPTNO0=[$t3], EXPR$0=[$t4]) EnumerableTableScan(table=[[scott, DEPT]]) !plan +!} +-------+-------+-----+-----+----------+-----+------+--------+ | EMPNO | ENAME | JOB | MGR | HIREDATE | SAL | COMM | DEPTNO | +-------+-------+-----+-----+----------+-----+------+--------+ @@ -3128,6 +3279,7 @@ from "scott".emp as emp1; select * from "scott".emp emp1 where emp1.comm <> some (select comm from "scott".emp emp2 where emp2.sal = emp1.sal); +!if (use_old_decorr) { EnumerableCalc(expr#0..13=[{inputs}], expr#14=[<>($t10, $t9)], expr#15=[1], expr#16=[<=($t11, $t15)], expr#17=[AND($t14, $t16)], expr#18=[=($t11, $t15)], expr#19=[OR($t17, $t18)], expr#20=[<>($t6, $t12)], expr#21=[IS NULL($t13)], expr#22=[IS NULL($t6)], expr#23=[0], expr#24=[=($t9, $t23)], expr#25=[OR($t21, $t22, $t24)], expr#26=[IS NOT TRUE($t25)], expr#27=[AND($t19, $t20, $t26)], expr#28=[IS NOT TRUE($t19)], expr#29=[AND($t26, $t28)], expr#30=[OR($t27, $t29)], proj#0..7=[{exprs}], $condition=[$t30]) EnumerableHashJoin(condition=[IS NOT DISTINCT FROM($5, $8)], joinType=[left]) EnumerableTableScan(table=[[scott, EMP]]) @@ -3142,6 +3294,7 @@ EnumerableCalc(expr#0..13=[{inputs}], expr#14=[<>($t10, $t9)], expr#15=[1], expr EnumerableCalc(expr#0..7=[{inputs}], expr#8=[IS NOT NULL($t5)], proj#0..7=[{exprs}], $condition=[$t8]) EnumerableTableScan(table=[[scott, EMP]]) !plan +!} +-------+--------+----------+------+------------+---------+---------+--------+ | EMPNO | ENAME | JOB | MGR | HIREDATE | SAL | COMM | DEPTNO | +-------+--------+----------+------+------------+---------+---------+--------+ @@ -3210,7 +3363,7 @@ where unique (select comm from "scott".emp where comm is not null); (4 rows) !ok - +!if (use_old_decorr) { EnumerableCalc(expr#0..1=[{inputs}], expr#2=[IS NULL($t1)], DEPTNO=[$t0], $condition=[$t2]) EnumerableNestedLoopJoin(condition=[true], joinType=[left]) EnumerableCalc(expr#0..2=[{inputs}], DEPTNO=[$t0]) @@ -3221,6 +3374,7 @@ EnumerableCalc(expr#0..1=[{inputs}], expr#2=[IS NULL($t1)], DEPTNO=[$t0], $condi EnumerableCalc(expr#0..7=[{inputs}], expr#8=[IS NOT NULL($t6)], proj#0..7=[{exprs}], $condition=[$t8]) EnumerableTableScan(table=[[scott, EMP]]) !plan +!} # Previous, as scalar sub-query. select deptno, unique (select comm from "scott".emp where comm is not null) as u @@ -3236,7 +3390,7 @@ from "scott".dept; (4 rows) !ok - +!if (use_old_decorr) { EnumerableCalc(expr#0..1=[{inputs}], expr#2=[IS NULL($t1)], DEPTNO=[$t0], U=[$t2]) EnumerableNestedLoopJoin(condition=[true], joinType=[left]) EnumerableCalc(expr#0..2=[{inputs}], DEPTNO=[$t0]) @@ -3247,6 +3401,7 @@ EnumerableCalc(expr#0..1=[{inputs}], expr#2=[IS NULL($t1)], DEPTNO=[$t0], U=[$t2 EnumerableCalc(expr#0..7=[{inputs}], expr#8=[IS NOT NULL($t6)], proj#0..7=[{exprs}], $condition=[$t8]) EnumerableTableScan(table=[[scott, EMP]]) !plan +!} # Previous, but NOT UNIQUE. select deptno, not unique (select comm from "scott".emp where comm is not null) as u @@ -3262,7 +3417,7 @@ from "scott".dept; (4 rows) !ok - +!if (use_old_decorr) { EnumerableCalc(expr#0..1=[{inputs}], expr#2=[IS NOT NULL($t1)], DEPTNO=[$t0], U=[$t2]) EnumerableNestedLoopJoin(condition=[true], joinType=[left]) EnumerableCalc(expr#0..2=[{inputs}], DEPTNO=[$t0]) @@ -3273,6 +3428,7 @@ EnumerableCalc(expr#0..1=[{inputs}], expr#2=[IS NOT NULL($t1)], DEPTNO=[$t0], U= EnumerableCalc(expr#0..7=[{inputs}], expr#8=[IS NOT NULL($t6)], proj#0..7=[{exprs}], $condition=[$t8]) EnumerableTableScan(table=[[scott, EMP]]) !plan +!} # singleton keys have unique value which includes partial null rows. select deptno @@ -3289,7 +3445,7 @@ where unique (select comm from "scott".emp); (4 rows) !ok - +!if (use_old_decorr) { EnumerableCalc(expr#0..1=[{inputs}], expr#2=[IS NULL($t1)], DEPTNO=[$t0], $condition=[$t2]) EnumerableNestedLoopJoin(condition=[true], joinType=[left]) EnumerableCalc(expr#0..2=[{inputs}], DEPTNO=[$t0]) @@ -3300,6 +3456,7 @@ EnumerableCalc(expr#0..1=[{inputs}], expr#2=[IS NULL($t1)], DEPTNO=[$t0], $condi EnumerableCalc(expr#0..7=[{inputs}], expr#8=[IS NOT NULL($t6)], proj#0..7=[{exprs}], $condition=[$t8]) EnumerableTableScan(table=[[scott, EMP]]) !plan +!} # Previous, as scalar sub-query. select deptno, unique (select comm from "scott".emp) as u @@ -3315,7 +3472,7 @@ from "scott".dept; (4 rows) !ok - +!if (use_old_decorr) { EnumerableCalc(expr#0..1=[{inputs}], expr#2=[IS NULL($t1)], DEPTNO=[$t0], U=[$t2]) EnumerableNestedLoopJoin(condition=[true], joinType=[left]) EnumerableCalc(expr#0..2=[{inputs}], DEPTNO=[$t0]) @@ -3326,6 +3483,7 @@ EnumerableCalc(expr#0..1=[{inputs}], expr#2=[IS NULL($t1)], DEPTNO=[$t0], U=[$t2 EnumerableCalc(expr#0..7=[{inputs}], expr#8=[IS NOT NULL($t6)], proj#0..7=[{exprs}], $condition=[$t8]) EnumerableTableScan(table=[[scott, EMP]]) !plan +!} # singleton keys which includes fully null rows. select deptno @@ -3342,10 +3500,11 @@ where unique (select comm from "scott".emp where comm is null); (4 rows) !ok - +!if (use_old_decorr) { EnumerableCalc(expr#0..2=[{inputs}], DEPTNO=[$t0]) EnumerableTableScan(table=[[scott, DEPT]]) !plan +!} # Previous, as scalar sub-query. select deptno, unique (select comm from "scott".emp where comm is null) as u @@ -3361,10 +3520,11 @@ from "scott".dept; (4 rows) !ok - +!if (use_old_decorr) { EnumerableCalc(expr#0..2=[{inputs}], expr#3=[true], DEPTNO=[$t0], U=[$t3]) EnumerableTableScan(table=[[scott, DEPT]]) !plan +!} # composite keys have unique value which excludes fully or partially null rows. select deptno @@ -3381,7 +3541,7 @@ where unique (select comm, sal from "scott".emp where comm is not null); (4 rows) !ok - +!if (use_old_decorr) { EnumerableCalc(expr#0..1=[{inputs}], expr#2=[IS NULL($t1)], DEPTNO=[$t0], $condition=[$t2]) EnumerableNestedLoopJoin(condition=[true], joinType=[left]) EnumerableCalc(expr#0..2=[{inputs}], DEPTNO=[$t0]) @@ -3392,6 +3552,7 @@ EnumerableCalc(expr#0..1=[{inputs}], expr#2=[IS NULL($t1)], DEPTNO=[$t0], $condi EnumerableCalc(expr#0..7=[{inputs}], expr#8=[IS NOT NULL($t6)], expr#9=[IS NOT NULL($t5)], expr#10=[AND($t8, $t9)], proj#0..7=[{exprs}], $condition=[$t10]) EnumerableTableScan(table=[[scott, EMP]]) !plan +!} # Previous, as scalar sub-query. select deptno, unique (select comm, sal from "scott".emp where comm is not null) as u @@ -3407,7 +3568,7 @@ from "scott".dept; (4 rows) !ok - +!if (use_old_decorr) { EnumerableCalc(expr#0..1=[{inputs}], expr#2=[IS NULL($t1)], DEPTNO=[$t0], U=[$t2]) EnumerableNestedLoopJoin(condition=[true], joinType=[left]) EnumerableCalc(expr#0..2=[{inputs}], DEPTNO=[$t0]) @@ -3418,7 +3579,7 @@ EnumerableCalc(expr#0..1=[{inputs}], expr#2=[IS NULL($t1)], DEPTNO=[$t0], U=[$t2 EnumerableCalc(expr#0..7=[{inputs}], expr#8=[IS NOT NULL($t6)], expr#9=[IS NOT NULL($t5)], expr#10=[AND($t8, $t9)], proj#0..7=[{exprs}], $condition=[$t10]) EnumerableTableScan(table=[[scott, EMP]]) !plan - +!} # composite keys have unique value which includes fully or partially null rows. @@ -3436,7 +3597,7 @@ where unique (select comm, sal from "scott".emp); (4 rows) !ok - +!if (use_old_decorr) { EnumerableCalc(expr#0..1=[{inputs}], expr#2=[IS NULL($t1)], DEPTNO=[$t0], $condition=[$t2]) EnumerableNestedLoopJoin(condition=[true], joinType=[left]) EnumerableCalc(expr#0..2=[{inputs}], DEPTNO=[$t0]) @@ -3447,6 +3608,7 @@ EnumerableCalc(expr#0..1=[{inputs}], expr#2=[IS NULL($t1)], DEPTNO=[$t0], $condi EnumerableCalc(expr#0..7=[{inputs}], expr#8=[IS NOT NULL($t6)], expr#9=[IS NOT NULL($t5)], expr#10=[AND($t8, $t9)], proj#0..7=[{exprs}], $condition=[$t10]) EnumerableTableScan(table=[[scott, EMP]]) !plan +!} # Previous, as scalar sub-query. select deptno, unique (select comm, sal from "scott".emp) as u @@ -3462,7 +3624,7 @@ from "scott".dept; (4 rows) !ok - +!if (use_old_decorr) { EnumerableCalc(expr#0..1=[{inputs}], expr#2=[IS NULL($t1)], DEPTNO=[$t0], U=[$t2]) EnumerableNestedLoopJoin(condition=[true], joinType=[left]) EnumerableCalc(expr#0..2=[{inputs}], DEPTNO=[$t0]) @@ -3473,6 +3635,7 @@ EnumerableCalc(expr#0..1=[{inputs}], expr#2=[IS NULL($t1)], DEPTNO=[$t0], U=[$t2 EnumerableCalc(expr#0..7=[{inputs}], expr#8=[IS NOT NULL($t6)], expr#9=[IS NOT NULL($t5)], expr#10=[AND($t8, $t9)], proj#0..7=[{exprs}], $condition=[$t10]) EnumerableTableScan(table=[[scott, EMP]]) !plan +!} # singleton keys have duplicate value select deptno @@ -3485,7 +3648,7 @@ where unique (select deptno from "scott".emp); (0 rows) !ok - +!if (use_old_decorr) { EnumerableCalc(expr#0..1=[{inputs}], expr#2=[IS NULL($t1)], DEPTNO=[$t0], $condition=[$t2]) EnumerableNestedLoopJoin(condition=[true], joinType=[left]) EnumerableCalc(expr#0..2=[{inputs}], DEPTNO=[$t0]) @@ -3496,6 +3659,7 @@ EnumerableCalc(expr#0..1=[{inputs}], expr#2=[IS NULL($t1)], DEPTNO=[$t0], $condi EnumerableCalc(expr#0..7=[{inputs}], expr#8=[IS NOT NULL($t7)], proj#0..7=[{exprs}], $condition=[$t8]) EnumerableTableScan(table=[[scott, EMP]]) !plan +!} # Previous, as scalar sub-query. select deptno, unique (select deptno from "scott".emp) as u @@ -3511,7 +3675,7 @@ from "scott".dept; (4 rows) !ok - +!if (use_old_decorr) { EnumerableCalc(expr#0..1=[{inputs}], expr#2=[IS NULL($t1)], DEPTNO=[$t0], U=[$t2]) EnumerableNestedLoopJoin(condition=[true], joinType=[left]) EnumerableCalc(expr#0..2=[{inputs}], DEPTNO=[$t0]) @@ -3522,6 +3686,7 @@ EnumerableCalc(expr#0..1=[{inputs}], expr#2=[IS NULL($t1)], DEPTNO=[$t0], U=[$t2 EnumerableCalc(expr#0..7=[{inputs}], expr#8=[IS NOT NULL($t7)], proj#0..7=[{exprs}], $condition=[$t8]) EnumerableTableScan(table=[[scott, EMP]]) !plan +!} # composite keys have duplicate value. select deptno @@ -3534,7 +3699,7 @@ where unique (select deptno, sal from "scott".emp where sal = 3000); (0 rows) !ok - +!if (use_old_decorr) { EnumerableCalc(expr#0..1=[{inputs}], expr#2=[IS NULL($t1)], DEPTNO=[$t0], $condition=[$t2]) EnumerableNestedLoopJoin(condition=[true], joinType=[left]) EnumerableCalc(expr#0..2=[{inputs}], DEPTNO=[$t0]) @@ -3545,6 +3710,7 @@ EnumerableCalc(expr#0..1=[{inputs}], expr#2=[IS NULL($t1)], DEPTNO=[$t0], $condi EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t5):DECIMAL(12, 2)], expr#9=[3000.00:DECIMAL(12, 2)], expr#10=[=($t8, $t9)], expr#11=[IS NOT NULL($t7)], expr#12=[AND($t10, $t11)], proj#0..7=[{exprs}], $condition=[$t12]) EnumerableTableScan(table=[[scott, EMP]]) !plan +!} # Previous, as scalar sub-query. select deptno, unique (select deptno, sal from "scott".emp where sal = 3000) as u @@ -3560,7 +3726,7 @@ from "scott".dept; (4 rows) !ok - +!if (use_old_decorr) { EnumerableCalc(expr#0..1=[{inputs}], expr#2=[IS NULL($t1)], DEPTNO=[$t0], U=[$t2]) EnumerableNestedLoopJoin(condition=[true], joinType=[left]) EnumerableCalc(expr#0..2=[{inputs}], DEPTNO=[$t0]) @@ -3571,6 +3737,7 @@ EnumerableCalc(expr#0..1=[{inputs}], expr#2=[IS NULL($t1)], DEPTNO=[$t0], U=[$t2 EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t5):DECIMAL(12, 2)], expr#9=[3000.00:DECIMAL(12, 2)], expr#10=[=($t8, $t9)], expr#11=[IS NOT NULL($t7)], expr#12=[AND($t10, $t11)], proj#0..7=[{exprs}], $condition=[$t12]) EnumerableTableScan(table=[[scott, EMP]]) !plan +!} # Previous, but NOT UNIQUE. select deptno, not unique (select deptno, sal from "scott".emp where sal = 3000) as u @@ -3586,7 +3753,7 @@ from "scott".dept; (4 rows) !ok - +!if (use_old_decorr) { EnumerableCalc(expr#0..1=[{inputs}], expr#2=[IS NOT NULL($t1)], DEPTNO=[$t0], U=[$t2]) EnumerableNestedLoopJoin(condition=[true], joinType=[left]) EnumerableCalc(expr#0..2=[{inputs}], DEPTNO=[$t0]) @@ -3597,6 +3764,7 @@ EnumerableCalc(expr#0..1=[{inputs}], expr#2=[IS NOT NULL($t1)], DEPTNO=[$t0], U= EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t5):DECIMAL(12, 2)], expr#9=[3000.00:DECIMAL(12, 2)], expr#10=[=($t8, $t9)], expr#11=[IS NOT NULL($t7)], expr#12=[AND($t10, $t11)], proj#0..7=[{exprs}], $condition=[$t12]) EnumerableTableScan(table=[[scott, EMP]]) !plan +!} # as above, but sub-query empty. select deptno @@ -3613,7 +3781,7 @@ where unique (select deptno from "scott".emp where deptno = 35); (4 rows) !ok - +!if (use_old_decorr) { EnumerableCalc(expr#0..1=[{inputs}], expr#2=[IS NULL($t1)], DEPTNO=[$t0], $condition=[$t2]) EnumerableNestedLoopJoin(condition=[true], joinType=[left]) EnumerableCalc(expr#0..2=[{inputs}], DEPTNO=[$t0]) @@ -3623,6 +3791,7 @@ EnumerableCalc(expr#0..1=[{inputs}], expr#2=[IS NULL($t1)], DEPTNO=[$t0], $condi EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t7):INTEGER], expr#9=[35], expr#10=[=($t8, $t9)], proj#0..7=[{exprs}], $condition=[$t10]) EnumerableTableScan(table=[[scott, EMP]]) !plan +!} # Previous, as scalar sub-query. select deptno, unique (select deptno from "scott".emp where deptno = 35) as u @@ -3638,7 +3807,7 @@ from "scott".dept; (4 rows) !ok - +!if (use_old_decorr) { EnumerableCalc(expr#0..1=[{inputs}], expr#2=[IS NULL($t1)], DEPTNO=[$t0], U=[$t2]) EnumerableNestedLoopJoin(condition=[true], joinType=[left]) EnumerableCalc(expr#0..2=[{inputs}], DEPTNO=[$t0]) @@ -3648,6 +3817,7 @@ EnumerableCalc(expr#0..1=[{inputs}], expr#2=[IS NULL($t1)], DEPTNO=[$t0], U=[$t2 EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t7):INTEGER], expr#9=[35], expr#10=[=($t8, $t9)], proj#0..7=[{exprs}], $condition=[$t10]) EnumerableTableScan(table=[[scott, EMP]]) !plan +!} # singleton keys which a uniqueness constraint indicates that the relation is already unique. select * @@ -3664,9 +3834,10 @@ where unique (select deptno from "scott".dept); (4 rows) !ok - +!if (use_old_decorr) { EnumerableTableScan(table=[[scott, DEPT]]) !plan +!} # as above, sub-query with limit. select * @@ -3683,9 +3854,10 @@ where unique (select deptno from "scott".emp limit 1); (4 rows) !ok - +!if (use_old_decorr) { EnumerableTableScan(table=[[scott, DEPT]]) !plan +!} # as above, sub-query with distinct. select deptno @@ -3702,10 +3874,11 @@ where unique (select distinct deptno, sal from "scott".emp where sal = 3000); (4 rows) !ok - +!if (use_old_decorr) { EnumerableCalc(expr#0..2=[{inputs}], DEPTNO=[$t0]) EnumerableTableScan(table=[[scott, DEPT]]) !plan +!} # as above, sub-query with group by. select deptno @@ -3722,10 +3895,11 @@ where unique (select job from "scott".emp group by job); (4 rows) !ok - +!if (use_old_decorr) { EnumerableCalc(expr#0..2=[{inputs}], DEPTNO=[$t0]) EnumerableTableScan(table=[[scott, DEPT]]) !plan +!} # Correlated UNIQUE predicate. select * @@ -3740,7 +3914,7 @@ where unique ( (1 row) !ok - +!if (use_old_decorr) { EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t3)], proj#0..2=[{exprs}], $condition=[$t5]) EnumerableMergeJoin(condition=[=($0, $4)], joinType=[left]) EnumerableTableScan(table=[[scott, DEPT]]) @@ -3750,6 +3924,7 @@ EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t3)], proj#0..2=[{exprs}], EnumerableCalc(expr#0..7=[{inputs}], expr#8=[IS NOT NULL($t7)], proj#0..7=[{exprs}], $condition=[$t8]) EnumerableTableScan(table=[[scott, EMP]]) !plan +!} # Previous, as scalar sub-query. select *, unique (select 1 from "scott".emp where dept.deptno = emp.deptno) as u @@ -3765,7 +3940,7 @@ from "scott".dept; (4 rows) !ok - +!if (use_old_decorr) { EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t3)], proj#0..2=[{exprs}], U=[$t5]) EnumerableMergeJoin(condition=[=($0, $4)], joinType=[left]) EnumerableTableScan(table=[[scott, DEPT]]) @@ -3775,6 +3950,7 @@ EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t3)], proj#0..2=[{exprs}], EnumerableCalc(expr#0..7=[{inputs}], expr#8=[IS NOT NULL($t7)], proj#0..7=[{exprs}], $condition=[$t8]) EnumerableTableScan(table=[[scott, EMP]]) !plan +!} # as above, but NOT UNIQUE. select * @@ -3791,7 +3967,7 @@ where not unique ( (3 rows) !ok - +!if (use_old_decorr) { EnumerableHashJoin(condition=[=($0, $3)], joinType=[semi]) EnumerableTableScan(table=[[scott, DEPT]]) EnumerableCalc(expr#0..1=[{inputs}], expr#2=[1], expr#3=[>($t1, $t2)], DEPTNO=[$t0], $condition=[$t3]) @@ -3799,6 +3975,7 @@ EnumerableHashJoin(condition=[=($0, $3)], joinType=[semi]) EnumerableCalc(expr#0..7=[{inputs}], expr#8=[IS NOT NULL($t7)], proj#0..7=[{exprs}], $condition=[$t8]) EnumerableTableScan(table=[[scott, EMP]]) !plan +!} # Previous, as scalar sub-query. select *, not unique (select 1 from "scott".emp where dept.deptno = emp.deptno) as u @@ -3814,7 +3991,7 @@ from "scott".dept; (4 rows) !ok - +!if (use_old_decorr) { EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NOT NULL($t3)], proj#0..2=[{exprs}], U=[$t5]) EnumerableMergeJoin(condition=[=($0, $4)], joinType=[left]) EnumerableTableScan(table=[[scott, DEPT]]) @@ -3824,6 +4001,7 @@ EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NOT NULL($t3)], proj#0..2=[{expr EnumerableCalc(expr#0..7=[{inputs}], expr#8=[IS NOT NULL($t7)], proj#0..7=[{exprs}], $condition=[$t8]) EnumerableTableScan(table=[[scott, EMP]]) !plan +!} # [CALCITE-4805] Calcite should convert a small IN-list as if the # user had written OR, even if the IN-list contains NULL. @@ -3839,10 +4017,11 @@ select * from "scott".emp where comm in (300, 500, null); (2 rows) !ok - +!if (use_old_decorr) { EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t6):DECIMAL(12, 2)], expr#9=[Sarg[300.00:DECIMAL(12, 2), 500.00:DECIMAL(12, 2)]:DECIMAL(12, 2)], expr#10=[SEARCH($t8, $t9)], proj#0..7=[{exprs}], $condition=[$t10]) EnumerableTableScan(table=[[scott, EMP]]) !plan +!} # Previous, as scalar sub-query. select *, comm in (300, 500, null) as i from "scott".emp; @@ -3867,10 +4046,11 @@ select *, comm in (300, 500, null) as i from "scott".emp; (14 rows) !ok - +!if (use_old_decorr) { EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t6):DECIMAL(12, 2)], expr#9=[Sarg[300.00:DECIMAL(12, 2), 500.00:DECIMAL(12, 2)]:DECIMAL(12, 2)], expr#10=[SEARCH($t8, $t9)], expr#11=[null:BOOLEAN], expr#12=[OR($t10, $t11)], proj#0..7=[{exprs}], I=[$t12]) EnumerableTableScan(table=[[scott, EMP]]) !plan +!} # As above, but NOT IN. select * from "scott".emp where comm not in (300, 500, null); @@ -3881,9 +4061,10 @@ select * from "scott".emp where comm not in (300, 500, null); (0 rows) !ok - +!if (use_old_decorr) { EnumerableValues(tuples=[[]]) !plan +!} # Previous, as scalar sub-query. select *, comm not in (300, 500, null) as i from "scott".emp; @@ -3908,10 +4089,11 @@ select *, comm not in (300, 500, null) as i from "scott".emp; (14 rows) !ok - +!if (use_old_decorr) { EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t6):DECIMAL(12, 2)], expr#9=[Sarg[(-∞..300.00:DECIMAL(12, 2)), (300.00:DECIMAL(12, 2)..500.00:DECIMAL(12, 2)), (500.00:DECIMAL(12, 2)..+∞)]:DECIMAL(12, 2)], expr#10=[SEARCH($t8, $t9)], expr#11=[null:BOOLEAN], expr#12=[AND($t10, $t11)], proj#0..7=[{exprs}], I=[$t12]) EnumerableTableScan(table=[[scott, EMP]]) !plan +!} # Previous NOT IN expressions in conjunction form select *, (comm <> 300 and comm <> 500 and comm <> null) as i from "scott".emp; @@ -3936,9 +4118,11 @@ select *, (comm <> 300 and comm <> 500 and comm <> null) as i from "scott".emp; (14 rows) !ok +!if (use_old_decorr) { EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t6):DECIMAL(12, 2)], expr#9=[Sarg[(-∞..300.00:DECIMAL(12, 2)), (300.00:DECIMAL(12, 2)..500.00:DECIMAL(12, 2)), (500.00:DECIMAL(12, 2)..+∞)]:DECIMAL(12, 2)], expr#10=[SEARCH($t8, $t9)], expr#11=[null:BOOLEAN], expr#12=[AND($t10, $t11)], proj#0..7=[{exprs}], I=[$t12]) EnumerableTableScan(table=[[scott, EMP]]) !plan +!} # The IN-list only contains null value. select * from "scott".emp where empno in (null); @@ -3949,9 +4133,10 @@ select * from "scott".emp where empno in (null); (0 rows) !ok - +!if (use_old_decorr) { EnumerableValues(tuples=[[]]) !plan +!} # Previous, as scalar sub-query. select *, empno in (null) as i from "scott".emp; @@ -3976,10 +4161,11 @@ select *, empno in (null) as i from "scott".emp; (14 rows) !ok - +!if (use_old_decorr) { EnumerableCalc(expr#0..7=[{inputs}], expr#8=[null:BOOLEAN], proj#0..8=[{exprs}]) EnumerableTableScan(table=[[scott, EMP]]) !plan +!} # As above, but NOT IN. select * from "scott".emp where empno not in (null); @@ -3990,9 +4176,10 @@ select * from "scott".emp where empno not in (null); (0 rows) !ok - +!if (use_old_decorr) { EnumerableValues(tuples=[[]]) !plan +!} # Previous, as scalar sub-query. select *, empno not in (null) as i from "scott".emp; @@ -4017,10 +4204,11 @@ select *, empno not in (null) as i from "scott".emp; (14 rows) !ok - +!if (use_old_decorr) { EnumerableCalc(expr#0..7=[{inputs}], expr#8=[null:BOOLEAN], proj#0..8=[{exprs}]) EnumerableTableScan(table=[[scott, EMP]]) !plan +!} # [CALCITE-4844] IN-list that references columns is wrongly converted to Values, and gives incorrect results @@ -4036,10 +4224,11 @@ SELECT empno, ename, mgr FROM "scott".emp WHERE 7782 IN (empno, mgr); (2 rows) !ok - +!if (use_old_decorr) { EnumerableCalc(expr#0..7=[{inputs}], expr#8=[7782], expr#9=[CAST($t0):INTEGER NOT NULL], expr#10=[=($t8, $t9)], expr#11=[CAST($t3):INTEGER], expr#12=[=($t8, $t11)], expr#13=[OR($t10, $t12)], proj#0..1=[{exprs}], MGR=[$t3], $condition=[$t13]) EnumerableTableScan(table=[[scott, EMP]]) !plan +!} SELECT empno, ename, mgr FROM "scott".emp WHERE (7782, 7839) IN ((empno, mgr), (mgr, empno)); +-------+-------+------+ @@ -4050,10 +4239,11 @@ SELECT empno, ename, mgr FROM "scott".emp WHERE (7782, 7839) IN ((empno, mgr), ( (1 row) !ok - +!if (use_old_decorr) { EnumerableCalc(expr#0..7=[{inputs}], expr#8=[7782], expr#9=[CAST($t0):INTEGER NOT NULL], expr#10=[=($t8, $t9)], expr#11=[7839], expr#12=[CAST($t3):INTEGER], expr#13=[=($t11, $t12)], expr#14=[AND($t10, $t13)], expr#15=[=($t8, $t12)], expr#16=[=($t11, $t9)], expr#17=[AND($t15, $t16)], expr#18=[OR($t14, $t17)], proj#0..1=[{exprs}], MGR=[$t3], $condition=[$t18]) EnumerableTableScan(table=[[scott, EMP]]) !plan +!} SELECT empno, ename, mgr FROM "scott".emp WHERE (7782, 7839) IN ((empno, 7839), (7782, mgr)); +-------+-------+------+ @@ -4066,10 +4256,11 @@ SELECT empno, ename, mgr FROM "scott".emp WHERE (7782, 7839) IN ((empno, 7839), (3 rows) !ok - +!if (use_old_decorr) { EnumerableCalc(expr#0..7=[{inputs}], expr#8=[7782], expr#9=[CAST($t0):INTEGER NOT NULL], expr#10=[=($t8, $t9)], expr#11=[7839], expr#12=[CAST($t3):INTEGER], expr#13=[=($t11, $t12)], expr#14=[OR($t10, $t13)], proj#0..1=[{exprs}], MGR=[$t3], $condition=[$t14]) EnumerableTableScan(table=[[scott, EMP]]) !plan +!} # Reset to default value 20 !set insubquerythreshold 20 @@ -4084,7 +4275,7 @@ select * from "scott".emp where empno not in (null, 7782); (0 rows) !ok - +!if (use_old_decorr) { EnumerableCalc(expr#0..12=[{inputs}], expr#13=[IS NULL($t12)], expr#14=[>=($t9, $t8)], expr#15=[AND($t13, $t14)], expr#16=[0], expr#17=[=($t8, $t16)], expr#18=[OR($t15, $t17)], proj#0..7=[{exprs}], $condition=[$t18]) EnumerableMergeJoin(condition=[=($10, $11)], joinType=[left]) EnumerableSort(sort0=[$10], dir0=[ASC]) @@ -4097,6 +4288,7 @@ EnumerableCalc(expr#0..12=[{inputs}], expr#13=[IS NULL($t12)], expr#14=[>=($t9, EnumerableCalc(expr#0=[{inputs}], expr#1=[true], proj#0..1=[{exprs}]) EnumerableValues(tuples=[[{ null }, { 7782 }]]) !plan +!} select * from "scott".emp where (empno, deptno) not in ((1, 2), (3, null)); +-------+--------+-----------+------+------------+---------+---------+--------+ @@ -4120,7 +4312,7 @@ select * from "scott".emp where (empno, deptno) not in ((1, 2), (3, null)); (14 rows) !ok - +!if (use_old_decorr) { EnumerableCalc(expr#0..14=[{inputs}], expr#15=[0], expr#16=[=($t8, $t15)], expr#17=[IS NULL($t14)], expr#18=[>=($t9, $t8)], expr#19=[IS NOT NULL($t7)], expr#20=[AND($t17, $t18, $t19)], expr#21=[OR($t16, $t20)], proj#0..7=[{exprs}], $condition=[$t21]) EnumerableMergeJoin(condition=[AND(=($10, $12), =($11, $13))], joinType=[left]) EnumerableSort(sort0=[$10], sort1=[$11], dir0=[ASC], dir1=[ASC]) @@ -4133,6 +4325,7 @@ EnumerableCalc(expr#0..14=[{inputs}], expr#15=[0], expr#16=[=($t8, $t15)], expr# EnumerableCalc(expr#0..1=[{inputs}], expr#2=[true], proj#0..2=[{exprs}]) EnumerableValues(tuples=[[{ 3, null }, { 1, 2 }]]) !plan +!} # As above, but the IN-list includes all null value select * from "scott".emp where (mgr, deptno) not in ((1, 2), (3, null), (cast(null as integer), cast(null as integer))); @@ -4143,7 +4336,7 @@ select * from "scott".emp where (mgr, deptno) not in ((1, 2), (3, null), (cast(n (0 rows) !ok - +!if (use_old_decorr) { EnumerableCalc(expr#0..14=[{inputs}], expr#15=[0], expr#16=[=($t8, $t15)], expr#17=[IS NULL($t14)], expr#18=[>=($t9, $t8)], expr#19=[IS NOT NULL($t3)], expr#20=[IS NOT NULL($t7)], expr#21=[AND($t17, $t18, $t19, $t20)], expr#22=[OR($t16, $t21)], proj#0..7=[{exprs}], $condition=[$t22]) EnumerableMergeJoin(condition=[AND(=($10, $12), =($11, $13))], joinType=[left]) EnumerableSort(sort0=[$10], sort1=[$11], dir0=[ASC], dir1=[ASC]) @@ -4157,6 +4350,7 @@ EnumerableCalc(expr#0..14=[{inputs}], expr#15=[0], expr#16=[=($t8, $t15)], expr# EnumerableCalc(expr#0..1=[{inputs}], expr#2=[true], proj#0..2=[{exprs}]) EnumerableValues(tuples=[[{ 3, null }, { null, null }, { 1, 2 }]]) !plan +!} select * from "scott".emp where (empno, deptno) not in ((1, 2), (3, null), (cast(null as integer), cast(null as integer))); +-------+-------+-----+-----+----------+-----+------+--------+ @@ -4187,7 +4381,7 @@ select * from "scott".emp where (empno, deptno) not in ((7369, 20), (7499, 30)); (12 rows) !ok - +!if (use_old_decorr) { EnumerableCalc(expr#0..14=[{inputs}], expr#15=[0], expr#16=[=($t8, $t15)], expr#17=[IS NULL($t14)], expr#18=[>=($t9, $t8)], expr#19=[IS NOT NULL($t7)], expr#20=[AND($t17, $t18, $t19)], expr#21=[OR($t16, $t20)], proj#0..7=[{exprs}], $condition=[$t21]) EnumerableMergeJoin(condition=[AND(=($10, $12), =($11, $13))], joinType=[left]) EnumerableSort(sort0=[$10], sort1=[$11], dir0=[ASC], dir1=[ASC]) @@ -4201,6 +4395,7 @@ EnumerableCalc(expr#0..14=[{inputs}], expr#15=[0], expr#16=[=($t8, $t15)], expr# EnumerableCalc(expr#0..1=[{inputs}], expr#2=[true], proj#0..2=[{exprs}]) EnumerableValues(tuples=[[{ 7369, 20 }, { 7499, 30 }]]) !plan +!} # Reset to default value 20 !set insubquerythreshold 20 @@ -4222,9 +4417,10 @@ where EXISTS (select count(*) from emp e where d.deptno = e.deptno); (4 rows) !ok - +!if (use_old_decorr) { EnumerableTableScan(table=[[scott, DEPT]]) !plan +!} # As above, but the filter condition always false select * @@ -4241,10 +4437,10 @@ where EXISTS (select count(*) from emp e where d.deptno = e.deptno and 1 = 2); (4 rows) !ok - +!if (use_old_decorr) { EnumerableTableScan(table=[[scott, DEPT]]) !plan - +!} # As above, but the Sum aggregation function select * @@ -4261,9 +4457,10 @@ where EXISTS (select sum(1) from emp e where d.deptno = e.deptno and 1 = 2); (4 rows) !ok - +!if (use_old_decorr) { EnumerableTableScan(table=[[scott, DEPT]]) !plan +!} # Test case about sub-query is guaranteed to produce no row select * @@ -4280,9 +4477,10 @@ where NOT EXISTS (select count(*) from emp e having false); (4 rows) !ok - +!if (use_old_decorr) { EnumerableTableScan(table=[[scott, DEPT]]) !plan +!} # Test case about nested row select (select (1, 2)); @@ -4294,10 +4492,11 @@ select (select (1, 2)); (1 row) !ok - +!if (use_old_decorr) { EnumerableCalc(expr#0=[{inputs}], expr#1=[1], expr#2=[2], expr#3=[ROW($t1, $t2)], expr#4=[CAST($t3):RecordType(INTEGER EXPR$0, INTEGER EXPR$1)], EXPR$0=[$t4]) EnumerableValues(tuples=[[{ 0 }]]) !plan +!} # Test case for correlated sub-query SELECT ARRAY(SELECT s.x) FROM (SELECT 1 as x) s; @@ -4595,6 +4794,7 @@ select deptno from dept d1 where exists ( (4 rows) !ok +!if (use_old_decorr) { EnumerableCalc(expr#0..3=[{inputs}], DEPTNO=[$t2]) EnumerableHashJoin(condition=[AND(=($0, $2), =($1, $3))], joinType=[inner]) EnumerableCalc(expr#0..2=[{inputs}], proj#0..1=[{exprs}]) @@ -4606,6 +4806,7 @@ EnumerableCalc(expr#0..3=[{inputs}], DEPTNO=[$t2]) EnumerableCalc(expr#0..2=[{inputs}], proj#0..1=[{exprs}]) EnumerableTableScan(table=[[scott, DEPT]]) !plan +!} # Test case for CALCITE-5683 which throws an exception during the de-correlation phase SELECT d1.dname, d1.deptno + ( @@ -4901,7 +5102,7 @@ select empno, empno in (7369, 7499, 7521) from emp; !ok - +!if (use_old_decorr) { EnumerableCalc(expr#0..3=[{inputs}], expr#4=[IS NOT NULL($t3)], EMPNO=[$t0], EXPR$1=[$t4]) EnumerableMergeJoin(condition=[=($1, $2)], joinType=[left]) EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t0):INTEGER NOT NULL], EMPNO=[$t0], EMPNO0=[$t8]) @@ -4909,6 +5110,7 @@ EnumerableCalc(expr#0..3=[{inputs}], expr#4=[IS NOT NULL($t3)], EMPNO=[$t0], EXP EnumerableCalc(expr#0=[{inputs}], expr#1=[true], proj#0..1=[{exprs}]) EnumerableValues(tuples=[[{ 7369 }, { 7499 }, { 7521 }]]) !plan +!} # Test LHS is nullable and RHS is not nullable select comm, comm in (500, 300, 0) from emp; @@ -4933,7 +5135,7 @@ select comm, comm in (500, 300, 0) from emp; (14 rows) !ok - +!if (use_old_decorr) { EnumerableCalc(expr#0..6=[{inputs}], expr#7=[IS NULL($t1)], expr#8=[null:BOOLEAN], expr#9=[0], expr#10=[<>($t2, $t9)], expr#11=[AND($t7, $t8, $t10)], expr#12=[IS NOT NULL($t6)], expr#13=[IS NOT NULL($t1)], expr#14=[AND($t12, $t10, $t13)], expr#15=[<($t3, $t2)], expr#16=[IS NULL($t6)], expr#17=[AND($t15, $t8, $t10, $t13, $t16)], expr#18=[OR($t11, $t14, $t17)], COMM=[$t1], EXPR$1=[$t18]) EnumerableMergeJoin(condition=[=($4, $5)], joinType=[left]) EnumerableSort(sort0=[$4], dir0=[ASC]) @@ -4948,6 +5150,7 @@ EnumerableCalc(expr#0..6=[{inputs}], expr#7=[IS NULL($t1)], expr#8=[null:BOOLEAN EnumerableCalc(expr#0=[{inputs}], expr#1=[true], proj#0..1=[{exprs}]) EnumerableValues(tuples=[[{ 500.00 }, { 300.00 }, { 0.00 }]]) !plan +!} # Test LHS is nullable and RHS is nullable @@ -4973,7 +5176,7 @@ select comm, comm in (500, 300, 0, null) from emp; (14 rows) !ok - +!if (use_old_decorr) { EnumerableCalc(expr#0..6=[{inputs}], expr#7=[IS NULL($t1)], expr#8=[null:BOOLEAN], expr#9=[0], expr#10=[<>($t2, $t9)], expr#11=[AND($t7, $t8, $t10)], expr#12=[IS NOT NULL($t6)], expr#13=[IS NOT NULL($t1)], expr#14=[AND($t12, $t10, $t13)], expr#15=[<($t3, $t2)], expr#16=[IS NULL($t6)], expr#17=[AND($t15, $t8, $t10, $t13, $t16)], expr#18=[OR($t11, $t14, $t17)], COMM=[$t1], EXPR$1=[$t18]) EnumerableMergeJoin(condition=[=($4, $5)], joinType=[left]) EnumerableSort(sort0=[$4], dir0=[ASC]) @@ -4987,6 +5190,7 @@ EnumerableCalc(expr#0..6=[{inputs}], expr#7=[IS NULL($t1)], expr#8=[null:BOOLEAN EnumerableCalc(expr#0=[{inputs}], expr#1=[true], proj#0..1=[{exprs}]) EnumerableValues(tuples=[[{ 500.00 }, { 300.00 }, { 0.00 }, { null }]]) !plan +!} # Reset to default value 20 !set insubquerythreshold 20 @@ -5015,7 +5219,7 @@ select empno, (empno, empno) in ((7369, 7369), (7499, 7499), (7521, 7521)) from (14 rows) !ok - +!if (use_old_decorr) { EnumerableCalc(expr#0..5=[{inputs}], expr#6=[IS NOT NULL($t5)], EMPNO=[$t0], EXPR$1=[$t6]) EnumerableMergeJoin(condition=[AND(=($1, $3), =($2, $4))], joinType=[left]) EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t0):INTEGER NOT NULL], EMPNO=[$t0], EMPNO0=[$t8], EMPNO1=[$t8]) @@ -5024,6 +5228,7 @@ EnumerableCalc(expr#0..5=[{inputs}], expr#6=[IS NOT NULL($t5)], EMPNO=[$t0], EXP EnumerableCalc(expr#0..1=[{inputs}], expr#2=[true], proj#0..2=[{exprs}]) EnumerableValues(tuples=[[{ 7369, 7369 }, { 7499, 7499 }, { 7521, 7521 }]]) !plan +!} # Test LHS is (nullable, nullable) and RHS is (not nullable, not nullable) @@ -5050,6 +5255,7 @@ select comm, (comm, comm) in ((500, 500), (300, 300), (0, 0)) from emp; !ok +!if (use_old_decorr) { EnumerableCalc(expr#0..8=[{inputs}], expr#9=[IS NULL($t1)], expr#10=[null:BOOLEAN], expr#11=[0], expr#12=[<>($t2, $t11)], expr#13=[AND($t9, $t10, $t12)], expr#14=[IS NOT NULL($t8)], expr#15=[IS NOT NULL($t1)], expr#16=[AND($t14, $t12, $t15)], expr#17=[<($t3, $t2)], expr#18=[IS NULL($t8)], expr#19=[AND($t17, $t10, $t12, $t15, $t18)], expr#20=[OR($t13, $t16, $t19)], COMM=[$t1], EXPR$1=[$t20]) EnumerableMergeJoin(condition=[AND(=($4, $6), =($5, $7))], joinType=[left]) EnumerableSort(sort0=[$4], sort1=[$5], dir0=[ASC], dir1=[ASC]) @@ -5063,6 +5269,7 @@ EnumerableCalc(expr#0..8=[{inputs}], expr#9=[IS NULL($t1)], expr#10=[null:BOOLEA EnumerableCalc(expr#0..1=[{inputs}], expr#2=[true], proj#0..2=[{exprs}]) EnumerableValues(tuples=[[{ 500.00, 500.00 }, { 300.00, 300.00 }, { 0.00, 0.00 }]]) !plan +!} # Test LHS is (nullable, nullable) and RHS is (nullable, nullable) @@ -5088,7 +5295,7 @@ select comm, (comm, comm) in ((500, 500), (300, 300), (0, 0), (null , null)) fro (14 rows) !ok - +!if (use_old_decorr) { EnumerableCalc(expr#0..8=[{inputs}], expr#9=[IS NULL($t1)], expr#10=[null:BOOLEAN], expr#11=[0], expr#12=[<>($t2, $t11)], expr#13=[AND($t9, $t10, $t12)], expr#14=[IS NOT NULL($t8)], expr#15=[IS NOT NULL($t1)], expr#16=[AND($t14, $t12, $t15)], expr#17=[<($t3, $t2)], expr#18=[IS NULL($t8)], expr#19=[AND($t17, $t10, $t12, $t15, $t18)], expr#20=[OR($t13, $t16, $t19)], COMM=[$t1], EXPR$1=[$t20]) EnumerableMergeJoin(condition=[AND(=($4, $6), =($5, $7))], joinType=[left]) EnumerableSort(sort0=[$4], sort1=[$5], dir0=[ASC], dir1=[ASC]) @@ -5103,6 +5310,7 @@ EnumerableCalc(expr#0..8=[{inputs}], expr#9=[IS NULL($t1)], expr#10=[null:BOOLEA EnumerableCalc(expr#0..1=[{inputs}], expr#2=[true], proj#0..2=[{exprs}]) EnumerableValues(tuples=[[{ 500.00, 500.00 }, { 300.00, 300.00 }, { 0.00, 0.00 }, { null, null }]]) !plan +!} # Reset to default value 20 !set insubquerythreshold 20 @@ -5122,7 +5330,7 @@ where deptno + 20 in (select deptno from dept); (2 rows) !ok - +!if (use_old_decorr) { EnumerableCalc(expr#0..3=[{inputs}], proj#0..2=[{exprs}]) EnumerableHashJoin(condition=[=($3, $4)], joinType=[semi]) EnumerableCalc(expr#0..2=[{inputs}], expr#3=[20], expr#4=[+($t0, $t3)], proj#0..2=[{exprs}], $f3=[$t4]) @@ -5130,6 +5338,7 @@ EnumerableCalc(expr#0..3=[{inputs}], proj#0..2=[{exprs}]) EnumerableCalc(expr#0..2=[{inputs}], expr#3=[CAST($t0):INTEGER NOT NULL], DEPTNO=[$t3]) EnumerableTableScan(table=[[scott, DEPT]]) !plan +!} # Test case about the IN sub-query left operand type is BIGINT and right operand type is TINYINT select * @@ -5146,7 +5355,7 @@ where cast(deptno as bigint) in (select deptno from dept); (4 rows) !ok - +!if (use_old_decorr) { EnumerableCalc(expr#0..3=[{inputs}], proj#0..2=[{exprs}]) EnumerableHashJoin(condition=[=($3, $4)], joinType=[semi]) EnumerableCalc(expr#0..2=[{inputs}], expr#3=[CAST($t0):BIGINT NOT NULL], proj#0..3=[{exprs}]) @@ -5154,6 +5363,7 @@ EnumerableCalc(expr#0..3=[{inputs}], proj#0..2=[{exprs}]) EnumerableCalc(expr#0..2=[{inputs}], expr#3=[CAST($t0):BIGINT NOT NULL], DEPTNO=[$t3]) EnumerableTableScan(table=[[scott, DEPT]]) !plan +!} # Test case about the IN sub-query left operand type is INTEGER and right operand type is BIGINT select * @@ -5167,7 +5377,7 @@ where deptno + 10 in (select count(*) + 10 from emp where comm is null); (1 row) !ok - +!if (use_old_decorr) { EnumerableCalc(expr#0..3=[{inputs}], proj#0..2=[{exprs}]) EnumerableHashJoin(condition=[=($3, $4)], joinType=[semi]) EnumerableCalc(expr#0..2=[{inputs}], expr#3=[10], expr#4=[+($t0, $t3)], expr#5=[CAST($t4):BIGINT NOT NULL], proj#0..2=[{exprs}], $f3=[$t5]) @@ -5177,6 +5387,7 @@ EnumerableCalc(expr#0..3=[{inputs}], proj#0..2=[{exprs}]) EnumerableCalc(expr#0..7=[{inputs}], expr#8=[IS NULL($t6)], proj#0..7=[{exprs}], $condition=[$t8]) EnumerableTableScan(table=[[scott, EMP]]) !plan +!} # Test case about the IN sub-query left operand type is SMALLINT and right operand type is TINYINT select * @@ -5190,7 +5401,7 @@ where cast(empno - 7349 as smallint) in (select deptno from emp) and ename = 'S (1 row) !ok - +!if (use_old_decorr) { EnumerableCalc(expr#0..8=[{inputs}], proj#0..7=[{exprs}]) EnumerableHashJoin(condition=[=($8, $9)], joinType=[semi]) EnumerableCalc(expr#0..7=[{inputs}], expr#8=[7349], expr#9=[-($t0, $t8)], expr#10=[CAST($t9):SMALLINT NOT NULL], expr#11=['SMITH':VARCHAR(10)], expr#12=[=($t1, $t11)], proj#0..7=[{exprs}], $f8=[$t10], $condition=[$t12]) @@ -5198,6 +5409,7 @@ EnumerableCalc(expr#0..8=[{inputs}], proj#0..7=[{exprs}]) EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t7):SMALLINT], DEPTNO=[$t8]) EnumerableTableScan(table=[[scott, EMP]]) !plan +!} # Test case about the IN sub-query left operand type is SMALLINT and right operand type is INTEGER select * @@ -5211,7 +5423,7 @@ where empno in (select deptno + 7349 from emp); (1 row) !ok - +!if (use_old_decorr) { EnumerableCalc(expr#0..8=[{inputs}], proj#0..7=[{exprs}]) EnumerableHashJoin(condition=[=($8, $9)], joinType=[semi]) EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t0):INTEGER NOT NULL], proj#0..8=[{exprs}]) @@ -5219,6 +5431,7 @@ EnumerableCalc(expr#0..8=[{inputs}], proj#0..7=[{exprs}]) EnumerableCalc(expr#0..7=[{inputs}], expr#8=[7349], expr#9=[+($t7, $t8)], EXPR$0=[$t9]) EnumerableTableScan(table=[[scott, EMP]]) !plan +!} # Test case about the IN sub-query left operand type is SMALLINT and right operand type is BIGINT select * @@ -5232,7 +5445,7 @@ where empno in (select cast(deptno + 7349 as bigint) from emp); (1 row) !ok - +!if (use_old_decorr) { EnumerableCalc(expr#0..8=[{inputs}], proj#0..7=[{exprs}]) EnumerableHashJoin(condition=[=($8, $9)], joinType=[semi]) EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t0):BIGINT NOT NULL], proj#0..8=[{exprs}]) @@ -5240,7 +5453,7 @@ EnumerableCalc(expr#0..8=[{inputs}], proj#0..7=[{exprs}]) EnumerableCalc(expr#0..7=[{inputs}], expr#8=[7349], expr#9=[+($t7, $t8)], expr#10=[CAST($t9):BIGINT], EXPR$0=[$t10]) EnumerableTableScan(table=[[scott, EMP]]) !plan - +!} # [CALCITE-6650] Optimize the IN sub-query and SOME sub-query by Metadata RowCount @@ -5255,9 +5468,10 @@ select * from emp where deptno > some(select deptno from dept where false); (0 rows) !ok - +!if (use_old_decorr) { EnumerableValues(tuples=[[]]) !plan +!} # Same as previous; but is Scalar sub-query select deptno, deptno > some(select deptno from dept where false) from emp; @@ -5277,10 +5491,11 @@ select deptno, deptno > some(select deptno from dept where false) from emp; (9 rows) !ok - +!if (use_old_decorr) { EnumerableCalc(expr#0=[{inputs}], expr#1=[false], expr#2=[CAST($t1):BOOLEAN], DEPTNO=[$t0], EXPR$1=[$t2]) EnumerableValues(tuples=[[{ 10 }, { 10 }, { 20 }, { 30 }, { 30 }, { 50 }, { 50 }, { 60 }, { null }]]) !plan +!} # Same as previous; but LHS is NULL select deptno, null > some(select deptno from dept where false) from emp; @@ -5300,10 +5515,11 @@ select deptno, null > some(select deptno from dept where false) from emp; (9 rows) !ok - +!if (use_old_decorr) { EnumerableCalc(expr#0=[{inputs}], expr#1=[false], expr#2=[CAST($t1):BOOLEAN], DEPTNO=[$t0], EXPR$1=[$t2]) EnumerableValues(tuples=[[{ 10 }, { 10 }, { 20 }, { 30 }, { 30 }, { 50 }, { 50 }, { 60 }, { null }]]) !plan +!} # Test case about ANY sub-query when sub-query return 0 row select * from emp where deptno > any(select deptno from dept where false); @@ -5314,9 +5530,10 @@ select * from emp where deptno > any(select deptno from dept where false); (0 rows) !ok - +!if (use_old_decorr) { EnumerableValues(tuples=[[]]) !plan +!} # Same as previous; but is Scalar sub-query select deptno, deptno > any(select deptno from dept where false) from emp; @@ -5336,10 +5553,11 @@ select deptno, deptno > any(select deptno from dept where false) from emp; (9 rows) !ok - +!if (use_old_decorr) { EnumerableCalc(expr#0=[{inputs}], expr#1=[false], expr#2=[CAST($t1):BOOLEAN], DEPTNO=[$t0], EXPR$1=[$t2]) EnumerableValues(tuples=[[{ 10 }, { 10 }, { 20 }, { 30 }, { 30 }, { 50 }, { 50 }, { 60 }, { null }]]) !plan +!} # Test case about UNIQUE sub-query when sub-query return 0 row select * from emp where unique (select deptno from dept where false); @@ -5359,9 +5577,10 @@ select * from emp where unique (select deptno from dept where false); (9 rows) !ok - +!if (use_old_decorr) { EnumerableValues(tuples=[[{ 'Jane ', 10, 'F' }, { 'Bob ', 10, 'M' }, { 'Eric ', 20, 'M' }, { 'Susan', 30, 'F' }, { 'Alice', 30, 'F' }, { 'Adam ', 50, 'M' }, { 'Eve ', 50, 'F' }, { 'Grace', 60, 'F' }, { 'Wilma', null, 'F' }]]) !plan +!} # Same as previous; but is Scalar sub-query select unique (select deptno from dept where false) from emp; @@ -5381,10 +5600,11 @@ select unique (select deptno from dept where false) from emp; (9 rows) !ok - +!if (use_old_decorr) { EnumerableCalc(expr#0=[{inputs}], expr#1=[true], EXPR$0=[$t1]) EnumerableValues(tuples=[[{ 10 }, { 10 }, { 20 }, { 30 }, { 30 }, { 50 }, { 50 }, { 60 }, { null }]]) !plan +!} # Test case about NOT UNIQUE sub-query when sub-query return 0 row @@ -5396,9 +5616,10 @@ select * from emp where not unique (select deptno from dept where false); (0 rows) !ok - +!if (use_old_decorr) { EnumerableValues(tuples=[[]]) !plan +!} # Same as previous; but is Scalar sub-query select not unique (select deptno from dept where false) from dept; @@ -5413,10 +5634,11 @@ select not unique (select deptno from dept where false) from dept; (4 rows) !ok - +!if (use_old_decorr) { EnumerableCalc(expr#0..1=[{inputs}], expr#2=[false], EXPR$0=[$t2]) EnumerableValues(tuples=[[{ 10, 'Sales ' }, { 20, 'Marketing ' }, { 30, 'Engineering' }, { 40, 'Empty ' }]]) !plan +!} # [CALCITE-4758] When SOME sub-query is SqlNodeList and converted to VALUES, Calcite returns incorrect result @@ -5450,7 +5672,7 @@ select 1 in (values(null), (null)); (1 row) !ok - +!if (use_old_decorr) { EnumerableCalc(expr#0..2=[{inputs}], expr#3=[IS FALSE($t1)], expr#4=[null:BOOLEAN], expr#5=[IS NOT NULL($t2)], expr#6=[AND($t3, $t4, $t5)], expr#7=[IS NOT NULL($t1)], expr#8=[IS NOT FALSE($t1)], expr#9=[AND($t7, $t5, $t8)], expr#10=[OR($t6, $t9)], EXPR$0=[$t10]) EnumerableNestedLoopJoin(condition=[true], joinType=[left]) EnumerableValues(tuples=[[{ 0 }]]) @@ -5460,6 +5682,7 @@ EnumerableCalc(expr#0..2=[{inputs}], expr#3=[IS FALSE($t1)], expr#4=[null:BOOLEA EnumerableCalc(expr#0=[{inputs}], expr#1=[IS NOT NULL($t0)], cs=[$t1]) EnumerableValues(tuples=[[{ null }, { null }]]) !plan +!} # [CALCITE-1583] Wrong results for query with correlated subqueries with aggregate subquery expression # Correlated sub-query with aggregate expression can optimized by Metadata RowCount @@ -5480,9 +5703,10 @@ select * from emp where exists (select count(deptno) from dept where dept.deptno (9 rows) !ok +!if (use_old_decorr) { EnumerableValues(tuples=[[{ 'Jane ', 10, 'F' }, { 'Bob ', 10, 'M' }, { 'Eric ', 20, 'M' }, { 'Susan', 30, 'F' }, { 'Alice', 30, 'F' }, { 'Adam ', 50, 'M' }, { 'Eve ', 50, 'F' }, { 'Grace', 60, 'F' }, { 'Wilma', null, 'F' }]]) !plan - +!} # Same as previous; but the sub-query with always false condition. select * from emp where exists (select count(deptno) from dept where dept.deptno = emp.deptno and 1 = 2); @@ -5502,9 +5726,10 @@ select * from emp where exists (select count(deptno) from dept where dept.deptno (9 rows) !ok +!if (use_old_decorr) { EnumerableValues(tuples=[[{ 'Jane ', 10, 'F' }, { 'Bob ', 10, 'M' }, { 'Eric ', 20, 'M' }, { 'Susan', 30, 'F' }, { 'Alice', 30, 'F' }, { 'Adam ', 50, 'M' }, { 'Eve ', 50, 'F' }, { 'Grace', 60, 'F' }, { 'Wilma', null, 'F' }]]) !plan - +!} # Same as previous; but the sub-query with true correlated condition sometimes and condition is always false. select * from emp where deptno <> (select count(deptno) from dept where dept.deptno = emp.deptno); @@ -5523,6 +5748,7 @@ select * from emp where deptno <> (select count(deptno) from dept where dept.dep (8 rows) !ok +!if (use_old_decorr) { EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t4)], expr#6=[CAST($t1):BIGINT], expr#7=[0:BIGINT], expr#8=[<>($t6, $t7)], expr#9=[AND($t5, $t8)], expr#10=[<>($t6, $t4)], expr#11=[OR($t9, $t10)], proj#0..2=[{exprs}], $condition=[$t11]) EnumerableHashJoin(condition=[IS NOT DISTINCT FROM($1, $3)], joinType=[left]) EnumerableValues(tuples=[[{ 'Jane ', 10, 'F' }, { 'Bob ', 10, 'M' }, { 'Eric ', 20, 'M' }, { 'Susan', 30, 'F' }, { 'Alice', 30, 'F' }, { 'Adam ', 50, 'M' }, { 'Eve ', 50, 'F' }, { 'Grace', 60, 'F' }, { 'Wilma', null, 'F' }]]) @@ -5533,6 +5759,7 @@ EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t4)], expr#6=[CAST($t1):BI EnumerableCalc(expr#0..1=[{inputs}], expr#2=[1:BIGINT], DEPTNO=[$t0], $f1=[$t2]) EnumerableValues(tuples=[[{ 10, 'Sales ' }, { 20, 'Marketing ' }, { 30, 'Engineering' }, { 40, 'Empty ' }]]) !plan +!} # Same as previous; but the sub-query with always false correlated condition and return true sometimes. select * from emp where deptno <> (select count(deptno) + 10 from dept where dept.deptno = emp.deptno and 1 = 2); @@ -5549,6 +5776,7 @@ select * from emp where deptno <> (select count(deptno) + 10 from dept where de (6 rows) !ok +!if (use_old_decorr) { EnumerableCalc(expr#0..4=[{inputs}], expr#5=[CAST($t1):BIGINT], expr#6=[IS NULL($t4)], expr#7=[0:BIGINT], expr#8=[CASE($t6, $t7, $t4)], expr#9=[10], expr#10=[+($t8, $t9)], expr#11=[<>($t5, $t10)], proj#0..2=[{exprs}], $condition=[$t11]) EnumerableHashJoin(condition=[IS NOT DISTINCT FROM($1, $3)], joinType=[left]) EnumerableValues(tuples=[[{ 'Jane ', 10, 'F' }, { 'Bob ', 10, 'M' }, { 'Eric ', 20, 'M' }, { 'Susan', 30, 'F' }, { 'Alice', 30, 'F' }, { 'Adam ', 50, 'M' }, { 'Eve ', 50, 'F' }, { 'Grace', 60, 'F' }, { 'Wilma', null, 'F' }]]) @@ -5556,6 +5784,7 @@ EnumerableCalc(expr#0..4=[{inputs}], expr#5=[CAST($t1):BIGINT], expr#6=[IS NULL( EnumerableAggregate(group=[{0}]) EnumerableValues(tuples=[[{ 10 }, { 10 }, { 20 }, { 30 }, { 30 }, { 50 }, { 50 }, { 60 }, { null }]]) !plan +!} # [CALCITE-5421] SqlToRelConverter should populate correlateId for join with correlated query in HAVING condition !use scott @@ -7834,9 +8063,11 @@ SELECT empno WHERE e2.deptno = d.deptno GROUP BY e2.deptno HAVING SUM(e2.sal) > 1000000)); +!if (use_old_decorr) { EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0]) EnumerableTableScan(table=[[scott, EMP]]) !plan +!} +-------+ | EMPNO | +-------+ @@ -7868,10 +8099,11 @@ SELECT empno WHERE e2.deptno = e.deptno GROUP BY e2.deptno HAVING SUM(e2.sal) > 1000000)); - +!if (use_old_decorr) { EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0]) EnumerableTableScan(table=[[scott, EMP]]) !plan +!} +-------+ | EMPNO | +-------+ @@ -7906,9 +8138,11 @@ SELECT empno WHERE e2.deptno = d.deptno GROUP BY e2.deptno HAVING SUM(e2.sal) > 1000000)); +!if (use_old_decorr) { EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0]) EnumerableTableScan(table=[[scott, EMP]]) !plan +!} +-------+ | EMPNO | +-------+ @@ -7941,9 +8175,11 @@ SELECT empno GROUP BY e2.deptno HAVING SUM(e2.sal) > 1000000)); +!if (use_old_decorr) { EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0]) EnumerableTableScan(table=[[scott, EMP]]) !plan +!} +-------+ | EMPNO | +-------+ @@ -8693,6 +8929,7 @@ from emp as e; (14 rows) !ok +!if (use_old_decorr) { EnumerableCalc(expr#0..6=[{inputs}], expr#7=[IS NULL($t6)], expr#8=[0:BIGINT], expr#9=[CASE($t7, $t8, $t6)], ENAME=[$t1], C=[$t9]) EnumerableHashJoin(condition=[AND(IS NOT DISTINCT FROM($2, $4), =($3, $5))], joinType=[left]) EnumerableCalc(expr#0..7=[{inputs}], expr#8=[IS NULL($t6)], proj#0..1=[{exprs}], COMM=[$t6], $f3=[$t8]) @@ -8710,6 +8947,7 @@ EnumerableCalc(expr#0..6=[{inputs}], expr#7=[IS NULL($t6)], expr#8=[0:BIGINT], e EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], COMM=[$t6]) EnumerableTableScan(table=[[scott, EMP]]) !plan +!} # Test case for [CALCITE-6452] Scalar sub-query that uses IS NOT DISTINCT FROM returns incorrect result select e.ename, From 63b98b407137b6dae0716df50d7afc767c5151de Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Thu, 12 Feb 2026 22:47:16 +0800 Subject: [PATCH 154/562] Included cases for [CALCITE-2359] --- core/src/test/resources/sql/cast.iq | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/core/src/test/resources/sql/cast.iq b/core/src/test/resources/sql/cast.iq index de08cd380792..9278af60c2e3 100644 --- a/core/src/test/resources/sql/cast.iq +++ b/core/src/test/resources/sql/cast.iq @@ -220,8 +220,6 @@ EnumerableCalc(expr#0=[{inputs}], expr#1=['TR'], expr#2=['UE'], expr#3=[||($t1, EnumerableValues(tuples=[[{ 0 }]]) !plan -!if (fixed.calcite2539) { - # In the following, that we get an error at run time, # and that the plan shows that the expression has not been reduced. values cast('null' as boolean); @@ -234,19 +232,19 @@ EnumerableCalc(expr#0=[{inputs}], expr#1=['null'], expr#2=[CAST($t1):BOOLEAN NOT # The following throw give an error (good!) # but throw java.lang.ExceptionInInitializerError (not great). values cast('' as date); -Caused by: java.lang.NumberFormatException: For input string: "" +Caused by: java.lang.IllegalArgumentException: Invalid DATE value, '' !error values cast('' as timestamp); -Caused by: java.lang.NumberFormatException: For input string: "" +Caused by: java.lang.IllegalArgumentException: Invalid DATE value, '' !error values cast('' as integer); -Caused by: java.lang.NumberFormatException: For input string: "" +java.lang.NumberFormatException: For input string: "" !error values cast('' as boolean); -Caused by: java.lang.RuntimeException: Invalid character for cast +Caused by: org.apache.calcite.runtime.CalciteException: Invalid character for cast !error values cast('' as double); @@ -256,7 +254,7 @@ Caused by: java.lang.NumberFormatException: empty String # Postgres fails: # ERROR: invalid input syntax for integer: "1.56" values cast('15.6' as integer); -Caused by: java.lang.NumberFormatException: For input string: "15.6" +java.lang.NumberFormatException: For input string: "15.6" !error # Postgres fails: @@ -277,14 +275,13 @@ Caused by: java.lang.NumberFormatException: Value out of range. Value:"50000" Ra # Out of INTEGER range (max 2.1e9) values cast('4567891234' as integer); -Caused by: java.lang.NumberFormatException: For input string: "4567891234" +java.lang.NumberFormatException: For input string: "4567891234" !error # Out of BIGINT range (max 9.2e18) values cast('12345678901234567890' as bigint); Caused by: java.lang.NumberFormatException: For input string: "12345678901234567890" !error -!} # Out of REAL range # (Should give an error, not infinity.) From 8153f8d025cef34fc07257a08610571bd09071a9 Mon Sep 17 00:00:00 2001 From: Terran Date: Wed, 11 Feb 2026 16:03:15 +0800 Subject: [PATCH 155/562] [CALCITE-7413] Add Concat and Substring function (enabled in Mongodb library) --- .../calcite/adapter/mongodb/MongoRules.java | 2 + .../adapter/mongodb/MongoAdapterTest.java | 85 +++++++++++++++++++ 2 files changed, 87 insertions(+) diff --git a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoRules.java b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoRules.java index 47ad046c142f..48e46e610d55 100644 --- a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoRules.java +++ b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoRules.java @@ -152,6 +152,8 @@ static class RexToMongoTranslator extends RexVisitorImpl { MONGO_OPERATORS.put(SqlStdOperatorTable.LESS_THAN, "$lt"); MONGO_OPERATORS.put(SqlStdOperatorTable.LESS_THAN_OR_EQUAL, "$lte"); MONGO_OPERATORS.put(SqlStdOperatorTable.ABS, "$abs"); + MONGO_OPERATORS.put(SqlStdOperatorTable.CONCAT, "$concat"); + MONGO_OPERATORS.put(SqlStdOperatorTable.SUBSTRING, "$substrCP"); } protected RexToMongoTranslator(JavaTypeFactory typeFactory, diff --git a/mongodb/src/test/java/org/apache/calcite/adapter/mongodb/MongoAdapterTest.java b/mongodb/src/test/java/org/apache/calcite/adapter/mongodb/MongoAdapterTest.java index ce83c6e1d831..7c79718e7984 100644 --- a/mongodb/src/test/java/org/apache/calcite/adapter/mongodb/MongoAdapterTest.java +++ b/mongodb/src/test/java/org/apache/calcite/adapter/mongodb/MongoAdapterTest.java @@ -1093,4 +1093,89 @@ private static Consumer mongoChecker(final String... expected) { "POP_A=17522", "POP_A=22576"); } + + /** Test case for + * [CALCITE-7413] + * Add Concat and Substring function (enabled in Mongodb library). */ + @Test void testConcat() { + assertModel(MODEL) + .query("SELECT city || ' ' || state from zips" + + " order by pop") + .limit(3) + .queryContains( + mongoChecker( + "{$project: {EXPR$0:{$concat:[{$concat:['$city',{$literal: ' '}]},'$state']},POP:'$pop'}}", + "{$sort:{POP:1}}")) + .returnsOrdered("EXPR$0=PENTAGON DC", + "EXPR$0=BRATTLEBORO VT", + "EXPR$0=RUTLAND VT"); + } + + /** Test case for + * [CALCITE-7413] + * Add Concat and Substring function (enabled in Mongodb library). */ + @Test void testAliasNameConcat() { + assertModel(MODEL) + .query("SELECT city || ' ' || state AS full_name from zips" + + " order by pop") + .limit(3) + .queryContains( + mongoChecker( + "{$project: {FULL_NAME:{$concat:[{$concat:['$city',{$literal: ' '}]},'$state']},POP:'$pop'}}", + "{$sort:{POP:1}}")) + .returnsOrdered("FULL_NAME=PENTAGON DC", + "FULL_NAME=BRATTLEBORO VT", + "FULL_NAME=RUTLAND VT"); + } + + /** Test case for + * [CALCITE-7413] + * Add Concat and Substring function (enabled in Mongodb library). */ + @Test void testAliasNameMultipleConcat() { + assertModel(MODEL) + .query("SELECT city || ',' || ',' || state AS full_name from zips" + + " order by pop") + .limit(3) + .queryContains( + mongoChecker( + "{$project:{FULL_NAME:{$concat:[{$concat:[{$concat:['$city',{$literal:','}]},{$literal:','}]},'$state']},POP:'$pop'}}", + "{$sort:{POP:1}}")) + .returnsOrdered("FULL_NAME=PENTAGON,,DC", + "FULL_NAME=BRATTLEBORO,,VT", + "FULL_NAME=RUTLAND,,VT"); + } + + /** Test case for + * [CALCITE-7413] + * Add Concat and Substring function (enabled in Mongodb library). */ + @Test void testSubstring() { + assertModel(MODEL) + .query("SELECT SUBSTRING(city FROM 1 FOR 2) from zips" + + " order by pop") + .limit(3) + .queryContains( + mongoChecker( + "{$project:{EXPR$0:{$substrCP:['$city',{$literal:1},{$literal:2}]},POP:'$pop'}}", + "{$sort:{POP:1}}")) + .returnsOrdered("EXPR$0=EN", + "EXPR$0=RA", + "EXPR$0=UT"); + } + + /** Test case for + * [CALCITE-7413] + * Add Concat and Substring function (enabled in Mongodb library). */ + @Test void testAliasNameSubstring() { + assertModel(MODEL) + .query("SELECT SUBSTRING(city FROM 1 FOR 2) AS city_substring from zips" + + " order by pop") + .limit(3) + .queryContains( + mongoChecker( + "{$project:{CITY_SUBSTRING:{$substrCP:['$city',{$literal:1},{$literal:2}]},POP:'$pop'}}", + "{$sort:{POP:1}}")) + .returnsOrdered("CITY_SUBSTRING=EN", + "CITY_SUBSTRING=RA", + "CITY_SUBSTRING=UT"); + } } From b253e29e69000cf1038d0014108953cd8e8b5f3c Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Tue, 17 Feb 2026 20:38:53 -0800 Subject: [PATCH 156/562] [CALCITE-7347] UNKNOWN type inferred for array element type Signed-off-by: Mihai Budiu --- babel/src/test/resources/sql/spark.iq | 17 +++++++++++++++++ .../calcite/sql/type/SqlTypeFactoryImpl.java | 2 +- .../apache/calcite/test/SqlOperatorTest.java | 2 +- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/babel/src/test/resources/sql/spark.iq b/babel/src/test/resources/sql/spark.iq index 5f0927961fa2..474775474a24 100644 --- a/babel/src/test/resources/sql/spark.iq +++ b/babel/src/test/resources/sql/spark.iq @@ -29,6 +29,23 @@ # # Returns BOOLEAN +# Test case for [CALCITE-7347] https://issues.apache.org/jira/browse/CALCITE-7347 +# UNKNOWN type inferred for array element type +SELECT DISTINCT t.f1, t.f2, CAST(t.f3 AS VARCHAR ARRAY) AS f3, t.f4 +FROM (VALUES + ('a', 1, ARRAY['by'], true), + ('b', 1, ARRAY(), false) +) AS t (f1, f2, f3, f4); +EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1], F1=[$t0], F2=[$t3], F3=[$t1], F4=[$t2]) + EnumerableAggregate(group=[{0, 1, 2}]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[CAST($t1):VARCHAR NOT NULL ARRAY NOT NULL], F1=[$t0], F3=[$t3], F4=[$t2]) + EnumerableUnion(all=[true]) + EnumerableCalc(expr#0=[{inputs}], expr#1=['a'], expr#2=['by'], expr#3=[ARRAY($t2)], expr#4=[true], EXPR$0=[$t1], EXPR$2=[$t3], EXPR$3=[$t4]) + EnumerableValues(tuples=[[{ 0 }]]) + EnumerableCalc(expr#0=[{inputs}], expr#1=['b'], expr#2=[ARRAY()], expr#3=[CAST($t2):CHAR(2) NOT NULL ARRAY NOT NULL], expr#4=[false], EXPR$0=[$t1], EXPR$2=[$t3], EXPR$3=[$t4]) + EnumerableValues(tuples=[[{ 0 }]]) +!plan + select NULL RLIKE 'abc*'; EXPR$0 null diff --git a/core/src/main/java/org/apache/calcite/sql/type/SqlTypeFactoryImpl.java b/core/src/main/java/org/apache/calcite/sql/type/SqlTypeFactoryImpl.java index 3dde2b430916..115b66fa215f 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/SqlTypeFactoryImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/type/SqlTypeFactoryImpl.java @@ -327,7 +327,7 @@ private static void assertBasic(SqlTypeName typeName) { RelDataTypeFamily family = type.getFamily(); final SqlTypeName typeName = type.getSqlTypeName(); - if (typeName == SqlTypeName.NULL) { + if (typeName == SqlTypeName.NULL || typeName == SqlTypeName.UNKNOWN) { continue; } diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java index c7144c92fbdc..5fb37c581253 100644 --- a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java +++ b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java @@ -9172,7 +9172,7 @@ void checkArrayReverseFunc(SqlOperatorFixture f0, SqlFunction function, f1.checkScalar("map_concat(map(1, 2), map(1, null))", "{1=null}", "(INTEGER NOT NULL, INTEGER) MAP NOT NULL"); f1.checkScalar("map_concat(map('foo', 1), map())", "{foo=1}", - "(UNKNOWN NOT NULL, UNKNOWN NOT NULL) MAP NOT NULL"); + "(CHAR(3) NOT NULL, INTEGER NOT NULL) MAP NOT NULL"); // test operand is null map f1.checkNull("map_concat(map('foo', 1), cast(null as map))"); From 487e81d7e754f3b321a415b6110f5f723858be95 Mon Sep 17 00:00:00 2001 From: zzwqqq Date: Tue, 17 Feb 2026 22:36:06 +0800 Subject: [PATCH 157/562] [CALCITE-7415] CalciteCatalogReader.lookupOperatorOverloads keeps original function identifier casing instead of resolved schema-path casing --- .../calcite/prepare/CalciteCatalogReader.java | 66 +++++++- .../prepare/LookupOperatorOverloadsTest.java | 158 ++++++++++++++++++ 2 files changed, 215 insertions(+), 9 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/prepare/CalciteCatalogReader.java b/core/src/main/java/org/apache/calcite/prepare/CalciteCatalogReader.java index 38f6f8e77865..df5e7ade813d 100644 --- a/core/src/main/java/org/apache/calcite/prepare/CalciteCatalogReader.java +++ b/core/src/main/java/org/apache/calcite/prepare/CalciteCatalogReader.java @@ -62,6 +62,7 @@ import org.apache.calcite.sql.validate.SqlUserDefinedTableMacro; import org.apache.calcite.sql.validate.SqlValidatorUtil; import org.apache.calcite.util.Optionality; +import org.apache.calcite.util.Pair; import org.apache.calcite.util.Util; import com.google.common.collect.ImmutableList; @@ -144,9 +145,10 @@ protected CalciteCatalogReader(CalciteSchema rootSchema, return config; } - private Collection getFunctionsFrom( - List names) { - final List functions2 = + private Collection> getFunctionsFrom( + SqlIdentifier identifier) { + final List names = identifier.names; + final List> functions2 = new ArrayList<>(); final List> schemaNameList = new ArrayList<>(); if (names.size() > 1) { @@ -171,14 +173,60 @@ private Collection getFunctionsFrom( SqlValidatorUtil.getSchema(rootSchema, Iterables.concat(schemaNames, Util.skipLast(names)), nameMatcher); if (schema != null) { - final String name = Util.last(names); - boolean caseSensitive = nameMatcher.isCaseSensitive(); - functions2.addAll(schema.getFunctions(name, caseSensitive)); + final String functionName = Util.last(names); + if (nameMatcher.isCaseSensitive()) { + addFunctions(functions2, schema, names, identifier.getParserPosition(), + functionName, true); + } else { + boolean hasMatchedFunctionName = false; + for (String candidateFunctionName : schema.getFunctionNames()) { + if (nameMatcher.matches(functionName, candidateFunctionName)) { + hasMatchedFunctionName = true; + // candidateFunctionName already has canonical case from schema. + // Use case-sensitive lookup to bind each function to that exact name. + addFunctions(functions2, schema, names, identifier.getParserPosition(), + candidateFunctionName, true); + } + } + if (!hasMatchedFunctionName) { + // Fallback for schemas where getFunctionNames() is incomplete but + // getFunctions(name, false) can still resolve functions. + addFunctions(functions2, schema, names, identifier.getParserPosition(), + functionName, false); + } + } } } return functions2; } + private static SqlIdentifier createResolvedIdentifier(CalciteSchema schema, + List names, String name, SqlParserPos pos) { + final List schemaPath = schema.path(null); + // Keep the same qualifier depth as the original call (e.g. schema.func + // stays 2-part, catalog.schema.func stays 3-part). + final int qualifierCount = names.size() - 1; + // Replace only the suffix that corresponds to the resolved schema path. + // Any leading qualifiers that are outside this schema path are kept as-is. + final int resolvedQualifierCount = Math.min(qualifierCount, schemaPath.size()); + final List resolvedNames = new ArrayList<>(names.size()); + resolvedNames.addAll(names.subList(0, qualifierCount - resolvedQualifierCount)); + resolvedNames.addAll( + schemaPath.subList(schemaPath.size() - resolvedQualifierCount, schemaPath.size())); + resolvedNames.add(name); + return new SqlIdentifier(resolvedNames, pos); + } + + private static void addFunctions( + List> functions, + CalciteSchema schema, List names, SqlParserPos pos, String functionName, + boolean caseSensitive) { + final SqlIdentifier functionIdentifier = + createResolvedIdentifier(schema, names, functionName, pos); + schema.getFunctions(functionName, caseSensitive).forEach(function -> + functions.add(Pair.of(functionIdentifier, function))); + } + @Override public @Nullable RelDataType getNamedType(SqlIdentifier typeName) { CalciteSchema.TypeEntry typeEntry = SqlValidatorUtil.getTypeEntry(getRootSchema(), typeName); if (typeEntry != null) { @@ -274,10 +322,10 @@ private static SqlMonikerImpl moniker(CalciteSchema schema, @Nullable String nam !(function instanceof TableMacro || function instanceof TableFunction); } - getFunctionsFrom(opName.names) + getFunctionsFrom(opName) .stream() - .filter(predicate) - .map(function -> toOp(opName, function, config)) + .filter(pair -> predicate.test(pair.right)) + .map(pair -> toOp(pair.left, pair.right, config)) .forEachOrdered(operatorList::add); } diff --git a/core/src/test/java/org/apache/calcite/prepare/LookupOperatorOverloadsTest.java b/core/src/test/java/org/apache/calcite/prepare/LookupOperatorOverloadsTest.java index cd340315fed6..a05dbfc82633 100644 --- a/core/src/test/java/org/apache/calcite/prepare/LookupOperatorOverloadsTest.java +++ b/core/src/test/java/org/apache/calcite/prepare/LookupOperatorOverloadsTest.java @@ -17,8 +17,13 @@ package org.apache.calcite.prepare; import org.apache.calcite.adapter.java.JavaTypeFactory; +import org.apache.calcite.config.CalciteConnectionConfigImpl; +import org.apache.calcite.config.CalciteConnectionProperty; import org.apache.calcite.jdbc.CalciteConnection; import org.apache.calcite.jdbc.CalcitePrepare; +import org.apache.calcite.jdbc.CalciteSchema; +import org.apache.calcite.jdbc.JavaTypeFactoryImpl; +import org.apache.calcite.schema.Function; import org.apache.calcite.schema.SchemaPlus; import org.apache.calcite.schema.TableFunction; import org.apache.calcite.schema.impl.AbstractSchema; @@ -35,7 +40,9 @@ import org.apache.calcite.util.Smalls; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMultimap; import com.google.common.collect.Lists; +import com.google.common.collect.Multimap; import org.checkerframework.checker.nullness.qual.Nullable; import org.junit.jupiter.api.Test; @@ -45,6 +52,8 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.List; +import java.util.Locale; +import java.util.Properties; import static org.apache.calcite.sql.SqlFunctionCategory.MATCH_RECOGNIZE; import static org.apache.calcite.sql.SqlFunctionCategory.USER_DEFINED_CONSTRUCTOR; @@ -58,6 +67,7 @@ import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.hasSize; import static java.util.Objects.requireNonNull; @@ -133,6 +143,154 @@ private static void check(List actuals, checkInternal(false); } + // Look up MyCatalog.MySchema.MyFUNC using a lowercase 3-part identifier. + @Test void testLookupQualifiedNameUsesResolvedCase() { + final String catalogName = "MyCatalog"; + final String schemaName = "MySchema"; + final String funcName = "MyFUNC"; + final CalciteSchema root = + CalciteSchema.createRootSchema(false, false, catalogName); + final CalciteSchema schema = root.add(schemaName, new AbstractSchema()); + final TableFunction table = + requireNonNull(TableFunctionImpl.create(Smalls.MAZE_METHOD)); + schema.plus().add(funcName, table); + + final JavaTypeFactory typeFactory = new JavaTypeFactoryImpl(); + final Properties properties = new Properties(); + properties.setProperty(CalciteConnectionProperty.CASE_SENSITIVE.camelName(), "false"); + final CalciteCatalogReader reader = + new CalciteCatalogReader(root, ImmutableList.of(), typeFactory, + new CalciteConnectionConfigImpl(properties)); + + final List operatorList = new ArrayList<>(); + final SqlIdentifier lowercaseIdentifier = + new SqlIdentifier( + Lists.newArrayList(catalogName.toLowerCase(Locale.ROOT), + schemaName.toLowerCase(Locale.ROOT), funcName.toLowerCase(Locale.ROOT)), + null, SqlParserPos.ZERO, null); + reader.lookupOperatorOverloads(lowercaseIdentifier, + SqlFunctionCategory.USER_DEFINED_TABLE_FUNCTION, SqlSyntax.FUNCTION, + operatorList, SqlNameMatchers.withCaseSensitive(false)); + + checkFunctionType(1, funcName, operatorList); + assertThat(operatorList.get(0).getNameAsId().names, + isListOf(catalogName, schemaName, funcName)); + } + + // Look up MySchema.MyFUNC using a lowercase 2-part identifier. + @Test void testLookupPartiallyQualifiedNameUsesResolvedCase() { + final String catalogName = "MyCatalog"; + final String schemaName = "MySchema"; + final String funcName = "MyFUNC"; + final CalciteSchema root = + CalciteSchema.createRootSchema(false, false, catalogName); + final CalciteSchema schema = root.add(schemaName, new AbstractSchema()); + final TableFunction table = + requireNonNull(TableFunctionImpl.create(Smalls.MAZE_METHOD)); + schema.plus().add(funcName, table); + + final JavaTypeFactory typeFactory = new JavaTypeFactoryImpl(); + final Properties properties = new Properties(); + properties.setProperty(CalciteConnectionProperty.CASE_SENSITIVE.camelName(), "false"); + final CalciteCatalogReader reader = + new CalciteCatalogReader(root, ImmutableList.of(), typeFactory, + new CalciteConnectionConfigImpl(properties)); + + final List operatorList = new ArrayList<>(); + final SqlIdentifier lowercaseIdentifier = + new SqlIdentifier( + Lists.newArrayList(schemaName.toLowerCase(Locale.ROOT), + funcName.toLowerCase(Locale.ROOT)), + null, SqlParserPos.ZERO, null); + reader.lookupOperatorOverloads(lowercaseIdentifier, + SqlFunctionCategory.USER_DEFINED_TABLE_FUNCTION, SqlSyntax.FUNCTION, + operatorList, SqlNameMatchers.withCaseSensitive(false)); + + checkFunctionType(1, funcName, operatorList); + assertThat(operatorList.get(0).getNameAsId().names, + isListOf(schemaName, funcName)); + } + + // Look up myfunc when both MyFUNC and myfunc exist in the same schema. + @Test void testLookupCaseInsensitiveUsesEachMatchedFunctionName() { + final String schemaName = "MySchema"; + final String upperFuncName = "MyFUNC"; + final String lowerFuncName = "myfunc"; + final CalciteSchema root = CalciteSchema.createRootSchema(false, true); + final CalciteSchema schema = root.add(schemaName, new AbstractSchema()); + final TableFunction table = + requireNonNull(TableFunctionImpl.create(Smalls.MAZE_METHOD)); + schema.plus().add(upperFuncName, table); + schema.plus().add(lowerFuncName, table); + + final JavaTypeFactory typeFactory = new JavaTypeFactoryImpl(); + final Properties properties = new Properties(); + properties.setProperty(CalciteConnectionProperty.CASE_SENSITIVE.camelName(), "false"); + final CalciteCatalogReader reader = + new CalciteCatalogReader(root, ImmutableList.of(), typeFactory, + new CalciteConnectionConfigImpl(properties)); + + final List operatorList = new ArrayList<>(); + final SqlIdentifier lowercaseIdentifier = + new SqlIdentifier( + Lists.newArrayList(schemaName.toLowerCase(Locale.ROOT), + lowerFuncName), + null, SqlParserPos.ZERO, null); + reader.lookupOperatorOverloads(lowercaseIdentifier, + SqlFunctionCategory.USER_DEFINED_TABLE_FUNCTION, SqlSyntax.FUNCTION, + operatorList, SqlNameMatchers.withCaseSensitive(false)); + + assertThat(operatorList, hasSize(2)); + boolean hasUpperName = false; + boolean hasLowerName = false; + for (SqlOperator operator : operatorList) { + if (operator.getNameAsId().names.equals(Lists.newArrayList(schemaName, upperFuncName))) { + hasUpperName = true; + } + if (operator.getNameAsId().names.equals(Lists.newArrayList(schemaName, lowerFuncName))) { + hasLowerName = true; + } + } + assertThat(hasUpperName, is(true)); + assertThat(hasLowerName, is(true)); + } + + // Example: lookup "myschema.myfunc" against a dynamic schema and resolve it + // to "MySchema.MyFUNC" (fresh function instances on each lookup). + @Test void testLookupImplicitFunctionUsesResolvedCase() { + final String schemaName = "MySchema"; + final String funcName = "MyFUNC"; + final CalciteSchema root = CalciteSchema.createRootSchema(false, true); + root.add(schemaName, new AbstractSchema() { + @Override protected Multimap getFunctionMultimap() { + // Return fresh instances to mimic dynamic schemas that do not preserve + // function identity across lookups. + return ImmutableMultimap.of(funcName, + requireNonNull(TableFunctionImpl.create(Smalls.MAZE_METHOD))); + } + }); + + final JavaTypeFactory typeFactory = new JavaTypeFactoryImpl(); + final Properties properties = new Properties(); + properties.setProperty(CalciteConnectionProperty.CASE_SENSITIVE.camelName(), "false"); + final CalciteCatalogReader reader = + new CalciteCatalogReader(root, ImmutableList.of(), typeFactory, + new CalciteConnectionConfigImpl(properties)); + + final List operatorList = new ArrayList<>(); + final SqlIdentifier lowercaseIdentifier = + new SqlIdentifier( + Lists.newArrayList(schemaName.toLowerCase(Locale.ROOT), + funcName.toLowerCase(Locale.ROOT)), + null, SqlParserPos.ZERO, null); + reader.lookupOperatorOverloads(lowercaseIdentifier, + SqlFunctionCategory.USER_DEFINED_TABLE_FUNCTION, SqlSyntax.FUNCTION, + operatorList, SqlNameMatchers.withCaseSensitive(false)); + + checkFunctionType(1, funcName, operatorList); + assertThat(operatorList.get(0).getNameAsId().names, isListOf(schemaName, funcName)); + } + private void checkInternal(boolean caseSensitive) throws SQLException { final SqlNameMatcher nameMatcher = SqlNameMatchers.withCaseSensitive(caseSensitive); From 92e02758545846b04596a2a5f7e6fbfcf8fcb269 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Tue, 17 Feb 2026 17:27:33 -0800 Subject: [PATCH 158/562] [CALCITE-7368] The validator accepts CAST(INT TO BINARY), but the runtime does not implement them Signed-off-by: Mihai Budiu --- .../enumerable/RexToLixTranslator.java | 14 +++++ .../apache/calcite/runtime/SqlFunctions.java | 61 +++++++++++++++++++ .../apache/calcite/util/BuiltInMethod.java | 1 + .../apache/calcite/test/SqlOperatorTest.java | 25 ++++++++ 4 files changed, 101 insertions(+) diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java index e7b8657bf542..f88170b1b58d 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java @@ -387,6 +387,7 @@ private Expression getConvertExpression( case VARBINARY: case BINARY: + int binaryPrecision = targetType.getPrecision(); switch (sourceType.getSqlTypeName()) { case CHAR: case VARCHAR: @@ -394,6 +395,19 @@ private Expression getConvertExpression( new ConstantExpression(Charset.class, sourceType.getCharset())); case UUID: return Expressions.call(BuiltInMethod.UUID_TO_BINARY.method, operand); + case BIGINT: + case INTEGER: + case SMALLINT: + case TINYINT: + case UBIGINT: + case UINTEGER: + case USMALLINT: + case UTINYINT: + return Expressions.call( + BuiltInMethod.INT_TO_BINARY.method, + operand, + Expressions.constant(binaryPrecision), + Expressions.constant(targetType.getSqlTypeName() == SqlTypeName.BINARY)); default: return defaultExpression.get(); } diff --git a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java index f9a74cec9a58..a729d34916f8 100644 --- a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java +++ b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java @@ -89,6 +89,7 @@ import java.net.URI; import java.net.URISyntaxException; import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.nio.charset.CharacterCodingException; import java.nio.charset.Charset; import java.nio.charset.CharsetDecoder; @@ -1272,6 +1273,66 @@ public static String formatNumber(BigDecimal value, int decimalVal) { return numberFormat.format(value); } + /** Implements casts from integer to binary values. + * + * @param value Value converted; an integer or unsigned value. + * @param resultSize Size of result in bytes; negative if unspecified. + * @param fixed True if the result type is BINARY; false for VARBINARY. + * @return A ByteString containing the conversion result. + * + *

    Most SQL dialects which support this feature seem to convert integers to big endian values, + * and then truncate or pad on the left when the size of the target BINARY does not exactly + * match the integer's size. + */ + public static ByteString intToBinary(Object value, int resultSize, boolean fixed) { + ByteBuffer buffer; + if (value instanceof Byte) { + buffer = ByteBuffer.allocate(1) + .order(ByteOrder.BIG_ENDIAN) + .put((byte) value); + } else if (value instanceof Short) { + buffer = ByteBuffer.allocate(2) + .order(ByteOrder.BIG_ENDIAN) + .putShort((short) value); + } else if (value instanceof Integer) { + buffer = ByteBuffer.allocate(4) + .order(ByteOrder.BIG_ENDIAN) + .putInt((Integer) value); + } else if (value instanceof Long) { + buffer = ByteBuffer.allocate(8) + .order(ByteOrder.BIG_ENDIAN) + .putLong((Long) value); + } else if (value instanceof UByte) { + buffer = ByteBuffer.allocate(1) + .order(ByteOrder.BIG_ENDIAN) + .put(((UByte) value).byteValue()); + } else if (value instanceof UShort) { + buffer = ByteBuffer.allocate(2) + .order(ByteOrder.BIG_ENDIAN) + .putShort(((UShort) value).shortValue()); + } else if (value instanceof UInteger) { + buffer = ByteBuffer.allocate(4) + .order(ByteOrder.BIG_ENDIAN) + .putInt(((UInteger) value).intValue()); + } else if (value instanceof ULong) { + buffer = ByteBuffer.allocate(8) + .order(ByteOrder.BIG_ENDIAN) + .putLong(((ULong) value).longValue()); + } else { + throw new IllegalArgumentException("Unexpected argument type " + value); + } + ByteString result = new ByteString(buffer.array()); + if (resultSize >= 0) { + if (resultSize < result.length()) { + result = SqlFunctions.right(result, resultSize); + } else if (fixed && resultSize > result.length()) { + // pad on left + result = new ByteString(new byte[resultSize - result.length()]).concat(result); + } + } + return result; + } + public static String formatNumber(long value, String format) { DecimalFormat numberFormat = getNumberFormat(format); return numberFormat.format(value); diff --git a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java index 465a89347e9d..7ff2995282b9 100644 --- a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java +++ b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java @@ -534,6 +534,7 @@ public enum BuiltInMethod { UUID_FROM_STRING(UUID.class, "fromString", String.class), UUID_TO_STRING(SqlFunctions.class, "uuidToString", UUID.class), UUID_TO_BINARY(SqlFunctions.class, "uuidToBinary", UUID.class), + INT_TO_BINARY(SqlFunctions.class, "intToBinary", Object.class, int.class, boolean.class), BINARY_TO_UUID(SqlFunctions.class, "binaryToUuid", ByteString.class), INITCAP(SqlFunctions.class, "initcap", String.class), SUBSTRING(SqlFunctions.class, "substring", String.class, int.class, diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java index 5fb37c581253..b3de8a3c51d5 100644 --- a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java +++ b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java @@ -4056,6 +4056,31 @@ void checkIsNull(SqlOperatorFixture f, SqlOperator operator) { f.setFor(SqlStdOperatorTable.EXISTS, VM_EXPAND); } + /** Test cases for [CALCITE-7368] + * The validator accepts CAST(INT TO BINARY), but the runtime does not implement them. */ + @Test void testCastIntToBinary() { + final SqlOperatorFixture f = fixture(); + f.checkNull("cast(CAST(NULL AS INT) AS VARBINARY)"); + f.checkScalar("cast(10 as BINARY(4))", "0000000a", "BINARY(4) NOT NULL"); + f.checkScalar("cast(10 AS BINARY(2))", "000a", "BINARY(2) NOT NULL"); + f.checkScalar("cast(10 as VARBINARY(4))", "0000000a", "VARBINARY(4) NOT NULL"); + f.checkScalar("cast(10 as VARBINARY(2))", "000a", "VARBINARY(2) NOT NULL"); + f.checkScalar("cast(cast(10 AS INT UNSIGNED) AS BINARY(4))", "0000000a", "BINARY(4) NOT NULL"); + f.checkScalar("cast(-1 AS BINARY(4))", "ffffffff", "BINARY(4) NOT NULL"); + f.checkScalar("cast(-1 AS VARBINARY(8))", "ffffffff", "VARBINARY(8) NOT NULL"); + f.checkScalar("cast(10 AS VARBINARY)", "0000000a", "VARBINARY NOT NULL"); + f.checkScalar("cast(cast(10 AS TINYINT) AS VARBINARY)", "0a", "VARBINARY NOT NULL"); + f.checkScalar("cast(cast(-1 AS TINYINT) AS VARBINARY)", "ff", "VARBINARY NOT NULL"); + f.checkScalar("cast(cast(-1 AS TINYINT) AS BINARY(4))", "000000ff", "BINARY(4) NOT NULL"); + f.checkScalar("cast(cast(10 AS BIGINT) AS VARBINARY(16))", "000000000000000a", + "VARBINARY(16) NOT NULL"); + f.checkScalar("cast(cast(10 AS BIGINT) AS VARBINARY(8))", "000000000000000a", + "VARBINARY(8) NOT NULL"); + f.checkScalar("cast(cast(10 AS BIGINT) AS VARBINARY(6))", "00000000000a", + "VARBINARY(6) NOT NULL"); + f.checkScalar("cast(cast(10 AS BIGINT) AS VARBINARY(4))", "0000000a", "VARBINARY(4) NOT NULL"); + } + @Test void testNotOperator() { final SqlOperatorFixture f = fixture(); f.setFor(SqlStdOperatorTable.NOT, VmName.EXPAND); From 3c876839840f8859ad94bb57e2bc6062fc196cfb Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Mon, 16 Feb 2026 21:21:20 -0800 Subject: [PATCH 159/562] [CALCITE-7418] SqlOverlapsOperator does not reject some illegal comparisons (e.g., TIME vs DATE) Signed-off-by: Mihai Budiu --- .../calcite/sql/fun/SqlOverlapsOperator.java | 58 ++++++++++++++++- .../apache/calcite/test/SqlOperatorTest.java | 63 +++++++++++++++++++ 2 files changed, 119 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlOverlapsOperator.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlOverlapsOperator.java index 355f5c2ad5ca..550b6c65c512 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlOverlapsOperator.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlOverlapsOperator.java @@ -33,6 +33,8 @@ import com.google.common.collect.ImmutableList; +import org.checkerframework.checker.nullness.qual.Nullable; + /** * SqlOverlapsOperator represents the SQL:1999 standard {@code OVERLAPS} * function. Determines whether two anchored time intervals overlap. @@ -77,8 +79,30 @@ void arg(SqlWriter writer, SqlCall call, int leftPrec, int rightPrec, int i) { return SqlOperandCountRanges.of(2); } + /** + * Returns a template describing how the operator signature is to be built. + * + * @param operandsCount is used with functions that can take a variable + * number of operands + * @return signature template, where {0} is the operator name and {1}, {2}, etc are operands + */ + @Override public @Nullable String getSignatureTemplate(final int operandsCount) { + // This function can be called in 3 ways: + // - as a binary operator; format like a binary operator left OP right + // - as a ternary operator, for (a, b) CONTAINS c + // - as a quaternary operator, for (a, b) OVERLAPS (c, d) + if (operandsCount == 2) { + return "{1} {0} {2}"; + } else if (operandsCount == 3) { + return "({1}, {2}) {0} {3}"; + } else if (operandsCount == 4) { + return "({1}, {2}) {0} ({3}, {4})"; + } + throw new IllegalArgumentException("Unexpected operand count " + operandsCount); + } + @Override public String getAllowedSignatures(String opName) { - final String d = "DATETIME"; + final String d = "DT"; final String i = "INTERVAL"; String[] typeNames = { d, d, @@ -96,6 +120,16 @@ void arg(SqlWriter writer, SqlCall call, int leftPrec, int rightPrec, int i) { SqlUtil.getAliasedSignature(this, opName, ImmutableList.of(d, typeNames[y], d, typeNames[y + 1]))); } + if (opName.equalsIgnoreCase("contains")) { + // Two more forms supported: (DT, DT) CONTAINS DT and (DT, INTERVAL) CONTAINS DT + ret.append(NL); + ret.append(SqlUtil.getAliasedSignature(this, opName, ImmutableList.of(d, d, d))); + ret.append(NL); + ret.append(SqlUtil.getAliasedSignature(this, opName, ImmutableList.of(d, i, d))); + } + ret.append(NL); + ret.append("Where 'DT' is one of 'DATE', 'TIME', or 'TIMESTAMP', " + + "the same for all arguments."); return ret.toString(); } @@ -108,11 +142,21 @@ void arg(SqlWriter writer, SqlCall call, int leftPrec, int rightPrec, int i) { final SqlSingleOperandTypeChecker rightChecker; switch (kind) { case CONTAINS: + // A ternary call of the form (a, b) CONTAINS c + // OR a quaternary call of the form (a, b) CONTAINS (c, d) rightChecker = OperandTypes.PERIOD_OR_DATETIME; break; - default: + case OVERLAPS: + case PRECEDES: + case IMMEDIATELY_PRECEDES: + case SUCCEEDS: + case IMMEDIATELY_SUCCEEDS: + case PERIOD_EQUALS: + // Always a quaternary call of the form (a, b) OVERLAPS (c, d) rightChecker = OperandTypes.PERIOD; break; + default: + throw new IllegalArgumentException("Unexpected operation " + kind); } if (!rightChecker.checkSingleOperandType(callBinding, callBinding.operand(1), 0, throwOnFailure)) { @@ -121,6 +165,7 @@ void arg(SqlWriter writer, SqlCall call, int leftPrec, int rightPrec, int i) { final RelDataType t0 = callBinding.getOperandType(0); final RelDataType t1 = callBinding.getOperandType(1); if (!SqlTypeUtil.isDatetime(t1)) { + // "quaternary" call, of the form (a, b) OVERLAPS (c, d) final RelDataType t00 = t0.getFieldList().get(0).getType(); final RelDataType t10 = t1.getFieldList().get(0).getType(); if (!SqlTypeUtil.sameNamedType(t00, t10)) { @@ -129,6 +174,15 @@ void arg(SqlWriter writer, SqlCall call, int leftPrec, int rightPrec, int i) { } return false; } + } else { + // "ternary" call, of the form (a, b) CONTAINS c + final RelDataType t00 = t0.getFieldList().get(0).getType(); + if (!SqlTypeUtil.sameNamedType(t00, t1)) { + if (throwOnFailure) { + throw callBinding.newValidationSignatureError(); + } + return false; + } } return true; } diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java index b3de8a3c51d5..41b45d3c761a 100644 --- a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java +++ b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java @@ -3307,6 +3307,69 @@ static void checkOverlaps(OverlapChecker c) { c.isTrue("($3,$0) IMMEDIATELY SUCCEEDS ($0,$0)"); } + /** Test cases for [CALCITE-7418] + * SqlOverlapsOperator does not reject some illegal comparisons (e.g., TIME vs DATE). */ + @Test void testNegativePeriodOperators() { + final String containsError = "Supported form\\(s\\): " + + "'\\(

    ,
    \\) CONTAINS \\(
    ,
    \\)'\\n" + + "'\\(
    ,
    \\) CONTAINS \\(
    , \\)'\\n" + + "'\\(
    , \\) CONTAINS \\(
    ,
    \\)'\\n" + + "'\\(
    , \\) CONTAINS \\(
    , \\)'\\n" + + "'\\(
    ,
    \\) CONTAINS
    '\\n" + + "'\\(
    , \\) CONTAINS
    '\\n" + + "Where 'DT' is one of 'DATE', 'TIME', or 'TIMESTAMP', the same for all arguments."; + final SqlOperatorFixture f = fixture(); + f.checkFails("^(DATE '2020-10-10', DATE '2021-10-10') CONTAINS TIME '10:00:00'^", + "Cannot apply 'CONTAINS' to arguments of type " + + "' CONTAINS '\\. " + + containsError, false); + f.checkFails("^(DATE '2020-10-10', DATE '2021-10-10') CONTAINS " + + "TIMESTAMP '2010-01-01 10:00:00'^", + "Cannot apply 'CONTAINS' to arguments of type " + + "' CONTAINS '\\. " + + containsError, false); + f.checkFails("^(DATE '2020-10-10', TIMESTAMP '2021-10-10 00:00:00') " + + "CONTAINS TIMESTAMP '2010-01-01 10:00:00'^", + "Cannot apply 'CONTAINS' to arguments of type " + + "' " + + "CONTAINS '\\. " + + containsError, false); + f.checkFails("^(TIME '10:10:10', DATE '2021-10-10') CONTAINS TIME '10:00:00'^", + "Cannot apply 'CONTAINS' to arguments of type " + + "' CONTAINS '\\. " + + containsError, false); + f.checkFails("^(TIME '10:10:10', DATE '2021-10-10') CONTAINS TIMESTAMP '2010-02-02 10:00:00'^", + "Cannot apply 'CONTAINS' to arguments of type " + + "' " + + "CONTAINS '\\. " + + containsError, false); + final String overlapsError = "Supported form\\(s\\): " + + "'\\(
    ,
    \\) OVERLAPS \\(
    ,
    \\)'\\n" + + "'\\(
    ,
    \\) OVERLAPS \\(
    , \\)'\\n" + + "'\\(
    , \\) OVERLAPS \\(
    ,
    \\)'\\n" + + "'\\(
    , \\) OVERLAPS \\(
    , \\)'\\n" + + "Where 'DT' is one of 'DATE', 'TIME', or 'TIMESTAMP', the same for all arguments."; + f.checkFails("^(TIME '10:10:10', DATE '2021-10-10') OVERLAPS " + + "(TIMESTAMP '2010-02-02 10:00:00', TIME '10:00:00')^", + "Cannot apply 'OVERLAPS' to arguments of type " + + "' " + + "OVERLAPS '\\. " + + overlapsError, false); + f.checkFails("^(TIME '10:10:10', DATE '2021-10-10') " + + "OVERLAPS (TIME '10:00:00', DATE '2020-01-01')^", + "Cannot apply 'OVERLAPS' to arguments of type " + + "' " + + "OVERLAPS '\\. " + + overlapsError, false); + final String precedesError = overlapsError.replace("OVERLAPS", "PRECEDES"); + f.checkFails("^(TIME '10:10:10', DATE '2021-10-10') " + + "PRECEDES (TIME '10:00:00', TIME '10:10:10')^", + "Cannot apply 'PRECEDES' to arguments of type " + + "' " + + "PRECEDES '\\. " + + precedesError, false); + } + @Test void testLessThanOperator() { final SqlOperatorFixture f = fixture(); f.setFor(SqlStdOperatorTable.LESS_THAN, VmName.EXPAND); From 03b890c8b0398cb81c5703f7db639ee029803c5d Mon Sep 17 00:00:00 2001 From: "wenzhuang.zwz" Date: Sat, 14 Feb 2026 10:50:26 +0800 Subject: [PATCH 160/562] [CALCITE-7416] Add firedRulesCache for HepPlanner --- .../apache/calcite/plan/hep/HepPlanner.java | 64 +++++++++++++++++++ .../apache/calcite/test/HepPlannerTest.java | 29 +++++++-- 2 files changed, 88 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/plan/hep/HepPlanner.java b/core/src/main/java/org/apache/calcite/plan/hep/HepPlanner.java index e9cb6da02adc..c5f8b5610fab 100644 --- a/core/src/main/java/org/apache/calcite/plan/hep/HepPlanner.java +++ b/core/src/main/java/org/apache/calcite/plan/hep/HepPlanner.java @@ -51,7 +51,9 @@ import org.apache.calcite.util.graph.Graphs; import org.apache.calcite.util.graph.TopologicalOrderIterator; +import com.google.common.collect.HashMultimap; import com.google.common.collect.ImmutableList; +import com.google.common.collect.Multimap; import org.checkerframework.checker.nullness.qual.Nullable; @@ -66,6 +68,7 @@ import java.util.Map; import java.util.Queue; import java.util.Set; +import java.util.stream.Collectors; import static com.google.common.base.Preconditions.checkArgument; @@ -114,6 +117,30 @@ public class HepPlanner extends AbstractRelOptPlanner { private final List materializations = new ArrayList<>(); + /** + * Cache of rules that have already been fired for a specific operand match, + * to avoid firing the same rule repeatedly. + * + *

    Key: the list of matched {@link RelNode} IDs (operand match). + * + *

    Value: the set of {@link RelOptRule}s already fired for that exact ID list. + */ + private final Multimap, RelOptRule> firedRulesCache = HashMultimap.create(); + + /** + * Reverse index for {@link #firedRulesCache}, used for cleanup/GC: + * maps a single {@link RelNode} ID to all match-key ID lists that include it, + * so related cache entries can be removed efficiently when a node is discarded. + * + *

    Key: {@link RelNode} ID. + * + *

    Value: match-key ID lists in {@link #firedRulesCache} that contain the key ID. + */ + private final Multimap> firedRulesCacheIndex = HashMultimap.create(); + + + private boolean enableFiredRulesCache = false; + //~ Constructors ----------------------------------------------------------- /** @@ -173,6 +200,8 @@ public HepPlanner( removeRule(rule); } this.materializations.clear(); + this.firedRulesCache.clear(); + this.firedRulesCacheIndex.clear(); } @Override public RelNode changeTraits(RelNode rel, RelTraitSet toTraits) { @@ -195,6 +224,17 @@ public HepPlanner( return buildFinalPlan(requireNonNull(root, "'root' must not be null")); } + /** + * Enables or disables the fire-rule cache. + * + *

    If enabled, a rule will not fire twice on the same {@code RelNode::getId()}. + * + * @param enable true to enable; false is default value. + */ + public void setEnableFiredRulesCache(boolean enable) { + enableFiredRulesCache = enable; + } + /** Top-level entry point for a program. Initializes state and then invokes * the program. */ private void executeProgram(HepProgram program) { @@ -519,6 +559,14 @@ private Iterator getGraphIterator( nodeChildren, parents); + List relIds = null; + if (enableFiredRulesCache) { + relIds = call.getRelList().stream().map(RelNode::getId).collect(Collectors.toList()); + if (firedRulesCache.get(relIds).contains(rule)) { + return null; + } + } + // Allow the rule to apply its own side-conditions. if (!rule.matches(call)) { return null; @@ -526,6 +574,13 @@ private Iterator getGraphIterator( fireRule(call); + if (relIds != null) { + firedRulesCache.put(relIds, rule); + for (Integer relId : relIds) { + firedRulesCacheIndex.put(relId, relIds); + } + } + if (!call.getResults().isEmpty()) { return applyTransformationResults( vertex, @@ -982,6 +1037,15 @@ private void collectGarbage() { // Clean up metadata cache too. sweepSet.forEach(this::clearCache); + + if (enableFiredRulesCache) { + sweepSet.forEach(rel -> { + for (List relIds : firedRulesCacheIndex.get(rel.getCurrentRel().getId())) { + firedRulesCache.removeAll(relIds); + } + firedRulesCacheIndex.removeAll(rel.getCurrentRel().getId()); + }); + } } private void assertNoCycles() { diff --git a/core/src/test/java/org/apache/calcite/test/HepPlannerTest.java b/core/src/test/java/org/apache/calcite/test/HepPlannerTest.java index 262bba031392..9af5e7fe8368 100644 --- a/core/src/test/java/org/apache/calcite/test/HepPlannerTest.java +++ b/core/src/test/java/org/apache/calcite/test/HepPlannerTest.java @@ -366,11 +366,29 @@ private void assertIncludesExactlyOnce(String message, String digest, } @Test void testRuleApplyCount() { - final long applyTimes1 = checkRuleApplyCount(HepMatchOrder.ARBITRARY); - assertThat(applyTimes1, is(316L)); + long applyTimes = checkRuleApplyCount(HepMatchOrder.ARBITRARY, false); + assertThat(applyTimes, is(316L)); - final long applyTimes2 = checkRuleApplyCount(HepMatchOrder.DEPTH_FIRST); - assertThat(applyTimes2, is(87L)); + applyTimes = checkRuleApplyCount(HepMatchOrder.DEPTH_FIRST, false); + assertThat(applyTimes, is(87L)); + + applyTimes = checkRuleApplyCount(HepMatchOrder.TOP_DOWN, false); + assertThat(applyTimes, is(295L)); + + applyTimes = checkRuleApplyCount(HepMatchOrder.BOTTOM_UP, false); + assertThat(applyTimes, is(296L)); + + applyTimes = checkRuleApplyCount(HepMatchOrder.ARBITRARY, true); + assertThat(applyTimes, is(65L)); + + applyTimes = checkRuleApplyCount(HepMatchOrder.DEPTH_FIRST, true); + assertThat(applyTimes, is(65L)); + + applyTimes = checkRuleApplyCount(HepMatchOrder.TOP_DOWN, true); + assertThat(applyTimes, is(65L)); + + applyTimes = checkRuleApplyCount(HepMatchOrder.BOTTOM_UP, true); + assertThat(applyTimes, is(65L)); } @Test void testMaterialization() { @@ -387,7 +405,7 @@ private void assertIncludesExactlyOnce(String message, String digest, assertThat(planner.getMaterializations(), empty()); } - private long checkRuleApplyCount(HepMatchOrder matchOrder) { + private long checkRuleApplyCount(HepMatchOrder matchOrder, boolean enableFiredRulesCache) { final HepProgramBuilder programBuilder = HepProgram.builder(); programBuilder.addMatchOrder(matchOrder); programBuilder.addRuleInstance(CoreRules.FILTER_REDUCE_EXPRESSIONS); @@ -397,6 +415,7 @@ private long checkRuleApplyCount(HepMatchOrder matchOrder) { HepPlanner planner = new HepPlanner(programBuilder.build()); planner.addListener(listener); planner.setRoot(sql(COMPLEX_UNION_TREE).toRel()); + planner.setEnableFiredRulesCache(enableFiredRulesCache); planner.findBestExp(); return listener.getApplyTimes(); } From 8e8f3dbe190179067f2ed9f30294642be202fc4a Mon Sep 17 00:00:00 2001 From: iwanttobepowerful <745778074@qq.com> Date: Wed, 28 Jan 2026 22:29:15 +0800 Subject: [PATCH 161/562] [CALCITE-5390] RelDecorrelator throws NullPointerException --- .../calcite/sql2rel/RelDecorrelator.java | 46 +++- .../sql2rel/TopDownGeneralDecorrelator.java | 3 - .../calcite/sql2rel/RelDecorrelatorTest.java | 89 ++++++++ core/src/test/resources/sql/sub-query.iq | 75 +++++++ .../apache/calcite/adapter/tpch/TpchTest.java | 212 ++++++++++++++++++ 5 files changed, 418 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java b/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java index 47c0040fc3e5..45596c1d7d28 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java @@ -918,7 +918,7 @@ protected RexNode removeCorrelationExpr( * CASE WHEN cnt0 IS NOT NULL THEN cnt0 ELSE 0 END AS cnt * FROM (SELECT deptno FROM dept GROUP BY deptno) d2 * LEFT JOIN ( - * SELECT deptno, COUNT(e.empno) cnt0 + * SELECT deptno, COUNT(emp.empno) cnt0 * FROM emp * WHERE deptno IS NOT NULL * GROUP BY deptno) e @@ -1421,7 +1421,8 @@ private static void shiftMapping(Map mapping, int startIndex, for (CorRef corVar : correlations) { final int oldCorVarOffset = corVar.field; - final RelNode oldInput = requireNonNull(getCorRel(corVar)); + final RelNode oldInput = findInputRel(corVar); + final Frame frame = requireNonNull(getOrCreateFrame(oldInput)); final RelNode newInput = frame.r; @@ -1453,7 +1454,7 @@ private static void shiftMapping(Map mapping, int startIndex, RelNode r = null; for (CorRef corVar : correlations) { - final RelNode oldInput = requireNonNull(getCorRel(corVar)); + final RelNode oldInput = findInputRel(corVar); final RelNode newInput = requireNonNull(getOrCreateFrame(oldInput).r); if (!joinedInputs.contains(newInput)) { @@ -1487,7 +1488,7 @@ private static void shiftMapping(Map mapping, int startIndex, for (CorRef corRef : correlations) { // The first input of a Correlate is always the rel defining // the correlated variables. - final RelNode oldInput = requireNonNull(getCorRel(corRef)); + final RelNode oldInput = findInputRel(corRef); final Frame frame = getOrCreateFrame(oldInput); final RelNode newInput = requireNonNull(frame.r); @@ -1533,6 +1534,39 @@ private RelNode getCorRel(CorRef corVar) { () -> "r.getInput(0) is null for " + r); } + /** + * Finds the RelNode that produces the given correlation variable. + * + *

    This method resolves correlation variables by inspecting the {@link #frameStack}, + * which maintains the active correlation contexts during the top-down traversal. + * + *

    The lookup logic implements Lexical Scoping (with Shadowing): + *

      + *
    • The {@code frameStack} is traversed from top to bottom (most recently pushed to + * least recently pushed). This ensures that if multiple nested queries use the same + * {@link CorrelationId}, the innermost definition takes precedence, shadowing outer ones. + *
      • + *
    + * + *

    If the variable is not found in the {@code frameStack} (e.g., it might be defined outside + * the current traversal path or in a global context), the method falls back to looking it up + * in the global {@link #cm} (CorelMap). + * + * @param corVar The correlation variable reference to resolve. + * @return The {@link RelNode} that produces the correlation variable. + */ + private RelNode findInputRel(CorRef corVar) { + final int oldCorVarOffset = corVar.field; + for (Pair pair : frameStack) { + if (pair.left.equals(corVar.corr)) { + if (oldCorVarOffset < pair.right.oldRel.getRowType().getFieldCount()) { + return pair.right.oldRel; + } + } + } + return getCorRel(corVar); + } + /** Adds a value generator to satisfy the correlating variables used by * a relational expression, if those variables are not already provided by * its input. */ @@ -3766,12 +3800,16 @@ private RexVisitorImpl rexVisitor(final RelNode rel) { * and where to find the output fields and correlation variables * among its output fields. */ static class Frame { + // The original relational expression before decorrelation + final RelNode oldRel; + // The decorrelated relational expression final RelNode r; final ImmutableSortedMap corDefOutputs; final ImmutableSortedMap oldToNewOutputs; Frame(RelNode oldRel, RelNode r, NavigableMap corDefOutputs, Map oldToNewOutputs) { + this.oldRel = requireNonNull(oldRel, "oldRel"); this.r = requireNonNull(r, "r"); this.corDefOutputs = ImmutableSortedMap.copyOf(corDefOutputs); this.oldToNewOutputs = ImmutableSortedMap.copyOf(oldToNewOutputs); diff --git a/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java b/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java index 82465799460f..c1f8494ebdce 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java @@ -973,8 +973,6 @@ public TopDownGeneralDecorrelator getVisitor() { * Unnesting information. */ static class UnnestedQuery extends Frame { - final RelNode oldRel; - /** * Creates a UnnestedQuery. * @@ -986,7 +984,6 @@ static class UnnestedQuery extends Frame { UnnestedQuery(RelNode oldRel, RelNode r, NavigableMap corDefOutputs, Map oldToNewOutputs) { super(oldRel, r, corDefOutputs, oldToNewOutputs); - this.oldRel = oldRel; } /** diff --git a/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java b/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java index 1fabc39d8cc7..2b406c3c0154 100644 --- a/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java +++ b/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java @@ -1669,6 +1669,95 @@ public static Frameworks.ConfigBuilder config() { assertThat(after, hasTree(planAfter)); } + /** Test case for [CALCITE-5390] + * RelDecorrelator throws NullPointerException. */ + @Test void testCorrelationLexicalScoping() { + final FrameworkConfig frameworkConfig = config().build(); + final RelBuilder builder = RelBuilder.create(frameworkConfig); + final RelOptCluster cluster = builder.getCluster(); + final Planner planner = Frameworks.getPlanner(frameworkConfig); + final String sql = "" + + "select deptno,\n" + + " (select min(1) from emp where empno > d.deptno) as i0,\n" + + " (select min(0) from emp where deptno = d.deptno and " + + "ename = 'SMITH' and d.deptno > 0) as i1\n" + + "from dept as d"; + final RelNode originalRel; + try { + final SqlNode parse = planner.parse(sql); + final SqlNode validate = planner.validate(parse); + originalRel = planner.rel(validate).rel; + } catch (Exception e) { + throw TestUtil.rethrow(e); + } + + final HepProgram hepProgram = HepProgram.builder() + .addRuleCollection( + ImmutableList.of( + // SubQuery program rules + CoreRules.FILTER_SUB_QUERY_TO_CORRELATE, + CoreRules.PROJECT_SUB_QUERY_TO_CORRELATE, + CoreRules.JOIN_SUB_QUERY_TO_CORRELATE)) + .build(); + final Program program = + Programs.of(hepProgram, true, + requireNonNull(cluster.getMetadataProvider())); + final RelNode before = + program.run(cluster.getPlanner(), originalRel, cluster.traitSet(), + Collections.emptyList(), Collections.emptyList()); + final String planBefore = "" + + "LogicalProject(DEPTNO=[$0], I0=[$3], I1=[$4])\n" + + " LogicalCorrelate(correlation=[$cor0], joinType=[left], requiredColumns=[{0}])\n" + + " LogicalCorrelate(correlation=[$cor0], joinType=[left], requiredColumns=[{0}])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n" + + " LogicalAggregate(group=[{}], EXPR$0=[MIN($0)])\n" + + " LogicalProject($f0=[1])\n" + + " LogicalFilter(condition=[>($0, CAST($cor0.DEPTNO):SMALLINT NOT NULL)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalAggregate(group=[{}], EXPR$0=[MIN($0)])\n" + + " LogicalProject($f0=[0])\n" + + " LogicalFilter(condition=[AND(=($7, $cor0.DEPTNO), =($1, 'SMITH'), >(CAST($cor0.DEPTNO):INTEGER NOT NULL, 0))])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + assertThat(before, hasTree(planBefore)); + + // Decorrelate without any rules, just "purely" decorrelation algorithm on RelDecorrelator + final RelNode after = + RelDecorrelator.decorrelateQuery(before, builder, RuleSets.ofList(Collections.emptyList()), + RuleSets.ofList(Collections.emptyList())); + final String planAfter = "" + + "LogicalProject(DEPTNO=[$0], I0=[$3], I1=[$8])\n" + + " LogicalJoin(condition=[AND(=($0, $6), =($5, $7))], joinType=[left])\n" + + " LogicalProject(DEPTNO=[$0], DNAME=[$1], LOC=[$2], EXPR$0=[$5], DEPTNO0=[$0], $f5=[>(CAST($0):INTEGER NOT NULL, 0)])\n" + + " LogicalJoin(condition=[=($3, $4)], joinType=[left])\n" + + " LogicalProject(DEPTNO=[$0], DNAME=[$1], LOC=[$2], DEPTNO0=[CAST($0):SMALLINT NOT NULL])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n" + + " LogicalAggregate(group=[{0}], EXPR$0=[MIN($1)])\n" + + " LogicalProject(DEPTNO0=[$8], $f0=[1])\n" + + " LogicalJoin(condition=[>($0, $8)], joinType=[inner])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalAggregate(group=[{0}])\n" + + " LogicalProject(DEPTNO0=[CAST($0):SMALLINT NOT NULL])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n" + + " LogicalAggregate(group=[{0, 1}], EXPR$0=[MIN($2)])\n" + + " LogicalProject(DEPTNO0=[$8], $f5=[$9], $f0=[0])\n" + + " LogicalJoin(condition=[=($7, $8)], joinType=[inner])\n" + + " LogicalFilter(condition=[=($1, 'SMITH')])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalFilter(condition=[$1])\n" + + " LogicalProject(DEPTNO=[$0], $f5=[>(CAST($0):INTEGER NOT NULL, 0)])\n" + + " LogicalJoin(condition=[=($3, $4)], joinType=[left])\n" + + " LogicalProject(DEPTNO=[$0], DNAME=[$1], LOC=[$2], DEPTNO0=[CAST($0):SMALLINT NOT NULL])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n" + + " LogicalAggregate(group=[{0}], EXPR$0=[MIN($1)])\n" + + " LogicalProject(DEPTNO0=[$8], $f0=[1])\n" + + " LogicalJoin(condition=[>($0, $8)], joinType=[inner])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalAggregate(group=[{0}])\n" + + " LogicalProject(DEPTNO0=[CAST($0):SMALLINT NOT NULL])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n"; + assertThat(after, hasTree(planAfter)); + } + /** Test case for [CALCITE-7320] * AggregateProjectMergeRule throws AssertionError when Project maps multiple grouping keys * to the same field. */ diff --git a/core/src/test/resources/sql/sub-query.iq b/core/src/test/resources/sql/sub-query.iq index 5fb06553726a..2b6e459cdacd 100644 --- a/core/src/test/resources/sql/sub-query.iq +++ b/core/src/test/resources/sql/sub-query.iq @@ -8967,4 +8967,79 @@ where e.deptno = 10; !ok +# [CALCITE-5390] RelDecorrelator throws NullPointerException +# Verified against PostgreSQL. +select deptno, + (select min(1) from emp where empno > d.deptno) as i0, + (select min(0) from emp where deptno = d.deptno and ename = 'SMITH' and d.deptno > 0) as i1 +from dept as d; ++--------+----+----+ +| DEPTNO | I0 | I1 | ++--------+----+----+ +| 10 | 1 | | +| 20 | 1 | 0 | +| 30 | 1 | | +| 40 | 1 | | ++--------+----+----+ +(4 rows) + +!ok + +# [CALCITE-5390] RelDecorrelator throws NullPointerException +# Verified against PostgreSQL. +SELECT + (SELECT 1 FROM emp d WHERE d.job = a.job LIMIT 1) AS t1, + (SELECT a.job = 'PRESIDENT' FROM emp s LIMIT 1) as t2 +FROM emp a; ++----+-------+ +| T1 | T2 | ++----+-------+ +| 1 | false | +| 1 | false | +| 1 | false | +| 1 | false | +| 1 | false | +| 1 | false | +| 1 | false | +| 1 | false | +| 1 | false | +| 1 | false | +| 1 | false | +| 1 | false | +| 1 | false | +| 1 | true | ++----+-------+ +(14 rows) + +!ok + +# [CALCITE-5390] RelDecorrelator throws NullPointerException +# Verified against PostgreSQL. +SELECT * +FROM emp e +WHERE e.ename NOT IN ( + SELECT d.dname + FROM dept d + WHERE e.deptno = d.deptno OR e.sal > 2000.0); ++-------+--------+-----------+------+------------+---------+---------+--------+ +| EMPNO | ENAME | JOB | MGR | HIREDATE | SAL | COMM | DEPTNO | ++-------+--------+-----------+------+------------+---------+---------+--------+ +| 7369 | SMITH | CLERK | 7902 | 1980-12-17 | 800.00 | | 20 | +| 7499 | ALLEN | SALESMAN | 7698 | 1981-02-20 | 1600.00 | 300.00 | 30 | +| 7521 | WARD | SALESMAN | 7698 | 1981-02-22 | 1250.00 | 500.00 | 30 | +| 7566 | JONES | MANAGER | 7839 | 1981-02-04 | 2975.00 | | 20 | +| 7654 | MARTIN | SALESMAN | 7698 | 1981-09-28 | 1250.00 | 1400.00 | 30 | +| 7698 | BLAKE | MANAGER | 7839 | 1981-01-05 | 2850.00 | | 30 | +| 7782 | CLARK | MANAGER | 7839 | 1981-06-09 | 2450.00 | | 10 | +| 7788 | SCOTT | ANALYST | 7566 | 1987-04-19 | 3000.00 | | 20 | +| 7839 | KING | PRESIDENT | | 1981-11-17 | 5000.00 | | 10 | +| 7844 | TURNER | SALESMAN | 7698 | 1981-09-08 | 1500.00 | 0.00 | 30 | +| 7876 | ADAMS | CLERK | 7788 | 1987-05-23 | 1100.00 | | 20 | +| 7900 | JAMES | CLERK | 7698 | 1981-12-03 | 950.00 | | 30 | +| 7902 | FORD | ANALYST | 7566 | 1981-12-03 | 3000.00 | | 20 | +| 7934 | MILLER | CLERK | 7782 | 1982-01-23 | 1300.00 | | 10 | ++-------+--------+-----------+------+------------+---------+---------+--------+ +(14 rows) + +!ok # End sub-query.iq diff --git a/plus/src/test/java/org/apache/calcite/adapter/tpch/TpchTest.java b/plus/src/test/java/org/apache/calcite/adapter/tpch/TpchTest.java index 5f4e93d9d7a1..39aa2f3f16e0 100644 --- a/plus/src/test/java/org/apache/calcite/adapter/tpch/TpchTest.java +++ b/plus/src/test/java/org/apache/calcite/adapter/tpch/TpchTest.java @@ -16,6 +16,7 @@ */ package org.apache.calcite.adapter.tpch; +import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.plan.RelOptPlanner; import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.plan.hep.HepPlanner; @@ -27,11 +28,16 @@ import org.apache.calcite.schema.SchemaPlus; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.parser.SqlParseException; +import org.apache.calcite.sql2rel.RelDecorrelator; import org.apache.calcite.test.CalciteAssert; import org.apache.calcite.tools.FrameworkConfig; import org.apache.calcite.tools.Frameworks; import org.apache.calcite.tools.Planner; +import org.apache.calcite.tools.Program; +import org.apache.calcite.tools.Programs; +import org.apache.calcite.tools.RelBuilder; import org.apache.calcite.tools.RelConversionException; +import org.apache.calcite.tools.RuleSets; import org.apache.calcite.tools.ValidationException; import org.apache.calcite.util.TestUtil; @@ -41,15 +47,19 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; +import java.util.Collections; import java.util.List; import java.util.concurrent.TimeUnit; import static org.apache.calcite.test.Matchers.containsStringLinux; +import static org.apache.calcite.test.Matchers.hasTree; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.not; import static org.hamcrest.MatcherAssert.assertThat; +import static java.util.Objects.requireNonNull; + /** Unit test for {@link org.apache.calcite.adapter.tpch.TpchSchema}. * *

    Because the TPC-H data generator takes time and memory to instantiate, @@ -1001,6 +1011,208 @@ private CalciteAssert.AssertThat with() { checkQuery(22); } + /** Test case for [CALCITE-5390] + * RelDecorrelator throws NullPointerException. */ + @Test public void test5390() + throws SqlParseException, ValidationException, RelConversionException { + SchemaPlus rootSchema = Frameworks.createRootSchema(true); + TpchSchema tpchSchema = new TpchSchema(1.0, 0, 1, false); + rootSchema.add("TPCH", tpchSchema); + FrameworkConfig config = Frameworks.newConfigBuilder() + .defaultSchema(rootSchema) + .build(); + final RelBuilder builder = RelBuilder.create(config); + final RelOptCluster cluster = builder.getCluster(); + + Planner planner = Frameworks.getPlanner(config); + + String sql = "select\n" + + " (select count(*) from tpch.part where p_partkey = tpch.partsupp.ps_partkey),\n" + + " (select count(*) from tpch.supplier\n" + + " where s_suppkey = case when s_acctbal > 0\n" + + " then tpch.partsupp.ps_partkey + 1\n" + + " else 1234 end)\n" + + "from tpch.partsupp"; + + SqlNode parsed = planner.parse(sql); + SqlNode validated = planner.validate(parsed); + RelRoot root = planner.rel(validated); + final RelNode originalRel = root.rel; + + final HepProgram hepProgram = HepProgram.builder() + .addRuleCollection( + ImmutableList.of( + // SubQuery program rules + CoreRules.FILTER_SUB_QUERY_TO_CORRELATE, + CoreRules.PROJECT_SUB_QUERY_TO_CORRELATE, + CoreRules.JOIN_SUB_QUERY_TO_CORRELATE)) + .build(); + final Program program = + Programs.of(hepProgram, true, + requireNonNull(cluster.getMetadataProvider())); + final RelNode before = + program.run(cluster.getPlanner(), originalRel, cluster.traitSet(), + Collections.emptyList(), Collections.emptyList()); + final String planBefore = "" + + "LogicalProject(EXPR$0=[$5], EXPR$1=[$6])\n" + + " LogicalCorrelate(correlation=[$cor0], joinType=[left], requiredColumns=[{0}])\n" + + " LogicalCorrelate(correlation=[$cor0], joinType=[left], requiredColumns=[{0}])\n" + + " LogicalTableScan(table=[[TPCH, PARTSUPP]])\n" + + " LogicalAggregate(group=[{}], EXPR$0=[COUNT()])\n" + + " LogicalFilter(condition=[=($0, $cor0.PS_PARTKEY)])\n" + + " LogicalTableScan(table=[[TPCH, PART]])\n" + + " LogicalAggregate(group=[{}], EXPR$0=[COUNT()])\n" + + " LogicalFilter(condition=[=(CAST($0):BIGINT, CASE(>(CAST($5):DOUBLE, CAST(0):DOUBLE NOT NULL), +($cor0.PS_PARTKEY, 1), 1234:BIGINT))])\n" + + " LogicalTableScan(table=[[TPCH, SUPPLIER]])\n"; + assertThat(before, hasTree(planBefore)); + + // Decorrelate without any rules, just "purely" decorrelation algorithm on RelDecorrelator + final RelNode after = + RelDecorrelator.decorrelateQuery(before, builder, + RuleSets.ofList(Collections.emptyList()), + RuleSets.ofList(Collections.emptyList())); + final String planAfter = "" + + "LogicalProject(EXPR$0=[$5], EXPR$1=[$8])\n" + + " LogicalJoin(condition=[IS NOT DISTINCT FROM($6, $7)], joinType=[left])\n" + + " LogicalProject(PS_PARTKEY=[$0], PS_SUPPKEY=[$1], PS_AVAILQTY=[$2], PS_SUPPLYCOST=[$3], PS_COMMENT=[$4], EXPR$0=[$6], $f6=[+($0, 1)])\n" + + " LogicalJoin(condition=[IS NOT DISTINCT FROM($0, $5)], joinType=[left])\n" + + " LogicalTableScan(table=[[TPCH, PARTSUPP]])\n" + + " LogicalProject(P_PARTKEY=[$0], EXPR$0=[CASE(IS NOT NULL($2), $2, 0)])\n" + + " LogicalJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left])\n" + + " LogicalAggregate(group=[{0}])\n" + + " LogicalProject(PS_PARTKEY=[$0])\n" + + " LogicalTableScan(table=[[TPCH, PARTSUPP]])\n" + + " LogicalAggregate(group=[{0}], EXPR$0=[COUNT()])\n" + + " LogicalProject(P_PARTKEY=[$0])\n" + + " LogicalFilter(condition=[IS NOT NULL($0)])\n" + + " LogicalTableScan(table=[[TPCH, PART]])\n" + + " LogicalProject($f6=[$0], EXPR$0=[CASE(IS NOT NULL($2), $2, 0)])\n" + + " LogicalJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left])\n" + + " LogicalAggregate(group=[{0}])\n" + + " LogicalProject($f6=[+($0, 1)])\n" + + " LogicalJoin(condition=[IS NOT DISTINCT FROM($0, $5)], joinType=[left])\n" + + " LogicalTableScan(table=[[TPCH, PARTSUPP]])\n" + + " LogicalProject(P_PARTKEY=[$0], EXPR$0=[CASE(IS NOT NULL($2), $2, 0)])\n" + + " LogicalJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left])\n" + + " LogicalAggregate(group=[{0}])\n" + + " LogicalProject(PS_PARTKEY=[$0])\n" + + " LogicalTableScan(table=[[TPCH, PARTSUPP]])\n" + + " LogicalAggregate(group=[{0}], EXPR$0=[COUNT()])\n" + + " LogicalProject(P_PARTKEY=[$0])\n" + + " LogicalFilter(condition=[IS NOT NULL($0)])\n" + + " LogicalTableScan(table=[[TPCH, PART]])\n" + + " LogicalAggregate(group=[{0}], EXPR$0=[COUNT()])\n" + + " LogicalProject($f6=[$7])\n" + + " LogicalJoin(condition=[=(CAST($0):BIGINT, CASE(>(CAST($5):DOUBLE, 0.0E0), $7, 1234:BIGINT))], joinType=[inner])\n" + + " LogicalTableScan(table=[[TPCH, SUPPLIER]])\n" + + " LogicalAggregate(group=[{0}])\n" + + " LogicalProject($f6=[+($0, 1)])\n" + + " LogicalJoin(condition=[IS NOT DISTINCT FROM($0, $5)], joinType=[left])\n" + + " LogicalTableScan(table=[[TPCH, PARTSUPP]])\n" + + " LogicalProject(P_PARTKEY=[$0], EXPR$0=[CASE(IS NOT NULL($2), $2, 0)])\n" + + " LogicalJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left])\n" + + " LogicalAggregate(group=[{0}])\n" + + " LogicalProject(PS_PARTKEY=[$0])\n" + + " LogicalTableScan(table=[[TPCH, PARTSUPP]])\n" + + " LogicalAggregate(group=[{0}], EXPR$0=[COUNT()])\n" + + " LogicalProject(P_PARTKEY=[$0])\n" + + " LogicalFilter(condition=[IS NOT NULL($0)])\n" + + " LogicalTableScan(table=[[TPCH, PART]])\n"; + assertThat(after, hasTree(planAfter)); + } + + @Test public void test53902() + throws SqlParseException, ValidationException, RelConversionException { + SchemaPlus rootSchema = Frameworks.createRootSchema(true); + TpchSchema tpchSchema = new TpchSchema(1.0, 0, 1, false); + rootSchema.add("TPCH", tpchSchema); + FrameworkConfig config = Frameworks.newConfigBuilder() + .defaultSchema(rootSchema) + .build(); + final RelBuilder builder = RelBuilder.create(config); + final RelOptCluster cluster = builder.getCluster(); + + Planner planner = Frameworks.getPlanner(config); + + String sql = "" + + "SELECT *\n" + + "FROM tpch.customer\n" + + "WHERE c_mktsegment = 'AUTOMOBILE'\n" + + " AND (SELECT COUNT(*)\n" + + " FROM tpch.orders\n" + + " WHERE o_custkey = c_custkey\n" + + " AND (SELECT SUM(l_extendedprice)\n" + + " FROM tpch.lineitem\n" + + " WHERE l_orderkey = o_orderkey\n" + + " ) > 300000\n" + + " ) > 5"; + + SqlNode parsed = planner.parse(sql); + SqlNode validated = planner.validate(parsed); + RelRoot root = planner.rel(validated); + final RelNode originalRel = root.rel; + + final HepProgram hepProgram = HepProgram.builder() + .addRuleCollection( + ImmutableList.of( + // SubQuery program rules + CoreRules.FILTER_SUB_QUERY_TO_CORRELATE, + CoreRules.PROJECT_SUB_QUERY_TO_CORRELATE, + CoreRules.JOIN_SUB_QUERY_TO_CORRELATE)) + .build(); + final Program program = + Programs.of(hepProgram, true, + requireNonNull(cluster.getMetadataProvider())); + final RelNode before = + program.run(cluster.getPlanner(), originalRel, cluster.traitSet(), + Collections.emptyList(), Collections.emptyList()); + final String planBefore = "" + + "LogicalProject(C_CUSTKEY=[$0], C_NAME=[$1], C_ADDRESS=[$2], C_NATIONKEY=[$3], C_PHONE=[$4], C_ACCTBAL=[$5], C_MKTSEGMENT=[$6], C_COMMENT=[$7])\n" + + " LogicalProject(C_CUSTKEY=[$0], C_NAME=[$1], C_ADDRESS=[$2], C_NATIONKEY=[$3], C_PHONE=[$4], C_ACCTBAL=[$5], C_MKTSEGMENT=[$6], C_COMMENT=[$7])\n" + + " LogicalFilter(condition=[AND(=(CAST($6):VARCHAR, 'AUTOMOBILE'), >($8, 5))])\n" + + " LogicalCorrelate(correlation=[$cor0], joinType=[left], requiredColumns=[{0}])\n" + + " LogicalTableScan(table=[[TPCH, CUSTOMER]])\n" + + " LogicalAggregate(group=[{}], EXPR$0=[COUNT()])\n" + + " LogicalProject(O_ORDERKEY=[$0], O_CUSTKEY=[$1], O_ORDERSTATUS=[$2], O_TOTALPRICE=[$3], O_ORDERDATE=[$4], O_ORDERPRIORITY=[$5], O_CLERK=[$6], O_SHIPPRIORITY=[$7], O_COMMENT=[$8])\n" + + " LogicalFilter(condition=[AND(=($1, $cor0.C_CUSTKEY), >(CAST($9):DOUBLE, 300000.0E0))])\n" + + " LogicalCorrelate(correlation=[$cor1], joinType=[left], requiredColumns=[{0}])\n" + + " LogicalTableScan(table=[[TPCH, ORDERS]])\n" + + " LogicalAggregate(group=[{}], EXPR$0=[SUM($0)])\n" + + " LogicalProject(L_EXTENDEDPRICE=[$5])\n" + + " LogicalFilter(condition=[=($0, $cor1.O_ORDERKEY)])\n" + + " LogicalTableScan(table=[[TPCH, LINEITEM]])\n"; + assertThat(before, hasTree(planBefore)); + + // Decorrelate without any rules, just "purely" decorrelation algorithm on RelDecorrelator + final RelNode after = + RelDecorrelator.decorrelateQuery(before, builder, + RuleSets.ofList(Collections.emptyList()), + RuleSets.ofList(Collections.emptyList())); + final String planAfter = "" + + "LogicalProject(C_CUSTKEY=[$0], C_NAME=[$1], C_ADDRESS=[$2], C_NATIONKEY=[$3], C_PHONE=[$4], C_ACCTBAL=[$5], C_MKTSEGMENT=[$6], C_COMMENT=[$7])\n" + + " LogicalProject(C_CUSTKEY=[$0], C_NAME=[$1], C_ADDRESS=[$2], C_NATIONKEY=[$3], C_PHONE=[$4], C_ACCTBAL=[$5], C_MKTSEGMENT=[$6], C_COMMENT=[$7], O_CUSTKEY9=[$8], EXPR$0=[CAST($9):BIGINT])\n" + + " LogicalJoin(condition=[IS NOT DISTINCT FROM($0, $8)], joinType=[inner])\n" + + " LogicalFilter(condition=[=(CAST($6):VARCHAR, 'AUTOMOBILE')])\n" + + " LogicalTableScan(table=[[TPCH, CUSTOMER]])\n" + + " LogicalFilter(condition=[>($1, 5)])\n" + + " LogicalProject(O_CUSTKEY9=[$0], EXPR$0=[CASE(IS NOT NULL($2), $2, 0)])\n" + + " LogicalJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left])\n" + + " LogicalAggregate(group=[{0}])\n" + + " LogicalProject(C_CUSTKEY=[$0])\n" + + " LogicalTableScan(table=[[TPCH, CUSTOMER]])\n" + + " LogicalAggregate(group=[{0}], EXPR$0=[COUNT()])\n" + + " LogicalProject(O_CUSTKEY9=[$1])\n" + + " LogicalJoin(condition=[=($0, $9)], joinType=[inner])\n" + + " LogicalFilter(condition=[IS NOT NULL($1)])\n" + + " LogicalTableScan(table=[[TPCH, ORDERS]])\n" + + " LogicalFilter(condition=[>(CAST($1):DOUBLE, 300000.0E0)])\n" + + " LogicalAggregate(group=[{0}], EXPR$0=[SUM($1)])\n" + + " LogicalProject(L_ORDERKEY=[$0], L_EXTENDEDPRICE=[$5])\n" + + " LogicalFilter(condition=[IS NOT NULL($0)])\n" + + " LogicalTableScan(table=[[TPCH, LINEITEM]])\n"; + assertThat(after, hasTree(planAfter)); + } + private void checkQuery(int i) { query(i).runs(); } From c403ed4ecdcad4651aab1fdb068f577a58dd4459 Mon Sep 17 00:00:00 2001 From: "wenzhuang.zwz" Date: Thu, 22 Jan 2026 18:40:11 +0800 Subject: [PATCH 162/562] [CALCITE-7393] Support RelDataTypeDigest Use structured innerDigest instead of string digest for composite/UDT types to reduce memory and improve hashCode/equals latency. Controlled by `calcite.disable.generate.rel.data.type.digest.string` (default: false). Legacy string digest is still used in hashCode/equals if explicitly set, ensuring backward compatibility. TestCase: TypeDigestBenchmark --- .../calcite/config/CalciteSystemProperty.java | 8 + .../apache/calcite/jdbc/JavaRecordType.java | 12 ++ .../apache/calcite/rel/HasDigestString.java | 24 +++ .../apache/calcite/rel/type/RelDataType.java | 22 +++ .../calcite/rel/type/RelDataTypeDigest.java | 26 ++++ .../calcite/rel/type/RelDataTypeImpl.java | 138 +++++++++++++++--- .../calcite/rel/type/RelRecordType.java | 34 +++++ .../type/SingleColumnAliasRelDataType.java | 12 ++ .../apache/calcite/sql/type/ArraySqlType.java | 19 +++ .../apache/calcite/sql/type/MapSqlType.java | 21 +++ .../calcite/sql/type/MultisetSqlType.java | 19 +++ .../apache/calcite/rex/RexBuilderTest.java | 1 + site/_docs/history.md | 4 + .../benchmarks/TypeDigestBenchmark.java | 126 ++++++++++++++++ 14 files changed, 448 insertions(+), 18 deletions(-) create mode 100644 core/src/main/java/org/apache/calcite/rel/HasDigestString.java create mode 100644 core/src/main/java/org/apache/calcite/rel/type/RelDataTypeDigest.java create mode 100644 ubenchmark/src/jmh/java/org/apache/calcite/benchmarks/TypeDigestBenchmark.java diff --git a/core/src/main/java/org/apache/calcite/config/CalciteSystemProperty.java b/core/src/main/java/org/apache/calcite/config/CalciteSystemProperty.java index e5ea8c2ea29f..70db4505d62f 100644 --- a/core/src/main/java/org/apache/calcite/config/CalciteSystemProperty.java +++ b/core/src/main/java/org/apache/calcite/config/CalciteSystemProperty.java @@ -138,6 +138,14 @@ public final class CalciteSystemProperty { public static final CalciteSystemProperty TOPDOWN_OPT = booleanProperty("calcite.planner.topdown.opt", false); + + /** Whether to disable generate rel data type digest string. + * + *

    Disable generate rel data type digest string for every type can + * reduce composite type's digest memory and digest relative operation's latency. */ + public static final CalciteSystemProperty DISABLE_GENERATE_REL_DATA_TYPE_DIGEST_STRING = + booleanProperty("calcite.disable.generate.rel.data.type.digest.string", false); + /** * Whether to run integration tests. */ diff --git a/core/src/main/java/org/apache/calcite/jdbc/JavaRecordType.java b/core/src/main/java/org/apache/calcite/jdbc/JavaRecordType.java index 72242d3934cf..2183c2ac28bc 100644 --- a/core/src/main/java/org/apache/calcite/jdbc/JavaRecordType.java +++ b/core/src/main/java/org/apache/calcite/jdbc/JavaRecordType.java @@ -51,4 +51,16 @@ public JavaRecordType(List fields, Class clazz) { @Override public int hashCode() { return Objects.hash(fieldList, clazz); } + + @Override public boolean deepEquals(@Nullable Object obj) { + return this == obj + || obj instanceof JavaRecordType + && Objects.equals(fieldList, ((JavaRecordType) obj).fieldList) + && clazz == ((JavaRecordType) obj).clazz + && this.isNullable() == ((JavaRecordType) obj).isNullable(); + } + + @Override public int deepHashCode() { + return Objects.hash(fieldList, this.isNullable(), clazz); + } } diff --git a/core/src/main/java/org/apache/calcite/rel/HasDigestString.java b/core/src/main/java/org/apache/calcite/rel/HasDigestString.java new file mode 100644 index 000000000000..5058c3876959 --- /dev/null +++ b/core/src/main/java/org/apache/calcite/rel/HasDigestString.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.rel; + +/** + * Interface for objects that have a digest string. + */ +public interface HasDigestString { + String getDigestString(); +} diff --git a/core/src/main/java/org/apache/calcite/rel/type/RelDataType.java b/core/src/main/java/org/apache/calcite/rel/type/RelDataType.java index 4188d04698f4..071a1152a108 100644 --- a/core/src/main/java/org/apache/calcite/rel/type/RelDataType.java +++ b/core/src/main/java/org/apache/calcite/rel/type/RelDataType.java @@ -322,4 +322,26 @@ default boolean equalsSansFieldNamesAndNullability(@Nullable RelDataType that) { default boolean isMeasure() { return getSqlTypeName() == SqlTypeName.MEASURE; } + + /** + * Returns the digest of this type. + * + * @return digest of this type + */ + RelDataTypeDigest getDigest(); + + /** + * Deep equality check for RelDataType digest. + * + * @return Whether the 2 RelDataTypes are equivalent or have the same digest. + * @see #deepHashCode() + */ + boolean deepEquals(@Nullable Object obj); + + /** + * Compute deep hash code for RelDataType digest. + * + * @see #deepEquals(Object) + */ + int deepHashCode(); } diff --git a/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeDigest.java b/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeDigest.java new file mode 100644 index 000000000000..ac805aae22ac --- /dev/null +++ b/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeDigest.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.rel.type; + +import org.apache.calcite.rel.HasDigestString; + +/** + * Digest of a RelDataType. + */ +public interface RelDataTypeDigest extends HasDigestString { + RelDataType getType(); +} diff --git a/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeImpl.java b/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeImpl.java index 67ab34101695..d896ece7e9bd 100644 --- a/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeImpl.java +++ b/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeImpl.java @@ -16,6 +16,7 @@ */ package org.apache.calcite.rel.type; +import org.apache.calcite.config.CalciteSystemProperty; import org.apache.calcite.sql.SqlCollation; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlIntervalQualifier; @@ -46,8 +47,8 @@ * RelDataTypeImpl is an abstract base for implementations of * {@link RelDataType}. * - *

    Identity is based upon the {@link #digest} field, which each derived class - * should set during construction. + *

    Identity is based upon the {@link #digest} or {@link #innerDigest} field, + * which each derived class should set {@link #digest} or {@link #innerDigest} during construction. */ public abstract class RelDataTypeImpl implements RelDataType, RelDataTypeFamily { @@ -60,7 +61,14 @@ public abstract class RelDataTypeImpl //~ Instance fields -------------------------------------------------------- protected final @Nullable List fieldList; - protected @Nullable String digest; + + /** + * Use {@link #innerDigest} instead. + * + * @deprecated See {@link CalciteSystemProperty#DISABLE_GENERATE_REL_DATA_TYPE_DIGEST_STRING}. + */ + protected @Deprecated @Nullable String digest; + protected @Nullable RelDataTypeDigest innerDigest; //~ Constructors ----------------------------------------------------------- @@ -232,18 +240,50 @@ private static void getFieldRecurse(List slots, RelDataType type, return fieldList != null; } + /** + * Gets the {@link RelDataTypeDigest} of this type. + * If a user has set the legacy string {@code digest} and {@code innerDigest} has not + * been initialized yet, this method computes and initializes it. + */ + @Override public RelDataTypeDigest getDigest() { + if (digest != null && innerDigest == null) { + innerDigest = new InnerRelDataTypeDigest(); + } + return requireNonNull(innerDigest, "innerDigest"); + } + @Override public boolean equals(@Nullable Object obj) { - return this == obj - || obj instanceof RelDataTypeImpl - && Objects.equals(this.digest, ((RelDataTypeImpl) obj).digest); + if (obj == this) { + return true; + } + if (obj instanceof RelDataTypeImpl) { + final RelDataTypeImpl that = (RelDataTypeImpl) obj; + return this.getDigest().equals(that.getDigest()); + } + return false; } @Override public int hashCode() { - return Objects.hashCode(digest); + return getDigest().hashCode(); + } + + @Override public boolean deepEquals(@Nullable Object obj) { + if (this == obj) { + return true; + } + if (obj == null || this.getClass() != obj.getClass()) { + return false; + } + return Objects.equals(this.getDigest().getDigestString(), + ((RelDataTypeImpl) obj).getDigest().getDigestString()); + } + + @Override public int deepHashCode() { + return Objects.hashCode(this.getDigest().getDigestString()); } @Override public String getFullTypeString() { - return requireNonNull(digest, "digest"); + return requireNonNull(this.getDigest().getDigestString(), "digest"); } @Override public boolean isNullable() { @@ -309,23 +349,85 @@ protected abstract void generateTypeString( boolean withDetail); /** - * Computes the digest field. This should be called in every non-abstract - * subclass constructor once the type is fully defined. + * Init the lazy digest computing field {@link #innerDigest}. + * This should be called in every non-abstract subclass + * constructor once the type is fully defined. */ @SuppressWarnings("method.invocation.invalid") protected void computeDigest(@UnknownInitialization RelDataTypeImpl this) { - StringBuilder sb = new StringBuilder(); - generateTypeString(sb, true); - if (!isNullable()) { - sb.append(NON_NULLABLE_SUFFIX); + digest = null; + innerDigest = new InnerRelDataTypeDigest(); + if (!CalciteSystemProperty.DISABLE_GENERATE_REL_DATA_TYPE_DIGEST_STRING.value()) { + digest = this.getDigest().getDigestString(); } - digest = sb.toString(); } @Override public String toString() { - StringBuilder sb = new StringBuilder(); - generateTypeString(sb, false); - return sb.toString(); + return getDigest().toString(); + } + + /** Implementation of {@link RelDataTypeDigest}. */ + private class InnerRelDataTypeDigest implements RelDataTypeDigest { + /** Cached hash code. */ + private int hash = 0; + /** Cached type string. */ + private @Nullable String digestWithDetail = null; // NOTE: shorter detail will be better + private @Nullable String digestWithoutDetail = null; + + @Override public RelDataType getType() { + return RelDataTypeImpl.this; + } + + @Override public boolean equals(@Nullable Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final RelDataTypeImpl.InnerRelDataTypeDigest otherDigest = + (RelDataTypeImpl.InnerRelDataTypeDigest) o; + if (digest != null) { + return digest.equals(otherDigest.getDigestString()); + } + return deepEquals(otherDigest.getType()); + } + + @Override public int hashCode() { + if (digest != null) { + return Objects.hashCode(digest); + } + if (hash == 0) { + hash = deepHashCode(); + } + return hash; + } + + @Override public String getDigestString() { + // return user defined digest by set legacy digest string field. + if (digest != null) { + return digest; + } + + if (digestWithDetail == null) { + StringBuilder sb = new StringBuilder(); + generateTypeString(sb, true); + if (!isNullable()) { + sb.append(NON_NULLABLE_SUFFIX); + } + digestWithDetail = sb.toString(); + } + return digestWithDetail; + } + + @Override public String toString() { + if (digestWithoutDetail == null || digest != null) { + StringBuilder sb = new StringBuilder(); + RelDataTypeImpl.this.generateTypeString(sb, false); + digestWithoutDetail = sb.toString(); + } + return digestWithoutDetail; + } } @Override public RelDataTypePrecedenceList getPrecedenceList() { diff --git a/core/src/main/java/org/apache/calcite/rel/type/RelRecordType.java b/core/src/main/java/org/apache/calcite/rel/type/RelRecordType.java index 4f253bf225d8..a63b107ca0cb 100644 --- a/core/src/main/java/org/apache/calcite/rel/type/RelRecordType.java +++ b/core/src/main/java/org/apache/calcite/rel/type/RelRecordType.java @@ -27,6 +27,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import static java.util.Objects.requireNonNull; @@ -146,6 +147,39 @@ public RelRecordType(List fields) { sb.append(")"); } + @Override public boolean deepEquals(@Nullable Object obj) { + if (this == obj) { + return true; + } + if (obj == null || this.getClass() != obj.getClass()) { + return false; + } + + RelRecordType that = (RelRecordType) obj; + if (kind != that.kind || nullable != that.nullable) { + return false; + } + + if (fieldList == null || that.fieldList == null) { + return fieldList == null && that.fieldList == null; + } + + if (fieldList.size() != that.fieldList.size()) { + return false; + } + + for (int i = 0; i < fieldList.size(); i++) { + if (!fieldList.get(i).equals(that.fieldList.get(i))) { + return false; + } + } + return true; + } + + @Override public int deepHashCode() { + return Objects.hash(kind.ordinal(), nullable, fieldList); + } + /** * Per {@link Serializable} API, provides a replacement object to be written * during serialization. diff --git a/core/src/main/java/org/apache/calcite/rel/type/SingleColumnAliasRelDataType.java b/core/src/main/java/org/apache/calcite/rel/type/SingleColumnAliasRelDataType.java index b09196b57789..28e0406f6717 100644 --- a/core/src/main/java/org/apache/calcite/rel/type/SingleColumnAliasRelDataType.java +++ b/core/src/main/java/org/apache/calcite/rel/type/SingleColumnAliasRelDataType.java @@ -136,4 +136,16 @@ public SingleColumnAliasRelDataType(RelDataType original, RelDataType alias) { @Override public boolean isDynamicStruct() { return original.isDynamicStruct(); } + + @Override public RelDataTypeDigest getDigest() { + return original.getDigest(); + } + + @Override public boolean deepEquals(@Nullable Object obj) { + return original.deepEquals(obj); + } + + @Override public int deepHashCode() { + return original.deepHashCode(); + } } diff --git a/core/src/main/java/org/apache/calcite/sql/type/ArraySqlType.java b/core/src/main/java/org/apache/calcite/sql/type/ArraySqlType.java index 1124939c7bec..7d1d1afc470f 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/ArraySqlType.java +++ b/core/src/main/java/org/apache/calcite/sql/type/ArraySqlType.java @@ -20,6 +20,10 @@ import org.apache.calcite.rel.type.RelDataTypeFamily; import org.apache.calcite.rel.type.RelDataTypePrecedenceList; +import org.checkerframework.checker.nullness.qual.Nullable; + +import java.util.Objects; + import static org.apache.calcite.sql.type.NonNullableAccessors.getComponentTypeOrThrow; import static java.util.Objects.requireNonNull; @@ -56,6 +60,21 @@ public ArraySqlType(RelDataType elementType, boolean isNullable) { sb.append(" ARRAY"); } + @Override public boolean deepEquals(@Nullable Object obj) { + if (this == obj) { + return true; + } + if (obj == null || this.getClass() != obj.getClass()) { + return false; + } + ArraySqlType that = (ArraySqlType) obj; + return this.isNullable() == that.isNullable() && elementType.equals(that.elementType); + } + + @Override public int deepHashCode() { + return Objects.hash(SqlTypeName.ARRAY.ordinal(), isNullable, elementType.hashCode()); + } + // implement RelDataType @Override public RelDataType getComponentType() { return elementType; diff --git a/core/src/main/java/org/apache/calcite/sql/type/MapSqlType.java b/core/src/main/java/org/apache/calcite/sql/type/MapSqlType.java index 960fec7ac4ef..9ad216057577 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/MapSqlType.java +++ b/core/src/main/java/org/apache/calcite/sql/type/MapSqlType.java @@ -19,6 +19,10 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFamily; +import org.checkerframework.checker.nullness.qual.Nullable; + +import java.util.Objects; + import static java.util.Objects.requireNonNull; /** @@ -69,6 +73,23 @@ public MapSqlType( .append(") MAP"); } + @Override public boolean deepEquals(@Nullable Object obj) { + if (this == obj) { + return true; + } + if (obj == null || this.getClass() != obj.getClass()) { + return false; + } + MapSqlType that = (MapSqlType) obj; + return this.isNullable() == that.isNullable() && keyType.equals(that.keyType) + && valueType.equals(that.valueType); + } + + @Override public int deepHashCode() { + return Objects.hash(SqlTypeName.MAP.ordinal(), this.isNullable, keyType.hashCode(), + valueType.hashCode()); + } + // implement RelDataType @Override public RelDataTypeFamily getFamily() { return this; diff --git a/core/src/main/java/org/apache/calcite/sql/type/MultisetSqlType.java b/core/src/main/java/org/apache/calcite/sql/type/MultisetSqlType.java index b12c836d89a0..cbea4062c2ea 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/MultisetSqlType.java +++ b/core/src/main/java/org/apache/calcite/sql/type/MultisetSqlType.java @@ -20,6 +20,10 @@ import org.apache.calcite.rel.type.RelDataTypeFamily; import org.apache.calcite.rel.type.RelDataTypePrecedenceList; +import org.checkerframework.checker.nullness.qual.Nullable; + +import java.util.Objects; + import static org.apache.calcite.sql.type.NonNullableAccessors.getComponentTypeOrThrow; import static java.util.Objects.requireNonNull; @@ -56,6 +60,21 @@ public MultisetSqlType(RelDataType elementType, boolean isNullable) { sb.append(" MULTISET"); } + @Override public boolean deepEquals(@Nullable Object obj) { + if (this == obj) { + return true; + } + if (obj == null || this.getClass() != obj.getClass()) { + return false; + } + MultisetSqlType that = (MultisetSqlType) obj; + return this.isNullable() == that.isNullable() && elementType.equals(that.elementType); + } + + @Override public int deepHashCode() { + return Objects.hash(SqlTypeName.MULTISET.ordinal(), this.isNullable, elementType.hashCode()); + } + // implement RelDataType @Override public RelDataType getComponentType() { return elementType; diff --git a/core/src/test/java/org/apache/calcite/rex/RexBuilderTest.java b/core/src/test/java/org/apache/calcite/rex/RexBuilderTest.java index 7653d5370c76..e23679eaf5f5 100644 --- a/core/src/test/java/org/apache/calcite/rex/RexBuilderTest.java +++ b/core/src/test/java/org/apache/calcite/rex/RexBuilderTest.java @@ -1477,6 +1477,7 @@ private void checkBigDecimalLiteral(RexBuilder builder, String val) { /** Emulate a user defined type. */ private static class UDT extends RelDataTypeImpl { + @SuppressWarnings("deprecation") UDT() { this.digest = "(udt)NOT NULL"; } diff --git a/site/_docs/history.md b/site/_docs/history.md index 89d91bd2664c..dba2c98c349c 100644 --- a/site/_docs/history.md +++ b/site/_docs/history.md @@ -49,6 +49,10 @@ other software versions as specified in gradle.properties. #### Breaking Changes {: #breaking-1-42-0} +* [CALCITE-7393] +`RelDataTypeImpl.digest` is deprecated. We recommend using `RelDataTypeImpl.innerDigest` instead. +See system property `CalciteSystemProperty.DISABLE_GENERATE_REL_DATA_TYPE_DIGEST_STRING`. + * [CALCITE-7301] Prior to this change, most `SqlNode`s in the `org.apache.calcite.sql.ddl` package could not be unparsed when created with `SqlOperator#createCall`. To fix this, those `SqlNode`s now implement their own `SqlOperator`. diff --git a/ubenchmark/src/jmh/java/org/apache/calcite/benchmarks/TypeDigestBenchmark.java b/ubenchmark/src/jmh/java/org/apache/calcite/benchmarks/TypeDigestBenchmark.java new file mode 100644 index 000000000000..a02a6e836a0b --- /dev/null +++ b/ubenchmark/src/jmh/java/org/apache/calcite/benchmarks/TypeDigestBenchmark.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.benchmarks; + +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.sql.SqlCollation; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.tools.Frameworks; +import org.apache.calcite.tools.RelBuilder; + +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Threads; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +import java.nio.charset.StandardCharsets; +import java.util.concurrent.TimeUnit; + +/** + * Benchmark for {@link RelDataType} digest generation and comparison. + */ +@Fork(value = 1, jvmArgsPrepend = "-Dcalcite.disable.generate.type.digest.string=true") +@Measurement(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS) +@Warmup(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MICROSECONDS) +@State(Scope.Thread) +@Threads(1) +public class TypeDigestBenchmark { + + @Param({"1", "50", "500", "5000", "50000"}) + int topN; + + RelDataType type; + RelDataType type2; + + @Setup(Level.Trial) + public void setup() { + type = createType(topN); + type2 = createType(topN); + } + + private RelDataType createType(int n) { + RelBuilder builder = + RelBuilder.create(Frameworks.newConfigBuilder() + .defaultSchema(Frameworks.createRootSchema(true)) + .build()); + final RelDataTypeFactory typeFactory = builder.getTypeFactory(); + + RelDataType varchar = + typeFactory.createTypeWithCharsetAndCollation(typeFactory + .createSqlType(SqlTypeName.VARCHAR, 100), + StandardCharsets.UTF_8, SqlCollation.IMPLICIT); + + RelDataType leafObj = typeFactory.builder().add("k", varchar).add("v", varchar) + .add("attrs", typeFactory.createMapType(varchar, varchar)) + .add("tags", typeFactory.createArrayType(varchar, -1)).build(); + + final RelDataTypeFactory.Builder root = typeFactory.builder(); + for (int i = 0; i < n; i++) { + int depth = 1 + (i % 8); + RelDataType t = leafObj; + + for (int d = 0; d < depth; d++) { + RelDataType arrObj = typeFactory.createArrayType(t, -1); + RelDataType mapObj = typeFactory.createMapType(varchar, t); + + t = + typeFactory.builder().add("lvl" + d, t).add("arr" + d, arrObj).add("map" + d, mapObj) + .add("s" + d, varchar).build(); + } + + if ((i % 11) == 0) { + root.add("f" + i, typeFactory.createArrayType(t, -1)); + } else if ((i % 11) == 1) { + root.add("f" + i, typeFactory.createMapType(varchar, t)); + } else { + root.add("f" + i, t); + } + } + + return root.build(); + } + + @Benchmark + public boolean testEquals() { + return type.hashCode() == type2.hashCode() && type.equals(type2); + } + + public static void main(String[] args) throws RunnerException { + Options opt = new OptionsBuilder() + .include(TypeDigestBenchmark.class.getSimpleName()) + .detectJvmArgs() + .build(); + + new Runner(opt).run(); + } +} From 2b1972fa372a602ba05ddc69bb85c6af879c7c98 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Mon, 16 Feb 2026 11:07:45 -0800 Subject: [PATCH 163/562] [CALCITE-7410] TIMESTAMP type for TUMBLE and HOP is hardwired to TIMESTAMP(3) Signed-off-by: Mihai Budiu --- .../calcite/sql/SqlWindowTableFunction.java | 29 +- .../calcite/test/SqlToRelConverterTest.java | 7 + .../calcite/test/SqlToRelConverterTest.xml | 48 ++-- core/src/test/resources/sql/stream.iq | 256 ++++++++++-------- site/_docs/history.md | 6 + 5 files changed, 208 insertions(+), 138 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql/SqlWindowTableFunction.java b/core/src/main/java/org/apache/calcite/sql/SqlWindowTableFunction.java index ca2fdd19a83f..601b38299450 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlWindowTableFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlWindowTableFunction.java @@ -40,6 +40,8 @@ import static org.apache.calcite.util.Static.RESOURCE; +import static java.util.Objects.requireNonNull; + /** * Base class for a table-valued function that computes windows. Examples * include {@code TUMBLE}, {@code HOP} and {@code SESSION}. @@ -107,14 +109,37 @@ public SqlWindowTableFunction(String name, SqlOperandMetadata operandMetadata) { private static RelDataType inferRowType(SqlOperatorBinding opBinding) { final RelDataType inputRowType = opBinding.getOperandType(0); final RelDataTypeFactory typeFactory = opBinding.getTypeFactory(); + int precision = getTimestampPrecision(opBinding); return typeFactory.builder() .kind(inputRowType.getStructKind()) .addAll(inputRowType.getFieldList()) - .add("window_start", SqlTypeName.TIMESTAMP, 3) - .add("window_end", SqlTypeName.TIMESTAMP, 3) + .add("window_start", SqlTypeName.TIMESTAMP, precision) + .add("window_end", SqlTypeName.TIMESTAMP, precision) .build(); } + /** Extract the precision for the start_window, end_window columns from the column supplied as + * DESCRIPTOR for the window function. */ + private static int getTimestampPrecision(SqlOperatorBinding opBinding) { + RelDataType inputRowType = opBinding.getOperandType(0); + SqlCallBinding callBinding = (SqlCallBinding) opBinding; + // Locate the "descriptor" argument + for (SqlNode operand : callBinding.operands()) { + if (operand instanceof SqlCall) { + SqlCall opCall = (SqlCall) operand; + if (opCall.getOperator().getKind() == SqlKind.DESCRIPTOR) { + SqlNode descriptor = opCall.operand(0); + SqlIdentifier id = (SqlIdentifier) descriptor; + RelDataTypeField field = + inputRowType.getField(id.getSimple(), false, false); + return requireNonNull(field, "field").getType().getPrecision(); + } + } + } + // Should be unreachable, since validation succeeded + throw new RuntimeException("Could not locate DESCRIPTOR column"); + } + /** Partial implementation of operand type checker. */ protected abstract static class AbstractOperandMetadata implements SqlOperandMetadata { diff --git a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java index f0d068d5bb23..dc18d05520ea 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java @@ -2502,6 +2502,13 @@ void checkCorrelatedMapSubQuery(boolean expand) { sql(sql).ok(); } + @Test void testTableFunctionTumbleConvert() { + final String sql = "with t as (select CAST(rowtime AS TIMESTAMP(2)) as rowtime FROM Shipments) " + + "select *\n" + + "from table(tumble(table t, descriptor(rowtime), INTERVAL '1.5' SECOND))"; + sql(sql).ok(); + } + @Test void testTableFunctionTumbleWithParamNames() { final String sql = "select *\n" + "from table(\n" diff --git a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml index c8c136c97f04..a4dc246f8e25 100644 --- a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml @@ -8404,7 +8404,7 @@ from table(hop(table Shipments, descriptor(rowtime), INTERVAL '1' MINUTE, INTERV @@ -8418,7 +8418,7 @@ from table(hop(table Shipments, descriptor(rowtime), INTERVAL '1' MINUTE, INTERV @@ -8437,7 +8437,7 @@ hop( @@ -8456,7 +8456,7 @@ hop( @@ -8470,7 +8470,7 @@ from table(hop((select * from Shipments), descriptor(rowtime), INTERVAL '1' MINU @@ -8484,7 +8484,7 @@ from table(session(table Shipments, descriptor(rowtime), descriptor(orderId), IN @@ -8498,7 +8498,7 @@ from table(session(table Orders, descriptor(rowtime), descriptor(orderId, produc @@ -8517,7 +8517,7 @@ session( @@ -8536,7 +8536,7 @@ session( @@ -8550,7 +8550,7 @@ from table(session((select * from Shipments), descriptor(rowtime), descriptor(or @@ -8585,9 +8585,23 @@ from table(tumble(table Shipments, descriptor(rowtime), INTERVAL '1' MINUTE))]]> + + + + + + + + @@ -8602,10 +8616,10 @@ on a.orderid = b.orderid]]> @@ -8620,7 +8634,7 @@ from table(tumble(table Shipments, descriptor(rowtime), @@ -8638,7 +8652,7 @@ tumble( @@ -8656,7 +8670,7 @@ tumble( @@ -8670,7 +8684,7 @@ from table(tumble((select * from Shipments), descriptor(rowtime), INTERVAL '1' M diff --git a/core/src/test/resources/sql/stream.iq b/core/src/test/resources/sql/stream.iq index 653632955d5b..f20a7fe6407b 100644 --- a/core/src/test/resources/sql/stream.iq +++ b/core/src/test/resources/sql/stream.iq @@ -17,81 +17,99 @@ # !use orinoco !set outputformat mysql + +# Test case for [CALCITE-7410] TIMESTAMP type for TUMBLE and HOP is hardwired to TIMESTAMP(3) +# Since we cast the input column to TIMESTAMP(3), we expect window_start to have the same type. +WITH S AS (SELECT *, CAST(ROWTIME AS TIMESTAMP(3)) + INTERVAL '0.5' SECONDS AS TS FROM ORDERS) +SELECT * FROM TABLE(TUMBLE((SELECT * FROM S), DESCRIPTOR(TS), INTERVAL '20:10.525' MINUTE TO SECOND)); ++---------------------+----+---------+-------+-------------------------+-------------------------+-------------------------+ +| ROWTIME | ID | PRODUCT | UNITS | TS | window_start | window_end | ++---------------------+----+---------+-------+-------------------------+-------------------------+-------------------------+ +| 2015-02-15 10:15:00 | 1 | paint | 10 | 2015-02-15 10:15:00.500 | 2015-02-15 10:10:31.125 | 2015-02-15 10:30:41.650 | +| 2015-02-15 10:24:15 | 2 | paper | 5 | 2015-02-15 10:24:15.500 | 2015-02-15 10:10:31.125 | 2015-02-15 10:30:41.650 | +| 2015-02-15 10:24:45 | 3 | brush | 12 | 2015-02-15 10:24:45.500 | 2015-02-15 10:10:31.125 | 2015-02-15 10:30:41.650 | +| 2015-02-15 10:58:00 | 4 | paint | 3 | 2015-02-15 10:58:00.500 | 2015-02-15 10:50:52.175 | 2015-02-15 11:11:02.700 | +| 2015-02-15 11:10:00 | 5 | paint | 3 | 2015-02-15 11:10:00.500 | 2015-02-15 10:50:52.175 | 2015-02-15 11:11:02.700 | ++---------------------+----+---------+-------+-------------------------+-------------------------+-------------------------+ +(5 rows) + +!ok + SELECT * FROM TABLE( TUMBLE( DATA => TABLE ORDERS, TIMECOL => DESCRIPTOR(ROWTIME), SIZE => INTERVAL '1' MINUTE)); -+---------------------+----+---------+-------+-------------------------+-------------------------+ -| ROWTIME | ID | PRODUCT | UNITS | window_start | window_end | -+---------------------+----+---------+-------+-------------------------+-------------------------+ -| 2015-02-15 10:15:00 | 1 | paint | 10 | 2015-02-15 10:15:00.000 | 2015-02-15 10:16:00.000 | -| 2015-02-15 10:24:15 | 2 | paper | 5 | 2015-02-15 10:24:00.000 | 2015-02-15 10:25:00.000 | -| 2015-02-15 10:24:45 | 3 | brush | 12 | 2015-02-15 10:24:00.000 | 2015-02-15 10:25:00.000 | -| 2015-02-15 10:58:00 | 4 | paint | 3 | 2015-02-15 10:58:00.000 | 2015-02-15 10:59:00.000 | -| 2015-02-15 11:10:00 | 5 | paint | 3 | 2015-02-15 11:10:00.000 | 2015-02-15 11:11:00.000 | -+---------------------+----+---------+-------+-------------------------+-------------------------+ ++---------------------+----+---------+-------+---------------------+---------------------+ +| ROWTIME | ID | PRODUCT | UNITS | window_start | window_end | ++---------------------+----+---------+-------+---------------------+---------------------+ +| 2015-02-15 10:15:00 | 1 | paint | 10 | 2015-02-15 10:15:00 | 2015-02-15 10:16:00 | +| 2015-02-15 10:24:15 | 2 | paper | 5 | 2015-02-15 10:24:00 | 2015-02-15 10:25:00 | +| 2015-02-15 10:24:45 | 3 | brush | 12 | 2015-02-15 10:24:00 | 2015-02-15 10:25:00 | +| 2015-02-15 10:58:00 | 4 | paint | 3 | 2015-02-15 10:58:00 | 2015-02-15 10:59:00 | +| 2015-02-15 11:10:00 | 5 | paint | 3 | 2015-02-15 11:10:00 | 2015-02-15 11:11:00 | ++---------------------+----+---------+-------+---------------------+---------------------+ (5 rows) !ok SELECT * FROM TABLE(TUMBLE(TABLE ORDERS, DESCRIPTOR(ROWTIME), INTERVAL '1' MINUTE)); -+---------------------+----+---------+-------+-------------------------+-------------------------+ -| ROWTIME | ID | PRODUCT | UNITS | window_start | window_end | -+---------------------+----+---------+-------+-------------------------+-------------------------+ -| 2015-02-15 10:15:00 | 1 | paint | 10 | 2015-02-15 10:15:00.000 | 2015-02-15 10:16:00.000 | -| 2015-02-15 10:24:15 | 2 | paper | 5 | 2015-02-15 10:24:00.000 | 2015-02-15 10:25:00.000 | -| 2015-02-15 10:24:45 | 3 | brush | 12 | 2015-02-15 10:24:00.000 | 2015-02-15 10:25:00.000 | -| 2015-02-15 10:58:00 | 4 | paint | 3 | 2015-02-15 10:58:00.000 | 2015-02-15 10:59:00.000 | -| 2015-02-15 11:10:00 | 5 | paint | 3 | 2015-02-15 11:10:00.000 | 2015-02-15 11:11:00.000 | -+---------------------+----+---------+-------+-------------------------+-------------------------+ ++---------------------+----+---------+-------+---------------------+---------------------+ +| ROWTIME | ID | PRODUCT | UNITS | window_start | window_end | ++---------------------+----+---------+-------+---------------------+---------------------+ +| 2015-02-15 10:15:00 | 1 | paint | 10 | 2015-02-15 10:15:00 | 2015-02-15 10:16:00 | +| 2015-02-15 10:24:15 | 2 | paper | 5 | 2015-02-15 10:24:00 | 2015-02-15 10:25:00 | +| 2015-02-15 10:24:45 | 3 | brush | 12 | 2015-02-15 10:24:00 | 2015-02-15 10:25:00 | +| 2015-02-15 10:58:00 | 4 | paint | 3 | 2015-02-15 10:58:00 | 2015-02-15 10:59:00 | +| 2015-02-15 11:10:00 | 5 | paint | 3 | 2015-02-15 11:10:00 | 2015-02-15 11:11:00 | ++---------------------+----+---------+-------+---------------------+---------------------+ (5 rows) !ok SELECT * FROM TABLE(TUMBLE((SELECT * FROM ORDERS), DESCRIPTOR(ROWTIME), INTERVAL '1' MINUTE)); -+---------------------+----+---------+-------+-------------------------+-------------------------+ -| ROWTIME | ID | PRODUCT | UNITS | window_start | window_end | -+---------------------+----+---------+-------+-------------------------+-------------------------+ -| 2015-02-15 10:15:00 | 1 | paint | 10 | 2015-02-15 10:15:00.000 | 2015-02-15 10:16:00.000 | -| 2015-02-15 10:24:15 | 2 | paper | 5 | 2015-02-15 10:24:00.000 | 2015-02-15 10:25:00.000 | -| 2015-02-15 10:24:45 | 3 | brush | 12 | 2015-02-15 10:24:00.000 | 2015-02-15 10:25:00.000 | -| 2015-02-15 10:58:00 | 4 | paint | 3 | 2015-02-15 10:58:00.000 | 2015-02-15 10:59:00.000 | -| 2015-02-15 11:10:00 | 5 | paint | 3 | 2015-02-15 11:10:00.000 | 2015-02-15 11:11:00.000 | -+---------------------+----+---------+-------+-------------------------+-------------------------+ ++---------------------+----+---------+-------+---------------------+---------------------+ +| ROWTIME | ID | PRODUCT | UNITS | window_start | window_end | ++---------------------+----+---------+-------+---------------------+---------------------+ +| 2015-02-15 10:15:00 | 1 | paint | 10 | 2015-02-15 10:15:00 | 2015-02-15 10:16:00 | +| 2015-02-15 10:24:15 | 2 | paper | 5 | 2015-02-15 10:24:00 | 2015-02-15 10:25:00 | +| 2015-02-15 10:24:45 | 3 | brush | 12 | 2015-02-15 10:24:00 | 2015-02-15 10:25:00 | +| 2015-02-15 10:58:00 | 4 | paint | 3 | 2015-02-15 10:58:00 | 2015-02-15 10:59:00 | +| 2015-02-15 11:10:00 | 5 | paint | 3 | 2015-02-15 11:10:00 | 2015-02-15 11:11:00 | ++---------------------+----+---------+-------+---------------------+---------------------+ (5 rows) !ok SELECT * FROM TABLE(TUMBLE((SELECT * FROM ORDERS), DESCRIPTOR(ROWTIME), INTERVAL '10' MINUTE, INTERVAL '3' MINUTE)); -+---------------------+----+---------+-------+-------------------------+-------------------------+ -| ROWTIME | ID | PRODUCT | UNITS | window_start | window_end | -+---------------------+----+---------+-------+-------------------------+-------------------------+ -| 2015-02-15 10:15:00 | 1 | paint | 10 | 2015-02-15 10:13:00.000 | 2015-02-15 10:23:00.000 | -| 2015-02-15 10:24:15 | 2 | paper | 5 | 2015-02-15 10:23:00.000 | 2015-02-15 10:33:00.000 | -| 2015-02-15 10:24:45 | 3 | brush | 12 | 2015-02-15 10:23:00.000 | 2015-02-15 10:33:00.000 | -| 2015-02-15 10:58:00 | 4 | paint | 3 | 2015-02-15 10:53:00.000 | 2015-02-15 11:03:00.000 | -| 2015-02-15 11:10:00 | 5 | paint | 3 | 2015-02-15 11:03:00.000 | 2015-02-15 11:13:00.000 | -+---------------------+----+---------+-------+-------------------------+-------------------------+ ++---------------------+----+---------+-------+---------------------+---------------------+ +| ROWTIME | ID | PRODUCT | UNITS | window_start | window_end | ++---------------------+----+---------+-------+---------------------+---------------------+ +| 2015-02-15 10:15:00 | 1 | paint | 10 | 2015-02-15 10:13:00 | 2015-02-15 10:23:00 | +| 2015-02-15 10:24:15 | 2 | paper | 5 | 2015-02-15 10:23:00 | 2015-02-15 10:33:00 | +| 2015-02-15 10:24:45 | 3 | brush | 12 | 2015-02-15 10:23:00 | 2015-02-15 10:33:00 | +| 2015-02-15 10:58:00 | 4 | paint | 3 | 2015-02-15 10:53:00 | 2015-02-15 11:03:00 | +| 2015-02-15 11:10:00 | 5 | paint | 3 | 2015-02-15 11:03:00 | 2015-02-15 11:13:00 | ++---------------------+----+---------+-------+---------------------+---------------------+ (5 rows) !ok SELECT * FROM TABLE(HOP(TABLE ORDERS, DESCRIPTOR(ROWTIME), INTERVAL '5' MINUTE, INTERVAL '10' MINUTE)); -+---------------------+----+---------+-------+-------------------------+-------------------------+ -| ROWTIME | ID | PRODUCT | UNITS | window_start | window_end | -+---------------------+----+---------+-------+-------------------------+-------------------------+ -| 2015-02-15 10:15:00 | 1 | paint | 10 | 2015-02-15 10:10:00.000 | 2015-02-15 10:20:00.000 | -| 2015-02-15 10:15:00 | 1 | paint | 10 | 2015-02-15 10:15:00.000 | 2015-02-15 10:25:00.000 | -| 2015-02-15 10:24:15 | 2 | paper | 5 | 2015-02-15 10:15:00.000 | 2015-02-15 10:25:00.000 | -| 2015-02-15 10:24:15 | 2 | paper | 5 | 2015-02-15 10:20:00.000 | 2015-02-15 10:30:00.000 | -| 2015-02-15 10:24:45 | 3 | brush | 12 | 2015-02-15 10:15:00.000 | 2015-02-15 10:25:00.000 | -| 2015-02-15 10:24:45 | 3 | brush | 12 | 2015-02-15 10:20:00.000 | 2015-02-15 10:30:00.000 | -| 2015-02-15 10:58:00 | 4 | paint | 3 | 2015-02-15 10:50:00.000 | 2015-02-15 11:00:00.000 | -| 2015-02-15 10:58:00 | 4 | paint | 3 | 2015-02-15 10:55:00.000 | 2015-02-15 11:05:00.000 | -| 2015-02-15 11:10:00 | 5 | paint | 3 | 2015-02-15 11:05:00.000 | 2015-02-15 11:15:00.000 | -| 2015-02-15 11:10:00 | 5 | paint | 3 | 2015-02-15 11:10:00.000 | 2015-02-15 11:20:00.000 | -+---------------------+----+---------+-------+-------------------------+-------------------------+ ++---------------------+----+---------+-------+---------------------+---------------------+ +| ROWTIME | ID | PRODUCT | UNITS | window_start | window_end | ++---------------------+----+---------+-------+---------------------+---------------------+ +| 2015-02-15 10:15:00 | 1 | paint | 10 | 2015-02-15 10:10:00 | 2015-02-15 10:20:00 | +| 2015-02-15 10:15:00 | 1 | paint | 10 | 2015-02-15 10:15:00 | 2015-02-15 10:25:00 | +| 2015-02-15 10:24:15 | 2 | paper | 5 | 2015-02-15 10:15:00 | 2015-02-15 10:25:00 | +| 2015-02-15 10:24:15 | 2 | paper | 5 | 2015-02-15 10:20:00 | 2015-02-15 10:30:00 | +| 2015-02-15 10:24:45 | 3 | brush | 12 | 2015-02-15 10:15:00 | 2015-02-15 10:25:00 | +| 2015-02-15 10:24:45 | 3 | brush | 12 | 2015-02-15 10:20:00 | 2015-02-15 10:30:00 | +| 2015-02-15 10:58:00 | 4 | paint | 3 | 2015-02-15 10:50:00 | 2015-02-15 11:00:00 | +| 2015-02-15 10:58:00 | 4 | paint | 3 | 2015-02-15 10:55:00 | 2015-02-15 11:05:00 | +| 2015-02-15 11:10:00 | 5 | paint | 3 | 2015-02-15 11:05:00 | 2015-02-15 11:15:00 | +| 2015-02-15 11:10:00 | 5 | paint | 3 | 2015-02-15 11:10:00 | 2015-02-15 11:20:00 | ++---------------------+----+---------+-------+---------------------+---------------------+ (10 rows) !ok @@ -102,72 +120,72 @@ SELECT * FROM TABLE( TIMECOL => DESCRIPTOR(ROWTIME), SLIDE => INTERVAL '5' MINUTE, SIZE => INTERVAL '10' MINUTE)); -+---------------------+----+---------+-------+-------------------------+-------------------------+ -| ROWTIME | ID | PRODUCT | UNITS | window_start | window_end | -+---------------------+----+---------+-------+-------------------------+-------------------------+ -| 2015-02-15 10:15:00 | 1 | paint | 10 | 2015-02-15 10:10:00.000 | 2015-02-15 10:20:00.000 | -| 2015-02-15 10:15:00 | 1 | paint | 10 | 2015-02-15 10:15:00.000 | 2015-02-15 10:25:00.000 | -| 2015-02-15 10:24:15 | 2 | paper | 5 | 2015-02-15 10:15:00.000 | 2015-02-15 10:25:00.000 | -| 2015-02-15 10:24:15 | 2 | paper | 5 | 2015-02-15 10:20:00.000 | 2015-02-15 10:30:00.000 | -| 2015-02-15 10:24:45 | 3 | brush | 12 | 2015-02-15 10:15:00.000 | 2015-02-15 10:25:00.000 | -| 2015-02-15 10:24:45 | 3 | brush | 12 | 2015-02-15 10:20:00.000 | 2015-02-15 10:30:00.000 | -| 2015-02-15 10:58:00 | 4 | paint | 3 | 2015-02-15 10:50:00.000 | 2015-02-15 11:00:00.000 | -| 2015-02-15 10:58:00 | 4 | paint | 3 | 2015-02-15 10:55:00.000 | 2015-02-15 11:05:00.000 | -| 2015-02-15 11:10:00 | 5 | paint | 3 | 2015-02-15 11:05:00.000 | 2015-02-15 11:15:00.000 | -| 2015-02-15 11:10:00 | 5 | paint | 3 | 2015-02-15 11:10:00.000 | 2015-02-15 11:20:00.000 | -+---------------------+----+---------+-------+-------------------------+-------------------------+ ++---------------------+----+---------+-------+---------------------+---------------------+ +| ROWTIME | ID | PRODUCT | UNITS | window_start | window_end | ++---------------------+----+---------+-------+---------------------+---------------------+ +| 2015-02-15 10:15:00 | 1 | paint | 10 | 2015-02-15 10:10:00 | 2015-02-15 10:20:00 | +| 2015-02-15 10:15:00 | 1 | paint | 10 | 2015-02-15 10:15:00 | 2015-02-15 10:25:00 | +| 2015-02-15 10:24:15 | 2 | paper | 5 | 2015-02-15 10:15:00 | 2015-02-15 10:25:00 | +| 2015-02-15 10:24:15 | 2 | paper | 5 | 2015-02-15 10:20:00 | 2015-02-15 10:30:00 | +| 2015-02-15 10:24:45 | 3 | brush | 12 | 2015-02-15 10:15:00 | 2015-02-15 10:25:00 | +| 2015-02-15 10:24:45 | 3 | brush | 12 | 2015-02-15 10:20:00 | 2015-02-15 10:30:00 | +| 2015-02-15 10:58:00 | 4 | paint | 3 | 2015-02-15 10:50:00 | 2015-02-15 11:00:00 | +| 2015-02-15 10:58:00 | 4 | paint | 3 | 2015-02-15 10:55:00 | 2015-02-15 11:05:00 | +| 2015-02-15 11:10:00 | 5 | paint | 3 | 2015-02-15 11:05:00 | 2015-02-15 11:15:00 | +| 2015-02-15 11:10:00 | 5 | paint | 3 | 2015-02-15 11:10:00 | 2015-02-15 11:20:00 | ++---------------------+----+---------+-------+---------------------+---------------------+ (10 rows) !ok SELECT * FROM TABLE(HOP(TABLE ORDERS, DESCRIPTOR(ROWTIME), INTERVAL '5' MINUTE, INTERVAL '10' MINUTE, INTERVAL '2' MINUTE)); -+---------------------+----+---------+-------+-------------------------+-------------------------+ -| ROWTIME | ID | PRODUCT | UNITS | window_start | window_end | -+---------------------+----+---------+-------+-------------------------+-------------------------+ -| 2015-02-15 10:15:00 | 1 | paint | 10 | 2015-02-15 10:07:00.000 | 2015-02-15 10:17:00.000 | -| 2015-02-15 10:15:00 | 1 | paint | 10 | 2015-02-15 10:12:00.000 | 2015-02-15 10:22:00.000 | -| 2015-02-15 10:24:15 | 2 | paper | 5 | 2015-02-15 10:17:00.000 | 2015-02-15 10:27:00.000 | -| 2015-02-15 10:24:15 | 2 | paper | 5 | 2015-02-15 10:22:00.000 | 2015-02-15 10:32:00.000 | -| 2015-02-15 10:24:45 | 3 | brush | 12 | 2015-02-15 10:17:00.000 | 2015-02-15 10:27:00.000 | -| 2015-02-15 10:24:45 | 3 | brush | 12 | 2015-02-15 10:22:00.000 | 2015-02-15 10:32:00.000 | -| 2015-02-15 10:58:00 | 4 | paint | 3 | 2015-02-15 10:52:00.000 | 2015-02-15 11:02:00.000 | -| 2015-02-15 10:58:00 | 4 | paint | 3 | 2015-02-15 10:57:00.000 | 2015-02-15 11:07:00.000 | -| 2015-02-15 11:10:00 | 5 | paint | 3 | 2015-02-15 11:02:00.000 | 2015-02-15 11:12:00.000 | -| 2015-02-15 11:10:00 | 5 | paint | 3 | 2015-02-15 11:07:00.000 | 2015-02-15 11:17:00.000 | -+---------------------+----+---------+-------+-------------------------+-------------------------+ ++---------------------+----+---------+-------+---------------------+---------------------+ +| ROWTIME | ID | PRODUCT | UNITS | window_start | window_end | ++---------------------+----+---------+-------+---------------------+---------------------+ +| 2015-02-15 10:15:00 | 1 | paint | 10 | 2015-02-15 10:07:00 | 2015-02-15 10:17:00 | +| 2015-02-15 10:15:00 | 1 | paint | 10 | 2015-02-15 10:12:00 | 2015-02-15 10:22:00 | +| 2015-02-15 10:24:15 | 2 | paper | 5 | 2015-02-15 10:17:00 | 2015-02-15 10:27:00 | +| 2015-02-15 10:24:15 | 2 | paper | 5 | 2015-02-15 10:22:00 | 2015-02-15 10:32:00 | +| 2015-02-15 10:24:45 | 3 | brush | 12 | 2015-02-15 10:17:00 | 2015-02-15 10:27:00 | +| 2015-02-15 10:24:45 | 3 | brush | 12 | 2015-02-15 10:22:00 | 2015-02-15 10:32:00 | +| 2015-02-15 10:58:00 | 4 | paint | 3 | 2015-02-15 10:52:00 | 2015-02-15 11:02:00 | +| 2015-02-15 10:58:00 | 4 | paint | 3 | 2015-02-15 10:57:00 | 2015-02-15 11:07:00 | +| 2015-02-15 11:10:00 | 5 | paint | 3 | 2015-02-15 11:02:00 | 2015-02-15 11:12:00 | +| 2015-02-15 11:10:00 | 5 | paint | 3 | 2015-02-15 11:07:00 | 2015-02-15 11:17:00 | ++---------------------+----+---------+-------+---------------------+---------------------+ (10 rows) !ok SELECT * FROM TABLE(HOP((SELECT * FROM ORDERS), DESCRIPTOR(ROWTIME), INTERVAL '5' MINUTE, INTERVAL '10' MINUTE)); -+---------------------+----+---------+-------+-------------------------+-------------------------+ -| ROWTIME | ID | PRODUCT | UNITS | window_start | window_end | -+---------------------+----+---------+-------+-------------------------+-------------------------+ -| 2015-02-15 10:15:00 | 1 | paint | 10 | 2015-02-15 10:10:00.000 | 2015-02-15 10:20:00.000 | -| 2015-02-15 10:15:00 | 1 | paint | 10 | 2015-02-15 10:15:00.000 | 2015-02-15 10:25:00.000 | -| 2015-02-15 10:24:15 | 2 | paper | 5 | 2015-02-15 10:15:00.000 | 2015-02-15 10:25:00.000 | -| 2015-02-15 10:24:15 | 2 | paper | 5 | 2015-02-15 10:20:00.000 | 2015-02-15 10:30:00.000 | -| 2015-02-15 10:24:45 | 3 | brush | 12 | 2015-02-15 10:15:00.000 | 2015-02-15 10:25:00.000 | -| 2015-02-15 10:24:45 | 3 | brush | 12 | 2015-02-15 10:20:00.000 | 2015-02-15 10:30:00.000 | -| 2015-02-15 10:58:00 | 4 | paint | 3 | 2015-02-15 10:50:00.000 | 2015-02-15 11:00:00.000 | -| 2015-02-15 10:58:00 | 4 | paint | 3 | 2015-02-15 10:55:00.000 | 2015-02-15 11:05:00.000 | -| 2015-02-15 11:10:00 | 5 | paint | 3 | 2015-02-15 11:05:00.000 | 2015-02-15 11:15:00.000 | -| 2015-02-15 11:10:00 | 5 | paint | 3 | 2015-02-15 11:10:00.000 | 2015-02-15 11:20:00.000 | -+---------------------+----+---------+-------+-------------------------+-------------------------+ ++---------------------+----+---------+-------+---------------------+---------------------+ +| ROWTIME | ID | PRODUCT | UNITS | window_start | window_end | ++---------------------+----+---------+-------+---------------------+---------------------+ +| 2015-02-15 10:15:00 | 1 | paint | 10 | 2015-02-15 10:10:00 | 2015-02-15 10:20:00 | +| 2015-02-15 10:15:00 | 1 | paint | 10 | 2015-02-15 10:15:00 | 2015-02-15 10:25:00 | +| 2015-02-15 10:24:15 | 2 | paper | 5 | 2015-02-15 10:15:00 | 2015-02-15 10:25:00 | +| 2015-02-15 10:24:15 | 2 | paper | 5 | 2015-02-15 10:20:00 | 2015-02-15 10:30:00 | +| 2015-02-15 10:24:45 | 3 | brush | 12 | 2015-02-15 10:15:00 | 2015-02-15 10:25:00 | +| 2015-02-15 10:24:45 | 3 | brush | 12 | 2015-02-15 10:20:00 | 2015-02-15 10:30:00 | +| 2015-02-15 10:58:00 | 4 | paint | 3 | 2015-02-15 10:50:00 | 2015-02-15 11:00:00 | +| 2015-02-15 10:58:00 | 4 | paint | 3 | 2015-02-15 10:55:00 | 2015-02-15 11:05:00 | +| 2015-02-15 11:10:00 | 5 | paint | 3 | 2015-02-15 11:05:00 | 2015-02-15 11:15:00 | +| 2015-02-15 11:10:00 | 5 | paint | 3 | 2015-02-15 11:10:00 | 2015-02-15 11:20:00 | ++---------------------+----+---------+-------+---------------------+---------------------+ (10 rows) !ok SELECT * FROM TABLE(SESSION(TABLE ORDERS, DESCRIPTOR(ROWTIME), DESCRIPTOR(PRODUCT), INTERVAL '20' MINUTE)); -+---------------------+----+---------+-------+-------------------------+-------------------------+ -| ROWTIME | ID | PRODUCT | UNITS | window_start | window_end | -+---------------------+----+---------+-------+-------------------------+-------------------------+ -| 2015-02-15 10:15:00 | 1 | paint | 10 | 2015-02-15 10:15:00.000 | 2015-02-15 10:35:00.000 | -| 2015-02-15 10:24:15 | 2 | paper | 5 | 2015-02-15 10:24:15.000 | 2015-02-15 10:44:15.000 | -| 2015-02-15 10:24:45 | 3 | brush | 12 | 2015-02-15 10:24:45.000 | 2015-02-15 10:44:45.000 | -| 2015-02-15 10:58:00 | 4 | paint | 3 | 2015-02-15 10:58:00.000 | 2015-02-15 11:30:00.000 | -| 2015-02-15 11:10:00 | 5 | paint | 3 | 2015-02-15 10:58:00.000 | 2015-02-15 11:30:00.000 | -+---------------------+----+---------+-------+-------------------------+-------------------------+ ++---------------------+----+---------+-------+---------------------+---------------------+ +| ROWTIME | ID | PRODUCT | UNITS | window_start | window_end | ++---------------------+----+---------+-------+---------------------+---------------------+ +| 2015-02-15 10:15:00 | 1 | paint | 10 | 2015-02-15 10:15:00 | 2015-02-15 10:35:00 | +| 2015-02-15 10:24:15 | 2 | paper | 5 | 2015-02-15 10:24:15 | 2015-02-15 10:44:15 | +| 2015-02-15 10:24:45 | 3 | brush | 12 | 2015-02-15 10:24:45 | 2015-02-15 10:44:45 | +| 2015-02-15 10:58:00 | 4 | paint | 3 | 2015-02-15 10:58:00 | 2015-02-15 11:30:00 | +| 2015-02-15 11:10:00 | 5 | paint | 3 | 2015-02-15 10:58:00 | 2015-02-15 11:30:00 | ++---------------------+----+---------+-------+---------------------+---------------------+ (5 rows) !ok @@ -178,29 +196,29 @@ SELECT * FROM TABLE( TIMECOL => DESCRIPTOR(ROWTIME), KEY => DESCRIPTOR(PRODUCT), SIZE => INTERVAL '20' MINUTE)); -+---------------------+----+---------+-------+-------------------------+-------------------------+ -| ROWTIME | ID | PRODUCT | UNITS | window_start | window_end | -+---------------------+----+---------+-------+-------------------------+-------------------------+ -| 2015-02-15 10:15:00 | 1 | paint | 10 | 2015-02-15 10:15:00.000 | 2015-02-15 10:35:00.000 | -| 2015-02-15 10:24:15 | 2 | paper | 5 | 2015-02-15 10:24:15.000 | 2015-02-15 10:44:15.000 | -| 2015-02-15 10:24:45 | 3 | brush | 12 | 2015-02-15 10:24:45.000 | 2015-02-15 10:44:45.000 | -| 2015-02-15 10:58:00 | 4 | paint | 3 | 2015-02-15 10:58:00.000 | 2015-02-15 11:30:00.000 | -| 2015-02-15 11:10:00 | 5 | paint | 3 | 2015-02-15 10:58:00.000 | 2015-02-15 11:30:00.000 | -+---------------------+----+---------+-------+-------------------------+-------------------------+ ++---------------------+----+---------+-------+---------------------+---------------------+ +| ROWTIME | ID | PRODUCT | UNITS | window_start | window_end | ++---------------------+----+---------+-------+---------------------+---------------------+ +| 2015-02-15 10:15:00 | 1 | paint | 10 | 2015-02-15 10:15:00 | 2015-02-15 10:35:00 | +| 2015-02-15 10:24:15 | 2 | paper | 5 | 2015-02-15 10:24:15 | 2015-02-15 10:44:15 | +| 2015-02-15 10:24:45 | 3 | brush | 12 | 2015-02-15 10:24:45 | 2015-02-15 10:44:45 | +| 2015-02-15 10:58:00 | 4 | paint | 3 | 2015-02-15 10:58:00 | 2015-02-15 11:30:00 | +| 2015-02-15 11:10:00 | 5 | paint | 3 | 2015-02-15 10:58:00 | 2015-02-15 11:30:00 | ++---------------------+----+---------+-------+---------------------+---------------------+ (5 rows) !ok SELECT * FROM TABLE(SESSION((SELECT * FROM ORDERS), DESCRIPTOR(ROWTIME), DESCRIPTOR(PRODUCT), INTERVAL '20' MINUTE)); -+---------------------+----+---------+-------+-------------------------+-------------------------+ -| ROWTIME | ID | PRODUCT | UNITS | window_start | window_end | -+---------------------+----+---------+-------+-------------------------+-------------------------+ -| 2015-02-15 10:15:00 | 1 | paint | 10 | 2015-02-15 10:15:00.000 | 2015-02-15 10:35:00.000 | -| 2015-02-15 10:24:15 | 2 | paper | 5 | 2015-02-15 10:24:15.000 | 2015-02-15 10:44:15.000 | -| 2015-02-15 10:24:45 | 3 | brush | 12 | 2015-02-15 10:24:45.000 | 2015-02-15 10:44:45.000 | -| 2015-02-15 10:58:00 | 4 | paint | 3 | 2015-02-15 10:58:00.000 | 2015-02-15 11:30:00.000 | -| 2015-02-15 11:10:00 | 5 | paint | 3 | 2015-02-15 10:58:00.000 | 2015-02-15 11:30:00.000 | -+---------------------+----+---------+-------+-------------------------+-------------------------+ ++---------------------+----+---------+-------+---------------------+---------------------+ +| ROWTIME | ID | PRODUCT | UNITS | window_start | window_end | ++---------------------+----+---------+-------+---------------------+---------------------+ +| 2015-02-15 10:15:00 | 1 | paint | 10 | 2015-02-15 10:15:00 | 2015-02-15 10:35:00 | +| 2015-02-15 10:24:15 | 2 | paper | 5 | 2015-02-15 10:24:15 | 2015-02-15 10:44:15 | +| 2015-02-15 10:24:45 | 3 | brush | 12 | 2015-02-15 10:24:45 | 2015-02-15 10:44:45 | +| 2015-02-15 10:58:00 | 4 | paint | 3 | 2015-02-15 10:58:00 | 2015-02-15 11:30:00 | +| 2015-02-15 11:10:00 | 5 | paint | 3 | 2015-02-15 10:58:00 | 2015-02-15 11:30:00 | ++---------------------+----+---------+-------+---------------------+---------------------+ (5 rows) !ok diff --git a/site/_docs/history.md b/site/_docs/history.md index dba2c98c349c..8a7da2a4f31e 100644 --- a/site/_docs/history.md +++ b/site/_docs/history.md @@ -49,6 +49,12 @@ other software versions as specified in gradle.properties. #### Breaking Changes {: #breaking-1-42-0} +* [CALCITE-7410] + Changes the type of the `WINDOW_START` and `WINDOW_END` columns for + the table functions `HOP`, `TUMBLE`, `SESSION` to match the original + type of the timestamp column. These types used to be hardwired to + `TIMESTAMP(3)`. + * [CALCITE-7393] `RelDataTypeImpl.digest` is deprecated. We recommend using `RelDataTypeImpl.innerDigest` instead. See system property `CalciteSystemProperty.DISABLE_GENERATE_REL_DATA_TYPE_DIGEST_STRING`. From 481cd2a2322faede6eab77e0017e2c731ea40287 Mon Sep 17 00:00:00 2001 From: Thomas Rebele Date: Mon, 23 Feb 2026 23:16:10 +0100 Subject: [PATCH 164/562] [CALCITE-5832] CyclicMetadataException thrown in complex JOIN --- .../apache/calcite/test/JdbcAdapterTest.java | 115 ++++++++++++++++++ 1 file changed, 115 insertions(+) diff --git a/core/src/test/java/org/apache/calcite/test/JdbcAdapterTest.java b/core/src/test/java/org/apache/calcite/test/JdbcAdapterTest.java index 989aac3604e3..98c8edff3fb9 100644 --- a/core/src/test/java/org/apache/calcite/test/JdbcAdapterTest.java +++ b/core/src/test/java/org/apache/calcite/test/JdbcAdapterTest.java @@ -36,6 +36,7 @@ import java.sql.Connection; import java.sql.DriverManager; +import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; @@ -1555,6 +1556,120 @@ private LockWrapper exclusiveCleanDb(Connection c) throws SQLException { .returnsCount(4); } + /** + * Test case for + * [CALCITE-5832] + * CyclicMetadataException thrown in complex JOIN. */ + @Test void testJdbcCyclicMetadata() throws Exception { + final String url = MultiJdbcSchemaJoinTest.TempDb.INSTANCE.getUrl(); + Connection baseConnection = DriverManager.getConnection(url); + Statement baseStmt = baseConnection.createStatement(); + baseStmt.execute("CREATE TABLE T1 (\n" + + "\"contentViewsCount\" INTEGER,\n" + + "\"isExpired\" BOOLEAN,\n" + + "\"metadataPreviewUrl\" VARCHAR(100),\n" + + "\"format\" VARCHAR(100),\n" + + "\"description\" VARCHAR(100),\n" + + "\"language\" VARCHAR(100),\n" + + "\"assetTitle\" VARCHAR(100),\n" + + "\"assetType\" VARCHAR(100),\n" + + "\"contentType\" VARCHAR(100),\n" + + "\"doi\" VARCHAR(100),\n" + + "\"crmBpn\" VARCHAR(100),\n" + + "PRIMARY KEY(\"doi\"))"); + baseStmt.execute("CREATE TABLE T2 (\n" + + "\"doi\" VARCHAR(100),\n" + + "\"industry\" VARCHAR(100),\n" + + "PRIMARY KEY(\"doi\", \"industry\"))"); + baseStmt.execute("CREATE TABLE T3 (\n" + + "\"semaphoreId\" VARCHAR(100),\n" + + "\"name\" VARCHAR(100),\n" + + "\"industryId\" VARCHAR(100),\n" + + "PRIMARY KEY(\"semaphoreId\"))"); + baseStmt.execute("CREATE TABLE T4 (\n" + + "\"contentViewsCount\" INTEGER,\n" + + "\"CRM_Account_ID\" VARCHAR(100),\n" + + "\"CRM_Account_Name\" VARCHAR(100),\n" + + "PRIMARY KEY(\"CRM_Account_ID\"))"); + baseStmt.close(); + baseConnection.commit(); + + Properties info = new Properties(); + info.put("model", + "inline:" + + "{\n" + + " version: '1.0',\n" + + " defaultSchema: 'BASEJDBC',\n" + + " schemas: [\n" + + " {\n" + + " type: 'jdbc',\n" + + " name: 'BASEJDBC',\n" + + " jdbcDriver: '" + jdbcDriver.class.getName() + "',\n" + + " jdbcUrl: '" + url + "',\n" + + " jdbcCatalog: null,\n" + + " jdbcSchema: null\n" + + " }\n" + + " ]\n" + + "}"); + + final Connection calciteConnection = + DriverManager.getConnection("jdbc:calcite:", info); + final PreparedStatement preparedStatement = calciteConnection + .prepareStatement("SELECT \"_metadata.status\", \"doi\", \"industry.title\", " + + "\"crm_account.crm_account_name\", \"assettitle\", \"description\", \"assettype\", " + + "\"format\", \"contentviewscount\", \"metadatapreviewurl\", \"language\", " + + "\"contenttype\", \"isexpired\" FROM (select\n" + + " \"A\".\"contentViewsCount\" \"contentviewscount\",\n" + + " \"A\".\"isExpired\" \"isexpired\",\n" + + " \"A\".\"metadataPreviewUrl\" \"metadatapreviewurl\",\n" + + " \"A\".\"format\" \"format\",\n" + + " \"A\".\"description\" \"description\",\n" + + " \"A\".\"language\" \"language\",\n" + + " \"A\".\"assetTitle\" \"assettitle\",\n" + + " \"A\".\"assetType\" \"assettype\",\n" + + " \"A\".\"contentType\" \"contenttype\",\n" + + " \"A\".\"doi\" \"doi\",\n" + + " null \"_metadata.status\",\n" + + " \"D\".\"industry.title\" \"industry.title\",\n" + + " \"F\".\"crm_account.crm_account_name\" \"crm_account.crm_account_name\"\n" + + "from \"T1\" \"A\"\n" + + " left outer join \"T2\" \"B\"\n" + + " on \"A\".\"doi\" = \"B\".\"doi\"\n" + + " left outer join (\n" + + " select\n" + + " \"C\".\"semaphoreId\" \"industry.semaphoreId\",\n" + + " \"C\".\"name\" \"industry.title\"\n" + + " from \"T3\" \"C\"\n" + + " ) \"D\"\n" + + " on \"B\".\"industry\" = \"D\".\"industry.semaphoreId\"\n" + + " left outer join (\n" + + " select\n" + + " \"E\".\"CRM_Account_ID\" \"crm_account.CRM_Account_ID\",\n" + + " \"E\".\"CRM_Account_Name\" \"crm_account.crm_account_name\"\n" + + " from \"T4\" \"E\"\n" + + " ) \"F\"\n" + + " on \"A\".\"crmBpn\" = \"F\".\"crm_account" + + ".CRM_Account_ID\")\n" + + "WHERE (\"isexpired\" = ?)\n" + + "AND (\"language\" IN (?, ?))\n" + + "AND (\"contenttype\" IN (?, ?))\n" + + "AND (\"doi\" IN (?))\n" + + "ORDER BY \"doi\" ASC\n" + + "LIMIT 500 OFFSET 0"); + preparedStatement.setBoolean(1, false); + preparedStatement.setString(2, "en"); + preparedStatement.setString(3, "de"); + preparedStatement.setString(4, "text/html"); + preparedStatement.setString(5, "text/plain"); + preparedStatement.setString(6, ""); + ResultSet rs = preparedStatement.executeQuery(); + + assertThat(rs.next(), is(false)); + + rs.close(); + calciteConnection.close(); + } + /** Acquires a lock, and releases it when closed. */ static class LockWrapper implements AutoCloseable { private final Lock lock; From 0f165d62487a066200dc69d72c0e5508d8959ebe Mon Sep 17 00:00:00 2001 From: Silun Date: Wed, 25 Feb 2026 14:23:41 +0800 Subject: [PATCH 165/562] [CALCITE-7386] An error occurred while using TopDownGeneralDecorrelator to process the aggregate(col) filter --- .../org/apache/calcite/test/CoreQuidemTest2.java | 14 -------------- core/src/test/resources/sql/measure.iq | 2 ++ 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/core/src/test/java/org/apache/calcite/test/CoreQuidemTest2.java b/core/src/test/java/org/apache/calcite/test/CoreQuidemTest2.java index 0a55d7764b96..46bb7895da2e 100644 --- a/core/src/test/java/org/apache/calcite/test/CoreQuidemTest2.java +++ b/core/src/test/java/org/apache/calcite/test/CoreQuidemTest2.java @@ -18,10 +18,6 @@ import org.apache.calcite.config.CalciteConnectionProperty; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; - /** * Test that runs Quidem files with the top-down decorrelator enabled. */ @@ -39,16 +35,6 @@ public static void main(String[] args) throws Exception { } } - @Override protected Collection data() { - final List paths = new ArrayList<>(super.data()); - // These remove operations are temporary and will be deleted - // once the new decorrelator can adapt to all scenarios. - - // TODO: Support measure - paths.remove("sql/measure.iq"); - return paths; - } - @Override protected CalciteAssert.AssertThat customize(CalciteAssert.AssertThat assertThat) { return super.customize(assertThat) .with(CalciteConnectionProperty.TOPDOWN_GENERAL_DECORRELATION_ENABLED, true); diff --git a/core/src/test/resources/sql/measure.iq b/core/src/test/resources/sql/measure.iq index aa8ee999fd77..624041d090b4 100644 --- a/core/src/test/resources/sql/measure.iq +++ b/core/src/test/resources/sql/measure.iq @@ -667,6 +667,7 @@ group by deptno, deptno2; !ok +!if (use_old_decorr) { # Measure with FILTER select job, c, @@ -689,6 +690,7 @@ group by job; (3 rows) !ok +!} !if (false) { # Null values in GROUP BY From e76240820250dcd72f7e636091ec2ed4f22e3feb Mon Sep 17 00:00:00 2001 From: iwanttobepowerful <745778074@qq.com> Date: Wed, 25 Feb 2026 11:32:38 +0800 Subject: [PATCH 166/562] [CALCITE-7423] Setop subquery without correlated variables triggers NullPointerException during decorrelation --- .../calcite/sql2rel/RelDecorrelator.java | 4 +++ core/src/test/resources/sql/sub-query.iq | 33 +++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java b/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java index 45596c1d7d28..c38268fb4d39 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java @@ -1258,6 +1258,10 @@ private static void shiftMapping(Map mapping, int startIndex, } final List corVarList = collectExternalCorVars(rel); + if (corVarList.isEmpty()) { + return decorrelateRel((RelNode) rel, true, parentPropagatesNullValues); + } + final NavigableMap valueGenCorDefOutputs = new TreeMap<>(); final RelNode valueGen = requireNonNull(createValueGenerator(corVarList, 0, valueGenCorDefOutputs)); diff --git a/core/src/test/resources/sql/sub-query.iq b/core/src/test/resources/sql/sub-query.iq index 2b6e459cdacd..2c47ab5d0538 100644 --- a/core/src/test/resources/sql/sub-query.iq +++ b/core/src/test/resources/sql/sub-query.iq @@ -9041,5 +9041,38 @@ WHERE e.ename NOT IN ( +-------+--------+-----------+------+------------+---------+---------+--------+ (14 rows) +!ok + +select empno +from emp e +where exists ( + select 1 + from ( + select deptno from dept + union all + select deptno from dept + ) u + where u.deptno = e.deptno +); ++-------+ +| EMPNO | ++-------+ +| 7369 | +| 7499 | +| 7521 | +| 7566 | +| 7654 | +| 7698 | +| 7782 | +| 7788 | +| 7839 | +| 7844 | +| 7876 | +| 7900 | +| 7902 | +| 7934 | ++-------+ +(14 rows) + !ok # End sub-query.iq From 99a597c1a599561a2b365df46cf687bae4a35849 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 20 Feb 2026 00:25:00 +0000 Subject: [PATCH 167/562] Bump nokogiri from 1.18.9 to 1.19.1 in /site Bumps [nokogiri](https://github.com/sparklemotion/nokogiri) from 1.18.9 to 1.19.1. - [Release notes](https://github.com/sparklemotion/nokogiri/releases) - [Changelog](https://github.com/sparklemotion/nokogiri/blob/main/CHANGELOG.md) - [Commits](https://github.com/sparklemotion/nokogiri/compare/v1.18.9...v1.19.1) --- updated-dependencies: - dependency-name: nokogiri dependency-version: 1.19.1 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- site/Gemfile | 2 +- site/Gemfile.lock | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/site/Gemfile b/site/Gemfile index 813430658d7a..ba74be552f09 100644 --- a/site/Gemfile +++ b/site/Gemfile @@ -16,7 +16,7 @@ source 'https://rubygems.org' gem 'jekyll', '~>4' gem "webrick", "~> 1.9.1" -gem "nokogiri", "~> 1.18.9" +gem "nokogiri", "~> 1.19.1" gem "csv", "~> 3.3.2" gem "base64", "~> 0.2.0" diff --git a/site/Gemfile.lock b/site/Gemfile.lock index fc2a68319ac6..bc0e0b41efa5 100644 --- a/site/Gemfile.lock +++ b/site/Gemfile.lock @@ -74,21 +74,21 @@ GEM rb-fsevent (~> 0.10, >= 0.10.3) rb-inotify (~> 0.9, >= 0.9.10) mercenary (0.4.0) - nokogiri (1.18.9-aarch64-linux-gnu) + nokogiri (1.19.1-aarch64-linux-gnu) racc (~> 1.4) - nokogiri (1.18.9-aarch64-linux-musl) + nokogiri (1.19.1-aarch64-linux-musl) racc (~> 1.4) - nokogiri (1.18.9-arm-linux-gnu) + nokogiri (1.19.1-arm-linux-gnu) racc (~> 1.4) - nokogiri (1.18.9-arm-linux-musl) + nokogiri (1.19.1-arm-linux-musl) racc (~> 1.4) - nokogiri (1.18.9-arm64-darwin) + nokogiri (1.19.1-arm64-darwin) racc (~> 1.4) - nokogiri (1.18.9-x86_64-darwin) + nokogiri (1.19.1-x86_64-darwin) racc (~> 1.4) - nokogiri (1.18.9-x86_64-linux-gnu) + nokogiri (1.19.1-x86_64-linux-gnu) racc (~> 1.4) - nokogiri (1.18.9-x86_64-linux-musl) + nokogiri (1.19.1-x86_64-linux-musl) racc (~> 1.4) pathutil (0.16.2) forwardable-extended (~> 2.6) @@ -140,7 +140,7 @@ DEPENDENCIES csv (~> 3.3.2) jekyll (~> 4) jekyll-redirect-from - nokogiri (~> 1.18.9) + nokogiri (~> 1.19.1) webrick (~> 1.9.1) BUNDLED WITH From e08179b97028f728207e328241e3e77861b1130a Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Thu, 26 Feb 2026 14:00:36 +0800 Subject: [PATCH 168/562] Included cases for [CALCITE-6828] --- core/src/test/resources/sql/misc.iq | 5 ----- core/src/test/resources/sql/operator.iq | 4 +--- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/core/src/test/resources/sql/misc.iq b/core/src/test/resources/sql/misc.iq index aa73ca9a08dd..05697b81185e 100644 --- a/core/src/test/resources/sql/misc.iq +++ b/core/src/test/resources/sql/misc.iq @@ -2143,7 +2143,6 @@ EnumerableAggregate(group=[{}], C=[COUNT()]) EnumerableValues(tuples=[[]]) !plan -!if (fixed.calcite6828) { # [CALCITE-1659] Simplifying CAST('YYYY-MM-DD hh:mm:ss.SSS' as TIMESTAMP) # should round the sub-second fraction select TIMESTAMP '2016-02-26 19:06:00.123456789', @@ -2165,9 +2164,6 @@ select TIMESTAMP '2016-02-26 19:06:00.123456789', EnumerableValues(tuples=[[{ 2016-02-26 19:06:00.123, 2016-02-26 19:06:00, 2016-02-26 19:06:00, 2016-02-26 19:06:00.1, 2016-02-26 19:06:00.12, 2016-02-26 19:06:00.123, 2016-02-26 19:06:00.123, 2016-02-26 19:06:00.123 }]]) !plan -!} - -!if (fixed.calcite6828) { # [CALCITE-1664] CAST('' as TIMESTAMP) adds part of sub-second fraction to the value select TIMESTAMP '2016-02-26 19:06:00.12345678', @@ -2184,7 +2180,6 @@ select !ok -!} # TIMESTAMPDIFF with 'flag' literal as time unit argument SELECT TIMESTAMPDIFF(quarter, TIMESTAMP '2008-12-25', TIMESTAMP '2008-09-25'); +--------+ diff --git a/core/src/test/resources/sql/operator.iq b/core/src/test/resources/sql/operator.iq index c0c9d0b42289..33731a08c4d2 100644 --- a/core/src/test/resources/sql/operator.iq +++ b/core/src/test/resources/sql/operator.iq @@ -276,7 +276,6 @@ order by 1,2; !ok -!if (fixed.calcite6828) { # FLOOR and CEIL of TIME select v, case when b then 'ceil' else 'floor' end as op, @@ -298,9 +297,8 @@ order by 1,2; | 12:34:56.7 | floor | 12:34:56.6 | 12:34:56.6 | 12:34:56.6 | 12:34:56.6 | 12:34:56.7 | 12:00:00.0 | 12:34:00.0 | 12:34:56.0 | +------------+-------+------------+------------+------------+------------+------------+------------+------------+------------+ (2 rows) -!ok -!} +!ok select "T"."X"[1] as x1 from (VALUES (ROW(ROW(3, 7), ROW(4, 8)))) as T(x, y); From 8ef68247cdd457a32554874476bfa042db435e54 Mon Sep 17 00:00:00 2001 From: xuzifu666 Date: Fri, 27 Feb 2026 18:03:57 +0800 Subject: [PATCH 169/562] Typo in index page --- site/community/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site/community/index.md b/site/community/index.md index 76ae5e05c48a..bb9edb7ea27d 100644 --- a/site/community/index.md +++ b/site/community/index.md @@ -105,7 +105,7 @@ At [Calcite Hybrid Meetup](https://www.meetup.com/apache-calcite/events/30562734 ## Optimizing Common Table Expressions in Apache Hive with Calcite At [Calcite Hybrid Meetup](https://www.meetup.com/apache-calcite/events/305627349), February 2025; -[[sumary](https://github.com/zabetak/slides/blob/master/2025/calcite-meetup-feb/optimizing-common-table-expressions-in-apache-hive-with-calcite.md)], +[[summary](https://github.com/zabetak/slides/blob/master/2025/calcite-meetup-feb/optimizing-common-table-expressions-in-apache-hive-with-calcite.md)], [[slides](https://www.slideshare.net/slideshow/optimizing-common-table-expressions-in-apache-hive-with-calcite/276219213)], [[pdf](https://github.com/zabetak/slides/blob/master/2025/calcite-meetup-feb/optimizing-common-table-expressions-in-apache-hive-with-calcite.pdf)], [[video](https://www.youtube.com/watch?v=PHm5vZ1A43I&t=6317s)]. From 2431cf7f91a918b6bcbafef351851c37eb8fcba5 Mon Sep 17 00:00:00 2001 From: Julian Hyde Date: Fri, 27 Feb 2026 22:36:08 -0800 Subject: [PATCH 170/562] [CALCITE-7424] In Lint, support sort specifications LintTest should check that files are sorted. A '// lint: sort' directive lets a file declare its own sort requirements. Add Sort and SortConsumer classes to parse and enforce the directive. Annotate .mailmap and contributors.yml files with the directive, and remove hard-coded tests that used to keep them sorted. A man page-style specification follows. LINT:SORT DIRECTIVE NAME lint:sort - Specification-based sorting directive for maintaining alphabetically ordered code sections SYNOPSIS // lint: sort [until 'END_PATTERN'] [where 'FILTER_PATTERN'] [erase 'ERASE_PATTERN'] DESCRIPTION The lint:sort directive enforces alphabetical ordering of lines within a specified code section. It extracts sort keys from matching lines, optionally filters and transforms them, then validates alphabetical order. PARAMETERS until 'END_PATTERN' Optional. Regular expression marking the end of the sorted section. Supports '#' placeholder (parent indent, -2 spaces) and '##' (current line indent). where 'FILTER_PATTERN' Optional. Regular expression to select lines for sorting. Only matching lines are checked. Supports '#' and '##' placeholders. If you want to match a literal '#' (as in a bash comment), write '[#]'. erase 'ERASE_PATTERN' Optional. Regular expression to remove from lines before comparing. Useful for ignoring type annotations, modifiers, etc. PLACEHOLDERS # Parent indentation level (current indent minus 2 spaces) ## Current line's indentation level MULTI-LINE SPECIFICATIONS Long directives can span multiple lines by ending each line with '\'. The backslash and newline are removed during parsing. Example: // lint: sort until '#}' \ // where '##private static final' \ // erase 'Applicable[0-4]*' EXAMPLES Sort cases of a switch: switch (x) { // lint: sort until '#}' where '##case ' case A: // some code case B: // some code } Sort Maven dependencies: Sort constants, ignoring the type of those constants // lint: sort until '#}' \ // where '##private static final [^ ]+ [^ ]+ =' \ // erase '##private static final [^ ]+ ' VIOLATIONS Violations report: - File path and line number - The out-of-order line - The line it should precede NOTES - Sorting is case-sensitive - Empty lines and comments between sorted items are preserved - The directive itself is not included in the sorted section - Use specific patterns to avoid false matches (e.g., exact indentation instead of \\s* for nested structures) - Multi-line specs help with readability and avoid formatter issues This feature is based on hydromatic/morel#316. Close apache/calcite#4813 --- .mailmap | 3 + .../org/apache/calcite/test/LintTest.java | 352 ++++++++++++++---- site/_data/contributors.yml | 3 + 3 files changed, 278 insertions(+), 80 deletions(-) diff --git a/.mailmap b/.mailmap index 094cc7b95cac..a0784207f8ad 100644 --- a/.mailmap +++ b/.mailmap @@ -15,6 +15,9 @@ # See the License for the specific language governing permissions and # limitations under the License. # +# The following directive tells LintTest to ensure that this file is sorted: +# // lint: sort where '^[^#]' +# Abhishek Dasgupta Adam Kennedy Alan Jin diff --git a/core/src/test/java/org/apache/calcite/test/LintTest.java b/core/src/test/java/org/apache/calcite/test/LintTest.java index 2d6953d13385..ff012bfa16c0 100644 --- a/core/src/test/java/org/apache/calcite/test/LintTest.java +++ b/core/src/test/java/org/apache/calcite/test/LintTest.java @@ -22,25 +22,16 @@ import org.apache.calcite.util.TestUnsafe; import org.apache.calcite.util.Util; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.JavaType; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLMapper; +import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import org.checkerframework.checker.nullness.qual.Nullable; import org.junit.jupiter.api.Test; -import java.io.BufferedReader; import java.io.File; -import java.io.IOException; import java.io.PrintWriter; import java.io.StringWriter; -import java.nio.file.Path; -import java.nio.file.Paths; import java.util.ArrayList; import java.util.Comparator; import java.util.HashSet; @@ -59,10 +50,10 @@ import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.startsWith; -import static org.junit.jupiter.api.Assertions.fail; import static org.junit.jupiter.api.Assumptions.assumeTrue; import static java.lang.Integer.parseInt; +import static java.util.Objects.requireNonNull; import static java.util.regex.Pattern.compile; /** Various automated checks on the code and git history. */ @@ -71,7 +62,7 @@ class LintTest { * space. */ private static final Pattern CALCITE_PATTERN = compile("^(\\[CALCITE-[0-9]{1,4}][ ]).*"); - private static final Path ROOT_PATH = Paths.get(System.getProperty("gradle.rootDir")); + private static final Pattern PATTERN = compile("^ *(// )?"); private static final String TERMINOLOGY_ERROR_MSG = "Message contains '%s' word; use one of the following instead: %s"; @@ -193,9 +184,63 @@ && isJava(line.filename()), line -> line.state().ulCount++) .add(line -> line.contains(""), line -> line.state().ulCount--) + + // Apply active sort consumer to each line. + .add(line -> line.state().sortConsumer != null, + line -> requireNonNull(line.state().sortConsumer).accept(line)) + + // Start sorting when a "// lint: sort ..." directive is found. + .add(line -> line.contains("// lint: sort") + && !line.source().fileOpt() + .filter(f -> f.getName().equals("LintTest.java")).isPresent(), + line -> { + line.state().sortConsumer = null; + boolean continued = line.line().endsWith("\\"); + if (continued) { + line.state().partialSort = ""; + } else { + line.state().partialSort = null; + Sort sort = Sort.parse(line.line()); + if (sort != null) { + line.state().sortConsumer = new SortConsumer(sort); + } + } + }) + + // Continue accumulating a multi-line sort specification. + .add(line -> line.state().partialSort != null, + line -> { + String thisLine = line.line(); + boolean continued = thisLine.endsWith("\\"); + if (continued) { + thisLine = skipLast(thisLine); + } + final String nextLine; + if (requireNonNull(line.state().partialSort).isEmpty()) { + nextLine = thisLine; + } else { + thisLine = PATTERN.matcher(thisLine).replaceAll(""); + nextLine = line.state().partialSort + thisLine; + } + if (continued) { + line.state().partialSort = nextLine; + } else { + line.state().partialSort = null; + Sort sort = Sort.parse(nextLine); + if (sort != null) { + line.state().sortConsumer = new SortConsumer(sort); + } + } + }) + .build(); } + /** Strips the last character from a string. */ + private static String skipLast(String s) { + return s.substring(0, s.length() - 1); + } + /** Returns whether we are currently in a region where lint rules should not * be applied. */ private static boolean skipping(Puffin.Line line) { @@ -395,7 +440,7 @@ private static final class TermRule { private final Set validTerms; TermRule(String regex, String... validTerms) { - this.termPattern = Pattern.compile(regex, Pattern.CASE_INSENSITIVE); + this.termPattern = compile(regex, Pattern.CASE_INSENSITIVE); this.validTerms = ImmutableSet.copyOf(validTerms); } @@ -497,67 +542,6 @@ private static void checkMessage(String subject, String body, } } - /** Ensures that the {@code contributors.yml} file is sorted by name. */ - @Test void testContributorsFileIsSorted() throws IOException { - final ObjectMapper mapper = new YAMLMapper(); - final File contributorsFile = ROOT_PATH.resolve("site/_data/contributors.yml").toFile(); - JavaType listType = - mapper.getTypeFactory() - .constructCollectionType(List.class, Contributor.class); - List contributors = - mapper.readValue(contributorsFile, listType); - Contributor contributor = - firstOutOfOrder(contributors, - Comparator.comparing(c -> c.name, String.CASE_INSENSITIVE_ORDER)); - if (contributor != null) { - fail("contributor '" + contributor.name + "' is out of order"); - } - } - - /** Ensures that the {@code .mailmap} file is sorted. */ - @Test void testMailmapFile() { - final File mailmapFile = ROOT_PATH.resolve(".mailmap").toFile(); - final List lines = new ArrayList<>(); - forEachLineIn(mailmapFile, line -> { - if (!line.startsWith("#")) { - lines.add(line); - } - }); - String line = firstOutOfOrder(lines, String.CASE_INSENSITIVE_ORDER); - if (line != null) { - fail("line '" + line + "' is out of order"); - } - } - - /** Performs an action for each line in a file. */ - private static void forEachLineIn(File file, Consumer consumer) { - try (BufferedReader r = Util.reader(file)) { - for (;;) { - String line = r.readLine(); - if (line == null) { - break; - } - consumer.accept(line); - } - } catch (IOException e) { - throw Util.throwAsRuntime(e); - } - } - - /** Returns the first element in a list that is out of order, or null if the - * list is sorted. */ - private static @Nullable E firstOutOfOrder(Iterable elements, - Comparator comparator) { - E previous = null; - for (E e : elements) { - if (previous != null && comparator.compare(previous, e) > 0) { - return e; - } - previous = e; - } - return null; - } - /** Warning that code is not as it should be. */ private static class Message { final Source source; @@ -591,6 +575,8 @@ private static class FileState { int javadocEndLine; int blockquoteCount; int ulCount; + @Nullable String partialSort; + @Nullable Consumer> sortConsumer; FileState(GlobalState global) { this.global = global; @@ -605,13 +591,219 @@ public boolean inJavadoc() { } } - /** Contributor element in "contributors.yaml" file. */ - @JsonIgnoreProperties(ignoreUnknown = true) - private static class Contributor { - final String name; + /** Tests the sort specification syntax. */ + @Test void testSort() { + // With "until" and "where": 'case b' arrives after 'case c', 'd'. + checkSortSpec( + "class Test {\n" + + " switch (x) {\n" + + " // lint: sort until '#}' where '##case '\n" + + " case a\n" + + " case c\n" + + " case d\n" + + " case b\n" + + " case e\n" + + " }\n" + + "}\n", + "GuavaCharSource{memory}:7:" + + "Lines must be sorted; ' case b' should be before ' case c'"); + + // Cases after "until" are checked against the same sorted list. + checkSortSpec( + "class Test {\n" + + " switch (x) {\n" + + " // lint: sort until '#}' where '##case '\n" + + " case x\n" + + " case y\n" + + " case z\n" + + " }\n" + + " switch (y) {\n" + + " case a\n" + + " }\n" + + "}\n", + "GuavaCharSource{memory}:9:" + + "Lines must be sorted; ' case a' should be before ' case x'"); + + // Change '#}' to '##}': consumer stops at the same-indent '}', so + // the second switch's cases are not compared. No violations. + checkSortSpec( + "class Test {\n" + + " switch (x) {\n" + + " // lint: sort until '##}' where '##case '\n" + + " case x\n" + + " case y\n" + + " case z\n" + + " }\n" + + " switch (y) {\n" + + " case a\n" + + " }\n" + + "}\n"); + + // Specification has "until", "where" and "erase" clauses. + checkSortSpec( + "class Test {\n" + + " // lint: sort until '#}' where '##A::' erase '^ .*::'\n" + + " A::c\n" + + " A::a\n" + + " A::b\n" + + " }\n" + + "}\n", + "GuavaCharSource{memory}:4:" + + "Lines must be sorted; 'a' should be before 'c'"); + + // Specification spread over multiple lines using '\' continuation. + checkSortSpec( + "class Test {\n" + + " // lint: sort until '#}'\\\n" + + " // where '##A::'\\\n" + + " // erase '^ .*::'\n" + + " A::c\n" + + " A::a\n" + + " A::b\n" + + " }\n" + + "}\n", + "GuavaCharSource{memory}:6:" + + "Lines must be sorted; 'a' should be before 'c'"); + } - @JsonCreator Contributor(@JsonProperty("name") String name) { - this.name = name; + private void checkSortSpec(String code, String... expectedMessages) { + final Puffin.Program program = makeProgram(); + final StringWriter sw = new StringWriter(); + final GlobalState g; + try (PrintWriter pw = new PrintWriter(sw)) { + g = program.execute(Stream.of(Sources.of(code)), pw); + } + assertThat(g.messages.toString(), + is("[" + String.join(", ", expectedMessages) + "]")); + } + + /** Specification for a sort directive, parsed from a comment like + * {@code // lint: sort until 'pattern' where 'filter' erase 'erasePattern'}. + * + *

    Supports indentation placeholders: + *

      + *
    • {@code ##} - the directive line's indentation (as a regex anchor) + *
    • {@code #} - one level up (indent minus 2 spaces) + *
    + */ + private static class Sort { + final @Nullable Pattern until; + final @Nullable Pattern where; + final @Nullable Pattern erase; + + Sort(@Nullable Pattern until, @Nullable Pattern where, @Nullable Pattern erase) { + this.until = until; + this.where = where; + this.erase = erase; + } + + /** Parses a sort directive from a line like + * {@code // lint: sort until 'X' where 'Y' erase 'Z'}. + * Returns null if parsing fails. */ + static @Nullable Sort parse(String line) { + int sortIndex = line.indexOf("lint: sort"); + if (sortIndex < 0) { + return null; + } + final String spec = + line.substring(sortIndex + "lint: sort".length()).trim(); + int indent = 0; + while (indent < line.length() && line.charAt(indent) == ' ') { + indent++; + } + Pattern until = extractPattern(spec, "until", indent); + Pattern where = extractPattern(spec, "where", indent); + Pattern erase = extractPattern(spec, "erase", indent); + return new Sort(until, where, erase); + } + + /** Extracts and compiles the pattern from a clause like + * {@code until 'pattern'}. Returns null if the keyword is absent or + * the pattern is malformed. */ + private static @Nullable Pattern extractPattern( + String spec, String keyword, int indent) { + int keywordIndex = spec.indexOf(keyword); + if (keywordIndex < 0) { + return null; + } + final String rest = + spec.substring(keywordIndex + keyword.length()).trim(); + if (rest.isEmpty() || rest.charAt(0) != '\'') { + return null; + } + int endQuote = rest.indexOf('\'', 1); + if (endQuote < 0) { + return null; + } + String pattern = rest.substring(1, endQuote); + if (!pattern.contains("[#")) { + pattern = pattern.replace("##", "^" + Strings.repeat(" ", indent)); + pattern = + pattern.replace("#", "^" + Strings.repeat(" ", Math.max(0, indent - 2))); + } + try { + return compile(pattern); + } catch (Exception e) { + return null; + } + } + } + + /** Checks that lines in a sorted region are in order. */ + private static class SortConsumer + implements Consumer> { + final Sort sort; + final Comparator comparator = String.CASE_INSENSITIVE_ORDER; + final List lines = new ArrayList<>(); + boolean done = false; + + SortConsumer(Sort sort) { + this.sort = sort; + } + + @Override public void accept(Puffin.Line line) { + if (done) { + return; + } + final String thisLine = line.line(); + + // End of sorted region. + if (sort.until != null && sort.until.matcher(thisLine).find()) { + done = true; + line.state().sortConsumer = null; + return; + } + + // If a "where" filter is present, skip non-matching lines. + if (sort.where != null && !sort.where.matcher(thisLine).find()) { + return; + } + + // Apply the "erase" pattern before comparing. + String compareLine = thisLine; + if (sort.erase != null) { + compareLine = sort.erase.matcher(thisLine).replaceAll(""); + } + + addLine(line, compareLine); + } + + private void addLine(Puffin.Line line, + String thisLine) { + if (!lines.isEmpty()) { + final String prevLine = lines.get(lines.size() - 1); + if (comparator.compare(prevLine, thisLine) > 0) { + final String earlierLine = + Util.filter(lines, s -> comparator.compare(s, thisLine) > 0) + .iterator().next(); + line.state().message( + String.format(Locale.ROOT, + "Lines must be sorted; '%s' should be before '%s'", + thisLine, earlierLine), + line); + } + } + lines.add(thisLine); } } } diff --git a/site/_data/contributors.yml b/site/_data/contributors.yml index f8347e026a59..8a075acd9445 100644 --- a/site/_data/contributors.yml +++ b/site/_data/contributors.yml @@ -17,7 +17,10 @@ # Database of contributors to Apache Calcite. # Pages such as developer.md use this data. +# # List must be sorted by first name, last name. +# The following directive tells LintTest to check: +# // lint: sort where ' name:' erase '^.*name:' # - name: Alan Gates emeritus: 2018/05/04 From 09869c9db4ed47012292aab2b0bce6e197e46211 Mon Sep 17 00:00:00 2001 From: Stamatis Zampetakis Date: Thu, 19 Feb 2026 11:30:01 +0100 Subject: [PATCH 171/562] [CALCITE-7362] Add rule to transform WHERE clauses into filtered aggregates --- ...ggregateFilterToFilteredAggregateRule.java | 105 +++++++++++ .../apache/calcite/rel/rules/CoreRules.java | 5 + ...gateFilterToFilteredAggregateRuleTest.java | 120 ++++++++++++ ...egateFilterToFilteredAggregateRuleTest.xml | 173 ++++++++++++++++++ 4 files changed, 403 insertions(+) create mode 100644 core/src/main/java/org/apache/calcite/rel/rules/AggregateFilterToFilteredAggregateRule.java create mode 100644 core/src/test/java/org/apache/calcite/test/AggregateFilterToFilteredAggregateRuleTest.java create mode 100644 core/src/test/resources/org/apache/calcite/test/AggregateFilterToFilteredAggregateRuleTest.xml diff --git a/core/src/main/java/org/apache/calcite/rel/rules/AggregateFilterToFilteredAggregateRule.java b/core/src/main/java/org/apache/calcite/rel/rules/AggregateFilterToFilteredAggregateRule.java new file mode 100644 index 000000000000..dd82f697622c --- /dev/null +++ b/core/src/main/java/org/apache/calcite/rel/rules/AggregateFilterToFilteredAggregateRule.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.rel.rules; + +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelRule; +import org.apache.calcite.rel.core.Aggregate; +import org.apache.calcite.rel.core.AggregateCall; +import org.apache.calcite.rel.core.Filter; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.tools.RelBuilder; + +import org.immutables.value.Value; + +import java.util.ArrayList; +import java.util.List; + +/** + * Rule that converts an aggregate on top of a filter into a filtered aggregate. + * + *

    Before + * 

    
+ *   SELECT SUM(salary)
+ *   FROM Emp
+ *   WHERE deptno = 10
+ *
    + * + *

    After + * 

    
+ *   SELECT SUM(salary) FILTER (WHERE deptno = 10)
+ *   FROM Emp
+ *
    + * + *

    The transformation is particularly useful in view-based rewriting. + * The removal of the {@code Filter} operators lifts some restrictions when using + * the {@link org.apache.calcite.rel.rules.materialize.MaterializedViewRules}. + * + *

    Filtered aggregates can be transformed to other equivalent forms via other + * transformation rules (e.g., {@link AggregateFilterToCaseRule}). + */ +@Value.Enclosing public class AggregateFilterToFilteredAggregateRule + extends RelRule { + + private AggregateFilterToFilteredAggregateRule(Config config) { + super(config); + } + + @Override public void onMatch(RelOptRuleCall call) { + Aggregate aggregate = call.rel(0); + Filter filter = call.rel(1); + if (!aggregate.getGroupSet().isEmpty()) { + // At the moment we only support the transformation for grand totals, i.e., + // aggregates with no grouping keys. + return; + } + RelBuilder builder = call.builder(); + builder.push(filter.getInput()); + List projects = new ArrayList<>(builder.fields()); + List newAggCalls = new ArrayList<>(); + for (AggregateCall aggCall : aggregate.getAggCallList()) { + if (!aggCall.getAggregation().allowsFilter()) { + return; + } + RexNode condition = filter.getCondition(); + // If the aggregate call has its own filter, combine it with the filter condition. + if (aggCall.hasFilter()) { + condition = builder.and(condition, builder.field(aggCall.filterArg)); + } + int pos = projects.indexOf(condition); + if (pos < 0) { + pos = projects.size(); + projects.add(condition); + } + newAggCalls.add(aggCall.withFilter(pos)); + } + builder.project(projects); + builder.aggregate(builder.groupKey(), newAggCalls); + call.transformTo(builder.build()); + } + + /** Rule configuration. */ + @Value.Immutable public interface Config extends RelRule.Config { + Config DEFAULT = ImmutableAggregateFilterToFilteredAggregateRule.Config.of() + .withOperandSupplier( + a -> a.operand(Aggregate.class).oneInput(f -> f.operand(Filter.class).anyInputs())); + + @Override default AggregateFilterToFilteredAggregateRule toRule() { + return new AggregateFilterToFilteredAggregateRule(this); + } + } +} diff --git a/core/src/main/java/org/apache/calcite/rel/rules/CoreRules.java b/core/src/main/java/org/apache/calcite/rel/rules/CoreRules.java index 21d5c971d641..444c89ccdd70 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/CoreRules.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/CoreRules.java @@ -959,6 +959,11 @@ private CoreRules() {} public static final AggregateFilterToCaseRule AGGREGATE_FILTER_TO_CASE = AggregateFilterToCaseRule.Config.DEFAULT.toRule(); + /** Rule that converts an aggregate on of a filter into a filtered aggregate. */ + public static final AggregateFilterToFilteredAggregateRule + AGGREGATE_FILTER_TO_FILTERED_AGGREGATE = + AggregateFilterToFilteredAggregateRule.Config.DEFAULT.toRule(); + /** Rule that remove duplicate {@link Sort} keys. */ public static final SortRemoveDuplicateKeysRule SORT_REMOVE_DUPLICATE_KEYS = SortRemoveDuplicateKeysRule.Config.DEFAULT.toRule(); diff --git a/core/src/test/java/org/apache/calcite/test/AggregateFilterToFilteredAggregateRuleTest.java b/core/src/test/java/org/apache/calcite/test/AggregateFilterToFilteredAggregateRuleTest.java new file mode 100644 index 000000000000..9e6cd3ea339a --- /dev/null +++ b/core/src/test/java/org/apache/calcite/test/AggregateFilterToFilteredAggregateRuleTest.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.test; + +import org.apache.calcite.plan.RelOptRule; +import org.apache.calcite.plan.hep.HepProgram; +import org.apache.calcite.rel.rules.AggregateFilterToFilteredAggregateRule; +import org.apache.calcite.rel.rules.CoreRules; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.apache.calcite.rel.rules.CoreRules.AGGREGATE_FILTER_TO_FILTERED_AGGREGATE; +import static org.apache.calcite.rel.rules.CoreRules.AGGREGATE_PROJECT_MERGE; +import static org.apache.calcite.rel.rules.CoreRules.PROJECT_FILTER_TRANSPOSE_WHOLE_PROJECT_EXPRESSIONS; + +/** + * Unit tests for {@link AggregateFilterToFilteredAggregateRule}. + * + *

    Relevant tickets: + *

    + */ +class AggregateFilterToFilteredAggregateRuleTest { + + private static RelOptFixture fixture() { + return RelOptFixture.DEFAULT.withDiffRepos( + DiffRepository.lookup(AggregateFilterToFilteredAggregateRuleTest.class)); + } + + private static RelOptFixture sql(String sql) { + return fixture().sql(sql); + } + + @Test void testSingleColumnAggregate() { + String sql = "select sum(sal) from emp where deptno = 10"; + sql(sql).withPreRule(AGGREGATE_PROJECT_MERGE) + .withRule(AGGREGATE_FILTER_TO_FILTERED_AGGREGATE).check(); + } + + @Test void testSingleStarAggregate() { + String sql = "select count(*) from emp where deptno = 10"; + sql(sql).withPreRule(AGGREGATE_PROJECT_MERGE) + .withRule(AGGREGATE_FILTER_TO_FILTERED_AGGREGATE).check(); + } + + @Test void testMultiAggregates() { + String sql = "select sum(sal), min(sal), max(sal), count(*) from emp where deptno = 10"; + sql(sql).withPreRule(AGGREGATE_PROJECT_MERGE) + .withRule(AGGREGATE_FILTER_TO_FILTERED_AGGREGATE).check(); + } + + @Test void testSingleColumnFilteredAggregate() { + String sql = "select sum(sal) filter (where ename = 'Bob') from emp where deptno = 10"; + List preRules = new ArrayList<>(); + preRules.add(AGGREGATE_PROJECT_MERGE); + preRules.add(PROJECT_FILTER_TRANSPOSE_WHOLE_PROJECT_EXPRESSIONS); + sql(sql).withPre(HepProgram.builder().addRuleCollection(preRules).build()) + .withRule(AGGREGATE_FILTER_TO_FILTERED_AGGREGATE, + CoreRules.PROJECT_MERGE).check(); + } + + @Test void testAggregateNoSupportingFilter() { + String sql = "select single_value(sal) from emp where deptno = 10"; + sql(sql).withPreRule(AGGREGATE_PROJECT_MERGE) + .withRule(AGGREGATE_FILTER_TO_FILTERED_AGGREGATE) + .checkUnchanged(); + } + + @Test void testSingleColumnAggregateWithGroupBy() { + String sql = "select sum(sal) from emp where deptno = 10 group by job"; + sql(sql).withPreRule(AGGREGATE_PROJECT_MERGE) + .withRule(AGGREGATE_FILTER_TO_FILTERED_AGGREGATE) + .checkUnchanged(); + } + + @Test void testSingleColumnAggregateWithGroupingSets() { + String sql = + "select sum(sal) from emp where deptno = 10 group by grouping sets ((job), (ename))"; + sql(sql).withPreRule(AGGREGATE_PROJECT_MERGE) + .withRule(AGGREGATE_FILTER_TO_FILTERED_AGGREGATE) + .checkUnchanged(); + } + + @Test void testSingleColumnAggregateWithEmptyGroupBy() { + String sql = "select sum(sal) from emp where deptno = 10 group by ()"; + sql(sql).withPreRule(AGGREGATE_PROJECT_MERGE) + .withRule(AGGREGATE_FILTER_TO_FILTERED_AGGREGATE).check(); + } + + @Test void testSingleColumnAggregateWithEmptyGroupingSets() { + String sql = "select sum(sal) from emp where deptno = 10 group by grouping sets (())"; + sql(sql).withPreRule(AGGREGATE_PROJECT_MERGE) + .withRule(AGGREGATE_FILTER_TO_FILTERED_AGGREGATE).check(); + } + + @AfterAll static void checkActualAndReferenceFiles() { + fixture().diffRepos.checkActualAndReferenceFiles(); + } +} diff --git a/core/src/test/resources/org/apache/calcite/test/AggregateFilterToFilteredAggregateRuleTest.xml b/core/src/test/resources/org/apache/calcite/test/AggregateFilterToFilteredAggregateRuleTest.xml new file mode 100644 index 000000000000..b31112c3f337 --- /dev/null +++ b/core/src/test/resources/org/apache/calcite/test/AggregateFilterToFilteredAggregateRuleTest.xml @@ -0,0 +1,173 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From ddc9bcf56d1059f6ac27d5c4ec2a09b1df427ce1 Mon Sep 17 00:00:00 2001 From: krooswu Date: Wed, 25 Feb 2026 23:49:08 +0800 Subject: [PATCH 172/562] [CALCITE-7343] RelToSqlConverter generate wrong sql when scalar correlated sub-query in Project --- .../rel/rel2sql/RelToSqlConverter.java | 74 +++++++++++++++- .../calcite/rel/rel2sql/SqlImplementor.java | 61 ++++++++++++-- .../rel/rel2sql/RelToSqlConverterTest.java | 84 +++++++++++++++++-- 3 files changed, 203 insertions(+), 16 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java b/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java index 592df51a0b1b..0404fa7bc51c 100644 --- a/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java +++ b/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java @@ -604,14 +604,82 @@ private static boolean selectListRequired(Context context) { return false; } - /** Visits a Project; called by {@link #dispatch} via reflection. */ + /** + * Extracts the table name from a SqlNode if it represents a simple table reference. + * Returns null for complex nodes like subqueries or joins. + * + *

    Examples: + *

      + *
    • "product" → "product"
      • + *
    • "foodmart.product" → "product"
      • + *
    • "SCOTT"."EMP" → "EMP"
      • + *
    • Subquery/Join/Other → null
      • + *
    + * + * @param node The SQL node to examine + * @return The table name if it's a simple identifier, null otherwise + */ + private @Nullable String unqualifiedName(SqlNode node) { + if (node instanceof SqlIdentifier) { + SqlIdentifier id = (SqlIdentifier) node; + // Return the last component (table name) + return id.names.get(id.names.size() - 1); + } + + // All other cases: return null + return null; + } + /** + * Visits a {@link Project} and converts it to a {@link SqlSelect}. + * + *

    If the project defines correlation variables (e.g., via {@code $cor0}), + * this method ensures that the input relation is assigned a stable alias + * (either the natural table name or a synthetic alias like 't'). + * This enables nested correlated subqueries to correctly qualify their + * column references back to this project's scope. + * + *

    For simple table scans, it avoids forcing an explicit 'AS' clause + * to maintain compatibility with DML statements (like UPDATE/DELETE) + * in certain SQL dialects. + */ public Result visit(Project e) { // If the input is a Sort, wrap SELECT is not required. final Result x; + final Set definedHere = e.getVariablesSet(); + boolean pushed = !definedHere.isEmpty(); + + // Visit input node + Result inputResult; if (e.getInput() instanceof Sort) { - x = visitInput(e, 0); + inputResult = visitInput(e, 0); + } else { + inputResult = visitInput(e, 0, Clause.SELECT); + } + + // If this Project defines correlations, fill in alias and force explicit generation + // Resolve the correlation alias using a three-level priority: + // 1. Use the existing alias from inputResult if it's already defined. + // 2. If not, extract the natural table name to maintain DML compatibility. + // 3. Fallback to 't' for anonymous relations (e.g., Joins or Sub-queries). + // + // 't' is safe and standard in Calcite's SqlImplementor because: + // - Anonymous relations in the FROM clause require an alias in most dialects. + // - SQL name shadowing rules ensure that nested sub-queries correctly bind + // to the nearest qualifying 't' in their scope. + if (pushed) { + String alias = inputResult.neededAlias; + if (alias != null) { + x = inputResult.resetAliasForCorrelation(alias, e.getInput().getRowType()); + } else { + alias = unqualifiedName(inputResult.node); + if (alias == null) { + alias = "t"; + } + x = inputResult.resetAliasForCorrelation + (alias, e.getInput().getRowType()); + } } else { - x = visitInput(e, 0, Clause.SELECT); + x = inputResult; } parseCorrelTable(e, x); final Builder builder = x.builder(e); diff --git a/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java b/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java index 6d60d52aa4e0..dec24ca1f24a 100644 --- a/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java +++ b/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java @@ -1910,16 +1910,24 @@ public class Result { private final @Nullable RelNode expectedRel; private final boolean needNew; + /** + * Whether to force explicit alias generation in FROM clause. + * Set to true when this Result is used in a correlation context + * where the table alias must be explicit even if the dialect + * normally supports implicit aliases. + */ + private final boolean forceExplicitAlias; + public Result(SqlNode node, Collection clauses, @Nullable String neededAlias, @Nullable RelDataType neededType, Map aliases) { this(node, clauses, neededAlias, neededType, aliases, false, false, - ImmutableSet.of(), null); + ImmutableSet.of(), null, false); } private Result(SqlNode node, Collection clauses, @Nullable String neededAlias, @Nullable RelDataType neededType, Map aliases, boolean anon, boolean ignoreClauses, Set expectedClauses, - @Nullable RelNode expectedRel) { + @Nullable RelNode expectedRel, boolean forceExplicitAlias) { this.node = node; this.neededAlias = neededAlias; this.neededType = neededType; @@ -1929,6 +1937,7 @@ private Result(SqlNode node, Collection clauses, @Nullable String needed this.ignoreClauses = ignoreClauses; this.expectedClauses = ImmutableSet.copyOf(expectedClauses); this.expectedRel = expectedRel; + this.forceExplicitAlias = forceExplicitAlias; final Set clauses2 = ignoreClauses ? ImmutableSet.of() : expectedClauses; this.needNew = expectedRel != null @@ -2241,9 +2250,23 @@ public SqlSelect subSelect() { * INTERSECT, EXCEPT) remain as is. */ public SqlSelect asSelect() { if (node instanceof SqlSelect) { - return (SqlSelect) node; + SqlSelect select = (SqlSelect) node; + // Check if we need to add explicit alias to FROM clause + if (forceExplicitAlias && neededAlias != null) { + SqlNode from = select.getFrom(); + + // Only add alias if FROM doesn't already have one + if (from != null && from.getKind() != SqlKind.AS) { + SqlNode newFrom = + SqlStdOperatorTable.AS.createCall(POS, from, + new SqlIdentifier(neededAlias, POS)); + select.setFrom(newFrom); + } + } + return select; } - if (!dialect.hasImplicitTableAlias() || hasConflictTableAlias(node)) { + // For non-SELECT nodes, wrap in SELECT * + if (forceExplicitAlias || !dialect.hasImplicitTableAlias() || hasConflictTableAlias(node)) { return wrapSelect(asFrom()); } return wrapSelect(node); @@ -2369,7 +2392,7 @@ public Result resetAlias() { } else { return new Result(node, clauses, neededAlias, neededType, ImmutableMap.of(neededAlias, castNonNull(neededType)), anon, ignoreClauses, - expectedClauses, expectedRel); + expectedClauses, expectedRel, false); } } @@ -2382,14 +2405,36 @@ public Result resetAlias() { public Result resetAlias(String alias, RelDataType type) { return new Result(node, clauses, alias, neededType, ImmutableMap.of(alias, type), anon, ignoreClauses, - expectedClauses, expectedRel); + expectedClauses, expectedRel, false); + } + + /** + * Sets the alias and forces explicit alias generation in FROM clause. + * Used when correlation requires an explicit table alias. + * + * @param alias New alias to use + * @param type Type of the node associated with the alias + * @return New Result with forced explicit alias + */ + public Result resetAliasForCorrelation(String alias, RelDataType type) { + return new Result( + node, + clauses, + alias, + neededType, + ImmutableMap.of(alias, type), + anon, + ignoreClauses, + expectedClauses, + expectedRel, + true); // Force explicit alias } /** Returns a copy of this Result, overriding the value of {@code anon}. */ Result withAnon(boolean anon) { return anon == this.anon ? this : new Result(node, clauses, neededAlias, neededType, aliases, anon, - ignoreClauses, expectedClauses, expectedRel); + ignoreClauses, expectedClauses, expectedRel, false); } /** Returns a copy of this Result, overriding the value of @@ -2401,7 +2446,7 @@ Result withExpectedClauses(boolean ignoreClauses, && expectedRel == this.expectedRel ? this : new Result(node, clauses, neededAlias, neededType, aliases, anon, - ignoreClauses, ImmutableSet.copyOf(expectedClauses), expectedRel); + ignoreClauses, ImmutableSet.copyOf(expectedClauses), expectedRel, false); } } diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java index a98f7fd1f8ff..4250181e6636 100644 --- a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java +++ b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java @@ -11107,7 +11107,7 @@ private void checkLiteral2(String expression, String expected) { + "FROM \"scott\".\"EMP\"\n" + "GROUP BY \"DEPTNO\"\n" + "HAVING \"DEPTNO\" = \"DEPT\".\"DEPTNO\") AS \"$f2\"\n" - + "FROM \"scott\".\"DEPT\""; + + "FROM \"scott\".\"DEPT\" AS \"DEPT\""; relFn(relFn).ok(expected); } @@ -11256,7 +11256,7 @@ private void checkLiteral2(String expression, String expected) { final String expected = "SELECT (SELECT COUNT(*)\n" + "FROM \"foodmart\".\"employee\"\n" + "WHERE \"product\".\"product_id\" >= 2), 3\n" - + "FROM \"foodmart\".\"product\""; + + "FROM \"foodmart\".\"product\" AS \"product\""; sql(sql).ok(expected); } @@ -11651,9 +11651,10 @@ public Sql schema(CalciteAssert.SchemaSpec schemaSpec) { } } - /** Test case for + /** Test cases for * [CALCITE-7279] - * ClickHouse dialect should wrap nested JOINs with explicit aliasing. */ + * ClickHouse dialect should wrap nested JOINs with explicit aliasing. + */ @Test void testClickHouseNestedJoin() { final String query = "SELECT e.empno, j.dname, j.loc\n" + "FROM emp e\n" @@ -11757,8 +11758,81 @@ public Sql schema(CalciteAssert.SchemaSpec schemaSpec) { sql(query) .schema(CalciteAssert.SchemaSpec.JDBC_SCOTT) - .withMysql() + .withMysql(); + } + + /** Test case for + * [CALCITE-7343] + RelToSqlConverter generate wrong sql when scalar correlated sub-query in Project . */ + @Test void testProjectScalarSubquery() { + final String sql = "SELECT \"EMPNO\",\n" + + " (SELECT COUNT(*) AS \"c\" FROM \"EMP\" WHERE \"MGR\" < \"m\".\"MGR\") AS \"$f1\"\n" + + "FROM \"EMP\" AS \"m\"\n" + + "WHERE \"SAL\" > 10"; + + final String expected = "SELECT \"EMPNO\", " + + "(SELECT COUNT(*) AS \"c\"\n" + + "FROM \"SCOTT\".\"EMP\"\n" + + "WHERE \"MGR\" < \"t\".\"MGR\") AS \"$f1\"\n" + + "FROM \"SCOTT\".\"EMP\" AS \"t\"\n" + + "WHERE CAST(\"SAL\" AS DECIMAL(12, 2)) > 10.00"; + + sql(sql) + .schema(CalciteAssert.SchemaSpec.JDBC_SCOTT) + .withCalcite().ok(expected); + } + @Test void testProjectDeeplyNestedScalarSubquery() { + final String sql = "SELECT \"EMPNO\",\n" + + " (SELECT MAX((SELECT COUNT(*) FROM \"DEPT\" " + + "WHERE \"DEPTNO\" = \"m\".\"DEPTNO\"))\n" + + " FROM \"DEPT\" WHERE \"LOC\" = 'NEW YORK') AS \"$f1\"\n" + + "FROM \"EMP\" AS \"m\""; + + final String expected = "SELECT \"EMPNO\", " + + "(SELECT MAX((SELECT COUNT(*)\n" + + "FROM \"SCOTT\".\"DEPT\"\n" + + "WHERE \"DEPTNO\" = \"EMP\".\"DEPTNO\"))\n" + + "FROM \"SCOTT\".\"DEPT\"\n" + + "WHERE \"LOC\" = 'NEW YORK') AS \"$f1\"\n" + + "FROM \"SCOTT\".\"EMP\" AS \"EMP\""; + + sql(sql) + .schema(CalciteAssert.SchemaSpec.JDBC_SCOTT) + .withCalcite() .ok(expected); } + @Test void testMultiLayerProjectCorrelation() { + final String sql = "SELECT \"EMPNO\" + 1, \n" + + " (SELECT \"DNAME\" FROM \"DEPT\" WHERE \"DEPTNO\" = \"sub\".\"DEPTNO\")\n" + + "FROM (SELECT * FROM \"EMP\") AS \"sub\""; + + final String expected = "SELECT \"EMPNO\" + 1, " + + "(SELECT \"DNAME\"\nFROM \"SCOTT\".\"DEPT\"\n" + + "WHERE \"DEPTNO\" = \"t\".\"DEPTNO\")\n" + + "FROM \"SCOTT\".\"EMP\" AS \"t\""; + + sql(sql).schema(CalciteAssert.SchemaSpec.JDBC_SCOTT).ok(expected); + } + + @Test void testMultiLevelCrossReference() { + final String sql = "SELECT \"e\".\"ENAME\",\n" + + " (SELECT COUNT(*)\n" + + " FROM (SELECT \"d\".\"DEPTNO\", \"d\".\"DNAME\" FROM \"DEPT\" \"d\" " + + " JOIN \"BONUS\" \"b\" ON \"d\".\"DEPTNO\" = \"e\".\"DEPTNO\") AS \"mid\"\n" + + " WHERE \"mid\".\"DNAME\" = (SELECT \"DNAME\" FROM \"DEPT\" " + + " WHERE \"DEPTNO\" = \"e\".\"DEPTNO\" " + + " AND \"LOC\" = \"mid\".\"DNAME\"))\n" + + "FROM \"EMP\" AS \"e\""; + final String expected = "SELECT \"ENAME\", (SELECT COUNT(*)\n" + + "FROM (SELECT \"DEPT\".\"DEPTNO\", \"DEPT\".\"DNAME\"\n" + + "FROM \"SCOTT\".\"DEPT\"\n" + + "INNER JOIN \"SCOTT\".\"BONUS\" ON \"DEPT\".\"DEPTNO\" = \"EMP\".\"DEPTNO\") AS \"t\"\n" + + "WHERE \"DNAME\" = (SELECT \"DNAME\"\n" + + "FROM \"SCOTT\".\"DEPT\"\n" + + "WHERE \"DEPTNO\" = \"EMP\".\"DEPTNO\" AND \"LOC\" = \"t\".\"DNAME\"))\n" + + "FROM \"SCOTT\".\"EMP\" AS \"EMP\""; + + sql(sql).schema(CalciteAssert.SchemaSpec.JDBC_SCOTT).ok(expected); + } } From 702afde094c7662972ea3e9704a0a5f7bffe6d01 Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Wed, 4 Mar 2026 21:55:26 +0800 Subject: [PATCH 173/562] [CALCITE-7427] Query with "ORDER BY NULL" throws "NoSuchMethodException: compareNullsLast" --- .../adapter/enumerable/PhysTypeImpl.java | 8 ++++++ core/src/test/resources/sql/sort.iq | 25 +++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/PhysTypeImpl.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/PhysTypeImpl.java index e3fdd4d36656..a44d4f69df70 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/PhysTypeImpl.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/PhysTypeImpl.java @@ -319,6 +319,10 @@ static PhysType of( body.add(Expressions.declare(mod, parameterC, null)); for (RelFieldCollation collation : collations) { final int index = collation.getFieldIndex(); + // NULL literal (Void) is always null; comparing null == null yields 0, skip. + if (fieldClass(index) == Void.class) { + continue; + } final RelDataType fieldType = rowType.getFieldList().get(index).getType(); final Expression fieldComparator = generateCollatorExpression(fieldType.getCollation()); Expression arg0 = fieldReference(parameterV0, index); @@ -443,6 +447,10 @@ private Expression generateComparator(RelCollation collation, body.add(Expressions.declare(mod, parameterC, null)); for (RelFieldCollation fieldCollation : collation.getFieldCollations()) { final int index = fieldCollation.getFieldIndex(); + // NULL literal (Void) is always null; comparing null == null yields 0, skip. + if (fieldClass(index) == Void.class) { + continue; + } final RelDataType fieldType = rowType.getFieldList().get(index).getType(); final Expression fieldComparator = generateCollatorExpression(fieldType.getCollation()); Expression arg0 = fieldReference(parameterV0, index); diff --git a/core/src/test/resources/sql/sort.iq b/core/src/test/resources/sql/sort.iq index b9e0412ff989..b93be76837bf 100644 --- a/core/src/test/resources/sql/sort.iq +++ b/core/src/test/resources/sql/sort.iq @@ -508,4 +508,29 @@ order by arr desc nulls last; !ok +# [CALCITE-7427] Query with "ORDER BY NULL" throws "NoSuchMethodException: compareNullsLast" +!use scott +SELECT * FROM emp ORDER BY deptno, null, empno; ++-------+--------+-----------+------+------------+---------+---------+--------+ +| EMPNO | ENAME | JOB | MGR | HIREDATE | SAL | COMM | DEPTNO | ++-------+--------+-----------+------+------------+---------+---------+--------+ +| 7782 | CLARK | MANAGER | 7839 | 1981-06-09 | 2450.00 | | 10 | +| 7839 | KING | PRESIDENT | | 1981-11-17 | 5000.00 | | 10 | +| 7934 | MILLER | CLERK | 7782 | 1982-01-23 | 1300.00 | | 10 | +| 7369 | SMITH | CLERK | 7902 | 1980-12-17 | 800.00 | | 20 | +| 7566 | JONES | MANAGER | 7839 | 1981-02-04 | 2975.00 | | 20 | +| 7788 | SCOTT | ANALYST | 7566 | 1987-04-19 | 3000.00 | | 20 | +| 7876 | ADAMS | CLERK | 7788 | 1987-05-23 | 1100.00 | | 20 | +| 7902 | FORD | ANALYST | 7566 | 1981-12-03 | 3000.00 | | 20 | +| 7499 | ALLEN | SALESMAN | 7698 | 1981-02-20 | 1600.00 | 300.00 | 30 | +| 7521 | WARD | SALESMAN | 7698 | 1981-02-22 | 1250.00 | 500.00 | 30 | +| 7654 | MARTIN | SALESMAN | 7698 | 1981-09-28 | 1250.00 | 1400.00 | 30 | +| 7698 | BLAKE | MANAGER | 7839 | 1981-01-05 | 2850.00 | | 30 | +| 7844 | TURNER | SALESMAN | 7698 | 1981-09-08 | 1500.00 | 0.00 | 30 | +| 7900 | JAMES | CLERK | 7698 | 1981-12-03 | 950.00 | | 30 | ++-------+--------+-----------+------+------------+---------+---------+--------+ +(14 rows) + +!ok + # End sort.iq From 162a04fafd094fa59e9baa092df650c5eec28dc3 Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Wed, 4 Mar 2026 22:38:38 +0800 Subject: [PATCH 174/562] [CALCITE-7429] Query with MINUS fails with "Unable to implement EnumerableMinus(all=[false])" --- .../enumerable/EnumerableIntersect.java | 4 --- .../adapter/enumerable/EnumerableMinus.java | 4 --- core/src/test/resources/sql/set-op.iq | 31 +++++++++++++++++++ 3 files changed, 31 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableIntersect.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableIntersect.java index e20a8ad765ea..01f9ab020698 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableIntersect.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableIntersect.java @@ -64,10 +64,6 @@ public EnumerableIntersect(RelOptCluster cluster, RelTraitSet traitSet, .appendIfNotNull(result.physType.comparer()) .append(Expressions.constant(all))); } - - // Once the first input has chosen its format, ask for the same for - // other inputs. - pref = pref.of(result.format); } builder.add(requireNonNull(intersectExp, "intersectExp")); diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMinus.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMinus.java index 33952937d1d3..fcb17cde742b 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMinus.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMinus.java @@ -65,10 +65,6 @@ public EnumerableMinus(RelOptCluster cluster, RelTraitSet traitSet, .appendIfNotNull(result.physType.comparer()) .append(Expressions.constant(all))); } - - // Once the first input has chosen its format, ask for the same for - // other inputs. - pref = pref.of(result.format); } builder.add( diff --git a/core/src/test/resources/sql/set-op.iq b/core/src/test/resources/sql/set-op.iq index 4b3917ad309d..1564650b1fa7 100644 --- a/core/src/test/resources/sql/set-op.iq +++ b/core/src/test/resources/sql/set-op.iq @@ -295,4 +295,35 @@ EnumerableCalc(expr#0=[{inputs}], expr#1=[3], proj#0..1=[{exprs}]) EnumerableValues(tuples=[[{ 30, 3 }, { 30, 3 }]]) !plan +# [CALCITE-7429] Query with MINUS fails with "Unable to implement EnumerableMinus(all=[false])" +!use scott +SELECT deptno FROM dept WHERE deptno > 12 +EXCEPT +SELECT deptno FROM emp e1 WHERE EXISTS ( + SELECT 1 FROM emp e2 + WHERE e2.comm = e1.comm); ++--------+ +| DEPTNO | ++--------+ +| 20 | +| 40 | ++--------+ +(2 rows) + +!ok + +SELECT deptno FROM dept WHERE deptno > 12 +INTERSECT +SELECT deptno FROM emp e1 WHERE EXISTS ( + SELECT 1 FROM emp e2 + WHERE e2.comm = e1.comm); ++--------+ +| DEPTNO | ++--------+ +| 30 | ++--------+ +(1 row) + +!ok + # End set-op.iq From 1afde6a36667abf0015e87fdf2436c9b7ebe935d Mon Sep 17 00:00:00 2001 From: wforget <643348094@qq.com> Date: Mon, 2 Mar 2026 10:52:36 +0800 Subject: [PATCH 175/562] [CALCITE-7425] Correct the logical inverse of SqlBetweenOperator Signed-off-by: wforget <643348094@qq.com> --- .../calcite/sql/fun/SqlBetweenOperator.java | 11 ++-- .../rel2sql/RelToSqlConverterStructsTest.java | 3 +- .../rel/rel2sql/RelToSqlConverterTest.java | 65 +++++++++++++++---- 3 files changed, 62 insertions(+), 17 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlBetweenOperator.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlBetweenOperator.java index 6f6234c3ed5b..2d77fd9df7b1 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlBetweenOperator.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlBetweenOperator.java @@ -112,7 +112,10 @@ public SqlBetweenOperator(Flag flag, boolean negated) { //~ Methods ---------------------------------------------------------------- @Override public boolean validRexOperands(int count, Litmus litmus) { - return litmus.fail("not a rex operator"); + if (count != 3) { + return litmus.fail("wrong operand count {} for {}", count, this); + } + return litmus.succeed(); } /** @@ -125,14 +128,14 @@ public boolean isNegated() { } @Override public SqlOperator not() { - return of(negated, flag == Flag.SYMMETRIC); + return of(!negated, flag == Flag.SYMMETRIC); } private static SqlBetweenOperator of(boolean negated, boolean symmetric) { if (symmetric) { return negated - ? SqlStdOperatorTable.SYMMETRIC_BETWEEN - : SqlStdOperatorTable.SYMMETRIC_NOT_BETWEEN; + ? SqlStdOperatorTable.SYMMETRIC_NOT_BETWEEN + : SqlStdOperatorTable.SYMMETRIC_BETWEEN; } else { return negated ? SqlStdOperatorTable.NOT_BETWEEN diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterStructsTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterStructsTest.java index dca08b4999e5..0472604f3050 100644 --- a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterStructsTest.java +++ b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterStructsTest.java @@ -18,6 +18,7 @@ import org.apache.calcite.sql.dialect.CalciteSqlDialect; import org.apache.calcite.sql.parser.SqlParser; +import org.apache.calcite.sql2rel.StandardConvertletTable; import org.apache.calcite.test.CalciteAssert; import com.google.common.collect.ImmutableList; @@ -36,7 +37,7 @@ class RelToSqlConverterStructsTest { private RelToSqlConverterTest.Sql sql(String sql) { return new RelToSqlConverterTest.Sql(CalciteAssert.SchemaSpec.MY_DB, sql, CalciteSqlDialect.DEFAULT, SqlParser.Config.DEFAULT, ImmutableSet.of(), - UnaryOperator.identity(), null, ImmutableList.of()); + UnaryOperator.identity(), null, ImmutableList.of(), StandardConvertletTable.INSTANCE); } @Test void testNestedSchemaSelectStar() { diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java index 4250181e6636..1b219255b439 100644 --- a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java +++ b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java @@ -74,6 +74,7 @@ import org.apache.calcite.sql.dialect.PostgresqlSqlDialect; import org.apache.calcite.sql.dialect.PrestoSqlDialect; import org.apache.calcite.sql.dialect.SqliteSqlDialect; +import org.apache.calcite.sql.fun.SqlBetweenOperator; import org.apache.calcite.sql.fun.SqlLibrary; import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.parser.SqlParser; @@ -82,7 +83,10 @@ import org.apache.calcite.sql.util.SqlShuttle; import org.apache.calcite.sql.validate.SqlConformance; import org.apache.calcite.sql.validate.SqlConformanceEnum; +import org.apache.calcite.sql2rel.SqlRexConvertlet; +import org.apache.calcite.sql2rel.SqlRexConvertletTable; import org.apache.calcite.sql2rel.SqlToRelConverter; +import org.apache.calcite.sql2rel.StandardConvertletTable; import org.apache.calcite.test.CalciteAssert; import org.apache.calcite.test.MockSqlOperatorTable; import org.apache.calcite.test.RelBuilderTest; @@ -135,7 +139,7 @@ class RelToSqlConverterTest { private Sql fixture() { return new Sql(CalciteAssert.SchemaSpec.JDBC_FOODMART, "?", CalciteSqlDialect.DEFAULT, SqlParser.Config.DEFAULT, ImmutableSet.of(), - UnaryOperator.identity(), null, ImmutableList.of()); + UnaryOperator.identity(), null, ImmutableList.of(), StandardConvertletTable.INSTANCE); } /** Initiates a test case with a given SQL query. */ @@ -153,12 +157,13 @@ private Sql relFn(Function relFn) { private static Planner getPlanner(List traitDefs, SqlParser.Config parserConfig, SchemaPlus schema, SqlToRelConverter.Config sqlToRelConf, Collection librarySet, - RelDataTypeSystem typeSystem, Program... programs) { + RelDataTypeSystem typeSystem, SqlRexConvertletTable convertletTable, Program... programs) { final FrameworkConfig config = Frameworks.newConfigBuilder() .parserConfig(parserConfig) .defaultSchema(schema) .traitDefs(traitDefs) .sqlToRelConverterConfig(sqlToRelConf) + .convertletTable(convertletTable) .programs(programs) .operatorTable(MockSqlOperatorTable.standard() .plus(librarySet) @@ -11293,12 +11298,14 @@ static class Sql { private final List> transforms; private final SqlParser.Config parserConfig; private final UnaryOperator config; + private final SqlRexConvertletTable convertletTable; Sql(CalciteAssert.SchemaSpec schemaSpec, String sql, SqlDialect dialect, SqlParser.Config parserConfig, Set librarySet, UnaryOperator config, @Nullable Function relFn, - List> transforms) { + List> transforms, + SqlRexConvertletTable convertletTable) { this.schemaSpec = schemaSpec; this.sql = sql; this.dialect = dialect; @@ -11307,21 +11314,22 @@ static class Sql { this.transforms = ImmutableList.copyOf(transforms); this.parserConfig = parserConfig; this.config = config; + this.convertletTable = convertletTable; } Sql withSql(String sql) { return new Sql(schemaSpec, sql, dialect, parserConfig, librarySet, config, - relFn, transforms); + relFn, transforms, convertletTable); } Sql dialect(SqlDialect dialect) { return new Sql(schemaSpec, sql, dialect, parserConfig, librarySet, config, - relFn, transforms); + relFn, transforms, convertletTable); } Sql relFn(Function relFn) { return new Sql(schemaSpec, sql, dialect, parserConfig, librarySet, config, - relFn, transforms); + relFn, transforms, convertletTable); } Sql withCalcite() { @@ -11564,12 +11572,12 @@ Sql withOracleModifiedTypeSystem() { Sql parserConfig(SqlParser.Config parserConfig) { return new Sql(schemaSpec, sql, dialect, parserConfig, librarySet, config, - relFn, transforms); + relFn, transforms, convertletTable); } Sql withConfig(UnaryOperator config) { return new Sql(schemaSpec, sql, dialect, parserConfig, librarySet, config, - relFn, transforms); + relFn, transforms, convertletTable); } final Sql withLibrary(SqlLibrary library) { @@ -11578,7 +11586,7 @@ final Sql withLibrary(SqlLibrary library) { Sql withLibrarySet(Iterable librarySet) { return new Sql(schemaSpec, sql, dialect, parserConfig, - ImmutableSet.copyOf(librarySet), config, relFn, transforms); + ImmutableSet.copyOf(librarySet), config, relFn, transforms, convertletTable); } Sql optimize(final RuleSet ruleSet, @@ -11595,7 +11603,12 @@ Sql optimize(final RuleSet ruleSet, ImmutableList.of(), ImmutableList.of()); }); return new Sql(schemaSpec, sql, dialect, parserConfig, librarySet, config, - relFn, transforms); + relFn, transforms, convertletTable); + } + + Sql withConvertletTable(SqlRexConvertletTable convertletTable) { + return new Sql(schemaSpec, sql, dialect, parserConfig, librarySet, config, + relFn, transforms, convertletTable); } Sql ok(String expectedQuery) { @@ -11631,7 +11644,8 @@ String exec() { .withTrimUnusedFields(false)); RelDataTypeSystem typeSystem = dialect.getTypeSystem(); final Planner planner = - getPlanner(null, parserConfig, defaultSchema, config, librarySet, typeSystem); + getPlanner(null, parserConfig, defaultSchema, config, librarySet, typeSystem, + convertletTable); SqlNode parse = planner.parse(sql); SqlNode validate = planner.validate(parse); rel = planner.rel(validate).project(); @@ -11647,7 +11661,7 @@ String exec() { public Sql schema(CalciteAssert.SchemaSpec schemaSpec) { return new Sql(schemaSpec, sql, dialect, parserConfig, librarySet, config, - relFn, transforms); + relFn, transforms, convertletTable); } } @@ -11835,4 +11849,31 @@ public Sql schema(CalciteAssert.SchemaSpec schemaSpec) { sql(sql).schema(CalciteAssert.SchemaSpec.JDBC_SCOTT).ok(expected); } + @Test void testNotBetween() { + Sql f = fixture().withConvertletTable(new SqlRexConvertletTable() { + @Override public @Nullable SqlRexConvertlet get(SqlCall call) { + // Override StandardConvertletTable::convertBetween to avoid converting SqlBetweenOperator + if (call != null && call.getOperator() instanceof SqlBetweenOperator) { + return StandardConvertletTable.INSTANCE::convertCall; + } + return StandardConvertletTable.INSTANCE.get(call); + } + }); + final String query1 = "SELECT empno FROM emp WHERE NOT (empno BETWEEN 1000 AND 2000)"; + final String expected1 = "SELECT \"EMPNO\"\n" + + "FROM \"SCOTT\".\"EMP\"\n" + + "WHERE CAST(\"EMPNO\" AS INTEGER) NOT BETWEEN ASYMMETRIC 1000 AND 2000"; + f.withSql(query1) + .schema(CalciteAssert.SchemaSpec.JDBC_SCOTT) + .ok(expected1); + + final String query2 = "SELECT empno FROM emp WHERE NOT (empno BETWEEN SYMMETRIC 1000 AND 2000)"; + final String expected2 = "SELECT \"EMPNO\"\n" + + "FROM \"SCOTT\".\"EMP\"\n" + + "WHERE CAST(\"EMPNO\" AS INTEGER) NOT BETWEEN SYMMETRIC 1000 AND 2000"; + f.withSql(query2) + .schema(CalciteAssert.SchemaSpec.JDBC_SCOTT) + .ok(expected2); + } + } From d9c10a9b7f25bc525c9af4a3200753b7494df0f3 Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Thu, 5 Mar 2026 06:43:08 +0800 Subject: [PATCH 176/562] [CALCITE-5132] Scalar IN subquery returns UNKNOWN instead of FALSE when key is partially NULL --- .../calcite/rel/rules/SubQueryRemoveRule.java | 100 ++++++++++-- .../apache/calcite/test/JdbcAdapterTest.java | 27 ++-- core/src/test/resources/sql/sub-query.iq | 147 +++++++++++++----- 3 files changed, 213 insertions(+), 61 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rel/rules/SubQueryRemoveRule.java b/core/src/main/java/org/apache/calcite/rel/rules/SubQueryRemoveRule.java index 9fa5f612ead8..a1b59e952f4f 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/SubQueryRemoveRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/SubQueryRemoveRule.java @@ -58,6 +58,7 @@ import java.util.List; import java.util.Set; import java.util.stream.Collectors; +import java.util.stream.IntStream; import static org.apache.calcite.util.Util.last; @@ -588,7 +589,7 @@ private static RexNode rewriteIn(RexSubQuery e, Set variablesSet, // when e.deptno is null then null -- (2) key NULL check // when dt.i is not null then true -- (3) match found // when ct.ck < ct.c then null -- (4) NULLs exist in subquery - // else false -- (5) no match + // else false -- (5) no match // end // from emp as e // left join ( @@ -602,7 +603,7 @@ private static RexNode rewriteIn(RexSubQuery e, Set variablesSet, // select e.deptno, // case // when dt.i is not null then true -- (3) match found - // else false -- (5) no match + // else false -- (5) no match // end // from emp as e // left join (select distinct deptno, true as i from emp) as dt @@ -622,6 +623,43 @@ private static RexNode rewriteIn(RexSubQuery e, Set variablesSet, // from emp as e // inner join (select distinct deptno from emp) as dt // on e.deptno = dt.deptno + // + // For multi-column IN where at least one key or RHS column is nullable, a + // single wildcard LEFT JOIN handles both exact and partial-null matches, + // while all-null RHS rows are excluded from dt and detected via ct.ck < ct.c: + // + // select e.empno, (e.empno, e.comm) in (select empno, comm from emp) + // from emp as e + // + // becomes + // + // select e.empno, + // case + // when ct.c = 0 then false -- (1) empty subquery check + // when e.comm is null then null -- (2) nullable key NULL check + // when dt.i is not null and dt.em then true -- (3) exact match (all cols non-null) + // when dt.i is not null then null -- (4) partial-null match (UNKNOWN) + // when ct.ck < ct.c then null -- (5) all-null row exists (UNKNOWN) + // else false -- (6) no match + // end + // from emp as e + // inner join ( + // select count(*) as c, + // count(*) filter (where not (empno is null and comm is null)) as ck + // from emp) as ct on true + // left join ( + // select empno, comm, true as i, + // max(empno is not null and comm is not null) as em + // from emp + // where empno is not null or comm is not null -- all-null rows excluded + // group by empno, comm) as dt + // on (e.empno = dt.empno or dt.empno is null) -- wildcard per-column condition + // and (e.comm = dt.comm or dt.comm is null) + // + // All-null rows (empno IS NULL AND comm IS NULL) are excluded from dt because + // the wildcard condition matches every LHS key, causing duplicate join output + // rows for LHS keys that also have exact group matches. They are instead + // caught by the global ct.ck < ct.c check (branch 5). builder.push(e.rel); final List fields = new ArrayList<>(builder.fields()); @@ -668,6 +706,7 @@ private static RexNode rewriteIn(RexSubQuery e, Set variablesSet, final RexLiteral unknownLiteral = builder.getRexBuilder().makeNullLiteral(trueLiteral.getType()); boolean needsNullSafety = false; + boolean needsNullRowJoin = false; if (allLiterals) { final List conditions = Pair.zip(expressionOperands, fields).stream() @@ -724,6 +763,7 @@ private static RexNode rewriteIn(RexSubQuery e, Set variablesSet, (logic == RelOptUtil.Logic.TRUE_FALSE_UNKNOWN || logic == RelOptUtil.Logic.UNKNOWN_AS_TRUE) && (!keyIsNulls.isEmpty() || anyFieldNullable); + needsNullRowJoin = needsNullSafety && fields.size() > 1; switch (logic) { case TRUE: @@ -762,8 +802,21 @@ private static RexNode rewriteIn(RexSubQuery e, Set variablesSet, } // fall through default: - builder.aggregate(builder.groupKey(fields), - builder.literalAgg(true).as("i")); + if (needsNullRowJoin) { + // Exclude all-null rows from dt (they are detected by ct.ck < ct.c instead). + // Add em (exact-match) column to distinguish exact groups from partial-null groups. + List anyFieldNotNull = + fields.stream().map(builder::isNotNull).collect(Collectors.toList()); + builder.filter(builder.or(anyFieldNotNull)); + RexNode allNotNull = + builder.and(fields.stream().map(builder::isNotNull).collect(Collectors.toList())); + builder.aggregate(builder.groupKey(fields), + builder.literalAgg(true).as("i"), + builder.max(allNotNull).as("em")); + } else { + builder.aggregate(builder.groupKey(fields), + builder.literalAgg(true).as("i")); + } } } @@ -773,10 +826,22 @@ private static RexNode rewriteIn(RexSubQuery e, Set variablesSet, } builder.as(dtAlias); int refOffset = offset; - final List conditions = - Pair.zip(expressionOperands, builder.fields()).stream() - .map(pair -> builder.equals(pair.left, RexUtil.shift(pair.right, refOffset))) - .collect(Collectors.toList()); + final List conditions; + if (needsNullRowJoin) { + // Per-column wildcard condition: (key = col OR col IS NULL). + final List dtFields = builder.fields(); + conditions = IntStream.range(0, expressionOperands.size()) + .mapToObj(k -> { + RexNode col = RexUtil.shift(dtFields.get(k), refOffset); + return builder.or(builder.equals(expressionOperands.get(k), col), + builder.isNull(col)); + }) + .collect(Collectors.toList()); + } else { + conditions = Pair.zip(expressionOperands, builder.fields()).stream() + .map(pair -> builder.equals(pair.left, RexUtil.shift(pair.right, refOffset))) + .collect(Collectors.toList()); + } switch (logic) { case TRUE: builder.join(JoinRelType.INNER, builder.and(conditions), variablesSet); @@ -826,16 +891,29 @@ private static RexNode rewriteIn(RexSubQuery e, Set variablesSet, operands.add(builder.isNotNull(builder.field(dtAlias, "cs")), trueLiteral); } else { - operands.add(builder.isNotNull(last(builder.fields())), - trueLiteral); + if (needsNullRowJoin) { + // em=true: exact match (all RHS cols non-null) → TRUE. + operands.add( + builder.and( + ImmutableList.of( + builder.isNotNull(builder.field(dtAlias, "i")), + builder.call(SqlStdOperatorTable.IS_TRUE, + builder.field(dtAlias, "em")))), + trueLiteral); + // em=false: partial-null match → UNKNOWN. + operands.add(builder.isNotNull(builder.field(dtAlias, "i")), b); + } else { + operands.add(builder.isNotNull(builder.field(dtAlias, "i")), + trueLiteral); + } } if (!allLiterals) { switch (logic) { case TRUE_FALSE_UNKNOWN: case UNKNOWN_AS_TRUE: - // only reference ctAlias if we created it if (needsNullSafety) { + // ct.ck < ct.c: RHS has a null (single-col) or all-null row (multi-col) → UNKNOWN. operands.add( builder.lessThan(builder.field(ctAlias, "ck"), builder.field(ctAlias, "c")), diff --git a/core/src/test/java/org/apache/calcite/test/JdbcAdapterTest.java b/core/src/test/java/org/apache/calcite/test/JdbcAdapterTest.java index 98c8edff3fb9..8d8d66371c8d 100644 --- a/core/src/test/java/org/apache/calcite/test/JdbcAdapterTest.java +++ b/core/src/test/java/org/apache/calcite/test/JdbcAdapterTest.java @@ -274,23 +274,20 @@ class JdbcAdapterTest { @Test void testNotPushDownNotIn() { CalciteAssert.model(JdbcTest.SCOTT_MODEL) .query("select * from dept where (deptno, dname) not in (select deptno, ename from emp)") - .explainContains("PLAN=EnumerableCalc(expr#0..7=[{inputs}], expr#8=[0], " - + "expr#9=[=($t3, $t8)], expr#10=[IS NULL($t7)], expr#11=[>=($t4, $t3)], " - + "expr#12=[IS NOT NULL($t1)], expr#13=[AND($t10, $t11, $t12)], " - + "expr#14=[OR($t9, $t13)], proj#0..2=[{exprs}], $condition=[$t14])\n" - + " EnumerableMergeJoin(condition=[AND(=($0, $5), =($1, $6))], joinType=[left])\n" - + " EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC])\n" - + " EnumerableNestedLoopJoin(condition=[true], joinType=[inner])\n" + .explainContains("EnumerableNestedLoopJoin(condition=[AND(OR(IS NULL($5)," + + " =($0, $5)), OR(IS NULL($6), =($1, $6)))], joinType=[left])\n" + + " EnumerableNestedLoopJoin(condition=[true], joinType=[inner])\n" + + " JdbcToEnumerableConverter\n" + + " JdbcTableScan(table=[[SCOTT, DEPT]])\n" + + " EnumerableAggregate(group=[{}], c=[COUNT()], ck=[COUNT() FILTER $0])\n" + " JdbcToEnumerableConverter\n" - + " JdbcTableScan(table=[[SCOTT, DEPT]])\n" - + " EnumerableAggregate(group=[{}], c=[COUNT()], ck=[COUNT() FILTER $0])\n" - + " JdbcToEnumerableConverter\n" - + " JdbcProject($f2=[OR(IS NOT NULL($7), IS NOT NULL($1))])\n" - + " JdbcTableScan(table=[[SCOTT, EMP]])\n" + + " JdbcProject($f2=[OR(IS NOT NULL($7), IS NOT NULL($1))])\n" + + " JdbcTableScan(table=[[SCOTT, EMP]])\n" + " JdbcToEnumerableConverter\n" - + " JdbcSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC])\n" - + " JdbcAggregate(group=[{0, 1}], i=[LITERAL_AGG(true)])\n" - + " JdbcProject(DEPTNO=[$7], ENAME=[CAST($1):VARCHAR(14)])\n" + + " JdbcAggregate(group=[{0, 1}], i=[LITERAL_AGG(true)], em=[MAX($2)])\n" + + " JdbcProject(DEPTNO=[$7], ENAME=[CAST($1):VARCHAR(14)]," + + " $f2=[AND(IS NOT NULL($7), IS NOT NULL($1))])\n" + + " JdbcFilter(condition=[OR(IS NOT NULL($7), IS NOT NULL($1))])\n" + " JdbcTableScan(table=[[SCOTT, EMP]])\n\n"); } diff --git a/core/src/test/resources/sql/sub-query.iq b/core/src/test/resources/sql/sub-query.iq index 2c47ab5d0538..51cab9f01e06 100644 --- a/core/src/test/resources/sql/sub-query.iq +++ b/core/src/test/resources/sql/sub-query.iq @@ -4313,16 +4313,16 @@ select * from "scott".emp where (empno, deptno) not in ((1, 2), (3, null)); !ok !if (use_old_decorr) { -EnumerableCalc(expr#0..14=[{inputs}], expr#15=[0], expr#16=[=($t8, $t15)], expr#17=[IS NULL($t14)], expr#18=[>=($t9, $t8)], expr#19=[IS NOT NULL($t7)], expr#20=[AND($t17, $t18, $t19)], expr#21=[OR($t16, $t20)], proj#0..7=[{exprs}], $condition=[$t21]) - EnumerableMergeJoin(condition=[AND(=($10, $12), =($11, $13))], joinType=[left]) - EnumerableSort(sort0=[$10], sort1=[$11], dir0=[ASC], dir1=[ASC]) - EnumerableCalc(expr#0..9=[{inputs}], expr#10=[CAST($t0):INTEGER NOT NULL], expr#11=[CAST($t7):INTEGER], proj#0..11=[{exprs}]) +EnumerableCalc(expr#0..14=[{inputs}], expr#15=[0], expr#16=[=($t8, $t15)], expr#17=[IS NULL($t7)], expr#18=[IS NOT NULL($t13)], expr#19=[AND($t14, $t18)], expr#20=[<($t9, $t8)], expr#21=[OR($t17, $t19, $t18, $t20)], expr#22=[IS NOT TRUE($t21)], expr#23=[OR($t16, $t22)], proj#0..7=[{exprs}], $condition=[$t23]) + EnumerableMergeJoin(condition=[AND(=($10, $11), OR(IS NULL($12), =(CAST($7):INTEGER, $12)))], joinType=[left]) + EnumerableSort(sort0=[$10], dir0=[ASC]) + EnumerableCalc(expr#0..9=[{inputs}], expr#10=[CAST($t0):INTEGER NOT NULL], proj#0..10=[{exprs}]) EnumerableNestedLoopJoin(condition=[true], joinType=[inner]) EnumerableTableScan(table=[[scott, EMP]]) EnumerableAggregate(group=[{}], c=[COUNT()], ck=[COUNT() FILTER $0]) EnumerableValues(tuples=[[{ true }, { true }]]) - EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) - EnumerableCalc(expr#0..1=[{inputs}], expr#2=[true], proj#0..2=[{exprs}]) + EnumerableSort(sort0=[$0], dir0=[ASC]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[true], expr#3=[IS NOT NULL($t1)], proj#0..3=[{exprs}]) EnumerableValues(tuples=[[{ 3, null }, { 1, 2 }]]) !plan !} @@ -4337,18 +4337,15 @@ select * from "scott".emp where (mgr, deptno) not in ((1, 2), (3, null), (cast(n !ok !if (use_old_decorr) { -EnumerableCalc(expr#0..14=[{inputs}], expr#15=[0], expr#16=[=($t8, $t15)], expr#17=[IS NULL($t14)], expr#18=[>=($t9, $t8)], expr#19=[IS NOT NULL($t3)], expr#20=[IS NOT NULL($t7)], expr#21=[AND($t17, $t18, $t19, $t20)], expr#22=[OR($t16, $t21)], proj#0..7=[{exprs}], $condition=[$t22]) - EnumerableMergeJoin(condition=[AND(=($10, $12), =($11, $13))], joinType=[left]) - EnumerableSort(sort0=[$10], sort1=[$11], dir0=[ASC], dir1=[ASC]) - EnumerableCalc(expr#0..9=[{inputs}], expr#10=[CAST($t3):INTEGER], expr#11=[CAST($t7):INTEGER], proj#0..11=[{exprs}]) - EnumerableNestedLoopJoin(condition=[true], joinType=[inner]) - EnumerableTableScan(table=[[scott, EMP]]) - EnumerableAggregate(group=[{}], c=[COUNT()], ck=[COUNT() FILTER $0]) - EnumerableCalc(expr#0..1=[{inputs}], expr#2=[IS NOT NULL($t0)], expr#3=[IS NOT NULL($t1)], expr#4=[OR($t2, $t3)], $f2=[$t4]) - EnumerableValues(tuples=[[{ 3, null }, { null, null }, { 1, 2 }]]) - EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) - EnumerableCalc(expr#0..1=[{inputs}], expr#2=[true], proj#0..2=[{exprs}]) - EnumerableValues(tuples=[[{ 3, null }, { null, null }, { 1, 2 }]]) +EnumerableCalc(expr#0..13=[{inputs}], expr#14=[0], expr#15=[=($t8, $t14)], expr#16=[IS NULL($t3)], expr#17=[IS NULL($t7)], expr#18=[IS NOT NULL($t12)], expr#19=[AND($t13, $t18)], expr#20=[<($t9, $t8)], expr#21=[OR($t16, $t17, $t19, $t18, $t20)], expr#22=[IS NOT TRUE($t21)], expr#23=[OR($t15, $t22)], proj#0..7=[{exprs}], $condition=[$t23]) + EnumerableNestedLoopJoin(condition=[AND(OR(IS NULL($10), =(CAST($3):INTEGER, $10)), OR(IS NULL($11), =(CAST($7):INTEGER, $11)))], joinType=[left]) + EnumerableNestedLoopJoin(condition=[true], joinType=[inner]) + EnumerableTableScan(table=[[scott, EMP]]) + EnumerableAggregate(group=[{}], c=[COUNT()], ck=[COUNT() FILTER $0]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[IS NOT NULL($t0)], expr#3=[IS NOT NULL($t1)], expr#4=[OR($t2, $t3)], $f2=[$t4]) + EnumerableValues(tuples=[[{ 3, null }, { null, null }, { 1, 2 }]]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[true], expr#3=[IS NOT NULL($t0)], expr#4=[IS NOT NULL($t1)], expr#5=[AND($t3, $t4)], expr#6=[OR($t3, $t4)], proj#0..2=[{exprs}], $f20=[$t5], $condition=[$t6]) + EnumerableValues(tuples=[[{ 3, null }, { null, null }, { 1, 2 }]]) !plan !} @@ -4382,7 +4379,7 @@ select * from "scott".emp where (empno, deptno) not in ((7369, 20), (7499, 30)); !ok !if (use_old_decorr) { -EnumerableCalc(expr#0..14=[{inputs}], expr#15=[0], expr#16=[=($t8, $t15)], expr#17=[IS NULL($t14)], expr#18=[>=($t9, $t8)], expr#19=[IS NOT NULL($t7)], expr#20=[AND($t17, $t18, $t19)], expr#21=[OR($t16, $t20)], proj#0..7=[{exprs}], $condition=[$t21]) +EnumerableCalc(expr#0..15=[{inputs}], expr#16=[0], expr#17=[=($t8, $t16)], expr#18=[IS NULL($t7)], expr#19=[IS NOT NULL($t14)], expr#20=[AND($t15, $t19)], expr#21=[<($t9, $t8)], expr#22=[OR($t18, $t20, $t19, $t21)], expr#23=[IS NOT TRUE($t22)], expr#24=[OR($t17, $t23)], proj#0..7=[{exprs}], $condition=[$t24]) EnumerableMergeJoin(condition=[AND(=($10, $12), =($11, $13))], joinType=[left]) EnumerableSort(sort0=[$10], sort1=[$11], dir0=[ASC], dir1=[ASC]) EnumerableCalc(expr#0..9=[{inputs}], expr#10=[CAST($t0):INTEGER NOT NULL], expr#11=[CAST($t7):INTEGER], proj#0..11=[{exprs}]) @@ -4392,7 +4389,7 @@ EnumerableCalc(expr#0..14=[{inputs}], expr#15=[0], expr#16=[=($t8, $t15)], expr# EnumerableCalc(expr#0..1=[{inputs}], expr#2=[true], $f2=[$t2]) EnumerableValues(tuples=[[{ 7369, 20 }, { 7499, 30 }]]) EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) - EnumerableCalc(expr#0..1=[{inputs}], expr#2=[true], proj#0..2=[{exprs}]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[true], proj#0..2=[{exprs}], $f20=[$t2]) EnumerableValues(tuples=[[{ 7369, 20 }, { 7499, 30 }]]) !plan !} @@ -5256,7 +5253,7 @@ select comm, (comm, comm) in ((500, 500), (300, 300), (0, 0)) from emp; !ok !if (use_old_decorr) { -EnumerableCalc(expr#0..8=[{inputs}], expr#9=[IS NULL($t1)], expr#10=[null:BOOLEAN], expr#11=[0], expr#12=[<>($t2, $t11)], expr#13=[AND($t9, $t10, $t12)], expr#14=[IS NOT NULL($t8)], expr#15=[IS NOT NULL($t1)], expr#16=[AND($t14, $t12, $t15)], expr#17=[<($t3, $t2)], expr#18=[IS NULL($t8)], expr#19=[AND($t17, $t10, $t12, $t15, $t18)], expr#20=[OR($t13, $t16, $t19)], COMM=[$t1], EXPR$1=[$t20]) +EnumerableCalc(expr#0..9=[{inputs}], expr#10=[IS NULL($t1)], expr#11=[null:BOOLEAN], expr#12=[0], expr#13=[<>($t2, $t12)], expr#14=[AND($t10, $t11, $t13)], expr#15=[IS NOT NULL($t8)], expr#16=[AND($t9, $t15)], expr#17=[IS TRUE($t16)], expr#18=[IS NOT NULL($t1)], expr#19=[AND($t17, $t13, $t18)], expr#20=[<($t3, $t2)], expr#21=[OR($t15, $t20)], expr#22=[IS NOT TRUE($t16)], expr#23=[AND($t21, $t11, $t13, $t18, $t22)], expr#24=[OR($t14, $t19, $t23)], COMM=[$t1], EXPR$1=[$t24]) EnumerableMergeJoin(condition=[AND(=($4, $6), =($5, $7))], joinType=[left]) EnumerableSort(sort0=[$4], sort1=[$5], dir0=[ASC], dir1=[ASC]) EnumerableCalc(expr#0..3=[{inputs}], expr#4=[CAST($t1):DECIMAL(12, 2)], proj#0..4=[{exprs}], COMM1=[$t4]) @@ -5266,7 +5263,7 @@ EnumerableCalc(expr#0..8=[{inputs}], expr#9=[IS NULL($t1)], expr#10=[null:BOOLEA EnumerableAggregate(group=[{}], c=[COUNT()], ck=[COUNT() FILTER $0]) EnumerableValues(tuples=[[{ true }, { true }, { true }]]) EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) - EnumerableCalc(expr#0..1=[{inputs}], expr#2=[true], proj#0..2=[{exprs}]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[true], proj#0..2=[{exprs}], $f20=[$t2]) EnumerableValues(tuples=[[{ 500.00, 500.00 }, { 300.00, 300.00 }, { 0.00, 0.00 }]]) !plan !} @@ -5296,19 +5293,16 @@ select comm, (comm, comm) in ((500, 500), (300, 300), (0, 0), (null , null)) fro !ok !if (use_old_decorr) { -EnumerableCalc(expr#0..8=[{inputs}], expr#9=[IS NULL($t1)], expr#10=[null:BOOLEAN], expr#11=[0], expr#12=[<>($t2, $t11)], expr#13=[AND($t9, $t10, $t12)], expr#14=[IS NOT NULL($t8)], expr#15=[IS NOT NULL($t1)], expr#16=[AND($t14, $t12, $t15)], expr#17=[<($t3, $t2)], expr#18=[IS NULL($t8)], expr#19=[AND($t17, $t10, $t12, $t15, $t18)], expr#20=[OR($t13, $t16, $t19)], COMM=[$t1], EXPR$1=[$t20]) - EnumerableMergeJoin(condition=[AND(=($4, $6), =($5, $7))], joinType=[left]) - EnumerableSort(sort0=[$4], sort1=[$5], dir0=[ASC], dir1=[ASC]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[CAST($t1):DECIMAL(12, 2)], proj#0..4=[{exprs}], COMM1=[$t4]) - EnumerableNestedLoopJoin(condition=[true], joinType=[inner]) - EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], COMM=[$t6]) - EnumerableTableScan(table=[[scott, EMP]]) - EnumerableAggregate(group=[{}], c=[COUNT()], ck=[COUNT() FILTER $0]) - EnumerableCalc(expr#0..1=[{inputs}], expr#2=[IS NOT NULL($t0)], expr#3=[IS NOT NULL($t1)], expr#4=[OR($t2, $t3)], $f2=[$t4]) - EnumerableValues(tuples=[[{ 500.00, 500.00 }, { 300.00, 300.00 }, { 0.00, 0.00 }, { null, null }]]) - EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) - EnumerableCalc(expr#0..1=[{inputs}], expr#2=[true], proj#0..2=[{exprs}]) - EnumerableValues(tuples=[[{ 500.00, 500.00 }, { 300.00, 300.00 }, { 0.00, 0.00 }, { null, null }]]) +EnumerableCalc(expr#0..7=[{inputs}], expr#8=[IS NULL($t1)], expr#9=[null:BOOLEAN], expr#10=[0], expr#11=[<>($t2, $t10)], expr#12=[AND($t8, $t9, $t11)], expr#13=[IS NOT NULL($t6)], expr#14=[AND($t7, $t13)], expr#15=[IS TRUE($t14)], expr#16=[IS NOT NULL($t1)], expr#17=[AND($t15, $t11, $t16)], expr#18=[<($t3, $t2)], expr#19=[OR($t13, $t18)], expr#20=[IS NOT TRUE($t14)], expr#21=[AND($t19, $t9, $t11, $t16, $t20)], expr#22=[OR($t12, $t17, $t21)], COMM=[$t1], EXPR$1=[$t22]) + EnumerableNestedLoopJoin(condition=[AND(OR(IS NULL($4), =(CAST($1):DECIMAL(12, 2), $4)), OR(IS NULL($5), =(CAST($1):DECIMAL(12, 2), $5)))], joinType=[left]) + EnumerableNestedLoopJoin(condition=[true], joinType=[inner]) + EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], COMM=[$t6]) + EnumerableTableScan(table=[[scott, EMP]]) + EnumerableAggregate(group=[{}], c=[COUNT()], ck=[COUNT() FILTER $0]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[IS NOT NULL($t0)], expr#3=[IS NOT NULL($t1)], expr#4=[OR($t2, $t3)], $f2=[$t4]) + EnumerableValues(tuples=[[{ 500.00, 500.00 }, { 300.00, 300.00 }, { 0.00, 0.00 }, { null, null }]]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[true], expr#3=[IS NOT NULL($t0)], expr#4=[IS NOT NULL($t1)], expr#5=[AND($t3, $t4)], expr#6=[OR($t3, $t4)], proj#0..2=[{exprs}], $f20=[$t5], $condition=[$t6]) + EnumerableValues(tuples=[[{ 500.00, 500.00 }, { 300.00, 300.00 }, { 0.00, 0.00 }, { null, null }]]) !plan !} @@ -9075,4 +9069,87 @@ where exists ( (14 rows) !ok + +# [CALCITE-5132] Scalar IN subquery returns UNKNOWN instead of FALSE when key is partially NULL. +# Case 1: Default insubquerythreshold=20 +!if (use_old_decorr) { +select empno, deptno, (empno, deptno) in ((7521, null)) from "scott".emp; ++-------+--------+--------+ +| EMPNO | DEPTNO | EXPR$2 | ++-------+--------+--------+ +| 7369 | 20 | false | +| 7499 | 30 | false | +| 7521 | 30 | | +| 7566 | 20 | false | +| 7654 | 30 | false | +| 7698 | 30 | false | +| 7782 | 10 | false | +| 7788 | 20 | false | +| 7839 | 10 | false | +| 7844 | 30 | false | +| 7876 | 20 | false | +| 7900 | 30 | false | +| 7902 | 20 | false | +| 7934 | 10 | false | ++-------+--------+--------+ +(14 rows) + +!ok + +select v, + row(v, 0) in (values (1, 0), (2, cast(null as integer))) as r +from (values (1), (2), (3)) as t(v); ++---+-------+ +| V | R | ++---+-------+ +| 1 | true | +| 2 | | +| 3 | false | ++---+-------+ +(3 rows) + +!ok + +# [CALCITE-5132] Scalar IN subquery returns UNKNOWN instead of FALSE when key is partially NULL. +# Case 2: insubquerythreshold=0 +!set insubquerythreshold 0 +select empno, deptno, (empno, deptno) in ((7521, null)) from "scott".emp; ++-------+--------+--------+ +| EMPNO | DEPTNO | EXPR$2 | ++-------+--------+--------+ +| 7369 | 20 | false | +| 7499 | 30 | false | +| 7521 | 30 | | +| 7566 | 20 | false | +| 7654 | 30 | false | +| 7698 | 30 | false | +| 7782 | 10 | false | +| 7788 | 20 | false | +| 7839 | 10 | false | +| 7844 | 30 | false | +| 7876 | 20 | false | +| 7900 | 30 | false | +| 7902 | 20 | false | +| 7934 | 10 | false | ++-------+--------+--------+ +(14 rows) + +!ok + +select v, + row(v, 0) in (values (1, 0), (2, cast(null as integer))) as r +from (values (1), (2), (3)) as t(v); ++---+-------+ +| V | R | ++---+-------+ +| 1 | true | +| 2 | | +| 3 | false | ++---+-------+ +(3 rows) + +!ok +!set insubquerythreshold 20 +!} + # End sub-query.iq From 9ae088bcaf4ebb0b93d08bdf90b6b67f2a4fe646 Mon Sep 17 00:00:00 2001 From: Silun Date: Fri, 6 Mar 2026 16:29:12 +0800 Subject: [PATCH 177/562] [CALCITE-7434] Error in new decorrelation algorithm caused by FilterJoinRule omitting variablesSet --- .../calcite/rel/rules/FilterJoinRule.java | 2 + .../apache/calcite/test/RelOptRulesTest.xml | 2 +- core/src/test/resources/sql/new-decorr.iq | 58 +++++++++++++++++++ 3 files changed, 61 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/calcite/rel/rules/FilterJoinRule.java b/core/src/main/java/org/apache/calcite/rel/rules/FilterJoinRule.java index 9c6b560bfa54..d4e1473ccbfc 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/FilterJoinRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/FilterJoinRule.java @@ -36,6 +36,7 @@ import org.apache.calcite.tools.RelBuilderFactory; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; import org.checkerframework.checker.nullness.qual.Nullable; @@ -249,6 +250,7 @@ protected void perform(RelOptRuleCall call, @Nullable Filter filter, // create a FilterRel on top of the join if needed relBuilder.filter( + filter == null ? ImmutableSet.of() : filter.getVariablesSet(), RexUtil.fixUp(rexBuilder, aboveFilters, RelOptUtil.getFieldTypeList(relBuilder.peek().getRowType()))); call.transformTo(relBuilder.build()); diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index e2f83bf2c199..a16c9200955d 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -5685,7 +5685,7 @@ LogicalAggregate(group=[{}], EXPR$0=[MIN($0)]) LogicalProject(DEPTNO=[$0]) LogicalFilter(condition=[=($0, $cor0.DEPTNO)]) LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) -}))]) +}))], variablesSet=[[$cor0]]) LogicalJoin(condition=[=($7, $9)], joinType=[inner]) LogicalFilter(condition=[>($0, 10)]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) diff --git a/core/src/test/resources/sql/new-decorr.iq b/core/src/test/resources/sql/new-decorr.iq index 73ed32f38648..e7fe98e17870 100644 --- a/core/src/test/resources/sql/new-decorr.iq +++ b/core/src/test/resources/sql/new-decorr.iq @@ -401,4 +401,62 @@ EnumerableCalc(expr#0..3=[{inputs}], ENAME=[$t1], JOB=[$t2], SAL=[$t3]) !plan !} +# [CALCITE-7434] Error in new decorrelation algorithm caused by FilterJoinRule omitting variablesSet +SELECT E.EMPNO +FROM EMP E +JOIN DEPT D ON E.DEPTNO = D.DEPTNO +WHERE E.EMPNO > 10 AND D.DEPTNO = ( + SELECT MIN(D_INNER.DEPTNO) + FROM DEPT D_INNER + WHERE D_INNER.DEPTNO = E.DEPTNO); ++-------+ +| EMPNO | ++-------+ +| 7369 | +| 7499 | +| 7521 | +| 7566 | +| 7654 | +| 7698 | +| 7782 | +| 7788 | +| 7839 | +| 7844 | +| 7876 | +| 7900 | +| 7902 | +| 7934 | ++-------+ +(14 rows) + +!ok + +SELECT e1.empno +FROM emp e1, + dept d1 +where e1.deptno = d1.deptno +and e1.deptno < 10 and d1.deptno < 15 +and e1.sal > (select avg(sal) from emp e2 where e1.empno = e2.empno); ++-------+ +| EMPNO | ++-------+ ++-------+ +(0 rows) + +!ok + +SELECT e1.empno +FROM emp e1, dept d1 +where e1.deptno = d1.deptno +and e1.deptno < 10 and d1.deptno < 15 +and e1.sal > (select avg(sal) from emp e2 where e1.empno = e2.empno) +order by e1.empno; ++-------+ +| EMPNO | ++-------+ ++-------+ +(0 rows) + +!ok + # End new-decorr.iq From 89abeb07834305c3787d88f5c3f45ab73a852ab8 Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Sun, 8 Mar 2026 00:04:34 +0800 Subject: [PATCH 178/562] [CALCITE-7392] Unable to implement EnumerableCollect for SQL queries containing UNNEST --- core/src/test/resources/sql/new-decorr.iq | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/core/src/test/resources/sql/new-decorr.iq b/core/src/test/resources/sql/new-decorr.iq index e7fe98e17870..987bd22299ac 100644 --- a/core/src/test/resources/sql/new-decorr.iq +++ b/core/src/test/resources/sql/new-decorr.iq @@ -459,4 +459,16 @@ order by e1.empno; !ok +# [CALCITE-7392] Unable to implement EnumerableCollect for SQL queries containing UNNEST +SELECT ARRAY(SELECT y + 1 FROM UNNEST(s.x) y) +FROM (SELECT ARRAY[1,2,3] as x) s; ++-----------+ +| EXPR$0 | ++-----------+ +| [2, 3, 4] | ++-----------+ +(1 row) + +!ok + # End new-decorr.iq From 5032d28183d4aa7093718ad82e9ed2a7a06c2bf4 Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Sat, 7 Mar 2026 20:01:01 +0800 Subject: [PATCH 179/562] [CALCITE-7386] An error occurred while using TopDownGeneralDecorrelator to process the aggregate(col) filter --- .../sql2rel/TopDownGeneralDecorrelator.java | 22 +++++++++++++++++-- core/src/test/resources/sql/measure.iq | 2 -- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java b/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java index c1f8494ebdce..c3d2bd924961 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java @@ -18,6 +18,7 @@ import org.apache.calcite.linq4j.function.Experimental; import org.apache.calcite.plan.RelOptCostImpl; +import org.apache.calcite.plan.RelOptRule; import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.plan.Strong; import org.apache.calcite.plan.hep.HepPlanner; @@ -35,6 +36,7 @@ import org.apache.calcite.rel.core.SetOp; import org.apache.calcite.rel.core.Sort; import org.apache.calcite.rel.rules.CoreRules; +import org.apache.calcite.rel.rules.FilterProjectTransposeRule; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexCorrelVariable; @@ -196,10 +198,26 @@ private TopDownGeneralDecorrelator createSubDecorrelator() { * @return Equivalent node without correlation */ public static RelNode decorrelateQuery(RelNode rel, RelBuilder builder) { + // Use a custom FILTER_PROJECT_TRANSPOSE that does not push filters through + // projects containing V2M (measure) expressions. Pushing a filter past a + // V2M-carrying project changes the scope of the measure computation and + // produces incorrect aggregate results. + RelOptRule filterProjectTransposeNoV2m = + CoreRules.FILTER_PROJECT_TRANSPOSE.config + .as(FilterProjectTransposeRule.Config.class) + .withOperandSupplier(b0 -> + b0.operand(Filter.class) + .predicate(f -> !RexUtil.containsCorrelation(f.getCondition())) + .oneInput(b1 -> + b1.operand(Project.class) + .predicate(p -> !RexUtil.find(SqlKind.V2M).inProject(p)) + .anyInputs())) + .as(FilterProjectTransposeRule.Config.class) + .toRule(); HepProgram preProgram = HepProgram.builder() .addRuleCollection( ImmutableList.of( - CoreRules.FILTER_PROJECT_TRANSPOSE, + filterProjectTransposeNoV2m, CoreRules.FILTER_INTO_JOIN, CoreRules.FILTER_CORRELATE)) .build(); @@ -226,7 +244,7 @@ public static RelNode decorrelateQuery(RelNode rel, RelBuilder builder) { HepProgram postProgram = HepProgram.builder() .addRuleCollection( ImmutableList.of( - CoreRules.FILTER_PROJECT_TRANSPOSE, + filterProjectTransposeNoV2m, CoreRules.FILTER_INTO_JOIN, CoreRules.MARK_TO_SEMI_OR_ANTI_JOIN_RULE, CoreRules.PROJECT_MERGE, diff --git a/core/src/test/resources/sql/measure.iq b/core/src/test/resources/sql/measure.iq index 624041d090b4..aa8ee999fd77 100644 --- a/core/src/test/resources/sql/measure.iq +++ b/core/src/test/resources/sql/measure.iq @@ -667,7 +667,6 @@ group by deptno, deptno2; !ok -!if (use_old_decorr) { # Measure with FILTER select job, c, @@ -690,7 +689,6 @@ group by job; (3 rows) !ok -!} !if (false) { # Null values in GROUP BY From d3b0712f0ed9cf86170eca3bd91432a117d458c1 Mon Sep 17 00:00:00 2001 From: xuzifu666 <1206332514@qq.com> Date: Thu, 5 Mar 2026 21:13:58 +0800 Subject: [PATCH 180/562] [CALCITE-7433] Invalid unparse for cast to map type in Spark --- .../calcite/sql/dialect/SparkSqlDialect.java | 3 ++ .../calcite/util/RelToSqlConverterUtil.java | 18 +++++++++ .../rel/rel2sql/RelToSqlConverterTest.java | 38 +++++++++++++++++++ 3 files changed, 59 insertions(+) diff --git a/core/src/main/java/org/apache/calcite/sql/dialect/SparkSqlDialect.java b/core/src/main/java/org/apache/calcite/sql/dialect/SparkSqlDialect.java index 42cd259e5154..6f6e3d3917b6 100644 --- a/core/src/main/java/org/apache/calcite/sql/dialect/SparkSqlDialect.java +++ b/core/src/main/java/org/apache/calcite/sql/dialect/SparkSqlDialect.java @@ -169,6 +169,9 @@ public SparkSqlDialect(SqlDialect.Context context) { case ARRAY: return RelToSqlConverterUtil.getCastSpecAngleBracketArrayType(this, type, SqlParserPos.ZERO); + case MAP: + return RelToSqlConverterUtil.getCastSpecSparkSqlMapType(this, type, + SqlParserPos.ZERO); case MULTISET: throw new UnsupportedOperationException("Spark dialect does not support cast to " + type.getSqlTypeName()); diff --git a/core/src/main/java/org/apache/calcite/util/RelToSqlConverterUtil.java b/core/src/main/java/org/apache/calcite/util/RelToSqlConverterUtil.java index ddb37e8dac3e..1f15adc7d61e 100644 --- a/core/src/main/java/org/apache/calcite/util/RelToSqlConverterUtil.java +++ b/core/src/main/java/org/apache/calcite/util/RelToSqlConverterUtil.java @@ -390,6 +390,24 @@ public static SqlDataTypeSpec getCastSpecAngleBracketArrayType(SqlDialect dialec return new SqlDataTypeSpec(sqlArrayTypeNameSpec, SqlParserPos.ZERO); } + /** + * Transformation Map type from {@code MAP} to {@code Map}. + */ + public static SqlDataTypeSpec getCastSpecSparkSqlMapType(SqlDialect dialect, + RelDataType type, SqlParserPos pos) { + MapSqlType mapSqlType = (MapSqlType) type; + SqlDataTypeSpec keySpec = (SqlDataTypeSpec) dialect.getCastSpec(mapSqlType.getKeyType()); + SqlDataTypeSpec valueSpec = + (SqlDataTypeSpec) dialect.getCastSpec(mapSqlType.getValueType()); + SqlDataTypeSpec nonNullKeySpec = + requireNonNull(keySpec, "keySpec").withNullable(false); + SqlDataTypeSpec nonNullValueSpec = + requireNonNull(valueSpec, "valueSpec").withNullable(false); + SqlMapTypeNameSpec sqlMapTypeNameSpec = + new SqlMapTypeNameSpec(nonNullKeySpec, nonNullValueSpec, pos); + return new SqlDataTypeSpec(sqlMapTypeNameSpec, SqlParserPos.ZERO); + } + /** * ClickHouseSqlMapTypeNameSpec to parse or unparse SQL MAP type to {@code Map(VARCHAR, VARCHAR)}. */ diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java index 1b219255b439..410543122027 100644 --- a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java +++ b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java @@ -2853,6 +2853,44 @@ private SqlDialect nonOrdinalDialect() { .withSpark().ok(expectedSpark1); } + /** Test case for + * [CALCITE-7433] + * Invalid unparse for cast to map type in Spark. + */ + @Test void testCastMapSpark() { + final String query = "select cast(MAP['a',1,'b',2,'c',3]" + + " as MAP)"; + final String expectedSpark = + "SELECT CAST(MAP ('a', 1, 'b', 2, 'c', 3) AS MAP< STRING, INTEGER >)\n" + + "FROM (VALUES (0)) `t` (`ZERO`)"; + sql(query) + .withSpark().ok(expectedSpark); + + final String query1 = "select cast(MAP['a',ARRAY[1,2,3]]" + + " as MAP)"; + final String expectedSpark1 = + "SELECT CAST(MAP ('a', ARRAY (1, 2, 3)) AS MAP< STRING, ARRAY< INTEGER > >)\n" + + "FROM (VALUES (0)) `t` (`ZERO`)"; + sql(query1) + .withSpark().ok(expectedSpark1); + + final String query2 = "select cast(MAP['a',ARRAY[1.0,2.0,3.0]]" + + " as MAP)"; + final String expectedSpark2 = + "SELECT CAST(MAP ('a', ARRAY (1.0, 2.0, 3.0)) AS MAP< STRING, ARRAY< REAL > >)\n" + + "FROM (VALUES (0)) `t` (`ZERO`)"; + sql(query2) + .withSpark().ok(expectedSpark2); + + final String query3 = "select cast(MAP['a',MAP['b','c']]" + + " as MAP>)"; + final String expectedSpark3 = + "SELECT CAST(MAP ('a', MAP ('b', 'c')) AS MAP< STRING, MAP< STRING, STRING > >)\n" + + "FROM (VALUES (0)) `t` (`ZERO`)"; + sql(query3) + .withSpark().ok(expectedSpark3); + } + /** Test case for * [CALCITE-7055] * Invalid unparse for cast to array type in StarRocks. From 7471ac5930127949f456973c22c6986d01aa7e1d Mon Sep 17 00:00:00 2001 From: "wenzhuang.zwz" Date: Sat, 14 Feb 2026 17:01:45 +0800 Subject: [PATCH 181/562] [CALCITE-7417] Add a large plan benchmark for HepPlanner --- .../benchmarks/LargePlanBenchmark.java | 184 ++++++++++++++++++ 1 file changed, 184 insertions(+) create mode 100644 ubenchmark/src/jmh/java/org/apache/calcite/benchmarks/LargePlanBenchmark.java diff --git a/ubenchmark/src/jmh/java/org/apache/calcite/benchmarks/LargePlanBenchmark.java b/ubenchmark/src/jmh/java/org/apache/calcite/benchmarks/LargePlanBenchmark.java new file mode 100644 index 000000000000..6d1cab341364 --- /dev/null +++ b/ubenchmark/src/jmh/java/org/apache/calcite/benchmarks/LargePlanBenchmark.java @@ -0,0 +1,184 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.benchmarks; + +import org.apache.calcite.plan.hep.HepMatchOrder; +import org.apache.calcite.plan.hep.HepPlanner; +import org.apache.calcite.plan.hep.HepProgram; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.logical.LogicalUnion; +import org.apache.calcite.rel.rules.CoreRules; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.schema.SchemaPlus; +import org.apache.calcite.schema.impl.AbstractTable; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.tools.Frameworks; +import org.apache.calcite.tools.RelBuilder; + +import com.google.common.collect.ImmutableList; + +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Threads; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +import java.util.concurrent.TimeUnit; + +/** + * Benchmark that constructs a synthetic query plan consisting of a large plan. + * + *

    This benchmark primarily measures planner overhead under heavy rule activity: matching and + * firing rules, replacing RelNodes, and traversing the evolving plan during optimization. + * It also simulates a multiphase optimization flow by running multiple HepPrograms sequentially. + * + *

    Each UNION input branch contains layers of Projects and Filters that are intentionally + * simplifiable, so that typical planner rules (e.g., Project/Filter simplification) + * are repeatedly applicable. + */ + +@Fork(value = 1, jvmArgsPrepend = {"-Xss200m"}) +@Measurement(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS) +@Warmup(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@State(Scope.Thread) +@Threads(1) +public class LargePlanBenchmark { + + @Param({"100", "1000", "5000", "10000"}) + int unionNum; + + private RelBuilder builder; + + @Setup(Level.Trial) + public void setup() { + SchemaPlus rootSchema = Frameworks.createRootSchema(true); + rootSchema.add("EMP", new AbstractTable() { + @Override public RelDataType getRowType(RelDataTypeFactory typeFactory) { + return typeFactory.builder() + .add("EMPNO", SqlTypeName.INTEGER) + .add("ENAME", SqlTypeName.VARCHAR) + .add("JOB", SqlTypeName.VARCHAR) + .add("MGR", SqlTypeName.INTEGER) + .add("HIREDATE", SqlTypeName.DATE) + .add("SAL", SqlTypeName.INTEGER) + .add("COMM", SqlTypeName.INTEGER) + .add("DEPTNO", SqlTypeName.INTEGER) + .build(); + } + }); + + builder = + RelBuilder.create(Frameworks.newConfigBuilder() + .defaultSchema(rootSchema) + .build()); + } + + // select ENAME, i as cat_id, 'i' as cat_name, i as require_free_postage, + // 0 as require_15return, 0 as require_48hour, 1 as require_insurance + // from emp + // where EMPNO = i and MGR >= 0 and MGR <= 0 and ENAME = 'Y' and SAL = i + private RelNode makeSelectBranch(int i) { + return builder.scan("EMP") + .filter( + builder.and( + builder.equals(builder.field("EMPNO"), builder.literal(i)), + builder.call(SqlStdOperatorTable.GREATER_THAN_OR_EQUAL, + builder.field("MGR"), builder.literal(0)), + builder.call(SqlStdOperatorTable.LESS_THAN_OR_EQUAL, + builder.field("MGR"), builder.literal(0)), + builder.equals(builder.field("ENAME"), builder.literal("Y")), + builder.equals(builder.field("SAL"), builder.literal(i)) + ) + ) + .project( + builder.field("ENAME"), + builder.alias(builder.literal(i), "cat_id"), + builder.alias(builder.literal(String.valueOf(i)), "cat_name"), + builder.alias(builder.literal(i), "require_free_postage"), + builder.alias(builder.literal(0), "require_15return"), + builder.alias(builder.literal(0), "require_48hour"), + builder.alias(builder.literal(1), "require_insurance") + ) + .build(); + } + + private RelNode makeUnionTree(int unionNum) { + RelNode union = makeSelectBranch(0); + for (int i = 1; i < unionNum; i++) { + RelNode right = makeSelectBranch(i); + union = LogicalUnion.create(ImmutableList.of(union, right), true); + } + union = LogicalUnion.create(ImmutableList.of(union, makeSelectBranch(unionNum)), true); + return union; + } + + @Benchmark + public void testLargeUnionPlan() { + RelNode root = makeUnionTree(unionNum); + + HepProgram filterReduce = HepProgram.builder() + .addMatchOrder(HepMatchOrder.DEPTH_FIRST) + .addRuleInstance(CoreRules.FILTER_REDUCE_EXPRESSIONS) + .build(); + + HepProgram projectReduce = HepProgram.builder() + .addMatchOrder(HepMatchOrder.DEPTH_FIRST) + .addRuleInstance(CoreRules.PROJECT_REDUCE_EXPRESSIONS) + .build(); + + // Phrase 1 + HepPlanner planner = new HepPlanner(filterReduce); + planner.setRoot(root); + root = planner.findBestExp(); + planner.clear(); + + // ... do some things cannot be done in planner.findBestExp() ... + // Phrase 2 + planner = new HepPlanner(projectReduce); + planner.setRoot(root); + root = planner.findBestExp(); + planner.clear(); + + // TODO LATER large plan optimization + // TODO LATER set "-Dcalcite.disable.generate.type.digest.string=true" + } + + public static void main(String[] args) throws RunnerException { + Options opt = new OptionsBuilder() + .include(LargePlanBenchmark.class.getSimpleName()) + .detectJvmArgs() + .build(); + + new Runner(opt).run(); + } +} From 60636fe58742636443e4936e9242dfa93df014a0 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Mon, 9 Mar 2026 14:35:04 -0700 Subject: [PATCH 182/562] [CALCITE-7435] WINDOW functions should allow ORDER BY fields of type INTERVAL Signed-off-by: Mihai Budiu --- .../calcite/sql/type/SqlTypeFamily.java | 4 ++++ core/src/test/resources/sql/winagg.iq | 24 +++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/core/src/main/java/org/apache/calcite/sql/type/SqlTypeFamily.java b/core/src/main/java/org/apache/calcite/sql/type/SqlTypeFamily.java index 0e9b14e35cf9..396932555a09 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/SqlTypeFamily.java +++ b/core/src/main/java/org/apache/calcite/sql/type/SqlTypeFamily.java @@ -160,6 +160,10 @@ public List allowableDifferenceTypes() { switch (this) { case NUMERIC: return ImmutableList.of(NUMERIC); + case INTERVAL_DAY_TIME: + return ImmutableList.of(INTERVAL_DAY_TIME); + case INTERVAL_YEAR_MONTH: + return ImmutableList.of(INTERVAL_YEAR_MONTH); case DATE: case TIME: case TIMESTAMP: diff --git a/core/src/test/resources/sql/winagg.iq b/core/src/test/resources/sql/winagg.iq index beb90d9fd4df..6a32b3b3f7b0 100644 --- a/core/src/test/resources/sql/winagg.iq +++ b/core/src/test/resources/sql/winagg.iq @@ -18,6 +18,30 @@ !use post !set outputformat mysql +# Test case for [CALCITE-7435] WINDOW functions should allow ORDER BY fields of type INTERVAL +# This result has been validated on Postgres by slightly changing the query (subtraction is different in Postgres) +WITH +T(ts, l) AS (VALUES(TIMESTAMP '2020-01-01 10:00:00', 10), + (TIMESTAMP '2020-02-01 10:00:00', 10), + (TIMESTAMP '2019-12-30 20:00:00', 10)), +IT AS (SELECT(ts - TIMESTAMP '2020-01-01 00:00:00') HOURS AS t, l FROM T) +SELECT *, + COUNT(*) OVER ( + PARTITION BY l + ORDER BY t + RANGE BETWEEN INTERVAL 2 DAYS PRECEDING AND INTERVAL 1 DAYS PRECEDING) AS c +FROM It; ++------+----+---+ +| T | L | C | ++------+----+---+ +| +10 | 10 | 1 | +| -28 | 10 | 0 | +| +754 | 10 | 0 | ++------+----+---+ +(3 rows) + +!ok + # Multiple window functions sharing a single window select count(*) over(partition by gender order by ename) as count1, count(*) over(partition by deptno order by ename) as count2, From 0b529c7f3ffaaa4bd2acf6d89f5facbb7383a227 Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Fri, 6 Mar 2026 15:07:04 +0800 Subject: [PATCH 183/562] [CALCITE-7426] Add a PR submission template to Calcite --- .github/pull_request_template.md | 43 ++++++++++++++++++++++++++++++++ .ratignore | 2 +- 2 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 .github/pull_request_template.md diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 000000000000..842d8fbd699a --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,43 @@ + + +## Jira Link + +[CALCITE-XXXX](https://issues.apache.org/jira/browse/CALCITE-XXXX) + +## Changes Proposed + diff --git a/.ratignore b/.ratignore index 65bd8f2cec84..50e978b59ac2 100644 --- a/.ratignore +++ b/.ratignore @@ -2,7 +2,7 @@ **/.editorconfig **/.gitignore **/.gitattributes -.github/workflows +.github/** .ratignore **/META-INF/services/java.sql.Driver **/src/test/resources/**/*.csv From c259f4a733983469dda9c4c15351c8c66da425d9 Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Thu, 12 Mar 2026 22:30:26 +0800 Subject: [PATCH 184/562] Test case for [CALCITE-3366] RelDecorrelator supports Union --- core/src/test/resources/sql/sub-query.iq | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/core/src/test/resources/sql/sub-query.iq b/core/src/test/resources/sql/sub-query.iq index 51cab9f01e06..ac2fe6d8aa11 100644 --- a/core/src/test/resources/sql/sub-query.iq +++ b/core/src/test/resources/sql/sub-query.iq @@ -9152,4 +9152,19 @@ from (values (1), (2), (3)) as t(v); !set insubquerythreshold 20 !} +# Test case for [CALCITE-3366] RelDecorrelator supports Union +SELECT deptno FROM dept where exists +(SELECT 1 FROM emp where sal < 100 and emp.deptno=dept.deptno +union all +SELECT 1 FROM emp where sal > 200 and emp.deptno=dept.deptno); ++--------+ +| DEPTNO | ++--------+ +| 10 | +| 20 | +| 30 | ++--------+ +(3 rows) + +!ok # End sub-query.iq From 45e1f50c39829a2d1d9b6197b8780f22ee2d9b70 Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Fri, 13 Mar 2026 20:06:36 +0800 Subject: [PATCH 185/562] Site: Add Zhen Chen as PMC --- site/_data/contributors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site/_data/contributors.yml b/site/_data/contributors.yml index 8a075acd9445..27328f4f7342 100644 --- a/site/_data/contributors.yml +++ b/site/_data/contributors.yml @@ -425,7 +425,7 @@ apacheId: zhenchen githubId: xiedeyantu org: - role: Committer + role: PMC - name: Zhen Wang apacheId: zhenw githubId: zinking From 8c4539ae98b4ebad6bba1c6dc64eb4f19a695738 Mon Sep 17 00:00:00 2001 From: Stamatis Zampetakis Date: Fri, 13 Mar 2026 15:59:21 +0100 Subject: [PATCH 186/562] [CALCITE-7441] AggregateFilterToFilteredAggregateRule fails when WHERE condition is nullable --- ...ggregateFilterToFilteredAggregateRule.java | 4 ++++ ...gateFilterToFilteredAggregateRuleTest.java | 6 ++++++ ...egateFilterToFilteredAggregateRuleTest.xml | 19 +++++++++++++++++++ 3 files changed, 29 insertions(+) diff --git a/core/src/main/java/org/apache/calcite/rel/rules/AggregateFilterToFilteredAggregateRule.java b/core/src/main/java/org/apache/calcite/rel/rules/AggregateFilterToFilteredAggregateRule.java index dd82f697622c..f814b6b9b8b9 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/AggregateFilterToFilteredAggregateRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/AggregateFilterToFilteredAggregateRule.java @@ -22,6 +22,7 @@ import org.apache.calcite.rel.core.AggregateCall; import org.apache.calcite.rel.core.Filter; import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.tools.RelBuilder; import org.immutables.value.Value; @@ -76,6 +77,9 @@ private AggregateFilterToFilteredAggregateRule(Config config) { return; } RexNode condition = filter.getCondition(); + if (condition.getType().isNullable()) { + condition = builder.call(SqlStdOperatorTable.IS_TRUE, condition); + } // If the aggregate call has its own filter, combine it with the filter condition. if (aggCall.hasFilter()) { condition = builder.and(condition, builder.field(aggCall.filterArg)); diff --git a/core/src/test/java/org/apache/calcite/test/AggregateFilterToFilteredAggregateRuleTest.java b/core/src/test/java/org/apache/calcite/test/AggregateFilterToFilteredAggregateRuleTest.java index 9e6cd3ea339a..326524e74741 100644 --- a/core/src/test/java/org/apache/calcite/test/AggregateFilterToFilteredAggregateRuleTest.java +++ b/core/src/test/java/org/apache/calcite/test/AggregateFilterToFilteredAggregateRuleTest.java @@ -58,6 +58,12 @@ private static RelOptFixture sql(String sql) { .withRule(AGGREGATE_FILTER_TO_FILTERED_AGGREGATE).check(); } + @Test void testSingleColumnAggregateWithFilterOnNullableColumn() { + String sql = "select sum(sal) from emp where mgr = 10"; + sql(sql).withPreRule(AGGREGATE_PROJECT_MERGE) + .withRule(AGGREGATE_FILTER_TO_FILTERED_AGGREGATE).check(); + } + @Test void testSingleStarAggregate() { String sql = "select count(*) from emp where deptno = 10"; sql(sql).withPreRule(AGGREGATE_PROJECT_MERGE) diff --git a/core/src/test/resources/org/apache/calcite/test/AggregateFilterToFilteredAggregateRuleTest.xml b/core/src/test/resources/org/apache/calcite/test/AggregateFilterToFilteredAggregateRuleTest.xml index b31112c3f337..47ca7c9bd138 100644 --- a/core/src/test/resources/org/apache/calcite/test/AggregateFilterToFilteredAggregateRuleTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/AggregateFilterToFilteredAggregateRuleTest.xml @@ -101,6 +101,25 @@ LogicalAggregate(group=[{}], EXPR$0=[SUM($5)]) LogicalAggregate(group=[{}], EXPR$0=[SUM($0) FILTER $1]) LogicalProject(SAL=[$5], $f9=[=($7, 10)]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +]]> + + + + + + + + + + + From 66dfefa464b1f8a452789da4c8c87a7392d7a79c Mon Sep 17 00:00:00 2001 From: "guohongyu.7" Date: Thu, 12 Mar 2026 22:52:58 +0800 Subject: [PATCH 187/562] [CALCITE-7259] Drop commons-lang3 dependency Replace commons-lang3 usages with local helpers and add lint checks to prevent re-introducing commons-lang3 imports. --- arrow/build.gradle.kts | 1 - bom/build.gradle.kts | 1 - .../calcite/test/CassandraExtension.java | 4 +- core/build.gradle.kts | 1 - .../rel/metadata/RelMdColumnUniqueness.java | 8 +- .../calcite/runtime/CompressionFunctions.java | 4 +- .../apache/calcite/runtime/SqlFunctions.java | 7 +- .../apache/calcite/runtime/XmlFunctions.java | 7 +- .../java/org/apache/calcite/util/Util.java | 101 ++++++++++++++++++ .../util/format/FormatElementEnum.java | 15 ++- .../plan/volcano/VolcanoPlannerTest.java | 18 +++- .../org/apache/calcite/test/LintTest.java | 27 +++++ .../org/apache/calcite/util/UtilTest.java | 39 +++++++ druid/build.gradle.kts | 1 - .../calcite/adapter/druid/DruidRules.java | 36 ++++--- file/build.gradle.kts | 1 - .../calcite/adapter/file/CsvEnumerator.java | 62 ++++++++--- geode/build.gradle.kts | 1 - .../adapter/geode/util/GeodeUtils.java | 7 +- gradle.properties | 1 - innodb/build.gradle.kts | 1 - .../calcite/adapter/innodb/InnodbSchema.java | 4 +- .../adapter/innodb/InnodbSchemaFactory.java | 4 +- .../innodb/InnodbAdapterDataTypesTest.java | 54 ++++++---- .../calcite/adapter/os/OsAdapterTest.java | 18 ++-- redis/build.gradle.kts | 1 - .../adapter/redis/RedisDataProcess.java | 6 +- .../adapter/redis/RedisEnumerator.java | 9 +- .../adapter/redis/RedisJedisManager.java | 3 +- .../calcite/adapter/redis/RedisSchema.java | 49 +++++++-- .../calcite/adapter/redis/RedisCaseBase.java | 7 +- testkit/build.gradle.kts | 1 - .../apache/calcite/test/CalciteAssert.java | 15 ++- 33 files changed, 383 insertions(+), 131 deletions(-) diff --git a/arrow/build.gradle.kts b/arrow/build.gradle.kts index ecbe01e2932d..598aa8a87972 100644 --- a/arrow/build.gradle.kts +++ b/arrow/build.gradle.kts @@ -29,7 +29,6 @@ dependencies { testImplementation("org.apache.arrow:arrow-jdbc") testImplementation("net.hydromatic:scott-data-hsqldb") - testImplementation("org.apache.commons:commons-lang3") testImplementation(project(":core")) testImplementation(project(":testkit")) } diff --git a/bom/build.gradle.kts b/bom/build.gradle.kts index 51c4177d108b..5a1ad75511d0 100644 --- a/bom/build.gradle.kts +++ b/bom/build.gradle.kts @@ -106,7 +106,6 @@ dependencies { apiv("org.apache.calcite.avatica:avatica-server", "calcite.avatica") apiv("org.apache.cassandra:cassandra-all") apiv("org.apache.commons:commons-dbcp2") - apiv("org.apache.commons:commons-lang3") apiv("org.apache.commons:commons-math3") apiv("org.apache.commons:commons-pool2") apiv("org.apache.commons:commons-collections4") diff --git a/cassandra/src/test/java/org/apache/calcite/test/CassandraExtension.java b/cassandra/src/test/java/org/apache/calcite/test/CassandraExtension.java index fa66f78edc50..03097925a5da 100644 --- a/cassandra/src/test/java/org/apache/calcite/test/CassandraExtension.java +++ b/cassandra/src/test/java/org/apache/calcite/test/CassandraExtension.java @@ -19,12 +19,12 @@ import org.apache.calcite.config.CalciteSystemProperty; import org.apache.calcite.util.Sources; import org.apache.calcite.util.TestUtil; +import org.apache.calcite.util.Util; import org.apache.cassandra.concurrent.Stage; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.service.CassandraDaemon; import org.apache.cassandra.service.StorageService; -import org.apache.commons.lang3.SystemUtils; import com.datastax.oss.driver.api.core.CqlSession; import com.google.common.collect.ImmutableMap; @@ -134,7 +134,7 @@ private static CassandraResource getOrCreate(ExtensionContext context) { boolean compatibleGuava = TestUtil.getGuavaMajorVersion() >= 23; // remove JVM check once Cassandra supports Eclipse OpenJ9 JVM boolean compatibleJVM = !"Eclipse OpenJ9".equals(TestUtil.getJavaVirtualMachineVendor()); - boolean compatibleOS = !SystemUtils.IS_OS_WINDOWS; + boolean compatibleOS = !Util.isWindows(); if (enabled && compatibleJdk && compatibleGuava && compatibleJVM && compatibleOS) { return ConditionEvaluationResult.enabled("Cassandra tests enabled"); } diff --git a/core/build.gradle.kts b/core/build.gradle.kts index 842a34fba69b..2cb75e9af0ba 100644 --- a/core/build.gradle.kts +++ b/core/build.gradle.kts @@ -69,7 +69,6 @@ dependencies { implementation("commons-codec:commons-codec") implementation("net.hydromatic:aggdesigner-algorithm") implementation("org.apache.commons:commons-dbcp2") - implementation("org.apache.commons:commons-lang3") implementation("org.apache.commons:commons-math3") implementation("org.apache.commons:commons-text") implementation("org.jooq:joou-java-6") diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdColumnUniqueness.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdColumnUniqueness.java index a4ac2e9c5adc..e69da277451c 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdColumnUniqueness.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdColumnUniqueness.java @@ -55,8 +55,6 @@ import org.apache.calcite.util.Pair; import org.apache.calcite.util.Util; -import org.apache.commons.lang3.mutable.MutableBoolean; - import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -571,17 +569,17 @@ private static void addInputRefIfOtherConstant(ImmutableBitSet.Builder builder, */ private static boolean isConstantScalarQuery(RexNode rexNode) { if (rexNode.getKind() == SqlKind.SCALAR_QUERY) { - MutableBoolean hasCorrelatingVars = new MutableBoolean(false); + final boolean[] hasCorrelatingVars = {false}; ((RexSubQuery) rexNode).rel.accept(new RelShuttleImpl() { @Override public RelNode visit(final LogicalFilter filter) { if (RexUtil.containsCorrelation(filter.getCondition())) { - hasCorrelatingVars.setTrue(); + hasCorrelatingVars[0] = true; return filter; } return super.visit(filter); } }); - return hasCorrelatingVars.isFalse(); + return !hasCorrelatingVars[0]; } return false; } diff --git a/core/src/main/java/org/apache/calcite/runtime/CompressionFunctions.java b/core/src/main/java/org/apache/calcite/runtime/CompressionFunctions.java index 1df63573eeb5..59bac5d5fd6c 100644 --- a/core/src/main/java/org/apache/calcite/runtime/CompressionFunctions.java +++ b/core/src/main/java/org/apache/calcite/runtime/CompressionFunctions.java @@ -18,8 +18,6 @@ import org.apache.calcite.avatica.util.ByteString; -import org.apache.commons.lang3.StringUtils; - import org.checkerframework.checker.nullness.qual.Nullable; import java.io.ByteArrayOutputStream; @@ -47,7 +45,7 @@ private CompressionFunctions() { if (data == null) { return null; } - if (StringUtils.isEmpty(data)) { + if (data.isEmpty()) { return new ByteString(new byte[0]); } ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); diff --git a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java index a729d34916f8..e2adce8ea1c7 100644 --- a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java +++ b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java @@ -60,7 +60,6 @@ import org.apache.commons.codec.binary.Hex; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.codec.language.Soundex; -import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.math3.util.CombinatoricsUtils; import org.apache.commons.text.StringEscapeUtils; import org.apache.commons.text.similarity.LevenshteinDistance; @@ -6521,7 +6520,7 @@ public static long customTimestampCeil(DataContext root, /** SQL {@code TRANSLATE(string, search_chars, replacement_chars)} * function. */ public static String translate3(String s, String search, String replacement) { - return org.apache.commons.lang3.StringUtils.replaceChars(s, search, replacement); + return Util.replaceChars(s, search, replacement); } /** SQL {@code REPLACE(string, search, replacement)} function. */ @@ -6534,7 +6533,7 @@ public static String replace(String s, String search, String replacement, return s.replace(search, replacement); } // for MSSQL's REPLACE function, search pattern is case-insensitive during matching - return org.apache.commons.lang3.Strings.CI.replace(s, search, replacement); + return Util.replaceIgnoreCase(s, search, replacement); } /** Helper for "array element reference". Caller has already ensured that @@ -7699,7 +7698,7 @@ private static String age(long timestamp1, long timestamp2) { sb.append( String.format(Locale.ROOT, "%02d:%02d:%02d.%s", hours, minutes, seconds, millisString)); - } else if (ObjectUtils.isNotEmpty(sb) + } else if (sb.length() != 0 && hours == 0 && minutes == 0 && seconds == 0 && millis == 0) { return sb.toString().trim(); } else { diff --git a/core/src/main/java/org/apache/calcite/runtime/XmlFunctions.java b/core/src/main/java/org/apache/calcite/runtime/XmlFunctions.java index 2a8af55e2360..00660a1b4643 100644 --- a/core/src/main/java/org/apache/calcite/runtime/XmlFunctions.java +++ b/core/src/main/java/org/apache/calcite/runtime/XmlFunctions.java @@ -18,8 +18,7 @@ import org.apache.calcite.util.SimpleNamespaceContext; import org.apache.calcite.util.TryThreadLocal; - -import org.apache.commons.lang3.StringUtils; +import org.apache.calcite.util.Util; import org.checkerframework.checker.nullness.qual.Nullable; import org.w3c.dom.Node; @@ -133,7 +132,7 @@ private XmlFunctions() { () -> "firstChild of node " + item); result.add(firstChild.getTextContent()); } - return StringUtils.join(result, " "); + return Util.joinNullable(result, " "); } catch (XPathExpressionException e) { return xpathExpression.evaluate(documentNode); } @@ -189,7 +188,7 @@ private XmlFunctions() { for (int i = 0; i < nodes.getLength(); i++) { result.add(convertNodeToString(castNonNull(nodes.item(i)))); } - return StringUtils.join(result, ""); + return Util.joinNullable(result, ""); } catch (XPathExpressionException e) { Node node = (Node) xpathExpression .evaluate(documentNode, XPathConstants.NODE); diff --git a/core/src/main/java/org/apache/calcite/util/Util.java b/core/src/main/java/org/apache/calcite/util/Util.java index d911103f2637..05b9927192ed 100644 --- a/core/src/main/java/org/apache/calcite/util/Util.java +++ b/core/src/main/java/org/apache/calcite/util/Util.java @@ -31,6 +31,7 @@ import org.apache.calcite.sql.util.SqlBasicVisitor; import com.google.common.base.Preconditions; +import com.google.common.base.Strings; import com.google.common.base.Throwables; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; @@ -642,6 +643,106 @@ public static String replace( return sb.toString(); } + /** Right-pads a string with {@code padChar} until it reaches {@code size}. + * + *

    Equivalent to {@code org.apache.commons.lang3.StringUtils#rightPad(String, int, char)} + * for non-null inputs. + */ + public static String rightPad(String s, int size, char padChar) { + return Strings.padEnd(s, size, padChar); + } + + /** Joins {@code parts} using {@code sep}. + * + *

    Null elements are treated as empty strings. + */ + public static String joinNullable(Iterable parts, String sep) { + final List strings = new ArrayList<>(); + for (Object o : parts) { + strings.add(o == null ? "" : o.toString()); + } + return String.join(sep, strings); + } + + /** Returns whether the current runtime is Windows. + * + *

    Derived from system property {@code os.name}, using case-insensitive + * prefix match against {@code "Windows"}. + * + *

    This method is intended to replace commons-lang3's + * {@code SystemUtils#IS_OS_WINDOWS}. + */ + public static boolean isWindows() { + final String osName = System.getProperty("os.name"); + return osName != null + && osName.regionMatches(true, 0, "Windows", 0, "Windows".length()); + } + + /** Replaces characters in {@code s} according to {@code search}/{@code replacement} mapping. + * + *

    Semantics are aligned with {@code org.apache.commons.lang3.StringUtils#replaceChars}: + * characters found in {@code search} are replaced by the character in the same position in + * {@code replacement}; if {@code replacement} is shorter, remaining matches are removed. + */ + public static @PolyNull String replaceChars(@PolyNull String s, @Nullable String search, + @Nullable String replacement) { + if (s == null || s.isEmpty() || search == null || search.isEmpty()) { + return s; + } + final String repl = replacement == null ? "" : replacement; + boolean modified = false; + final StringBuilder b = new StringBuilder(s.length()); + for (int i = 0; i < s.length(); i++) { + final char ch = s.charAt(i); + final int j = search.indexOf(ch); + if (j >= 0) { + modified = true; + if (j < repl.length()) { + b.append(repl.charAt(j)); + } + // else: delete character + } else { + b.append(ch); + } + } + return modified ? b.toString() : s; + } + + /** Case-insensitive replace of all occurrences of {@code search} in {@code s}. + * + *

    Equivalent to commons-lang's {@code StringUtils.replaceIgnoreCase}, but only supports + * non-null inputs. + */ + public static String replaceIgnoreCase(String s, String search, String replacement) { + if (search.isEmpty()) { + return s; + } + final int replLength = search.length(); + int start = 0; + int end = indexOfIgnoreCase(s, search, start); + if (end < 0) { + return s; + } + final StringBuilder out = new StringBuilder(s.length()); + while (end >= 0) { + out.append(s, start, end).append(replacement); + start = end + replLength; + end = indexOfIgnoreCase(s, search, start); + } + out.append(s, start, s.length()); + return out.toString(); + } + + private static int indexOfIgnoreCase(String str, String search, int fromIndex) { + final int max = str.length() - search.length(); + for (int i = Math.max(0, fromIndex); i <= max; i++) { + if (str.regionMatches(true, i, search, 0, search.length())) { + return i; + } + } + return -1; + } + /** * Creates a file-protocol URL for the given file. */ diff --git a/core/src/main/java/org/apache/calcite/util/format/FormatElementEnum.java b/core/src/main/java/org/apache/calcite/util/format/FormatElementEnum.java index 0e289cf678e0..66061d3596da 100644 --- a/core/src/main/java/org/apache/calcite/util/format/FormatElementEnum.java +++ b/core/src/main/java/org/apache/calcite/util/format/FormatElementEnum.java @@ -18,8 +18,7 @@ import org.apache.calcite.avatica.util.DateTimeUtils; import org.apache.calcite.util.TryThreadLocal; - -import org.apache.commons.lang3.StringUtils; +import org.apache.calcite.util.Util; import java.text.DateFormat; import java.text.SimpleDateFormat; @@ -147,7 +146,7 @@ public enum FormatElementEnum implements FormatElement { // Padding zeroes to right as SimpleDateFormat supports precision only up to 3 places. // Refer to // [CALCITE-6269] Fix missing/broken BigQuery date-time format elements. - sb.append(StringUtils.rightPad(work.sssFormat.format(date), 4, "0")); + sb.append(Util.rightPad(work.sssFormat.format(date), 4, '0')); } }, FF5("S", "Fractional seconds to 5 digits") { @@ -156,7 +155,7 @@ public enum FormatElementEnum implements FormatElement { // Padding zeroes to right as SimpleDateFormat supports precision only up to 3 places. // Refer to // [CALCITE-6269] Fix missing/broken BigQuery date-time format elements. - sb.append(StringUtils.rightPad(work.sssFormat.format(date), 5, "0")); + sb.append(Util.rightPad(work.sssFormat.format(date), 5, '0')); } }, FF6("S", "Fractional seconds to 6 digits") { @@ -165,7 +164,7 @@ public enum FormatElementEnum implements FormatElement { // Padding zeroes to right as SimpleDateFormat supports precision only up to 3 places. // Refer to // [CALCITE-6269] Fix missing/broken BigQuery date-time format elements. - sb.append(StringUtils.rightPad(work.sssFormat.format(date), 6, "0")); + sb.append(Util.rightPad(work.sssFormat.format(date), 6, '0')); } }, FF7("S", "Fractional seconds to 6 digits") { @@ -174,7 +173,7 @@ public enum FormatElementEnum implements FormatElement { // Padding zeroes to right as SimpleDateFormat supports precision only up to 3 places. // Refer to // [CALCITE-6269] Fix missing/broken BigQuery date-time format elements. - sb.append(StringUtils.rightPad(work.sssFormat.format(date), 7, "0")); + sb.append(Util.rightPad(work.sssFormat.format(date), 7, '0')); } }, FF8("S", "Fractional seconds to 6 digits") { @@ -183,7 +182,7 @@ public enum FormatElementEnum implements FormatElement { // Padding zeroes to right as SimpleDateFormat supports precision only up to 3 places. // Refer to // [CALCITE-6269] Fix missing/broken BigQuery date-time format elements. - sb.append(StringUtils.rightPad(work.sssFormat.format(date), 8, "0")); + sb.append(Util.rightPad(work.sssFormat.format(date), 8, '0')); } }, FF9("S", "Fractional seconds to 6 digits") { @@ -192,7 +191,7 @@ public enum FormatElementEnum implements FormatElement { // Padding zeroes to right as SimpleDateFormat supports precision only up to 3 places. // Refer to // [CALCITE-6269] Fix missing/broken BigQuery date-time format elements. - sb.append(StringUtils.rightPad(work.sssFormat.format(date), 9, "0")); + sb.append(Util.rightPad(work.sssFormat.format(date), 9, '0')); } }, HH12("h", "The hour (12-hour clock) as a decimal number (01-12)") { diff --git a/core/src/test/java/org/apache/calcite/plan/volcano/VolcanoPlannerTest.java b/core/src/test/java/org/apache/calcite/plan/volcano/VolcanoPlannerTest.java index 909312dd4e3f..12c5fbeba3c2 100644 --- a/core/src/test/java/org/apache/calcite/plan/volcano/VolcanoPlannerTest.java +++ b/core/src/test/java/org/apache/calcite/plan/volcano/VolcanoPlannerTest.java @@ -42,8 +42,6 @@ import org.apache.calcite.tools.RelBuilder; import org.apache.calcite.util.Pair; -import org.apache.commons.lang3.exception.ExceptionUtils; - import org.immutables.value.Value; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; @@ -362,7 +360,7 @@ public interface Config extends RelRule.Config { "Should throw exception fail since the type mismatches after " + "applying rule."); - Throwable exception = ExceptionUtils.getRootCause(ex); + Throwable exception = getRootCause(ex); assertThat(exception, instanceOf(IllegalArgumentException.class)); assertThat( exception.getMessage(), isLinux("Type mismatch:\n" @@ -372,6 +370,20 @@ public interface Config extends RelRule.Config { + "this: JavaType(class java.lang.Integer) -> JavaType(void) NOT NULL\n")); } + public static Throwable getRootCause(final Throwable throwable) { + final List list = getThrowableList(throwable); + return list.isEmpty() ? null : list.get(list.size() - 1); + } + + public static List getThrowableList(Throwable throwable) { + final List list = new ArrayList<>(); + while (throwable != null && !list.contains(throwable)) { + list.add(throwable); + throwable = throwable.getCause(); + } + return list; + } + private static List sort(List list) { final List list2 = new ArrayList<>(list); Collections.sort(list2); diff --git a/core/src/test/java/org/apache/calcite/test/LintTest.java b/core/src/test/java/org/apache/calcite/test/LintTest.java index ff012bfa16c0..089d76edbaa4 100644 --- a/core/src/test/java/org/apache/calcite/test/LintTest.java +++ b/core/src/test/java/org/apache/calcite/test/LintTest.java @@ -49,6 +49,7 @@ import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.hasToString; import static org.hamcrest.Matchers.startsWith; import static org.junit.jupiter.api.Assumptions.assumeTrue; @@ -64,6 +65,10 @@ class LintTest { compile("^(\\[CALCITE-[0-9]{1,4}][ ]).*"); private static final Pattern PATTERN = compile("^ *(// )?"); + private static final Pattern COMMONS_LANG3_IMPORT_PATTERN = + compile("^\\s*import\\s+(static\\s+)?" + + "org\\.apache\\.commons\\.lang3\\..*;\\s*$"); + private static final String TERMINOLOGY_ERROR_MSG = "Message contains '%s' word; use one of the following instead: %s"; private static final List TERM_RULES = initTerminologyRules(); @@ -94,6 +99,13 @@ private Puffin.Program makeProgram() { && !skipping(line), line -> line.state().message("Tab", line)) + // Forbid importing commons-lang3 (Calcite should not use it). + .add(line -> isJava(line.filename()) + && line.matches(COMMONS_LANG3_IMPORT_PATTERN.pattern()), + line -> line.state().message( + "Forbidden import from 'org.apache.commons.lang3' (commons-lang3 is not allowed)", + line)) + // Comment without space .add(line -> line.matches(".* //[^ ].*") && !line.source().fileOpt() @@ -236,6 +248,21 @@ && isJava(line.filename()), .build(); } + @Test void testNoCommonsLang3Import() { + final Puffin.Program program = makeProgram(); + final String code = "import org.apache.commons.lang3.StringUtils;\n" + + "class X {}\n"; + final StringWriter sw = new StringWriter(); + final GlobalState g; + try (PrintWriter pw = new PrintWriter(sw)) { + g = program.execute(Stream.of(Sources.of(code)), pw); + } + final String expected = "[GuavaCharSource{memory}:1:" + + "Forbidden import from 'org.apache.commons.lang3' " + + "(commons-lang3 is not allowed)]"; + assertThat(g.messages, hasToString(expected)); + } + /** Strips the last character from a string. */ private static String skipLast(String s) { return s.substring(0, s.length() - 1); diff --git a/core/src/test/java/org/apache/calcite/util/UtilTest.java b/core/src/test/java/org/apache/calcite/util/UtilTest.java index 7d8797f19caf..2d7ab0f606b5 100644 --- a/core/src/test/java/org/apache/calcite/util/UtilTest.java +++ b/core/src/test/java/org/apache/calcite/util/UtilTest.java @@ -241,6 +241,45 @@ class UtilTest { assertThatScientific("-0.0", is("-0.0E0")); } + @Test void testRightPad() { + assertThat(Util.rightPad("a", 1, 'x'), is("a")); + assertThat(Util.rightPad("a", 2, 'x'), is("ax")); + assertThat(Util.rightPad("a", 4, 'x'), is("axxx")); + assertThat(Util.rightPad("", 3, 'x'), is("xxx")); + } + + @Test void testJoinNullable() { + final List<@Nullable Object> parts = Arrays.asList("a", null, "b"); + assertThat(Util.joinNullable(parts, ":"), is("a::b")); + assertThat(Util.joinNullable(Collections.emptyList(), ","), is("")); + assertThat(Util.joinNullable(Arrays.asList(null, null), ":"), is(":")); + } + + @Test void testReplaceChars() { + assertNull(Util.replaceChars((String) null, "ab", "x")); + assertThat(Util.replaceChars("", "ab", "x"), is("")); + + final String s = "abc"; + assertThat(Util.replaceChars(s, null, "x"), sameInstance(s)); + assertThat(Util.replaceChars(s, "", "x"), sameInstance(s)); + assertThat(Util.replaceChars(s, "x", "y"), sameInstance(s)); + + assertThat(Util.replaceChars("abc", "ab", null), is("c")); + assertThat(Util.replaceChars("abc", "abc", "12"), is("12")); + assertThat(Util.replaceChars("abc", "a", "a"), is("abc")); + assertNotSame("abc", Util.replaceChars("abc", "a", "a")); + } + + @Test void testReplaceIgnoreCase() { + final String s = "abc"; + assertThat(Util.replaceIgnoreCase(s, "", "x"), sameInstance(s)); + assertThat(Util.replaceIgnoreCase(s, "ZZ", "x"), sameInstance(s)); + + assertThat(Util.replaceIgnoreCase("aBAba", "ab", "x"), is("xxa")); + assertThat(Util.replaceIgnoreCase("xxxx", "X", "yz"), is("yzyzyzyz")); + assertThat(Util.replaceIgnoreCase("aaaa", "aa", "b"), is("bb")); + } + @Test void testToJavaId() throws UnsupportedEncodingException { assertThat(Util.toJavaId("foo", 0), is("ID$0$foo")); assertThat(Util.toJavaId("foo bar", 0), is("ID$0$foo_20_bar")); diff --git a/druid/build.gradle.kts b/druid/build.gradle.kts index 1b1fcf712e10..34cae5bd05ef 100644 --- a/druid/build.gradle.kts +++ b/druid/build.gradle.kts @@ -32,7 +32,6 @@ dependencies { implementation("com.fasterxml.jackson.core:jackson-databind") implementation("com.google.guava:guava") - implementation("org.apache.commons:commons-lang3") testImplementation(project(":testkit")) testImplementation("org.mockito:mockito-core") diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java index d313e30e93ab..ff64d44ed2c1 100644 --- a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java +++ b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java @@ -54,9 +54,6 @@ import org.apache.calcite.util.Util; import org.apache.calcite.util.trace.CalciteTrace; -import org.apache.commons.lang3.tuple.ImmutableTriple; -import org.apache.commons.lang3.tuple.Triple; - import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; @@ -220,15 +217,15 @@ protected DruidFilterRule(DruidFilterRuleConfig config) { query.getRowType().getFieldNames() .indexOf(query.druidTable.timestampFieldName); RelNode newDruidQuery = query; - final Triple, List, List> triple = + final SplitFiltersResult split = splitFilters(validPreds, nonValidPreds, timestampFieldIdx); - if (triple.getLeft().isEmpty() && triple.getMiddle().isEmpty()) { + if (split.timeRangeNodes.isEmpty() && split.pushableNodes.isEmpty()) { // it sucks, nothing to push return; } - final List residualPreds = new ArrayList<>(triple.getRight()); + final List residualPreds = new ArrayList<>(split.nonPushableNodes); List intervals = null; - if (!triple.getLeft().isEmpty()) { + if (!split.timeRangeNodes.isEmpty()) { final CalciteConnectionConfig connectionConfig = requireNonNull( cluster.getPlanner().getContext() @@ -236,17 +233,17 @@ protected DruidFilterRule(DruidFilterRuleConfig config) { requireNonNull(connectionConfig.timeZone()); intervals = DruidDateTimeUtils.createInterval( - RexUtil.composeConjunction(rexBuilder, triple.getLeft())); + RexUtil.composeConjunction(rexBuilder, split.timeRangeNodes)); if (intervals == null || intervals.isEmpty()) { // Case we have a filter with extract that can not be written as interval push down - triple.getMiddle().addAll(triple.getLeft()); + split.pushableNodes.addAll(split.timeRangeNodes); } } - if (!triple.getMiddle().isEmpty()) { + if (!split.pushableNodes.isEmpty()) { final RelNode newFilter = filter.copy(filter.getTraitSet(), Util.last(query.rels), - RexUtil.composeConjunction(rexBuilder, triple.getMiddle())); + RexUtil.composeConjunction(rexBuilder, split.pushableNodes)); newDruidQuery = DruidQuery.extendQuery(query, newFilter); } if (intervals != null && !intervals.isEmpty()) { @@ -269,7 +266,20 @@ protected DruidFilterRule(DruidFilterRuleConfig config) { * 2-m) condition filters that can be pushed to Druid, * 3-r) condition filters that cannot be pushed to Druid. */ - private static Triple, List, List> splitFilters( + private static final class SplitFiltersResult { + final List timeRangeNodes; + final List pushableNodes; + final List nonPushableNodes; + + private SplitFiltersResult(List timeRangeNodes, List pushableNodes, + List nonPushableNodes) { + this.timeRangeNodes = timeRangeNodes; + this.pushableNodes = pushableNodes; + this.nonPushableNodes = nonPushableNodes; + } + } + + private static SplitFiltersResult splitFilters( final List validPreds, final List nonValidPreds, final int timestampFieldIdx) { final List timeRangeNodes = new ArrayList<>(); @@ -286,7 +296,7 @@ private static Triple, List, List> splitFilters( pushableNodes.add(conj); } } - return ImmutableTriple.of(timeRangeNodes, pushableNodes, nonPushableNodes); + return new SplitFiltersResult(timeRangeNodes, pushableNodes, nonPushableNodes); } /** Rule configuration. */ diff --git a/file/build.gradle.kts b/file/build.gradle.kts index 37373ccf92f5..04036a888eae 100644 --- a/file/build.gradle.kts +++ b/file/build.gradle.kts @@ -31,7 +31,6 @@ dependencies { implementation("net.sf.opencsv:opencsv") implementation("org.apache.calcite.avatica:avatica-core") implementation("commons-io:commons-io") - implementation("org.apache.commons:commons-lang3") implementation("org.jsoup:jsoup") implementation("com.fasterxml.jackson.core:jackson-core") implementation("com.fasterxml.jackson.core:jackson-databind") diff --git a/file/src/main/java/org/apache/calcite/adapter/file/CsvEnumerator.java b/file/src/main/java/org/apache/calcite/adapter/file/CsvEnumerator.java index 4762c20424b4..012be6145085 100644 --- a/file/src/main/java/org/apache/calcite/adapter/file/CsvEnumerator.java +++ b/file/src/main/java/org/apache/calcite/adapter/file/CsvEnumerator.java @@ -27,8 +27,6 @@ import org.apache.calcite.util.Source; import org.apache.calcite.util.trace.CalciteLogger; -import org.apache.commons.lang3.time.FastDateFormat; - import au.com.bytecode.opencsv.CSVReader; import com.google.common.annotations.VisibleForTesting; @@ -40,6 +38,7 @@ import java.math.BigDecimal; import java.math.RoundingMode; import java.text.ParseException; +import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Date; import java.util.List; @@ -76,20 +75,43 @@ public class CsvEnumerator implements Enumerator { private final RowConverter rowConverter; private @Nullable E current; - private static final FastDateFormat TIME_FORMAT_DATE; - private static final FastDateFormat TIME_FORMAT_TIME; - private static final FastDateFormat TIME_FORMAT_TIMESTAMP; + private static final TimeZone GMT = TimeZone.getTimeZone("GMT"); + + // FastDateFormat is thread-safe and lenient; mimic with ThreadLocal(SimpleDateFormat). + private static final ThreadLocal TIME_FORMAT_DATE = + ThreadLocal.withInitial(() -> { + SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd", Locale.ROOT); + f.setTimeZone(GMT); + f.setLenient(true); + return f; + }); + + private static final ThreadLocal TIME_FORMAT_TIME = + ThreadLocal.withInitial(() -> { + SimpleDateFormat f = new SimpleDateFormat("HH:mm:ss", Locale.ROOT); + f.setTimeZone(GMT); + f.setLenient(true); + return f; + }); + + private static final ThreadLocal TIME_FORMAT_TIMESTAMP = + ThreadLocal.withInitial(() -> { + SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ROOT); + f.setTimeZone(GMT); + f.setLenient(true); + return f; + }); + + /** Clears per-thread cached date/time formatters to avoid ThreadLocal leaks in long-lived + * thread pools or container environments. Must be called on the same thread that used them. */ + private static void clearTimeFormats() { + TIME_FORMAT_DATE.remove(); + TIME_FORMAT_TIME.remove(); + TIME_FORMAT_TIMESTAMP.remove(); + } private static final Pattern DECIMAL_TYPE_PATTERN = Pattern .compile("\"decimal\\(([0-9]+),([0-9]+)\\)"); - static { - final TimeZone gmt = TimeZone.getTimeZone("GMT"); - TIME_FORMAT_DATE = FastDateFormat.getInstance("yyyy-MM-dd", gmt); - TIME_FORMAT_TIME = FastDateFormat.getInstance("HH:mm:ss", gmt); - TIME_FORMAT_TIMESTAMP = - FastDateFormat.getInstance("yyyy-MM-dd HH:mm:ss", gmt); - } - public CsvEnumerator(Source source, AtomicBoolean cancelFlag, List fieldTypes, List fields) { //noinspection unchecked @@ -252,7 +274,11 @@ static CSVReader openCsv(Source source) throws IOException { continue; } current = null; - reader.close(); + try { + reader.close(); + } finally { + clearTimeFormats(); + } return false; } if (filterValues != null) { @@ -282,6 +308,8 @@ static CSVReader openCsv(Source source) throws IOException { reader.close(); } catch (IOException e) { throw new RuntimeException("Error closing CSV reader", e); + } finally { + clearTimeFormats(); } } @@ -357,7 +385,7 @@ abstract static class RowConverter { return null; } try { - Date date = TIME_FORMAT_DATE.parse(string); + Date date = TIME_FORMAT_DATE.get().parse(string); return (int) (date.getTime() / DateTimeUtils.MILLIS_PER_DAY); } catch (ParseException e) { return null; @@ -367,7 +395,7 @@ abstract static class RowConverter { return null; } try { - Date date = TIME_FORMAT_TIME.parse(string); + Date date = TIME_FORMAT_TIME.get().parse(string); return (int) date.getTime(); } catch (ParseException e) { return null; @@ -377,7 +405,7 @@ abstract static class RowConverter { return null; } try { - Date date = TIME_FORMAT_TIMESTAMP.parse(string); + Date date = TIME_FORMAT_TIMESTAMP.get().parse(string); return date.getTime(); } catch (ParseException e) { return null; diff --git a/geode/build.gradle.kts b/geode/build.gradle.kts index bf51ecdb589d..0ef17741679e 100644 --- a/geode/build.gradle.kts +++ b/geode/build.gradle.kts @@ -30,7 +30,6 @@ dependencies { implementation("com.google.guava:guava") implementation("org.apache.calcite.avatica:avatica-core") - implementation("org.apache.commons:commons-lang3") testImplementation(project(":testkit")) testImplementation("com.fasterxml.jackson.core:jackson-databind") diff --git a/geode/src/main/java/org/apache/calcite/adapter/geode/util/GeodeUtils.java b/geode/src/main/java/org/apache/calcite/adapter/geode/util/GeodeUtils.java index 06b74f20b19f..2aa212dc5a92 100644 --- a/geode/src/main/java/org/apache/calcite/adapter/geode/util/GeodeUtils.java +++ b/geode/src/main/java/org/apache/calcite/adapter/geode/util/GeodeUtils.java @@ -22,7 +22,6 @@ import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.util.Util; -import org.apache.commons.lang3.Strings; import org.apache.geode.cache.CacheClosedException; import org.apache.geode.cache.GemFireCache; import org.apache.geode.cache.Region; @@ -84,7 +83,7 @@ public static synchronized ClientCache createClientCache(String locatorHost, int locatorPort, String autoSerializerPackagePath, boolean readSerialized) { if (locatorPort != currentLocatorPort - || !Strings.CI.equals(currentLocatorHost, locatorHost)) { + || !equalsIgnoreCase(currentLocatorHost, locatorHost)) { LOGGER.info("Close existing ClientCache [" + currentLocatorHost + ":" + currentLocatorPort + "] for new Locator connection at: [" + locatorHost + ":" + locatorPort + "]"); @@ -117,6 +116,10 @@ public static synchronized void closeClientCache() { REGION_MAP.clear(); } + private static boolean equalsIgnoreCase(@Nullable String a, @Nullable String b) { + return a == null ? b == null : b != null && a.equalsIgnoreCase(b); + } + /** * Obtains a proxy pointing to an existing Region on the server. * diff --git a/gradle.properties b/gradle.properties index 974408d02b58..3005b96424c5 100644 --- a/gradle.properties +++ b/gradle.properties @@ -92,7 +92,6 @@ chinook-data-hsqldb.version=0.2 commons-codec.version=1.16.0 commons-dbcp2.version=2.11.0 commons-io.version=2.15.0 -commons-lang3.version=3.18.0 commons-math3.version=3.6.1 commons-pool2.version=2.12.0 commons-collections4.version=4.4 diff --git a/innodb/build.gradle.kts b/innodb/build.gradle.kts index 88b00aa8090e..81143af455ed 100644 --- a/innodb/build.gradle.kts +++ b/innodb/build.gradle.kts @@ -31,7 +31,6 @@ dependencies { api("com.google.guava:guava") implementation("org.apache.calcite.avatica:avatica-core") - implementation("org.apache.commons:commons-lang3") implementation("org.slf4j:slf4j-api") testImplementation(project(":testkit")) diff --git a/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbSchema.java b/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbSchema.java index c6dbf0139a7b..759f5cd60739 100644 --- a/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbSchema.java +++ b/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbSchema.java @@ -25,8 +25,6 @@ import org.apache.calcite.sql.type.SqlTypeFactoryImpl; import org.apache.calcite.sql.type.SqlTypeName; -import org.apache.commons.lang3.StringUtils; - import com.alibaba.innodb.java.reader.TableReaderFactory; import com.alibaba.innodb.java.reader.column.ColumnType; import com.alibaba.innodb.java.reader.schema.Column; @@ -57,7 +55,7 @@ public InnodbSchema(List sqlFilePathList, String ibdDataFileBasePath) { checkArgument(sqlFilePathList != null && !sqlFilePathList.isEmpty(), "SQL file path list cannot be empty"); - checkArgument(StringUtils.isNotEmpty(ibdDataFileBasePath), + checkArgument(ibdDataFileBasePath != null && !ibdDataFileBasePath.isEmpty(), "InnoDB data file with ibd suffix cannot be empty"); this.sqlFilePathList = sqlFilePathList; this.ibdDataFileBasePath = ibdDataFileBasePath; diff --git a/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbSchemaFactory.java b/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbSchemaFactory.java index 7d9599b04973..00b0b287ba00 100644 --- a/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbSchemaFactory.java +++ b/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbSchemaFactory.java @@ -20,8 +20,6 @@ import org.apache.calcite.schema.SchemaFactory; import org.apache.calcite.schema.SchemaPlus; -import org.apache.commons.lang3.StringUtils; - import java.util.List; import java.util.Map; @@ -37,7 +35,7 @@ public InnodbSchemaFactory() { final List sqlFilePathList = (List) operand.get("sqlFilePath"); final String ibdDataFileBasePath = (String) operand.get("ibdDataFileBasePath"); final String timeZone = (String) operand.get("timeZone"); - if (StringUtils.isNotEmpty(timeZone)) { + if (timeZone != null && !timeZone.isEmpty()) { System.setProperty("innodb.java.reader.server.timezone", timeZone); } diff --git a/innodb/src/test/java/org/apache/calcite/adapter/innodb/InnodbAdapterDataTypesTest.java b/innodb/src/test/java/org/apache/calcite/adapter/innodb/InnodbAdapterDataTypesTest.java index 588e0fa87499..d04300303e05 100644 --- a/innodb/src/test/java/org/apache/calcite/adapter/innodb/InnodbAdapterDataTypesTest.java +++ b/innodb/src/test/java/org/apache/calcite/adapter/innodb/InnodbAdapterDataTypesTest.java @@ -19,8 +19,6 @@ import org.apache.calcite.test.CalciteAssert; import org.apache.calcite.util.Sources; -import org.apache.commons.lang3.StringUtils; - import com.alibaba.innodb.java.reader.util.Utils; import com.google.common.collect.ImmutableMap; @@ -119,18 +117,18 @@ public class InnodbAdapterDataTypesTest { + "f_decimal4=123.100; " + "f_decimal5=12346; " + "f_decimal6=12345.1234567890123456789012345; " - + "f_varchar=c" + StringUtils.repeat('x', 31) + "; " - + "f_varchar_overflow=c" + StringUtils.repeat("データ", 300) + "; " + + "f_varchar=c" + repeat('x', 31) + "; " + + "f_varchar_overflow=c" + repeat("データ", 300) + "; " + "f_varchar_null=null; " - + "f_char_32=c" + StringUtils.repeat("данные", 2) + "; " - + "f_char_255=c" + StringUtils.repeat("数据", 100) + "; " + + "f_char_32=c" + repeat("данные", 2) + "; " + + "f_char_255=c" + repeat("数据", 100) + "; " + "f_char_null=null; " + "f_boolean=false; " + "f_bool=true; " - + "f_tinytext=c" + StringUtils.repeat("Data", 50) + "; " - + "f_text=c" + StringUtils.repeat("Daten", 200) + "; " - + "f_mediumtext=c" + StringUtils.repeat("Datos", 200) + "; " - + "f_longtext=c" + StringUtils.repeat("Les données", 800) + "; " + + "f_tinytext=c" + repeat("Data", 50) + "; " + + "f_text=c" + repeat("Daten", 200) + "; " + + "f_mediumtext=c" + repeat("Datos", 200) + "; " + + "f_longtext=c" + repeat("Les données", 800) + "; " + "f_tinyblob=" + genByteArrayString("63", (byte) 0x0a, 100) + "; " + "f_blob=" @@ -164,18 +162,18 @@ public class InnodbAdapterDataTypesTest { + "f_decimal4=456.000; " + "f_decimal5=0; " + "f_decimal6=-0.0123456789012345678912345; " - + "f_varchar=d" + StringUtils.repeat('y', 31) + "; " - + "f_varchar_overflow=d" + StringUtils.repeat("データ", 300) + "; " + + "f_varchar=d" + repeat('y', 31) + "; " + + "f_varchar_overflow=d" + repeat("データ", 300) + "; " + "f_varchar_null=null; " - + "f_char_32=d" + StringUtils.repeat("данные", 2) + "; " - + "f_char_255=d" + StringUtils.repeat("数据", 100) + "; " + + "f_char_32=d" + repeat("данные", 2) + "; " + + "f_char_255=d" + repeat("数据", 100) + "; " + "f_char_null=null; " + "f_boolean=false; " + "f_bool=true; " - + "f_tinytext=d" + StringUtils.repeat("Data", 50) + "; " - + "f_text=d" + StringUtils.repeat("Daten", 200) + "; " - + "f_mediumtext=d" + StringUtils.repeat("Datos", 200) + "; " - + "f_longtext=d" + StringUtils.repeat("Les données", 800) + "; " + + "f_tinytext=d" + repeat("Data", 50) + "; " + + "f_text=d" + repeat("Daten", 200) + "; " + + "f_mediumtext=d" + repeat("Datos", 200) + "; " + + "f_longtext=d" + repeat("Les données", 800) + "; " + "f_tinyblob=" + genByteArrayString("64", (byte) 0x0a, 100) + "; " + "f_blob=" @@ -192,6 +190,26 @@ public class InnodbAdapterDataTypesTest { + "f_set=a,e,i,o,u"); } + private static String repeat(String s, int n) { + if (n <= 0) { + return ""; + } + StringBuilder b = new StringBuilder(s.length() * n); + for (int i = 0; i < n; i++) { + b.append(s); + } + return b.toString(); + } + + private static String repeat(char c, int n) { + if (n <= 0) { + return ""; + } + char[] a = new char[n]; + java.util.Arrays.fill(a, c); + return String.valueOf(a); + } + private String genByteArrayString(String prefix, byte b, int repeat) { StringBuilder str = new StringBuilder(); str.append(prefix); diff --git a/plus/src/test/java/org/apache/calcite/adapter/os/OsAdapterTest.java b/plus/src/test/java/org/apache/calcite/adapter/os/OsAdapterTest.java index 9ff5c3f99cb1..e1158d5aa66d 100644 --- a/plus/src/test/java/org/apache/calcite/adapter/os/OsAdapterTest.java +++ b/plus/src/test/java/org/apache/calcite/adapter/os/OsAdapterTest.java @@ -71,12 +71,6 @@ * */ class OsAdapterTest { - private static final String OS_NAME = System.getProperty("os.name"); - - private static boolean isWindows() { - return OS_NAME.startsWith("Windows"); - } - /** Returns whether there is a ".git" directory in this directory or in a * directory between this directory and root. */ private static boolean hasGit() { @@ -114,7 +108,7 @@ private static boolean checkProcessExists(String command) { } @Test void testDu() { - assumeFalse(isWindows(), "Skip: the 'du' table does not work on Windows"); + assumeFalse(Util.isWindows(), "Skip: the 'du' table does not work on Windows"); assumeToolExists("du"); sql("select * from du") .returns(r -> { @@ -130,7 +124,7 @@ private static boolean checkProcessExists(String command) { } @Test void testDuFilterSortLimit() { - assumeFalse(isWindows(), "Skip: the 'du' table does not work on Windows"); + assumeFalse(Util.isWindows(), "Skip: the 'du' table does not work on Windows"); assumeToolExists("du"); sql("select * from du where path like '%/src/test/java/%'\n" + "order by 1 limit 2") @@ -149,14 +143,14 @@ private static boolean checkProcessExists(String command) { } @Test void testFiles() { - assumeFalse(isWindows(), "Skip: the 'files' table does not work on Windows"); + assumeFalse(Util.isWindows(), "Skip: the 'files' table does not work on Windows"); sql("select distinct type from files") .returnsUnordered("type=d", "type=f"); } @Test void testPs() { - assumeFalse(isWindows(), "Skip: the 'ps' table does not work on Windows"); + assumeFalse(Util.isWindows(), "Skip: the 'ps' table does not work on Windows"); assumeToolExists("ps"); sql("select * from ps") .returns(r -> { @@ -176,7 +170,7 @@ private static boolean checkProcessExists(String command) { } @Test void testPsDistinct() { - assumeFalse(isWindows(), "Skip: the 'ps' table does not work on Windows"); + assumeFalse(Util.isWindows(), "Skip: the 'ps' table does not work on Windows"); assumeToolExists("ps"); sql("select distinct `user` from ps") .returns(r -> { @@ -229,7 +223,7 @@ private static boolean checkProcessExists(String command) { } @Test void testVmstat() { - assumeFalse(isWindows(), "Skip: the 'files' table does not work on Windows"); + assumeFalse(Util.isWindows(), "Skip: the 'files' table does not work on Windows"); assumeToolExists("vmstat"); sql("select * from vmstat") .returns(r -> { diff --git a/redis/build.gradle.kts b/redis/build.gradle.kts index 0cbc43112fe4..11eb99f93d48 100644 --- a/redis/build.gradle.kts +++ b/redis/build.gradle.kts @@ -24,7 +24,6 @@ dependencies { implementation("com.google.guava:guava") implementation("commons-io:commons-io") implementation("org.apache.calcite.avatica:avatica-core") - implementation("org.apache.commons:commons-lang3") implementation("org.apache.commons:commons-pool2") implementation("org.slf4j:slf4j-api") diff --git a/redis/src/main/java/org/apache/calcite/adapter/redis/RedisDataProcess.java b/redis/src/main/java/org/apache/calcite/adapter/redis/RedisDataProcess.java index 853507cd1ad3..024a0e7df91f 100644 --- a/redis/src/main/java/org/apache/calcite/adapter/redis/RedisDataProcess.java +++ b/redis/src/main/java/org/apache/calcite/adapter/redis/RedisDataProcess.java @@ -16,8 +16,6 @@ */ package org.apache.calcite.adapter.redis; -import org.apache.commons.lang3.StringUtils; - import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -77,7 +75,7 @@ public List read() { } private Object[] parseJson(String value) { - assert StringUtils.isNotEmpty(value); + assert value != null && !value.isEmpty(); Object[] arr = new Object[fields.size()]; try { JsonNode jsonNode = objectMapper.readTree(value); @@ -97,7 +95,7 @@ private Object[] parseJson(String value) { } private Object[] parseCsv(String value) { - assert StringUtils.isNotEmpty(value); + assert value != null && !value.isEmpty(); String[] values = value.split(keyDelimiter); Object[] arr = new Object[fields.size()]; assert values.length == arr.length; diff --git a/redis/src/main/java/org/apache/calcite/adapter/redis/RedisEnumerator.java b/redis/src/main/java/org/apache/calcite/adapter/redis/RedisEnumerator.java index 0643d804feea..dede17cd48f0 100644 --- a/redis/src/main/java/org/apache/calcite/adapter/redis/RedisEnumerator.java +++ b/redis/src/main/java/org/apache/calcite/adapter/redis/RedisEnumerator.java @@ -19,8 +19,6 @@ import org.apache.calcite.linq4j.Enumerator; import org.apache.calcite.linq4j.Linq4j; -import org.apache.commons.lang3.StringUtils; - import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -38,13 +36,14 @@ class RedisEnumerator implements Enumerator { RedisEnumerator(RedisConfig redisConfig, RedisSchema schema, String tableName) { RedisTableFieldInfo tableFieldInfo = schema.getTableFieldInfo(tableName); + String password = redisConfig.getPassword(); RedisJedisManager redisManager = new RedisJedisManager(redisConfig.getHost(), redisConfig.getPort(), - redisConfig.getDatabase(), redisConfig.getPassword()); + redisConfig.getDatabase(), password); try (Jedis jedis = redisManager.getResource()) { - if (StringUtils.isNotEmpty(redisConfig.getPassword())) { - jedis.auth(redisConfig.getPassword()); + if (password != null && !password.isEmpty()) { + jedis.auth(password); } RedisDataProcess dataProcess = new RedisDataProcess(jedis, tableFieldInfo); List objs = dataProcess.read(); diff --git a/redis/src/main/java/org/apache/calcite/adapter/redis/RedisJedisManager.java b/redis/src/main/java/org/apache/calcite/adapter/redis/RedisJedisManager.java index 92647d6c1495..226af4b7b779 100644 --- a/redis/src/main/java/org/apache/calcite/adapter/redis/RedisJedisManager.java +++ b/redis/src/main/java/org/apache/calcite/adapter/redis/RedisJedisManager.java @@ -18,7 +18,6 @@ import org.apache.calcite.util.trace.CalciteTrace; -import org.apache.commons.lang3.StringUtils; import org.apache.commons.pool2.impl.GenericObjectPoolConfig; import com.google.common.cache.CacheBuilder; @@ -75,7 +74,7 @@ public Jedis getResource() { private JedisPool createConsumer() { String pwd = password; - if (StringUtils.isEmpty(pwd)) { + if (pwd == null || pwd.isEmpty()) { pwd = null; } return new JedisPool(jedisPoolConfig, host, port, Protocol.DEFAULT_TIMEOUT, diff --git a/redis/src/main/java/org/apache/calcite/adapter/redis/RedisSchema.java b/redis/src/main/java/org/apache/calcite/adapter/redis/RedisSchema.java index 1aad9ae459d5..42c623895bc8 100644 --- a/redis/src/main/java/org/apache/calcite/adapter/redis/RedisSchema.java +++ b/redis/src/main/java/org/apache/calcite/adapter/redis/RedisSchema.java @@ -20,19 +20,20 @@ import org.apache.calcite.schema.Table; import org.apache.calcite.schema.impl.AbstractSchema; -import org.apache.commons.lang3.ObjectUtils; -import org.apache.commons.lang3.StringUtils; - import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; +import org.checkerframework.checker.nullness.qual.Nullable; + import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -92,7 +93,7 @@ public RedisTableFieldInfo getTableFieldInfo(String tableName) { if (jsonCustomTable.name.equals(tableName)) { Map map = requireNonNull(jsonCustomTable.operand, OPERAND); - if (ObjectUtils.isEmpty(map.get(DATA_FORMAT))) { + if (isEmptyObject(map.get(DATA_FORMAT))) { throw new RuntimeException("dataFormat is invalid, it must be raw, csv or json"); } RedisDataFormat dataFormatEnum = @@ -100,7 +101,7 @@ public RedisTableFieldInfo getTableFieldInfo(String tableName) { if (dataFormatEnum == null) { throw new RuntimeException("dataFormat is invalid, it must be raw, csv or json"); } - if (ObjectUtils.isEmpty(map.get(FIELDS))) { + if (isEmptyObject(map.get(FIELDS))) { throw new RuntimeException("fields is null"); } dataFormat = map.get(DATA_FORMAT).toString(); @@ -114,9 +115,45 @@ public RedisTableFieldInfo getTableFieldInfo(String tableName) { tableFieldInfo.setTableName(tableName); tableFieldInfo.setDataFormat(dataFormat); tableFieldInfo.setFields(fields); - if (StringUtils.isNotEmpty(keyDelimiter)) { + if (!keyDelimiter.isEmpty()) { tableFieldInfo.setKeyDelimiter(keyDelimiter); } return tableFieldInfo; } + + /** Returns whether an object should be considered "empty" for configuration/validation. + * + *

    Semantics are aligned with + * {@code org.apache.commons.lang3.ObjectUtils#isEmptyObject(Object)}. + *

      + *
    • {@code null} is empty + *
    • {@link CharSequence}: {@code length()==0} is empty + *
    • {@link Collection}/{@link Map}: {@code isEmpty()} is empty + *
    • {@link Optional}: {@code !isPresent()} is empty + *
    • Arrays: {@code length==0} is empty + *
    + */ + public static boolean isEmptyObject(@Nullable Object o) { + if (o == null) { + return true; + } + if (o instanceof CharSequence) { + return ((CharSequence) o).length() == 0; + } + if (o instanceof Collection) { + return ((Collection) o).isEmpty(); + } + if (o instanceof Map) { + return ((Map) o).isEmpty(); + } + if (o instanceof Optional) { + return !((Optional) o).isPresent(); + } + final Class c = o.getClass(); + if (c.isArray()) { + return java.lang.reflect.Array.getLength(o) == 0; + } + return false; + } + } diff --git a/redis/src/test/java/org/apache/calcite/adapter/redis/RedisCaseBase.java b/redis/src/test/java/org/apache/calcite/adapter/redis/RedisCaseBase.java index 44d82385aa8d..adfc7dd6c6f3 100644 --- a/redis/src/test/java/org/apache/calcite/adapter/redis/RedisCaseBase.java +++ b/redis/src/test/java/org/apache/calcite/adapter/redis/RedisCaseBase.java @@ -17,6 +17,7 @@ package org.apache.calcite.adapter.redis; import org.apache.calcite.config.CalciteSystemProperty; +import org.apache.calcite.util.Util; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; @@ -71,7 +72,7 @@ public static void startRedisContainer() { @BeforeEach public void createRedisServer() throws IOException { if (!REDIS_CONTAINER.isRunning()) { - if (isWindows()) { + if (Util.isWindows()) { redisServer = RedisServer.builder().port(PORT).setting(MAX_HEAP).build(); } else { redisServer = new RedisServer(PORT); @@ -81,10 +82,6 @@ public void createRedisServer() throws IOException { } } - private static boolean isWindows() { - return System.getProperty("os.name").startsWith("Windows"); - } - @AfterEach public void stopRedisServer() { if (!REDIS_CONTAINER.isRunning()) { diff --git a/testkit/build.gradle.kts b/testkit/build.gradle.kts index 2154cbf82c14..613526ebd06d 100644 --- a/testkit/build.gradle.kts +++ b/testkit/build.gradle.kts @@ -30,7 +30,6 @@ dependencies { implementation("net.hydromatic:scott-data-hsqldb") implementation("net.hydromatic:steelwheels-data-hsqldb") implementation("org.apache.commons:commons-dbcp2") - implementation("org.apache.commons:commons-lang3") implementation("org.apache.commons:commons-pool2") implementation("org.hamcrest:hamcrest") implementation("org.hsqldb:hsqldb::jdk8") diff --git a/testkit/src/main/java/org/apache/calcite/test/CalciteAssert.java b/testkit/src/main/java/org/apache/calcite/test/CalciteAssert.java index 8a846da87eac..cbc2f6c7999c 100644 --- a/testkit/src/main/java/org/apache/calcite/test/CalciteAssert.java +++ b/testkit/src/main/java/org/apache/calcite/test/CalciteAssert.java @@ -137,8 +137,6 @@ import static org.apache.calcite.test.Matchers.containsStringLinux; import static org.apache.calcite.test.Matchers.isLinux; -import static org.apache.commons.lang3.StringUtils.countMatches; - import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.hasItem; @@ -500,6 +498,19 @@ public static Consumer checkResultContains( }; } + private static int countMatches(@Nullable String str, @Nullable String sub) { + if (str == null || sub == null || sub.isEmpty()) { + return 0; + } + int count = 0; + int idx = 0; + while ((idx = str.indexOf(sub, idx)) >= 0) { + count++; + idx += sub.length(); + } + return count; + } + public static Consumer checkMaskedResultContains( final String expected) { return s -> { From e216ef1c0048ff5369dc5ff07f4af6209e627293 Mon Sep 17 00:00:00 2001 From: krooswu Date: Sat, 14 Mar 2026 21:56:09 +0800 Subject: [PATCH 188/562] [CALCITE-7432] NumberFormatException when convert `NaN` literal to sql --- .../calcite/rel/rel2sql/SqlImplementor.java | 14 ++++++-- .../rel/rel2sql/RelToSqlConverterTest.java | 34 +++++++++++++++++++ 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java b/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java index dec24ca1f24a..410cc5ea33fe 100644 --- a/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java +++ b/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java @@ -66,8 +66,10 @@ import org.apache.calcite.sql.JoinType; import org.apache.calcite.sql.SqlAggFunction; import org.apache.calcite.sql.SqlBasicCall; +import org.apache.calcite.sql.SqlBasicTypeNameSpec; import org.apache.calcite.sql.SqlBinaryOperator; import org.apache.calcite.sql.SqlCall; +import org.apache.calcite.sql.SqlDataTypeSpec; import org.apache.calcite.sql.SqlDialect; import org.apache.calcite.sql.SqlDynamicParam; import org.apache.calcite.sql.SqlIdentifier; @@ -1536,8 +1538,16 @@ public static SqlNode toSql(RexLiteral literal) { case NUMERIC: case EXACT_NUMERIC: { if (SqlTypeName.APPROX_TYPES.contains(typeName)) { - return SqlLiteral.createApproxNumeric( - castNonNull(literal.getValueAs(Double.class)).toString(), POS); + final Double d = castNonNull(literal.getValueAs(Double.class)); + // BigDecimal cannot represent IEEE 754 special values (NaN, ±Infinity). + if (!Double.isFinite(d)) { + final SqlNode strLiteral = + SqlLiteral.createCharString(d.toString(), POS); + final SqlDataTypeSpec typeSpec = + new SqlDataTypeSpec(new SqlBasicTypeNameSpec(typeName, POS), POS); + return SqlStdOperatorTable.CAST.createCall(POS, strLiteral, typeSpec); + } + return SqlLiteral.createApproxNumeric(d.toString(), POS); } else { return SqlLiteral.createExactNumeric( castNonNull(literal.getValueAs(BigDecimal.class)).toPlainString(), POS); diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java index 410543122027..2deca5f9d863 100644 --- a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java +++ b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java @@ -11913,5 +11913,39 @@ public Sql schema(CalciteAssert.SchemaSpec schemaSpec) { .schema(CalciteAssert.SchemaSpec.JDBC_SCOTT) .ok(expected2); } + /** + * Test case for + * [CALCITE-7432] + * NumberFormatException when convert `NaN` literal to sql. + * + *

    Tests support for all IEEE 754 floating-point special values: + * NaN, positive/negative infinity, signed zero, and subnormal values. + */ + @Test void testCastFloatingPointSpecialValuesToDouble() { + // Test NaN + sql("select cast('NaN' as DOUBLE)") + .ok("SELECT *\n" + + "FROM (VALUES (CAST('NaN' AS DOUBLE))) AS \"t\" (\"EXPR$0\")"); + + // Test Positive Infinity + sql("select cast('Infinity' as DOUBLE)") + .ok("SELECT *\n" + + "FROM (VALUES (CAST('Infinity' AS DOUBLE))) AS \"t\" (\"EXPR$0\")"); + + // Test Negative Infinity + sql("select cast('-Infinity' as DOUBLE)") + .ok("SELECT *\n" + + "FROM (VALUES (CAST('-Infinity' AS DOUBLE))) AS \"t\" (\"EXPR$0\")"); + + // Test Negative Zero + sql("select cast('-0.0' as DOUBLE)") + .ok("SELECT *\n" + + "FROM (VALUES (0E0)) AS \"t\" (\"EXPR$0\")"); + + // Test Subnormal values + sql("select cast('1e-310' as DOUBLE)") + .ok("SELECT *\n" + + "FROM (VALUES (1.0E-310)) AS \"t\" (\"EXPR$0\")"); + } } From cfc7a0b06134bc64d08cd8e028b8481ea403198d Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Thu, 19 Mar 2026 08:50:00 +0800 Subject: [PATCH 189/562] [CALCITE-7318] Execution fails when the JOIN ON condition contains references to columns from both the left and right sides --- .../calcite/rel/rules/SubQueryRemoveRule.java | 56 +++++++++++++++++-- .../apache/calcite/test/RelOptRulesTest.java | 2 +- .../apache/calcite/test/RelOptRulesTest.xml | 15 +++++ core/src/test/resources/sql/sub-query.iq | 17 ++++++ 4 files changed, 85 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rel/rules/SubQueryRemoveRule.java b/core/src/main/java/org/apache/calcite/rel/rules/SubQueryRemoveRule.java index a1b59e952f4f..43d5adc27e2e 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/SubQueryRemoveRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/SubQueryRemoveRule.java @@ -1050,10 +1050,58 @@ private static void matchJoin(SubQueryRemoveRule rule, RelOptRuleCall call) { boolean inputIntersectsRightSide = inputSet.intersects(ImmutableBitSet.range(nFieldsLeft, nFieldsLeft + nFieldsRight)); if (inputIntersectsLeftSide && inputIntersectsRightSide) { - // The current existential rewrite needs to make join with one side of the origin join and - // generate a new condition to replace the on clause. But for RexNode whose operands are - // on either side of the join, we can't push them into join. So this rewriting is not - // supported. + if (join.getJoinType() != JoinRelType.INNER) { + // Rewriting requires flattening the join into a cross-product first (see below). + // That transformation is only valid for INNER JOIN (A JOIN B ON c ≡ (A × B) WHERE c). + // For OUTER JOINs the semantics differ (NULL-padding), so we bail out. + return; + } + + // The sub-query operands span both sides of the join, e.g.: + // SELECT empno FROM emp JOIN dept + // ON emp.deptno + dept.deptno >= SOME(SELECT deptno FROM dept) + // + // Because the sub-query references fields from both the left (emp) and right (dept) + // inputs, we cannot attach the sub-query rewrite to either side alone. + // The logic is to exploit the INNER JOIN equivalence: + // L INNER JOIN R ON cond ≡ (L CROSS JOIN R) WHERE cond + + // Step 1 – flatten to a cross-product so every field is visible in one relation: + // + // Before: After (builder stack): + // LogicalJoin(INNER, cond) → LogicalJoin(INNER, true) ← cross-product + // LogicalTableScan(EMP) LogicalTableScan(EMP) + // LogicalTableScan(DEPT) LogicalTableScan(DEPT) + builder.push(join.getLeft()); + builder.push(join.getRight()); + builder.join(JoinRelType.INNER, builder.literal(true)); + + // Step 2 – expand the sub-query against the flattened relation (rule.apply). + // The sub-query rewrite pushes an auxiliary relation onto the stack, e.g. for SOME: + // + // LogicalJoin(INNER, true) ← sub-query auxiliary join + // LogicalJoin(INNER, true) ← cross-product from step 1 + // LogicalTableScan(EMP) + // LogicalTableScan(DEPT) + // LogicalAggregate(m, c, d) ← aggregate over sub-query + // LogicalTableScan(DEPT) + final int count = builder.peek().getRowType().getFieldCount(); + final RelOptUtil.Logic logic = + LogicVisitor.find(RelOptUtil.Logic.TRUE, ImmutableList.of(join.getCondition()), e); + final RexNode target = + rule.apply(e, variablesSet, logic, builder, 1, count, 0); + + // Step 3 – replace the sub-query in the ON condition with the expression (target) + // produced by rule.apply (e.g. a CASE expression for SOME, IS NOT NULL for EXISTS), + // then materialize it as a Filter (valid because the cross-product is already in place). + final RexShuttle shuttle = new ReplaceSubQueryShuttle(e, target); + final RexNode newCond = shuttle.apply(join.getCondition()); + builder.filter(newCond); + + // Step 4 – project away the auxiliary columns introduced by the sub-query rewrite, + // restoring the original output schema (fields 0 .. count-1). + builder.project(fields(builder, count)); + call.transformTo(builder.build()); return; } diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index 98beae87c775..5b9b9d8aba8d 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -10427,7 +10427,7 @@ public interface Config extends RelRule.Config { + "emp.deptno + dept.deptno >= SOME(SELECT deptno FROM dept)"; sql(sql) .withRule(CoreRules.JOIN_SUB_QUERY_TO_CORRELATE) - .checkUnchanged(); + .check(); } /** Test case for diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index a16c9200955d..9cb281d0a53b 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -8737,6 +8737,21 @@ LogicalProject(DEPTNO=[$0]) })], joinType=[inner]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) +]]> + + + =(+($7, $9), $11)), <>($12, 0)), AND(>($12, $13), null, <>($12, 0), IS NOT TRUE(>=(+($7, $9), $11))), AND(>=(+($7, $9), $11), <>($12, 0), IS NOT TRUE(>=(+($7, $9), $11)), <=($12, $13)))]) + LogicalJoin(condition=[true], joinType=[inner]) + LogicalJoin(condition=[true], joinType=[inner]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) + LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) + LogicalProject(m=[$0], c=[$1], d=[$1]) + LogicalAggregate(group=[{}], m=[MIN($0)], c=[COUNT()]) + LogicalProject(DEPTNO=[$0]) + LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) ]]> diff --git a/core/src/test/resources/sql/sub-query.iq b/core/src/test/resources/sql/sub-query.iq index ac2fe6d8aa11..b9d4774cefd0 100644 --- a/core/src/test/resources/sql/sub-query.iq +++ b/core/src/test/resources/sql/sub-query.iq @@ -9070,6 +9070,23 @@ where exists ( !ok +# [CALCITE-7318] Execution fails when the JOIN ON condition contains references to columns from both the left and right sides +!use scott +select Header.Name from ( VALUES (1, 'A'), (2, 'B')) as Header(Id, Name) +join (values (11, 1), (12, 1), (21, 2)) as Version(Id, Parent) +on not exists (select 1 from (values (11, 1), (12, 1), (21, 2)) as Version2(Id, Parent) +where Version2.Parent = Header.Id and Version2.Id > Version.Id); ++------+ +| NAME | ++------+ +| A | +| A | +| B | ++------+ +(3 rows) + +!ok + # [CALCITE-5132] Scalar IN subquery returns UNKNOWN instead of FALSE when key is partially NULL. # Case 1: Default insubquerythreshold=20 !if (use_old_decorr) { From 655bb1a60ac959c5b14a2b420ec24d87ee424899 Mon Sep 17 00:00:00 2001 From: Niels Pardon Date: Fri, 20 Mar 2026 14:57:09 +0100 Subject: [PATCH 190/562] [CALCITE-7447] RelRoot.project() adds Project for DDL nodes Signed-off-by: Niels Pardon --- .../java/org/apache/calcite/rel/RelRoot.java | 5 +- .../org/apache/calcite/rel/RelRootTest.java | 50 +++++++++++++++++++ 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/calcite/rel/RelRoot.java b/core/src/main/java/org/apache/calcite/rel/RelRoot.java index 4eb495e70a0a..7fe6955910c7 100644 --- a/core/src/main/java/org/apache/calcite/rel/RelRoot.java +++ b/core/src/main/java/org/apache/calcite/rel/RelRoot.java @@ -159,10 +159,13 @@ public RelNode project() { /** Returns the root relational expression as a {@link LogicalProject}. * - * @param force Create a Project even if all fields are used */ + * @param force Create a Project even if all fields are used + * @return the root relational expression + */ public RelNode project(boolean force) { if (isRefTrivial() && (SqlKind.DML.contains(kind) + || SqlKind.DDL.contains(kind) || !force || (rel instanceof LogicalProject && isNameTrivial()))) { return rel; diff --git a/core/src/test/java/org/apache/calcite/rel/RelRootTest.java b/core/src/test/java/org/apache/calcite/rel/RelRootTest.java index 9c13336cee51..9c195b7f2b6b 100644 --- a/core/src/test/java/org/apache/calcite/rel/RelRootTest.java +++ b/core/src/test/java/org/apache/calcite/rel/RelRootTest.java @@ -16,6 +16,8 @@ */ package org.apache.calcite.rel; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.logical.LogicalProject; import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.rel.type.RelDataTypeFieldImpl; @@ -29,6 +31,8 @@ import org.apache.calcite.tools.RelBuilder; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; import java.util.Collections; import java.util.List; @@ -119,4 +123,50 @@ public class RelRootTest { final RelNode forceProject = root.project(true); assertThat(forceProject, equalTo(project)); } + + static SqlKind[] ddlSqlKinds() { + return SqlKind.DDL.toArray(new SqlKind[0]); + } + + /** Test case for + * [CALCITE-7447] + * RelRoot.project() adds Project for DDL nodes. */ + @ParameterizedTest + @MethodSource("ddlSqlKinds") + void testRelRootProjectDdl(SqlKind ddlKind) { + final SchemaPlus rootSchema = Frameworks.createRootSchema(true); + final SchemaPlus defaultSchema = + CalciteAssert.addSchema(rootSchema, CalciteAssert.SchemaSpec.HR); + final FrameworkConfig frameworkConfig = RelBuilderTest.config() + .defaultSchema(defaultSchema) + .build(); + final RelBuilder relBuilder = RelBuilder.create(frameworkConfig); + final RelNode scanRel = relBuilder.scan("emps") + .project(relBuilder.fields(Collections.singletonList("empid"))).build(); + + final RelNode inputRel = new DummyDdlRelNode(relBuilder.getCluster(), scanRel); + + final RelRoot root = RelRoot.of(inputRel, ddlKind); + + final RelNode project = root.project(); + assertThat(project, equalTo(inputRel)); + assertThat(project, instanceOf(DummyDdlRelNode.class)); + + // regular project() and force project() are the same + final RelNode forceProject = root.project(true); + assertThat(forceProject, equalTo(project)); + } + + /** + * Dummy DDL RelNode for testing. + */ + static class DummyDdlRelNode extends SingleRel { + protected DummyDdlRelNode(RelOptCluster cluster, RelNode input) { + this(cluster, cluster.traitSet(), input); + } + + protected DummyDdlRelNode(RelOptCluster cluster, RelTraitSet traits, RelNode input) { + super(cluster, traits, input); + } + } } From c54132acf493b9754423a9f3abfe64e54a04fd1d Mon Sep 17 00:00:00 2001 From: Terran Date: Wed, 4 Mar 2026 11:07:57 +0800 Subject: [PATCH 191/562] [CALCITE-7428] Support regexp function change regexp operator for Hive library --- .../calcite/sql/dialect/HiveSqlDialect.java | 3 ++ .../calcite/sql/fun/SqlLibraryOperators.java | 7 ++-- .../calcite/util/RelToSqlConverterUtil.java | 15 +++++++ .../rel/rel2sql/RelToSqlConverterTest.java | 41 +++++++++++++++++++ site/_docs/reference.md | 2 +- .../apache/calcite/test/SqlOperatorTest.java | 1 + 6 files changed, 64 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql/dialect/HiveSqlDialect.java b/core/src/main/java/org/apache/calcite/sql/dialect/HiveSqlDialect.java index db57e5affedc..689ecc1ecf60 100644 --- a/core/src/main/java/org/apache/calcite/sql/dialect/HiveSqlDialect.java +++ b/core/src/main/java/org/apache/calcite/sql/dialect/HiveSqlDialect.java @@ -129,6 +129,9 @@ public HiveSqlDialect(Context context) { case TRIM: RelToSqlConverterUtil.unparseHiveTrim(writer, call, leftPrec, rightPrec); break; + case RLIKE: + RelToSqlConverterUtil.unparseRegexp(writer, call, leftPrec, rightPrec); + break; default: super.unparseCall(writer, call, leftPrec, rightPrec); } diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java index 947223b682a8..2479a92ba247 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java @@ -705,11 +705,10 @@ static RelDataType deriveTypeSplit(SqlOperatorBinding operatorBinding, public static final SqlFunction REGEXP_SUBSTR = REGEXP_EXTRACT.withName("REGEXP_SUBSTR"); /** The "REGEXP(value, regexp)" function, equivalent to {@link #RLIKE}. */ - @LibraryOperator(libraries = {SPARK}) + @LibraryOperator(libraries = {SPARK, HIVE}) public static final SqlFunction REGEXP = - SqlBasicFunction.create("REGEXP", ReturnTypes.BOOLEAN_NULLABLE, - OperandTypes.STRING_STRING, - SqlFunctionCategory.STRING); + SqlBasicFunction.create("REGEXP", SqlKind.RLIKE, ReturnTypes.BOOLEAN_NULLABLE, + OperandTypes.STRING_STRING); /** The "REGEXP_LIKE(value, regexp)" function, equivalent to {@link #RLIKE}. */ @LibraryOperator(libraries = {SPARK, MYSQL, POSTGRESQL, ORACLE}) diff --git a/core/src/main/java/org/apache/calcite/util/RelToSqlConverterUtil.java b/core/src/main/java/org/apache/calcite/util/RelToSqlConverterUtil.java index 1f15adc7d61e..db417179afc3 100644 --- a/core/src/main/java/org/apache/calcite/util/RelToSqlConverterUtil.java +++ b/core/src/main/java/org/apache/calcite/util/RelToSqlConverterUtil.java @@ -463,4 +463,19 @@ public ClickHouseSqlArrayTypeNameSpec(SqlTypeNameSpec elementTypeName, writer.endList(frame); } } + + /** + * Unparses REGEXP function calls by converting from function call format + * (e.g., REGEXP(column, pattern)) to infix operator format (e.g., column REGEXP pattern). + */ + public static void unparseRegexp(SqlWriter writer, SqlCall call, int leftPrec, int rightPrec) { + if ("REGEXP".equals(call.getOperator().getName())) { + final SqlWriter.Frame frame = writer.startList(SqlWriter.FrameTypeEnum.SIMPLE, "(", ")"); + call.operand(0).unparse(writer, leftPrec, rightPrec); + writer.sep("REGEXP", true); + call.operand(1).unparse(writer, leftPrec, rightPrec); + writer.endList(frame); + } + } + } diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java index 2deca5f9d863..f2a5b4b383cc 100644 --- a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java +++ b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java @@ -11948,4 +11948,45 @@ public Sql schema(CalciteAssert.SchemaSpec schemaSpec) { + "FROM (VALUES (1.0E-310)) AS \"t\" (\"EXPR$0\")"); } + /** Test case for + * [CALCITE-7428] + * Support regexp function change regexp operator for Hive library. */ + @Test void testRegexpWithHive() { + final String query = "select \"brand_name\"\n" + + "from \"product\" where REGEXP(\"brand_name\",'[a-zA-Z]') "; + final String expectedHive = "SELECT `brand_name`\nFROM " + + "`foodmart`.`product`\nWHERE (`brand_name` REGEXP '[a-zA-Z]')"; + final String expectedSpark = "SELECT `brand_name`\nFROM `foodmart`.`product`\n" + + "WHERE REGEXP(`brand_name`, '[a-zA-Z]')"; + sql(query).withLibrary(SqlLibrary.HIVE).withHive().ok(expectedHive); + sql(query).withLibrary(SqlLibrary.SPARK).withSpark().ok(expectedSpark); + } + + /** Test case for + * [CALCITE-7428] + * Support regexp function change regexp operator for Hive library. */ + @Test void testRegexpWithHiveIsNotNull() { + final String query = "select \"brand_name\"\n" + + "from \"product\" where REGEXP(\"brand_name\",'[a-zA-Z]') is not null "; + final String expectedHive = "SELECT `brand_name`\nFROM " + + "`foodmart`.`product`\nWHERE (`brand_name` REGEXP '[a-zA-Z]') IS NOT NULL"; + final String expectedSpark = "SELECT `brand_name`\nFROM `foodmart`.`product`\n" + + "WHERE REGEXP(`brand_name`, '[a-zA-Z]') IS NOT NULL"; + sql(query).withLibrary(SqlLibrary.HIVE).withHive().ok(expectedHive); + sql(query).withLibrary(SqlLibrary.SPARK).withSpark().ok(expectedSpark); + } + + /** Test case for + * [CALCITE-7428] + * Support regexp function change regexp operator for Hive library. */ + @Test void testSelectRegexpWithHiveIsNotNull() { + final String query = "select REGEXP(\"brand_name\",'[a-zA-Z]') is not null \n" + + "from \"product\""; + final String expectedHive = "SELECT (`brand_name` REGEXP '[a-zA-Z]') IS NOT NULL\n" + + "FROM `foodmart`.`product`"; + final String expectedSpark = "SELECT REGEXP(`brand_name`, '[a-zA-Z]') IS NOT NULL\n" + + "FROM `foodmart`.`product`"; + sql(query).withLibrary(SqlLibrary.HIVE).withHive().ok(expectedHive); + sql(query).withLibrary(SqlLibrary.SPARK).withSpark().ok(expectedSpark); + } } diff --git a/site/_docs/reference.md b/site/_docs/reference.md index e1ea8e720697..ee3d2d739f8a 100644 --- a/site/_docs/reference.md +++ b/site/_docs/reference.md @@ -3004,7 +3004,7 @@ In the following: | b s | POW(numeric1, numeric2) | Returns *numeric1* raised to the power *numeric2* | b c h q m o f s p r | POWER(numeric1, numeric2) | Returns *numeric1* raised to the power of *numeric2* | p r | RANDOM() | Generates a random double between 0 and 1 inclusive -| s | REGEXP(string, regexp) | Equivalent to `string1 RLIKE string2` +| s h | REGEXP(string, regexp) | Equivalent to `string1 RLIKE string2` | b | REGEXP_CONTAINS(string, regexp) | Returns whether *string* is a partial match for the *regexp* | b | REGEXP_EXTRACT(string, regexp [, position [, occurrence]]) | Returns the substring in *string* that matches the *regexp*, starting search at *position* (default 1), and until locating the nth *occurrence* (default 1). Returns NULL if there is no match | b | REGEXP_EXTRACT_ALL(string, regexp) | Returns an array of all substrings in *string* that matches the *regexp*. Returns an empty array if there is no match diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java index 41b45d3c761a..b6731f9e63f1 100644 --- a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java +++ b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java @@ -4232,6 +4232,7 @@ void checkIsNull(SqlOperatorFixture f, SqlOperator operator) { checkRlikeFunc(f, SqlLibrary.HIVE, SqlLibraryOperators.RLIKE); checkRlikeFunc(f, SqlLibrary.SPARK, SqlLibraryOperators.RLIKE); checkRlikeFunc(f, SqlLibrary.SPARK, SqlLibraryOperators.REGEXP); + checkRlikeFunc(f, SqlLibrary.HIVE, SqlLibraryOperators.REGEXP); checkRlikeFunc(f, SqlLibrary.MYSQL, SqlLibraryOperators.RLIKE); checkNotRlikeFunc(f.withLibrary(SqlLibrary.HIVE)); checkNotRlikeFunc(f.withLibrary(SqlLibrary.SPARK)); From 12c696d0504c6c7cad8f59caf4d35a00c6cd033c Mon Sep 17 00:00:00 2001 From: Darpan Date: Wed, 25 Mar 2026 11:52:51 +0530 Subject: [PATCH 192/562] [CALCITE-7450] ValuesReduceRule incorrectly drops tuples when filter condition is irreducible When the filter condition contains a UDF that cannot be reduced to a RexLiteral (e.g., no Janino CallImplementor), isAlwaysTrue() returns false for the unreduced RexCall, causing the rule to incorrectly drop the tuple. The fix checks whether the reduced value is a RexLiteral before evaluating isAlwaysTrue(); if it is not, the rule bails out and leaves the plan unchanged. --- .../calcite/rel/rules/ValuesReduceRule.java | 14 +++- .../rel/rules/ValuesReduceRuleTest.java | 69 +++++++++++++++++++ 2 files changed, 80 insertions(+), 3 deletions(-) create mode 100644 core/src/test/java/org/apache/calcite/rel/rules/ValuesReduceRuleTest.java diff --git a/core/src/main/java/org/apache/calcite/rel/rules/ValuesReduceRule.java b/core/src/main/java/org/apache/calcite/rel/rules/ValuesReduceRule.java index 3b8d03e57c7f..acd80a81d977 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/ValuesReduceRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/ValuesReduceRule.java @@ -186,9 +186,17 @@ protected void apply(RelOptRuleCall call, @Nullable LogicalProject project, final RexNode reducedValue = reducibleExps.get((row * fieldsPerRow) + i); ++i; - if (!reducedValue.isAlwaysTrue()) { - ++changeCount; - continue; + // Condition reduced to a literal (or CAST(NULL AS type)); + // evaluate it to decide whether to keep or drop the tuple. + if (reducedValue instanceof RexLiteral + || RexUtil.isNullLiteral(reducedValue, true)) { + if (!reducedValue.isAlwaysTrue()) { + ++changeCount; + continue; + } + } else { + // Condition could not be reduced to a literal + return; } } diff --git a/core/src/test/java/org/apache/calcite/rel/rules/ValuesReduceRuleTest.java b/core/src/test/java/org/apache/calcite/rel/rules/ValuesReduceRuleTest.java new file mode 100644 index 000000000000..81472fce76b1 --- /dev/null +++ b/core/src/test/java/org/apache/calcite/rel/rules/ValuesReduceRuleTest.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.rel.rules; + +import org.apache.calcite.plan.RelOptUtil; +import org.apache.calcite.plan.hep.HepPlanner; +import org.apache.calcite.plan.hep.HepProgram; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.sql.parser.SqlParser; +import org.apache.calcite.tools.FrameworkConfig; +import org.apache.calcite.tools.Frameworks; +import org.apache.calcite.tools.Planner; + +import org.junit.jupiter.api.Test; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; + +/** + * Tests for {@link ValuesReduceRule}. + */ +class ValuesReduceRuleTest { + + /** Test case for + * [CALCITE-7450] + * ValuesReduceRule incorrectly drops tuples when filter condition is + * irreducible. + * + *

    {@code RAND()} function, is non-deterministic + * therefore not reduced by {@code ReduceExpressionsRule}. */ + @Test void testFilterWithNonDeterministicConditionDoesNotDropTuples() + throws Exception { + final FrameworkConfig config = Frameworks.newConfigBuilder() + .defaultSchema(Frameworks.createRootSchema(true)) + .parserConfig(SqlParser.config().withCaseSensitive(false)) + .build(); + + final Planner planner = Frameworks.getPlanner(config); + final String sql = "SELECT * FROM (VALUES (0, 1, 2), (3, 4, 5)) " + + "AS t(a, b, c) WHERE RAND(t.a) > 0.5"; + final RelNode planBefore = + planner.rel(planner.validate(planner.parse(sql))).rel; + + final HepProgram program = HepProgram.builder() + .addRuleInstance(CoreRules.PROJECT_FILTER_VALUES_MERGE) + .build(); + final HepPlanner hepPlanner = new HepPlanner(program); + hepPlanner.setRoot(planBefore); + final RelNode planAfter = hepPlanner.findBestExp(); + + // RAND() is non-deterministic, so the condition cannot be reduced. + // The plan must remain unchanged. + assertThat(RelOptUtil.toString(planAfter), is(RelOptUtil.toString(planBefore))); + } +} From 5e65c4b709a22094edce760799b467ca9bb15186 Mon Sep 17 00:00:00 2001 From: "wenzhuang.zwz" Date: Tue, 24 Feb 2026 18:46:25 +0800 Subject: [PATCH 193/562] [CALCITE-7422] Support large plan optimization mode for HepPlanner Key optimizations of large plan mode: 1. Reusable graph, avoid reinit. 2. Efficient traversal, skip stable subtree. 3. Fine-grained GC. Usage: see comments of HepPlanner() Perf result of LargePlanBenchmark: Match Order Union Num Node Count Rule Transforms Time (ms) -------------------------------------------------------------------- ARBITRARY 1000 4000 6006 1043 ARBITRARY 3000 12000 18006 1306 ARBITRARY 10000 40000 60006 3655 ARBITRARY 30000 120000 180006 13040 DEPTH_FIRST 1000 4000 6006 347 DEPTH_FIRST 3000 12000 18006 1068 DEPTH_FIRST 10000 40000 60006 4165 DEPTH_FIRST 30000 120000 180006 12898 BOTTOM_UP 1000 4000 6006 1145 BOTTOM_UP 3000 12000 18006 10152 TOP_DOWN 1000 4000 6006 1193 TOP_DOWN 3000 12000 18006 8074 --- build.gradle.kts | 42 ++ .../calcite/config/CalciteSystemProperty.java | 8 + .../calcite/plan/AbstractRelOptPlanner.java | 41 +- .../apache/calcite/plan/hep/HepPlanner.java | 284 +++++++++-- .../calcite/plan/hep/HepVertexIterator.java | 92 ++++ .../apache/calcite/test/HepPlannerTest.java | 96 +++- .../benchmarks/LargePlanBenchmark.java | 465 ++++++++++++++++-- 7 files changed, 964 insertions(+), 64 deletions(-) create mode 100644 core/src/main/java/org/apache/calcite/plan/hep/HepVertexIterator.java diff --git a/build.gradle.kts b/build.gradle.kts index 33ff8905c808..63ac833db686 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -90,6 +90,28 @@ val werror by props(true) // treat javac warnings as errors // Inherited from stage-vote-release-plugin: skipSign, useGpgCmd // Inherited from gradle-extensions-plugin: slowSuiteLogThreshold=0L, slowTestLogThreshold=2000L +val hepLargePlanModeTestIncludes = mapOf( + ":core" to listOf( + "**/org/apache/calcite/test/HepPlannerTest.class", + "**/org/apache/calcite/test/RelOptRulesTest.class", + "**/org/apache/calcite/test/RelMetadataTest.class", + "**/org/apache/calcite/sql2rel/RelFieldTrimmerTest.class", + "**/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.class", + "**/org/apache/calcite/test/SqlToRelConverterTest.class", + "**/org/apache/calcite/test/SqlHintsConverterTest.class", + "**/org/apache/calcite/test/InterpreterTest.class", + "**/org/apache/calcite/test/MaterializedViewSubstitutionVisitorTest.class", + "**/org/apache/calcite/test/RuleMatchVisualizerTest.class", + "**/org/apache/calcite/test/enumerable/EnumerableJoinTest.class", + "**/org/apache/calcite/test/enumerable/EnumerableHashJoinTest.class", + "**/org/apache/calcite/test/enumerable/EnumerableCorrelateTest.class" + ), + ":plus" to listOf( + "**/org/apache/calcite/adapter/tpch/TpchTest.class", + "**/org/apache/calcite/sql2rel/TpcdsSqlToRelTest.class" + ) +) + // Java versions prior to 1.8.0u202 have known issues that cause invalid bytecode in certain patterns // of annotation usage. // So we require at least 1.8.0u202 @@ -137,6 +159,11 @@ val buildVersion = "calcite".v + releaseParams.snapshotSuffix println("Building Apache Calcite $buildVersion") +val testHepLargePlanMode by tasks.registering() { + group = LifecycleBasePlugin.VERIFICATION_GROUP + description = "Runs HepPlanner regression tests with large-plan mode enabled by default." +} + releaseArtifacts { fromProject(":release") } @@ -906,6 +933,21 @@ allprojects { } jvmArgs("-Xmx6g") } + hepLargePlanModeTestIncludes[project.path]?.let { includes -> + val hepLargePlanModeTask = register("testHepLargePlanMode") { + group = LifecycleBasePlugin.VERIFICATION_GROUP + description = + "Runs HepPlanner-heavy tests with calcite.hep.large.plan.mode=true." + testClassesDirs = sourceSets["test"].output.classesDirs + classpath = sourceSets["test"].runtimeClasspath + include(includes) + systemProperty("calcite.hep.large.plan.mode", "true") + shouldRunAfter("test") + } + rootProject.tasks.named("testHepLargePlanMode") { + dependsOn(hepLargePlanModeTask) + } + } configureEach { group = LifecycleBasePlugin.VERIFICATION_GROUP if (enableSpotBugs) { diff --git a/core/src/main/java/org/apache/calcite/config/CalciteSystemProperty.java b/core/src/main/java/org/apache/calcite/config/CalciteSystemProperty.java index 70db4505d62f..f38d405e3475 100644 --- a/core/src/main/java/org/apache/calcite/config/CalciteSystemProperty.java +++ b/core/src/main/java/org/apache/calcite/config/CalciteSystemProperty.java @@ -138,6 +138,14 @@ public final class CalciteSystemProperty { public static final CalciteSystemProperty TOPDOWN_OPT = booleanProperty("calcite.planner.topdown.opt", false); + /** Whether {@link org.apache.calcite.plan.hep.HepPlanner} should enable + * large-plan mode by default. + * + *

    This property only affects planners that do not call + * {@code setLargePlanMode} explicitly. */ + public static final CalciteSystemProperty HEP_PLANNER_LARGE_PLAN_MODE = + booleanProperty("calcite.hep.large.plan.mode", false); + /** Whether to disable generate rel data type digest string. * diff --git a/core/src/main/java/org/apache/calcite/plan/AbstractRelOptPlanner.java b/core/src/main/java/org/apache/calcite/plan/AbstractRelOptPlanner.java index 6cb7c4fed484..654f25689395 100644 --- a/core/src/main/java/org/apache/calcite/plan/AbstractRelOptPlanner.java +++ b/core/src/main/java/org/apache/calcite/plan/AbstractRelOptPlanner.java @@ -28,6 +28,7 @@ import org.apache.calcite.util.trace.CalciteTrace; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import org.checkerframework.checker.initialization.qual.UnknownInitialization; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; @@ -119,6 +120,17 @@ protected AbstractRelOptPlanner(RelOptCostFactory costFactory, addListener(new RuleEventLogger()); } + /** + * Explicitly enables rule attempts tracking regardless of log level. + * This is useful for benchmarks and testing. + */ + public void enableRuleAttemptsTracking() { + if (this.ruleAttemptsListener == null) { + this.ruleAttemptsListener = new RuleAttemptsListener(); + addListener(this.ruleAttemptsListener); + } + } + //~ Methods ---------------------------------------------------------------- @Override public void clear() {} @@ -308,13 +320,31 @@ protected void onNewClass(RelNode node) { // do nothing } - protected void dumpRuleAttemptsInfo() { + public void dumpRuleAttemptsInfo() { if (this.ruleAttemptsListener != null) { RULE_ATTEMPTS_LOGGER.debug("Rule Attempts Info for " + this.getClass().getSimpleName()); RULE_ATTEMPTS_LOGGER.debug(this.ruleAttemptsListener.dump()); } } + /** + * Returns the rule attempts information as a map. + * The map key is the rule string representation, + * and the value is a Pair of (attemptCount, totalTimeMicros). + * + *

    This is useful for programmatic access to rule execution statistics, + * e.g., in benchmarks to verify that rules have been applied. + * + * @return Map of rule to the Pair of (attempt count, total time in microseconds), + * or empty map if rule attempts tracking is not enabled + */ + public Map> getRuleAttemptsInfo() { + if (this.ruleAttemptsListener == null) { + return ImmutableMap.of(); + } + return this.ruleAttemptsListener.getRuleAttempts(); + } + /** * Fires a rule, taking care of tracing and listener notification. * @@ -488,6 +518,15 @@ private static class RuleAttemptsListener implements RelOptListener { @Override public void relChosen(RelChosenEvent event) { } + /** + * Returns a copy of the rule attempts map. + * The map key is the rule string representation, + * and the value is a Pair of (attemptCount, totalTimeMicros). + */ + public Map> getRuleAttempts() { + return ImmutableMap.copyOf(this.ruleAttempts); + } + public String dump() { // Sort rules by number of attempts descending, then by rule elapsed time descending, // then by rule name ascending. diff --git a/core/src/main/java/org/apache/calcite/plan/hep/HepPlanner.java b/core/src/main/java/org/apache/calcite/plan/hep/HepPlanner.java index c5f8b5610fab..d2a39d4a43c9 100644 --- a/core/src/main/java/org/apache/calcite/plan/hep/HepPlanner.java +++ b/core/src/main/java/org/apache/calcite/plan/hep/HepPlanner.java @@ -16,6 +16,7 @@ */ package org.apache.calcite.plan.hep; +import org.apache.calcite.config.CalciteSystemProperty; import org.apache.calcite.linq4j.function.Function2; import org.apache.calcite.linq4j.function.Functions; import org.apache.calcite.plan.AbstractRelOptPlanner; @@ -40,6 +41,7 @@ import org.apache.calcite.rel.metadata.RelMetadataProvider; import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.util.ImmutableIntList; import org.apache.calcite.util.Pair; import org.apache.calcite.util.Util; import org.apache.calcite.util.graph.BreadthFirstIterator; @@ -54,21 +56,23 @@ import com.google.common.collect.HashMultimap; import com.google.common.collect.ImmutableList; import com.google.common.collect.Multimap; +import com.google.common.collect.Sets; import org.checkerframework.checker.nullness.qual.Nullable; import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Collection; +import java.util.Deque; import java.util.HashMap; import java.util.HashSet; +import java.util.IdentityHashMap; import java.util.Iterator; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Queue; import java.util.Set; -import java.util.stream.Collectors; import static com.google.common.base.Preconditions.checkArgument; @@ -125,7 +129,7 @@ public class HepPlanner extends AbstractRelOptPlanner { * *

    Value: the set of {@link RelOptRule}s already fired for that exact ID list. */ - private final Multimap, RelOptRule> firedRulesCache = HashMultimap.create(); + private final Multimap firedRulesCache = HashMultimap.create(); /** * Reverse index for {@link #firedRulesCache}, used for cleanup/GC: @@ -136,11 +140,17 @@ public class HepPlanner extends AbstractRelOptPlanner { * *

    Value: match-key ID lists in {@link #firedRulesCache} that contain the key ID. */ - private final Multimap> firedRulesCacheIndex = HashMultimap.create(); - + private final Multimap firedRulesCacheIndex = HashMultimap.create(); private boolean enableFiredRulesCache = false; + /** Enables optimizations for large plans. + * This optimization improves performance for plans of any scale. + * To be removed in the future; the optimization will become the default behavior. + */ + private boolean largePlanMode = false; + + //~ Constructors ----------------------------------------------------------- /** @@ -181,12 +191,55 @@ public HepPlanner( this.mainProgram = requireNonNull(program, "program"); this.onCopyHook = Util.first(onCopyHook, Functions.ignore2()); this.noDag = noDag; + this.largePlanMode = CalciteSystemProperty.HEP_PLANNER_LARGE_PLAN_MODE.value(); + } + + /** + * Create a new {@code HepPlanner} capable of executing multiple HepPrograms + * with (noDag = false, isLargePlanMode = true, enableFiredRulesCache = true). + * + *

    Unlike planners that require setRoot for every optimization pass, + * this planner preserves the internal graph structure and optimized plan across + * successive executions. This allows for multiphase optimization where the + * output of one {@link HepProgram} serves as the immediate starting point for the next. + * + *

    Usage Example: + * 

    {@code
+   *   HepPlanner planner = new HepPlanner();
+   *   // or use other constructor and set isLargePlanMode/enableFiredRulesCache = true
+   *   // HepPlanner planner = new HepPlanner(new HepProgramBuilder().build(), ...);
+   *   // planner.setEnableFiredRulesCache(true);
+   *   // planner.setLargePlanMode(true);
+   *   planner.setRoot(initPlanRoot);
+   *   planner.executeProgram(phase1Program);
+   *   planner.dumpRuleAttemptsInfo(); // optional
+   *   planner.clearRules(); // clear the rules and rule match caches, the graph is preserved
+   *   // other logic ...
+   *   planner.executeProgram(phase2Program);
+   *   planner.clearRules();
+   *   ...
+   *   RelNode optimized = planner.buildFinalPlan();
+   * }
    + * + * @see #setRoot(RelNode) + * @see #executeProgram(HepProgram) + * @see #dumpRuleAttemptsInfo() + * @see #clearRules() + * @see #buildFinalPlan() + */ + public HepPlanner() { + this(HepProgram.builder().build(), null, false, null, RelOptCostImpl.FACTORY); + this.largePlanMode = true; + this.enableFiredRulesCache = true; } //~ Methods ---------------------------------------------------------------- @Override public void setRoot(RelNode rel) { - root = addRelToGraph(rel); + // initRelToVertexCache is used to quickly skip common nodes before traversing its inputs + IdentityHashMap initRelToVertexCache = (isLargePlanMode() && !noDag) + ? new IdentityHashMap<>() : null; + root = addRelToGraph(rel, initRelToVertexCache); dumpGraph(); } @@ -196,14 +249,30 @@ public HepPlanner( @Override public void clear() { super.clear(); + this.materializations.clear(); + clearRules(); + } + + /** Clears the rules and rule match caches while preserving the internal graph + * structure. This is useful for multiphase optimization where the graph should + * be reused across successive {@link HepProgram} executions. + */ + public void clearRules() { for (RelOptRule rule : getRules()) { removeRule(rule); } - this.materializations.clear(); this.firedRulesCache.clear(); this.firedRulesCacheIndex.clear(); } + public boolean isLargePlanMode() { + return largePlanMode; + } + + public void setLargePlanMode(final boolean largePlanMode) { + this.largePlanMode = largePlanMode; + } + @Override public RelNode changeTraits(RelNode rel, RelTraitSet toTraits) { // Ignore traits, except for the root, where we remember // what the final conversion should be. @@ -224,6 +293,10 @@ public HepPlanner( return buildFinalPlan(requireNonNull(root, "'root' must not be null")); } + public RelNode buildFinalPlan() { + return buildFinalPlan(requireNonNull(root, "'root' must not be null")); + } + /** * Enables or disables the fire-rule cache. * @@ -237,7 +310,7 @@ public void setEnableFiredRulesCache(boolean enable) { /** Top-level entry point for a program. Initializes state and then invokes * the program. */ - private void executeProgram(HepProgram program) { + public void executeProgram(HepProgram program) { final HepInstruction.PrepareContext px = HepInstruction.PrepareContext.create(this); final HepState state = program.prepare(px); @@ -249,7 +322,7 @@ void executeProgram(HepProgram instruction, HepProgram.State state) { state.instructionStates.forEach(instructionState -> { instructionState.execute(); int delta = nTransformations - nTransformationsLastGC; - if (delta > graphSizeLastGC) { + if (!isLargePlanMode() && delta > graphSizeLastGC) { // The number of transformations performed since the last // garbage collection is greater than the number of vertices in // the graph at that time. That means there should be a @@ -444,13 +517,20 @@ private void applyRules(HepProgram.State programState, final boolean fullRestartAfterTransformation = programState.matchOrder != HepMatchOrder.ARBITRARY && programState.matchOrder != HepMatchOrder.DEPTH_FIRST; + final boolean useHepVertexIterator = (programState.matchOrder == HepMatchOrder.ARBITRARY + || programState.matchOrder == HepMatchOrder.DEPTH_FIRST) && isLargePlanMode(); int nMatches = 0; boolean fixedPoint; do { - Iterator iter = - getGraphIterator(programState, requireNonNull(root, "root")); + Iterator iter; + if (!useHepVertexIterator) { + iter = getGraphIterator(programState, requireNonNull(root, "root")); + } else { + iter = HepVertexIterator.of(requireNonNull(root, "root"), new HashSet<>()).iterator(); + } + fixedPoint = true; while (iter.hasNext()) { HepRelVertex vertex = iter.next(); @@ -470,7 +550,21 @@ private void applyRules(HepProgram.State programState, // To the extent possible, pick up where we left // off; have to create a new iterator because old // one was invalidated by transformation. - iter = getGraphIterator(programState, newVertex); + if (!useHepVertexIterator) { + iter = getGraphIterator(programState, newVertex); + } else { + // Continue from newVertex and keep previous iterator status. + // It prevents revisiting the large plan's stable subgraph from root. + // A stable subgraph is a part of the DAG to which no rules will be applied. + // For a plan like this, every node replacement in subgraph2 may reset the iterator + // to the root, so subgraph1, although stable, will be visited repeatedly. + // root + // <- subgraph1_root (stable) + // <- ... other nodes in subgraph1 (stable) + // <- subgraph2_root + // <- ... other nodes in subgraph2 (with many node replacements) + iter = ((HepVertexIterator) iter).continueFrom(newVertex); + } if (programState.matchOrder == HepMatchOrder.DEPTH_FIRST) { nMatches = depthFirstApply(programState, iter, rules, forceConversions, nMatches); @@ -493,11 +587,18 @@ private Iterator getGraphIterator( switch (requireNonNull(programState.matchOrder, "programState.matchOrder")) { case ARBITRARY: case DEPTH_FIRST: + if (isLargePlanMode()) { + return HepVertexIterator.of(start, new HashSet<>()).iterator(); + } return DepthFirstIterator.of(graph, start).iterator(); case TOP_DOWN: case BOTTOM_UP: assert start == root; - collectGarbage(); + if (!isLargePlanMode()) { + // NOTE: We do not need to run garbage collection for the whole graph here. + // tryCleanVertices already cleans up potentially removed vertices. + collectGarbage(); + } return TopologicalOrderIterator.of(graph, programState.matchOrder).iterator(); default: throw new @@ -551,6 +652,20 @@ private Iterator getGraphIterator( return null; } + // Cache the fired rule before constructing a HepRuleCall. + ImmutableIntList relIds = null; + if (enableFiredRulesCache) { + int[] ids = new int[bindings.size()]; + for (int i = 0; i < bindings.size(); i++) { + ids[i] = bindings.get(i).getId(); + } + relIds = ImmutableIntList.of(ids); + Collection rules = firedRulesCache.get(relIds); + if (rules.contains(rule)) { + return null; + } + } + HepRuleCall call = new HepRuleCall( this, @@ -559,14 +674,6 @@ private Iterator getGraphIterator( nodeChildren, parents); - List relIds = null; - if (enableFiredRulesCache) { - relIds = call.getRelList().stream().map(RelNode::getId).collect(Collectors.toList()); - if (firedRulesCache.get(relIds).contains(rule)) { - return null; - } - } - // Allow the rule to apply its own side-conditions. if (!rule.matches(call)) { return null; @@ -576,8 +683,8 @@ private Iterator getGraphIterator( if (relIds != null) { firedRulesCache.put(relIds, rule); - for (Integer relId : relIds) { - firedRulesCacheIndex.put(relId, relIds); + for (int i = 0; i < relIds.size(); i++) { + firedRulesCacheIndex.put(relIds.getInt(i), relIds); } } @@ -774,7 +881,9 @@ private HepRelVertex applyTransformationResults( parents.add(parent); } - HepRelVertex newVertex = addRelToGraph(bestRel); + HepRelVertex newVertex = addRelToGraph(bestRel, null); + // LinkedHashSet preserves insertion order during iteration. it is debugging-friendly. + Set garbageVertexSet = new LinkedHashSet<>(); // There's a chance that newVertex is the same as one // of the parents due to common subexpression recognition @@ -785,10 +894,12 @@ private HepRelVertex applyTransformationResults( if (iParentMatch != -1) { newVertex = parents.get(iParentMatch); } else { - contractVertices(newVertex, vertex, parents); + contractVertices(newVertex, vertex, parents, garbageVertexSet); } - if (getListener() != null) { + if (isLargePlanMode()) { + collectGarbage(garbageVertexSet); + } else if (getListener() != null) { // Assume listener doesn't want to see garbage. collectGarbage(); } @@ -824,19 +935,26 @@ private HepRelVertex applyTransformationResults( } private HepRelVertex addRelToGraph( - RelNode rel) { + RelNode rel, @Nullable IdentityHashMap initRelToVertexCache) { // Check if a transformation already produced a reference // to an existing vertex. if (graph.vertexSet().contains(rel)) { return (HepRelVertex) rel; } + // Fast equiv vertex for set root, before add children. + if (initRelToVertexCache != null && initRelToVertexCache.containsKey(rel)) { + HepRelVertex vertex = initRelToVertexCache.get(rel); + assert vertex != null; + return vertex; + } + // Recursively add children, replacing this rel's inputs // with corresponding child vertices. final List inputs = rel.getInputs(); final List newInputs = new ArrayList<>(); for (RelNode input1 : inputs) { - HepRelVertex childVertex = addRelToGraph(input1); + HepRelVertex childVertex = addRelToGraph(input1, initRelToVertexCache); newInputs.add(childVertex); } @@ -868,6 +986,10 @@ private HepRelVertex addRelToGraph( graph.addEdge(newVertex, (HepRelVertex) input); } + if (initRelToVertexCache != null) { + initRelToVertexCache.put(rel, newVertex); + } + nTransformations++; return newVertex; } @@ -875,7 +997,8 @@ private HepRelVertex addRelToGraph( private void contractVertices( HepRelVertex preservedVertex, HepRelVertex discardedVertex, - List parents) { + List parents, + Set garbageVertexSet) { if (preservedVertex == discardedVertex) { // Nop. return; @@ -897,6 +1020,18 @@ private void contractVertices( } clearCache(parent); graph.removeEdge(parent, discardedVertex); + + if (!noDag && isLargePlanMode()) { + // Recursive merge parent path + HepRelVertex addedVertex = mapDigestToVertex.get(parentRel.getRelDigest()); + if (addedVertex != null && addedVertex != parent) { + List parentCopy = // contractVertices will change predecessorList + new ArrayList<>(Graphs.predecessorListOf(graph, parent)); + contractVertices(addedVertex, parent, parentCopy, garbageVertexSet); + continue; + } + } + graph.addEdge(parent, preservedVertex); updateVertex(parent, parentRel); } @@ -904,10 +1039,13 @@ private void contractVertices( // NOTE: we don't actually do graph.removeVertex(discardedVertex), // because it might still be reachable from preservedVertex. // Leave that job for garbage collection. + // If isLargePlanMode is true, we will do fine-grained GC in tryCleanVertices + // by tracking discarded vertex subtree's inward references. if (discardedVertex == root) { root = preservedVertex; } + garbageVertexSet.add(discardedVertex); } /** @@ -992,6 +1130,58 @@ private RelNode buildFinalPlan(HepRelVertex vertex) { return rel; } + /** Try to remove discarded vertices recursively. */ + private void tryCleanVertices(HepRelVertex vertex) { + if (vertex == root || !graph.vertexSet().contains(vertex) + || !graph.getInwardEdges(vertex).isEmpty()) { + return; + } + + // rel is the root of a subtree with no inward edges. + RelNode rel = vertex.getCurrentRel(); + notifyDiscard(rel); + + Set outVertices = new LinkedHashSet<>(); + List outEdges = graph.getOutwardEdges(vertex); + for (DefaultEdge outEdge : outEdges) { + outVertices.add((HepRelVertex) outEdge.target); + } + + for (HepRelVertex child : outVertices) { + graph.removeEdge(vertex, child); + } + assert graph.getInwardEdges(vertex).isEmpty(); + assert graph.getOutwardEdges(vertex).isEmpty(); + graph.vertexSet().remove(vertex); + mapDigestToVertex.remove(rel.getRelDigest()); + + for (HepRelVertex child : outVertices) { + tryCleanVertices(child); + } + clearCache(vertex); + + if (enableFiredRulesCache) { + for (ImmutableIntList relIds : firedRulesCacheIndex.get(rel.getId())) { + firedRulesCache.removeAll(relIds); + } + } + } + + private void collectGarbage(final Set garbageVertexSet) { + for (HepRelVertex vertex : garbageVertexSet) { + tryCleanVertices(vertex); + } + + if (LOGGER.isTraceEnabled()) { + int currentGraphSize = graph.vertexSet().size(); + collectGarbage(); + int currentGraphSize2 = graph.vertexSet().size(); + if (currentGraphSize != currentGraphSize2) { + throw new AssertionError("Graph size changed after garbage collection"); + } + } + } + private void collectGarbage() { if (nTransformations == nTransformationsLastGC) { // No modifications have taken place since the last gc, @@ -1040,7 +1230,7 @@ private void collectGarbage() { if (enableFiredRulesCache) { sweepSet.forEach(rel -> { - for (List relIds : firedRulesCacheIndex.get(rel.getCurrentRel().getId())) { + for (ImmutableIntList relIds : firedRulesCacheIndex.get(rel.getCurrentRel().getId())) { firedRulesCache.removeAll(relIds); } firedRulesCacheIndex.removeAll(rel.getCurrentRel().getId()); @@ -1061,12 +1251,48 @@ private void assertNoCycles() { + cyclicVertices); } + private void assertGraphConsistent() { + int liveNum = 0; + for (HepRelVertex vertex : BreadthFirstIterator.of(graph, requireNonNull(root, "root"))) { + if (graph.getOutwardEdges(vertex).size() + != Sets.newHashSet(requireNonNull(vertex, "vertex").getCurrentRel().getInputs()).size()) { + throw new AssertionError("HepPlanner:outward edge num is different " + + "from input node num, " + vertex); + } + for (DefaultEdge edge : graph.getInwardEdges(vertex)) { + if (!((HepRelVertex) edge.source).getCurrentRel().getInputs().contains(vertex)) { + throw new AssertionError("HepPlanner:inward edge target is not in input node list, " + + vertex); + } + } + liveNum++; + } + + Set validSet = new HashSet<>(); + Deque nodes = new ArrayDeque<>(); + nodes.push(requireNonNull(requireNonNull(root, "root").getCurrentRel())); + while (!nodes.isEmpty()) { + RelNode node = nodes.pop(); + validSet.add(node); + for (RelNode input : node.getInputs()) { + nodes.push(((HepRelVertex) input).getCurrentRel()); + } + } + + if (liveNum == validSet.size()) { + return; + } + throw new AssertionError("HepPlanner:Query graph live node num is different from root" + + " input valid node num, liveNodeNum: " + liveNum + ", validNodeNum: " + validSet.size()); + } + private void dumpGraph() { if (!LOGGER.isTraceEnabled()) { return; } assertNoCycles(); + assertGraphConsistent(); HepRelVertex root = this.root; if (root == null) { diff --git a/core/src/main/java/org/apache/calcite/plan/hep/HepVertexIterator.java b/core/src/main/java/org/apache/calcite/plan/hep/HepVertexIterator.java new file mode 100644 index 000000000000..7831d845d6d3 --- /dev/null +++ b/core/src/main/java/org/apache/calcite/plan/hep/HepVertexIterator.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.plan.hep; + +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.SingleRel; + +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.Iterator; +import java.util.NoSuchElementException; +import java.util.Set; + +/** + * Iterates over the vertices in a HepVertex graph in depth-first order. + * In a HepVertex graph, every HepVertex.getCurrentRel().getInputs() is a + * List<HepRelVertex>. + * + * @param Vertex type + */ +public class HepVertexIterator + implements Iterator { + private final Deque deque = new ArrayDeque<>(); + private final Set visitedSet; + + private HepVertexIterator(V root, Set visitedSet) { + this.deque.push(root); + this.visitedSet = visitedSet; + } + + /** + * Creates a HepVertexIterator for a given HepVertex root. + * + * @param root Root of iteration. + * @param visitedSet Set of HepVertex IDs to exclude from iteration; next() will add more + * items to it. + */ + protected static Iterable of( + final V root, final Set visitedSet) { + return () -> new HepVertexIterator<>(root, visitedSet); + } + + public Iterator continueFrom(V newVertex) { + this.deque.push(newVertex); + return this; + } + + @Override public boolean hasNext() { + return !deque.isEmpty(); + } + + @Override public V next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + V v = deque.pop(); + + RelNode current = v.getCurrentRel(); + if (current instanceof SingleRel) { + @SuppressWarnings("unchecked") V target = (V) ((SingleRel) current).getInput(); + if (visitedSet.add(target.getId())) { + deque.push(target); + } + } else { + for (RelNode input : current.getInputs()) { + @SuppressWarnings("unchecked") V target = (V) input; + if (visitedSet.add(target.getId())) { + deque.push(target); + } + } + } + return v; + } + + @Override public void remove() { + throw new UnsupportedOperationException(); + } +} diff --git a/core/src/test/java/org/apache/calcite/test/HepPlannerTest.java b/core/src/test/java/org/apache/calcite/test/HepPlannerTest.java index 9af5e7fe8368..1729817e368d 100644 --- a/core/src/test/java/org/apache/calcite/test/HepPlannerTest.java +++ b/core/src/test/java/org/apache/calcite/test/HepPlannerTest.java @@ -16,8 +16,10 @@ */ package org.apache.calcite.test; +import org.apache.calcite.config.CalciteSystemProperty; import org.apache.calcite.plan.RelOptListener; import org.apache.calcite.plan.RelOptMaterialization; +import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.plan.hep.HepMatchOrder; import org.apache.calcite.plan.hep.HepPlanner; import org.apache.calcite.plan.hep.HepProgram; @@ -30,6 +32,7 @@ import org.apache.calcite.rel.rules.CoerceInputsRule; import org.apache.calcite.rel.rules.CoreRules; import org.apache.calcite.sql.SqlExplainLevel; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.tools.RelBuilder; import com.google.common.collect.ImmutableList; @@ -366,8 +369,11 @@ private void assertIncludesExactlyOnce(String message, String digest, } @Test void testRuleApplyCount() { + final boolean largePlanMode = + CalciteSystemProperty.HEP_PLANNER_LARGE_PLAN_MODE.value(); + long applyTimes = checkRuleApplyCount(HepMatchOrder.ARBITRARY, false); - assertThat(applyTimes, is(316L)); + assertThat(applyTimes, is(largePlanMode ? 87L : 316L)); applyTimes = checkRuleApplyCount(HepMatchOrder.DEPTH_FIRST, false); assertThat(applyTimes, is(87L)); @@ -391,6 +397,22 @@ private void assertIncludesExactlyOnce(String message, String digest, assertThat(applyTimes, is(65L)); } + @Test void testOrderSensitivePrograms() { + diffRepos = DiffRepository.lookup(HepPlannerTest.class); + + final String topDownPlan = + runUnion(HepMatchOrder.TOP_DOWN, 1, false); + final String bottomUpPlan = + runUnion(HepMatchOrder.BOTTOM_UP, 1, false); + + assertThat(topDownPlan.equals(bottomUpPlan), is(false)); + + final String legacyPlan = runToCalc(false); + final String largePlanModePlan = runToCalc(true); + + assertThat(largePlanModePlan.equals(legacyPlan), is(false)); + } + @Test void testMaterialization() { HepPlanner planner = new HepPlanner(HepProgram.builder().build()); RelNode tableRel = sql("select * from dept").toRel(); @@ -420,6 +442,78 @@ private long checkRuleApplyCount(HepMatchOrder matchOrder, boolean enableFiredRu return listener.getApplyTimes(); } + private String runUnion(HepMatchOrder matchOrder, int matchLimit, + boolean largePlanMode) { + HepProgram program = HepProgram.builder() + .addMatchOrder(matchOrder) + .addMatchLimit(matchLimit) + .addRuleInstance(CoreRules.UNION_TO_DISTINCT) + .build(); + HepPlanner planner = new HepPlanner(program); + planner.setLargePlanMode(largePlanMode); + planner.setRoot(unionPlan()); + return RelOptUtil.toString(planner.findBestExp()); + } + + private String runToCalc(boolean largePlanMode) { + HepProgram program = HepProgram.builder() + .addMatchLimit(1) + .addRuleInstance(CoreRules.PROJECT_TO_CALC) + .build(); + HepPlanner planner = new HepPlanner(program); + planner.setLargePlanMode(largePlanMode); + planner.setRoot(toCalcPlan()); + return RelOptUtil.toString(planner.findBestExp()); + } + + private RelNode unionPlan() { + final RelBuilder builder = RelBuilderTest.createBuilder(); + final RelNode dept = + builder.scan("DEPT") + .project(builder.field("DNAME")) + .build(); + final RelNode emp = + builder.scan("EMP") + .project(builder.field("ENAME")) + .build(); + final RelNode bonus = + builder.scan("BONUS") + .project(builder.field("ENAME")) + .build(); + final RelNode left = + builder.push(dept) + .push(emp) + .union(false) + .build(); + return builder.push(left) + .push(bonus) + .union(false) + .build(); + } + + private RelNode toCalcPlan() { + final RelBuilder builder = RelBuilderTest.createBuilder(); + final RelNode scan = builder.scan("EMP").build(); + final RelNode upper = + builder.push(scan) + .project( + builder.alias( + builder.call(SqlStdOperatorTable.UPPER, builder.field("ENAME")), + "EXPR$0")) + .build(); + final RelNode lower = + builder.push(scan) + .project( + builder.alias( + builder.call(SqlStdOperatorTable.LOWER, builder.field("ENAME")), + "EXPR$0")) + .build(); + return builder.push(upper) + .push(lower) + .union(true) + .build(); + } + /** Listener for HepPlannerTest; counts how many times rules fire. */ private static class HepTestListener implements RelOptListener { private long applyTimes; diff --git a/ubenchmark/src/jmh/java/org/apache/calcite/benchmarks/LargePlanBenchmark.java b/ubenchmark/src/jmh/java/org/apache/calcite/benchmarks/LargePlanBenchmark.java index 6d1cab341364..91455da17263 100644 --- a/ubenchmark/src/jmh/java/org/apache/calcite/benchmarks/LargePlanBenchmark.java +++ b/ubenchmark/src/jmh/java/org/apache/calcite/benchmarks/LargePlanBenchmark.java @@ -26,10 +26,10 @@ import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.schema.SchemaPlus; import org.apache.calcite.schema.impl.AbstractTable; -import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.tools.Frameworks; import org.apache.calcite.tools.RelBuilder; +import org.apache.calcite.util.Pair; import com.google.common.collect.ImmutableList; @@ -46,11 +46,13 @@ import org.openjdk.jmh.annotations.State; import org.openjdk.jmh.annotations.Threads; import org.openjdk.jmh.annotations.Warmup; -import org.openjdk.jmh.runner.Runner; import org.openjdk.jmh.runner.RunnerException; -import org.openjdk.jmh.runner.options.Options; -import org.openjdk.jmh.runner.options.OptionsBuilder; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; import java.util.concurrent.TimeUnit; /** @@ -65,19 +67,37 @@ * are repeatedly applicable. */ -@Fork(value = 1, jvmArgsPrepend = {"-Xss200m"}) -@Measurement(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS) -@Warmup(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) +@Fork(value = 1, jvmArgsPrepend = {"-Xss200m", + "-Dcalcite.disable.generate.type.digest.string=true"}) +@Measurement(iterations = 1, time = 1, timeUnit = TimeUnit.SECONDS) +@Warmup(iterations = 1, time = 1, timeUnit = TimeUnit.SECONDS) @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MILLISECONDS) @State(Scope.Thread) @Threads(1) public class LargePlanBenchmark { - @Param({"100", "1000", "5000", "10000"}) + @Param({"100", "1000", "10000", "100000"}) int unionNum; - private RelBuilder builder; + // For large plans, "DEPTH_FIRST", "BOTTOM_UP", and "TOP_DOWN" are slower than ARBITRARY + @Param({"ARBITRARY"}) + String matchOrder; + + // Enable validation mode to verify rule application counts across different orders. + boolean enableValidation = false; + + boolean isLargePlanMode = true; // false is very slow in 10000 unions + boolean isEnableFiredRulesCache = true; + private static RelBuilder builder; + + // All available match orders for validation + private static final String[] ALL_MATCH_ORDERS = { + "ARBITRARY", "DEPTH_FIRST", "BOTTOM_UP", "TOP_DOWN" + }; + + // Validation sizes: 1, 10, 100, 1000 + private static final int[] VALIDATION_SIZES = {1, 10, 100, 1000}; @Setup(Level.Trial) public void setup() { @@ -112,9 +132,9 @@ private RelNode makeSelectBranch(int i) { .filter( builder.and( builder.equals(builder.field("EMPNO"), builder.literal(i)), - builder.call(SqlStdOperatorTable.GREATER_THAN_OR_EQUAL, + builder.greaterThanOrEqual( builder.field("MGR"), builder.literal(0)), - builder.call(SqlStdOperatorTable.LESS_THAN_OR_EQUAL, + builder.lessThanOrEqual( builder.field("MGR"), builder.literal(0)), builder.equals(builder.field("ENAME"), builder.literal("Y")), builder.equals(builder.field("SAL"), builder.literal(i)) @@ -144,41 +164,420 @@ private RelNode makeUnionTree(int unionNum) { @Benchmark public void testLargeUnionPlan() { + testLargeUnionPlan(unionNum, matchOrder, false); + } + + /** + * Executes the optimization with the given parameters. + * + * @param unionNum number of union branches + * @param matchOrder the match order to use + * @param collectStats whether to collect and return rule statistics (for validation) + * @return rule statistics map if collectStats is true, otherwise empty map + */ + public Map>> testLargeUnionPlan( + int unionNum, String matchOrder, boolean collectStats) { + RelNode root = makeUnionTree(unionNum); + HepMatchOrder hepMatchOrder = HepMatchOrder.valueOf(matchOrder); HepProgram filterReduce = HepProgram.builder() - .addMatchOrder(HepMatchOrder.DEPTH_FIRST) + .addMatchOrder(hepMatchOrder) .addRuleInstance(CoreRules.FILTER_REDUCE_EXPRESSIONS) .build(); HepProgram projectReduce = HepProgram.builder() - .addMatchOrder(HepMatchOrder.DEPTH_FIRST) + .addMatchOrder(hepMatchOrder) .addRuleInstance(CoreRules.PROJECT_REDUCE_EXPRESSIONS) .build(); - // Phrase 1 - HepPlanner planner = new HepPlanner(filterReduce); - planner.setRoot(root); - root = planner.findBestExp(); - planner.clear(); - - // ... do some things cannot be done in planner.findBestExp() ... - // Phrase 2 - planner = new HepPlanner(projectReduce); - planner.setRoot(root); - root = planner.findBestExp(); - planner.clear(); - - // TODO LATER large plan optimization - // TODO LATER set "-Dcalcite.disable.generate.type.digest.string=true" + Map>> stats = new HashMap<>(); + + if (!isLargePlanMode) { + // Phase 1 + HepPlanner planner = new HepPlanner(filterReduce); + if (collectStats) { + planner.enableRuleAttemptsTracking(); + } + planner.setRoot(root); + planner.setEnableFiredRulesCache(isEnableFiredRulesCache); + root = planner.findBestExp(); + if (collectStats) { + stats.put("FILTER", snapshotRuleAttempts(planner)); + } + + // Phase 2 + planner = new HepPlanner(projectReduce); + if (collectStats) { + planner.enableRuleAttemptsTracking(); + } + planner.setEnableFiredRulesCache(isEnableFiredRulesCache); + planner.setRoot(root); + root = planner.findBestExp(); + if (collectStats) { + stats.put("PROJECT", snapshotRuleAttempts(planner)); + } + } else { + HepPlanner planner = new HepPlanner(); + planner.setEnableFiredRulesCache(isEnableFiredRulesCache); + planner.setLargePlanMode(isLargePlanMode); + if (collectStats) { + planner.enableRuleAttemptsTracking(); + } + planner.setRoot(root); + + // Phase 1: Execute FILTER_REDUCE_EXPRESSIONS + Map> beforeFilter = + collectStats ? snapshotRuleAttempts(planner) : null; + planner.executeProgram(filterReduce); + if (collectStats) { + stats.put("FILTER", + subtractRuleAttempts(snapshotRuleAttempts(planner), beforeFilter)); + } + planner.clearRules(); + + // Phase 2: Execute PROJECT_REDUCE_EXPRESSIONS + Map> beforeProject = + collectStats ? snapshotRuleAttempts(planner) : null; + planner.executeProgram(projectReduce); + if (collectStats) { + stats.put("PROJECT", + subtractRuleAttempts(snapshotRuleAttempts(planner), beforeProject)); + } + planner.clearRules(); + + root = planner.buildFinalPlan(); + } + + return stats; + } + + /** + * Returns a string repeated n times (Java 8 compatible). + */ + private static String repeat(String str, int count) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < count; i++) { + sb.append(str); + } + return sb.toString(); + } + + /** + * Runs validation mode to verify that different match orders produce + * the same rule application counts. + */ + public void runValidation() { + this.enableValidation = true; + System.out.println("\n" + + repeat("=", 80)); + System.out.println("VALIDATION MODE: Verifying rule application counts across match orders"); + System.out.println(repeat("=", 80) + "\n"); + + Map>>>> allStats = + new HashMap<>(); + + for (String order : ALL_MATCH_ORDERS) { + System.out.println("Testing match order: " + order); + Map>>> orderStats = new HashMap<>(); + + for (int size : VALIDATION_SIZES) { + System.out.println(" Size: " + size); + Map>> stats = + testLargeUnionPlan(size, order, true); + orderStats.put(size, stats); + } + + allStats.put(order, orderStats); + System.out.println(); + } + + System.out.println("\n" + + repeat("-", 80)); + System.out.println("VALIDATION RESULTS"); + System.out.println(repeat("-", 80) + "\n"); + + boolean allPassed = true; + Map>>> baselineStats = + allStats.get("ARBITRARY"); + + for (String order : ALL_MATCH_ORDERS) { + if (order.equals("ARBITRARY")) { + continue; + } + + System.out.println("Comparing " + order + " against ARBITRARY:"); + Map>>> orderStats = + allStats.get(order); + + for (int size : VALIDATION_SIZES) { + boolean sizePassed = + validateSizeStats(order, size, baselineStats.get(size), orderStats.get(size)); + if (!sizePassed) { + allPassed = false; + } + } + System.out.println(); + } + + System.out.println("\n" + + repeat("=", 80)); + if (allPassed) { + System.out.println("VALIDATION PASSED: All match orders produce " + + "consistent rule application counts"); + } else { + System.out.println("VALIDATION FAILED: Some match orders have " + + "inconsistent rule application counts"); + } + System.out.println(repeat("=", 80) + "\n"); + } + + /** + * Validates that the rule statistics for a specific size match between + * the baseline (ARBITRARY) and the test order. + */ + private boolean validateSizeStats(String order, int size, + Map>> baseline, + Map>> test) { + + boolean passed = true; + StringBuilder sb = new StringBuilder(); + sb.append(String.format(Locale.ROOT, " Size %4d: ", size)); + + Map> baselineFilter = baseline.get("FILTER"); + Map> testFilter = test.get("FILTER"); + if (!comparePhaseStats("FILTER", baselineFilter, testFilter, sb)) { + passed = false; + } + + Map> baselineProject = baseline.get("PROJECT"); + Map> testProject = test.get("PROJECT"); + if (!comparePhaseStats("PROJECT", baselineProject, testProject, sb)) { + passed = false; + } + if (passed) { + sb.append("PASSED"); + } else { + sb.append("FAILED"); + } + + System.out.println(sb.toString()); + return passed; + } + + private boolean comparePhaseStats(String phase, + Map> baseline, + Map> test, + StringBuilder sb) { + + if (baseline == null && test == null) { + return true; + } + if (baseline == null || test == null) { + sb.append(phase).append("(null mismatch) "); + return false; + } + + boolean passed = true; + for (String rule : baseline.keySet()) { + Pair baselineCount = baseline.get(rule); + Pair testCount = test.get(rule); + + if (testCount == null) { + sb.append(phase).append("/").append(rule).append("(missing) "); + passed = false; + continue; + } + + if (!baselineCount.left.equals(testCount.left)) { + sb.append( + String.format(Locale.ROOT, "%s/%s(%d vs %d) ", + phase, rule, baselineCount.left, testCount.left)); + passed = false; + } + } + + return passed; + } + + private static Map> snapshotRuleAttempts(HepPlanner planner) { + return new HashMap<>(planner.getRuleAttemptsInfo()); + } + + private static Map> subtractRuleAttempts( + Map> current, + Map> previous) { + Map> delta = new HashMap<>(); + for (Map.Entry> entry : current.entrySet()) { + Pair oldValue = previous.get(entry.getKey()); + long oldAttempts = oldValue == null ? 0L : oldValue.left; + long oldTime = oldValue == null ? 0L : oldValue.right; + long deltaAttempts = entry.getValue().left - oldAttempts; + long deltaTime = entry.getValue().right - oldTime; + if (deltaAttempts != 0L || deltaTime != 0L) { + delta.put(entry.getKey(), Pair.of(deltaAttempts, deltaTime)); + } + } + return delta; + } + + /** + * Runs benchmark mode for performance testing. + * Tests different union sizes based on the match order's scalability. + */ + public void runBenchmark() { + System.out.println("\n" + + repeat("=", 80)); + System.out.println("BENCHMARK MODE: Performance testing"); + System.out.println(repeat("=", 80) + "\n"); + + // Define size ranges for each match order + // ARBITRARY: 1K, 3K, 10K, 30K, 100K, 300K (best scalability) + // DEPTH_FIRST: 1K, 3K, 10K, 30K (good scalability) + // BOTTOM_UP/TOP_DOWN: 1K, 3K (limited scalability) + Map orderSizes = new HashMap<>(); + orderSizes.put("ARBITRARY", new int[]{1000, 3000, 10000, 30000}); + orderSizes.put("DEPTH_FIRST", new int[]{1000, 3000, 10000, 30000}); + orderSizes.put("BOTTOM_UP", new int[]{1000, 3000}); + orderSizes.put("TOP_DOWN", new int[]{1000, 3000}); + + List results = new ArrayList<>(); + results.add( + String.format(Locale.ROOT, + "%-15s %-10s %-15s %-15s %-15s", + "Match Order", "Union Num", "Node Count", "Rule Attempts", "Time (ms)")); + results.add(repeat("-", 85)); + + for (String order : ALL_MATCH_ORDERS) { + System.out.println("Testing match order: " + order); + int[] sizes = orderSizes.get(order); + + for (int size : sizes) { + int nodeCount = 4 * size + 3; + + // Warmup + testLargeUnionPlan(size, order, false); + + // Actual measurement with rule stats collection + long startTime = System.currentTimeMillis(); + try { + Map>> stats = + testLargeUnionPlan(size, order, true); + long endTime = System.currentTimeMillis(); + long elapsed = endTime - startTime; + + long totalRuleAttempts = 0; + for (Map> phaseStats : stats.values()) { + for (Pair ruleStat : phaseStats.values()) { + totalRuleAttempts += ruleStat.left; + } + } + + results.add( + String.format(Locale.ROOT, + "%-15s %-10d %-15d %-15d %-15d", + order, size, nodeCount, totalRuleAttempts, elapsed)); + System.out.println(" Size " + size + ": " + elapsed + " ms, " + + "nodes=" + nodeCount + ", ruleAttempts=" + totalRuleAttempts); + } catch (Exception e) { + results.add( + String.format(Locale.ROOT, + "%-15s %-10d %-15s %-15s %-15s", + order, size, "N/A", "N/A", "N/A")); + System.out.println(" Size " + size + ": FAILED - " + e.getMessage()); + } + } + System.out.println(); + } + + // Print summary report + System.out.println("\n" + + repeat("=", 85)); + System.out.println("BENCHMARK REPORT"); + System.out.println(repeat("=", 85)); + for (String line : results) { + System.out.println(line); + } + System.out.println(repeat("=", 85) + "\n"); } public static void main(String[] args) throws RunnerException { - Options opt = new OptionsBuilder() - .include(LargePlanBenchmark.class.getSimpleName()) - .detectJvmArgs() - .build(); + LargePlanBenchmark benchmark = new LargePlanBenchmark(); + benchmark.setup(); + benchmark.isLargePlanMode = true; + benchmark.isEnableFiredRulesCache = true; + + // Check command line arguments for mode selection + boolean runValidation = false; + boolean runBenchmark = false; + boolean runProfile = false; + + for (String arg : args) { + if ("--validate".equals(arg) || "-v".equals(arg)) { + runValidation = true; + } else if ("--benchmark".equals(arg) || "-b".equals(arg)) { + runBenchmark = true; + } else if ("--both".equals(arg)) { + runValidation = true; + runBenchmark = true; + } else if ("--profile".equals(arg) || "-p".equals(arg)) { + runProfile = true; + } + } - new Runner(opt).run(); + // Profile mode takes precedence (specialized mode) + if (runProfile) { + benchmark.runProfileMode(); + return; + } + + // Default: run benchmark mode if no arguments specified + if (!runValidation && !runBenchmark) { + runBenchmark = true; + } + + // Run validation first (if requested) + if (runValidation) { + benchmark.runValidation(); + } + + // Then run benchmark (if requested) + if (runBenchmark) { + benchmark.runBenchmark(); + } + } + + /** + * Runs profile mode for performance tuning with ARBITRARY order at 100K unions. + * This mode is optimized for generating perf/flame graph records. + */ + public void runProfileMode() { + System.out.println("\n" + + repeat("=", 80)); + System.out.println("PROFILE MODE: ARBITRARY order with 100000 unions"); + System.out.println("Optimized for perf/flame graph recording"); + System.out.println(repeat("=", 80) + "\n"); + + String order = "ARBITRARY"; + int size = 100000; + + long startTime = System.currentTimeMillis(); + try { + testLargeUnionPlan(size, order, false); + long endTime = System.currentTimeMillis(); + long elapsed = endTime - startTime; + + System.out.println("\n" + + repeat("=", 80)); + System.out.println("PROFILE RUN COMPLETED"); + System.out.println(repeat("=", 80)); + System.out.println("Match Order: " + order); + System.out.println("Union Size: " + size); + System.out.println("Time: " + elapsed + " ms (" + (elapsed / 1000.0) + " s)"); + System.out.println(repeat("=", 80) + "\n"); + } catch (Exception e) { + System.err.println("Profile run failed: " + e.getMessage()); + e.printStackTrace(); + } } } From f33798d8fb434a5daa964145e0130a295836d5d7 Mon Sep 17 00:00:00 2001 From: "cc.cai" <2356672992@qq.com> Date: Mon, 23 Mar 2026 23:35:51 +0800 Subject: [PATCH 194/562] [CALCITE-6646] Support timestamp data type in Arrow adapter --- .../arrow/AbstractArrowEnumerator.java | 45 ++++++++- .../adapter/arrow/ArrowFieldTypeFactory.java | 21 ++++ .../arrow/ArrowAdapterDataTypesTest.java | 96 ++++++++++++++++--- .../adapter/arrow/ArrowAdapterTest.java | 18 ++-- .../calcite/adapter/arrow/ArrowDataTest.java | 82 ++++++++++++++++ 5 files changed, 240 insertions(+), 22 deletions(-) diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/AbstractArrowEnumerator.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/AbstractArrowEnumerator.java index 6351dca36ffc..8cc08b990475 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/AbstractArrowEnumerator.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/AbstractArrowEnumerator.java @@ -20,11 +20,14 @@ import org.apache.calcite.util.ImmutableIntList; import org.apache.calcite.util.Util; +import org.apache.arrow.vector.TimeStampVector; import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.arrow.vector.VectorUnloader; import org.apache.arrow.vector.ipc.ArrowFileReader; import org.apache.arrow.vector.ipc.message.ArrowRecordBatch; +import org.apache.arrow.vector.types.TimeUnit; +import org.apache.arrow.vector.types.pojo.ArrowType; import java.io.IOException; import java.util.ArrayList; @@ -66,16 +69,54 @@ protected void loadNextArrowBatch() { @Override public Object current() { if (fields.size() == 1) { - return this.valueVectors.get(0).getObject(currRowIndex); + return getValue(this.valueVectors.get(0), currRowIndex); } Object[] current = new Object[valueVectors.size()]; for (int i = 0; i < valueVectors.size(); i++) { ValueVector vector = this.valueVectors.get(i); - current[i] = vector.getObject(currRowIndex); + current[i] = getValue(vector, currRowIndex); } return current; } + /** Extracts a value from a vector at the given index. + * + *

    For {@link TimeStampVector}, converts the raw value to + * milliseconds since epoch, which is the representation used by + * Calcite's Enumerable runtime for TIMESTAMP types. */ + private static Object getValue(ValueVector vector, int index) { + if (vector instanceof TimeStampVector) { + if (vector.isNull(index)) { + return null; + } + final TimeStampVector tsVector = (TimeStampVector) vector; + final long rawValue = tsVector.get(index); + final ArrowType.Timestamp tsType = + (ArrowType.Timestamp) vector.getField().getType(); + return toMillis(rawValue, tsType.getUnit()); + } + return vector.getObject(index); + } + + /** Converts a raw timestamp value to milliseconds since epoch. + * + *

    Note: for {@link TimeUnit#MICROSECOND} and {@link TimeUnit#NANOSECOND}, + * this conversion is lossy because sub-millisecond precision is truncated. */ + private static long toMillis(long rawValue, TimeUnit unit) { + switch (unit) { + case SECOND: + return rawValue * 1000L; + case MILLISECOND: + return rawValue; + case MICROSECOND: + return rawValue / 1000L; + case NANOSECOND: + return rawValue / 1_000_000L; + default: + throw new IllegalArgumentException("Unsupported TimeUnit: " + unit); + } + } + @Override public void reset() { throw new UnsupportedOperationException(); } diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowFieldTypeFactory.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowFieldTypeFactory.java index ad993c813dc1..1693637e8c05 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowFieldTypeFactory.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowFieldTypeFactory.java @@ -82,6 +82,27 @@ private static RelDataType of(ArrowType arrowType, JavaTypeFactory typeFactory) ((ArrowType.Decimal) arrowType).getScale()); case Time: return typeFactory.createSqlType(SqlTypeName.TIME); + case Timestamp: + ArrowType.Timestamp timestampType = (ArrowType.Timestamp) arrowType; + int timestampPrecision; + switch (timestampType.getUnit()) { + case SECOND: + timestampPrecision = 0; + break; + case MILLISECOND: + timestampPrecision = 3; + break; + case MICROSECOND: + timestampPrecision = 6; + break; + case NANOSECOND: + timestampPrecision = 9; + break; + default: + throw new IllegalArgumentException("Unsupported Timestamp unit: " + + timestampType.getUnit()); + } + return typeFactory.createSqlType(SqlTypeName.TIMESTAMP, timestampPrecision); default: throw new IllegalArgumentException("Unsupported type: " + arrowType); } diff --git a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterDataTypesTest.java b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterDataTypesTest.java index 46e440493700..566b4c531ee9 100644 --- a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterDataTypesTest.java +++ b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterDataTypesTest.java @@ -68,7 +68,7 @@ static void initializeArrowState(@TempDir Path sharedTempDir) String sql = "select \"tinyIntField\" from arrowdatatype"; String plan = "PLAN=ArrowToEnumerableConverter\n" + " ArrowProject(tinyIntField=[$0])\n" - + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]])\n\n"; + + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]])\n\n"; String result = "tinyIntField=0\ntinyIntField=1\n"; CalciteAssert.that() .with(arrow) @@ -82,7 +82,7 @@ static void initializeArrowState(@TempDir Path sharedTempDir) String sql = "select \"smallIntField\" from arrowdatatype"; String plan = "PLAN=ArrowToEnumerableConverter\n" + " ArrowProject(smallIntField=[$1])\n" - + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]])\n\n"; + + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]])\n\n"; String result = "smallIntField=0\nsmallIntField=1\n"; CalciteAssert.that() .with(arrow) @@ -96,7 +96,7 @@ static void initializeArrowState(@TempDir Path sharedTempDir) String sql = "select \"intField\" from arrowdatatype"; String plan = "PLAN=ArrowToEnumerableConverter\n" + " ArrowProject(intField=[$2])\n" - + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]])\n\n"; + + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]])\n\n"; String result = "intField=0\nintField=1\n"; CalciteAssert.that() .with(arrow) @@ -110,7 +110,7 @@ static void initializeArrowState(@TempDir Path sharedTempDir) String sql = "select \"longField\" from arrowdatatype"; String plan = "PLAN=ArrowToEnumerableConverter\n" + " ArrowProject(longField=[$5])\n" - + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]])\n\n"; + + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]])\n\n"; String result = "longField=0\nlongField=1\n"; CalciteAssert.that() .with(arrow) @@ -124,7 +124,7 @@ static void initializeArrowState(@TempDir Path sharedTempDir) String sql = "select \"floatField\" from arrowdatatype"; String plan = "PLAN=ArrowToEnumerableConverter\n" + " ArrowProject(floatField=[$4])\n" - + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]])\n\n"; + + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]])\n\n"; String result = "floatField=0.0\nfloatField=1.0\n"; CalciteAssert.that() .with(arrow) @@ -138,7 +138,7 @@ static void initializeArrowState(@TempDir Path sharedTempDir) String sql = "select \"doubleField\" from arrowdatatype"; String plan = "PLAN=ArrowToEnumerableConverter\n" + " ArrowProject(doubleField=[$6])\n" - + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]])\n\n"; + + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]])\n\n"; String result = "doubleField=0.0\ndoubleField=1.0\n"; CalciteAssert.that() .with(arrow) @@ -152,7 +152,7 @@ static void initializeArrowState(@TempDir Path sharedTempDir) String sql = "select \"decimalField\" from arrowdatatype"; String plan = "PLAN=ArrowToEnumerableConverter\n" + " ArrowProject(decimalField=[$8])\n" - + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]])\n\n"; + + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]])\n\n"; String result = "decimalField=0.00\ndecimalField=1.00\n"; CalciteAssert.that() .with(arrow) @@ -166,7 +166,7 @@ static void initializeArrowState(@TempDir Path sharedTempDir) String sql = "select \"dateField\" from arrowdatatype"; String plan = "PLAN=ArrowToEnumerableConverter\n" + " ArrowProject(dateField=[$9])\n" - + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]])\n\n"; + + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]])\n\n"; String result = "dateField=1970-01-01\n" + "dateField=1970-01-02\n"; CalciteAssert.that() @@ -181,7 +181,7 @@ static void initializeArrowState(@TempDir Path sharedTempDir) String sql = "select \"booleanField\" from arrowdatatype"; String plan = "PLAN=ArrowToEnumerableConverter\n" + " ArrowProject(booleanField=[$7])\n" - + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]])\n\n"; + + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]])\n\n"; String result = "booleanField=null\nbooleanField=true\nbooleanField=false\n"; CalciteAssert.that() .with(arrow) @@ -198,7 +198,7 @@ static void initializeArrowState(@TempDir Path sharedTempDir) String sql = "select \"decimalField2\" from arrowdatatype"; String plan = "PLAN=ArrowToEnumerableConverter\n" + " ArrowProject(decimalField2=[$10])\n" - + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]])\n\n"; + + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]])\n\n"; String result = "decimalField2=20.000\ndecimalField2=21.000\n"; CalciteAssert.that() .with(arrow) @@ -212,7 +212,7 @@ static void initializeArrowState(@TempDir Path sharedTempDir) String sql = "select \"timeField\" from arrowdatatype"; String plan = "PLAN=ArrowToEnumerableConverter\n" + " ArrowProject(timeField=[$11])\n" - + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]])\n\n"; + + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]])\n\n"; String result = "timeField=00:00:00\n" + "timeField=00:00:01\n"; CalciteAssert.that() @@ -222,4 +222,78 @@ static void initializeArrowState(@TempDir Path sharedTempDir) .returns(result) .explainContains(plan); } + + @Test void testTimestampSecProject() { + String sql = "select \"timestampSecField\" from arrowdatatype"; + String plan = "PLAN=ArrowToEnumerableConverter\n" + + " ArrowProject(timestampSecField=[$12])\n" + + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]])\n\n"; + String result = "timestampSecField=2024-01-01 00:00:00\n" + + "timestampSecField=2024-01-02 00:00:00\n"; + CalciteAssert.that() + .with(arrow) + .query(sql) + .limit(2) + .returns(result) + .explainContains(plan); + } + + @Test void testTimestampMilliProject() { + String sql = "select \"timestampMilliField\" from arrowdatatype"; + String plan = "PLAN=ArrowToEnumerableConverter\n" + + " ArrowProject(timestampMilliField=[$13])\n" + + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]])\n\n"; + String result = "timestampMilliField=2024-01-01 00:00:00.000\n" + + "timestampMilliField=2024-01-02 00:00:00.000\n"; + CalciteAssert.that() + .with(arrow) + .query(sql) + .limit(2) + .returns(result) + .explainContains(plan); + } + + /** Test case for + * [CALCITE-6646] + * Support timestamp data type in Arrow adapter. + * + *

    The source data is {@code 2024-01-01 00:00:00.123456} in microseconds. + * The Enumerable runtime only supports millisecond precision, so the result + * is truncated to {@code 2024-01-01 00:00:00.123}. */ + @Test void testTimestampMicroProject() { + String sql = "select \"timestampMicroField\" from arrowdatatype"; + String plan = "PLAN=ArrowToEnumerableConverter\n" + + " ArrowProject(timestampMicroField=[$14])\n" + + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]])\n\n"; + String result = "timestampMicroField=2024-01-01 00:00:00.123\n" + + "timestampMicroField=2024-01-02 00:00:00.123\n"; + CalciteAssert.that() + .with(arrow) + .query(sql) + .limit(2) + .returns(result) + .explainContains(plan); + } + + /** Test case for + * [CALCITE-6646] + * Arrow adapter should support Timestamp data type. + * + *

    The source data is {@code 2024-01-01 00:00:00.123456789} in nanoseconds. + * The Enumerable runtime only supports millisecond precision, so the result + * is truncated to {@code 2024-01-01 00:00:00.123}. */ + @Test void testTimestampNanoProject() { + String sql = "select \"timestampNanoField\" from arrowdatatype"; + String plan = "PLAN=ArrowToEnumerableConverter\n" + + " ArrowProject(timestampNanoField=[$15])\n" + + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]])\n\n"; + String result = "timestampNanoField=2024-01-01 00:00:00.123\n" + + "timestampNanoField=2024-01-02 00:00:00.123\n"; + CalciteAssert.that() + .with(arrow) + .query(sql) + .limit(2) + .returns(result) + .explainContains(plan); + } } diff --git a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java index 6027e2448803..14f387509c9e 100644 --- a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java +++ b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java @@ -860,7 +860,7 @@ static void initializeArrowState(@TempDir Path sharedTempDir) String plan = "PLAN=ArrowToEnumerableConverter\n" + " ArrowProject(booleanField=[$7])\n" + " ArrowFilter(condition=[$7])\n" - + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]])\n\n"; + + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]])\n\n"; String result = "booleanField=true\nbooleanField=true\n"; CalciteAssert.that() @@ -878,7 +878,7 @@ static void initializeArrowState(@TempDir Path sharedTempDir) String plan = "PLAN=ArrowToEnumerableConverter\n" + " ArrowProject(intField=[$2])\n" + " ArrowFilter(condition=[>($2, 10)])\n" - + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]])\n\n"; + + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]])\n\n"; String result = "intField=11\nintField=12\n"; CalciteAssert.that() @@ -896,7 +896,7 @@ static void initializeArrowState(@TempDir Path sharedTempDir) String plan = "PLAN=ArrowToEnumerableConverter\n" + " ArrowProject(booleanField=[$7])\n" + " ArrowFilter(condition=[NOT($7)])\n" - + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]])\n\n"; + + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]])\n\n"; String result = "booleanField=false\nbooleanField=false\n"; CalciteAssert.that() @@ -915,7 +915,7 @@ static void initializeArrowState(@TempDir Path sharedTempDir) String plan = "PLAN=ArrowToEnumerableConverter\n" + " ArrowProject(booleanField=[$7])\n" + " ArrowFilter(condition=[IS NOT TRUE($7)])\n" - + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]])\n\n"; + + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]])\n\n"; String result = "booleanField=null\nbooleanField=false\n"; CalciteAssert.that() @@ -933,7 +933,7 @@ static void initializeArrowState(@TempDir Path sharedTempDir) String plan = "PLAN=ArrowToEnumerableConverter\n" + " ArrowProject(booleanField=[$7])\n" + " ArrowFilter(condition=[IS NOT FALSE($7)])\n" - + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]])\n\n"; + + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]])\n\n"; String result = "booleanField=null\nbooleanField=true\n"; CalciteAssert.that() @@ -951,7 +951,7 @@ static void initializeArrowState(@TempDir Path sharedTempDir) String plan = "PLAN=ArrowToEnumerableConverter\n" + " ArrowProject(booleanField=[$7])\n" + " ArrowFilter(condition=[IS NULL($7)])\n" - + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]])\n\n"; + + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]])\n\n"; String result = "booleanField=null\n"; CalciteAssert.that() @@ -972,7 +972,7 @@ static void initializeArrowState(@TempDir Path sharedTempDir) String plan = "PLAN=ArrowToEnumerableConverter\n" + " ArrowProject(decimalField=[$8])\n" + " ArrowFilter(condition=[=($8, 1.00)])\n" - + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]])\n\n"; + + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]])\n\n"; String result = "decimalField=1.00\n"; CalciteAssert.that() @@ -989,7 +989,7 @@ static void initializeArrowState(@TempDir Path sharedTempDir) String plan = "PLAN=ArrowToEnumerableConverter\n" + " ArrowProject(doubleField=[$6])\n" + " ArrowFilter(condition=[=($6, 1.0E0)])\n" - + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]])\n\n"; + + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]])\n\n"; String result = "doubleField=1.0\n"; CalciteAssert.that() @@ -1006,7 +1006,7 @@ static void initializeArrowState(@TempDir Path sharedTempDir) String plan = "PLAN=ArrowToEnumerableConverter\n" + " ArrowProject(stringField=[$3])\n" + " ArrowFilter(condition=[=($3, '1')])\n" - + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]])\n\n"; + + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]])\n\n"; String result = "stringField=1\n"; CalciteAssert.that() diff --git a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowDataTest.java b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowDataTest.java index 0d92da040e8e..8241a80dd10a 100644 --- a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowDataTest.java +++ b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowDataTest.java @@ -16,6 +16,8 @@ */ package org.apache.calcite.adapter.arrow; +import org.apache.calcite.avatica.util.DateTimeUtils; + import org.apache.arrow.adapter.jdbc.ArrowVectorIterator; import org.apache.arrow.adapter.jdbc.JdbcToArrow; import org.apache.arrow.adapter.jdbc.JdbcToArrowConfig; @@ -32,6 +34,10 @@ import org.apache.arrow.vector.IntVector; import org.apache.arrow.vector.SmallIntVector; import org.apache.arrow.vector.TimeSecVector; +import org.apache.arrow.vector.TimeStampMicroVector; +import org.apache.arrow.vector.TimeStampMilliVector; +import org.apache.arrow.vector.TimeStampNanoVector; +import org.apache.arrow.vector.TimeStampSecVector; import org.apache.arrow.vector.TinyIntVector; import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.VectorSchemaRoot; @@ -67,6 +73,10 @@ */ public class ArrowDataTest { + /** 2024-01-01 00:00:00 UTC, in epoch milliseconds. */ + private static final long BASE_EPOCH_MILLIS = + DateTimeUtils.unixTimestamp(2024, 1, 1, 0, 0, 0); + private final int batchSize; private final int entries; private byte tinyIntValue; @@ -111,6 +121,14 @@ private Schema makeArrowDateTypeSchema() { FieldType decimalType2 = FieldType.nullable(new ArrowType.Decimal(12, 3, 128)); FieldType dateType = FieldType.nullable(new ArrowType.Date(DateUnit.DAY)); FieldType timeType = FieldType.nullable(new ArrowType.Time(TimeUnit.SECOND, 32)); + FieldType timestampSecType = + FieldType.nullable(new ArrowType.Timestamp(TimeUnit.SECOND, null)); + FieldType timestampMilliType = + FieldType.nullable(new ArrowType.Timestamp(TimeUnit.MILLISECOND, null)); + FieldType timestampMicroType = + FieldType.nullable(new ArrowType.Timestamp(TimeUnit.MICROSECOND, null)); + FieldType timestampNanoType = + FieldType.nullable(new ArrowType.Timestamp(TimeUnit.NANOSECOND, null)); childrenBuilder.add(new Field("tinyIntField", tinyIntType, null)); childrenBuilder.add(new Field("smallIntField", smallIntType, null)); @@ -124,6 +142,10 @@ private Schema makeArrowDateTypeSchema() { childrenBuilder.add(new Field("dateField", dateType, null)); childrenBuilder.add(new Field("decimalField2", decimalType2, null)); childrenBuilder.add(new Field("timeField", timeType, null)); + childrenBuilder.add(new Field("timestampSecField", timestampSecType, null)); + childrenBuilder.add(new Field("timestampMilliField", timestampMilliType, null)); + childrenBuilder.add(new Field("timestampMicroField", timestampMicroType, null)); + childrenBuilder.add(new Field("timestampNanoField", timestampNanoType, null)); return new Schema(childrenBuilder.build(), null); } @@ -282,6 +304,18 @@ public void writeArrowDataType(File file) throws IOException { case "timeField": timeField(vector, numRows); break; + case "timestampSecField": + timestampSecField(vector, numRows); + break; + case "timestampMilliField": + timestampMilliField(vector, numRows); + break; + case "timestampMicroField": + timestampMicroField(vector, numRows); + break; + case "timestampNanoField": + timestampNanoField(vector, numRows); + break; default: throw new IllegalStateException("Not supported type yet: " + vector.getMinorType()); } @@ -430,4 +464,52 @@ private void timeField(FieldVector fieldVector, int rowCount) { } fieldVector.setValueCount(rowCount); } + + private void timestampSecField(FieldVector fieldVector, int rowCount) { + TimeStampSecVector tsVector = (TimeStampSecVector) fieldVector; + tsVector.setInitialCapacity(rowCount); + tsVector.allocateNew(); + for (int i = 0; i < rowCount; i++) { + tsVector.set(i, + BASE_EPOCH_MILLIS / DateTimeUtils.MILLIS_PER_SECOND + + i * DateTimeUtils.SECONDS_PER_DAY); + } + fieldVector.setValueCount(rowCount); + } + + private void timestampMilliField(FieldVector fieldVector, int rowCount) { + TimeStampMilliVector tsVector = (TimeStampMilliVector) fieldVector; + tsVector.setInitialCapacity(rowCount); + tsVector.allocateNew(); + for (int i = 0; i < rowCount; i++) { + tsVector.set(i, BASE_EPOCH_MILLIS + i * DateTimeUtils.MILLIS_PER_DAY); + } + fieldVector.setValueCount(rowCount); + } + + private void timestampMicroField(FieldVector fieldVector, int rowCount) { + // Sub-millisecond part (.000456) will be truncated by the adapter. + TimeStampMicroVector tsVector = (TimeStampMicroVector) fieldVector; + tsVector.setInitialCapacity(rowCount); + tsVector.allocateNew(); + for (int i = 0; i < rowCount; i++) { + tsVector.set(i, + BASE_EPOCH_MILLIS * 1000L + 123456L + + i * DateTimeUtils.MILLIS_PER_DAY * 1000L); + } + fieldVector.setValueCount(rowCount); + } + + private void timestampNanoField(FieldVector fieldVector, int rowCount) { + // Sub-millisecond part (.000456789) will be truncated by the adapter. + TimeStampNanoVector tsVector = (TimeStampNanoVector) fieldVector; + tsVector.setInitialCapacity(rowCount); + tsVector.allocateNew(); + for (int i = 0; i < rowCount; i++) { + tsVector.set(i, + BASE_EPOCH_MILLIS * DateTimeUtils.NANOS_PER_MILLI + 123456789L + + i * DateTimeUtils.MILLIS_PER_DAY * DateTimeUtils.NANOS_PER_MILLI); + } + fieldVector.setValueCount(rowCount); + } } From 269e8c7eb1c26bc3d8446a46511592df5bc1378e Mon Sep 17 00:00:00 2001 From: Darpan Date: Fri, 27 Mar 2026 09:36:05 +0530 Subject: [PATCH 195/562] [CALCITE-7450] Refactor ValuesReduceRuleTest inside RelOptRulesTest (addendum) --- .../rel/rules/ValuesReduceRuleTest.java | 69 ------------------- .../apache/calcite/test/RelOptRulesTest.java | 15 ++++ .../apache/calcite/test/RelOptRulesTest.xml | 12 ++++ 3 files changed, 27 insertions(+), 69 deletions(-) delete mode 100644 core/src/test/java/org/apache/calcite/rel/rules/ValuesReduceRuleTest.java diff --git a/core/src/test/java/org/apache/calcite/rel/rules/ValuesReduceRuleTest.java b/core/src/test/java/org/apache/calcite/rel/rules/ValuesReduceRuleTest.java deleted file mode 100644 index 81472fce76b1..000000000000 --- a/core/src/test/java/org/apache/calcite/rel/rules/ValuesReduceRuleTest.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.calcite.rel.rules; - -import org.apache.calcite.plan.RelOptUtil; -import org.apache.calcite.plan.hep.HepPlanner; -import org.apache.calcite.plan.hep.HepProgram; -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.sql.parser.SqlParser; -import org.apache.calcite.tools.FrameworkConfig; -import org.apache.calcite.tools.Frameworks; -import org.apache.calcite.tools.Planner; - -import org.junit.jupiter.api.Test; - -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.MatcherAssert.assertThat; - -/** - * Tests for {@link ValuesReduceRule}. - */ -class ValuesReduceRuleTest { - - /** Test case for - * [CALCITE-7450] - * ValuesReduceRule incorrectly drops tuples when filter condition is - * irreducible. - * - *

    {@code RAND()} function, is non-deterministic - * therefore not reduced by {@code ReduceExpressionsRule}. */ - @Test void testFilterWithNonDeterministicConditionDoesNotDropTuples() - throws Exception { - final FrameworkConfig config = Frameworks.newConfigBuilder() - .defaultSchema(Frameworks.createRootSchema(true)) - .parserConfig(SqlParser.config().withCaseSensitive(false)) - .build(); - - final Planner planner = Frameworks.getPlanner(config); - final String sql = "SELECT * FROM (VALUES (0, 1, 2), (3, 4, 5)) " - + "AS t(a, b, c) WHERE RAND(t.a) > 0.5"; - final RelNode planBefore = - planner.rel(planner.validate(planner.parse(sql))).rel; - - final HepProgram program = HepProgram.builder() - .addRuleInstance(CoreRules.PROJECT_FILTER_VALUES_MERGE) - .build(); - final HepPlanner hepPlanner = new HepPlanner(program); - hepPlanner.setRoot(planBefore); - final RelNode planAfter = hepPlanner.findBestExp(); - - // RAND() is non-deterministic, so the condition cannot be reduced. - // The plan must remain unchanged. - assertThat(RelOptUtil.toString(planAfter), is(RelOptUtil.toString(planBefore))); - } -} diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index 5b9b9d8aba8d..1009d3688e16 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -12267,4 +12267,19 @@ private void checkLoptOptimizeJoinRule(LoptOptimizeJoinRule rule) { .withTopDownGeneralDecorrelate(true) .check(); } + + /** Test case for + * [CALCITE-7450] + * ValuesReduceRule incorrectly drops tuples when filter condition is + * irreducible. + * + *

    {@code RAND()} is non-deterministic and therefore not reduced by + * {@code ReduceExpressionsRule}. The rule must leave the plan unchanged. */ + @Test void testFilterWithNonDeterministicConditionDoesNotDropTuples() { + final String sql = "SELECT * FROM (VALUES (0, 1, 2), (3, 4, 5)) " + + "AS t(a, b, c) WHERE RAND(t.a) > 0.5"; + sql(sql) + .withRule(CoreRules.PROJECT_FILTER_VALUES_MERGE) + .checkUnchanged(); + } } diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index 9cb281d0a53b..f04ced869df0 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -6058,6 +6058,18 @@ LogicalProject(COMM=[$6]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) }))]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +]]> + + + + + 0.5]]> + + + (RAND($0), CAST(0.5:DECIMAL(2, 1)):DOUBLE NOT NULL)]) + LogicalValues(tuples=[[{ 0, 1, 2 }, { 3, 4, 5 }]]) ]]> From 98407e9bcd30cc4e4b2df1efdcb209ed0ab45478 Mon Sep 17 00:00:00 2001 From: hongyu guo Date: Sat, 28 Mar 2026 00:58:45 +0800 Subject: [PATCH 196/562] [CALCITE-7316] The POSITION function in SQLite is missing the FROM clause --- .../calcite/sql/dialect/SqliteSqlDialect.java | 58 +++++++++++++++++-- .../rel/rel2sql/RelToSqlConverterTest.java | 36 ++++++++++++ 2 files changed, 88 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql/dialect/SqliteSqlDialect.java b/core/src/main/java/org/apache/calcite/sql/dialect/SqliteSqlDialect.java index 18e865f7e957..82376ae576ab 100644 --- a/core/src/main/java/org/apache/calcite/sql/dialect/SqliteSqlDialect.java +++ b/core/src/main/java/org/apache/calcite/sql/dialect/SqliteSqlDialect.java @@ -18,12 +18,21 @@ import org.apache.calcite.config.NullCollation; import org.apache.calcite.rel.core.JoinRelType; +import org.apache.calcite.sql.SqlBasicFunction; import org.apache.calcite.sql.SqlCall; import org.apache.calcite.sql.SqlDialect; +import org.apache.calcite.sql.SqlFunction; +import org.apache.calcite.sql.SqlFunctionCategory; +import org.apache.calcite.sql.SqlLiteral; import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlNodeList; import org.apache.calcite.sql.SqlWriter; +import org.apache.calcite.sql.fun.SqlCase; import org.apache.calcite.sql.fun.SqlLibraryOperators; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.parser.SqlParserPos; +import org.apache.calcite.sql.type.OperandTypes; +import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.util.RelToSqlConverterUtil; import org.checkerframework.checker.nullness.qual.Nullable; @@ -32,6 +41,17 @@ * A SqliteSqlDialect implementation for the SQLite database. */ public class SqliteSqlDialect extends SqlDialect { + // Use plain function nodes here so the SQLite POSITION rewrite does not + // re-enter dialect-specific operator handling during unparsing. + private static final SqlFunction INSTR = + SqlBasicFunction.create("INSTR", ReturnTypes.INTEGER_NULLABLE, + OperandTypes.STRING_STRING, + SqlFunctionCategory.STRING); + + private static final SqlFunction SUBSTR = + SqlBasicFunction.create("SUBSTR", ReturnTypes.ARG0_NULLABLE_VARYING, + OperandTypes.STRING_INTEGER_OPTIONAL_INTEGER, + SqlFunctionCategory.STRING); public static final SqlDialect.Context DEFAULT_CONTEXT = SqlDialect.EMPTY_CONTEXT .withDatabaseProduct(SqlDialect.DatabaseProduct.DUCKDB) @@ -85,16 +105,42 @@ public SqliteSqlDialect(SqlDialect.Context context) { RelToSqlConverterUtil.unparseTrimLR(writer, call, leftPrec, rightPrec); break; case POSITION: - final SqlWriter.Frame frame = writer.startFunCall("INSTR"); - writer.sep(","); - call.operand(1).unparse(writer, leftPrec, rightPrec); - writer.sep(","); - call.operand(0).unparse(writer, leftPrec, rightPrec); - writer.endFunCall(frame); + switch (call.operandCount()) { + case 2: + INSTR.createCall(call.getParserPosition(), call.operand(1), call.operand(0)) + .unparse(writer, leftPrec, rightPrec); + break; + case 3: + positionWithFromCall(call).unparse(writer, leftPrec, rightPrec); + break; + default: + super.unparseCall(writer, call, leftPrec, rightPrec); + } break; default: super.unparseCall(writer, call, leftPrec, rightPrec); } } + private static SqlNode positionWithFromCall(SqlCall call) { + final SqlParserPos pos = call.getParserPosition(); + final SqlNode source = call.operand(1); + final SqlNode search = call.operand(0); + final SqlNode start = call.operand(2); + final SqlNode relativePosition = + INSTR.createCall(pos, SUBSTR.createCall(pos, source, start), search); + final SqlNode startAdjustment = + SqlStdOperatorTable.MINUS.createCall(pos, + SqlNode.clone(start), + SqlLiteral.createExactNumeric("1", pos)); + final SqlNode whenCondition = + SqlStdOperatorTable.GREATER_THAN.createCall(pos, + SqlNode.clone(relativePosition), + SqlLiteral.createExactNumeric("0", pos)); + final SqlNode thenResult = + SqlStdOperatorTable.PLUS.createCall(pos, relativePosition, startAdjustment); + return new SqlCase(pos, null, SqlNodeList.of(whenCondition), + SqlNodeList.of(thenResult), SqlNode.clone(relativePosition)); + } + } diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java index f2a5b4b383cc..598fb7500ec0 100644 --- a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java +++ b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java @@ -3692,11 +3692,47 @@ private SqlDialect nonOrdinalDialect() { sql(query1).withClickHouse().ok(expected1); } + /** Test case for + * [CALCITE-7316] + * The POSITION function in SQLite is missing the FROM clause. */ @Test void testPositionFunctionForSqlite() { final String query = "select position('A' IN 'ABC') from \"product\""; final String expected = "SELECT INSTR('ABC', 'A')\n" + "FROM \"foodmart\".\"product\""; sql(query).withSQLite().ok(expected); + + final String query1 = "select position('C' IN 'ABCABC' FROM 4) from \"product\""; + final String expected1 = + "SELECT CASE WHEN INSTR(SUBSTR('ABCABC', 4), 'C') > 0 THEN " + + "INSTR(SUBSTR('ABCABC', 4), 'C') + (4 - 1) " + + "ELSE INSTR(SUBSTR('ABCABC', 4), 'C') END\n" + + "FROM \"foodmart\".\"product\""; + sql(query1).withSQLite().ok(expected1); + + final String query2 = "select position('A' IN 'ABC' FROM 2) from \"product\""; + final String expected2 = + "SELECT CASE WHEN INSTR(SUBSTR('ABC', 2), 'A') > 0 THEN " + + "INSTR(SUBSTR('ABC', 2), 'A') + (2 - 1) " + + "ELSE INSTR(SUBSTR('ABC', 2), 'A') END\n" + + "FROM \"foodmart\".\"product\""; + sql(query2).withSQLite().ok(expected2); + } + + /** Test case for + * [CALCITE-7316] + * The POSITION function in SQLite is missing the FROM clause. */ + @Test void testPositionFunctionForSqliteInWhereClause() { + final String query = "select * from \"product\"\n" + + "where \"product_class_id\" = " + + "position(\"product_name\" IN \"brand_name\" FROM 2)"; + final String expected = + "SELECT *\n" + + "FROM \"foodmart\".\"product\"\n" + + "WHERE \"product_class_id\" = CASE WHEN INSTR(SUBSTR(\"brand_name\", 2), " + + "\"product_name\") > 0 THEN INSTR(SUBSTR(\"brand_name\", 2), " + + "\"product_name\") + (2 - 1) ELSE INSTR(SUBSTR(\"brand_name\", 2), " + + "\"product_name\") END"; + sql(query).withSQLite().ok(expected); } @Test void testPositionFunctionForHive() { From b482e331971e5985158d0570d077af2d690c3774 Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Mon, 30 Mar 2026 17:42:25 +0800 Subject: [PATCH 197/562] [CALCITE-7431] RelTraitSet#getTrait seems to mishandle RelCompositeTrait --- .../enumerable/EnumerableMergeUnion.java | 14 ++-- .../org/apache/calcite/plan/RelTraitSet.java | 70 ++++++++++++++++--- .../org/apache/calcite/plan/RelTraitTest.java | 35 ++++++++++ 3 files changed, 102 insertions(+), 17 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeUnion.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeUnion.java index d06fa31661a1..09d5a9e221e8 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeUnion.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeUnion.java @@ -22,10 +22,8 @@ import org.apache.calcite.linq4j.tree.Expressions; import org.apache.calcite.linq4j.tree.ParameterExpression; import org.apache.calcite.plan.RelOptCluster; -import org.apache.calcite.plan.RelTrait; import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.RelCollation; -import org.apache.calcite.rel.RelCollationTraitDef; import org.apache.calcite.rel.RelNode; import org.apache.calcite.util.BuiltInMethod; import org.apache.calcite.util.Pair; @@ -49,13 +47,17 @@ protected EnumerableMergeUnion(RelOptCluster cluster, RelTraitSet traitSet, throw new IllegalArgumentException("EnumerableMergeUnion with no collation"); } for (RelNode input : inputs) { - final RelTrait inputCollationTrait = - input.getTraitSet().getTrait(RelCollationTraitDef.INSTANCE); + // Use getCollations() rather than getTrait() so that we handle the case + // where the input's collation slot holds a RelCompositeTrait (multiple + // collations). For each required collation, at least one of the input's + // collations must satisfy it. + final List inputCollations = input.getTraitSet().getCollations(); for (RelCollation collation : collations) { - if (inputCollationTrait == null || !inputCollationTrait.satisfies(collation)) { + boolean satisfied = inputCollations.stream().anyMatch(ic -> ic.satisfies(collation)); + if (!satisfied) { throw new IllegalArgumentException("EnumerableMergeUnion input does " + "not satisfy collation. EnumerableMergeUnion collation: " - + collation + ". Input collation: " + inputCollationTrait + ". Input: " + + collation + ". Input collations: " + inputCollations + ". Input: " + input); } } diff --git a/core/src/main/java/org/apache/calcite/plan/RelTraitSet.java b/core/src/main/java/org/apache/calcite/plan/RelTraitSet.java index 65747426062c..fff7eea4f93b 100644 --- a/core/src/main/java/org/apache/calcite/plan/RelTraitSet.java +++ b/core/src/main/java/org/apache/calcite/plan/RelTraitSet.java @@ -87,7 +87,13 @@ public static RelTraitSet createEmpty() { * {@link #size()} or less than 0. */ public RelTrait getTrait(int index) { - return traits[index]; + final RelTrait trait = traits[index]; + if (trait instanceof RelCompositeTrait) { + throw new IllegalStateException("Trait index " + index + + " has multiple values in this trait set; " + + "use getTraits(RelTraitDef) instead of getTrait(RelTraitDef)"); + } + return trait; } /** @@ -110,21 +116,29 @@ public List getTraits(int index) { } @Override public RelTrait get(int index) { - return getTrait(index); + return traits[index]; } /** * Returns whether a given kind of trait is enabled. */ public boolean isEnabled(RelTraitDef traitDef) { - return getTrait(traitDef) != null; + return findIndex(traitDef) >= 0; } /** * Retrieves a RelTrait of the given type from the set. * + *

    If this trait def supports multiple values (i.e. its trait implements + * {@link RelMultipleTrait}), the underlying slot may contain a + * {@link RelCompositeTrait} when more than one value is present. In that + * case this method throws {@link IllegalStateException}; use + * {@link #getTraits(RelTraitDef)} instead. + * * @param traitDef the type of RelTrait to retrieve * @return the RelTrait, or null if not found + * @throws IllegalStateException if the slot holds a composite (multiple) + * trait; use {@link #getTraits(RelTraitDef)} in that case */ public @Nullable T getTrait(RelTraitDef traitDef) { int index = findIndex(traitDef); @@ -375,17 +389,44 @@ public RelTraitSet getDefaultSansConvention() { * {@link RelDistributionTraitDef#INSTANCE}, or null if the * {@link RelDistributionTraitDef#INSTANCE} is not registered * in this traitSet. + * + *

    If this trait set contains multiple distributions (a composite trait), + * this method throws {@link IllegalStateException}. Use + * {@link #getDistributions()} to handle both the single and multi-distribution + * cases uniformly. */ @SuppressWarnings("unchecked") public @Nullable T getDistribution() { return (@Nullable T) getTrait(RelDistributionTraitDef.INSTANCE); } + /** + * Returns {@link RelDistribution} traits defined by + * {@link RelDistributionTraitDef#INSTANCE}. + * + *

    Returns an empty list when the trait def is not registered, a + * singleton list for the common single-distribution case, and a list with + * more than one element when a {@link RelCompositeTrait} is present. + */ + @SuppressWarnings("unchecked") + public List getDistributions() { + int index = findIndex(RelDistributionTraitDef.INSTANCE); + if (index < 0) { + return ImmutableList.of(); + } + return (List) (List) getTraits(index); + } + /** * Returns {@link RelCollation} trait defined by * {@link RelCollationTraitDef#INSTANCE}, or null if the * {@link RelCollationTraitDef#INSTANCE} is not registered * in this traitSet. + * + *

    If this trait set contains multiple collations (a composite trait), + * this method throws {@link IllegalStateException}. Use + * {@link #getCollations()} to handle both the single and multi-collation + * cases uniformly. */ @SuppressWarnings("unchecked") public @Nullable T getCollation() { @@ -395,17 +436,19 @@ public RelTraitSet getDefaultSansConvention() { /** * Returns {@link RelCollation} traits defined by * {@link RelCollationTraitDef#INSTANCE}. + * + *

    Returns an empty list when the trait def is not registered, a + * singleton list for the common single-collation case, and a list with + * more than one element when a {@link RelCompositeTrait} is present. */ @SuppressWarnings("unchecked") public List getCollations() { - RelCollation trait = getTrait(RelCollationTraitDef.INSTANCE); - if (trait == null) { + int index = findIndex(RelCollationTraitDef.INSTANCE); + if (index < 0) { return ImmutableList.of(); } - if (trait instanceof RelCompositeTrait) { - return ((RelCompositeTrait) trait).traitList(); - } - return ImmutableList.of(trait); + // getTraits(int) already unwraps RelCompositeTrait transparently. + return (List) (List) getTraits(index); } /** @@ -577,8 +620,13 @@ public boolean contains(RelTrait trait) { */ public boolean containsIfApplicable(RelTrait trait) { // Note that '==' is sufficient, because trait should be canonized. - final RelTrait trait1 = getTrait(trait.getTraitDef()); - return trait1 == null || trait1 == trait; + int index = findIndex(trait.getTraitDef()); + if (index < 0) { + // TraitDef not registered in this set → treat as "not applicable" → true + return true; + } + final RelTrait stored = get(index); + return stored == trait; } /** diff --git a/core/src/test/java/org/apache/calcite/plan/RelTraitTest.java b/core/src/test/java/org/apache/calcite/plan/RelTraitTest.java index 6a9ddbaa49a5..2760b1e157e7 100644 --- a/core/src/test/java/org/apache/calcite/plan/RelTraitTest.java +++ b/core/src/test/java/org/apache/calcite/plan/RelTraitTest.java @@ -20,6 +20,10 @@ import org.apache.calcite.rel.RelCollation; import org.apache.calcite.rel.RelCollationTraitDef; import org.apache.calcite.rel.RelCollations; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.test.RelBuilderTest; +import org.apache.calcite.tools.FrameworkConfig; +import org.apache.calcite.tools.RelBuilder; import com.google.common.collect.ImmutableList; @@ -33,6 +37,7 @@ import static org.hamcrest.Matchers.hasSize; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static java.lang.Integer.toHexString; @@ -98,4 +103,34 @@ private void assertCanonical(String message, Supplier> collat RelTraitSet traits3 = traits2.replace(RelCollations.of(1)); assertFalse(traits3.equalsSansConvention(traits2)); } + + /** Test for + * [CALCITE-7431] + * RelTraitSet#getTrait seems to mishandle RelCompositeTrait. */ + @Test void testRelCompositeTrait() { + // Build: EMP -> Sort(MGR asc) -> Project(MGR, MGR as MGR2) + // The project maps both output columns 0 and 1 back to input column 3 + // (MGR), so the planner derives two collations: [0 ASC] and [1 ASC], which + // are stored as a RelCompositeTrait in the output trait set. + final FrameworkConfig config = RelBuilderTest.config().build(); + final RelBuilder b = RelBuilder.create(config); + final RelNode in = b + .scan("EMP") + .sort(3) // MGR asc + .project(b.field(3), b.alias(b.field(3), "MGR2")) // MGR, MGR as MGR2 + .build(); + + final RelTraitSet traitSet = in.getTraitSet(); + + final List collations = traitSet.getCollations(); + assertTrue(collations.size() >= 2, + "getCollations() should expose all composite collations"); + + assertThrows(IllegalStateException.class, traitSet::getCollation, + "getCollation() should throw when a RelCompositeTrait is present"); + + assertThrows(IllegalStateException.class, + () -> traitSet.getTrait(RelCollationTraitDef.INSTANCE), + "getTrait() should throw when a RelCompositeTrait is present"); + } } From 1b806d10441b3bb5afe5dcbd01b4a343079f2dea Mon Sep 17 00:00:00 2001 From: hongyu guo Date: Sat, 28 Mar 2026 23:57:22 +0800 Subject: [PATCH 198/562] [CALCITE-7187] Java UDF byte arrays cannot be mapped to VARBINARY --- .../calcite/adapter/enumerable/EnumUtils.java | 12 +++++++ .../apache/calcite/runtime/SqlFunctions.java | 16 +++++++++ .../apache/calcite/util/BuiltInMethod.java | 2 ++ .../java/org/apache/calcite/test/UdfTest.java | 35 +++++++++++++++++++ site/_docs/reference.md | 4 +++ .../java/org/apache/calcite/util/Smalls.java | 18 ++++++++++ 6 files changed, 87 insertions(+) diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java index 8787e682cc3e..018ba8bb7551 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java @@ -17,6 +17,7 @@ package org.apache.calcite.adapter.enumerable; import org.apache.calcite.adapter.java.JavaTypeFactory; +import org.apache.calcite.avatica.util.ByteString; import org.apache.calcite.avatica.util.DateTimeUtils; import org.apache.calcite.linq4j.AbstractEnumerable; import org.apache.calcite.linq4j.Enumerable; @@ -306,6 +307,8 @@ private static Expression toInternal(Expression operand, } else if (targetType == Long.class) { return Expressions.call(BuiltInMethod.TIMESTAMP_TO_LONG_OPTIONAL.method, operand); } + } else if (fromType == byte[].class && targetType == ByteString.class) { + return Expressions.call(BuiltInMethod.BYTE_ARRAY_TO_BYTE_STRING.method, operand); } return operand; } @@ -346,6 +349,8 @@ private static Expression fromInternal(Expression operand, if (isA(fromType, Primitive.LONG)) { return Expressions.call(BuiltInMethod.INTERNAL_TO_TIMESTAMP.method, operand); } + } else if (targetType == byte[].class && fromType == ByteString.class) { + return Expressions.call(BuiltInMethod.BYTE_STRING_TO_BYTE_ARRAY.method, operand); } if (Primitive.is(operand.type) && Primitive.isBox(targetType)) { @@ -437,6 +442,13 @@ public static Expression convert(Expression operand, Type fromType, return operand; } + if (fromType == byte[].class && toType == ByteString.class) { + return Expressions.call(BuiltInMethod.BYTE_ARRAY_TO_BYTE_STRING.method, operand); + } + if (fromType == ByteString.class && toType == byte[].class) { + return Expressions.call(BuiltInMethod.BYTE_STRING_TO_BYTE_ARRAY.method, operand); + } + // TODO use Expressions#convertChecked to throw exception in case of overflow (CALCITE-6366) // E.g. from "Short" to "int". diff --git a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java index e2adce8ea1c7..0d1db4f855d1 100644 --- a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java +++ b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java @@ -6010,6 +6010,22 @@ public static int time(long timestampMillis, String timeZone) { } } + public static @PolyNull ByteString byteArrayToByteString(byte @PolyNull [] bytes) { + if (bytes == null) { + return null; + } else { + return new ByteString(bytes); + } + } + + public static byte @PolyNull [] byteStringToByteArray(@PolyNull ByteString s) { + if (s == null) { + return null; + } else { + return s.getBytes(); + } + } + /** Helper for CAST(... AS VARBINARY(maxLength)). */ public static @PolyNull ByteString truncate(@PolyNull ByteString s, int maxLength) { if (s == null) { diff --git a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java index 7ff2995282b9..61aebe15f51e 100644 --- a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java +++ b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java @@ -720,6 +720,8 @@ public enum BuiltInMethod { STRING_TO_TIMESTAMP_WITH_LOCAL_TIME_ZONE(SqlFunctions.class, "toTimestampWithLocalTimeZone", String.class), STRING_TO_BINARY(SqlFunctions.class, "stringToBinary", String.class, Charset.class), + BYTE_ARRAY_TO_BYTE_STRING(SqlFunctions.class, "byteArrayToByteString", byte[].class), + BYTE_STRING_TO_BYTE_ARRAY(SqlFunctions.class, "byteStringToByteArray", ByteString.class), TIMESTAMP_STRING_TO_TIMESTAMP_WITH_LOCAL_TIME_ZONE(SqlFunctions.class, "toTimestampWithLocalTimeZone", String.class, TimeZone.class), TIME_WITH_LOCAL_TIME_ZONE_TO_TIME(SqlFunctions.class, "timeWithLocalTimeZoneToTime", diff --git a/core/src/test/java/org/apache/calcite/test/UdfTest.java b/core/src/test/java/org/apache/calcite/test/UdfTest.java index 60ed8e5be10e..63492af2bcfc 100644 --- a/core/src/test/java/org/apache/calcite/test/UdfTest.java +++ b/core/src/test/java/org/apache/calcite/test/UdfTest.java @@ -190,6 +190,18 @@ private CalciteAssert.AssertThat withUdf() { + "'\n" + " },\n" + " {\n" + + " name: 'BYTEARRAY',\n" + + " className: '" + + Smalls.ByteArrayFunction.class.getName() + + "'\n" + + " },\n" + + " {\n" + + " name: 'BYTEARRAY_LENGTH',\n" + + " className: '" + + Smalls.ByteArrayLengthFunction.class.getName() + + "'\n" + + " },\n" + + " {\n" + " name: 'CHARACTERARRAY',\n" + " className: '" + Smalls.CharacterArrayFunction.class.getName() @@ -1107,6 +1119,29 @@ private static CalciteAssert.AssertThat withBadUdf(Class clazz) { withUdf().query(sql2).returns("C=true\n"); } + /** Test case for + * [CALCITE-7187] + * Java UDF byte arrays cannot be mapped to VARBINARY. */ + @Test void testByteArrayDirectComparison() { + final String testString = "test"; + final String testHex = "74657374"; + + final String sql = "values \"adhoc\".bytearray('" + testString + "')"; + withUdf().query(sql).typeIs("[EXPR$0 VARBINARY]"); + + final String sql2 = "select \"adhoc\".bytearray(cast('" + testString + + "' as varchar)) = x'" + testHex + "' as C\n"; + withUdf().query(sql2).returns("C=true\n"); + } + + /** Test case for + * [CALCITE-7187] + * Java UDF byte arrays cannot be mapped to VARBINARY. */ + @Test void testByteArrayParameter() { + withUdf().query("values \"adhoc\".bytearray_length(x'74657374')") + .returns("EXPR$0=4\n"); + } + /** * Test for [CALCITE-7186] * Add mapping from Character[] to VARCHAR in Java UDF. diff --git a/site/_docs/reference.md b/site/_docs/reference.md index ee3d2d739f8a..96eb1a25b5a6 100644 --- a/site/_docs/reference.md +++ b/site/_docs/reference.md @@ -3426,6 +3426,10 @@ can specify the name and optionality of each parameter using the [Parameter]({{ site.apiRoot }}/org/apache/calcite/linq4j/function/Parameter.html) annotation. +For Java UDFs, `byte[]` and `ByteString` are supported Java representations of +SQL `VARBINARY` values for parameters and return types. Boxed byte arrays +(`Byte[]`) are not supported. + ### Calling functions with named and optional parameters Usually when you call a function, you need to specify all of its parameters, diff --git a/testkit/src/main/java/org/apache/calcite/util/Smalls.java b/testkit/src/main/java/org/apache/calcite/util/Smalls.java index 3b34d1a88dd9..a6c52e466f6c 100644 --- a/testkit/src/main/java/org/apache/calcite/util/Smalls.java +++ b/testkit/src/main/java/org/apache/calcite/util/Smalls.java @@ -69,6 +69,7 @@ import java.io.IOException; import java.lang.reflect.Method; import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; import java.sql.Date; import java.sql.Time; import java.sql.Timestamp; @@ -1493,6 +1494,23 @@ public static ByteString eval(String s) { } } + /** User-defined function with return type byte[]. */ + public static class ByteArrayFunction { + public static byte[] eval(String s) { + if (s == null) { + return null; + } + return s.getBytes(StandardCharsets.UTF_8); + } + } + + /** User-defined function with parameter type byte[]. */ + public static class ByteArrayLengthFunction { + public static int eval(byte[] bytes) { + return bytes.length; + } + } + /** User-defined function with return type Character[]. */ public static class CharacterArrayFunction { public static Character[] eval(String s) { From dedce96bc2ad74c38834011e6073824133cdcc70 Mon Sep 17 00:00:00 2001 From: "cc.cai" <2356672992@qq.com> Date: Fri, 27 Mar 2026 21:31:24 +0800 Subject: [PATCH 199/562] [CALCITE-6300] Function MAP_VALUES/MAP_KEYS gives exception when mapVauleType and mapKeyType not equals map Biggest mapKeytype or mapValueType --- .../apache/calcite/sql/type/OperandTypes.java | 30 +++++++++++++++++ .../apache/calcite/test/SqlOperatorTest.java | 32 ++++++++++++++++++- 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/calcite/sql/type/OperandTypes.java b/core/src/main/java/org/apache/calcite/sql/type/OperandTypes.java index b67899d761a0..66b4aab9b66d 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/OperandTypes.java +++ b/core/src/main/java/org/apache/calcite/sql/type/OperandTypes.java @@ -31,6 +31,8 @@ import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.SqlOperatorBinding; import org.apache.calcite.sql.SqlUtil; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.sql.util.SqlBasicVisitor; import org.apache.calcite.sql.validate.SqlLambdaScope; import org.apache.calcite.sql.validate.SqlValidator; @@ -1555,9 +1557,37 @@ private static class MapFunctionOperandTypeChecker } return false; } + // Insert implicit casts for operands whose SqlTypeName differs + // from the inferred key/value type. + coerceOperands(callBinding, argTypes, + componentType.left, componentType.right); return true; } + /** Casts operands whose {@code SqlTypeName} differs from the + * target key or value type. Operands at even positions are keys, + * odd positions are values. */ + private static void coerceOperands(SqlCallBinding callBinding, + List operandTypes, + RelDataType keyType, RelDataType valueType) { + final SqlValidator validator = callBinding.getValidator(); + final SqlCall call = callBinding.getCall(); + final List operands = call.getOperandList(); + for (int i = 0; i < operands.size(); i++) { + final RelDataType targetType = i % 2 == 0 ? keyType : valueType; + if (operandTypes.get(i).getSqlTypeName() + != targetType.getSqlTypeName()) { + final SqlNode castNode = + SqlStdOperatorTable.CAST.createCall(SqlParserPos.ZERO, + operands.get(i), + SqlTypeUtil.convertTypeToSpec(targetType) + .withNullable(targetType.isNullable())); + call.setOperand(i, castNode); + validator.setValidatedNodeType(castNode, targetType); + } + } + } + /** * Extract the key type and value type of arg types. */ diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java index b6731f9e63f1..e79bbe197b75 100644 --- a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java +++ b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java @@ -9356,6 +9356,20 @@ void checkArrayReverseFunc(SqlOperatorFixture f0, SqlFunction function, f1.checkScalar("map_keys(map('foo', 1, 'bar', 2))", "[foo, bar]", "CHAR(3) NOT NULL ARRAY NOT NULL"); + // [CALCITE-6300] MAP function with mixed key/value types + f1.checkScalar("map_keys(map(cast(1 as tinyint), 1, 2, 2))", "[1, 2]", + "INTEGER NOT NULL ARRAY NOT NULL"); + f1.checkScalar("map_keys(map(cast(1 as tinyint), 1, cast(2 as double), 2))", + "[1.0, 2.0]", + "DOUBLE NOT NULL ARRAY NOT NULL"); + f1.checkScalar("map_keys(map(cast(1 as tinyint), 1, cast(2 as float), 2))", + "[1.0, 2.0]", + "FLOAT NOT NULL ARRAY NOT NULL"); + f1.checkFails("map_keys(map(cast(1 as tinyint), 1, cast(null as float), 2))", + "Illegal arguments for MAP_KEYS function: " + + "using a map with a null key is not allowed", + true); + f.checkFails("map_keys(map['foo', 1, null, 2])", "Illegal arguments for MAP_KEYS function: using a map with a null key is not allowed", true); @@ -9395,6 +9409,21 @@ void checkArrayReverseFunc(SqlOperatorFixture f0, SqlFunction function, f1.checkScalar("map_values(map('foo', 1, 'bar', cast(null as integer)))", "[1, null]", "INTEGER ARRAY NOT NULL"); + // [CALCITE-6300] MAP function with mixed key/value types + f1.checkScalar("map_values(map('foo', null))", "[null]", + "NULL ARRAY NOT NULL"); + f1.checkScalar("map_values(map('foo', 1, 'bar', cast(1 as tinyint)))", "[1, 1]", + "INTEGER NOT NULL ARRAY NOT NULL"); + f1.checkScalar("map_values(map('foo', 1, 'bar', cast(1 as double)))", + "[1.0, 1.0]", + "DOUBLE NOT NULL ARRAY NOT NULL"); + f1.checkScalar("map_values(map('foo', 1, 'bar', cast(1 as float)))", + "[1.0, 1.0]", + "FLOAT NOT NULL ARRAY NOT NULL"); + f1.checkScalar("map_values(map('foo', 1, 'bar', cast(null as float)))", + "[1.0, null]", + "FLOAT ARRAY NOT NULL"); + f.checkFails("map_values(map['foo', 1, null, 2])", "Illegal arguments for MAP_VALUES function: using a map with a null key is not allowed", true); @@ -13880,8 +13909,9 @@ private static void checkArrayConcatAggFuncFails(SqlOperatorFixture t) { f1.checkScalar("map('washington', 1, 'obama', 44)", "{washington=1, obama=44}", "(CHAR(10) NOT NULL, INTEGER NOT NULL) MAP NOT NULL"); + // [CALCITE-6300] values are coerced to DECIMAL(11, 1) f1.checkScalar("map('k1', 1, 'k2', 2.0)", - "{k1=1, k2=2.0}", + "{k1=1.0, k2=2.0}", "(CHAR(2) NOT NULL, DECIMAL(11, 1) NOT NULL) MAP NOT NULL"); } From e4ea136524c24982c6d655827467a02237058fd5 Mon Sep 17 00:00:00 2001 From: "cc.cai" <2356672992@qq.com> Date: Thu, 26 Mar 2026 22:33:44 +0800 Subject: [PATCH 200/562] [CALCITE-6636] Support CNF condition of Arrow ArrowAdapter --- .../calcite/adapter/arrow/ArrowFilter.java | 2 +- .../calcite/adapter/arrow/ArrowRel.java | 15 +- .../calcite/adapter/arrow/ArrowRules.java | 8 +- .../calcite/adapter/arrow/ArrowTable.java | 58 ++++--- .../arrow/ArrowToEnumerableConverter.java | 20 ++- .../adapter/arrow/ArrowTranslator.java | 88 +++++----- .../calcite/adapter/arrow/ConditionToken.java | 97 +++++++++++ .../adapter/arrow/ArrowAdapterTest.java | 160 ++++++++++++++---- .../java/org/apache/calcite/util/Bug.java | 10 -- 9 files changed, 342 insertions(+), 116 deletions(-) create mode 100644 arrow/src/main/java/org/apache/calcite/adapter/arrow/ConditionToken.java diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowFilter.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowFilter.java index 9774318ea925..9617ac6b3cb0 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowFilter.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowFilter.java @@ -34,7 +34,7 @@ * relational expression in Arrow. */ class ArrowFilter extends Filter implements ArrowRel { - private final List match; + private final List> match; ArrowFilter(RelOptCluster cluster, RelTraitSet traitSet, RelNode input, RexNode condition) { super(cluster, traitSet, input, condition); diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowRel.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowRel.java index 5b002bdc2dcd..944c17d867dd 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowRel.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowRel.java @@ -41,15 +41,24 @@ public interface ArrowRel extends RelNode { * {@link ArrowRel} nodes into a SQL query. */ class Implementor { @Nullable List selectFields; - final List whereClause = new ArrayList<>(); + final List> whereClause = new ArrayList<>(); @Nullable RelOptTable table; @Nullable ArrowTable arrowTable; /** Adds new predicates. * - * @param predicates Predicates + *

    The structure is two levels of nesting: + *

      + *
    • Outer list: conjunction (AND) of clauses + *
    • Inner list: disjunction (OR) of conditions within a clause + *
    + * + *

    Each {@link ConditionToken} represents a single unary or binary + * predicate condition. + * + * @param predicates Predicates in CNF form */ - void addFilters(List predicates) { + void addFilters(List> predicates) { whereClause.addAll(predicates); } diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowRules.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowRules.java index b70e70964837..6e268d646928 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowRules.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowRules.java @@ -29,6 +29,8 @@ import org.apache.calcite.rel.logical.LogicalFilter; import org.apache.calcite.rel.logical.LogicalProject; import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexUtil; import org.apache.calcite.sql.validate.SqlValidatorUtil; import com.google.common.collect.ImmutableList; @@ -97,9 +99,13 @@ protected ArrowFilterRule(Config config) { RelNode convert(Filter filter) { final RelTraitSet traitSet = filter.getTraitSet().replace(ArrowRel.CONVENTION); + // Expand SEARCH (e.g. IN, BETWEEN) before pushing to Arrow, + // since Gandiva does not support SEARCH natively. + final RexNode condition = + RexUtil.expandSearch(filter.getCluster().getRexBuilder(), null, filter.getCondition()); return new ArrowFilter(filter.getCluster(), traitSet, convert(filter.getInput(), ArrowRel.CONVENTION), - filter.getCondition()); + condition); } /** Rule configuration. */ diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTable.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTable.java index ba459c7b48b8..358a08fb2500 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTable.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTable.java @@ -97,7 +97,7 @@ public class ArrowTable extends AbstractTable * {@link org.apache.calcite.adapter.arrow.ArrowMethod#ARROW_QUERY}. */ @SuppressWarnings("unused") public Enumerable query(DataContext root, ImmutableIntList fields, - List conditions) { + List>> conditions) { requireNonNull(fields, "fields"); final Projector projector; final Filter filter; @@ -119,30 +119,26 @@ public Enumerable query(DataContext root, ImmutableIntList fields, } else { projector = null; - final List conditionNodes = new ArrayList<>(conditions.size()); - for (String condition : conditions) { - String[] data = condition.split(" "); - List treeNodes = new ArrayList<>(2); - treeNodes.add( - TreeBuilder.makeField(schema.getFields() - .get(schema.getFields().indexOf(schema.findField(data[0]))))); - - // if the split condition has more than two parts it's a binary operator - // with an additional literal node - if (data.length > 2) { - treeNodes.add(makeLiteralNode(data[2], data[3])); + final List conjuncts = new ArrayList<>(conditions.size()); + for (List> orGroup : conditions) { + final List disjuncts = new ArrayList<>(orGroup.size()); + for (List conditionParts : orGroup) { + disjuncts.add( + convertConditionToGandiva( + ConditionToken.fromTokenList(conditionParts))); + } + if (disjuncts.size() == 1) { + conjuncts.add(disjuncts.get(0)); + } else { + conjuncts.add(TreeBuilder.makeOr(disjuncts)); } - - String operator = data[1]; - conditionNodes.add( - TreeBuilder.makeFunction(operator, treeNodes, new ArrowType.Bool())); } final Condition filterCondition; - if (conditionNodes.size() == 1) { - filterCondition = TreeBuilder.makeCondition(conditionNodes.get(0)); + if (conjuncts.size() == 1) { + filterCondition = TreeBuilder.makeCondition(conjuncts.get(0)); } else { - TreeNode treeNode = TreeBuilder.makeAnd(conditionNodes); - filterCondition = TreeBuilder.makeCondition(treeNode); + filterCondition = + TreeBuilder.makeCondition(TreeBuilder.makeAnd(conjuncts)); } try { @@ -184,6 +180,26 @@ private static RelDataType deduceRowType(Schema schema, return builder.build(); } + /** Converts a single {@link ConditionToken} into a Gandiva {@link TreeNode}. */ + private TreeNode convertConditionToGandiva(ConditionToken token) { + final List treeNodes = new ArrayList<>(2); + treeNodes.add( + TreeBuilder.makeField(schema.getFields() + .get( + schema.getFields().indexOf( + schema.findField(token.fieldName))))); + + if (token.isBinary()) { + treeNodes.add( + makeLiteralNode( + requireNonNull(token.value, "value"), + requireNonNull(token.valueType, "valueType"))); + } + + return TreeBuilder.makeFunction( + token.operator, treeNodes, new ArrowType.Bool()); + } + private static TreeNode makeLiteralNode(String literal, String type) { if (type.startsWith("decimal")) { String[] typeParts = diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowToEnumerableConverter.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowToEnumerableConverter.java index 3b90dfd890e3..bd0e2c2e8c10 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowToEnumerableConverter.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowToEnumerableConverter.java @@ -35,6 +35,7 @@ import com.google.common.primitives.Ints; +import java.util.ArrayList; import java.util.List; import static java.util.Objects.requireNonNull; @@ -84,6 +85,23 @@ protected ArrowToEnumerableConverter(RelOptCluster cluster, : Expressions.call( BuiltInMethod.IMMUTABLE_INT_LIST_IDENTITY.method, Expressions.constant(fieldCount)), - Expressions.constant(arrowImplementor.whereClause)))); + Expressions.constant( + toTokenLists(arrowImplementor.whereClause))))); + } + + /** Converts structured {@link ConditionToken} conditions to nested string + * lists for serialization through {@link Expressions#constant}. */ + private static List>> toTokenLists( + List> conditions) { + final List>> result = + new ArrayList<>(conditions.size()); + for (List orGroup : conditions) { + final List> group = new ArrayList<>(orGroup.size()); + for (ConditionToken token : orGroup) { + group.add(token.toTokenList()); + } + result.add(group); + } + return result; } } diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTranslator.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTranslator.java index 1102ce205692..cb27096a0984 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTranslator.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTranslator.java @@ -41,7 +41,7 @@ import static java.util.Objects.requireNonNull; /** - * Translates a {@link RexNode} expression to a Gandiva string. + * Translates a {@link RexNode} expression to Gandiva predicate tokens. */ class ArrowTranslator { final RexBuilder rexBuilder; @@ -61,13 +61,30 @@ public static ArrowTranslator create(RexBuilder rexBuilder, return new ArrowTranslator(rexBuilder, rowType); } - List translateMatch(RexNode condition) { - List disjunctions = RelOptUtil.disjunctions(condition); - if (disjunctions.size() == 1) { - return translateAnd(disjunctions.get(0)); - } else { - throw new UnsupportedOperationException("Unsupported disjunctive condition " + condition); + /** The maximum number of nodes allowed during CNF conversion. + * + *

    If exceeded, {@link RexUtil#toCnf(RexBuilder, int, RexNode)} returns + * the original expression unchanged, which may cause the subsequent + * translation to Gandiva predicates to fail with an + * {@link UnsupportedOperationException}. When invoked by the Arrow adapter + * module, the exception is caught and the plan falls back to + * an Enumerable convention. */ + private static final int MAX_CNF_NODE_COUNT = 256; + + List> translateMatch(RexNode condition) { + // Convert to CNF; SEARCH nodes are already expanded + // by ArrowFilterRule before reaching here. + final RexNode cnf = RexUtil.toCnf(rexBuilder, MAX_CNF_NODE_COUNT, condition); + + final List> result = new ArrayList<>(); + for (RexNode conjunct : RelOptUtil.conjunctions(cnf)) { + final List orGroup = new ArrayList<>(); + for (RexNode disjunct : RelOptUtil.disjunctions(conjunct)) { + orGroup.add(translateMatch2(disjunct)); + } + result.add(orGroup); } + return result; } /** @@ -93,34 +110,14 @@ private static Object literalValue(RexLiteral literal) { } } - /** - * Translate a conjunctive predicate to a SQL string. - * - * @param condition A conjunctive predicate - * - * @return SQL string for the predicate - */ - private List translateAnd(RexNode condition) { - List predicates = new ArrayList<>(); - for (RexNode node : RelOptUtil.conjunctions(condition)) { - if (node.getKind() == SqlKind.SEARCH) { - final RexNode node2 = RexUtil.expandSearch(rexBuilder, null, node); - predicates.addAll(translateMatch(node2)); - } else { - predicates.add(translateMatch2(node)); - } - } - return predicates; - } - /** * Translates a binary or unary relation. * * @param node A RexNode that always evaluates to a boolean expression. * Currently, this method is only called from translateAnd. - * @return The translated SQL string for the relation. + * @return The translated condition token for the relation. */ - private String translateMatch2(RexNode node) { + private ConditionToken translateMatch2(RexNode node) { switch (node.getKind()) { case EQUALS: return translateBinary("equal", "=", (RexCall) node); @@ -144,7 +141,7 @@ private String translateMatch2(RexNode node) { return translateUnary("isnotfalse", (RexCall) node); case INPUT_REF: final RexInputRef inputRef = (RexInputRef) node; - return fieldNames.get(inputRef.getIndex()) + " istrue"; + return ConditionToken.unary(fieldNames.get(inputRef.getIndex()), "istrue"); case NOT: return translateUnary("isfalse", (RexCall) node); default: @@ -156,10 +153,10 @@ private String translateMatch2(RexNode node) { * Translates a call to a binary operator, reversing arguments if * necessary. */ - private String translateBinary(String op, String rop, RexCall call) { + private ConditionToken translateBinary(String op, String rop, RexCall call) { final RexNode left = call.operands.get(0); final RexNode right = call.operands.get(1); - @Nullable String expression = translateBinary2(op, left, right); + @Nullable ConditionToken expression = translateBinary2(op, left, right); if (expression != null) { return expression; } @@ -171,7 +168,8 @@ private String translateBinary(String op, String rop, RexCall call) { } /** Translates a call to a binary operator. Returns null on failure. */ - private @Nullable String translateBinary2(String op, RexNode left, RexNode right) { + private @Nullable ConditionToken translateBinary2(String op, RexNode left, + RexNode right) { if (right.getKind() != SqlKind.LITERAL) { return null; } @@ -189,26 +187,29 @@ private String translateBinary(String op, String rop, RexCall call) { } } - /** Combines a field name, operator, and literal to produce a predicate string. */ - private String translateOp2(String op, String name, RexLiteral right) { + /** Combines a field name, operator, and literal to produce a binary + * condition token. */ + private ConditionToken translateOp2(String op, String name, + RexLiteral right) { Object value = literalValue(right); String valueString = value.toString(); String valueType = getLiteralType(right.getType()); if (value instanceof String) { - final RelDataTypeField field = requireNonNull(rowType.getField(name, true, false), "field"); + final RelDataTypeField field = + requireNonNull(rowType.getField(name, true, false), "field"); SqlTypeName typeName = field.getType().getSqlTypeName(); if (typeName != SqlTypeName.CHAR) { valueString = "'" + valueString + "'"; } } - return name + " " + op + " " + valueString + " " + valueType; + return ConditionToken.binary(name, op, valueString, valueType); } /** Translates a call to a unary operator. */ - private String translateUnary(String op, RexCall call) { + private ConditionToken translateUnary(String op, RexCall call) { final RexNode opNode = call.operands.get(0); - @Nullable String expression = translateUnary2(op, opNode); + @Nullable ConditionToken expression = translateUnary2(op, opNode); if (expression != null) { return expression; @@ -218,21 +219,16 @@ private String translateUnary(String op, RexCall call) { } /** Translates a call to a unary operator. Returns null on failure. */ - private @Nullable String translateUnary2(String op, RexNode opNode) { + private @Nullable ConditionToken translateUnary2(String op, RexNode opNode) { if (opNode.getKind() == SqlKind.INPUT_REF) { final RexInputRef inputRef = (RexInputRef) opNode; final String name = fieldNames.get(inputRef.getIndex()); - return translateUnaryOp(op, name); + return ConditionToken.unary(name, op); } return null; } - /** Combines a field name and a unary operator to produce a predicate string. */ - private static String translateUnaryOp(String op, String name) { - return name + " " + op; - } - private static String getLiteralType(RelDataType type) { if (type.getSqlTypeName() == SqlTypeName.DECIMAL) { return "decimal" + "(" + type.getPrecision() + "," + type.getScale() + ")"; diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ConditionToken.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ConditionToken.java new file mode 100644 index 000000000000..44d3facea77f --- /dev/null +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ConditionToken.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.adapter.arrow; + +import com.google.common.collect.ImmutableList; + +import org.checkerframework.checker.nullness.qual.Nullable; + +import java.util.List; + +import static java.util.Objects.requireNonNull; + +/** + * A structured representation of a single Gandiva predicate condition. + * + *

    A condition is either unary (e.g. {@code IS NULL}) or binary + * (e.g. {@code =}, {@code <}). Unary conditions have a field name + * and operator; binary conditions additionally have a literal value + * and its type. + * + * @see ArrowTranslator + */ +class ConditionToken { + final String fieldName; + final String operator; + final @Nullable String value; + final @Nullable String valueType; + + private ConditionToken(String fieldName, String operator, + @Nullable String value, @Nullable String valueType) { + this.fieldName = requireNonNull(fieldName, "fieldName"); + this.operator = requireNonNull(operator, "operator"); + this.value = value; + this.valueType = valueType; + } + + /** Creates a binary condition token + * (e.g. {@code intField equal 12 integer}). */ + static ConditionToken binary(String fieldName, String operator, + String value, String valueType) { + return new ConditionToken(fieldName, operator, + requireNonNull(value, "value"), + requireNonNull(valueType, "valueType")); + } + + /** Creates a unary condition token + * (e.g. {@code intField isnull}). */ + static ConditionToken unary(String fieldName, String operator) { + return new ConditionToken(fieldName, operator, null, null); + } + + /** Returns whether this is a binary condition. */ + boolean isBinary() { + return value != null; + } + + /** Converts this token to a string list for serialization + * through code generation. + * + *

    The result is either {@code [fieldName, operator]} for unary + * conditions or {@code [fieldName, operator, value, valueType]} for + * binary conditions. */ + List toTokenList() { + if (isBinary()) { + return ImmutableList.of(fieldName, operator, + requireNonNull(value, "value"), + requireNonNull(valueType, "valueType")); + } + return ImmutableList.of(fieldName, operator); + } + + /** Creates a {@code ConditionToken} from a serialized string list. */ + static ConditionToken fromTokenList(List tokens) { + final int size = tokens.size(); + if (size == 4) { + return binary(tokens.get(0), tokens.get(1), + tokens.get(2), tokens.get(3)); + } else if (size == 2) { + return unary(tokens.get(0), tokens.get(1)); + } + throw new IllegalArgumentException("Invalid condition tokens: " + tokens); + } +} diff --git a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java index 14f387509c9e..67b3075b4e1d 100644 --- a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java +++ b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java @@ -22,7 +22,6 @@ import org.apache.calcite.rel.type.RelDataTypeSystem; import org.apache.calcite.schema.Table; import org.apache.calcite.test.CalciteAssert; -import org.apache.calcite.util.Bug; import org.apache.calcite.util.Sources; import com.google.common.collect.ImmutableMap; @@ -218,7 +217,7 @@ static void initializeArrowState(@TempDir Path sharedTempDir) + "where \"intField\" > 1 and \"intField\" < 4"; String plan = "PLAN=ArrowToEnumerableConverter\n" + " ArrowProject(intField=[$0], stringField=[$1])\n" - + " ArrowFilter(condition=[SEARCH($0, Sarg[(1..4)])])\n" + + " ArrowFilter(condition=[AND(>($0, 1), <($0, 4))])\n" + " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n"; String result = "intField=2; stringField=2\n" + "intField=3; stringField=3\n"; @@ -251,20 +250,10 @@ static void initializeArrowState(@TempDir Path sharedTempDir) String sql = "select \"intField\", \"stringField\"\n" + "from arrowdata\n" + "where \"intField\"=12 or \"stringField\"='12'"; - String plan; - if (Bug.CALCITE_6293_FIXED) { - plan = "PLAN=ArrowToEnumerableConverter\n" - + " ArrowProject(intField=[$0], stringField=[$1])\n" - + " ArrowFilter(condition=[OR(=($0, 12), =($1, '12'))])\n" - + " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n"; - } else { - plan = "PLAN=EnumerableCalc(expr#0..1=[{inputs}], expr#2=[12], " - + "expr#3=[=($t0, $t2)], expr#4=['12':VARCHAR], expr#5=[=($t1, $t4)], " - + "expr#6=[OR($t3, $t5)], proj#0..1=[{exprs}], $condition=[$t6])\n" - + " ArrowToEnumerableConverter\n" - + " ArrowProject(intField=[$0], stringField=[$1])\n" - + " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n"; - } + String plan = "PLAN=ArrowToEnumerableConverter\n" + + " ArrowProject(intField=[$0], stringField=[$1])\n" + + " ArrowFilter(condition=[OR(=($0, 12), =($1, '12'))])\n" + + " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n"; String result = "intField=12; stringField=12\n"; CalciteAssert.that() @@ -274,23 +263,84 @@ static void initializeArrowState(@TempDir Path sharedTempDir) .explainContains(plan); } + /** Test case for + * [CALCITE-6636] + * Support CNF condition of Arrow adapter. */ + @Test void testArrowProjectFieldsWithCnfFilter() { + String sql = "select \"intField\", \"stringField\"\n" + + "from arrowdata\n" + + "where (\"intField\" > 1 and \"stringField\" = '2') or \"intField\" = 0"; + String plan = "PLAN=ArrowToEnumerableConverter\n" + + " ArrowProject(intField=[$0], stringField=[$1])\n" + + " ArrowFilter(condition=[OR(AND(>($0, 1), =($1, '2')), =($0, 0))])\n" + + " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n"; + String result = "intField=0; stringField=0\n" + + "intField=2; stringField=2\n"; + + CalciteAssert.that() + .with(arrow) + .query(sql) + .returns(result) + .explainContains(plan); + } + + /** Test case for + * [CALCITE-6636] + * Support CNF condition of Arrow adapter. + * + *

    Tests deeply nested conditions: {@code (A AND B) OR (C AND D)}, + * which in CNF becomes {@code (A OR C) AND (A OR D) AND (B OR C) AND (B OR D)}. */ + @Test void testArrowProjectFieldsWithDeepCnfFilter() { + String sql = "select \"intField\", \"stringField\"\n" + + "from arrowdata\n" + + "where (\"intField\" = 2 and \"stringField\" = '2')" + + " or (\"intField\" = 3 and \"stringField\" = '3')"; + String plan = "PLAN=ArrowToEnumerableConverter\n" + + " ArrowProject(intField=[$0], stringField=[$1])\n" + + " ArrowFilter(condition=[OR(AND(=($0, 2), =($1, '2')), AND(=($0, 3), =($1, '3')))])\n" + + " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n"; + String result = "intField=2; stringField=2\n" + + "intField=3; stringField=3\n"; + + CalciteAssert.that() + .with(arrow) + .query(sql) + .returns(result) + .explainContains(plan); + } + + /** Test case for + * [CALCITE-6636] + * Support CNF condition of Arrow adapter. + * + *

    Tests triple OR: {@code A OR B OR C}. */ + @Test void testArrowProjectFieldsWithTripleOrFilter() { + String sql = "select \"intField\", \"stringField\"\n" + + "from arrowdata\n" + + "where \"intField\" = 1 or \"intField\" = 2 or \"intField\" = 3"; + String plan = "PLAN=ArrowToEnumerableConverter\n" + + " ArrowProject(intField=[$0], stringField=[$1])\n" + + " ArrowFilter(condition=[OR(=($0, 1), =($0, 2), =($0, 3))])\n" + + " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n"; + String result = "intField=1; stringField=1\n" + + "intField=2; stringField=2\n" + + "intField=3; stringField=3\n"; + + CalciteAssert.that() + .with(arrow) + .query(sql) + .returns(result) + .explainContains(plan); + } + @Test void testArrowProjectFieldsWithInFilter() { String sql = "select \"intField\", \"stringField\"\n" + "from arrowdata\n" + "where \"intField\" in (0, 1, 2)"; - String plan; - if (Bug.CALCITE_6294_FIXED) { - plan = "PLAN=ArrowToEnumerableConverter\n" - + " ArrowProject(intField=[$0], stringField=[$1])\n" - + " ArrowFilter(condition=[OR(=($0, 0), =($0, 1), =($0, 2))])\n" - + " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n"; - } else { - plan = "PLAN=EnumerableCalc(expr#0..1=[{inputs}], expr#2=[Sarg[0, 1, 2]], " - + "expr#3=[SEARCH($t0, $t2)], proj#0..1=[{exprs}], $condition=[$t3])\n" - + " ArrowToEnumerableConverter\n" - + " ArrowProject(intField=[$0], stringField=[$1])\n" - + " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n"; - } + String plan = "PLAN=ArrowToEnumerableConverter\n" + + " ArrowProject(intField=[$0], stringField=[$1])\n" + + " ArrowFilter(condition=[OR(=($0, 0), =($0, 1), =($0, 2))])\n" + + " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n"; String result = "intField=0; stringField=0\n" + "intField=1; stringField=1\n" + "intField=2; stringField=2\n"; @@ -387,7 +437,7 @@ static void initializeArrowState(@TempDir Path sharedTempDir) + "where \"intField\" between 1 and 3"; String plan = "PLAN=ArrowToEnumerableConverter\n" + " ArrowProject(intField=[$0], stringField=[$1])\n" - + " ArrowFilter(condition=[SEARCH($0, Sarg[[1..3]])])\n" + + " ArrowFilter(condition=[AND(>=($0, 1), <=($0, 3))])\n" + " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n"; String result = "intField=1; stringField=1\n" + "intField=2; stringField=2\n" @@ -530,14 +580,13 @@ static void initializeArrowState(@TempDir Path sharedTempDir) .explainContains(plan); } - @Disabled("literal with space is not supported") @Test void testLiteralWithSpace() { String sql = "select \"intField\", \"stringField\" as \"my Field\"\n" + "from arrowdata\n" + "where \"stringField\" = 'literal with space'"; String plan = "PLAN=ArrowToEnumerableConverter\n" - + " ArrowProject(intField=[$0], my Field=[$1])\n" - + " ArrowFilter(condition=[=($1, '2')])\n" + + " ArrowProject(intField=[$0], stringField=[$1])\n" + + " ArrowFilter(condition=[=($1, 'literal with space')])\n" + " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n"; String result = ""; @@ -565,6 +614,23 @@ static void initializeArrowState(@TempDir Path sharedTempDir) .explainContains(plan); } + @Test void testLiteralWithEmptyString() { + String sql = "select \"intField\", \"stringField\"\n" + + "from arrowdata\n" + + "where \"stringField\" = ''"; + String plan = "PLAN=ArrowToEnumerableConverter\n" + + " ArrowProject(intField=[$0], stringField=[$1])\n" + + " ArrowFilter(condition=[=($1, '')])\n" + + " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n"; + String result = ""; + + CalciteAssert.that() + .with(arrow) + .query(sql) + .returns(result) + .explainContains(plan); + } + @Test void testTinyIntProject() { String sql = "select DEPTNO from DEPT"; String plan = "PLAN=ArrowToEnumerableConverter\n" @@ -962,6 +1028,34 @@ static void initializeArrowState(@TempDir Path sharedTempDir) .explainContains(plan); } + /** When a filter condition exceeds the CNF node limit, the Arrow adapter + * falls back to the Enumerable convention (EnumerableCalc) instead of + * using ArrowFilter. The query should still return correct results. */ + @Test void testCnfExceedsLimitFallsBackToEnumerable() { + StringBuilder sb = new StringBuilder(); + sb.append("select \"intField\", \"stringField\" from arrowdata\nwhere "); + for (int i = 0; i < 45; i++) { + if (i > 0) { + sb.append(" or "); + } + sb.append("(\"intField\" = ").append(i) + .append(" and \"stringField\" = '").append(i).append("')"); + } + String sql = sb.toString(); + + String planPrefix = "PLAN=EnumerableCalc("; + String arrowInputPlan = "ArrowToEnumerableConverter" + + "\n ArrowProject(intField=[$0], stringField=[$1])" + + "\n ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])"; + + CalciteAssert.that() + .with(arrow) + .query(sql) + .returnsCount(45) + .explainContains(planPrefix) + .explainContains(arrowInputPlan); + } + /** Test case for * [CALCITE-6684] * Arrow adapter should supports filter conditions of Decimal type. */ diff --git a/core/src/main/java/org/apache/calcite/util/Bug.java b/core/src/main/java/org/apache/calcite/util/Bug.java index 7eb756d5d24a..8aaebeb4af06 100644 --- a/core/src/main/java/org/apache/calcite/util/Bug.java +++ b/core/src/main/java/org/apache/calcite/util/Bug.java @@ -204,16 +204,6 @@ public abstract class Bug { * is fixed. */ public static final boolean CALCITE_6391_FIXED = false; - /** Whether - * - * [CALCITE-6293] Support OR condition in Arrow adapter is fixed. */ - public static final boolean CALCITE_6293_FIXED = false; - - /** Whether - * - * [CALCITE-6294] Support IN filter in Arrow adapter is fixed. */ - public static final boolean CALCITE_6294_FIXED = false; - /** Whether * [CALCITE-6328] * The BigQuery functions SAFE_* do not match the BigQuery specification From 10a25350a334beeeba63fe4b35c8b35990045f52 Mon Sep 17 00:00:00 2001 From: Soumyakanti Das Date: Thu, 2 Apr 2026 08:41:12 -0700 Subject: [PATCH 201/562] [CALCITE-7458] Upgrade Jackson to 2.18.6 due to CVE --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 3005b96424c5..2cc99e709cac 100644 --- a/gradle.properties +++ b/gradle.properties @@ -120,7 +120,7 @@ hutool-all.version=5.8.23 hydromatic.tpcds.version=0.4 immutables.version=2.8.8 innodb-java-reader.version=1.0.10 -jackson.version=2.18.4.1 +jackson.version=2.18.6 janino.version=3.1.12 java-diff.version=1.1.2 jcip-annotations.version=1.0-1 From a033b534f374d29be31338fed6f98947ea7c4133 Mon Sep 17 00:00:00 2001 From: "cc.cai" <2356672992@qq.com> Date: Thu, 2 Apr 2026 16:22:25 +0800 Subject: [PATCH 202/562] [CALCITE-7456] Enable the TRY_CAST function to support the MSSQL dialect Co-Authored-By: junjie --- .../calcite/sql/dialect/MssqlSqlDialect.java | 8 +++++++ .../rel/rel2sql/RelToSqlConverterTest.java | 21 +++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/core/src/main/java/org/apache/calcite/sql/dialect/MssqlSqlDialect.java b/core/src/main/java/org/apache/calcite/sql/dialect/MssqlSqlDialect.java index 2387ed54d9d4..a88f78b48f8e 100644 --- a/core/src/main/java/org/apache/calcite/sql/dialect/MssqlSqlDialect.java +++ b/core/src/main/java/org/apache/calcite/sql/dialect/MssqlSqlDialect.java @@ -37,6 +37,7 @@ import org.apache.calcite.sql.SqlSyntax; import org.apache.calcite.sql.SqlUtil; import org.apache.calcite.sql.SqlWriter; +import org.apache.calcite.sql.fun.SqlLibraryOperators; import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.sql.type.OperandTypes; @@ -185,6 +186,13 @@ public MssqlSqlDialect(Context context) { SqlOperator op = SqlStdOperatorTable.PERCENT_REMAINDER; SqlSyntax.BINARY.unparse(writer, op, call, leftPrec, rightPrec); break; + case SAFE_CAST: + // MSSQL uses TRY_CAST instead of SAFE_CAST (BigQuery) + super.unparseCall(writer, + SqlLibraryOperators.TRY_CAST.createCall( + call.getParserPosition(), call.getOperandList()), + leftPrec, rightPrec); + break; default: super.unparseCall(writer, call, leftPrec, rightPrec); } diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java index 598fb7500ec0..2f881236d1ce 100644 --- a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java +++ b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java @@ -3135,6 +3135,27 @@ private SqlDialect nonOrdinalDialect() { sql(query).withLibrary(SqlLibrary.BIG_QUERY).ok(expected); } + /** Test case for + * [CALCITE-7456] + * Enable the TRY_CAST function to support the MSSQL dialect. */ + @Test void testMssqlTryCast() { + final String query = "select try_cast(\"product_name\" as date) " + + "from \"foodmart\".\"product\""; + final String expected = "SELECT TRY_CAST([product_name] AS DATE)\n" + + "FROM [foodmart].[product]"; + + sql(query).withLibrary(SqlLibrary.MSSQL).withMssql().ok(expected); + } + + @Test void testSafeCastToMssqlTryCast() { + final String query = "select safe_cast(\"product_name\" as date) " + + "from \"foodmart\".\"product\""; + final String expected = "SELECT TRY_CAST([product_name] AS DATE)\n" + + "FROM [foodmart].[product]"; + + sql(query).withLibrary(SqlLibrary.BIG_QUERY).withMssql().ok(expected); + } + /** Test case for * [CALCITE-6150] * JDBC adapter for ClickHouse generates incorrect SQL for certain units in From 5cf836a699708c2bf59685bc571a907675689e90 Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Sat, 4 Apr 2026 00:16:32 +0800 Subject: [PATCH 203/562] Add test cases for SetOpToFilterRule to verify that PROJECT containing non-deterministic expressions and subqueries are not merged --- .../apache/calcite/test/RelOptRulesTest.java | 26 ++++++++++ .../apache/calcite/test/RelOptRulesTest.xml | 52 +++++++++++++++++++ 2 files changed, 78 insertions(+) diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index 1009d3688e16..bd983836d216 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -11395,6 +11395,32 @@ private void checkLoptOptimizeJoinRule(LoptOptimizeJoinRule rule) { .check(); } + // If the PROJECT clause contains non-deterministic expressions, + // they will not be merged. + @Test void testUnionToFilterRuleWithNonDeterministicProject() { + final String sql = "SELECT mgr, comm, rand() FROM emp WHERE mgr = 12\n" + + "UNION\n" + + "SELECT mgr, comm, rand() FROM emp WHERE comm = 5\n"; + sql(sql) + .withPreRule(CoreRules.PROJECT_FILTER_TRANSPOSE) + .withRule(CoreRules.UNION_FILTER_TO_FILTER) + .checkUnchanged(); + } + + // If the projection contains a subquery, merging will not be performed. + @Test void testUnionToFilterRuleWithSubqueryProject() { + final String sql = "SELECT 1, (SELECT COUNT(*) FROM dept)\n" + + "FROM emp WHERE mgr = 12\n" + + "UNION\n" + + "SELECT 1, (SELECT COUNT(*) FROM dept)\n" + + "FROM emp WHERE comm = 5\n"; + + sql(sql) + .withPreRule(CoreRules.PROJECT_FILTER_TRANSPOSE) + .withRule(CoreRules.UNION_FILTER_TO_FILTER) + .checkUnchanged(); + } + /** Test case of * [CALCITE-7002] * Create an optimization rule to eliminate UNION diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index f04ced869df0..ece2ad5258aa 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -21495,6 +21495,27 @@ LogicalUnion(all=[false]) LogicalFilter(condition=[SEARCH($0, Sarg[5, 10])]) LogicalProject(DEPTNO=[$0]) LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) +]]> + + + + + + + + @@ -21520,6 +21541,37 @@ LogicalUnion(all=[false]) LogicalAggregate(group=[{0, 1}]) LogicalProject(MGR=[$3], COMM=[$6]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +]]> + + + + + + + + From 906e2e9e91723e267043c13a89878d67e658da10 Mon Sep 17 00:00:00 2001 From: "cc.cai" <2356672992@qq.com> Date: Fri, 3 Apr 2026 17:10:51 +0800 Subject: [PATCH 204/562] [CALCITE-7461] Add @Strict to ByteArrayFunction and ByteArrayLengthFunction --- testkit/src/main/java/org/apache/calcite/util/Smalls.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testkit/src/main/java/org/apache/calcite/util/Smalls.java b/testkit/src/main/java/org/apache/calcite/util/Smalls.java index a6c52e466f6c..6f33f00669da 100644 --- a/testkit/src/main/java/org/apache/calcite/util/Smalls.java +++ b/testkit/src/main/java/org/apache/calcite/util/Smalls.java @@ -31,6 +31,7 @@ import org.apache.calcite.linq4j.function.Deterministic; import org.apache.calcite.linq4j.function.Parameter; import org.apache.calcite.linq4j.function.SemiStrict; +import org.apache.calcite.linq4j.function.Strict; import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.linq4j.tree.Expressions; import org.apache.calcite.linq4j.tree.MethodCallExpression; @@ -1495,16 +1496,15 @@ public static ByteString eval(String s) { } /** User-defined function with return type byte[]. */ + @Strict public static class ByteArrayFunction { public static byte[] eval(String s) { - if (s == null) { - return null; - } return s.getBytes(StandardCharsets.UTF_8); } } /** User-defined function with parameter type byte[]. */ + @Strict public static class ByteArrayLengthFunction { public static int eval(byte[] bytes) { return bytes.length; From 45dbbbc6e2012dedd5c04608a6b25d8e2e481e88 Mon Sep 17 00:00:00 2001 From: hongyu guo Date: Mon, 30 Mar 2026 21:59:07 +0800 Subject: [PATCH 205/562] [CALCITE-6968] SqlUpdate#getOperandList omits sourceSelect operand --- .../org/apache/calcite/sql/SqlUpdate.java | 8 +- .../calcite/sql/SqlCallOperandsTest.java | 141 ++++++++++++++++++ 2 files changed, 145 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql/SqlUpdate.java b/core/src/main/java/org/apache/calcite/sql/SqlUpdate.java index 3d1ad09c5231..82985f2bd721 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlUpdate.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlUpdate.java @@ -47,8 +47,8 @@ public class SqlUpdate extends SqlCall { (SqlNodeList) operands[1], (SqlNodeList) operands[2], operands[3], - null, - (SqlIdentifier) operands[4]); + (SqlSelect) operands[4], + (SqlIdentifier) operands[5]); } }; @@ -93,7 +93,7 @@ public SqlUpdate(SqlParserPos pos, @SuppressWarnings("nullness") @Override public List<@Nullable SqlNode> getOperandList() { return ImmutableNullableList.of(targetTable, targetColumnList, - sourceExpressionList, condition, alias); + sourceExpressionList, condition, sourceSelect, alias); } @SuppressWarnings("assignment.type.incompatible") @@ -113,7 +113,7 @@ public SqlUpdate(SqlParserPos pos, condition = operand; break; case 4: - sourceExpressionList = requireNonNull((SqlNodeList) operand); + sourceSelect = (SqlSelect) operand; break; case 5: alias = (SqlIdentifier) operand; diff --git a/core/src/test/java/org/apache/calcite/sql/SqlCallOperandsTest.java b/core/src/test/java/org/apache/calcite/sql/SqlCallOperandsTest.java index f829b805ce17..d98716713466 100644 --- a/core/src/test/java/org/apache/calcite/sql/SqlCallOperandsTest.java +++ b/core/src/test/java/org/apache/calcite/sql/SqlCallOperandsTest.java @@ -17,6 +17,8 @@ package org.apache.calcite.sql; import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.parser.SqlParseException; +import org.apache.calcite.sql.parser.SqlParser; import org.apache.calcite.sql.parser.SqlParserPos; import org.junit.jupiter.api.Test; @@ -24,6 +26,7 @@ import java.util.List; import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; @@ -70,4 +73,142 @@ public class SqlCallOperandsTest { assertThat(sqlDelete.getSourceSelect(), equalTo(operandList.get(2))); assertThat(sqlDelete.getAlias(), equalTo(operandList.get(3))); } + + /** Test case for + * [CALCITE-6968] + * SqlUpdate#getOperandList return operands' missing 'sourceSelect'. */ + @Test void testSqlUpdateGetOperandsMatchWithSetOperand() { + SqlUpdate sqlUpdate = + new SqlUpdate(SqlParserPos.ZERO, new SqlIdentifier("table1", SqlParserPos.ZERO), + SqlNodeList.EMPTY, + SqlNodeList.EMPTY, + null, + null, + null); + SqlNode targetTable = new SqlIdentifier("table2", SqlParserPos.ZERO); + final SqlIdentifier field1 = new SqlIdentifier("field1", SqlParserPos.ZERO); + final SqlIdentifier field2 = new SqlIdentifier("field2", SqlParserPos.ZERO); + final SqlIdentifier field3 = new SqlIdentifier("field3", SqlParserPos.ZERO); + final SqlNodeList targetColumnList = SqlNodeList.of(field2); + final SqlNodeList sourceExpressionList = SqlNodeList.of(field3); + SqlNode condition = + SqlStdOperatorTable.EQUALS.createCall(SqlParserPos.ZERO, field1, + SqlLiteral.createCharString("field1Value", SqlParserPos.ZERO)); + SqlSelect sourceSelect = + new SqlSelect(SqlParserPos.ZERO, null, + SqlNodeList.of(field1), + null, + null, + null, + null, + null, + null, + null, + null, + null, + null); + SqlIdentifier alias = new SqlIdentifier("alias", SqlParserPos.ZERO); + sqlUpdate.setOperand(0, targetTable); + sqlUpdate.setOperand(1, targetColumnList); + sqlUpdate.setOperand(2, sourceExpressionList); + sqlUpdate.setOperand(3, condition); + sqlUpdate.setOperand(4, sourceSelect); + sqlUpdate.setOperand(5, alias); + final List operandList = sqlUpdate.getOperandList(); + assertThat(operandList, hasSize(6)); + assertThat(sqlUpdate.getTargetTable(), equalTo(operandList.get(0))); + assertThat(sqlUpdate.getTargetColumnList(), equalTo(operandList.get(1))); + assertThat(sqlUpdate.getSourceExpressionList(), equalTo(operandList.get(2))); + assertThat(sqlUpdate.getCondition(), equalTo(operandList.get(3))); + assertThat(sqlUpdate.getSourceSelect(), equalTo(operandList.get(4))); + assertThat(sqlUpdate.getAlias(), equalTo(operandList.get(5))); + } + + /** Test case for + * [CALCITE-6968] + * SqlUpdate#getOperandList return operands' missing 'sourceSelect'. */ + @Test void testSqlUpdateClonePreservesSourceSelect() { + final SqlIdentifier field1 = new SqlIdentifier("field1", SqlParserPos.ZERO); + final SqlSelect sourceSelect = + new SqlSelect(SqlParserPos.ZERO, null, + SqlNodeList.of(field1), + null, + null, + null, + null, + null, + null, + null, + null, + null, + null); + final SqlIdentifier alias = new SqlIdentifier("alias", SqlParserPos.ZERO); + final SqlUpdate sqlUpdate = + new SqlUpdate(SqlParserPos.ZERO, new SqlIdentifier("table1", SqlParserPos.ZERO), + SqlNodeList.of(field1), + SqlNodeList.of(SqlLiteral.createCharString("field1Value", SqlParserPos.ZERO)), + null, + sourceSelect, + alias); + final SqlUpdate cloned = (SqlUpdate) sqlUpdate.clone(SqlParserPos.ZERO); + assertThat(cloned.getOperandList(), hasSize(6)); + assertThat(cloned.getSourceSelect(), equalTo(sourceSelect)); + assertThat(cloned.getAlias(), equalTo(alias)); + } + + /** Test case for + * [CALCITE-6968] + * SqlUpdate#getOperandList return operands' missing 'sourceSelect'. */ + @Test void testSqlUpdateUnparseIgnoresSourceSelect() { + final SqlIdentifier targetColumn = new SqlIdentifier("field1", SqlParserPos.ZERO); + final SqlSelect sourceSelect = + new SqlSelect(SqlParserPos.ZERO, null, + SqlNodeList.of(new SqlIdentifier("internalField", SqlParserPos.ZERO)), + null, + null, + null, + null, + null, + null, + null, + null, + null, + null); + final SqlUpdate sqlUpdate = + new SqlUpdate(SqlParserPos.ZERO, new SqlIdentifier("table1", SqlParserPos.ZERO), + SqlNodeList.of(targetColumn), + SqlNodeList.of(SqlLiteral.createCharString("field1Value", SqlParserPos.ZERO)), + null, + sourceSelect, + new SqlIdentifier("alias", SqlParserPos.ZERO)); + final String sql = + sqlUpdate.toSqlString(c -> c.withClauseStartsLine(false)).getSql(); + assertThat(sql, containsString("UPDATE")); + assertThat(sql, containsString("field1Value")); + assertThat(sql.contains("internalField"), equalTo(false)); + assertThat(sql.contains("SELECT"), equalTo(false)); + } + + /** Test case for + * [CALCITE-6968] + * SqlUpdate#getOperandList return operands' missing 'sourceSelect'. */ + @Test void testSqlUpdateUnparseIgnoresSourceSelectAfterParsingSql() + throws SqlParseException { + final SqlUpdate sqlUpdate = + (SqlUpdate) SqlParser.create("UPDATE table1 AS alias " + + "SET field1 = 'field1Value' " + + "WHERE field1 = 'field1Value'") + .parseStmt(); + final SqlSelect sourceSelect = + (SqlSelect) SqlParser.create("SELECT INTERNAL_MARKER FROM INTERNAL_SOURCE") + .parseQuery(); + sqlUpdate.setSourceSelect(sourceSelect); + final String sql = + sqlUpdate.toSqlString(c -> c.withClauseStartsLine(false)).getSql(); + assertThat(sql, containsString("UPDATE")); + assertThat(sql, containsString("field1Value")); + assertThat(sql.contains("INTERNAL_MARKER"), equalTo(false)); + assertThat(sql.contains("INTERNAL_SOURCE"), equalTo(false)); + assertThat(sql.contains("SELECT"), equalTo(false)); + } } From 697c5a2543e204f357413085c3125c2ad62413a2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 8 Apr 2026 09:24:24 +0000 Subject: [PATCH 206/562] Bump addressable from 2.8.7 to 2.9.0 in /site Bumps [addressable](https://github.com/sporkmonger/addressable) from 2.8.7 to 2.9.0. - [Changelog](https://github.com/sporkmonger/addressable/blob/main/CHANGELOG.md) - [Commits](https://github.com/sporkmonger/addressable/compare/addressable-2.8.7...addressable-2.9.0) --- updated-dependencies: - dependency-name: addressable dependency-version: 2.9.0 dependency-type: indirect ... Signed-off-by: dependabot[bot] --- site/Gemfile.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/site/Gemfile.lock b/site/Gemfile.lock index bc0e0b41efa5..a4cc1b4b44d5 100644 --- a/site/Gemfile.lock +++ b/site/Gemfile.lock @@ -1,8 +1,8 @@ GEM remote: https://rubygems.org/ specs: - addressable (2.8.7) - public_suffix (>= 2.0.2, < 7.0) + addressable (2.9.0) + public_suffix (>= 2.0.2, < 8.0) base64 (0.2.0) bigdecimal (3.1.9) colorator (1.1.0) @@ -92,7 +92,7 @@ GEM racc (~> 1.4) pathutil (0.16.2) forwardable-extended (~> 2.6) - public_suffix (6.0.1) + public_suffix (7.0.5) racc (1.8.1) rake (13.2.1) rb-fsevent (0.11.2) From d478564b2c7a9461733393bfb85ea29b01b01729 Mon Sep 17 00:00:00 2001 From: xuzifu666 <1206332514@qq.com> Date: Tue, 7 Apr 2026 15:18:25 +0800 Subject: [PATCH 207/562] [CALCITE-4645] In Elasticsearch adapter, a range predicate should be translated to a range query --- .../java/org/apache/calcite/util/Bug.java | 6 -- .../elasticsearch/PredicateAnalyzer.java | 81 ++++++++++++++++++- .../elasticsearch/AggregationAndSortTest.java | 29 ++++--- .../ElasticSearchAdapterTest.java | 3 - 4 files changed, 98 insertions(+), 21 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/util/Bug.java b/core/src/main/java/org/apache/calcite/util/Bug.java index 8aaebeb4af06..6d13a13d67a9 100644 --- a/core/src/main/java/org/apache/calcite/util/Bug.java +++ b/core/src/main/java/org/apache/calcite/util/Bug.java @@ -165,12 +165,6 @@ public abstract class Bug { * fixed. */ public static final boolean CALCITE_4213_FIXED = false; - /** Whether - * [CALCITE-4645] - * In Elasticsearch adapter, a range predicate should be translated to a range query is - * fixed. */ - public static final boolean CALCITE_4645_FIXED = false; - /** Whether * [CALCITE-4868] * Elasticsearch adapter fails if GROUP BY is followed by ORDER BY is diff --git a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/PredicateAnalyzer.java b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/PredicateAnalyzer.java index cfc8f20d82db..de51a90013e1 100644 --- a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/PredicateAnalyzer.java +++ b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/PredicateAnalyzer.java @@ -33,6 +33,7 @@ import org.apache.calcite.util.Sarg; import com.google.common.base.Throwables; +import com.google.common.collect.BoundType; import com.google.common.collect.Range; import java.util.ArrayList; @@ -215,7 +216,8 @@ private static boolean supportedRexCall(RexCall call) { * @return true if it isSearchWithPoints or isSearchWithComplementedPoints, other false */ static boolean canBeTranslatedToTermsQuery(RexCall search) { - return isSearchWithPoints(search) || isSearchWithComplementedPoints(search); + return isSearchWithPoints(search) || isSearchWithComplementedPoints(search) + || isSearchWithRange(search); } static boolean isSearchWithPoints(RexCall search) { @@ -230,6 +232,12 @@ static boolean isSearchWithComplementedPoints(RexCall search) { return sarg.isComplementedPoints(); } + static boolean isSearchWithRange(RexCall search) { + RexLiteral literal = (RexLiteral) search.getOperands().get(1); + final Sarg sarg = requireNonNull(literal.getValueAs(Sarg.class), "Sarg"); + return !sarg.isPoints() && !sarg.isComplementedPoints(); + } + @Override public Expression visitCall(RexCall call) { SqlSyntax syntax = call.getOperator().getSyntax(); @@ -406,8 +414,10 @@ private QueryExpression binary(RexCall call) { case SEARCH: if (isSearchWithComplementedPoints(call)) { return QueryExpression.create(pair.getKey()).notIn(pair.getValue()); - } else { + } else if (isSearchWithPoints(call)) { return QueryExpression.create(pair.getKey()).in(pair.getValue()); + } else { + return QueryExpression.create(pair.getKey()).range(pair.getValue()); } default: break; @@ -601,6 +611,8 @@ public boolean isPartial() { public abstract QueryExpression notIn(LiteralExpression literal); + public abstract QueryExpression range(LiteralExpression literal); + public abstract QueryExpression notEquals(LiteralExpression literal); public abstract QueryExpression gt(LiteralExpression literal); @@ -761,6 +773,10 @@ private CompoundQueryExpression(boolean partial, BoolQueryBuilder builder) { @Override public QueryExpression notIn(LiteralExpression literal) { throw new PredicateAnalyzerException("notIn cannot be applied to a compound expression"); } + + @Override public QueryExpression range(LiteralExpression literal) { + throw new PredicateAnalyzerException("range cannot be applied to a compound expression"); + } } /** @@ -899,6 +915,67 @@ private SimpleQueryExpression(NamedFieldExpression rel) { builder = boolQuery().mustNot(termsQuery(getFieldReference(), iterable)); return this; } + + @Override public QueryExpression range(LiteralExpression literal) { + final Sarg sarg = requireNonNull(literal.literal.getValueAs(Sarg.class), "Sarg"); + final Set> ranges = sarg.rangeSet.asRanges(); + + if (ranges.isEmpty()) { + throw new PredicateAnalyzerException("Range query expects at least one range"); + } + + if (ranges.size() == 1) { + // Single range, create a simple range query + final Range range = ranges.iterator().next(); + builder = createRangeQuery(range, literal); + } else { + // Multiple ranges, create bool query with should clauses (OR) + final BoolQueryBuilder boolQuery = boolQuery(); + for (Range range : ranges) { + boolQuery.should(createRangeQuery(range, literal)); + } + builder = boolQuery; + } + return this; + } + + private RangeQueryBuilder createRangeQuery(Range range, LiteralExpression literal) { + final RangeQueryBuilder rangeQuery = rangeQuery(getFieldReference()); + + // Handle lower bound + if (range.hasLowerBound()) { + final Object lowerValue = + literal.sargPointValue(range.lowerEndpoint(), + literal.literal.getType().getSqlTypeName()); + if (range.lowerBoundType() == BoundType.CLOSED) { + rangeQuery.gte(lowerValue); + } else { + rangeQuery.gt(lowerValue); + } + // Directly check if lower bound endpoint is GregorianCalendar + if (range.lowerEndpoint() instanceof GregorianCalendar) { + rangeQuery.format("date_time"); + } + } + + // Handle upper bound + if (range.hasUpperBound()) { + final Object upperValue = + literal.sargPointValue(range.upperEndpoint(), + literal.literal.getType().getSqlTypeName()); + if (range.upperBoundType() == BoundType.CLOSED) { + rangeQuery.lte(upperValue); + } else { + rangeQuery.lt(upperValue); + } + // Directly check if upper bound endpoint is GregorianCalendar + if (range.upperEndpoint() instanceof GregorianCalendar) { + rangeQuery.format("date_time"); + } + } + + return rangeQuery; + } } diff --git a/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/AggregationAndSortTest.java b/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/AggregationAndSortTest.java index 27658bdfee38..1d4dd0e1cdb6 100644 --- a/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/AggregationAndSortTest.java +++ b/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/AggregationAndSortTest.java @@ -115,21 +115,15 @@ private static Connection createConnection() throws SQLException { return connection; } - /** - * Currently the patterns like below will be converted to Search in range - * which is not supported in elastic search adapter. - * (val1 >= 10 and val1 <= 20) - * (val1 <= 10 or val1 >=20) - * (val1 <= 10) or (val1 > 15 and val1 <= 20) - * So disable this test case until the translation from Search in range - * to rang Query in ES is implemented. + /** Test case for + * [CALCITE-4645] + * In Elasticsearch adapter, a range predicate should be translated to a range query. */ @Test void searchInRange() { - Assumptions.assumeTrue(Bug.CALCITE_4645_FIXED, "CALCITE-4645"); CalciteAssert.that() .with(AggregationAndSortTest::createConnection) .query("select count(*) from view where val1 >= 10 and val1 <=20") - .returns("EXPR$0=1\n"); + .returns("EXPR$0=0\n"); CalciteAssert.that() .with(AggregationAndSortTest::createConnection) @@ -140,6 +134,21 @@ private static Connection createConnection() throws SQLException { .with(AggregationAndSortTest::createConnection) .query("select count(*) from view where val1 <= 10 or (val1 > 15 and val1 <= 20)") .returns("EXPR$0=2\n"); + + CalciteAssert.that() + .with(AggregationAndSortTest::createConnection) + .query("select count(*) from view where val1 = 1 or (val1 > 15 and val1 <= 20)") + .returns("EXPR$0=1\n"); + + CalciteAssert.that() + .with(AggregationAndSortTest::createConnection) + .query("select count(*) from view where cat1 <= 'e' and cat1 >= 'a'") + .returns("EXPR$0=2\n"); + + CalciteAssert.that() + .with(AggregationAndSortTest::createConnection) + .query("select count(*) from view where cat4 >= '2017-12-22' and cat4 <= '2018-02-01'") + .returns("EXPR$0=1\n"); } @Test void countStar() { diff --git a/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/ElasticSearchAdapterTest.java b/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/ElasticSearchAdapterTest.java index 0f31965fe594..b548d511540b 100644 --- a/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/ElasticSearchAdapterTest.java +++ b/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/ElasticSearchAdapterTest.java @@ -23,7 +23,6 @@ import org.apache.calcite.schema.impl.ViewTable; import org.apache.calcite.test.CalciteAssert; import org.apache.calcite.test.ElasticsearchChecker; -import org.apache.calcite.util.Bug; import org.apache.calcite.util.TestUtil; import org.apache.http.HttpHost; @@ -33,7 +32,6 @@ import com.google.common.io.LineProcessor; import com.google.common.io.Resources; -import org.junit.jupiter.api.Assumptions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.parallel.ResourceAccessMode; @@ -651,7 +649,6 @@ private static Consumer sortedResultSetChecker(String column, } @Test void testFilterSortDesc() { - Assumptions.assumeTrue(Bug.CALCITE_4645_FIXED, "CALCITE-4645"); final String sql = "select * from zips\n" + "where pop BETWEEN 95000 AND 100000\n" + "order by state desc, pop"; From 0b6b1b940da5463e2d888fea72758b2d03e95eaa Mon Sep 17 00:00:00 2001 From: Sergey Nuyanzin Date: Fri, 10 Apr 2026 22:48:00 +0200 Subject: [PATCH 208/562] [CALCITE-7466] Unparse of `MATCH_RECOGNIZE` produces duplicate aliases --- .../sql/validate/SqlValidatorImpl.java | 4 +--- .../apache/calcite/test/SqlValidatorTest.java | 19 +++++++++++++++++-- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java index ae5c8ee792d0..45896295a38f 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java @@ -6428,9 +6428,7 @@ private PairList validateMeasure(SqlMatchRecognize mr, setValidatedNodeType(measure, type); fields.add(alias, type); - sqlNodes.add( - SqlStdOperatorTable.AS.createCall(SqlParserPos.ZERO, expand, - new SqlIdentifier(alias, SqlParserPos.ZERO))); + sqlNodes.add(expand); } SqlNodeList list = new SqlNodeList(sqlNodes, measures.getParserPosition()); diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index dd66362819a7..6cc204716093 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -2468,7 +2468,7 @@ void testLikeAndSimilarFails() { sql(sql).fails("In UNPIVOT, cannot derive type for axis 'A0'"); } - @Test void testMatchRecognizeWithDistinctAggregation() { + @Test void testMatchRecognize() { final String sql = "SELECT *\n" + "FROM emp\n" + "MATCH_RECOGNIZE (\n" @@ -2481,13 +2481,28 @@ void testLikeAndSimilarFails() { + ") AS T"; sql(sql).fails("DISTINCT/ALL not allowed with " + "COUNT\\(DISTINCT `A`\\.`DEPTNO`\\) function"); + + sql("SELECT *\n" + + "FROM emp\n" + + "MATCH_RECOGNIZE (\n" + + " MEASURES\n" + + " FINAL COUNT(A.deptno) AS deptno\n" + + " PATTERN (A B)\n" + + " DEFINE\n" + + " A AS A.empno = 123\n" + + ") AS T") + .rewritesTo("SELECT *\n" + + "FROM `EMP` MATCH_RECOGNIZE(\n" + + "MEASURES FINAL COUNT(`A`.`DEPTNO`) AS `DEPTNO`\n" + + "PATTERN (`A` `B`)\n" + + "DEFINE `A` AS (PREV(`A`.`EMPNO`, 0) = 123 AS `A`)) AS `T`"); } @Test void testIntervalTimeUnitEnumeration() { // Since there is validation code relaying on the fact that the // enumerated time unit ordinals in SqlIntervalQualifier starts with 0 // and ends with 5, this test is here to make sure that if someone - // changes how the time untis are setup, an early feedback will be + // changes how the time units are setup, an early feedback will be // generated by this test. assertThat(TimeUnit.YEAR.ordinal(), is(0)); assertThat(TimeUnit.MONTH.ordinal(), is(1)); From 228525fb40d2518d684b6af2a23ad5fae911915b Mon Sep 17 00:00:00 2001 From: Sergey Nuyanzin Date: Sat, 11 Apr 2026 21:04:00 +0200 Subject: [PATCH 209/562] [CALCITE-7470] Unparse of `DEFINE` in `MATCH_RECOGNIZE` leads to incorrect SQL --- .../sql/validate/SqlValidatorImpl.java | 6 ++---- .../apache/calcite/test/SqlValidatorTest.java | 21 ++++++++++++------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java index 45896295a38f..bf277669d523 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java @@ -6483,11 +6483,9 @@ private void validateDefinitions(SqlMatchRecognize mr, inferUnknownTypes(booleanType, scope, expand); expand.validate(this, scope); - // Some extra work need required here. // In PREV, NEXT, FINAL and LAST, only one pattern variable is allowed. - sqlNodes.add( - SqlStdOperatorTable.AS.createCall(SqlParserPos.ZERO, expand, - new SqlIdentifier(alias, SqlParserPos.ZERO))); + // It is already parsed into AS operator, see PatternDefinition in Parser.jj + sqlNodes.add(expand); final RelDataType type = deriveType(scope, expand); if (!SqlTypeUtil.inBooleanFamily(type)) { diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index 6cc204716093..9c57293ed980 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -2482,7 +2482,7 @@ void testLikeAndSimilarFails() { sql(sql).fails("DISTINCT/ALL not allowed with " + "COUNT\\(DISTINCT `A`\\.`DEPTNO`\\) function"); - sql("SELECT *\n" + final String simpleMatchRecognize = "SELECT *\n" + "FROM emp\n" + "MATCH_RECOGNIZE (\n" + " MEASURES\n" @@ -2490,12 +2490,19 @@ void testLikeAndSimilarFails() { + " PATTERN (A B)\n" + " DEFINE\n" + " A AS A.empno = 123\n" - + ") AS T") - .rewritesTo("SELECT *\n" - + "FROM `EMP` MATCH_RECOGNIZE(\n" - + "MEASURES FINAL COUNT(`A`.`DEPTNO`) AS `DEPTNO`\n" - + "PATTERN (`A` `B`)\n" - + "DEFINE `A` AS (PREV(`A`.`EMPNO`, 0) = 123 AS `A`)) AS `T`"); + + ") AS T"; + + final String simpleMatchRecognizeExpected = "SELECT *\n" + + "FROM `EMP` MATCH_RECOGNIZE(\n" + + "MEASURES FINAL COUNT(`A`.`DEPTNO`) AS `DEPTNO`\n" + + "PATTERN (`A` `B`)\n" + + "DEFINE `A` AS PREV(`A`.`EMPNO`, 0) = 123) AS `T`"; + + sql(simpleMatchRecognize) + .rewritesTo(simpleMatchRecognizeExpected); + + sql(simpleMatchRecognizeExpected) + .withParserConfig(c -> c.withQuoting(Quoting.BACK_TICK)).ok(); } @Test void testIntervalTimeUnitEnumeration() { From 9adbf0c4cde8da13425353a6016e97c8e6feee82 Mon Sep 17 00:00:00 2001 From: Sergey Nuyanzin Date: Fri, 10 Apr 2026 22:48:00 +0200 Subject: [PATCH 210/562] [CALCITE-7467] `MATCH_RECOGNIZE` does not support aliases for tableRef --- core/src/main/codegen/templates/Parser.jj | 17 +++- .../apache/calcite/test/SqlValidatorTest.java | 82 +++++++++++++++++-- core/src/test/resources/sql/lateral.iq | 2 - 3 files changed, 92 insertions(+), 9 deletions(-) diff --git a/core/src/main/codegen/templates/Parser.jj b/core/src/main/codegen/templates/Parser.jj index d87339e6602a..cd1fbdc38560 100644 --- a/core/src/main/codegen/templates/Parser.jj +++ b/core/src/main/codegen/templates/Parser.jj @@ -2331,7 +2331,10 @@ SqlNode TableRef3(ExprContext exprContext, boolean lateral) : [ tableRef = ExtendTable(tableRef) ] tableRef = Over(tableRef) [ tableRef = Snapshot(tableRef) ] - [ tableRef = MatchRecognize(tableRef) ] + [ + LOOKAHEAD(3) + tableRef = MatchRecognize(tableRef) + ] ) | LOOKAHEAD(2) @@ -2339,7 +2342,10 @@ SqlNode TableRef3(ExprContext exprContext, boolean lateral) : tableRef = ParenthesizedExpression(exprContext) tableRef = Over(tableRef) tableRef = addLateral(tableRef, lateral) - [ tableRef = MatchRecognize(tableRef) ] + [ + LOOKAHEAD(3) + tableRef = MatchRecognize(tableRef) + ] | LOOKAHEAD(2) [ ] // "LATERAL" is implicit with "UNNEST", so ignore @@ -3251,6 +3257,7 @@ void AddUnpivotValue(List list) : SqlMatchRecognize MatchRecognize(SqlNode tableRef) : { final Span s, s0, s1, s2; + final SqlIdentifier aliasBeforeMatch; final SqlNodeList measureList; final SqlNodeList partitionList; final SqlNodeList orderList; @@ -3265,6 +3272,12 @@ SqlMatchRecognize MatchRecognize(SqlNode tableRef) : final SqlLiteral isStrictEnds; } { + [ + aliasBeforeMatch = SimpleIdentifier() { + tableRef = SqlStdOperatorTable.AS.createCall( + Span.of(tableRef).end(this), tableRef, aliasBeforeMatch); + } + ] { s = span(); checkNotJoin(tableRef); } ( { s2 = span(); } diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index 9c57293ed980..3b869edc9fe3 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -2482,7 +2482,9 @@ void testLikeAndSimilarFails() { sql(sql).fails("DISTINCT/ALL not allowed with " + "COUNT\\(DISTINCT `A`\\.`DEPTNO`\\) function"); - final String simpleMatchRecognize = "SELECT *\n" + // Test case for [CALCITE-7466] https://issues.apache.org/jira/browse/CALCITE-7466 + // Unparse of `MATCH_RECOGNIZE` produces duplicate aliases + final String sql2 = "SELECT *\n" + "FROM emp\n" + "MATCH_RECOGNIZE (\n" + " MEASURES\n" @@ -2492,16 +2494,86 @@ void testLikeAndSimilarFails() { + " A AS A.empno = 123\n" + ") AS T"; - final String simpleMatchRecognizeExpected = "SELECT *\n" + final String expected2 = "SELECT *\n" + "FROM `EMP` MATCH_RECOGNIZE(\n" + "MEASURES FINAL COUNT(`A`.`DEPTNO`) AS `DEPTNO`\n" + "PATTERN (`A` `B`)\n" + "DEFINE `A` AS PREV(`A`.`EMPNO`, 0) = 123) AS `T`"; - sql(simpleMatchRecognize) - .rewritesTo(simpleMatchRecognizeExpected); + sql(sql2) + .rewritesTo(expected2); + + sql(expected2) + .withParserConfig(c -> c.withQuoting(Quoting.BACK_TICK)).ok(); + + // Test cases for [CALCITE-7467] https://issues.apache.org/jira/browse/CALCITE-7467 + // `MATCH_RECOGNIZE` does not support alias for table before + final String sql3 = "SELECT *\n" + + "FROM sales.emp\n" + + "MATCH_RECOGNIZE (\n" + + " MEASURES\n" + + " FINAL COUNT(A.deptno) AS deptno\n" + + " PATTERN (A B)\n" + + " DEFINE\n" + + " A AS A.empno = 123\n" + + ") AS T"; + final String expected3 = "SELECT `T`.`DEPTNO`\n" + + "FROM `CATALOG`.`SALES`.`EMP` AS `EMP` MATCH_RECOGNIZE(\n" + + "MEASURES FINAL COUNT(`A`.`DEPTNO`) AS `DEPTNO`\n" + + "PATTERN (`A` `B`)\n" + + "DEFINE `A` AS PREV(`A`.`EMPNO`, 0) = 123) AS `T`"; + + sql(sql3) + .withValidatorConfig(c -> c.withIdentifierExpansion(true)) + .rewritesTo(expected3); + sql(expected3) + .withParserConfig(c -> c.withQuoting(Quoting.BACK_TICK)).ok(); + + // Identifier with alias for table before MATCH_RECOGNIZE should pass parser + final String sql4 = "SELECT *\n" + + "FROM sales.emp AS emp_alias\n" + + "MATCH_RECOGNIZE (\n" + + " MEASURES\n" + + " FINAL COUNT(A.deptno) AS deptno\n" + + " PATTERN (A B)\n" + + " DEFINE\n" + + " A AS A.empno = 123\n" + + ") AS T"; + + final String expected4 = "SELECT `T`.`DEPTNO`\n" + + "FROM `CATALOG`.`SALES`.`EMP` AS `EMP_ALIAS` MATCH_RECOGNIZE(\n" + + "MEASURES FINAL COUNT(`A`.`DEPTNO`) AS `DEPTNO`\n" + + "PATTERN (`A` `B`)\n" + + "DEFINE `A` AS PREV(`A`.`EMPNO`, 0) = 123) AS `T`"; - sql(simpleMatchRecognizeExpected) + sql(sql4) + .withValidatorConfig(c -> c.withIdentifierExpansion(true)) + .rewritesTo(expected4); + sql(expected4) + .withParserConfig(c -> c.withQuoting(Quoting.BACK_TICK)).ok(); + + // Identifier with alias for table before MATCH_RECOGNIZE should pass parser + final String sql5 = "SELECT emp.empno, T.*\n" + + "FROM emp JOIN sales.emp AS emp_alias\n" + + "MATCH_RECOGNIZE (\n" + + " MEASURES\n" + + " FINAL COUNT(A.deptno) AS deptno\n" + + " PATTERN (A B)\n" + + " DEFINE\n" + + " A AS A.empno = 123\n" + + ") AS T on emp.deptno = T.deptno"; + final String expected5 = + "SELECT `EMP`.`EMPNO`, `T`.`DEPTNO`\n" + + "FROM `CATALOG`.`SALES`.`EMP` AS `EMP`\n" + + "INNER JOIN `CATALOG`.`SALES`.`EMP` AS `EMP_ALIAS` MATCH_RECOGNIZE(\n" + + "MEASURES FINAL COUNT(`A`.`DEPTNO`) AS `DEPTNO`\n" + + "PATTERN (`A` `B`)\n" + + "DEFINE `A` AS PREV(`A`.`EMPNO`, 0) = 123) AS `T` ON CAST(`EMP`.`DEPTNO` AS BIGINT) = `T`.`DEPTNO`"; + + sql(sql5) + .withValidatorConfig(c -> c.withIdentifierExpansion(true)) + .rewritesTo(expected5); + sql(expected5) .withParserConfig(c -> c.withQuoting(Quoting.BACK_TICK)).ok(); } diff --git a/core/src/test/resources/sql/lateral.iq b/core/src/test/resources/sql/lateral.iq index 8af6a5edc126..5c82727b8930 100644 --- a/core/src/test/resources/sql/lateral.iq +++ b/core/src/test/resources/sql/lateral.iq @@ -26,7 +26,6 @@ Was expecting one of: "CROSS" ... "EXTEND" ... "FOR" ... - "MATCH_RECOGNIZE" ... "OUTER" ... "TABLESAMPLE" ... !error @@ -39,7 +38,6 @@ Was expecting one of: "CROSS" ... "EXTEND" ... "FOR" ... - "MATCH_RECOGNIZE" ... "OUTER" ... "TABLESAMPLE" ... !error From 3b6c246acae8b7c0bbbb257c4dbd27de89a3d941 Mon Sep 17 00:00:00 2001 From: xuzifu666 <1206332514@qq.com> Date: Fri, 10 Apr 2026 10:01:52 +0800 Subject: [PATCH 211/562] [CALCITE-4868] Elasticsearch adapter fails if GROUP BY is followed by ORDER BY --- .../java/org/apache/calcite/util/Bug.java | 6 --- .../elasticsearch/ElasticsearchTable.java | 6 ++- .../elasticsearch/AggregationAndSortTest.java | 43 ++++++++++++++++--- 3 files changed, 43 insertions(+), 12 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/util/Bug.java b/core/src/main/java/org/apache/calcite/util/Bug.java index 6d13a13d67a9..8f23ce7b1cf8 100644 --- a/core/src/main/java/org/apache/calcite/util/Bug.java +++ b/core/src/main/java/org/apache/calcite/util/Bug.java @@ -165,12 +165,6 @@ public abstract class Bug { * fixed. */ public static final boolean CALCITE_4213_FIXED = false; - /** Whether - * [CALCITE-4868] - * Elasticsearch adapter fails if GROUP BY is followed by ORDER BY is - * fixed. */ - public static final boolean CALCITE_4868_FIXED = false; - /** Whether * [CALCITE-5422] * MILLISECOND and MICROSECOND units in INTERVAL literal is fixed. */ diff --git a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchTable.java b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchTable.java index db8441636be9..29ad974fd572 100644 --- a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchTable.java +++ b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchTable.java @@ -208,8 +208,12 @@ private Enumerable aggregate(List ops, // due to ES aggregation format. fields in "order by" clause should go first // if "order by" is missing. order in "group by" is un-important + // Only include fields that are actually in groupBy list, exclude aggregation aliases final Set orderedGroupBy = new LinkedHashSet<>(); - orderedGroupBy.addAll(sort.stream().map(Map.Entry::getKey).collect(Collectors.toList())); + sort.stream() + .map(Map.Entry::getKey) + .filter(groupBy::contains) + .forEach(orderedGroupBy::add); orderedGroupBy.addAll(groupBy); // construct nested aggregations node(s) diff --git a/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/AggregationAndSortTest.java b/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/AggregationAndSortTest.java index 1d4dd0e1cdb6..3ddf3c82cc3e 100644 --- a/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/AggregationAndSortTest.java +++ b/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/AggregationAndSortTest.java @@ -21,14 +21,12 @@ import org.apache.calcite.schema.impl.ViewTable; import org.apache.calcite.test.CalciteAssert; import org.apache.calcite.test.ElasticsearchChecker; -import org.apache.calcite.util.Bug; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableMap; -import org.junit.jupiter.api.Assumptions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.parallel.ResourceAccessMode; @@ -86,8 +84,13 @@ public static void setupInstance() throws Exception { } private static Connection createConnection() throws SQLException { + return createConnectionWithConformance("JAVA", "DEFAULT"); + } + + private static Connection createConnectionWithConformance(String lex, String conformance) + throws SQLException { final Connection connection = - DriverManager.getConnection("jdbc:calcite:lex=JAVA"); + DriverManager.getConnection("jdbc:calcite:lex=" + lex + ";conformance=" + conformance); final SchemaPlus root = connection.unwrap(CalciteConnection.class).getRootSchema(); @@ -468,9 +471,11 @@ private static Connection createConnection() throws SQLException { + "cat6=null; cat5=2\n"); } + /** Test case for + * [CALCITE-4868] + * Elasticsearch adapter fails if GROUP BY is followed by ORDER BY. + */ @Test void testOrderByWithGroupBy() { - // Once CALCITE-4868 is fixed, we can enable this test - Assumptions.assumeTrue(Bug.CALCITE_4868_FIXED, "CALCITE-4868"); CalciteAssert.that() .with(AggregationAndSortTest::createConnection) .query("select cat6, cat5 from view group by cat6, cat5 " @@ -479,4 +484,32 @@ private static Connection createConnection() throws SQLException { + "cat6=null; cat5=1\n" + "cat6=text1; cat5=null\n"); } + + /** Test case for + * [CALCITE-4868] + * Elasticsearch adapter fails if GROUP BY is followed by ORDER BY. + */ + @Test void testSortAggregation() { + // Test ORDER BY alias (isSortByAlias) + CalciteAssert.that() + .with(AggregationAndSortTest::createConnection) + .query("select cat5, max(val1) as MAX_VAL1 from view" + + " group by cat5 order by MAX_VAL1 desc, cat5 desc") + .returns("cat5=2; MAX_VAL1=7.0\ncat5=1; MAX_VAL1=1.0\ncat5=null; MAX_VAL1=null\n"); + + // Test ORDER BY ordinal (isSortByOrdinal) + CalciteAssert.that() + .with(AggregationAndSortTest::createConnection) + .query("select cat5, max(val1) as MAX_VAL1 from view" + + " group by cat5 order by 2 desc, 1 desc") + .returns("cat5=2; MAX_VAL1=7.0\ncat5=1; MAX_VAL1=1.0\ncat5=null; MAX_VAL1=null\n"); + + // Test GROUP BY alias (isGroupByAlias) with ORDER BY alias + // Uses BABEL conformance which supports GROUP BY alias + CalciteAssert.that() + .with(() -> createConnectionWithConformance("JAVA", "BABEL")) + .query("select cat5 as CAT, max(val1) as MAX_VAL1 from view" + + " group by CAT order by MAX_VAL1 desc, CAT desc") + .returns("CAT=2; MAX_VAL1=7.0\nCAT=1; MAX_VAL1=1.0\nCAT=null; MAX_VAL1=null\n"); + } } From 4a8a2160c907dc1949fd7b41382279007158b110 Mon Sep 17 00:00:00 2001 From: "cc.cai" <2356672992@qq.com> Date: Sun, 5 Apr 2026 14:45:03 +0800 Subject: [PATCH 212/562] [CALCITE-6298] Support UNION in Arrow adapter --- .../adapter/arrow/ArrowEnumerable.java | 12 ++-- .../adapter/arrow/ArrowFilterEnumerator.java | 8 ++- .../adapter/arrow/ArrowProjectEnumerator.java | 6 +- .../calcite/adapter/arrow/ArrowSchema.java | 21 ++++--- .../calcite/adapter/arrow/ArrowTable.java | 60 +++++++++++++++---- .../adapter/arrow/ArrowAdapterTest.java | 60 ++++++++++++++++++- 6 files changed, 139 insertions(+), 28 deletions(-) diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowEnumerable.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowEnumerable.java index 142f18c2f6a0..516822567eb8 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowEnumerable.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowEnumerable.java @@ -35,22 +35,26 @@ class ArrowEnumerable extends AbstractEnumerable { private final ImmutableIntList fields; private final @Nullable Projector projector; private final @Nullable Filter filter; - + private final Runnable onClose; ArrowEnumerable(ArrowFileReader arrowFileReader, ImmutableIntList fields, - @Nullable Projector projector, @Nullable Filter filter) { + @Nullable Projector projector, @Nullable Filter filter, + Runnable onClose) { this.arrowFileReader = arrowFileReader; this.projector = projector; this.filter = filter; this.fields = fields; + this.onClose = onClose; } @Override public Enumerator enumerator() { try { if (projector != null) { - return new ArrowProjectEnumerator(arrowFileReader, fields, projector); + return new ArrowProjectEnumerator(arrowFileReader, fields, projector, + onClose); } else if (filter != null) { - return new ArrowFilterEnumerator(arrowFileReader, fields, filter); + return new ArrowFilterEnumerator(arrowFileReader, fields, filter, + onClose); } throw new IllegalArgumentException( "The arrow enumerator must have either a filter or a projection"); diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowFilterEnumerator.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowFilterEnumerator.java index e54a8c6ed05b..5eddec224909 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowFilterEnumerator.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowFilterEnumerator.java @@ -45,10 +45,14 @@ class ArrowFilterEnumerator extends AbstractArrowEnumerator { private @Nullable SelectionVector selectionVector; private int selectionVectorIndex; - ArrowFilterEnumerator(ArrowFileReader arrowFileReader, ImmutableIntList fields, Filter filter) { + private final Runnable onClose; + + ArrowFilterEnumerator(ArrowFileReader arrowFileReader, ImmutableIntList fields, + Filter filter, Runnable onClose) { super(arrowFileReader, fields); this.allocator = new RootAllocator(Long.MAX_VALUE); this.filter = filter; + this.onClose = onClose; } @Override void evaluateOperator(ArrowRecordBatch arrowRecordBatch) { @@ -98,6 +102,8 @@ class ArrowFilterEnumerator extends AbstractArrowEnumerator { filter.close(); } catch (GandivaException e) { throw Util.toUnchecked(e); + } finally { + onClose.run(); } } } diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowProjectEnumerator.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowProjectEnumerator.java index 2426810bbc13..0895f36cf15f 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowProjectEnumerator.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowProjectEnumerator.java @@ -31,11 +31,13 @@ */ class ArrowProjectEnumerator extends AbstractArrowEnumerator { private final Projector projector; + private final Runnable onClose; ArrowProjectEnumerator(ArrowFileReader arrowFileReader, ImmutableIntList fields, - Projector projector) { + Projector projector, Runnable onClose) { super(arrowFileReader, fields); this.projector = projector; + this.onClose = onClose; } @Override protected void evaluateOperator(ArrowRecordBatch arrowRecordBatch) { @@ -71,6 +73,8 @@ class ArrowProjectEnumerator extends AbstractArrowEnumerator { projector.close(); } catch (GandivaException e) { throw Util.toUnchecked(e); + } finally { + onClose.run(); } } } diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowSchema.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowSchema.java index c40cf4439a18..510adfb8428f 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowSchema.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowSchema.java @@ -24,6 +24,7 @@ import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.ipc.ArrowFileReader; import org.apache.arrow.vector.ipc.SeekableReadChannel; +import org.apache.arrow.vector.types.pojo.Schema; import com.google.common.base.Suppliers; import com.google.common.collect.ImmutableMap; @@ -34,7 +35,7 @@ import java.io.File; import java.io.FileInputStream; -import java.io.FileNotFoundException; +import java.io.IOException; import java.util.HashMap; import java.util.Locale; import java.util.Map; @@ -96,21 +97,19 @@ private static Map deduceTableMap(File baseDirectory) { final Map tables = new HashMap<>(); for (File file : files) { final File arrowFile = new File(Sources.of(file).path()); - final FileInputStream fileInputStream; - try { - fileInputStream = new FileInputStream(arrowFile); - } catch (FileNotFoundException e) { + final Schema arrowSchema; + try (FileInputStream fis = new FileInputStream(arrowFile); + ArrowFileReader reader = + new ArrowFileReader(new SeekableReadChannel(fis.getChannel()), + new RootAllocator())) { + arrowSchema = reader.getVectorSchemaRoot().getSchema(); + } catch (IOException e) { throw Util.toUnchecked(e); } - final SeekableReadChannel seekableReadChannel = - new SeekableReadChannel(fileInputStream.getChannel()); - final RootAllocator allocator = new RootAllocator(); - final ArrowFileReader arrowFileReader = - new ArrowFileReader(seekableReadChannel, allocator); final String tableName = trim(file.getName(), ".arrow").toUpperCase(Locale.ROOT); final ArrowTable table = - new ArrowTable(null, arrowFileReader); + new ArrowTable(null, arrowFile, arrowSchema); tables.put(tableName, table); } diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTable.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTable.java index 358a08fb2500..fa8d59389b88 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTable.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTable.java @@ -43,13 +43,17 @@ import org.apache.arrow.gandiva.expression.ExpressionTree; import org.apache.arrow.gandiva.expression.TreeBuilder; import org.apache.arrow.gandiva.expression.TreeNode; +import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.ipc.ArrowFileReader; +import org.apache.arrow.vector.ipc.SeekableReadChannel; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.Schema; import org.checkerframework.checker.nullness.qual.Nullable; +import java.io.File; +import java.io.FileInputStream; import java.io.IOException; import java.lang.reflect.Type; import java.util.ArrayList; @@ -62,23 +66,34 @@ import static java.util.Objects.requireNonNull; /** - * Arrow Table. + * Table backed by an Apache Arrow file. + * + *

    Reads data from an Arrow IPC file on disk and supports projection + * and filter push-down via the Gandiva expression compiler. + * + *

    Implements {@link TranslatableTable} so that it can be converted into + * an {@link ArrowTableScan} for query planning, and {@link QueryableTable} + * so that it can be used via the {@link org.apache.calcite.linq4j} API. */ public class ArrowTable extends AbstractTable implements TranslatableTable, QueryableTable { private final @Nullable RelProtoDataType protoRowType; /** Arrow schema. (In Calcite terminology, more like a row type than a Schema.) */ private final Schema schema; - private final ArrowFileReader arrowFileReader; + private final File arrowFile; - ArrowTable(@Nullable RelProtoDataType protoRowType, ArrowFileReader arrowFileReader) { - try { - this.schema = arrowFileReader.getVectorSchemaRoot().getSchema(); - } catch (IOException e) { - throw Util.toUnchecked(e); - } + /** Creates an ArrowTable. + * + * @param protoRowType Optional row type override; if null, the row type is + * deduced from the Arrow schema + * @param arrowFile Arrow IPC file on disk + * @param schema Arrow schema of the file + */ + ArrowTable(@Nullable RelProtoDataType protoRowType, File arrowFile, + Schema schema) { this.protoRowType = protoRowType; - this.arrowFileReader = arrowFileReader; + this.arrowFile = arrowFile; + this.schema = schema; } @Override public RelDataType getRowType(RelDataTypeFactory typeFactory) { @@ -148,7 +163,23 @@ public Enumerable query(DataContext root, ImmutableIntList fields, } } - return new ArrowEnumerable(arrowFileReader, fields, projector, filter); + FileInputStream fis = null; + try { + fis = new FileInputStream(arrowFile); + final ArrowFileReader reader = + new ArrowFileReader(new SeekableReadChannel(fis.getChannel()), + new RootAllocator()); + final FileInputStream fisRef = fis; + final Runnable onClose = () -> closeSilently(fisRef); + fis = null; // ownership transferred to onClose + return new ArrowEnumerable(reader, fields, projector, filter, onClose); + } catch (IOException e) { + throw Util.toUnchecked(e); + } finally { + if (fis != null) { + closeSilently(fis); + } + } } @Override public Queryable asQueryable(QueryProvider queryProvider, @@ -200,6 +231,15 @@ private TreeNode convertConditionToGandiva(ConditionToken token) { token.operator, treeNodes, new ArrowType.Bool()); } + /** Closes an {@link AutoCloseable} without throwing. */ + private static void closeSilently(AutoCloseable closeable) { + try { + closeable.close(); + } catch (Exception e) { + // ignore + } + } + private static TreeNode makeLiteralNode(String literal, String type) { if (type.startsWith("decimal")) { String[] typeParts = diff --git a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java index 67b3075b4e1d..e8d09d6bd961 100644 --- a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java +++ b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java @@ -536,7 +536,9 @@ static void initializeArrowState(@TempDir Path sharedTempDir) .explainContains(plan); } - @Disabled("UNION does not work") + /** Test case for + * [CALCITE-6298] + * Support UNION in Arrow adapter. */ @Test void testArrowUnion() { String sql = "(select \"intField\"\n" + "from arrowdata\n" @@ -563,6 +565,62 @@ static void initializeArrowState(@TempDir Path sharedTempDir) .explainContains(plan); } + /** Test case for + * [CALCITE-6298] + * Support UNION in Arrow adapter. + * + *

    Tests three-way UNION to verify that multiple concurrent scans + * on the same table work correctly. */ + @Test void testArrowUnionThreeWay() { + String sql = "(select \"intField\"\n" + + "from arrowdata\n" + + "where \"intField\" = 1)\n" + + " union \n" + + "(select \"intField\"\n" + + "from arrowdata\n" + + "where \"intField\" = 2)\n" + + " union \n" + + "(select \"intField\"\n" + + "from arrowdata\n" + + "where \"intField\" = 3)\n"; + String result = "intField=1\nintField=2\nintField=3\n"; + + CalciteAssert.that() + .with(arrow) + .query(sql) + .returns(result); + } + + /** Test case for + * [CALCITE-6298] + * Support UNION in Arrow adapter. + * + *

    Tests four-way UNION ALL to verify that repeated scans + * on the same table work correctly without deduplication. */ + @Test void testArrowUnionAllFourWay() { + String sql = "(select \"intField\"\n" + + "from arrowdata\n" + + "where \"intField\" = 1)\n" + + " union all\n" + + "(select \"intField\"\n" + + "from arrowdata\n" + + "where \"intField\" = 1)\n" + + " union all\n" + + "(select \"intField\"\n" + + "from arrowdata\n" + + "where \"intField\" = 2)\n" + + " union all\n" + + "(select \"intField\"\n" + + "from arrowdata\n" + + "where \"intField\" = 2)\n"; + String result = "intField=1\nintField=1\nintField=2\nintField=2\n"; + + CalciteAssert.that() + .with(arrow) + .query(sql) + .returns(result); + } + @Test void testFieldWithSpace() { String sql = "select \"my Field\" from (select \"intField\", \"stringField\" as \"my Field\"\n" + "from arrowdata)\n" From 591171ba31c9d044472ef25e655f7f153b3a582d Mon Sep 17 00:00:00 2001 From: Sergey Nuyanzin Date: Sat, 11 Apr 2026 21:41:46 +0200 Subject: [PATCH 213/562] [CALCITE-7020] Upgrade gradle from 8.7 to 8.14.4 Other dependencies also bumped com.github.vlsi.vlsi-release-plugins from 1.90 to 3.0.1 kotlin from 1.9.22 to 2.0.21 org.jetbrains.gradle.plugin.idea-ext from 0.5 to 1.4.1 jandex from 2.2.3.Final to 3.5.3 asm from 9.6 to 9.9.1 byte-buddy from 1.14.15 to 1.18.8 junit5 from 5.9.1 to 5.10.5 --- .ratignore | 4 ++++ build.gradle.kts | 14 +++++++---- buildSrc/build.gradle.kts | 8 +++++-- .../calcite/test/CassandraExtension.java | 2 +- gradle.properties | 14 +++++------ gradle/wrapper/gradle-wrapper.jar | Bin 43462 -> 43504 bytes gradle/wrapper/gradle-wrapper.properties | 4 ++-- gradlew | 7 ++++-- gradlew.bat | 22 ++++++++++-------- site/_docs/howto.md | 2 +- 10 files changed, 48 insertions(+), 29 deletions(-) diff --git a/.ratignore b/.ratignore index 50e978b59ac2..71048d405c5c 100644 --- a/.ratignore +++ b/.ratignore @@ -50,3 +50,7 @@ site/favicon.ico # jenv .jenv-version + +# Kotlin compiler session and log files +**/.kotlin/sessions/kotlin-compiler*.salive +**/.kotlin/errors/errors*.log diff --git a/build.gradle.kts b/build.gradle.kts index 63ac833db686..95c38f1a0978 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -198,7 +198,7 @@ reporting { reports { if (enableJacoco) { val jacocoAggregateTestReport by creating(JacocoCoverageReport::class) { - testType.set(TestSuiteType.UNIT_TEST) + testSuiteName = "test" } } } @@ -289,7 +289,8 @@ dependencies { } if (enableJacoco) { for (p in subprojects) { - if (p.name != "bom") { + val hasTests = p.file("src/test/java").isDirectory || p.file("src/test/kotlin").isDirectory + if (p.name != "bom" && hasTests) { jacocoAggregation(p) } } @@ -418,6 +419,7 @@ allprojects { val testRuntimeOnly by configurations testImplementation(platform("org.junit:junit-bom")) testImplementation("org.junit.jupiter:junit-jupiter") + testRuntimeOnly("org.junit.platform:junit-platform-launcher") testImplementation("org.hamcrest:hamcrest") if (project.props.bool("junit4", default = false)) { // Allow projects to opt-out of junit dependency, so they can be JUnit5-only @@ -535,8 +537,12 @@ allprojects { // Ensure builds are reproducible isPreserveFileTimestamps = false isReproducibleFileOrder = true - dirMode = "775".toInt(8) - fileMode = "664".toInt(8) + dirPermissions { + unix("775") + } + filePermissions { + unix("664") + } } tasks { diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts index fdf111eef790..62a9436fb4f0 100644 --- a/buildSrc/build.gradle.kts +++ b/buildSrc/build.gradle.kts @@ -42,8 +42,12 @@ allprojects { // Ensure builds are reproducible isPreserveFileTimestamps = false isReproducibleFileOrder = true - dirMode = "775".toInt(8) - fileMode = "664".toInt(8) + dirPermissions { + unix("775") + } + filePermissions { + unix("664") + } } java { diff --git a/cassandra/src/test/java/org/apache/calcite/test/CassandraExtension.java b/cassandra/src/test/java/org/apache/calcite/test/CassandraExtension.java index 03097925a5da..681a38e8a470 100644 --- a/cassandra/src/test/java/org/apache/calcite/test/CassandraExtension.java +++ b/cassandra/src/test/java/org/apache/calcite/test/CassandraExtension.java @@ -157,7 +157,7 @@ private static CassandraResource getOrCreate(ExtensionContext context) { /** Cassandra resource. */ private static class CassandraResource - implements ExtensionContext.Store.CloseableResource { + implements AutoCloseable { private final CqlSession session; private CassandraResource() { diff --git a/gradle.properties b/gradle.properties index 2cc99e709cac..297c059964ef 100644 --- a/gradle.properties +++ b/gradle.properties @@ -46,14 +46,14 @@ org.checkerframework.version=0.5.16 com.github.autostyle.version=3.0 com.github.johnrengelman.shadow.version=5.1.0 com.github.spotbugs.version=2.0.0 -com.github.vlsi.vlsi-release-plugins.version=1.90 +com.github.vlsi.vlsi-release-plugins.version=3.0.1 com.google.protobuf.version=0.8.10 de.thetaphi.forbiddenapis.version=3.7 jacoco.version=0.8.12 -kotlin.version=1.9.22 +kotlin.version=2.0.21 net.ltgt.errorprone.version=1.3.0 me.champeau.jmh.version=0.7.2 -org.jetbrains.gradle.plugin.idea-ext.version=0.5 +org.jetbrains.gradle.plugin.idea-ext.version=1.4.1 org.nosphere.apache.rat.version=0.8.1 org.owasp.dependencycheck.version=6.1.6 org.sonarqube.version=3.5.0.2730 @@ -75,7 +75,7 @@ checkstyle.version=8.28 spotbugs.version=3.1.11 errorprone.version=2.5.1 # The property is used in https://github.com/wildfly/jandex regression testing, so avoid renaming -jandex.version=2.2.3.Final +jandex.version=3.5.3 # We support Guava versions as old as 21.0 but prefer more recent versions. # elasticsearch does not like asm:6.2.1+ @@ -83,8 +83,8 @@ aggdesigner-algorithm.version=6.1 apiguardian-api.version=1.1.2 arrow-gandiva.version=15.0.0 arrow.version=15.0.0 -asm.version=9.6 -byte-buddy.version=1.14.15 +asm.version=9.9.1 +byte-buddy.version=1.18.8 cassandra-all.version=4.1.6 cassandra-java-driver-core.version=4.18.1 cassandra-unit.version=4.3.1.0 @@ -137,7 +137,7 @@ json-smart.version=2.6.0 jsr305.version=3.0.2 jsoup.version=1.11.3 junit4.version=4.13.2 -junit5.version=5.9.1 +junit5.version=5.10.5 kafka-clients.version=2.1.1 kerby.version=1.1.1 log4j2.version=2.17.1 diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index d64cd4917707c1f8861d8cb53dd15194d4248596..2c3521197d7c4586c843d1d3e9090525f1898cde 100644 GIT binary patch delta 34463 zcmY(qRX`kF)3u#IAjsf0xCD212@LM;?(PINyAue(f;$XO2=4Cg1P$=#e%|lo zKk1`B>Q#GH)wNd-&cI#Hz}3=WfYndTeo)CyX{fOHsQjGa<{e=jamMNwjdatD={CN3>GNchOE9OGPIqr)3v>RcKWR3Z zF-guIMjE2UF0Wqk1)21791y#}ciBI*bAenY*BMW_)AeSuM5}vz_~`+1i!Lo?XAEq{TlK5-efNFgHr6o zD>^vB&%3ZGEWMS>`?tu!@66|uiDvS5`?bF=gIq3rkK(j<_TybyoaDHg8;Y#`;>tXI z=tXo~e9{U!*hqTe#nZjW4z0mP8A9UUv1}C#R*@yu9G3k;`Me0-BA2&Aw6f`{Ozan2 z8c8Cs#dA-7V)ZwcGKH}jW!Ja&VaUc@mu5a@CObzNot?b{f+~+212lwF;!QKI16FDS zodx>XN$sk9;t;)maB^s6sr^L32EbMV(uvW%or=|0@U6cUkE`_!<=LHLlRGJx@gQI=B(nn z-GEjDE}*8>3U$n(t^(b^C$qSTI;}6q&ypp?-2rGpqg7b}pyT zOARu2x>0HB{&D(d3sp`+}ka+Pca5glh|c=M)Ujn_$ly^X6&u z%Q4Y*LtB_>i6(YR!?{Os-(^J`(70lZ&Hp1I^?t@~SFL1!m0x6j|NM!-JTDk)%Q^R< z@e?23FD&9_W{Bgtr&CG&*Oer3Z(Bu2EbV3T9FeQ|-vo5pwzwQ%g&=zFS7b{n6T2ZQ z*!H(=z<{D9@c`KmHO&DbUIzpg`+r5207}4D=_P$ONIc5lsFgn)UB-oUE#{r+|uHc^hzv_df zV`n8&qry%jXQ33}Bjqcim~BY1?KZ}x453Oh7G@fA(}+m(f$)TY%7n=MeLi{jJ7LMB zt(mE*vFnep?YpkT_&WPV9*f>uSi#n#@STJmV&SLZnlLsWYI@y+Bs=gzcqche=&cBH2WL)dkR!a95*Ri)JH_4c*- zl4pPLl^as5_y&6RDE@@7342DNyF&GLJez#eMJjI}#pZN{Y8io{l*D+|f_Y&RQPia@ zNDL;SBERA|B#cjlNC@VU{2csOvB8$HzU$01Q?y)KEfos>W46VMh>P~oQC8k=26-Ku)@C|n^zDP!hO}Y z_tF}0@*Ds!JMt>?4y|l3?`v#5*oV-=vL7}zehMON^=s1%q+n=^^Z{^mTs7}*->#YL z)x-~SWE{e?YCarwU$=cS>VzmUh?Q&7?#Xrcce+jeZ|%0!l|H_=D_`77hBfd4Zqk&! zq-Dnt_?5*$Wsw8zGd@?woEtfYZ2|9L8b>TO6>oMh%`B7iBb)-aCefM~q|S2Cc0t9T zlu-ZXmM0wd$!gd-dTtik{bqyx32%f;`XUvbUWWJmpHfk8^PQIEsByJm+@+-aj4J#D z4#Br3pO6z1eIC>X^yKk|PeVwX_4B+IYJyJyc3B`4 zPrM#raacGIzVOexcVB;fcsxS=s1e&V;Xe$tw&KQ`YaCkHTKe*Al#velxV{3wxx}`7@isG zp6{+s)CG%HF#JBAQ_jM%zCX5X;J%-*%&jVI?6KpYyzGbq7qf;&hFprh?E5Wyo=bZ) z8YNycvMNGp1836!-?nihm6jI`^C`EeGryoNZO1AFTQhzFJOA%Q{X(sMYlzABt!&f{ zoDENSuoJQIg5Q#@BUsNJX2h>jkdx4<+ipUymWKFr;w+s>$laIIkfP6nU}r+?J9bZg zUIxz>RX$kX=C4m(zh-Eg$BsJ4OL&_J38PbHW&7JmR27%efAkqqdvf)Am)VF$+U3WR z-E#I9H6^)zHLKCs7|Zs<7Bo9VCS3@CDQ;{UTczoEprCKL3ZZW!ffmZFkcWU-V|_M2 zUA9~8tE9<5`59W-UgUmDFp11YlORl3mS3*2#ZHjv{*-1#uMV_oVTy{PY(}AqZv#wF zJVks)%N6LaHF$$<6p8S8Lqn+5&t}DmLKiC~lE{jPZ39oj{wR&fe*LX-z0m}9ZnZ{U z>3-5Bh{KKN^n5i!M79Aw5eY=`6fG#aW1_ZG;fw7JM69qk^*(rmO{|Z6rXy?l=K=#_ zE-zd*P|(sskasO(cZ5L~_{Mz&Y@@@Q)5_8l<6vB$@226O+pDvkFaK8b>%2 zfMtgJ@+cN@w>3)(_uR;s8$sGONbYvoEZ3-)zZk4!`tNzd<0lwt{RAgplo*f@Z)uO` zzd`ljSqKfHJOLxya4_}T`k5Ok1Mpo#MSqf~&ia3uIy{zyuaF}pV6 z)@$ZG5LYh8Gge*LqM_|GiT1*J*uKes=Oku_gMj&;FS`*sfpM+ygN&yOla-^WtIU#$ zuw(_-?DS?6DY7IbON7J)p^IM?N>7x^3)(7wR4PZJu(teex%l>zKAUSNL@~{czc}bR z)I{XzXqZBU3a;7UQ~PvAx8g-3q-9AEd}1JrlfS8NdPc+!=HJ6Bs( zCG!0;e0z-22(Uzw>hkEmC&xj?{0p|kc zM}MMXCF%RLLa#5jG`+}{pDL3M&|%3BlwOi?dq!)KUdv5__zR>u^o|QkYiqr(m3HxF z6J*DyN#Jpooc$ok=b7{UAVM@nwGsr6kozSddwulf5g1{B=0#2)zv!zLXQup^BZ4sv*sEsn)+MA?t zEL)}3*R?4(J~CpeSJPM!oZ~8;8s_=@6o`IA%{aEA9!GELRvOuncE`s7sH91 zmF=+T!Q6%){?lJn3`5}oW31(^Of|$r%`~gT{eimT7R~*Mg@x+tWM3KE>=Q>nkMG$U za7r>Yz2LEaA|PsMafvJ(Y>Xzha?=>#B!sYfVob4k5Orb$INFdL@U0(J8Hj&kgWUlO zPm+R07E+oq^4f4#HvEPANGWLL_!uF{nkHYE&BCH%l1FL_r(Nj@M)*VOD5S42Gk-yT z^23oAMvpA57H(fkDGMx86Z}rtQhR^L!T2iS!788E z+^${W1V}J_NwdwdxpXAW8}#6o1(Uu|vhJvubFvQIH1bDl4J4iDJ+181KuDuHwvM?` z%1@Tnq+7>p{O&p=@QT}4wT;HCb@i)&7int<0#bj8j0sfN3s6|a(l7Bj#7$hxX@~iP z1HF8RFH}irky&eCN4T94VyKqGywEGY{Gt0Xl-`|dOU&{Q;Ao;sL>C6N zXx1y^RZSaL-pG|JN;j9ADjo^XR}gce#seM4QB1?S`L*aB&QlbBIRegMnTkTCks7JU z<0(b+^Q?HN1&$M1l&I@>HMS;!&bb()a}hhJzsmB?I`poqTrSoO>m_JE5U4=?o;OV6 zBZjt;*%1P>%2{UL=;a4(aI>PRk|mr&F^=v6Fr&xMj8fRCXE5Z2qdre&;$_RNid5!S zm^XiLK25G6_j4dWkFqjtU7#s;b8h?BYFxV?OE?c~&ME`n`$ix_`mb^AWr+{M9{^^Rl;~KREplwy2q;&xe zUR0SjHzKVYzuqQ84w$NKVPGVHL_4I)Uw<$uL2-Ml#+5r2X{LLqc*p13{;w#E*Kwb*1D|v?e;(<>vl@VjnFB^^Y;;b3 z=R@(uRj6D}-h6CCOxAdqn~_SG=bN%^9(Ac?zfRkO5x2VM0+@_qk?MDXvf=@q_* z3IM@)er6-OXyE1Z4sU3{8$Y$>8NcnU-nkyWD&2ZaqX1JF_JYL8y}>@V8A5%lX#U3E zet5PJM`z79q9u5v(OE~{by|Jzlw2<0h`hKpOefhw=fgLTY9M8h+?37k@TWpzAb2Fc zQMf^aVf!yXlK?@5d-re}!fuAWu0t57ZKSSacwRGJ$0uC}ZgxCTw>cjRk*xCt%w&hh zoeiIgdz__&u~8s|_TZsGvJ7sjvBW<(C@}Y%#l_ID2&C`0;Eg2Z+pk;IK}4T@W6X5H z`s?ayU-iF+aNr5--T-^~K~p;}D(*GWOAYDV9JEw!w8ZYzS3;W6*_`#aZw&9J ziXhBKU3~zd$kKzCAP-=t&cFDeQR*_e*(excIUxKuD@;-twSlP6>wWQU)$|H3Cy+`= z-#7OW!ZlYzZxkdQpfqVDFU3V2B_-eJS)Fi{fLtRz!K{~7TR~XilNCu=Z;{GIf9KYz zf3h=Jo+1#_s>z$lc~e)l93h&RqW1VHYN;Yjwg#Qi0yzjN^M4cuL>Ew`_-_wRhi*!f zLK6vTpgo^Bz?8AsU%#n}^EGigkG3FXen3M;hm#C38P@Zs4{!QZPAU=m7ZV&xKI_HWNt90Ef zxClm)ZY?S|n**2cNYy-xBlLAVZ=~+!|7y`(fh+M$#4zl&T^gV8ZaG(RBD!`3?9xcK zp2+aD(T%QIgrLx5au&TjG1AazI;`8m{K7^!@m>uGCSR;Ut{&?t%3AsF{>0Cm(Kf)2 z?4?|J+!BUg*P~C{?mwPQ#)gDMmro20YVNsVx5oWQMkzQ? zsQ%Y>%7_wkJqnSMuZjB9lBM(o zWut|B7w48cn}4buUBbdPBW_J@H7g=szrKEpb|aE>!4rLm+sO9K%iI75y~2HkUo^iw zJ3se$8$|W>3}?JU@3h@M^HEFNmvCp|+$-0M?RQ8SMoZ@38%!tz8f8-Ptb@106heiJ z^Bx!`0=Im z1!NUhO=9ICM*+||b3a7w*Y#5*Q}K^ar+oMMtekF0JnO>hzHqZKH0&PZ^^M(j;vwf_ z@^|VMBpcw8;4E-9J{(u7sHSyZpQbS&N{VQ%ZCh{c1UA5;?R} z+52*X_tkDQ(s~#-6`z4|Y}3N#a&dgP4S_^tsV=oZr4A1 zaSoPN1czE(UIBrC_r$0HM?RyBGe#lTBL4~JW#A`P^#0wuK)C-2$B6TvMi@@%K@JAT_IB^T7Zfqc8?{wHcSVG_?{(wUG%zhCm=%qP~EqeqKI$9UivF zv+5IUOs|%@ypo6b+i=xsZ=^G1yeWe)z6IX-EC`F=(|_GCNbHbNp(CZ*lpSu5n`FRA zhnrc4w+Vh?r>her@Ba_jv0Omp#-H7avZb=j_A~B%V0&FNi#!S8cwn0(Gg-Gi_LMI{ zCg=g@m{W@u?GQ|yp^yENd;M=W2s-k7Gw2Z(tsD5fTGF{iZ%Ccgjy6O!AB4x z%&=6jB7^}pyftW2YQpOY1w@%wZy%}-l0qJlOSKZXnN2wo3|hujU+-U~blRF!^;Tan z0w;Srh0|Q~6*tXf!5-rCD)OYE(%S|^WTpa1KHtpHZ{!;KdcM^#g8Z^+LkbiBHt85m z;2xv#83lWB(kplfgqv@ZNDcHizwi4-8+WHA$U-HBNqsZ`hKcUI3zV3d1ngJP-AMRET*A{> zb2A>Fk|L|WYV;Eu4>{a6ESi2r3aZL7x}eRc?cf|~bP)6b7%BnsR{Sa>K^0obn?yiJ zCVvaZ&;d_6WEk${F1SN0{_`(#TuOOH1as&#&xN~+JDzX(D-WU_nLEI}T_VaeLA=bc zl_UZS$nu#C1yH}YV>N2^9^zye{rDrn(rS99>Fh&jtNY7PP15q%g=RGnxACdCov47= zwf^9zfJaL{y`R#~tvVL#*<`=`Qe zj_@Me$6sIK=LMFbBrJps7vdaf_HeX?eC+P^{AgSvbEn?n<}NDWiQGQG4^ZOc|GskK z$Ve2_n8gQ-KZ=s(f`_X!+vM5)4+QmOP()2Fe#IL2toZBf+)8gTVgDSTN1CkP<}!j7 z0SEl>PBg{MnPHkj4wj$mZ?m5x!1ePVEYI(L_sb0OZ*=M%yQb?L{UL(2_*CTVbRxBe z@{)COwTK1}!*CK0Vi4~AB;HF(MmQf|dsoy(eiQ>WTKcEQlnKOri5xYsqi61Y=I4kzAjn5~{IWrz_l))|Ls zvq7xgQs?Xx@`N?f7+3XKLyD~6DRJw*uj*j?yvT3}a;(j_?YOe%hUFcPGWRVBXzpMJ zM43g6DLFqS9tcTLSg=^&N-y0dXL816v&-nqC0iXdg7kV|PY+js`F8dm z2PuHw&k+8*&9SPQ6f!^5q0&AH(i+z3I7a?8O+S5`g)>}fG|BM&ZnmL;rk)|u{1!aZ zEZHpAMmK_v$GbrrWNP|^2^s*!0waLW=-h5PZa-4jWYUt(Hr@EA(m3Mc3^uDxwt-me^55FMA9^>hpp26MhqjLg#^Y7OIJ5%ZLdNx&uDgIIqc zZRZl|n6TyV)0^DDyVtw*jlWkDY&Gw4q;k!UwqSL6&sW$B*5Rc?&)dt29bDB*b6IBY z6SY6Unsf6AOQdEf=P1inu6(6hVZ0~v-<>;LAlcQ2u?wRWj5VczBT$Op#8IhppP-1t zfz5H59Aa~yh7EN;BXJsLyjkjqARS5iIhDVPj<=4AJb}m6M@n{xYj3qsR*Q8;hVxDyC4vLI;;?^eENOb5QARj#nII5l$MtBCI@5u~(ylFi$ zw6-+$$XQ}Ca>FWT>q{k)g{Ml(Yv=6aDfe?m|5|kbGtWS}fKWI+})F6`x@||0oJ^(g|+xi zqlPdy5;`g*i*C=Q(aGeDw!eQg&w>UUj^{o?PrlFI=34qAU2u@BgwrBiaM8zoDTFJ< zh7nWpv>dr?q;4ZA?}V}|7qWz4W?6#S&m>hs4IwvCBe@-C>+oohsQZ^JC*RfDRm!?y zS4$7oxcI|##ga*y5hV>J4a%HHl^t$pjY%caL%-FlRb<$A$E!ws?8hf0@(4HdgQ!@> zds{&g$ocr9W4I84TMa9-(&^_B*&R%^=@?Ntxi|Ejnh;z=!|uVj&3fiTngDPg=0=P2 zB)3#%HetD84ayj??qrxsd9nqrBem(8^_u_UY{1@R_vK-0H9N7lBX5K(^O2=0#TtUUGSz{ z%g>qU8#a$DyZ~EMa|8*@`GOhCW3%DN%xuS91T7~iXRr)SG`%=Lfu%U~Z_`1b=lSi?qpD4$vLh$?HU6t0MydaowUpb zQr{>_${AMesCEffZo`}K0^~x>RY_ZIG{(r39MP>@=aiM@C;K)jUcfQV8#?SDvq>9D zI{XeKM%$$XP5`7p3K0T}x;qn)VMo>2t}Ib(6zui;k}<<~KibAb%p)**e>ln<=qyWU zrRDy|UXFi9y~PdEFIAXejLA{K)6<)Q`?;Q5!KsuEw({!#Rl8*5_F{TP?u|5(Hijv( ztAA^I5+$A*+*e0V0R~fc{ET-RAS3suZ}TRk3r)xqj~g_hxB`qIK5z(5wxYboz%46G zq{izIz^5xW1Vq#%lhXaZL&)FJWp0VZNO%2&ADd?+J%K$fM#T_Eke1{dQsx48dUPUY zLS+DWMJeUSjYL453f@HpRGU6Dv)rw+-c6xB>(=p4U%}_p>z^I@Ow9`nkUG21?cMIh9}hN?R-d)*6%pr6d@mcb*ixr7 z)>Lo<&2F}~>WT1ybm^9UO{6P9;m+fU^06_$o9gBWL9_}EMZFD=rLJ~&e?fhDnJNBI zKM=-WR6g7HY5tHf=V~6~QIQ~rakNvcsamU8m28YE=z8+G7K=h%)l6k zmCpiDInKL6*e#)#Pt;ANmjf`8h-nEt&d}(SBZMI_A{BI#ck-_V7nx)K9_D9K-p@?Zh81#b@{wS?wCcJ%og)8RF*-0z+~)6f#T` zWqF7_CBcnn=S-1QykC*F0YTsKMVG49BuKQBH%WuDkEy%E?*x&tt%0m>>5^HCOq|ux zuvFB)JPR-W|%$24eEC^AtG3Gp4qdK%pjRijF5Sg3X}uaKEE z-L5p5aVR!NTM8T`4|2QA@hXiLXRcJveWZ%YeFfV%mO5q#($TJ`*U>hicS+CMj%Ip# zivoL;dd*araeJK9EA<(tihD50FHWbITBgF9E<33A+eMr2;cgI3Gg6<-2o|_g9|> zv5}i932( zYfTE9?4#nQhP@a|zm#9FST2 z!y+p3B;p>KkUzH!K;GkBW}bWssz)9b>Ulg^)EDca;jDl+q=243BddS$hY^fC6lbpM z(q_bo4V8~eVeA?0LFD6ZtKcmOH^75#q$Eo%a&qvE8Zsqg=$p}u^|>DSWUP5i{6)LAYF4E2DfGZuMJ zMwxxmkxQf}Q$V3&2w|$`9_SQS^2NVbTHh;atB>=A%!}k-f4*i$X8m}Ni^ppZXk5_oYF>Gq(& z0wy{LjJOu}69}~#UFPc;$7ka+=gl(FZCy4xEsk);+he>Nnl>hb5Ud-lj!CNicgd^2 z_Qgr_-&S7*#nLAI7r()P$`x~fy)+y=W~6aNh_humoZr7MWGSWJPLk}$#w_1n%(@? z3FnHf1lbxKJbQ9c&i<$(wd{tUTX6DAKs@cXIOBv~!9i{wD@*|kwfX~sjKASrNFGvN zrFc=!0Bb^OhR2f`%hrp2ibv#KUxl)Np1aixD9{^o=)*U%n%rTHX?FSWL^UGpHpY@7 z74U}KoIRwxI#>)Pn4($A`nw1%-D}`sGRZD8Z#lF$6 zOeA5)+W2qvA%m^|$WluUU-O+KtMqd;Pd58?qZj})MbxYGO<{z9U&t4D{S2G>e+J9K ztFZ?}ya>SVOLp9hpW)}G%kTrg*KXXXsLkGdgHb+R-ZXqdkdQC0_)`?6mqo8(EU#d( zy;u&aVPe6C=YgCRPV!mJ6R6kdY*`e+VGM~`VtC>{k27!9vAZT)x2~AiX5|m1Rq}_= z;A9LX^nd$l-9&2%4s~p5r6ad-siV`HtxKF}l&xGSYJmP=z!?Mlwmwef$EQq~7;#OE z)U5eS6dB~~1pkj#9(}T3j!((8Uf%!W49FfUAozijoxInUE7z`~U3Y^}xc3xp){#9D z<^Tz2xw}@o@fdUZ@hnW#dX6gDOj4R8dV}Dw`u!h@*K)-NrxT8%2`T}EvOImNF_N1S zy?uo6_ZS>Qga4Xme3j#aX+1qdFFE{NT0Wfusa$^;eL5xGE_66!5_N8!Z~jCAH2=${ z*goHjl|z|kbmIE{cl-PloSTtD+2=CDm~ZHRgXJ8~1(g4W=1c3=2eF#3tah7ho`zm4 z05P&?nyqq$nC?iJ-nK_iBo=u5l#|Ka3H7{UZ&O`~t-=triw=SE7ynzMAE{Mv-{7E_ zViZtA(0^wD{iCCcg@c{54Ro@U5p1QZq_XlEGtdBAQ9@nT?(zLO0#)q55G8_Ug~Xnu zR-^1~hp|cy&52iogG@o?-^AD8Jb^;@&Ea5jEicDlze6%>?u$-eE};bQ`T6@(bED0J zKYtdc?%9*<<$2LCBzVx9CA4YV|q-qg*-{yQ;|0=KIgI6~z0DKTtajw2Oms3L zn{C%{P`duw!(F@*P)lFy11|Z&x`E2<=$Ln38>UR~z6~za(3r;45kQK_^QTX%!s zNzoIFFH8|Y>YVrUL5#mgA-Jh>j7)n)5}iVM4%_@^GSwEIBA2g-;43* z*)i7u*xc8jo2z8&=8t7qo|B-rsGw)b8UXnu`RgE4u!(J8yIJi(5m3~aYsADcfZ!GG zzqa7p=sg`V_KjiqI*LA-=T;uiNRB;BZZ)~88 z`C%p8%hIev2rxS12@doqsrjgMg3{A&N8A?%Ui5vSHh7!iC^ltF&HqG~;=16=h0{ygy^@HxixUb1XYcR36SB}}o3nxu z_IpEmGh_CK<+sUh@2zbK9MqO!S5cao=8LSQg0Zv4?ju%ww^mvc0WU$q@!oo#2bv24 z+?c}14L2vlDn%Y0!t*z=$*a!`*|uAVu&NO!z_arim$=btpUPR5XGCG0U3YU`v>yMr z^zmTdcEa!APX zYF>^Q-TP11;{VgtMqC}7>B^2gN-3KYl33gS-p%f!X<_Hr?`rG8{jb9jmuQA9U;BeG zHj6Pk(UB5c6zwX%SNi*Py*)gk^?+729$bAN-EUd*RKN7{CM4`Q65a1qF*-QWACA&m zrT)B(M}yih{2r!Tiv5Y&O&=H_OtaHUz96Npo_k0eN|!*s2mLe!Zkuv>^E8Xa43ZwH zOI058AZznYGrRJ+`*GmZzMi6yliFmGMge6^j?|PN%ARns!Eg$ufpcLc#1Ns!1@1 zvC7N8M$mRgnixwEtX{ypBS^n`k@t2cCh#_6L6WtQb8E~*Vu+Rr)YsKZRX~hzLG*BE zaeU#LPo?RLm(Wzltk79Jd1Y$|6aWz1)wf1K1RtqS;qyQMy@H@B805vQ%wfSJB?m&&=^m4i* zYVH`zTTFbFtNFkAI`Khe4e^CdGZw;O0 zqkQe2|NG_y6D%h(|EZNf&77_!NU%0y={^E=*gKGQ=)LdKPM3zUlM@otH2X07Awv8o zY8Y7a1^&Yy%b%m{mNQ5sWNMTIq96Wtr>a(hL>Qi&F(ckgKkyvM0IH<_}v~Fv-GqDapig=3*ZMOx!%cYY)SKzo7ECyem z9Mj3C)tCYM?C9YIlt1?zTJXNOo&oVxu&uXKJs7i+j8p*Qvu2PAnY}b`KStdpi`trk ztAO}T8eOC%x)mu+4ps8sYZ=vYJp16SVWEEgQyFKSfWQ@O5id6GfL`|2<}hMXLPszS zgK>NWOoR zBRyKeUPevpqKKShD|MZ`R;~#PdNMB3LWjqFKNvH9k+;(`;-pyXM55?qaji#nl~K8m z_MifoM*W*X9CQiXAOH{cZcP0;Bn10E1)T@62Um>et2ci!J2$5-_HPy(AGif+BJpJ^ ziHWynC_%-NlrFY+(f7HyVvbDIM$5ci_i3?22ZkF>Y8RPBhgx-7k3M2>6m5R24C|~I z&RPh9xpMGzhN4bii*ryWaN^d(`0 zTOADlU)g`1p+SVMNLztd)c+;XjXox(VHQwqzu>FROvf0`s&|NEv26}(TAe;@=FpZq zaVs6mp>W0rM3Qg*6x5f_bPJd!6dQGmh?&v0rpBNfS$DW-{4L7#_~-eA@7<2BsZV=X zow){3aATmLZOQrs>uzDkXOD=IiX;Ue*B(^4RF%H zeaZ^*MWn4tBDj(wj114r(`)P96EHq4th-;tWiHhkp2rDlrklX}I@ib-nel0slFoQO zOeTc;Rh7sMIebO`1%u)=GlEj+7HU;c|Nj>2j)J-kpR)s3#+9AiB zd$hAk6;3pu9(GCR#)#>aCGPYq%r&i02$0L9=7AlIGYdlUO5%eH&M!ZWD&6^NBAj0Y9ZDcPg@r@8Y&-}e!aq0S(`}NuQ({;aigCPnq75U9cBH&Y7 ze)W0aD>muAepOKgm7uPg3Dz7G%)nEqTUm_&^^3(>+eEI;$ia`m>m0QHEkTt^=cx^JsBC68#H(3zc~Z$E9I)oSrF$3 zUClHXhMBZ|^1ikm3nL$Z@v|JRhud*IhOvx!6X<(YSX(9LG#yYuZeB{=7-MyPF;?_8 zy2i3iVKG2q!=JHN>~!#Bl{cwa6-yB@b<;8LSj}`f9pw7#x3yTD>C=>1S@H)~(n_K4 z2-yr{2?|1b#lS`qG@+823j;&UE5|2+EdU4nVw5=m>o_gj#K>>(*t=xI7{R)lJhLU{ z4IO6!x@1f$aDVIE@1a0lraN9!(j~_uGlks)!&davUFRNYHflp<|ENwAxsp~4Hun$Q z$w>@YzXp#VX~)ZP8`_b_sTg(Gt7?oXJW%^Pf0UW%YM+OGjKS}X`yO~{7WH6nX8S6Z ztl!5AnM2Lo*_}ZLvo%?iV;D2z>#qdpMx*xY2*GGlRzmHCom`VedAoR=(A1nO)Y>;5 zCK-~a;#g5yDgf7_phlkM@)C8s!xOu)N2UnQhif-v5kL$*t=X}L9EyBRq$V(sI{90> z=ghTPGswRVbTW@dS2H|)QYTY&I$ljbpNPTc_T|FEJkSW7MV!JM4I(ksRqQ8)V5>}v z2Sf^Z9_v;dKSp_orZm09jb8;C(vzFFJgoYuWRc|Tt_&3k({wPKiD|*m!+za$(l*!gNRo{xtmqjy1=kGzFkTH=Nc>EL@1Um0BiN1)wBO$i z6rG={bRcT|%A3s3xh!Bw?=L&_-X+6}L9i~xRj2}-)7fsoq0|;;PS%mcn%_#oV#kAp zGw^23c8_0~ ze}v9(p};6HM0+qF5^^>BBEI3d=2DW&O#|(;wg}?3?uO=w+{*)+^l_-gE zSw8GV=4_%U4*OU^hibDV38{Qb7P#Y8zh@BM9pEM_o2FuFc2LWrW2jRRB<+IE)G=Vx zuu?cp2-`hgqlsn|$nx@I%TC!`>bX^G00_oKboOGGXLgyLKXoo$^@L7v;GWqfUFw3< zekKMWo0LR;TaFY}Tt4!O$3MU@pqcw!0w0 zA}SnJ6Lb597|P5W8$OsEHTku2Kw9y4V=hx*K%iSn!#LW9W#~OiWf^dXEP$^2 zaok=UyGwy3GRp)bm6Gqr>8-4h@3=2`Eto2|JE6Sufh?%U6;ut1v1d@#EfcQP2chCt z+mB{Bk5~()7G>wM3KYf7Xh?LGbwg1uWLotmc_}Z_o;XOUDyfU?{9atAT$={v82^w9 z(MW$gINHt4xB3{bdbhRR%T}L?McK?!zkLK3(e>zKyei(yq%Nsijm~LV|9mll-XHavFcc$teX7v);H>=oN-+E_Q{c|! zp

      JV~-9AH}jxf6IF!PxrB9is{_9s@PYth^`pb%DkwghLdAyDREz(csf9)HcVRq z+2Vn~>{(S&_;bq_qA{v7XbU?yR7;~JrLfo;g$Lkm#ufO1P`QW_`zWW+4+7xzQZnO$ z5&GyJs4-VGb5MEDBc5=zxZh9xEVoY(|2yRv&!T7LAlIs@tw+4n?v1T8M>;hBv}2n) zcqi+>M*U@uY>4N3eDSAH2Rg@dsl!1py>kO39GMP#qOHipL~*cCac2_vH^6x@xmO|E zkWeyvl@P$2Iy*mCgVF+b{&|FY*5Ygi8237i)9YW#Fp& z?TJTQW+7U)xCE*`Nsx^yaiJ0KSW}}jc-ub)8Z8x(|K7G>`&l{Y&~W=q#^4Gf{}aJ%6kLXsmv6cr=Hi*uB`V26;dr4C$WrPnHO>g zg1@A%DvIWPDtXzll39kY6#%j;aN7grYJP9AlJgs3FnC?crv$wC7S4_Z?<_s0j;MmE z75yQGul2=bY%`l__1X3jxju2$Ws%hNv75ywfAqjgFO7wFsFDOW^)q2%VIF~WhwEW0 z45z^+r+}sJ{q+>X-w(}OiD(!*&cy4X&yM`!L0Fe+_RUfs@=J{AH#K~gArqT=#DcGE z!FwY(h&+&811rVCVoOuK)Z<-$EX zp`TzcUQC256@YWZ*GkE@P_et4D@qpM92fWA6c$MV=^qTu7&g)U?O~-fUR&xFqNiY1 zRd=|zUs_rmFZhKI|H}dcKhy%Okl(#y#QuMi81zsY56Y@757xBQqDNkd+XhLQhp2BB zBF^aJ__D676wLu|yYo6jNJNw^B+Ce;DYK!f$!dNs1*?D^97u^jKS++7S z5qE%zG#HY-SMUn^_yru=T6v`)CM%K<>_Z>tPe|js`c<|y7?qol&)C=>uLWkg5 zmzNcSAG_sL)E9or;i+O}tY^70@h7+=bG1;YDlX{<4zF_?{)K5B&?^tKZ6<$SD%@>F zY0cl2H7)%zKeDX%Eo7`ky^mzS)s;842cP{_;dzFuyd~Npb4u!bwkkhf8-^C2e3`q8>MuPhgiv0VxHxvrN9_`rJv&GX0fWz-L-Jg^B zrTsm>)-~j0F1sV=^V?UUi{L2cp%YwpvHwwLaSsCIrGI#({{QfbgDxMqR1Z0TcrO*~ z;`z(A$}o+TN+QHHSvsC2`@?YICZ>s8&hY;SlR#|0PKaZIauCMS*cOpAMn@6@g@rZ+ z+GT--(uT6#mL8^*mMf7BE`(AVj?zLY-2$aI%TjtREu}5AWdGlcWLvfz(%wn72tGczwUOgGD3RXpWs%onuMxs9!*D^698AupW z9qTDQu4`!>n|)e35b4t+d(+uOx+>VC#nXCiRex_Fq4fu1f`;C`>g;IuS%6KgEa3NK z<8dsc`?SDP0g~*EC3QU&OZH-QpPowNEUd4rJF9MGAgb@H`mjRGq;?wFRDVQY7mMpm z3yoB7eQ!#O#`XIBDXqU>Pt~tCe{Q#awQI4YOm?Q3muUO6`nZ4^zi5|(wb9R)oyarG?mI|I@A0U!+**&lW7_bYKF2biJ4BDbi~*$h?kQ`rCC(LG-oO(nPxMU zfo#Z#n8t)+3Ph87roL-y2!!U4SEWNCIM16i~-&+f55;kxC2bL$FE@jH{5p$Z8gxOiP%Y`hTTa_!v{AKQz&- ztE+dosg?pN)leO5WpNTS>IKdEEn21zMm&?r28Q52{$e2tGL44^Ys=^?m6p=kOy!gJ zWm*oFGKS@mqj~{|SONA*T2)3XC|J--en+NrnPlNhAmXMqmiXs^*154{EVE{Uc%xqF zrbcQ~sezg;wQkW;dVezGrdC0qf!0|>JG6xErVZ8_?B(25cZrr-sL&=jKwW>zKyYMY zdRn1&@Rid0oIhoRl)+X4)b&e?HUVlOtk^(xldhvgf^7r+@TXa!2`LC9AsB@wEO&eU2mN) z(2^JsyA6qfeOf%LSJx?Y8BU1m=}0P;*H3vVXSjksEcm>#5Xa`}jj5D2fEfH2Xje-M zUYHgYX}1u_p<|fIC+pI5g6KGn%JeZPZ-0!!1})tOab>y=S>3W~x@o{- z6^;@rhHTgRaoor06T(UUbrK4+@5bO?r=!vckDD+nwK+>2{{|{u4N@g}r(r z#3beB`G2`XrO(iR6q2H8yS9v;(z-=*`%fk%CVpj%l#pt?g4*)yP|xS-&NBKOeW5_5 zXkVr;A)BGS=+F;j%O|69F0Lne?{U*t=^g?1HKy7R)R*<>%xD>K zelPqrp$&BF_?^mZ&U<*tWDIuhrw3HJj~--_0)GL8jxYs2@VLev2$;`DG7X6UI9Z)P zq|z`w46OtLJ1=V3U8B%9@FSsRP+Ze)dQ@;zLq|~>(%J5G-n}dRZ6&kyH|cQ!{Vil( zBUvQvj*~0_A1JCtaGZW|?6>KdP}!4A%l>(MnVv>A%d;!|qA>*t&-9-JFU4GZhn`jG z8GrgNsQJ%JSLgNFP`5;(=b+M9GO8cg+ygIz^4i?=eR@IY>IcG?+on?I4+Y47p-DB8 zjrlar)KtoI{#kBcqL&4?ub@Df+zMt*USCD_T8O$J$~oMrC6*TP7j@H5trGV$r0P6I zV7EZ{MWH`5`DrX*wx&`d;C`jjYoc_PMSqNB290QXlRn_4*F{5hBmEE4DHBC$%EsbR zQGb7p;)4MAjY@Bd*2F3L?<8typrrUykb$JXr#}c1|BL*QF|18D{ZTYBZ_=M&Ec6IS ziv{(%>CbeR(9Aog)}hA!xSm1p@K?*ce*-6R%odqGGk?I4@6q3dmHq)4jbw+B?|%#2 zbX;ioJ_tcGO*#d0v?il&mPAi+AKQvsQnPf*?8tX6qfOPsf-ttT+RZX6Dm&RF6beP3 zdotcJDI1Kn7wkq=;Au=BIyoGfXCNVjCKTj+fxU@mxp*d*7aHec0GTUPt`xbN8x%fe zikv87g)u~0cpQaf zd<7Mi9GR0B@*S&l&9pCl-HEaNX?ZY8MoXaYHGDf}733;(88<{E%)< z^k)X#To3=_O2$lKPsc9P-MkDAhJ~{x<=xTJw2aRY5SSZIA6Gij5cFzsGk@S)4@C65 zwN^6CwOI9`5c(3?cqRrH_gSq+ox(wtSBZc-Jr5N%^t3N&WB|TT_i4!i3lxwI=*p)Y zn7fb%HlXhf8OGjhzswj!=Crh~YwQYb+p~UaV@s%YPgiH_);$|Gx3{{v5v?7s<)+cb zxlT0Bb!OwtE!K>gx6c4v^M9mL0F=It*NfQL0J0O$RCpt746=H1pPNG#AZC|Y`SZt( zG`yKMBPV_0I|S?}?$t7GU%;*_39bCGO*x3+R|<=9WNe!8jH- zw5ZJS(k@wws?6w1rejjyZ>08aizReJBo%IRb3b3|VuR6Uo&sL?L5j(isqs%CYe@@b zIID7kF*hyqmy+7D(SPa^xNVm54hVF3{;4I9+mh)F22+_YFP>ux`{F)8l;uRX>1-cH zXqPnGsFRr|UZwJtjG=1x2^l_tF-mS0@sdC38kMi$kDw8W#zceJowZuV=@agQ_#l5w znB`g+sb1mhkrXh$X4y(<-CntwmVwah5#oA_p-U<_5$ zGDc%(b6Z=!QQ%w6YZS&HWovIaN8wMw1B-9N+Vyl=>(yIgy}BrAhpc2}8YL-i*_KY7 ztV+`WKcC?{RKA@t3pu*BtqZJFSd2d)+cc07-Z#4x&7Dnd{yg6)lz@`z%=Sl-`9Z~*io zck_Lshk9JRJs=t>1jmKB~>`6+(J z@(S}J2Q{Q{a-ASTnIViecW(FIagWQ%G41y?zS)gpooM z@c<2$7TykMs4LH*UUYfts(!Ncn`?eZl}f zg)wx@0N0J(X(OJ^=$2()HLn)=Cn~=zx(_9(B@L04%{F_Zn}5!~5Ec5D4ibN6G_AD} zzxY^T_JF##qM8~B%aZ1OC}X^kQu`JDwaRaZnt!YcRrP7fq>eIihJW1UY{Xhkn>NdX zKy|<6-wD*;GtE08sLYryW<-e)?7k;;B>e$u?v!QhU9jPK6*Y$o8{Tl`N`+QvG ze}71rVC)fis9TZ<>EJ2JR`80F^2rkB7dihm$1Ta2bR?&wz>e`)w<4)1{3SfS$uKfV z3R=JT!eY+i7+IIfl3SIgiR|KvBWH*s;OEuF5tq~wLOB^xP_Dc7-BbNjpC|dHYJrZCWj-ucmv4;YS~eN!LvwER`NCd`R4Xh5%zP$V^nU>j zdOkNvbyB_117;mhiTiL_TBcy&Grvl->zO_SlCCX5dFLd`q7x-lBj*&ykj^ zR3@z`y0<8XlBHEhlCk7IV=ofWsuF|d)ECS}qnWf?I#-o~5=JFQM8u+7I!^>dg|wEb zbu4wp#rHGayeYTT>MN+(x3O`nFMpOSERQdpzQv2ui|Z5#Qd zB(+GbXda|>CW55ky@mG13K0wfXAm8yoek3MJG!Hujn$5)Q(6wWb-l4ogu?jj2Q|srw?r z-TG0$OfmDx%(qcX`Fc`D!WS{3dN*V%SZas3$vFXQy98^y3oT~8Yv>$EX0!uiRae?m z_}pvK=rBy5Z_#_!8QEmix_@_*w8E8(2{R5kf^056;GzbLOPr2uqFYaG6Fkrv($n_51%7~QN<>9$WdjE=H}>(a41KM%d2x#e@K3{W|+=-h*mR&2C01e z2sMP;YjU)9h+1kxOKJ+g*W=&D@=$q4jF%@HyRtCwOmEmpS|Rr9V_2br*NOd^ z4LN#oxd5yL=#MPWN{9Vo^X-Wo{a7IF2hvYWB%eUCkAZq+=NQ=iLI9?~@ zr+|ky4Rgm7yEDuc2dIe941~qc8V_$7;?7|XLk6+nbrh}e&Tt20EWZ@dRFDoYbwhkn zjJ$th974Z0F${3wtVLk_Ty;*J-Pi zP0IwrAT!Lj34GcoSB8g?IKPt%!iLD-$s+f_eZg@9q!2Si?`F#fUqY`!{bM0O7V^G%VB|A zyMM>SKNg|KKP}+>>?n6|5MlPK3Vto&;nxppD;yk@z4DXPm0z9hxb+U&Fv4$y&G>q= z799L0$A2&#>CfSgCuu$+9W>s<-&yq3!C{F9N!{d?I|g|+Qd9@*d;GplgY5Fk$LOV+ zoMealKns!!80PWsJ%(}L61B!7l?j1_5P#LRrVv%NBhs{R`;aufHYb&b+mF%A+DGl5 zBemAHtbLFi++KT(wv9*?;awp>ROX~P?e<4#Uf5RKIV{c3NxmUz!LYO#Cxdz*CoRQp zSvX|#NN06=q_eTU5-T!RmUJ?Ht=XQF8t)f+GnY5nY5>-}WLR1+R5pou?l@Y|F@KEX zk=jh-yq=Rn9;riE*;Slo}PfNKhXO#;FrZCf%VZ9h7W z<63YWE^s_SlAVQh6B(En9i<9%4AT|2bTQ4Ph2)pI?f2S`$j?bp`>_3(`Fz&?ig-FJ zoO7KAh@4BDOU>sBXV84Eajr9;>wlbW&OSUt&dug?oAV;`+3oBzpI18%%1wA4blzmb z-{QPYJmn_2-F$A5JI!a8+-p8Bk*^U?^f5j7uZ}jEz0E3;XbahB2iZwS&l4jj4WRS6 z3O&!w=ymQSl~7LUE99noXd2y1)9E>yK`+ouR%sTOQ@Qjt@<;lErGLk1wrw7r zV)M})+amJXs_9hQa++&vrqgU&Xr8T)=G&5Vy6vOnvt37L*nU7&ws&ZO-9`)TGA**t zpby#0X|df;etRud+s~#Y_7zlPZ=_oLg%q&wraF6s>g@;VO#2sUseO=^+3%&Z?61(- z_IKzU`+Kw;Blil&LR#qv&{rzQnG|%i(Q3zLI@gh)2FE^H;~1dx9G|AOj(e%mSwT(C z71Zp!jar*i3S|_ik_3{n0L4KavYWWZ2x3MhyU!66E$h=L+A&-s$9X_w9Q_e;+`-{ZW# z^Zn2H_I~`}!vGeFRRY^DyKK#pORBr{&?X}ut`1a(x__(dt3y_-*Np0pX~q39D{Rns z!iXBWZO~+oZu>($Mrf0rjM>$JZar!n_0_!*e@yT7n=HfVT6#jbYZ0wYEXnTgPDZ0N zVE5?$1-v94G2@1jFyj##-E1Um(naG-8WuGy@rRAg)t9Oe0$RJ3OoWV8X4DXvW+ftx zk%S(O8h?#_3B9-1NHn&@ZAXtr=PXcAATV*GzFBXK>hVb9*`iMM-zvA6RwMH#2^901uxUFh&4fT% zmP?pjNsiRIMD)<6xZyOeThl_DN_ZJ*?KUIHgnx{vz`WKxj&!7HbM8{w?{Rued(M1v zKHsK{_q=YI88@Bf0*RW@cIV@=<{eGsG21xrTrWycT7*KBd!eD2zb1R(O@H~k7>Duv zHPwp=n8;t#1>7~fuM9IaD5w%BpwLtNCe_Sq9eal4oj2DB1#<+(MGR-P&Ig%3t%=!< zS$|KxI1a~an2Q>L$s;1$9nQJal4dk)Box$YsAKgCiEGni##jr|%So6Y4J@pYBF!;~ zhXwpKhc7&QZ$=e~Sb&ABZ4o)&U~N*dSU`2G^eQh-WCe9tA}~Ae369btLlB{GjOKB@yEDH!C7Q&df^#X zi~?{rCuAE|kAjKzt+r#t6s)1h840@A<%i5(O;$Q&tD(opg0)yzgm#=ucf4CSqkqYS zaTdivk5I~#=1Z9K5M*uV6H??6s9*ynT`vzr2@%Tkr4k+Tr_ib40$fPP7$yLA$cwJ@ zF@`94=op)$x^0t+QAsNY$pi!4e7hp~gO=|yD=^8JTvTiC(HAamYEQ}t z+hR~QoKTOz%)IHEg&6iC4vP=3mw&u4wvcSwi$vNBGQE5RoSUs^l+u{A+6s~aMMkXG z+1g4wD8^Y27Oe4f``K{+tm76n(*d6BUA4;pLa26`6RD6?Rq?2K1yMXVAk`&xbks*~{+``Mhg4cQEuw+aM zaI9{}9en8DCh*S9CojIk)qh|k?#iNiCQ}rAmr&iYRJiND ztt+j*c+}Fv&6x&7U~!(Sb1eAz1N@Nf`w?YxGJdhy+seiNNZEYIG1_<^?&pm^P8W?d ze(p@$nWC`Pxqpf8d&AIGNJn#Ty)j z1NbA^Y}pNQ>OfTdiAp+WR>C6390IrFj;YZglitGH8r7(GvVRpWjZd7|r24M{u66B) zs#VS$?R*!1FT&sO-ssvW8s5jh$-O=^9=7^y z75||~QA6zLW}Lu!YOZh1J$j46m zNH|;^a$U_RKgla5h>5(igl^ek(~2nL5a_0}ipvA_Xf0k*E-ExJNld0{LZ;F^DzqAL+IZGJ7<3i1szf zxMRkQ(|@;wj9%I7h{c*{;?g%giylU}Dz{iwb(1vGK<-vlnKs!|Mb9}iTt)Rl&NZka zkkugrMiY(ng3QseY!npaOf1jo3|r35nK+eTYh*`DHabuv@IFy zG7@V!LWE0&)bvqgQ8=-L-(vt#Z-&xaOj3G@Nqw1FfbNQ`!bFEl@z)0)+#Z5e#_hQ|Rd!KrEoRn^aFz zkzYzz%hher>ixcg6fW`=rr>Nx@enQ!sQqYR{<2^|eUfw?e8;B_`T)Kxkp8${U>g?k*VhCd zp^yYLvi}<#5TDjrx@{0U$jx*tQn+mhcXsq2e46a@44^-Sd;C6S2=}sK1LQ_OUhgO` z^4yN+e9Dv9TQ64y1Bw)0i4u)98(^+@R~eUUsG!Ye84 zFa7-?x3cqUXX)$G<2MgYiGWhjq?Q-CE(|sm-68_z>h_O2vME5nX;RodIf)=No(={I z_<&3QJcPg8kAI}_Vd+OH4z{NsFMmjv3;kunMSh94VNnqD?85uOps%nq=q?kU_JT5@ zwih;eQlhxr)7d^K#-~InWlc&<*#?{A(8f^+C_WmRR{B&Yh3pxhLU9-toLz%rCPi}} zE!cw^pQlXB3aACUpacU&ZlBUl(Jo4fxpbDVwDn^m{VG||ar9B)9}@K`(SJxmAWro& z_3yzfUqLoXg`H($!I;FTudPdo6FTJm2@^S|&42H(XbSRW7!)V&=I`{;mWicu@BT7z zQs!)F9t-K|aFaMsoJ_6z-ICrzjW5#yJRs>~)bugki)ST$8T%!D4F@EBliCNSA5!fl zN;OuKbR3m0rj=rrq}5`nq<<%iHIl|euXt6QA}$hFNqV)oR?_Rm4oPnoLy|ru_DQ-= zJTDFa;zjY2p{sg zWqz0I5y>-U{xR1Rl4r{NQ?6Ge&y@N7t~Vsll=-(^?@FF2^Y6JnkbgW==09{7N}eh4 z?h`%x-LM8D}+*41ZA#EG0D9KQjc2#z59Pq zO9u!y^MeiK3jhHB6_epc9Fs0q7m}w4lLmSnf6Gb(F%*XXShZTmYQ1gTje=G?4qg`Z zf*U~;6hT37na-R}qnQiIv@S#+#J6xEf(swOhZ4_JMMMtdob%^9e?s#9@%jc}19Jk8 z4-eKFdIEVQN4T|=j2t&EtMI{9_E$cx)DHN2-1mG28IEdMq557#dRO3U?22M($g zlriC81f!!ELd`)1V?{MBFnGYPgmrGp{4)cn6%<#sg5fMU9E|fi%iTOm9KgiN)zu3o zSD!J}c*e{V&__#si_#}hO9u$51d|3zY5@QM=aUgu9h0?tNPn1w)HWnB7LQ^GRUjeP z(zSg-y4St;3UIQ}ZX?^;ZtL2n4`>^*Y>Trk?aBtSQ(D-o$(D8Px^?ZI-PUB?*1fv! z{YdHme3Fc8%cR@*@zc5A_nq&2=R47Hp@$-JF4Fz*;SLw5}|ID{W__bHvfJIivHmqmPXlPJd^=<$8K97bHK^(i8eAy)&m< zBc1z)P8b<4NOeqgIeTQpaF|x5YV1#`#T`tctbN+b*?N{~O)bV<K z^y>s-s;V!}b2i=5=M-ComP? zju>8FPIq0VrdV5*EH$|!Ot;e=VudJExcb;2wST}N#u?M~TxGC_!?ccCHCjt|F*PgJ zf@kJB`|Ml}cmsyrAjO#Kjr^E5p29w+#>$C`Q|54BoDv$fQ9D?3n32P9LPMIzu?LjNqggOH=1@T{9bMn*u8(GI!;MGs%MKpd@c!?|2x+D-Rsw10~pU|Rn@A}C1xOlxCribxes0~+n26qDaI zA2$?e`opx3_KW!rAgbpzU)gFdjAKXh|5w``#F0R|c)Y)Du0_Ihhz^S?k^pk%P>9|p zIDx)xHH^_~+aA=^$M!<8K~Hy(71nJG(ov0$3Fg{n+QicHk{UcoFg0-esGM}1X@Ad~ zBS?mZCLw;l4W4a+D8qc)XJS`pUJ5X-f^1ytxwr`@si$lAE?{4G|o;O0l>` zrr?;~c;{ZEFJ!!3=7=FdGJ?Q^xfNQh4A?i;IJ4}B+A?4olTK(fN++3CRBP97jTJnI zF!X$o@{%29Dqq5zt&v4zmF$4E8GqYQko@>U1_;EC_6ig|Drn@=DMV9YEUSCaIf$kH zei3(u#zm9I!Jf(4t`Vm1lltJ&lVHy(eIXE8sy9sUpmz%I_gA#8x^Zv8%w?r2{GdkX z1SkzRIr>prRK@rqn9j2wG|rUv%t7pQ!2SrmOQRpAcS|Wp-{6gg=|^e5#DDOQVM?H4 z;eM-QeRFr06@ifV(ocvk?_)~N@1c2ien56UjWXid6W%6i zevIh)>dk|rIs##^kY67ib8Kw%#-oVFaXG7$ERyA9(NSJUvWiOA5H(!{uOpcWg&-?i zqPhds%3%tFspHDqqr;A!N0fU`!IdoMs=lv7E*9NYeVfBht~=W5wtrfcc#o#+l8s8! z(|NMeqjsy@0x{8^j0d00SqRZjp{Kj)&4UHYGxG+z9b-)72I*&J70?+8e?p_@=>-(> zl6z5vYlP~<2%DU02b!mA{7mS)NS_eLe=CB zc62^$j+OeC%Nkvg?0*n6EKlkPQ)EUvvfC=;4M&*|I!w}(@V_)eUKLA_t^%`o0PM9L zV|UKTLnk|?M3u!|f2S0?UqZsEIH9*NJS-8lzu;A6-rr-ot=dg9SASoluZUkFH$7X;P=?kY zX!K?JL-b~<#7wU;b;eS)O;@?h%sPPk{4xEBxb{!sm0AY|>CXVS(_RT9YPMpChUjl310o*$QocjGdf>jS%%kn_+Y;Ztbauie*k&Q@=9;erLneIoel2C zfCMiPTmYnjjxjV!Ar1h1yQ-31h=b@RZt-play?)#cs=ZxOt;5oX)|*e=7k*ASmQ;r zO4_`=Z&gX-C2$fitvq+iGK1U*^*#IW!Bo{nON%KSxQv@MZsO%Lx21x78z740FSW!f zJ%f-?XMgR#xdurqd6mWyUX2uh=Si>bnwg#gssR#jDVN{uEi3n(PZ%PFZ|6J25_rBf z0-u>e4sFe0*Km49ATi7>Kn0f9!uc|rRMR1Dtt6m1LW8^>qFlo}h$@br=Rmpi;mI&> zOF64Ba2v-pj&TB}f&A09bMg?1id{fne%>Q?9GLm{i~p^lAn!%ZtF$I~>39XVZxk0bROh^B zk9cE0AJBLozZIEmy7xG(yHWGztvfnr0(2ro1%>zsGMS^EMu+S$r=_;9WwZkg z)ww}6KOsH_)RkMh?x@N2R^3(SICQNAzP7(RdB{@@`v*GfeSYLv=cfmTC%s2_T@_Cso2168v@AU^NzL&qv?6hZBJEdb)g=X=dVg9? zYf78=0c@!QU6_a$>CPiXT7QAGDM}7Z(0z#_ZA=fmLUj{2z7@Ypo71UDy8GHr-&TLK zf6a5WCf@Adle3VglBt4>Z>;xF}}-S~B7<(%B;Y0QR55 z{z-buw>8ilNM3u6I+D$S%?)(p>=eBx-HpvZj{7c*_?K=d()*7r74N{MulF2dQ*rGJ8Al=QJ~zb`)MPYedy2kVl9jXxdnmn`&r8ut0w>q?93 zus}1dq%FAFYLsW8ZTQ_XZLh`P2*6(NgS}qGcfGXVWpwsp#Rs}IuKbk*`2}&)I^Vsk z6S&Q4@oYS?dJ`NwMVBs6!1v<013>Q(y%%a0i}Y#1 z-F3m;Ieh#Y12UgW?-R)|eX>ZuF-2cc!1>~NS|XSF-6In>zBoZg+ml!6%fk7Uw0LHc zz8VQk(jOJ+Yu)|^|15ufl$KQd_1eUZZzj`aC%umU6F1&D5XVWcUvDqcUtW@*>xfVd z@!G2_v`obR5 zU*UT{eMHfcUo`jw*u?4r2s_$`}U{?NjvEm(u&<>B|%mq$Q3weshzrh!=m4 zH~yPq{qO0O>o|+xpE_i3$yVP%gs2l20HBh&_;PzZtwMPqQDk4~L}0tfu;d4uxUM8h zx$5GP@d7%rg(9Y8!9@i+9&2l=3<|?le_)g9Z)PQ5ESCo?x4680QstTl-CH_ z5m)j*Epfqj7I|G0-*vpm?U#8&k?((2zg;QYNszIUs?zAIGUr9}em3I$Fhb*w9-ci~gV$1;8(U;p&SDZE^3_CNLX1zM3@E|W%A=rX4; zwOlLm!AP*(*Bl0rL_(L=6`Hv5>_8;g?VljGOuMhr8|fxKG|7jrCnCW}AbEe8A8O*a z;rbQWArFQUVyZaIdGyF7WbZ8lvQ6v;yEgG7uqYA&H#G5ad?wWuhnhHBvUGfsN3K^( zewji7_p=ede8DTP$FEa_M(6|&v8m{z@NJ&XsIgEPpP?ss9mYaeWBd+!UX6vy_yzie z8Vi;2C+U(J3ze}%uZ)Gt_+?D`yc!FY@z?1aYAjU7Z=eB`u~3ZJ#|<)8RL1SxrN%;K zoZ+XHo~5{G1p40!tUgK$I7L3rV9Y8@Eg;`_0Z>Z^2tPilXQ&PU0NNXq;YJ*jtBNjv zYflqF6o%gs=t3z%xd|2&*IQdyR=^LH8WYpRgrrep4Mx6Aw}fxhSE$jN z_`x6Gk20R2MM&C)-R$h{nfE#GnVgwFe}DZ3unAM(^yK7C>62cU)*<-~eOtHo^)=lJ zyq4q2*a>{Y3mU}nkX(`x@nlm*hSem0>o7{ZNZ;OQ5dw>RYT0 zOXvK4;<_A&n$p-%65n=wqR{bejviAOu@}cn>s#w3qd~{|=TQiObS+3ii(WV`2`mPo zZQ7x1xMY3^WvfM@Sq*HPLJh+LQwQ=`ny&P1^Hu$TtXM-zVD=*VoC&`n>n>@37!?>f zN*sy>#GXLvspC8GGlAj!USU^YC|}skAcN~^Xqe0(jqx#zAj>muU<=IUs~34|v06u2 zahGbSeT-uAG|Vv*Bw$#pf8#qXFtMfw|VuC{UeT)2WpJ6&O+E6jF; z;~n9>cf~Ip6j-_@&PGFD0%Vu*QJ@Ht`C7Og!xt#L>mqlJGEh<%*ATJUmZc(FfNSB## zfy_`Y-70r{Iv3jEfR|~Ii!xC44vZ(KNj#>kjsE86E3FB*OayD~$|}3Y&(h6^X|1(TcJ}8{Ua3yL1loSfg!2gTekn ztVO7WNyFQCfwF2ti$UvL8C6{{IPBg01XK~$ThIQx{)~aw>(9F2L#G36*kRDPqA$P* znq=!@bbQ#RzDpVIfYc*x9=}2N^*2z1E%3epP)i30>M4^xlbnuWe_MAGRTTb?O*?TC zw6v5$6bS)qZqo=w4J~*9i;eVx4NwO!crrOjhE8U(&P-ZZU9$We^ubqNd73QDTJqqV z55D;u{1?`JQre~$mu9WZ%=z|x?{A;q|NiAy0GH5U*nIM2xww(4aBEe#)zoy#s-^NN z%WJl5hX=Oj8cnY%e+ZYt5!@FfY;fPO8p2xj+f6?;UE_`~@~KwcX!4d}D<7hA<#M$$ zMY^)MV_$1K4gr3H8yA&|Ten>yr0v!TT@%u$ScDfRrzVR=Rjj3cjDj)fWv?wQanp7L zL)Me^LS6EzBMR%1w^~9L%8&g(G;d3f4uLKFIqs5JYKSlle?R1Fyx?%RURbI;6jq>N zh+(uYf`e8J=hO2&ZQCoTU^AKRV>_^&!W{P-3%oVMaQqOcL1!4cYP)vuF~dMQb1#lK zj_HWu4TgBXPYuJQYWv&8km~(7Mlh=5I8HE}*mJ#?mxhx%#+9e>eorO0)eg#m6uhb7 zG^KSg`Cbxlf9XizZH9>B@hZcqJ*7VTp6)w1tHLB11}(?)MI0$rLIUS0;Z^atECLmz zzb6FE#PKdBl;L{}$M%UdWEi4$AS4ew$#8O?ZRr(G4syuHkcGi8a#*gRz@QP|7R93= zj*A$L;eA}9id+JyWjkK`Mod00;{&DlA!QJFR3&ljf1vI*O1ec{(V=0QA?ELLVls-W z``ELsu7M`3`vI4MzhVcpJ!9#^KGjq|#b-J`!F7h${dUEFmBLuMbYu>nV^(S3q+UC; z7s@e_qZG#+N=oo0o$G1>6Y0a{9@&9;EU2+8k|7P6p?HMh|8#X5UnwpxGbHw;%WXHX zn_~8ne zdvw09V+G$(lhoq7L}=qb+OaPSD&;$TuUtG(4;py(h)8|Nord(*d1ZH-Dmw1MqU&RK ziI)26r-hE(pqnmo4uixe^`qea7(_HA_ zR2KjdJ4$g!)7ve&Q^b1Tf+{(Vd6vInCd>i725IomG^(Ez( zD8L!4qlUAX=)EV9!3JfWLB4n1z)!ums&0UuuVLUHP)i30*5f6tnvk?lbhL{|8I78X7|_c zA3p(L9<~X5y1L3{K8Sf*xL|5gToDT;aYig?m8z^zQ`XdEMJqC#*O|ho!7x~+MzT<5 zg$turF~pS;RSY&GR;6TxR)3Q+&%yG`3&ngIwR*qK&t{TERu@0|fDrKKw3=RE&t-)Xh-$i&l5|>BSn5)z)hg3d?<~8msU=ye z>CHWR!9yT;PU|$KP*qADf(V?zj^n^g~nykv^I)Uz3{78Ty81{n~ZsS&7WH)#Ach3%UyVD1s=Ahvw9*%Wt<42vTt%|niux3Zww13+oK)-d~ zG>VKHM0ov>KXKaUH(Cc)#9GFVSc4EoUbnRudxi}T8J!VNY=4g*Y7C*Ho7#^wUVt&< zKN3&ugs1Ur<767&ea4^1oBw%@h^+YZ+eK^VI5573*KZosq? zpMj(u5257?^lBu&LF9`ao`sYf9&zx;uK2iv&$;8{4nFUSFF5$3JHFuHORo5YgFkV{ zCmcNEicdQDvO7NM;484|f=_+6!)x%g1CL;L9DE%%T=1xaKZ8v-+-@x1OZ;|0_a9J8 z2MFd71j+6K002-1li@}jlN6Rde_awnSQ^R>8l%uQO&WF!6qOdxN;eu7Q-nHAUeckH znK(0P3kdECiu+2%6$MdLP?%OK@`LB_gMXCA`(~0RX;Tm9uJ&d7>n%9A~GP*{Zrpyh7B^|a-)|8b<&(!>OhWQ08 z$LV}WQ`RD4Od8d3O-;%vhK7#W<7u;XvbxQo0JX@fY(C0RS6^zcd>jo287k@<4tg;k z3q5e5hLHE@&4ooC)S|`w7N|jm>3tns$G}U4o!(2g=!}xLHp?+qFvj$ztd<%96=4tCKGG@ADSX{=m zNZ@ho6rr?EOQ1(G2i@2;GXb&S#U3YtCuVwc*4rJcPm$kZf2+|!X~X6%(QMj{4u)mZ zOi!(P(dF3hX4ra9l=RKQ$v(kJFS#;ib+z9K^#Gle6LKa>&4oMFJ4C&NBJ7hhPSIjc zOno$M6iq+l;ExpH9rF68@D3-EgCCf}JJSgVPbI1$?JjPPX!_88InA}KX&=#cFH#s3 zIx<6LeY==wf5DK*jP`hqF%u+|sI)3HfyywfAj=0OMNUX2pLR;T(8c+$g&}Z#q9L>( zD~t~l&X^VFXp@&w92f8tq+KXMZ&o!an%$#uo^hJh^9-RjEvqE_s%H8{qw(juo4?SC z{YhO*`|H*ibxm%ZF6r=2QC)bE`d3oZ(~?;a-(mX) zb!|i%p!VVP>DN6tg*Ry97gUPUJj<}OxaYL1nXE}hxs-O{twImUw43Eo6nJ4_RTDIQALB8H!3nq37 zcE6>oNG;jZZhXh!vORPsMKfzJ8_*?O7DfGmcrL8A(_NAhSH+JE?u?`xR1|ZThDb;2 zDt`9hC;UQ%94^20-MA*;<$KO0{3b&9y(ENIe@&xj6>X23)Ftc?ax=4pL5FZ06CPOj zgG%2*F$-x6 z&si`nj955%8LK)caVl1M8?IPaMPtM85o>MvPUn@(X=!wZq0)at}MK|kJ&KJggGx6y?Ey21qiw~76MoISk z+LyUR=2+oJK1IoYOX~R}S1x>iblZ|_oAmqhyU+NpxvjQb;Ht{pO_xn4T+UO<73|gD zaq0Wtdz^7GoZq-Fu+;61dX%|tud0myO`{vHTlP*oes5OaTBV$=y?3V{mRnFLdQ!Hj z)lErp+uBchtEPv?ao=?feR1oRVaUdpIVC}+xkgTxPYSGDyR2Zw++VdTe(-~Oh=P%c zFD5UUvx;?cLREy~~@9BnQ?{+kh7j7^BGZ3r}vC zuRPgbSbFk*%f8<`nm*%=sYP!wJk1uNV$&qN0K`bt|AMMaWeMf&qirQ!Dt0FDJ8`4KXRTiO^HPz`BO1{-ofSrz0YR`9K0lLHorGM!h0O0Z3yut19ieErkD1!7DO zG~nX@7pO{uE-YFOTtaXT=wTxi=Y>zUU+BjIx>jcL#D!u^>AGNjXBL{vAZ}$~KnuVC z1E3-$;H5MCAlFEP4~z$T=^-$HP(wOqa`hr78Te`EKnLicSpL~^a?K*8$-ft=N<+?q zW?-0u5gn^0TQByPK^#BKz~G2th_L-+o5j*dCr4Ycg3q*_+`m|qNyu^Xvc-|obKpm+ zGBD_)==PZ0utaRK!4gv$&;gX1%nS@qfG$9_!NzrRSv~>`eq9tbPbwj5K&x^fX&o_o$H1U~ zqIOd?L@oQ|Bg^Gwz#}riv?K=%D|r-k8@s@c6Ir1u0~(i50a^-LyMmf7oO;2EvR3Fw zgF8gPQ1=7g{c3<>(&5P)SNO;vnvv+PKQakyh~7$L8Bq2Q1{!dbhk-!@#SpP+P(|#M SXRcJ{65?fGI57uQ5&!`B?F@7P delta 34554 zcmX7vV`H6d(}mmEwr$(CZQE$vU^m*aZQE(=WXEZ2+l}qF_w)XN>&rEBu9;)4xt<3b zo(HR^Mh47P)@z^^pH!4#b(O8!;$>N+S+v5K5f8RrQ+Qv0_oH#e!pI2>yt4ij>fI9l zW&-hsVAQg%dpn3NRy$kb_vbM2sr`>bZ48b35m{D=OqX;p8A${^Dp|W&J5mXvUl#_I zN!~GCBUzj~C%K?<7+UZ_q|L)EGG#_*2Zzko-&Kck)Qd2%CpS3{P1co1?$|Sj1?E;PO z7alI9$X(MDly9AIEZ-vDLhpAKd1x4U#w$OvBtaA{fW9)iD#|AkMrsSaNz(69;h1iM1#_ z?u?O_aKa>vk=j;AR&*V-p3SY`CI}Uo%eRO(Dr-Te<99WQhi>y&l%UiS%W2m(d#woD zW?alFl75!1NiUzVqgqY98fSQNjhX3uZ&orB08Y*DFD;sjIddWoJF;S_@{Lx#SQk+9 zvSQ-620z0D7cy8-u_7u?PqYt?R0m2k%PWj%V(L|MCO(@3%l&pzEy7ijNv(VXU9byn z@6=4zL|qk*7!@QWd9imT9i%y}1#6+%w=s%WmsHbw@{UVc^?nL*GsnACaLnTbr9A>B zK)H-$tB`>jt9LSwaY+4!F1q(YO!E7@?SX3X-Ug4r($QrmJnM8m#;#LN`kE>?<{vbCZbhKOrMpux zTU=02hy${;n&ikcP8PqufhT9nJU>s;dyl;&~|Cs+o{9pCu{cRF+0{iyuH~6=tIZXVd zR~pJBC3Hf-g%Y|bhTuGyd~3-sm}kaX5=T?p$V?48h4{h2;_u{b}8s~Jar{39PnL7DsXpxcX#3zx@f9K zkkrw9s2*>)&=fLY{=xeIYVICff2Id5cc*~l7ztSsU@xuXYdV1(lLGZ5)?mXyIDf1- zA7j3P{C5s?$Y-kg60&XML*y93zrir8CNq*EMx)Kw)XA(N({9t-XAdX;rjxk`OF%4-0x?ne@LlBQMJe5+$Ir{Oj`@#qe+_-z!g5qQ2SxKQy1ex_x^Huj%u+S@EfEPP-70KeL@7@PBfadCUBt%`huTknOCj{ z;v?wZ2&wsL@-iBa(iFd)7duJTY8z-q5^HR-R9d*ex2m^A-~uCvz9B-1C$2xXL#>ow z!O<5&jhbM&@m=l_aW3F>vjJyy27gY}!9PSU3kITbrbs#Gm0gD?~Tub8ZFFK$X?pdv-%EeopaGB#$rDQHELW!8bVt`%?&>0 zrZUQ0!yP(uzVK?jWJ8^n915hO$v1SLV_&$-2y(iDIg}GDFRo!JzQF#gJoWu^UW0#? z*OC-SPMEY!LYcIZO95!sv{#-t!3Z!CfomqgzFJld>~CTFKGcr^sUai5s-y^vI5K={ z)cmQthQuKS07e8nLfaIYQ5f}PJQqcmokx?%yzFH*`%k}RyXCt1Chfv5KAeMWbq^2MNft;@`hMyhWg50(!jdAn;Jyx4Yt)^^DVCSu?xRu^$*&&=O6#JVShU_N3?D)|$5pyP8A!f)`| z>t0k&S66T*es5(_cs>0F=twYJUrQMqYa2HQvy)d+XW&rai?m;8nW9tL9Ivp9qi2-` zOQM<}D*g`28wJ54H~1U!+)vQh)(cpuf^&8uteU$G{9BUhOL| zBX{5E1**;hlc0ZAi(r@)IK{Y*ro_UL8Ztf8n{Xnwn=s=qH;fxkK+uL zY)0pvf6-iHfX+{F8&6LzG;&d%^5g`_&GEEx0GU=cJM*}RecV-AqHSK@{TMir1jaFf&R{@?|ieOUnmb?lQxCN!GnAqcii9$ z{a!Y{Vfz)xD!m2VfPH=`bk5m6dG{LfgtA4ITT?Sckn<92rt@pG+sk>3UhTQx9ywF3 z=$|U(bN<=6-B4+UbYWxfQUOe8cmEDY3QL$;mOw&X2;q9x9qNz3J97)3^jb zdlzkDYLKm^5?3IV>t3fdWwNpq3qY;hsj=pk9;P!wVmjP|6Dw^ez7_&DH9X33$T=Q{>Nl zv*a*QMM1-2XQ)O=3n@X+RO~S`N13QM81^ZzljPJIFBh%x<~No?@z_&LAl)ap!AflS zb{yFXU(Uw(dw%NR_l7%eN2VVX;^Ln{I1G+yPQr1AY+0MapBnJ3k1>Zdrw^3aUig*! z?xQe8C0LW;EDY(qe_P!Z#Q^jP3u$Z3hQpy^w7?jI;~XTz0ju$DQNc4LUyX}+S5zh> zGkB%~XU+L?3pw&j!i|x6C+RyP+_XYNm9`rtHpqxvoCdV_MXg847oHhYJqO+{t!xxdbsw4Ugn($Cwkm^+36&goy$vkaFs zrH6F29eMPXyoBha7X^b+N*a!>VZ<&Gf3eeE+Bgz7PB-6X7 z_%2M~{sTwC^iQVjH9#fVa3IO6E4b*S%M;#WhHa^L+=DP%arD_`eW5G0<9Tk=Ci?P@ z6tJXhej{ZWF=idj32x7dp{zmQY;;D2*11&-(~wifGXLmD6C-XR=K3c>S^_+x!3OuB z%D&!EOk;V4Sq6eQcE{UEDsPMtED*;qgcJU^UwLwjE-Ww54d73fQ`9Sv%^H>juEKmxN+*aD=0Q+ZFH1_J(*$~9&JyUJ6!>(Nj zi3Z6zWC%Yz0ZjX>thi~rH+lqv<9nkI3?Ghn7@!u3Ef){G(0Pvwnxc&(YeC=Kg2-7z zr>a^@b_QClXs?Obplq@Lq-l5>W);Y^JbCYk^n8G`8PzCH^rnY5Zk-AN6|7Pn=oF(H zxE#8LkI;;}K7I^UK55Z)c=zn7OX_XVgFlEGSO}~H^y|wd7piw*b1$kA!0*X*DQ~O` z*vFvc5Jy7(fFMRq>XA8Tq`E>EF35{?(_;yAdbO8rrmrlb&LceV%;U3haVV}Koh9C| zTZnR0a(*yN^Hp9u*h+eAdn)d}vPCo3k?GCz1w>OOeme(Mbo*A7)*nEmmUt?eN_vA; z=~2}K_}BtDXJM-y5fn^v>QQo+%*FdZQFNz^j&rYhmZHgDA-TH47#Wjn_@iH4?6R{J z%+C8LYIy>{3~A@|y4kN8YZZp72F8F@dOZWp>N0-DyVb4UQd_t^`P)zsCoygL_>>x| z2Hyu7;n(4G&?wCB4YVUIVg0K!CALjRsb}&4aLS|}0t`C}orYqhFe7N~h9XQ_bIW*f zGlDCIE`&wwyFX1U>}g#P0xRRn2q9%FPRfm{-M7;}6cS(V6;kn@6!$y06lO>8AE_!O z{|W{HEAbI0eD$z9tQvWth7y>qpTKQ0$EDsJkQxAaV2+gE28Al8W%t`Pbh zPl#%_S@a^6Y;lH6BfUfZNRKwS#x_keQ`;Rjg@qj zZRwQXZd-rWngbYC}r6X)VCJ-=D54A+81%(L*8?+&r7(wOxDSNn!t(U}!;5|sjq zc5yF5$V!;%C#T+T3*AD+A({T)#p$H_<$nDd#M)KOLbd*KoW~9E19BBd-UwBX1<0h9 z8lNI&7Z_r4bx;`%5&;ky+y7PD9F^;Qk{`J@z!jJKyJ|s@lY^y!r9p^75D)_TJ6S*T zLA7AA*m}Y|5~)-`cyB+lUE9CS_`iB;MM&0fX**f;$n($fQ1_Zo=u>|n~r$HvkOUK(gv_L&@DE0b4#ya{HN)8bNQMl9hCva zi~j0v&plRsp?_zR zA}uI4n;^_Ko5`N-HCw_1BMLd#OAmmIY#ol4M^UjLL-UAat+xA+zxrFqKc@V5Zqan_ z+LoVX-Ub2mT7Dk_ z<+_3?XWBEM84@J_F}FDe-hl@}x@v-s1AR{_YD!_fMgagH6s9uyi6pW3gdhauG>+H? zi<5^{dp*5-9v`|m*ceT&`Hqv77oBQ+Da!=?dDO&9jo;=JkzrQKx^o$RqAgzL{ zjK@n)JW~lzxB>(o(21ibI}i|r3e;17zTjdEl5c`Cn-KAlR7EPp84M@!8~CywES-`mxKJ@Dsf6B18_!XMIq$Q3rTDeIgJ3X zB1)voa#V{iY^ju>*Cdg&UCbx?d3UMArPRHZauE}c@Fdk;z85OcA&Th>ZN%}=VU%3b9={Q(@M4QaeuGE(BbZ{U z?WPDG+sjJSz1OYFpdImKYHUa@ELn%n&PR9&I7B$<-c3e|{tPH*u@hs)Ci>Z@5$M?lP(#d#QIz}~()P7mt`<2PT4oHH}R&#dIx4uq943D8gVbaa2&FygrSk3*whGr~Jn zR4QnS@83UZ_BUGw;?@T zo5jA#potERcBv+dd8V$xTh)COur`TQ^^Yb&cdBcesjHlA3O8SBeKrVj!-D3+_p6%P zP@e{|^-G-C(}g+=bAuAy8)wcS{$XB?I=|r=&=TvbqeyXiuG43RR>R72Ry7d6RS;n^ zO5J-QIc@)sz_l6%Lg5zA8cgNK^GK_b-Z+M{RLYk5=O|6c%!1u6YMm3jJg{TfS*L%2 zA<*7$@wgJ(M*gyTzz8+7{iRP_e~(CCbGB}FN-#`&1ntct@`5gB-u6oUp3#QDxyF8v zOjxr}pS{5RpK1l7+l(bC)0>M;%7L?@6t}S&a zx0gP8^sXi(g2_g8+8-1~hKO;9Nn%_S%9djd*;nCLadHpVx(S0tixw2{Q}vOPCWvZg zjYc6LQ~nIZ*b0m_uN~l{&2df2*ZmBU8dv`#o+^5p>D5l%9@(Y-g%`|$%nQ|SSRm0c zLZV)45DS8d#v(z6gj&6|ay@MP23leodS8-GWIMH8_YCScX#Xr)mbuvXqSHo*)cY9g z#Ea+NvHIA)@`L+)T|f$Etx;-vrE3;Gk^O@IN@1{lpg&XzU5Eh3!w;6l=Q$k|%7nj^ z|HGu}c59-Ilzu^w<93il$cRf@C(4Cr2S!!E&7#)GgUH@py?O;Vl&joXrep=2A|3Vn zH+e$Ctmdy3B^fh%12D$nQk^j|v=>_3JAdKPt2YVusbNW&CL?M*?`K1mK*!&-9Ecp~>V1w{EK(429OT>DJAV21fG z=XP=%m+0vV4LdIi#(~XpaUY$~fQ=xA#5?V%xGRr_|5WWV=uoG_Z&{fae)`2~u{6-p zG>E>8j({w7njU-5Lai|2HhDPntQ(X@yB z9l?NGoKB5N98fWrkdN3g8ox7Vic|gfTF~jIfXkm|9Yuu-p>v3d{5&hC+ZD%mh|_=* zD5v*u(SuLxzX~owH!mJQi%Z=ALvdjyt9U6baVY<88B>{HApAJ~>`buHVGQd%KUu(d z5#{NEKk6Vy08_8*E(?hqZe2L?P2$>!0~26N(rVzB9KbF&JQOIaU{SumX!TsYzR%wB z<5EgJXDJ=1L_SNCNZcBWBNeN+Y`)B%R(wEA?}Wi@mp(jcw9&^1EMSM58?68gwnXF` zzT0_7>)ep%6hid-*DZ42eU)tFcFz7@bo=<~CrLXpNDM}tv*-B(ZF`(9^RiM9W4xC%@ZHv=>w(&~$Wta%)Z;d!{J;e@z zX1Gkw^XrHOfYHR#hAU=G`v43E$Iq}*gwqm@-mPac0HOZ0 zVtfu7>CQYS_F@n6n#CGcC5R%4{+P4m7uVlg3axX}B(_kf((>W?EhIO&rQ{iUO$16X zv{Abj3ZApUrcar7Ck}B1%RvnR%uocMlKsRxV9Qqe^Y_5C$xQW@9QdCcF%W#!zj;!xWc+0#VQ*}u&rJ7)zc+{vpw+nV?{tdd&Xs`NV zKUp|dV98WbWl*_MoyzM0xv8tTNJChwifP!9WM^GD|Mkc75$F;j$K%Y8K@7?uJjq-w zz*|>EH5jH&oTKlIzueAN2926Uo1OryC|CmkyoQZABt#FtHz)QmQvSX35o`f z<^*5XXxexj+Q-a#2h4(?_*|!5Pjph@?Na8Z>K%AAjNr3T!7RN;7c)1SqAJfHY|xAV z1f;p%lSdE8I}E4~tRH(l*rK?OZ>mB4C{3e%E-bUng2ymerg8?M$rXC!D?3O}_mka? zm*Y~JMu+_F7O4T;#nFv)?Ru6 z92r|old*4ZB$*6M40B;V&2w->#>4DEu0;#vHSgXdEzm{+VS48 z7U1tVn#AnQ3z#gP26$!dmS5&JsXsrR>~rWA}%qd{92+j zu+wYAqrJYOA%WC9nZ>BKH&;9vMSW_59z5LtzS4Q@o5vcrWjg+28#&$*8SMYP z!l5=|p@x6YnmNq>23sQ(^du5K)TB&K8t{P`@T4J5cEFL@qwtsCmn~p>>*b=37y!kB zn6x{#KjM{S9O_otGQub*K)iIjtE2NfiV~zD2x{4r)IUD(Y8%r`n;#)ujIrl8Sa+L{ z>ixGoZJ1K@;wTUbRRFgnltN_U*^EOJS zRo4Y+S`cP}e-zNtdl^S5#%oN#HLjmq$W^(Y6=5tM#RBK-M14RO7X(8Gliy3+&9fO; zXn{60%0sWh1_g1Z2r0MuGwSGUE;l4TI*M!$5dm&v9pO7@KlW@j_QboeDd1k9!7S)jIwBza-V#1)(7ht|sjY}a19sO!T z2VEW7nB0!zP=Sx17-6S$r=A)MZikCjlQHE)%_Ka|OY4+jgGOw=I3CM`3ui^=o0p7u z?xujpg#dRVZCg|{%!^DvoR*~;QBH8ia6%4pOh<#t+e_u!8gjuk_Aic=|*H24Yq~Wup1dTRQs0nlZOy+30f16;f7EYh*^*i9hTZ`h`015%{i|4 z?$7qC3&kt#(jI#<76Biz=bl=k=&qyaH>foM#zA7}N`Ji~)-f-t&tR4^do)-5t?Hz_Q+X~S2bZx{t+MEjwy3kGfbv(ij^@;=?H_^FIIu*HP_7mpV)NS{MY-Rr7&rvWo@Wd~{Lt!8|66rq`GdGu% z@<(<7bYcZKCt%_RmTpAjx=TNvdh+ZiLkMN+hT;=tC?%vQQGc7WrCPIYZwYTW`;x|N zrlEz1yf95FiloUU^(onr3A3>+96;;6aL?($@!JwiQ2hO|^i)b4pCJ7-y&a~B#J`#FO!3uBp{5GLQfhOAOMUV7$0|d$=_y&jl>va$3u-H z_+H*|UXBPLe%N2Ukwu1*)kt!$Y>(IH3`YbEt; znb1uB*{UgwG{pQnh>h@vyCE!6B~!k}NxEai#iY{$!_w54s5!6jG9%pr=S~3Km^EEA z)sCnnau+ZY)(}IK#(3jGGADw8V7#v~<&y5cF=5_Ypkrs3&7{}%(4KM7) zuSHVqo~g#1kzNwXc39%hL8atpa1Wd#V^uL=W^&E)fvGivt)B!M)?)Y#Ze&zU6O_I?1wj)*M;b*dE zqlcwgX#eVuZj2GKgBu@QB(#LHMd`qk<08i$hG1@g1;zD*#(9PHjVWl*5!;ER{Q#A9 zyQ%fu<$U?dOW=&_#~{nrq{RRyD8upRi}c-m!n)DZw9P>WGs>o1vefI}ujt_`O@l#Z z%xnOt4&e}LlM1-0*dd?|EvrAO-$fX8i{aTP^2wsmSDd!Xc9DxJB=x1}6|yM~QQPbl z0xrJcQNtWHgt*MdGmtj%x6SWYd?uGnrx4{m{6A9bYx`m z$*UAs@9?3s;@Jl19%$!3TxPlCkawEk12FADYJClt0N@O@Pxxhj+Kk(1jK~laR0*KGAc7%C4nI^v2NShTc4#?!p{0@p0T#HSIRndH;#Ts0YECtlSR}~{Uck+keoJq6iH)(Zc~C!fBe2~4(Wd> zR<4I1zMeW$<0xww(@09!l?;oDiq zk8qjS9Lxv$<5m#j(?4VLDgLz;8b$B%XO|9i7^1M;V{aGC#JT)c+L=BgCfO5k>CTlI zOlf~DzcopV29Dajzt*OcYvaUH{UJPaD$;spv%>{y8goE+bDD$~HQbON>W*~JD`;`- zZEcCPSdlCvANe z=?|+e{6AW$f(H;BND>uy1MvQ`pri>SafK5bK!YAE>0URAW9RS8#LWUHBOc&BNQ9T+ zJpg~Eky!u!9WBk)!$Z?!^3M~o_VPERYnk1NmzVYaGH;1h+;st==-;jzF~2LTn+x*k zvywHZg7~=aiJe=OhS@U>1fYGvT1+jsAaiaM;) zay2xsMKhO+FIeK?|K{G4SJOEt*eX?!>K8jpsZWW8c!X|JR#v(1+Ey5NM^TB1n|_40 z@Db2gH}PNT+3YEyqXP8U@)`E|Xat<{K5K;eK7O0yV72m|b!o43!e-!P>iW>7-9HN7 zmmc7)JX0^lPzF#>$#D~nU^3f!~Q zQWly&oZEb1847&czU;dg?=dS>z3lJkADL1innNtE(f?~OxM`%A_PBp?Lj;zDDomdw zoC=eKBnzA5DamDVIk!-AoSMv~QchAOt&5fk#G=s!$FD}9rL0yDjwDkw<9>|UUuyVm z&o7y|6Ut5WI0!G$M?NiMUy%;s3ugPKJU_+B!Z$eMFm}A**6Z8jHg)_qVmzG-uG7bj zfb6twRQ2wVgd)WY00}ux=jqy@YH4ldI*;T^2iAk+@0u`r_Fu(hmc3}!u-Pb>BDIf{ zCNDDv_Ko`U@})TZvuE=#74~E4SUh)<>8kxZ=7`E?#|c zdDKEoHxbEq;VVpkk^b&~>-y`uO~mX=X0bmP!=F1G1YiluyeEg!D*8Fq-h=NyE-2S;^F6j=QMtUzN4oPedvc*q(BCpbg~*As!D@U z3(sz|;Pe1hn08P_cDQ(klZ6 z;P`q(5_V?*kJYBBrA1^yDgJD|)X1FV_*~sO>?8Sy~I9WdK5K8bc7aeNC zDb{Fe>y3N^{mrD1+GyH{F?@9}YQ2Om3t`nt zQ(}MS8M?6Vk>B=*j*yibz6QCdR=ALgTUcKx61){O@1WkPp-v$$4}e#KgK`HG~2@#A?`BF8em`ah6+8hH-DNA2>@02WWk9(fzhL_iz|~H~qEViQ(*{ zV;3tjb<%&r!whm6B`XtWmmrMWi=#ZO&`{h9`->HVxQ)^_oOS{W z!BzVRjdx5@pCXl#87ovlp<^QU;s<*d$)+|vI;Ai(!8Tjll^mi6!o~CpnlgZAK>6=V zm38^kT`D$_$v@UYeFyVhnsMZI1m`E&8<{V07>bBEI1=fg3cji*N?7pBzuamD`X|^^ zm!)2v?s|6T&H-_^y`KM&$!0!9tai9x&)5<(&sY6B`3D{$$KMAX3@&`SW;X0 zB-}obt^I;|#o_bR>eOv?P>=UC6CGTXIM+lSu?Uy+R9~O;q|c2+FafBP;E)B5M9HJgRIpF|GvRi*E+JTBI~T?T*X}r) zefUd*(+3n_YHZZS(g8)+7=pNV9QR^>Qs8t+iEpbJS!9;wio&9rn=19C0G#Ax zM-tWHp_YlJvXWsUqJUr^`OYFA4wkgL`cSOV;w4?tp>GT1jq}-qPoN zp&G}*;+#+Zh&vqDOp>gRL#^O7;s2yWqs+U4_+R4`{l9rEt-ud(kZ*JZm#0M{4K(OH zb<7kgkgbakPE=G&!#cNkvSgpU{KLkc6)dNU$}BQelv+t+gemD5;)F-0(%cjYUFcm{ zxaUt??ycI({X5Gkk@KIR$WCqy4!wkeO_j)?O7=lFL@zJDfz zrJJRDePaPzCAB)hPOL%05T5D*hq|L5-GG&s5sB97pCT23toUrTxRB{!lejfX_xg(y z;VQ+X91I;EUOB;=mTkswkW0~F$ zS%M}ATlKkIg??F?I|%gdYBhU(h$LqkhE!Xx$7kPS{2U4wLujF_4O+d8^ej{ zgSo(;vA)|(KT8R_n_aQ$YqDQaI9Stqi7u=+l~~*u^3-WsfA$=w=VX6H%gf!6X|O#X z*U6Wg#naq%yrf&|`*$O!?cS94GD zk}Gx%{UU!kx|HFb+{f(RA2h+t#A!32`fxL}QlXUM{QF3m&{=7+hz@aXMq*FirZk?W zoQ~ZCOx>S?o>3`+tC&N0x4R`%m)%O$b@BkW;6zE+aBzeYi47~78w$d~uypaV*p$kQ zJf34Q+pp~vg6)yeTT&qWbnR2|SifwK2gA7fzy#W(DyM^bdCjnee42Ws>5mM9W6_`j zC(|n5Fa&=MT$$@?p~)!IlLezYa}=Uw21^Fz-I#?_AOk(7Ttxm;#>RDD_9EloqhvrS z&7fpbd$q_e21Al+bcz|o{(^p}AG>jX0B}ZZRfzk$WLbNLC{y|lZ|&a(=bOE6Mxum{ zM=Nd+-I2A-N&2giWM2oAH`O&QecJn6%uYl0GWlpx&2*)BIfl3h&2E(>#ODt4oG}Dq z__73?sw2-TOWq@d&gmYKdh`a}-_6YQ5```}bEBEmWLj))O z?*eUM4tw0Cwrr+4Ml^9JkKW9e4|_^oal0*sS-u_Xovjo8RJ18x_m7v!j$eR@-{2(Y z?&K4ZR8^T{MGHL#C(+ZAs6&k}r07Xqo1WzaMLo9V;I<9a6jx2wH2qeU?kv25MJxoj zJKzX`Un|;_e&KY%R2jU~<5lm-`$EjIJLDP~11_5?&W#t3I{~+0Ze++pOh2B4c1Mde zSgj$ODQQm7gk&w{wwfE1_@V(g!C=2Hd%Gwj{{-_K4S|nZu+vk}@k(?&13iccsLkQo z_t8#Ah$HVB-MRyzpab*OHOp zl`$tEcUcF9_=3*qh8KTaW$znGztA7Obzb`QW5IQN+8XC=l%+$FVgZ|*XCU?G4w)}! zmEY+2!(!%R5;h`>W(ACqB|7`GTSp4{d)eEC8O)Mhsr$dQG}WVBk$aN1->sTSV7E)K zBqr;^#^bZJJX4E_{9gdPo8e?Ry>ZrE&qM)zF5z20DP0`)IIm_!vm&s2mzl z2;EPI{HgFH-Mp&fIL^6f74>19^>o^AOj`uyL0+Nb##Slvi9K4LQSs>f+$j?cn9Z__C zAkyZ9C;#uRi3cDYoTA>AT<|*pt{K70oZKG*S1F$r?KE=$4~W3!u53yUvh~(kMrClS zXC?Dmgv4iS`>~wBPJJFL_C8x2tEg*PCDX2=rHQ@z+Zs)Kkr;FYG`GnbUXqdipzvHE z1aZ>G6|e`}Q#)Kru0)(SZnUCN#dN2H zd1}r&xGsaAeEed9#?|0HzMGA7pl2=aehy_zsRV8RKV6+^I8woDd%4J8v9hs$x{ zl*V61wSumovRVWtetd1eJ%i^#z`_~~^B;aeuD`6LgHL66F0b^G5@om^&_3REtGmhz z%j^9{U`BH7-~P_>c_yu9sE+kk)|2`C)-ygYhR?g~gH`OK@JFAGg0O)ng-JzSZMjw< z2f&vA7@qAhrVyoz64A!JaTVa>jb5=I0cbRuTv;gMF@4bX3DVV#!VWZEo>PWHeMQtU!!7ptMzb{H ze`E4ZG!rr4A8>j2AK(A0Vh6mNY0|*1BbLhs4?>jmi6fRaQwed-Z?0d=eT@Hg zLS(%af5#q%h@txY2KaYmJBu>}ZESUv-G02~cJ-(ADz6u8rLVECbAR7+KV~a!DI83H zd!Z(Ekz%vjA-|%4-YpgfymMzxm_RjZg%ruo zT4^x)f*%Ufvg_n`&55cK;~QChP6~Fy_Z67HA`UtdW)@$Xk-2+|opk6A@y0~3Qb;V% z%+B@ArKl|Q^DJW&xuBZD#~SurH7XXf*uE0@|ccNd&MA%Ts*1 zg7TU!xY}~*AOY+tAnFR(Fu)e@^9V!Rm65$;G$-?6e%7w7p9WT098%-R?u#J+zLot@ z4H7R>G8;q~_^uxC_Z=-548YRA`r`CsPDL!^$v0Yy<^KSoKwiJaCt&dlW?p^7Y_<9c z3n#cMWFUe@W@4ffE`}pQduRZ)I5v`G8On2RI zL)V5k)PMBq(Zfb6Ruig;_SMwaM9t)2JfUafW-6F8V+PjKM#9iD1~v!uOfWiNL=R_j z$xKbCPfuiw`kKN1U{W6p#s!Vo+Suw#*7O24y`hNTmrEqDkQvZ}tMO{2`r|3XNXJwC zSUqB-GdK(D8yYTd*bs~vM{3@r5;JMtW-c8ywtvPG2Gepg-QU=s)?*2y@n~8f95m96 z+pO1p_FIP@Pbnlb&AnDXqBkb=RDa{H-fN9$Rv{OYoWwrU{J??m#C~^HFtMrjN~Spz zt1SsVlTk=x^7b3q-DxumB4DxAv}x1?YHb=BBbrOcvqOzjVK#ZlL$frhpxI1I&JL^4 zTz{rnIH(26vL$9Zf7%ffyC7agUX3bg9@D~^pcIOgp^SvS@0_fS0rHL9Zq*vjT4ZZ-;< zjl1>i0E~DMlLHLFe*&dK6lIzW57ySu#Tu=qwMh#+h*$yk2HIFb z>nT*!OJPT$OPLhmOCaK*%WUy42dzuvsd)CXDdLTLrH7iRS)E$Zzgab4TrcDG#Hg058>HuG9V=$qMph{<;l?`Ri zEyGDUBkrQzLi1NJtvoj(mN?yl$vw8i+u{fXdFV>oD0cQS`6mT>G!chOCzE!M}POG4yVkcsa=D@;o&t554oCp+<>_TZ~ZFu!frP4 zU=Fl`17;Hbhh*q72kj_XUp7O8XXeU24I1gAe!Z;8OmghWKbAdr6WwUEq^k(Y&_8z zj%SeljzOqyBkQ*T{RNL0@|%7B?116lab<@;U^MhM_=By8;asX*oe`l13GJ8z5* z5VjTi4+vl>1TM8OFqzvHGm)^9If&dr@6zaY`cEcbpgfH2v+vgE7J84UMd4{&7eL;p z(c9_$OzU1R7?w91eP-GY=k8o@VPB!Un6?GZ;t-tik9u# zvqoC)70K;GOln-bWzDpZYO;db3+qtNN9djk`Y?U8NTp<7p^qb*p}pudj%BUzM(7UH zy%qEc`XuT^%33b1Ck5~E(5L7=0rzR9`q$N${pil>S#W+o{57c$^%{6jXLl7mylgTC zJD;ToHF|(P$0P-VDu1113cl`fO??oskdG7^5dmB%MB4r5SOQ*GRGZ)={o>ds z>9kPUQ%r0Ab$o@MK{hL}EBvA<4GAv_oC7bVTzr|H)#yv~6@O3*T%M^d=yP+!DwVzl zmBv#szT%!L@ zp@s&_ia!GxNcwyFgCOxoHX+X@7dgvR{(Rc?n~*xScUt%qyo=g)w5da7a@kfkHC5f{IFx%*o4ng~rPm)5Yw; zw2^`5jQ4|6i@zwi9u9D=8;Zrap%z2I!`5JN3kOAh$h0K~vqK(kg#U3hW2TTZ@#_r_ zuYrSM;o@m|cf2&M;Y$Pr=7tL7cfFCjZdTPi91>|OQHV-$Uwc{<^Jl;4rh{n0WYMi;%o-qsd8G>t` zQ-2D8(zo(95gXe{3}cf6_?9yO@>*O2@DnMi0IM0|s|7 zttz7!JH98}Y&!xefmFwP>`Q>D`_oUYE!S7_mAp^my?hl~!ZN3Z&HjFI$bM0J_S;+@ z)c61&5|i&S#33B9Mvme=0gk(Yj(KKL8KhQ>V+m7_DV!+plI5r>jJ{+xCiSCc z`tY83(lA9*;dT!X@^x-D8ExhQ@OlJNOt(y3UP_9ldOS+k8hnRVig8sESest%o% z;j}Clsg_Ca5_>KG)G$OIMXfS(ocFQ<>%6$;u%x@EBc{_~MsPZjH3YcHB?RH<~ z;dk0a0@D>EH({DmGJ2n}HyvkMGJnIh%sA;g_+3K57^-Gv&8F^__Vz-f!0)!MQ5b`i zqoef_mEQ*sEWHiuFftjv-)N2Z8=|Bgx097+l$5w-TRn5KDo+Fae1PxP_%6mQq=HuS zP*%8{9H>3e?BNgbhlQLUK_uk{V@U3p*8>NdMN#@Fe@vi#yja%I#t$?$$AA0VQ(42x z0mDFwS%-M|lb{3O|He|F-NJ`0?$h{Q{SHul5z+L*m&!#!fJJqj;3jztr>O#Fy-E!z~0 zLOmUN3K~L8HkR|Nwiywi&40)E3vRgB<4otz96rleEBpjg`mCW*>Nn*WDNrlBS2nlV zdOxl4ll+uzZtGeG6`^DdE!@@cGyElu6#g>Yp&=1HtTN^eSMqQSqq&E_W@quQ!v*8$ z+|%d|%rshx=j?UN8s|+=?8>FG$a<4ngKuN*X)$w&m{snhX#>vXAAhv&&-}3>HGiL( z_9x8fVZXSs^sD>=(;RT!)SEFAxvXK^@SkiV<(^P-nfQ+mo2Io4{LcX;>*{6kT1 zf8-?bXHN4L2l2NaD^3zncNc1-nY1lw-EQ*FFcGJZs{9L$e=aJlCR8<`r&0!z{?fpt ztJbK!nz3wF0D;ur zV^Cy@9RmCxjK=X*#$+N#;gcRdLx}GuB`W$sS&0-$g7}56F@GLO#-t)SB+Mj^M7&p( z6cp|#ig#l@GT+ik-Xx2!!l_e8s;ehRK%E%3_0F#P1+Hc zYSW_5-U2TRC4ZkLEs)OhP@Dbhd?Cw$($5_;U|V4>EzzV(=>k+4Eezv|b9qyP_f% zJ<_EjASxvcKW!7qG9kWy8P-j=tyX_g&Hf!tUH*8gxIDQ$`d6;VtZYyv@r?#q71eqQ zuVwU8hJV-Mv?Dc1&FBmyML`_H0h2++J;ImVNPoF!}q{<%zspm zX8~m8`|*10*R2fZ&ze^H4}rQEqeM{`zr#4%AJ6!6_9qfm>cr6#TEf6N09|0P_S;v9 z5PmmirL$iSA{@-4#TOxVGx|!+=_0&Hxs(;xvNvL&VY_&!l9JH6|vKHhzEX6SO zrIYcL;g1S;8$`*n#4IE;{|-Iv?@OCWf7FZ_y^yVFseR%m<}9p51Z(??En=Zh=pMqj ze{7=8N(YOdYb_d`rseakM&DL5mx|f;i}F&b&b&8JY8k~4Uf_O$iai1BXmeU zNxJh9s*6M%Rncy_%IMBhysGXbnZ?!Xuz#8ntNV&8IjkHNE0L-p09L)>B;7blH;>WV zBO!T=Zixg>&~16TbA;YILdVDG1Cfw3=#xk2gAdWim_ja}>mfoTdz?@EoZ|Oqm>vV^ zkdmhp$NA$vr7ADPq{=ZG1+G9H8$Rw{GzH3e!l(4)>FGRuHRK#VbAKQ9 zzi#a}i2b>n^YpEC0Bo1` zLID4d1?(E8iZS|GWQ2ZxDhM<{hEz!HQ}gtz<1|mu62FVQ%?%c4hui|nZ9%=o=NzM# zB0hId)o(}WcX@g_Pk#}6PebTD{eS&9d5ePDY`pf24==BVoX&M>wd#YqUc2YDlRjs) zDqkZctyV2jL#jnqEg@?&^J)knJ~ada!)H#xPI@V`uZmNmGxAjcXcicGX7PKSPX<#g zkFwS|Mz@3W5w57p<$3lA_U3v1gte)?#MWM3nCC^2b?V(zDd>55ah{j%8-G6YoX--) zr#PxrA&nwmQ!ur){W+f;35p|ERz-!Lc=o;%TqhP9j#IY}4!Akwtcqei5^`BQtd?&Q zK4HJCl|M=ggxlfGk>~Yb22nFi#u#smczM$ZUwX>^d71e6Ah+!Ea@#1k^- zbokLQ!dK^6Kkj&9jH8iA{TMHcjBsp(`%m!UjxkOGJXn8%GqA)cAMF|8>&N(wkq$)O z7~cSr&bkqPb8v*;3iwFp34Vv5Pg}sSmv7DUZIN}#-NLbF`&`ww&VPmNynK6cPlHU# zFwOG09My_tnP3EDM)}S>zc-|M`Te8(!AQsrU*dc6{E0EX7fvLv!|SK2RWS6Kxy$qX zfaO~XUOx-Z5=Ya^J+_a96k$B|1fKvE=+#OBn$H<>55q^WVx(5L#`f>KZr zI>8T((-L7Jh(V!(nt%HQe?Ah@iqzabXIO}+6^X5^_qppP5js^$sPNM@PV)qRag3jg zgnbaxC)Y!tPv`krD+Nb7M37unh#gD59TthNj$>mx(wXOP+(oN{!k9D*k8fG|#6QN* zM+9ztkC(qA;*P&p#QXj!?&J_+?8o!?CrK~=^k#j%lS7J6d4G!b7FOpw-+ec2ALE}# ztl;`(JvjJPo_}k3(VrrnPtg*DIcU6szm@d#&7=IO+);m;_KZoDk%M7CROO}W4*3yU9C6flk4lU3(&7=xKPoN9$pNpl zDlau)w;~dDc%_TFz0zu|UxF0{E33L0Z=3ezrOQ4m^kyyZbkqTC%c@bSRj6zl^W1r= zsACw%D{Zxm^V7W4?v-{5E4xcnzA9MM);O9^>+wn*c7IOvO1mat#{t|k0PGYHUg?Te zBhsEzlQ^yi$5$3Po+8Or#dQlAm{o6SPc$)6{MSG`t;S{}Nwk|Bw4Y=$(D1~` zMMG$NZbZZLE;Ks#kVdGb^hxs2eKd>ir`hy1nnTagT-KhaQJDVV+HvfwRE0i9W8RS(D{ztwAe8~OMe_Gy1?;P@;lx^OC8^&8pq#gne3qD zvO+85Idq|1MJwe11>}0FmDkcLc|Fz1O;j&mMM3!xHONtFly9bsZp= z6aWB?DU;C^9FxIqIe*i8dz(GluG`YRvTlQ}ZQ8wBMi`H+11Xd;){T;FQf`ym_HIdT zxw%<4ULqnQiUNY#fhed{bPCKaEfg4_ZZJSmR31)Vg5U#DR8+vtbG{^9+GV)@e(AaA z`@Zu&-#O>ofAE2a0W1-#1$JC<#oFbUR(9&)Ek-<28LSLhbRSb2~R1VMjrsz%03% zbj)ad*oudfwr#|n`X(aNJEMjIl?b=$(fLs;tVcJPy=iF^TO^rj)iZvQKrx?*m$vcIFG^5a1P{u+&```@)4cGezkFUy zz(oF<;l(6O=C4@-?kc7$!yF9?`~n5!dh*|ts)a4%V@TF{bB$0iUtmJF;jGa)km+bm z&Jt!V^?%|x9Is&kssyGTX4&R&&aFzC(THIysMb)!;uT`os>h7+8l;aCvjFOtSv`50 zeGrcb1gefacqDB`6tP&0B`j?z8DD2@QPCivI#&9W7bmcQ8Y~x>mp6iAq)68VSs~6# zGeH?ij0XzQs=bD^bVyf2kC6uJu)YXwIG^r#mu^Or zwtsOB`9bfdlqt=ZFc%=i(l$_~$iq;0# zo#`-!DS0T2O;J6OAQ5AdRxXkX2DP1kIRVJqUWIC#Beg@3V)cqhED(^in`<%f%NlNF6p8k5w7f}}u^ z5$kofw-5#SIBTIi$!la_AGT@O3d;JTD6Oz~;#g9(aO3z|a49Zhd6#FSA-SxyZC$cg z@Cgl9avgB%k;u4kWQq{qs;lrRK6f?cz*t=rTto3N9fRCxQ4&oZqiu6$o%FaCpMNdJ zXK)=EbmYE*&r?!Re{D6kIbM7LrxfFQe36P{TrS**dAx8F`7vsBcN-*VM!q}LA~#9e z&A6qA9RFpqdNrpHrIkODEfszhU*$5=!DVNMfbXcB6x>FhA(39(&d0xouan2q2`PJF z$+#3?U)_N_Iq2V{;+>mMUVNLo!GC7lm96TTOi}P1s_KrlvaPAPIa?IJ%XR5)e2+Xz zGlJQ*eYMpWk6L=9DKmfwG~~HD$5KDPj~}pp_fR$`555d62BlN?n!g>VGn9BeK@e zWxskjn>ZPbvg?oJ34&}Ak7;-mKjI28x|^oS?Egf=9_*#$rK%KZp_$B!$Jv-YctXGv zj#>#?d6L`o9y~=!(qtv05r5or{9Szg{gkaeekuo)O+Te{%#%aekSTbEJd)76jP*8E znb}q23dMMD`~uHv_&I(#u7A;Huj5BH+Fx@{KPMpSRJ=gOk;w@w9wa4yldS-fa$S#Y z^`(cv-*UGwoJ>*o;$`;2OL&EJwi0!5nhjLEM$MLEZd+uSLuKcM&0B0 z+1`_`9Gr3_`Yi$1`nJ(NlCwvYf5e}P@CW>PY}b-}75s%1a;z4skALboP3MOd%H@$) zp}*p98s5RXWL}>ck63*P75^Yl(WvU^W}M3Cj9lBAdUU(ZxHxIV!|Ch&9{$Dj|0b_> zn(<7`RlF}S{V)|diid^KY3oBysUCU}s5nR!<%EU?8okLdZe)7gikqabyimd=2NL1t zQo8Xd1Ca1&_^+V(-hV?~-*&ic=bD-kev((HqKHpwbVrWZR)m*bpqtJaT)1g^YW9kW zVv;5%h{=@i*-O(L?@eZUcjnHCQfdRFdCm?^nmJ==&ITzlMU*qospO!lyhqYDP1i)3 z@QrCxq*zRM92Pl46Eo$sydbe4u8P^z3A*I2z=}Mnxbdj>W`8VWQqM2u5^qt-0+x@- zHM%2Yup$;vdCt6@(o5rK<@74?I$l(1;yAI8ngq=^G*u;g9j~aNB0{UR0@a6$NWyUZ z#x^6Ibodtf=~~6i1iu9nTvX`7iaHicj2)xZ=#!JISR{uBv6!aS!_wC#PH>XOr>8%D1|eI(Gogm5a)$j_o8sX^+C-p zv=ft!DSzlGMB1xEp-ps}PE2nd#LQp;kp(@2m>mih)~3+YK8RRQaW|@kjYR>;T`gDp zq16U_1u0zY^Q7SHK=Cjx3918VX8ej!P~Ate4!!MDM{s2*s14zh4>uOO8@=V;^5Q!& z$ETKimxO{7q|(Jc%|~CKZok?q1`fUA(}Jo`y?-B{6G(sDAkdGc{PiV)N5~~Xjr9Kt zJH)4Tl=ctdRx&f~ixj>wjBm9M9D0KED;&f?3OfTnWf=FeVuNJH0A6e_FDkqPdwt42 zJX$MHg@TG?r?7)l7-H|0pInr4lHx!P8Nr^=CZ>3lv>U>Y zhkvjyh5bP_g{OULP#Hig`>Dvs3wvrqSwobL(w~tb!}wJS&zHV9YE5=u?I=AU4SjWV zO9YjIMzy@iby29X=ytKFT-|Z-qHN^pH&Zg(nG=7i2(%pv7I0ike>aRbcj4_6{$Bde z6#mms5yO+xQcs}t1F}Z6j^Mwc!iVrqD1YShbcEcchuR9tglO|L7N$f&d0|J}kWf;h zm{KJrO8T*djc*+hWg#CeOdApvWc`SkN&7=$7P)ReIeIUue1&CVPEaj)2udhe+5W`X$bg@!MQ?OPnF&J6-okoFU`8T)QRCknthc6B1|0_*1TDCC-rX z7hEq%oFU_{xL%hyL&o29y(@8sj30EnCC-p=s)kKe88@Q>JiDAt)wLaNY+XbFz1BVS zL@dNLRAFy|io2*{eh7_dip6SpMK>mh7$&+JFv)c`CcD<5#I*sXt_xA-axlexD$3nw zVXAu#rn%Q+y88n7+?%8vx2)ps{{c`-2M9FbluW}5006p^;dxnq+e!m55QhI)wOUte zJ>7V>3ZA+y^#Dc18$lElK|$~`-JNcu*#pV8UWh)3Z{dXqUibh$lsH=z5gEwL{Q2fj zNZvnQ-vDf2PT=w3;k&^Ae^^@j$M1ODMq|d0-FZ_2|XiKHLhEB;^88I<+^6PSu7q?|oxD=%8&Ue1^o%27B&#!&!lh=u83+I?Fo;!DF z$CE8Xdghd2Wm~#iGQ%zHEg3sMe`e-%&$O*%-p(4BcZ{5&y9O3VbvKzAH8Q8%Lf&oZ z9@cZN(cUsPlFaL4NmFEG@6K-Cwq*#s&W_6d;X*El33pUaZpP5CMoh~v9Mc-X>}kVs zaTexxbZqU|k<1#WTb>FLGiif%!O0j8m^p)Kwe5^_jyQTYXLO!%^szC+f9dSETu;yC zg5+mfeo{ZJcjk0!r1QYgNh9M0sg9{GXOD~+4%3=cjr}RLxRWWAwa-{NThB7BtHrpx zybRXW#@S4+;F_nEUOkzN;kx^DOIN3K*4n&h!3_{scdu!g-Y%v`W4F-omO9m1Jg9r4 zJ+5oyhjQ57_Arw#*7k6if0oj6je^v`l>A?58l)zTR!~Ej!nCBG0<oPUP+Nxx!$(>=ko$io(N14La#|EhdE-=oTuIDNfJrbr3)T+^Xf4YmQS+N#8GuPQ? z=W@UlaOwsr##C?Q$Gq_r_Axb9PE?#ShXdo3(5Q{t!J5O29EKAbVr|D}-#bhl)G6n| zUQIJndK^br;)AqBqpjkw#iqO4bfARojE8AkNz3ifTF(Nu&9T(n0N5$F*+KWn{%)qF zvvmy8y-Y#V-6IzXf732%T}=1U{Y;NPs7xNsg2^$53UcY_##VP@G;14f)Uv&3#(fwb~OKgwcQ~c3ABsH``hMQBut0th^QhVpEHL-^bWxZ^lhtQ zj9%OJpr$^y4~h+Xy5kwnhRs1brqOZ1T-$7$SbAPkgC{Aa296(-lTI-0eQN~C@wy{d zoyJnM#xC4fe`i{W5@8OHR}x-dx&AP1tAUcYb|PRu_)t%B%eL(yf&{+ER1R_iIhUs1OZsGmziq=&(?k$+PtW<^X)#$tcrD2An z-|`GqF}@F`^X!L=v!y-r5IY^PKR`dI(f892Nx4RE;Ejgqhv|UC@Q+|hpkm>EYh!)$ zcb64`e~|amkBKhtLuFgoLksNufb4t*WyG^9x~_=TRQ1Q{L&E!EsT%Jrp!*5aMai(c z=_6u5^hq9U`q5HyewJw&u+uZ-+PQ*fNKFpYb0T3q{Ur0~!vbqFqgt(~JzOgQqQg3n zkiE0jYPHhnhHCQU_3`Mae%go*8HN@0^gKcve|hAL>5X=@T79-PY&!X!L1F`^r* zHxG{L2!z2xeq(gZv9Zw`k0Kh!<*ZV&NS2dDM|mB|3i$~-m@b0Xk<5fbkd-Y_-GOT5 zFonU?apmpNVaLuR$~~vxN|tj~Z`UCgi|($z%@HTp9c^`6txCK{Q+CNlrRnKBS?NQ& ze^qXQm}pPNgHPrygy^Txx6OF-P{H!dyn$}V7!$cc`k6TebXLNj(C7tv5rw?uUKHUP zq525ICa2ng=II(g8#*u1$Heg;57W=l&ueIxK7k-CSWlRU?K^7Lo|!x_s~5qJ&PU9# zQvY&AqpOk~f`;Wu9bt;hYDe~1g}mV?fAc|yNtzP=muJbVVhPeUU=~gOKHD+&m+#s2*K)+1CBJ974%so%*Jy3HzNWTt^5gPkZP{QifeO9B_f9SX6 zWOPw=`BSK}xa;qfV)qM3I29-K7KVo5d9q!qfY+= z?z-RuCP?3qcElbD(>Eoa{)zq>+4c|~l@iq<`qxT%Q$9L8>ey%WA%XY5LowKW{sP8e9jV>_n~qo~*gnHu*n%<7JA~&RICDgu;o;t?QVYd9(L!PI-dS%ggq9&d+y&sH zSryoqrsgK|(kwjrHtx~*e(uEv)0N)NaSCH7zhT~uOo^2}0g`{qiEt8ngb@e9DlbgK zl0S*ucdNf$Y}joKf9r*uR~a9ivmNL6^Ioyz0MpL@hoB(uL(QwSCV1(11-EY$7d2Gp zymzm7;{YGjct5`#nQXfEIHS8!bLQ3^As*D|O?nYJ5u$=Zd=#0?QBR}8c9_#r+t)MN zfrjebpqif$9|!8nEnRoiE4exv3-M#p-qvW2t0VexiDX{3@+VT%}0+Ra$dd!Ka?q z(z?xqH*%k(y;3l#N#nu6&8U;AKVZ+wa# z8n{M#(tN%9 zvvSp*zVO>1;x%OAdf4OmZigNp}k(KWD zCno8ge+|p&Q=#ra#4i>*liptUEHx%00bg@nk)E7@wdn)Rb&D>E*}syE_=|L|NZ*6~ z=dpj1p7w1IGzXH`pQnywb6{%&-8?r%?@4!K^N-@bizEK!n~L=QqY#g&4<0=qfJ45} zE^;oU_ZR6WE- z#SK`XnO4&_+-xn%v(PsDZkx8(Qg8%dulK=Tui?91+V3(td$HmJ-5yu|N`m}?xM_p$ zzO@P5X03QOo>;pDj-8^*7b)O->HH$-{suTNy;KG+nx(Rhx0j>i`D=7Fo!$pEi$(gR zf8g$h;O;y=evJW{&!qQ@WSBl#q~DmL&ne)1{sJwNOa1QAiJPCFpkwXHYxG6o{8Cyx zGf7{L1SaW^iu9Fke}jLHzdl0CD*k$X;^x9UjF!2gMx?;eQbq&IG~7wIoA%g+r& zsD^m$RTf&I=qidT+Cr_0#%Q~u_s}jyOZU)TMN@P@(L;1x(c^Ri)+N$uSkY0k6)n(v z6qR4$dp~_x(UM;@_ygF)>LTQhuT^Y_xuD7z2NUg6^w*cu`{U^=6cMB)PBeaflh243 ze@`_2n_~U%>6IHei{PI+WN*n<-$I0_6BhxXlDYUwdpxZ|c_2|_U+F|(yU4KQ2b;LA zBucsJ($Vrk?I)Tzgp;OtX^|T$I;`0*=0@gXpSY8|{oEZ;EUOR{;??e;xD^2TvUrr& z3EB}?@;@zc!FLvULlfV1qR8!6cvF$@e^$R;MegnnG{oTieMP=+yT86GRNtjV0__R~ zVMM4m#eGG7;37S~Qd=2n4nKXoE2MYfQ^&^&elTDE%ttA_Qfu}<{meyLm0T&4Mpx(x zr!cirEApX8u-(@j29QKTm(~@UxcS^bB-rhrAh%4ruhE<7CO$mLM{Xn{!AKx^e}x}z z;&;}6w@uRRP5&}0g@d1%2%RK{KxGFDW^?cAlt zLS>xcXOy0$xM&3W-wv!kMvFK_KF(mwDoZUQ-?sr!O9u!`Lm;-F4gdhY8;4O&V%U42cOzgT@++{5Rb_Y!~)Y_JT1+9)zb* zqnP-I58y)?&(IzX7bl6gWOQdQ<(RH>I^tfvvCW)~>#y zTcO`}J(;*+VECa;9FNE&852*oWNcV1vVZpD)Q|P`UFpTNqPHExmu^|J zwNdqq-%UM_193|l6&_OHxB*e*1`bCLDT>*Pb*8!6ELqrE-i8iy7Ij%u-2E|-0W*uxf<$W z`9N7d`evT{Ki4BcStVHJs&4Qp6v);2&~2rDlcKi@M}=#uL12{Myecx^iy{8c zVw`(}N3*!b4ak(=|HMS$2PVHlJ$X!Fx~nO4HM#P4Odcci4L6rhaQjTSgiAYJVW}(3 zcZ6dd;k|d|FB}wD<$jpIV3ES^cd=y*as#G1*to(L7Ee&T3=W)vrT%_}6Rcdu_!2Ox zdYK3HJOTg!9+QD19g|)V50gKZ2$Phk9FvcY6@RORP(={Ilb|T{zS&HZ zZ8w{+o7RKa2k|XD2_Ad^A4;5v9-M{w_q z=X}6rk(Ww~N);x^iv)>V)F>R%WhPu8Gn7lW${nB1g?2dLWg6t73{<@%IZZ~BaZFho z{msu;S`%=Y2!BRo(WJ^CT4hqAYqXBuA|4G-hEb5X+gsK4vi|+ax`Y)QE>yX5GbXw0?()rHg zp2v6Y?|;Ai6~Hta44Y4$EEhLYRc@>br(frOjAV;0o1acsC^@* zn3r)y+I>hF1TIxce;hk#yN!}<5g)5iP-2MryPTMe;_5#3Y?~{f39EjFts-NL=6`$fd!<&A)>c385EL}b_hc7TIt#4AVZQ2VNn8;C%V-97h_=;pxPGBN^ zxZEQv^u1TyF>`Dd|Y+WNVk^$vUz2S`^>>OG|rnzOP~h-%^w0;yXlW?LXSF zFAFN=d;B0nJdh6>c=m{s`j9&f&t2!$-EFF>xC?`>kKH9&>Z_j?I&y<d)Ov7vpfIa?C#9&uirm@0zd|~2z#gaHD7ORz-qEb_-YRO7fVmPlel~IFXuuP3)vCN9+M!jN)Dp22H6{lT-VJ zGgdUc&`&^+6vNb&LY?af1om1gjhU%`gWT>aQtk0gJTQUq-oH$Flkd1w_lBBf0;BCy z`7+HcE$8bM0^avZ&C0|*OB=uyFRJ?aTcyIPb&~+uB{0^Ysv=R7ZMP*l&{d2c6X;)4 zG{sye&>M>%3NQkre(=Ig+{%mG#`fOM=|O%cclvVw)s7Fw1@Oa-0qBDX0)tL}srdd3 zAKVr|u!4652w2`d0fsD36d(v8?%fw448z=eKw!vV=Ju7+g<@B0$2aAJ0j^IF7?!W< ztpbe1;%>zpHr&Lcv2JbrusgL?(as#!?0ARvZ(9Tyw9dPLBI6nnUO(iIo%Z>S_JI|# zma!w&AcT?E9qq-QVS__Pcf=Ea+vSIvKgxKI!0TcYM;pGp_iegD<(`iw?f*icdNCBX@kt!LzRTw1Yo($EO{91y)_~ zna_534W4x25$ukGuftOpJnG=jV8ac!8;kc6zdg|V2T)4~2x;QgE$@>LmS2BOn-Id% zPzQ28t;HPLr2p=wv3&Oj;JfT|seQL0nM~MJ-CF6-0jU9DeYR z@_64&(j;x_;hdb@dGFotF5i9czW2|+H~#{#7PlELoIc&#dNMd5B?h^g3~ml4Qo(RA zp=EQjBAK$LMzUIx)4a|VE*XEE7Bi9&No06p(8y7msI>(K*_+;xm6@}{P{;bNG3R2q_^ill$0qum2XdBSv~ zj!flrjWkV}8w?9NY@NI*E76{b`7I2yOInW8*^Z{HMa7sj>JplolG6-L9n;6tX6xj2 zn?nKGDyy>jD8s78N_*AgXzF9AX>98AVK(M^;YK|n@6nqZ^So$4y$?Rjnt@s@@WF!_ z;%ku)Ud$9Xi~Bio)1CH@sgE?7-s2Q zO70|>uI<+qhK9zbjuQPbQ&f114=b=z09Fwo&CMQ3=c?)OJGTfZGU7uMLc(z~Lu*;i zHb=5*a$S{_V&=AIc_1$mC;vnQ?IluiBSJ+^IKxRw46Caap*(-$LQE<*qx*Z?DW)h^ zd(nb5408-#VUeM}u~J*qZ5`H&Dr}$xlV!>~=nQ%A2*bQ|r4_N@!zMvf12!|v6f`-E zA159fr-nFf(3Q+@#Wuk_ZM}KMRF@3%tC$uEJdW)mlpT{2=#k8f2Ro-GAQpVs?IiHT zRBz6DyJPh!@>_pyHI|XqZrB*hXFcd(STxD>#HtTnj{R zI_co4MD?WI#m!+&AKWKrxt2HWBiimm8X2J@Gq@Vt#l(MB42sNXkJlShK|+a2t3nf~ z9K#Z_+$Sk=QZo6ZQ{saz&VK_8f$J9yVJq^&_z>ZYX>pD=c{zsT0)B$DOC{*dt0qOW z>sW&4oM!brL%2=LE6ISWnE}yg0)_4tD7E51O4qW1RV$2DEgqb%=t39~8?^CDDrIS&Wms6= zbK2Eh-Xx=3%DVAZsfQF>l4J92FV5i|>Z;Xl2{+y&vIS$bk4x|}%eIvd@Szv)LD%aOMWyPXmsD3iJHYjQVmo3Dol!SE z@M=&mE`Iu|7uUWm=}AD+4I&bA=>HbL+*kq^&HmjSY7T`%@iF*sp&=gc8pHfiEF8t+ zQ7pCa;CWn%gd*{&Kf;B_@vw!)P77iBTx)+}qra5~Tf#>yJZ7QIzl%ms7DjvgoiyqR zAE~hrv(V>%nuZ4pi--Ns(kM|Fr7Rq^khSof1=GT?g_BpXtn(I5#a*}Ij(62GJN`%C z<=Drl3ZC?LG0U$s-Dq50A)NbSTPi=_%})kwxho&E==wkE(LH}@{{)3qO|C%#YF=3$ zdiA?ni$9)wR*=E-zD>6#=i#B!N#gG&-1E6KkNw7xOU%m~-nh!XQ{HJ=8J4JS5MC7j80GfF1F!!W{h{y?1Y6gJv#Es?z-Mhy6*8qFYB=KY5fJ$eA5$JDWZC&|wm9Vh`;wc1 z=hdk(0FO+816Kit$%z66lMChx$ilBF2VOs5jG{_Fm|^llWu?h^^R#6V_b)Rr*r2Go zCJIq?W1a~s_?F7ag7Zb0%OoM9-t$dmLAMF|0NpViXalO=LkbX8`{$d;BCcg)V6a88 zp-~y6${p-l#0_8!3>GM=&ZvP@X-rJ1|U_6z{_d)L2hS-94p_r zNR&C&lwq=fmEz=Gi{xeDN1+4Vql040S4)s8GqAtmXGCMf(rRml$p-dPz{AsxWx*#7 z1I<|s^p_oqSz`7Kll2`vz-A#%!)0L5M^WYL$S|3)N@Q}Svnp66{FqRnt&S)votz;m zA;;+IfmI{UMr2?xK~eqK4W?QPtP=SQA4L?Exn2;JaX#W;mGFaPfWAVFN$n7b${>49 zkV+ZQVI((!sx|@ru8U%(NZ90nWgaq!b@vPmS||zvBY+B|C!b%YB@17*Bg(*_graC; zF33Ka$q#Y`z%B!>QGqN`0osXb-`Pr#N^_7ZX~ZBQ1A_vJd9x>9Ty7%^AMXK%usn+V z#4d>c>{h7C!iPA3cA@5E9CIF-wP*MN@ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index bcc1d6e79ea9..436598da212b 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -16,8 +16,8 @@ # distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionSha256Sum=544c35d6bd849ae8a5ed0bcea39ba677dc40f49df7d1835561582da2009b961d -distributionUrl=https\://services.gradle.org/distributions/gradle-8.7-bin.zip +distributionSha256Sum=f1771298a70f6db5a29daf62378c4e18a17fc33c9ba6b14362e0cdf40610380d +distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.4-bin.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/gradlew b/gradlew index 1aa94a426907..f5feea6d6b11 100755 --- a/gradlew +++ b/gradlew @@ -15,6 +15,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # +# SPDX-License-Identifier: Apache-2.0 +# ############################################################################## # @@ -55,7 +57,7 @@ # Darwin, MinGW, and NonStop. # # (3) This script is generated from the Groovy template -# https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt # within the Gradle project. # # You can find Gradle at https://github.com/gradle/gradle/. @@ -84,7 +86,8 @@ done # shellcheck disable=SC2034 APP_BASE_NAME=${0##*/} # Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) -APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit +APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s +' "$PWD" ) || exit # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD=maximum diff --git a/gradlew.bat b/gradlew.bat index 93e3f59f135d..9d21a21834d5 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -13,6 +13,8 @@ @rem See the License for the specific language governing permissions and @rem limitations under the License. @rem +@rem SPDX-License-Identifier: Apache-2.0 +@rem @if "%DEBUG%"=="" @echo off @rem ########################################################################## @@ -43,11 +45,11 @@ set JAVA_EXE=java.exe %JAVA_EXE% -version >NUL 2>&1 if %ERRORLEVEL% equ 0 goto execute -echo. -echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. +echo. 1>&2 +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 goto fail @@ -57,11 +59,11 @@ set JAVA_EXE=%JAVA_HOME%/bin/java.exe if exist "%JAVA_EXE%" goto execute -echo. -echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. +echo. 1>&2 +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 goto fail diff --git a/site/_docs/howto.md b/site/_docs/howto.md index 430c530c8fb8..9035a2c5524f 100644 --- a/site/_docs/howto.md +++ b/site/_docs/howto.md @@ -32,7 +32,7 @@ adapters. ## Building from a source distribution Prerequisite is Java (JDK 8, 11, 17, 21 or 23) -and Gradle (version 8.7) on your path. +and Gradle (version 8.14.4) on your path. Unpack the source distribution `.tar.gz` file, `cd` to the root directory of the unpacked source, From 4ee6afeedb15b79453a6366668a862c31e06dbe8 Mon Sep 17 00:00:00 2001 From: Sergey Nuyanzin Date: Mon, 13 Apr 2026 00:31:31 +0200 Subject: [PATCH 214/562] [CALCITE-7471] Alias is not auto generated for `MATCH_RECOGNIZE` --- .../sql/validate/SqlValidatorImpl.java | 2 +- .../apache/calcite/test/SqlValidatorTest.java | 25 +++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java index bf277669d523..6195ffd90448 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java @@ -2640,7 +2640,7 @@ private SqlNode registerFrom( case MATCH_RECOGNIZE: registerMatchRecognize(parentScope, usingScope, (SqlMatchRecognize) node, enclosingNode, alias, forceNullable); - return node; + return newNode; case PIVOT: registerPivot(parentScope, usingScope, (SqlPivot) node, enclosingNode, diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index 3b869edc9fe3..87383d00b317 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -2575,6 +2575,31 @@ void testLikeAndSimilarFails() { .rewritesTo(expected5); sql(expected5) .withParserConfig(c -> c.withQuoting(Quoting.BACK_TICK)).ok(); + + // Test cases for [CALCITE-7471] https://issues.apache.org/jira/browse/CALCITE-7471 + // Alias is not auto generated for `MATCH_RECOGNIZE` + final String sql6 = "SELECT *\n" + + "FROM emp\n" + + "MATCH_RECOGNIZE (\n" + + " MEASURES\n" + + " FINAL COUNT(A.deptno) AS deptno\n" + + " PATTERN (A B)\n" + + " DEFINE\n" + + " A AS A.empno = 123\n" + + ")"; + + final String expected6 = "SELECT `EXPR$0`.`DEPTNO`\n" + + "FROM `CATALOG`.`SALES`.`EMP` AS `EMP` MATCH_RECOGNIZE(\n" + + "MEASURES FINAL COUNT(`A`.`DEPTNO`) AS `DEPTNO`\n" + + "PATTERN (`A` `B`)\n" + + "DEFINE `A` AS PREV(`A`.`EMPNO`, 0) = 123) AS `EXPR$0`"; + + sql(sql6) + .withValidatorConfig(c -> c.withIdentifierExpansion(true)) + .rewritesTo(expected6); + + sql(expected6) + .withParserConfig(c -> c.withQuoting(Quoting.BACK_TICK)).ok(); } @Test void testIntervalTimeUnitEnumeration() { From 62fec9af405cd35a5ea4e3bf6010a92d69270221 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Mon, 13 Apr 2026 17:49:00 -0700 Subject: [PATCH 215/562] [CALCITE-7468] The SPLIT_PART implementation is incorrect for regex patterns Signed-off-by: Mihai Budiu --- babel/src/test/resources/sql/postgresql.iq | 37 ++++++++++++++++++- .../apache/calcite/runtime/SqlFunctions.java | 13 +++++-- .../apache/calcite/test/SqlFunctionsTest.java | 11 ++++-- 3 files changed, 52 insertions(+), 9 deletions(-) diff --git a/babel/src/test/resources/sql/postgresql.iq b/babel/src/test/resources/sql/postgresql.iq index ef9edb7b53b3..af230bd21bd6 100644 --- a/babel/src/test/resources/sql/postgresql.iq +++ b/babel/src/test/resources/sql/postgresql.iq @@ -60,7 +60,7 @@ EXPR$0 false !ok -#Test string function split_part +#Test string function split_part, validated on Postgres select split_part('abc~@~def~@~ghi', '~@~', 2); EXPR$0 def @@ -71,6 +71,41 @@ EXPR$0 ghi !ok +select split_part('abc.def', '.', 1); +EXPR$0 +abc +!ok + +select split_part('abc.def', '', 1); +EXPR$0 +abc.def +!ok + +select split_part('abc.def', '', 2); +EXPR$0 + +!ok + +select split_part(NULL, '.', 1); +EXPR$0 +null +!ok + +select split_part(NULL, NULL, NULL); +EXPR$0 +null +!ok + +select split_part('abc.abc', '.', NULL); +EXPR$0 +null +!ok + +select split_part('abc', NULL, 1); +EXPR$0 +null +!ok + # Test string_to_array function select string_to_array('a,b,c', ',', 'd'); EXPR$0 diff --git a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java index 0d1db4f855d1..cc6b56520db7 100644 --- a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java +++ b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java @@ -1004,11 +1004,18 @@ public static List split(String s, String delimiter) { /** SQL {@code SPLIT_PART(string, string, int)} function. */ public static String splitPart(String s, String delimiter, int n) { - if (Strings.isNullOrEmpty(s) || Strings.isNullOrEmpty(delimiter)) { + // Function is strict, so arguments cannot be null + if (s.isEmpty()) { return ""; } - String[] parts = s.split(delimiter, -1); + + String[] parts; + if (delimiter.isEmpty()) { + parts = new String[] { s }; + } else { + parts = s.split(Pattern.quote(delimiter), -1); + } int partCount = parts.length; if (n < 0) { @@ -1022,8 +1029,6 @@ public static String splitPart(String s, String delimiter, int n) { return parts[n - 1]; } - - /** SQL {@code SPLIT(string)} function. */ public static List split(String s) { return split(s, ","); diff --git a/core/src/test/java/org/apache/calcite/test/SqlFunctionsTest.java b/core/src/test/java/org/apache/calcite/test/SqlFunctionsTest.java index 4c5f6cc2b7f1..9c7f4b4a216c 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlFunctionsTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlFunctionsTest.java @@ -1085,14 +1085,17 @@ private void checkCeil(int x, int y, int result) { assertThat(SqlFunctions.splitPart("abc,,ghi", ",", 2), is("")); assertThat(SqlFunctions.splitPart("", ",", 1), is("")); - assertThat(SqlFunctions.splitPart("abc", "", 1), is("")); - - assertThat(SqlFunctions.splitPart(null, ",", 1), is("")); - assertThat(SqlFunctions.splitPart("abc,def", null, 1), is("")); + // Tested on Postgres 17: empty delimiter + assertThat(SqlFunctions.splitPart("abc", "", 1), is("abc")); + assertThat(SqlFunctions.splitPart("abc", "", 2), is("")); assertThat(SqlFunctions.splitPart("abc,def", ",", 0), is("")); assertThat(SqlFunctions.splitPart("abc,def", ",", 3), is("")); assertThat(SqlFunctions.splitPart("abc,def", ",", -3), is("")); + + // Test case for https://issues.apache.org/jira/browse/CALCITE-7468 + // The SPLIT_PART implementation is incorrect for regex patterns + assertThat(SqlFunctions.splitPart("abc.def", ".", 1), is("abc")); } @Test void testByteString() { From 58970860e96bee2a054bf2882b8915e2d3aa47f2 Mon Sep 17 00:00:00 2001 From: xuzifu666 <1206332514@qq.com> Date: Mon, 13 Apr 2026 14:55:17 +0800 Subject: [PATCH 216/562] Test case for [CALCITE-5161] NPE when inserting a null value into a decimal column --- .../apache/calcite/test/JdbcAdapterTest.java | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/core/src/test/java/org/apache/calcite/test/JdbcAdapterTest.java b/core/src/test/java/org/apache/calcite/test/JdbcAdapterTest.java index 8d8d66371c8d..df12fbe8a8d1 100644 --- a/core/src/test/java/org/apache/calcite/test/JdbcAdapterTest.java +++ b/core/src/test/java/org/apache/calcite/test/JdbcAdapterTest.java @@ -80,6 +80,35 @@ class JdbcAdapterTest { .returnsCount(14); } + /** Test case for + * [CALCITE-5161] + * NPE when inserting a null value into a decimal column. */ + @Test void testInsertNull() { + // Insert data with null values + CalciteAssert.model(FoodmartSchema.FOODMART_MODEL) + .query("insert into \"foodmart\".\"promotion\" " + + "values (9999, 111, 'Test', NULL, NULL, NULL, NULL)") + .updates(1); + + // Verify data was inserted + CalciteAssert.model(FoodmartSchema.FOODMART_MODEL) + .query("select \"promotion_id\", \"promotion_district_id\", \"promotion_name\", " + + "\"media_type\", \"cost\", \"start_date\", \"end_date\" " + + "from \"foodmart\".\"promotion\" where \"promotion_id\" = 9999") + .returns("promotion_id=9999; promotion_district_id=111; promotion_name=Test; " + + "media_type=null; cost=null; start_date=null; end_date=null\n"); + + // Delete the inserted data + CalciteAssert.model(FoodmartSchema.FOODMART_MODEL) + .query("delete from \"foodmart\".\"promotion\" where \"promotion_id\" = 9999") + .updates(1); + + // Verify data was deleted + CalciteAssert.model(FoodmartSchema.FOODMART_MODEL) + .query("select count(*) as c from \"foodmart\".\"promotion\" where \"promotion_id\" = 9999") + .returns("C=0\n"); + } + /** Test case for * [CALCITE-6462] * VolcanoPlanner internal valid may throw exception when log trace is enabled. */ From 0c375899a4cf936e700f10faf561e8e8c43c261c Mon Sep 17 00:00:00 2001 From: xuzifu666 <1206332514@qq.com> Date: Wed, 15 Apr 2026 14:35:43 +0800 Subject: [PATCH 217/562] Test case for [CALCITE-5124] LIMIT won't work when GROUP BY two or more columns in Elasticsearch Adapter --- .../elasticsearch/AggregationAndSortTest.java | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/AggregationAndSortTest.java b/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/AggregationAndSortTest.java index 3ddf3c82cc3e..e41c81048b18 100644 --- a/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/AggregationAndSortTest.java +++ b/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/AggregationAndSortTest.java @@ -512,4 +512,47 @@ private static Connection createConnectionWithConformance(String lex, String con + " group by CAT order by MAX_VAL1 desc, CAT desc") .returns("CAT=2; MAX_VAL1=7.0\nCAT=1; MAX_VAL1=1.0\nCAT=null; MAX_VAL1=null\n"); } + + /** Test case for + * [CALCITE-5124] + * LIMIT won't work when GROUP BY two or more columns in Elasticsearch Adapter. + */ + @Test void testGroupByAggregationLimit() { + CalciteAssert.that() + .with(AggregationAndSortTest::createConnection) + .query("select val1, cat4 from view group by val1, cat4 limit 1") + .returnsCount(1); + + CalciteAssert.that() + .with(AggregationAndSortTest::createConnection) + .query("select val1, cat4 from view group by val1, cat4 limit 2") + .returnsCount(2); + + CalciteAssert.that() + .with(AggregationAndSortTest::createConnection) + .query("select val1, cat4 from view group by val1, cat1, cat4 limit 2") + .returnsCount(2); + + CalciteAssert.that() + .with(AggregationAndSortTest::createConnection) + .query("select val1 from view group by val1 limit 2") + .returnsCount(2); + + CalciteAssert.that() + .with(AggregationAndSortTest::createConnection) + .query("select val1, cat4 from view group by val1, cat4 limit 2") + .returns("val1=null; cat4=1576108800000\nval1=1; cat4=1514764800000\n"); + + // Test GROUP BY alias with LIMIT (requires BABEL conformance) + CalciteAssert.that() + .with(() -> createConnectionWithConformance("JAVA", "BABEL")) + .query("select val1 as V, cat4 as C from view group by V, C limit 2") + .returnsCount(2); + + CalciteAssert.that() + .with(() -> createConnectionWithConformance("JAVA", "BABEL")) + .query("select cat5 as CAT, max(val1) as MAX_VAL1 from view" + + " group by CAT order by MAX_VAL1 desc, CAT desc limit 2") + .returns("CAT=2; MAX_VAL1=7.0\nCAT=1; MAX_VAL1=1.0\n"); + } } From bcd801348c7e20e3382966b6ad538f9e759fa733 Mon Sep 17 00:00:00 2001 From: Yash Limbad Date: Wed, 18 Mar 2026 16:26:45 +0530 Subject: [PATCH 218/562] [CALCITE-7442] Correlated variable has wrong index inside subquery --- .../org/apache/calcite/plan/RelOptUtil.java | 87 +++++- .../apache/calcite/rel/rules/CoreRules.java | 2 +- .../java/org/apache/calcite/rex/RexUtil.java | 49 ++- .../calcite/sql2rel/RelDecorrelatorTest.java | 295 ++++++++++++++++++ 4 files changed, 423 insertions(+), 10 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java b/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java index 274e91f14de1..9b24cf2ad201 100644 --- a/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java +++ b/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java @@ -2934,15 +2934,39 @@ public static boolean classifyFilters( ImmutableBitSet rightBitmap = ImmutableBitSet.range(nSysFields + nFieldsLeft, nTotalFields); + // Correlation variables introduced by this join itself: i.e. ids whose + // binding is established by *this* join. A predicate that references any + // of these cannot be pushed to either input -- the binder lives on the + // join, so pushing the reference below it would strand the variable. + // Such predicates must stay on the join itself. + final Set joinCorrelationIds = joinRel instanceof Join + ? joinRel.getVariablesSet() + : ImmutableSet.of(); + final List filtersToRemove = new ArrayList<>(); for (RexNode filter : filters) { - final InputFinder inputFinder = InputFinder.analyze(filter); + + // Only consider correlation ids bound by *this* join when computing + // the input bitmap of a sub-query inside the predicate. Foreign + // correlation ids are bound by an outer scope and their + // correlationColumns indices would otherwise alias onto unrelated + // columns of this join's row type, mis-classifying the predicate. + final InputFinder inputFinder = InputFinder.analyze(filter, joinCorrelationIds); final ImmutableBitSet inputBits = inputFinder.build(); + // Block pushing to either input for filters that reference a + // CorrelationId bound by this join; they must remain on the join. + // pushing down correlated subqueries carries risks and involves extremely complex logic, + // and therefore pushing down is prohibited. + final boolean blockPush = + RexUtil.containsCorrelation(filter, joinCorrelationIds); + final boolean effectivePushLeft = pushLeft && !blockPush && leftBitmap.contains(inputBits); + final boolean effectivePushRight = pushRight && !blockPush && rightBitmap.contains(inputBits); + // REVIEW - are there any expressions that need special handling // and therefore cannot be pushed? - if (pushLeft && leftBitmap.contains(inputBits)) { + if (effectivePushLeft) { // ignore filters that always evaluate to true if (!filter.isAlwaysTrue()) { // adjust the field references in the filter to reflect @@ -2962,7 +2986,7 @@ public static boolean classifyFilters( leftFilters.add(shiftedFilter); } filtersToRemove.add(filter); - } else if (pushRight && rightBitmap.contains(inputBits)) { + } else if (effectivePushRight) { if (!filter.isAlwaysTrue()) { // adjust the field references in the filter to reflect // that fields in the right now shift over to the left @@ -4678,12 +4702,29 @@ public RexCorrelVariableMapShuttle(final CorrelationId correlationId, public static class InputFinder extends RexVisitorImpl { private final ImmutableBitSet.Builder bitBuilder; private final @Nullable Set extraFields; + /** Correlation ids whose binder is the current scope. When non-null, + * {@link #visitSubQuery} projects bits for each id in this set by looking + * up its {@code correlationColumns} against the sub-query's inner plan + * and adding those column indices to the bitmap. Correlation ids bound + * by an outer scope are skipped, since their column indices are relative + * to a foreign row type and would otherwise alias onto unrelated columns + * of the current scope. When null, {@link #visitSubQuery} contributes no + * correlation-related bits and simply descends into the sub-query's + * operands (legacy behaviour). */ + private final @Nullable Set localCorrelationIds; private InputFinder(@Nullable Set extraFields, - ImmutableBitSet.Builder bitBuilder) { + ImmutableBitSet.Builder bitBuilder, + @Nullable Set localCorrelationIds) { super(true); this.bitBuilder = bitBuilder; this.extraFields = extraFields; + this.localCorrelationIds = localCorrelationIds; + } + + private InputFinder(@Nullable Set extraFields, + ImmutableBitSet.Builder bitBuilder) { + this(extraFields, bitBuilder, null); } public InputFinder() { @@ -4706,6 +4747,22 @@ public static InputFinder analyze(RexNode node) { return inputFinder; } + /** Returns an input finder that has analyzed a given expression, + * treating {@code localCorrelationIds} as the set of correlation ids + * bound by the current scope. For each nested {@link RexSubQuery}, + * any correlation id used inside it that belongs to this set + * contributes its {@code correlationColumns} indices to the bitmap; + * correlation ids bound by an outer scope are ignored, because their + * indices are relative to a foreign row type and would otherwise + * alias onto unrelated columns of the current scope. */ + public static InputFinder analyze(RexNode node, + Set localCorrelationIds) { + final InputFinder inputFinder = + new InputFinder(null, ImmutableBitSet.builder(), localCorrelationIds); + node.accept(inputFinder); + return inputFinder; + } + /** * Returns a bit set describing the inputs used by an expression. */ @@ -4752,6 +4809,28 @@ public ImmutableBitSet build() { } return super.visitCall(call); } + + @Override public Void visitSubQuery(RexSubQuery subQuery) { + if (localCorrelationIds == null) { + return super.visitSubQuery(subQuery); + } + + final Set variablesSet = RelOptUtil.getVariablesUsed(subQuery.rel); + for (CorrelationId id : variablesSet) { + // Skip correlation ids that are not bound by the *current* scope. + // Their requiredColumns indices are relative to whichever outer + // RelNode produces them and would otherwise alias onto unrelated + // columns of the current row type. + if (!localCorrelationIds.contains(id)) { + continue; + } + ImmutableBitSet requiredColumns = RelOptUtil.correlationColumns(id, subQuery.rel); + for (int index : requiredColumns) { + bitBuilder.set(index); + } + } + return super.visitSubQuery(subQuery); + } } /** diff --git a/core/src/main/java/org/apache/calcite/rel/rules/CoreRules.java b/core/src/main/java/org/apache/calcite/rel/rules/CoreRules.java index 444c89ccdd70..0337b1d5abb3 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/CoreRules.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/CoreRules.java @@ -284,7 +284,7 @@ private CoreRules() {} * {@link org.apache.calcite.rel.rules.FilterProjectTransposeRule}. * *

      It does not allow a Filter to be pushed past the Project if - * {@link RexUtil#containsCorrelation there is a correlation condition} + * {@link RexUtil#containsCorrelation(org.apache.calcite.rex.RexNode) there is a correlation condition} * anywhere in the Filter, since in some cases it can prevent a * {@link Correlate} from being de-correlated. */ diff --git a/core/src/main/java/org/apache/calcite/rex/RexUtil.java b/core/src/main/java/org/apache/calcite/rex/RexUtil.java index 8c45ffc97ae6..3604e98dfd5b 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexUtil.java +++ b/core/src/main/java/org/apache/calcite/rex/RexUtil.java @@ -2412,6 +2412,23 @@ public static boolean containsCorrelation(RexNode condition) { } } + /** Returns whether an expression references a {@link RexCorrelVariable} + * whose id is in {@code ids}, either directly (typically through a + * {@link RexFieldAccess}) or transitively via the inner plan of a + * {@link RexSubQuery}. */ + public static boolean containsCorrelation(RexNode condition, + Set ids) { + if (ids.isEmpty()) { + return false; + } + try { + condition.accept(new CorrelationFinder(ids)); + return false; + } catch (Util.FoundOne e) { + return true; + } + } + /** * Given an expression, it will swap the table references contained in its * {@link RexTableInputRef} using the contents in the map. @@ -3116,14 +3133,28 @@ private static class RexShiftShuttle extends RexShuttle { /** Visitor that throws {@link org.apache.calcite.util.Util.FoundOne} if * applied to an expression that contains a {@link RexCorrelVariable}. */ private static class CorrelationFinder extends RexVisitorImpl { - static final CorrelationFinder INSTANCE = new CorrelationFinder(); + static final CorrelationFinder INSTANCE = new CorrelationFinder(null); - private CorrelationFinder() { + /** Optional filter: when non-null, only correlation ids in this set + * trigger a match; when null, every correlation id matches. */ + private final @Nullable Set ids; + + /** + * Creates a CorrelationFinder. + * + * @param ids correlation ids to look for; pass {@code null} to match any + * {@link RexCorrelVariable} regardless of its id + */ + private CorrelationFinder(@Nullable Set ids) { super(true); + this.ids = ids; } @Override public Void visitCorrelVariable(RexCorrelVariable var) { - throw Util.FoundOne.NULL; + if (ids == null || ids.contains(var.id)) { + throw Util.FoundOne.NULL; + } + return null; } @Override public Void visitSubQuery(RexSubQuery subQuery) { @@ -3135,8 +3166,16 @@ private CorrelationFinder() { operand.accept(this); } - if (!RelOptUtil.getVariablesUsed(subQuery.rel).isEmpty()) { - throw Util.FoundOne.NULL; + Set used = RelOptUtil.getVariablesUsed(subQuery.rel); + if (!used.isEmpty()) { + if (ids == null) { + throw Util.FoundOne.NULL; + } + for (CorrelationId id : used) { + if (ids.contains(id)) { + throw Util.FoundOne.NULL; + } + } } return null; } diff --git a/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java b/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java index 2b406c3c0154..e91b13886139 100644 --- a/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java +++ b/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java @@ -1830,4 +1830,299 @@ public static Frameworks.ConfigBuilder config() { + " LogicalTableScan(table=[[scott, EMP]])\n"; assertThat(after, hasTree(planAfter)); } + + /** Test case for [CALCITE-7442] + * Getting Wrong index of Correlated variable inside Subquery after FilterJoinRule. */ + @Test void testCorrelatedVariableIndexForInClause() { + final FrameworkConfig frameworkConfig = config().build(); + final RelBuilder builder = RelBuilder.create(frameworkConfig); + final RelOptCluster cluster = builder.getCluster(); + final Planner planner = Frameworks.getPlanner(frameworkConfig); + final String sql = "select e.empno, d.dname, b.ename\n" + + "from emp e\n" + + "inner join dept d\n" + + " on d.deptno = e.deptno\n" + + "inner join bonus b\n" + + " on e.ename = b.ename\n" + + " and b.job in (\n" + + " select b2.job\n" + + " from bonus b2\n" + + " where b2.ename = b.ename)\n" + + "where e.sal > 1000 and d.dname = 'SALES'"; + + final RelNode originalRel; + try { + final SqlNode parse = planner.parse(sql); + final SqlNode validate = planner.validate(parse); + originalRel = planner.rel(validate).rel; + } catch (Exception e) { + throw TestUtil.rethrow(e); + } + + final HepProgram hepProgram = HepProgram.builder() + .addRuleCollection(ImmutableList.of(CoreRules.FILTER_INTO_JOIN)) + .build(); + final Program program = + Programs.of(hepProgram, true, + requireNonNull(cluster.getMetadataProvider())); + final RelNode afterFilterIntoJoin = + program.run(cluster.getPlanner(), originalRel, cluster.traitSet(), + Collections.emptyList(), Collections.emptyList()); + + final String planAfterFilterIntoJoin = "LogicalProject(EMPNO=[$0], DNAME=[$9], ENAME=[$11])\n" + + " LogicalJoin(condition=[AND(=($1, $11), IN($12, {\n" + + "LogicalProject(JOB=[$1])\n" + + " LogicalFilter(condition=[=($0, $cor0.ENAME0)])\n" + + " LogicalTableScan(table=[[scott, BONUS]])\n" + + "}))], joinType=[inner], variablesSet=[[$cor0]])\n" + + " LogicalJoin(condition=[=($8, $7)], joinType=[inner])\n" + + " LogicalFilter(condition=[>(CAST($5):DECIMAL(12, 2), 1000.00)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalFilter(condition=[=($1, 'SALES')])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n" + + " LogicalTableScan(table=[[scott, BONUS]])\n"; + assertThat(afterFilterIntoJoin, hasTree(planAfterFilterIntoJoin)); + + final HepProgram hepProgram1 = HepProgram.builder() + .addRuleCollection(ImmutableList.of(CoreRules.JOIN_SUB_QUERY_TO_CORRELATE)) + .build(); + final Program program1 = + Programs.of(hepProgram1, true, + requireNonNull(cluster.getMetadataProvider())); + final RelNode afterJoinSubqueryCorrelate = + program1.run(cluster.getPlanner(), afterFilterIntoJoin, cluster.traitSet(), + Collections.emptyList(), Collections.emptyList()); + + final String planAfterJoinSubqueryCorrelate = + "LogicalProject(EMPNO=[$0], DNAME=[$9], ENAME=[$11])\n" + + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4]," + + " SAL=[$5], COMM=[$6], DEPTNO=[$7], DEPTNO0=[$8], DNAME=[$9], LOC=[$10]," + + " ENAME0=[$11], JOB0=[$12], SAL0=[$13], COMM0=[$14])\n" + + " LogicalJoin(condition=[=($1, $11)], joinType=[inner])\n" + + " LogicalJoin(condition=[=($8, $7)], joinType=[inner])\n" + + " LogicalFilter(condition=[>(CAST($5):DECIMAL(12, 2), 1000.00)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalFilter(condition=[=($1, 'SALES')])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n" + + " LogicalFilter(condition=[=($1, $4)])\n" + + " LogicalCorrelate(correlation=[$cor0], joinType=[inner]," + + " requiredColumns=[{0}])\n" + + " LogicalTableScan(table=[[scott, BONUS]])\n" + + " LogicalProject(JOB=[$1])\n" + + " LogicalFilter(condition=[=($0, $cor0.ENAME)])\n" + + " LogicalTableScan(table=[[scott, BONUS]])\n"; + assertThat(afterJoinSubqueryCorrelate, hasTree(planAfterJoinSubqueryCorrelate)); + + final RelNode afterDecorrelation = + RelDecorrelator.decorrelateQuery(afterJoinSubqueryCorrelate, builder, + RuleSets.ofList(Collections.emptyList()), RuleSets.ofList(Collections.emptyList())); + final String planAfterDecorrelation = "LogicalProject(EMPNO=[$0], DNAME=[$9], ENAME=[$11])\n" + + " LogicalJoin(condition=[=($1, $11)], joinType=[inner])\n" + + " LogicalJoin(condition=[=($8, $7)], joinType=[inner])\n" + + " LogicalFilter(condition=[>(CAST($5):DECIMAL(12, 2), 1000.00)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalFilter(condition=[=($1, 'SALES')])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n" + + " LogicalJoin(condition=[AND(=($0, $5), =($1, $4))], joinType=[inner])\n" + + " LogicalTableScan(table=[[scott, BONUS]])\n" + + " LogicalProject(JOB=[$1], ENAME=[$0])\n" + + " LogicalFilter(condition=[IS NOT NULL($0)])\n" + + " LogicalTableScan(table=[[scott, BONUS]])\n"; + assertThat(afterDecorrelation, hasTree(planAfterDecorrelation)); + } + + /** Test case for [CALCITE-7442] + * Getting Wrong index of Correlated variable inside Subquery after FilterJoinRule. + * Same as {@link #testCorrelatedVariableIndexForInClause()} but uses EXISTS + * instead of IN. */ + @Test void testCorrelatedVariableIndexForExistsClause() { + final FrameworkConfig frameworkConfig = config().build(); + final RelBuilder builder = RelBuilder.create(frameworkConfig); + final RelOptCluster cluster = builder.getCluster(); + final Planner planner = Frameworks.getPlanner(frameworkConfig); + final String sql = "select e.empno, d.dname, b.ename\n" + + "from emp e\n" + + "inner join dept d\n" + + " on d.deptno = e.deptno\n" + + "inner join bonus b\n" + + " on e.ename = b.ename\n" + + " and exists (\n" + + " select b2.job\n" + + " from bonus b2\n" + + " where b2.ename = b.ename\n" + + " and b2.job = b.job)\n" + + "where e.sal > 1000 and d.dname = 'SALES'"; + + final RelNode originalRel; + try { + final SqlNode parse = planner.parse(sql); + final SqlNode validate = planner.validate(parse); + originalRel = planner.rel(validate).rel; + } catch (Exception e) { + throw TestUtil.rethrow(e); + } + + final HepProgram hepProgram = HepProgram.builder() + .addRuleCollection(ImmutableList.of(CoreRules.FILTER_INTO_JOIN)) + .build(); + final Program program = + Programs.of(hepProgram, true, + requireNonNull(cluster.getMetadataProvider())); + final RelNode afterFilterIntoJoin = + program.run(cluster.getPlanner(), originalRel, cluster.traitSet(), + Collections.emptyList(), Collections.emptyList()); + + final String planAfterFilterIntoJoin = "LogicalProject(EMPNO=[$0], DNAME=[$9], ENAME=[$11])\n" + + " LogicalJoin(condition=[AND(=($1, $11), EXISTS({\n" + + "LogicalFilter(condition=[AND(=($0, $cor0.ENAME0), =($1, $cor0.JOB0))])\n" + + " LogicalTableScan(table=[[scott, BONUS]])\n" + + "}))], joinType=[inner], variablesSet=[[$cor0]])\n" + + " LogicalJoin(condition=[=($8, $7)], joinType=[inner])\n" + + " LogicalFilter(condition=[>(CAST($5):DECIMAL(12, 2), 1000.00)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalFilter(condition=[=($1, 'SALES')])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n" + + " LogicalTableScan(table=[[scott, BONUS]])\n"; + assertThat(afterFilterIntoJoin, hasTree(planAfterFilterIntoJoin)); + + final HepProgram hepProgram1 = HepProgram.builder() + .addRuleCollection(ImmutableList.of(CoreRules.JOIN_SUB_QUERY_TO_CORRELATE)) + .build(); + final Program program1 = + Programs.of(hepProgram1, true, + requireNonNull(cluster.getMetadataProvider())); + final RelNode afterJoinSubqueryCorrelate = + program1.run(cluster.getPlanner(), afterFilterIntoJoin, cluster.traitSet(), + Collections.emptyList(), Collections.emptyList()); + + final String planAfterJoinSubqueryCorrelate = + "LogicalProject(EMPNO=[$0], DNAME=[$9], ENAME=[$11])\n" + + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4]," + + " SAL=[$5], COMM=[$6], DEPTNO=[$7], DEPTNO0=[$8], DNAME=[$9], LOC=[$10]," + + " ENAME0=[$11], JOB0=[$12], SAL0=[$13], COMM0=[$14])\n" + + " LogicalJoin(condition=[=($1, $11)], joinType=[inner])\n" + + " LogicalJoin(condition=[=($8, $7)], joinType=[inner])\n" + + " LogicalFilter(condition=[>(CAST($5):DECIMAL(12, 2), 1000.00)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalFilter(condition=[=($1, 'SALES')])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n" + + " LogicalCorrelate(correlation=[$cor0], joinType=[inner], " + + "requiredColumns=[{0, 1}])\n" + + " LogicalTableScan(table=[[scott, BONUS]])\n" + + " LogicalAggregate(group=[{0}])\n" + + " LogicalProject(i=[true])\n" + + " LogicalFilter(condition=[AND(=($0, $cor0.ENAME), =($1, $cor0.JOB))])\n" + + " LogicalTableScan(table=[[scott, BONUS]])\n"; + assertThat(afterJoinSubqueryCorrelate, hasTree(planAfterJoinSubqueryCorrelate)); + + final RelNode afterDecorrelation = + RelDecorrelator.decorrelateQuery(afterJoinSubqueryCorrelate, builder, + RuleSets.ofList(Collections.emptyList()), RuleSets.ofList(Collections.emptyList())); + final String planAfterDecorrelation = "LogicalProject(EMPNO=[$0], DNAME=[$9], ENAME=[$11])\n" + + " LogicalJoin(condition=[=($1, $11)], joinType=[inner])\n" + + " LogicalJoin(condition=[=($8, $7)], joinType=[inner])\n" + + " LogicalFilter(condition=[>(CAST($5):DECIMAL(12, 2), 1000.00)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalFilter(condition=[=($1, 'SALES')])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n" + + " LogicalJoin(condition=[AND(=($0, $4), =($1, $5))], joinType=[inner])\n" + + " LogicalTableScan(table=[[scott, BONUS]])\n" + + " LogicalProject(ENAME=[$0], JOB=[$1], $f2=[true])\n" + + " LogicalFilter(condition=[AND(IS NOT NULL($0), IS NOT NULL($1))])\n" + + " LogicalTableScan(table=[[scott, BONUS]])\n"; + assertThat(afterDecorrelation, hasTree(planAfterDecorrelation)); + } + + /** Test case for [CALCITE-7442] + * Getting Wrong index of Correlated variable inside Subquery after FilterJoinRule. + * Same as {@link #testCorrelatedVariableIndexForInClause()} EXISTS edge case */ + @Test void testCorrelatedVariableIndexForExistsClause2() { + final FrameworkConfig frameworkConfig = config().build(); + final RelBuilder builder = RelBuilder.create(frameworkConfig); + final RelOptCluster cluster = builder.getCluster(); + final Planner planner = Frameworks.getPlanner(frameworkConfig); + final String sql = "select e1.empno \n" + + "from emp e1, \n" + + "lateral(\n" + + " select d.deptno \n" + + " from emp e2 inner join dept d \n" + + " on exists(\n" + + " select e3.empno from emp e3 where e3.empno = e2.empno and e3.ename = e1.ename\n" + + " )\n" + + ")"; + + final RelNode originalRel; + try { + final SqlNode parse = planner.parse(sql); + final SqlNode validate = planner.validate(parse); + originalRel = planner.rel(validate).rel; + } catch (Exception e) { + throw TestUtil.rethrow(e); + } + + final HepProgram hepProgram = HepProgram.builder() + .addRuleCollection(ImmutableList.of(CoreRules.JOIN_CONDITION_PUSH)) + .build(); + final Program program = + Programs.of(hepProgram, true, + requireNonNull(cluster.getMetadataProvider())); + final RelNode afterJoinConditionPush = + program.run(cluster.getPlanner(), originalRel, cluster.traitSet(), + Collections.emptyList(), Collections.emptyList()); + + final String planAfterJoinConditionPush = "LogicalProject(EMPNO=[$0])\n" + + " LogicalCorrelate(correlation=[$cor1], joinType=[inner], requiredColumns=[{1}])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalProject(DEPTNO=[$8])\n" + + " LogicalJoin(condition=[EXISTS({\n" + + "LogicalFilter(condition=[AND(=($0, $cor0.EMPNO), =($1, $cor1.ENAME))])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + "})], joinType=[inner], variablesSet=[[$cor0]])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n"; + assertThat(afterJoinConditionPush, hasTree(planAfterJoinConditionPush)); + + final HepProgram hepProgram1 = HepProgram.builder() + .addRuleCollection(ImmutableList.of(CoreRules.JOIN_SUB_QUERY_TO_CORRELATE)) + .build(); + final Program program1 = + Programs.of(hepProgram1, true, + requireNonNull(cluster.getMetadataProvider())); + final RelNode afterJoinSubqueryCorrelate = + program1.run(cluster.getPlanner(), afterJoinConditionPush, cluster.traitSet(), + Collections.emptyList(), Collections.emptyList()); + + final String planAfterJoinSubqueryCorrelate = "LogicalProject(EMPNO=[$0])\n" + + " LogicalCorrelate(correlation=[$cor1], joinType=[inner], requiredColumns=[{1}])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalProject(DEPTNO=[$8])\n" + + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4]," + + " SAL=[$5], COMM=[$6], DEPTNO=[$7], DEPTNO0=[$9], DNAME=[$10], LOC=[$11])\n" + + " LogicalJoin(condition=[true], joinType=[inner])\n" + + " LogicalJoin(condition=[true], joinType=[inner]," + + " variablesSet=[[$cor0, $cor1]])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalAggregate(group=[{0}])\n" + + " LogicalProject(i=[true])\n" + + " LogicalFilter(condition=[AND(=($0, $cor0.EMPNO), =($1, $cor1.ENAME))])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n"; + assertThat(afterJoinSubqueryCorrelate, hasTree(planAfterJoinSubqueryCorrelate)); + + final RelNode afterDecorrelation = + RelDecorrelator.decorrelateQuery(afterJoinSubqueryCorrelate, builder, + RuleSets.ofList(Collections.emptyList()), RuleSets.ofList(Collections.emptyList())); + final String planAfterDecorrelation = "LogicalProject(EMPNO=[$0])\n" + + " LogicalJoin(condition=[=($1, $10)], joinType=[inner])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalProject(DEPTNO=[$11], EMPNO0=[$8], ENAME0=[$9])\n" + + " LogicalJoin(condition=[true], joinType=[inner])\n" + + " LogicalJoin(condition=[true], joinType=[inner])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalProject(EMPNO=[$0], ENAME=[$1], $f2=[true])\n" + + " LogicalFilter(condition=[IS NOT NULL($1)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n"; + assertThat(afterDecorrelation, hasTree(planAfterDecorrelation)); + } } From ae321958d7a13a646c5cfce3a80dcd407e390ad2 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Tue, 24 Mar 2026 17:55:14 -0700 Subject: [PATCH 219/562] [CALCITE-7443] Incorrect simplification for large interval Signed-off-by: Mihai Budiu --- .../org/apache/calcite/prepare/Prepare.java | 13 +++-- .../apache/calcite/runtime/SqlFunctions.java | 20 ++++++- .../calcite/sql2rel/ConvertToChecked.java | 54 ++++++++++++++++--- .../sql2rel/StandardConvertletTable.java | 2 +- core/src/test/resources/sql/scalar.iq | 26 +++++++++ 5 files changed, 100 insertions(+), 15 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/prepare/Prepare.java b/core/src/main/java/org/apache/calcite/prepare/Prepare.java index 586cce838536..1c6d8875cb16 100644 --- a/core/src/main/java/org/apache/calcite/prepare/Prepare.java +++ b/core/src/main/java/org/apache/calcite/prepare/Prepare.java @@ -256,11 +256,14 @@ public PreparedResult prepareSql( RelRoot root = sqlToRelConverter.convertQuery(sqlQuery, needsValidation, true); - if (this.context.config().conformance().checkedArithmetic()) { - ConvertToChecked checkedConv = new ConvertToChecked(root.rel.getCluster().getRexBuilder()); - RelNode rel = checkedConv.visit(root.rel); - root = root.withRel(rel); - } + boolean convertToChecked = this.context.config().conformance().checkedArithmetic(); + // Convert some operations to use checked arithmetic: + // - all arithmetic operations on exact types if the conformance requires checked arithmetic + // - all arithmetic that produces INTERVAL results, regardless of the conformance + ConvertToChecked checkedConv = + new ConvertToChecked(root.rel.getCluster().getRexBuilder(), convertToChecked); + RelNode rel = checkedConv.visit(root.rel); + root = root.withRel(rel); Hook.CONVERTED.run(root.rel); if (timingTracer != null) { diff --git a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java index cc6b56520db7..c1ebafe8edd4 100644 --- a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java +++ b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java @@ -2976,10 +2976,14 @@ public static long checkedDivide(long b0, long b1) { if ((b0 & b1 & q) >= 0) { return q; } else { - throw new ArithmeticException("integer overflow"); + throw new ArithmeticException("long overflow"); } } + public static double checkedDivide(int b0, double b1) { + return b0 / b1; + } + public static UByte checkedDivide(UByte b0, UByte b1) { return UByte.valueOf(b0.intValue() / b1.intValue()); } @@ -2996,6 +3000,16 @@ public static ULong checkedDivide(ULong b0, ULong b1) { return ULong.valueOf(UnsignedType.toBigInteger(b0).divide(UnsignedType.toBigInteger(b1))); } + // The definition of this function must match the divide function with the same signature + public static int checkedDivide(int b0, BigDecimal b1) { + return BigDecimal.valueOf(b0) + .divide(b1, RoundingMode.HALF_DOWN).intValueExact(); + } + + public static BigDecimal checkedDivide(BigDecimal b0, BigDecimal b1) { + return b0.divide(b1, RoundingMode.HALF_DOWN); + } + // * /** SQL * operator applied to int values. */ @@ -3118,6 +3132,10 @@ public static ULong checkedMultiply(ULong b0, ULong b1) { return ULong.valueOf(UnsignedType.toBigInteger(b0).multiply(UnsignedType.toBigInteger(b1))); } + public static BigDecimal checkedMultiply(BigDecimal b0, long b1) { + return b0.multiply(BigDecimal.valueOf(b1)); + } + /** SQL SAFE_ADD function applied to long values. */ public static @Nullable Long safeAdd(long b0, long b1) { try { diff --git a/core/src/main/java/org/apache/calcite/sql2rel/ConvertToChecked.java b/core/src/main/java/org/apache/calcite/sql2rel/ConvertToChecked.java index a5e658e54a83..e8c4697c12ac 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/ConvertToChecked.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/ConvertToChecked.java @@ -38,8 +38,8 @@ public class ConvertToChecked extends RelHomogeneousShuttle { final ConvertRexToChecked converter; - public ConvertToChecked(RexBuilder builder) { - this.converter = new ConvertRexToChecked(builder); + public ConvertToChecked(RexBuilder builder, boolean allArithmetic) { + this.converter = new ConvertRexToChecked(builder, allArithmetic); } @Override public RelNode visit(RelNode other) { @@ -48,14 +48,36 @@ public ConvertToChecked(RexBuilder builder) { } /** - * Visitor which rewrites an expression tree such that all - * arithmetic operations that produce numeric values use checked arithmetic. + * Visitor which rewrites an expression tree such that arithmetic operations + * use checked arithmetic. */ class ConvertRexToChecked extends RexShuttle { private final RexBuilder builder; + // If true all arithmetic operations are converted. + // Otherwise, only arithmetic operations on INTERVAL values is checked. + private final boolean allArithmetic; + /** + * Create a visitor which converts all arithmetic operations to checked. + * + * @deprecated Use #ConvertRexToChecked(RexBuilder, boolean). + */ + @Deprecated ConvertRexToChecked(RexBuilder builder) { + this(builder, true); + } + + /** + * Create a converter that replaces arithmetic with checked arithmetic. + * + * @param builder RexBuilder to use. + * @param allArithmetic If true all exact arithmetic operations are converted to checked. + * If false, only operations that produce INTERVAL-typed results + * are converted to checked. + */ + ConvertRexToChecked(RexBuilder builder, boolean allArithmetic) { this.builder = builder; + this.allArithmetic = allArithmetic; } @Override public RexNode visitSubQuery(RexSubQuery subQuery) { @@ -72,6 +94,22 @@ class ConvertRexToChecked extends RexShuttle { List clonedOperands = visitList(call.operands, update); SqlKind kind = call.getKind(); SqlOperator operator = call.getOperator(); + SqlTypeName resultType = call.getType().getSqlTypeName(); + boolean anyOperandIsInterval = false; + for (RexNode op : call.getOperands()) { + if (SqlTypeName.INTERVAL_TYPES.contains(op.getType().getSqlTypeName())) { + anyOperandIsInterval = true; + break; + } + } + boolean resultIsInterval = SqlTypeName.INTERVAL_TYPES.contains(resultType); + boolean rewrite = + // Do not rewrite operator if the type is e.g., DOUBLE or DATE + (this.allArithmetic && SqlTypeName.EXACT_TYPES.contains(resultType)) + // But always rewrite if the type is an INTERVAL and any operand is INTERVAL + // This will not rewrite date subtraction, for example + || (resultIsInterval && anyOperandIsInterval); + switch (kind) { case PLUS: operator = SqlStdOperatorTable.CHECKED_PLUS; @@ -91,8 +129,7 @@ class ConvertRexToChecked extends RexShuttle { default: break; } - SqlTypeName resultType = call.getType().getSqlTypeName(); - if (resultType == SqlTypeName.DECIMAL) { + if (resultType == SqlTypeName.DECIMAL && this.allArithmetic) { // Checked decimal arithmetic is implemented using unchecked // arithmetic followed by a CAST, which is always checked RexCall result; @@ -102,8 +139,9 @@ class ConvertRexToChecked extends RexShuttle { result = call; } return builder.makeCast(call.getParserPosition(), call.getType(), result); - } else if (!SqlTypeName.EXACT_TYPES.contains(resultType)) { - // Do not rewrite operator if the type is e.g., DOUBLE or DATE + } + + if (!rewrite) { operator = call.getOperator(); } update[0] = update[0] || operator != call.getOperator(); diff --git a/core/src/main/java/org/apache/calcite/sql2rel/StandardConvertletTable.java b/core/src/main/java/org/apache/calcite/sql2rel/StandardConvertletTable.java index e8b1bcbd7a04..739c3e30fae2 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/StandardConvertletTable.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/StandardConvertletTable.java @@ -552,7 +552,7 @@ private static RexNode convertInterval(SqlRexContext cx, SqlCall call) { SqlLiteral.createInterval(1, "1", intervalQualifier, call.getParserPosition()); final SqlCall multiply = - SqlStdOperatorTable.MULTIPLY.createCall(call.getParserPosition(), n, + SqlStdOperatorTable.CHECKED_MULTIPLY.createCall(call.getParserPosition(), n, literal); return cx.convertExpression(multiply); } diff --git a/core/src/test/resources/sql/scalar.iq b/core/src/test/resources/sql/scalar.iq index d82d69f58901..e4ef19c0d3f7 100644 --- a/core/src/test/resources/sql/scalar.iq +++ b/core/src/test/resources/sql/scalar.iq @@ -18,6 +18,32 @@ !set outputformat mysql !use scott +# 5 test cases for [CALCITE-7443] Incorrect simplification for large interval +SELECT -(INTERVAL -2147483648 months); +java.lang.ArithmeticException: integer overflow + +!error + +SELECT INTERVAL 2147483647 years; +java.lang.ArithmeticException: integer overflow + +!error + +SELECT -(INTERVAL -9223372036854775.808 SECONDS); +java.lang.ArithmeticException: long overflow + +!error + +SELECT INTERVAL 3000000 months * 1000; +java.lang.ArithmeticException: integer overflow + +!error + +SELECT INTERVAL 3000000 months / .0001; +java.lang.ArithmeticException: Overflow + +!error + select deptno, (select min(empno) from "scott".emp where deptno = dept.deptno) as x from "scott".dept; +--------+------+ | DEPTNO | X | From 4ff2dbb6aca342b3bf38e18e7398bbcfa86401a1 Mon Sep 17 00:00:00 2001 From: xuzifu666 <1206332514@qq.com> Date: Wed, 15 Apr 2026 18:40:43 +0800 Subject: [PATCH 220/562] [CALCITE-7472] Arrow adapter should support LIKE operator push down --- .../adapter/arrow/ArrowTranslator.java | 2 + .../adapter/arrow/ArrowAdapterTest.java | 84 +++++++++++++++++++ 2 files changed, 86 insertions(+) diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTranslator.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTranslator.java index cb27096a0984..0ec680405270 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTranslator.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTranslator.java @@ -144,6 +144,8 @@ private ConditionToken translateMatch2(RexNode node) { return ConditionToken.unary(fieldNames.get(inputRef.getIndex()), "istrue"); case NOT: return translateUnary("isfalse", (RexCall) node); + case LIKE: + return translateBinary("like", null, (RexCall) node); default: throw new UnsupportedOperationException("Unsupported operator " + node); } diff --git a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java index e8d09d6bd961..25237eed2769 100644 --- a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java +++ b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java @@ -1167,4 +1167,88 @@ static void initializeArrowState(@TempDir Path sharedTempDir) .returns(result) .explainContains(plan); } + + /** Test case for + * [CALCITE-7472] + * Arrow adapter should support LIKE operator push down. */ + @Test void testArrowProjectFieldsWithLikePrefixFilter() { + String sql = "select \"stringField\"\n" + + "from arrowdatatype\n" + + "where \"stringField\" like '1%'"; + String plan = "PLAN=ArrowToEnumerableConverter\n" + + " ArrowProject(stringField=[$3])\n" + + " ArrowFilter(condition=[LIKE($3, '1%')])\n" + + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], " + + "fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]])\n\n"; + String result = "stringField=1\nstringField=10\n"; + + CalciteAssert.that() + .with(arrow) + .query(sql) + .limit(2) + .returns(result) + .explainContains(plan); + } + + /** Test case for + * [CALCITE-7472] + * Arrow adapter should support LIKE operator push down. */ + @Test void testArrowProjectFieldsWithLikeSuffixFilter() { + String sql = "select \"stringField\"\n" + + "from arrowdatatype\n" + + "where \"stringField\" like '%5'"; + String plan = "PLAN=ArrowToEnumerableConverter\n" + + " ArrowProject(stringField=[$3])\n" + + " ArrowFilter(condition=[LIKE($3, '%5')])\n" + + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], " + + "fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]])\n\n"; + + CalciteAssert.that() + .with(arrow) + .query(sql) + .returnsCount(5) // 5, 15, 25, 35, 45 + .explainContains(plan); + } + + /** Test case for + * [CALCITE-7472] + * Arrow adapter should support LIKE operator push down. */ + @Test void testArrowProjectFieldsWithLikeContainsFilter() { + String sql = "select \"stringField\"\n" + + "from arrowdatatype\n" + + "where \"stringField\" like '%2%'"; + String plan = "PLAN=ArrowToEnumerableConverter\n" + + " ArrowProject(stringField=[$3])\n" + + " ArrowFilter(condition=[LIKE($3, '%2%')])\n" + + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]]," + + " fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]])\n\n"; + + CalciteAssert.that() + .with(arrow) + .query(sql) + .returnsCount(14) // 2, 12, 20-29, 32, 42 + .explainContains(plan); + } + + /** Test case for + * [CALCITE-7472] + * Arrow adapter should support LIKE operator push down. */ + @Test void testArrowProjectFieldsWithLikeSingleCharFilter() { + String sql = "select \"stringField\"\n" + + "from arrowdatatype\n" + + "where \"stringField\" like '1_'"; + String plan = "PLAN=ArrowToEnumerableConverter\n" + + " ArrowProject(stringField=[$3])\n" + + " ArrowFilter(condition=[LIKE($3, '1_')])\n" + + " ArrowTableScan(table=[[ARROW, ARROWDATATYPE]], " + + "fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]])\n\n"; + String result = "stringField=10\nstringField=11\n"; + + CalciteAssert.that() + .with(arrow) + .query(sql) + .limit(2) + .returns(result) + .explainContains(plan); + } } From c59bbdb9ef92c37563649c70615d09b529fcc934 Mon Sep 17 00:00:00 2001 From: Sergey Nuyanzin Date: Wed, 15 Apr 2026 21:55:35 +0200 Subject: [PATCH 221/562] [CALCITE-7474] `LAST` in `MATCH_RECOGNIZE` might return wrong result --- .../adapter/enumerable/MatchUtils.java | 19 +++++++- .../adapter/enumerable/RexImpTable.java | 45 ++++++------------- .../apache/calcite/util/BuiltInMethod.java | 2 + core/src/test/resources/sql/match.iq | 16 +++++++ 4 files changed, 50 insertions(+), 32 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/MatchUtils.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/MatchUtils.java index 254ffc289aa6..d7096f0fb1c9 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/MatchUtils.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/MatchUtils.java @@ -27,7 +27,7 @@ private MatchUtils() { } /** - * Returns the row with the highest index whose corresponding symbol matches, null otherwise. + * Returns the highest index whose corresponding symbol matches, -1 otherwise. * * @param symbol Target Symbol * @param rows List of passed rows @@ -44,6 +44,23 @@ public static int lastWithSymbol(String symbol, List rows, List s return -1; } + /** + * Returns the highest index whose corresponding symbol matches, startIndex otherwise. + * + * @param symbol Target Symbol + * @param symbols Corresponding symbols to rows + * @return index or startIndex + */ + public static int lastWithSymbolOrLast(String symbol, List symbols, + int startIndex) { + for (int i = startIndex; i >= 0; i--) { + if (symbol.equals(symbols.get(i))) { + return i; + } + } + return startIndex; + } + public static void print(int s) { System.out.println(s); } diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java index 42afb720fce6..549bddbf724d 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java @@ -22,7 +22,6 @@ import org.apache.calcite.avatica.util.DateTimeUtils; import org.apache.calcite.avatica.util.TimeUnit; import org.apache.calcite.avatica.util.TimeUnitRange; -import org.apache.calcite.linq4j.tree.BinaryExpression; import org.apache.calcite.linq4j.tree.BlockBuilder; import org.apache.calcite.linq4j.tree.BlockStatement; import org.apache.calcite.linq4j.tree.ConstantExpression; @@ -4173,43 +4172,27 @@ private static class LastImplementor implements MatchImplementor { final String alpha = ((RexPatternFieldRef) call.getOperands().get(0)).getAlpha(); - // TODO: verify if the variable is needed - @SuppressWarnings("unused") - final BinaryExpression lastIndex = - Expressions.subtract( - Expressions.call(rows, BuiltInMethod.COLLECTION_SIZE.method), - Expressions.constant(1)); - // Just take the last one, if exists if ("*".equals(alpha)) { setInputGetterIndex(translator, i); - // Important, unbox the node / expression to avoid NullAs.NOT_POSSIBLE - final RexPatternFieldRef ref = (RexPatternFieldRef) node; - final RexPatternFieldRef newRef = - new RexPatternFieldRef(ref.getAlpha(), - ref.getIndex(), - translator.typeFactory.createTypeWithNullability(ref.getType(), - true)); - final Expression expression = translator.translate(newRef, NullAs.NULL); - setInputGetterIndex(translator, null); - return expression; } else { // Alpha != "*" so we have to search for a specific one to find and use that, if found + // otherwise pick the last one setInputGetterIndex(translator, - Expressions.call(BuiltInMethod.MATCH_UTILS_LAST_WITH_SYMBOL.method, - Expressions.constant(alpha), rows, symbols, i)); - - // Important, unbox the node / expression to avoid NullAs.NOT_POSSIBLE - final RexPatternFieldRef ref = (RexPatternFieldRef) node; - final RexPatternFieldRef newRef = - new RexPatternFieldRef(ref.getAlpha(), - ref.getIndex(), - translator.typeFactory.createTypeWithNullability(ref.getType(), - true)); - final Expression expression = translator.translate(newRef, NullAs.NULL); - setInputGetterIndex(translator, null); - return expression; + Expressions.call(BuiltInMethod.MATCH_UTILS_LAST_WITH_SYMBOL_OR_LAST.method, + Expressions.constant(alpha), symbols, i)); } + + // Important, unbox the node / expression to avoid NullAs.NOT_POSSIBLE + final RexPatternFieldRef ref = (RexPatternFieldRef) node; + final RexPatternFieldRef newRef = + new RexPatternFieldRef(ref.getAlpha(), + ref.getIndex(), + translator.typeFactory.createTypeWithNullability(ref.getType(), + true)); + final Expression expression = translator.translate(newRef, NullAs.NULL); + setInputGetterIndex(translator, null); + return expression; } private static void setInputGetterIndex(RexToLixTranslator translator, @Nullable Expression o) { diff --git a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java index 61aebe15f51e..98b67a02bbf3 100644 --- a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java +++ b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java @@ -245,6 +245,8 @@ public enum BuiltInMethod { MATCHER_BUILDER_BUILD(Matcher.Builder.class, "build"), MATCH_UTILS_LAST_WITH_SYMBOL(MatchUtils.class, "lastWithSymbol", String.class, List.class, List.class, int.class), + MATCH_UTILS_LAST_WITH_SYMBOL_OR_LAST(MatchUtils.class, "lastWithSymbolOrLast", String.class, + List.class, int.class), EMITTER_EMIT(Enumerables.Emitter.class, "emit", List.class, List.class, List.class, int.class, Consumer.class), MERGE_JOIN(EnumerableDefaults.class, "mergeJoin", Enumerable.class, diff --git a/core/src/test/resources/sql/match.iq b/core/src/test/resources/sql/match.iq index 887c781c15ec..edee26b9696d 100644 --- a/core/src/test/resources/sql/match.iq +++ b/core/src/test/resources/sql/match.iq @@ -140,6 +140,22 @@ C EMPID !ok +# Test Simple LAST with expanded column name +# Test case for CALCITE-7474, the behavior is similar to BigQuery +select * +from "hr"."emps" match_recognize ( + order by "empid" desc + measures "commission" as c, + LAST("hr"."emps"."empid") as empid + pattern (s up) + define up as up."commission" < prev(up."commission")); +C EMPID +---- ----- +1000 100 + 500 200 + +!ok + # Test LAST with Classifier select * from "hr"."emps" match_recognize ( From 872eacb367fb9095bf01bf1aadb93409681efb1d Mon Sep 17 00:00:00 2001 From: Sergey Nuyanzin Date: Sun, 12 Apr 2026 20:42:27 +0200 Subject: [PATCH 222/562] [CALCITE-7480] Unparse of `MATCH_RECOGNIZE` with `PARTITION BY` or `ORDER BY` produces invalid SQL --- .../sql/validate/MatchRecognizeScope.java | 15 ++++- .../rel/rel2sql/RelToSqlConverterTest.java | 4 +- .../apache/calcite/test/SqlValidatorTest.java | 55 +++++++++++++++++++ 3 files changed, 71 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql/validate/MatchRecognizeScope.java b/core/src/main/java/org/apache/calcite/sql/validate/MatchRecognizeScope.java index 89df7fbb9517..df5d8de14db2 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/MatchRecognizeScope.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/MatchRecognizeScope.java @@ -18,6 +18,9 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.StructKind; +import org.apache.calcite.sql.SqlBasicCall; +import org.apache.calcite.sql.SqlIdentifier; +import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlMatchRecognize; import org.apache.calcite.sql.SqlNode; @@ -70,7 +73,17 @@ public void addPatternVar(String str) { for (ScopeChild child : children) { final RelDataType rowType = child.namespace.getRowType(); if (nameMatcher.field(rowType, columnName) != null) { - map.put(STAR, child); + SqlNode tableRef = matchRecognize.getTableRef(); + + assert tableRef instanceof SqlIdentifier || tableRef.getKind() == SqlKind.AS; + + String tableName; + if (tableRef.getKind() == SqlKind.AS) { + tableName = ((SqlBasicCall) tableRef).getOperandList().get(1).toString(); + } else { + tableName = tableRef.toString(); + } + map.put(tableName, child); } } switch (map.size()) { diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java index 2f881236d1ce..3c5dc7c4b3df 100644 --- a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java +++ b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java @@ -7343,8 +7343,8 @@ private void checkLiteral2(String expression, String expected) { + "MEASURES " + "FINAL \"STRT\".\"net_weight\" AS \"START_NW\", " + "FINAL COUNT(\"UP\".\"net_weight\") AS \"UP_CNT\", " - + "FINAL COUNT(\"*\".\"net_weight\") AS \"DOWN_CNT\", " - + "FINAL (RUNNING COUNT(\"*\".\"net_weight\")) AS \"RUNNING_CNT\"\n" + + "FINAL COUNT(\"product\".\"net_weight\") AS \"DOWN_CNT\", " + + "FINAL (RUNNING COUNT(\"product\".\"net_weight\")) AS \"RUNNING_CNT\"\n" + "ONE ROW PER MATCH\n" + "AFTER MATCH SKIP TO NEXT ROW\n" + "PATTERN (\"STRT\" \"DOWN\" + \"UP\" +)\n" diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index 87383d00b317..361f12eb34a5 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -2600,6 +2600,61 @@ void testLikeAndSimilarFails() { sql(expected6) .withParserConfig(c -> c.withQuoting(Quoting.BACK_TICK)).ok(); + + // Test cases for [CALCITE-7480] https://issues.apache.org/jira/browse/CALCITE-7480 + // Unparse of MATCH_RECOGNIZE with PARTITION BY or ORDER BY produces invalid SQL + // Accepted by BigQuery (both simple and expanded column name under ORDER BY, PARTITION BY) + // Oracle accepts only simple column name + final String sql7 = "SELECT *\n" + + "FROM sales.emp AS emp_alias\n" + + "MATCH_RECOGNIZE (\n" + + " PARTITION BY empno\n" + + " ORDER BY deptno\n" + + " MEASURES\n" + + " FINAL COUNT(A.deptno) AS deptno\n" + + " PATTERN (A B)\n" + + " DEFINE\n" + + " A AS A.empno = 123\n" + + ") AS T"; + final String expected7 = "SELECT `T`.`EMPNO`, `T`.`DEPTNO`\n" + + "FROM `CATALOG`.`SALES`.`EMP` AS `EMP_ALIAS` MATCH_RECOGNIZE(\n" + + "PARTITION BY `EMP_ALIAS`.`EMPNO`\n" + + "ORDER BY `EMP_ALIAS`.`DEPTNO`\n" + + "MEASURES FINAL COUNT(`A`.`DEPTNO`) AS `DEPTNO`\n" + + "PATTERN (`A` `B`)\n" + + "DEFINE `A` AS PREV(`A`.`EMPNO`, 0) = 123) AS `T`"; + + sql(sql7) + .withValidatorConfig(c -> c.withIdentifierExpansion(true)) + .rewritesTo(expected7); + sql(expected7) + .withParserConfig(c -> c.withQuoting(Quoting.BACK_TICK)).ok(); + + final String sql8 = "SELECT *\n" + + "FROM (SELECT empno, deptno from sales.emp)\n" + + "MATCH_RECOGNIZE (\n" + + " PARTITION BY empno\n" + + " ORDER BY deptno\n" + + " MEASURES\n" + + " FINAL COUNT(A.deptno) AS deptno\n" + + " PATTERN (A B)\n" + + " DEFINE\n" + + " A AS A.empno = 123\n" + + ")"; + final String expected8 = "SELECT `EXPR$0`.`EMPNO`, `EXPR$0`.`DEPTNO`\n" + + "FROM (SELECT `EMP`.`EMPNO`, `EMP`.`DEPTNO`\n" + + "FROM `CATALOG`.`SALES`.`EMP` AS `EMP`) AS `EXPR$1` MATCH_RECOGNIZE(\n" + + "PARTITION BY `EXPR$1`.`EMPNO`\n" + + "ORDER BY `EXPR$1`.`DEPTNO`\n" + + "MEASURES FINAL COUNT(`A`.`DEPTNO`) AS `DEPTNO`\n" + + "PATTERN (`A` `B`)\n" + + "DEFINE `A` AS PREV(`A`.`EMPNO`, 0) = 123) AS `EXPR$0`"; + + sql(sql8) + .withValidatorConfig(c -> c.withIdentifierExpansion(true)) + .rewritesTo(expected8); + sql(expected8) + .withParserConfig(c -> c.withQuoting(Quoting.BACK_TICK)).ok(); } @Test void testIntervalTimeUnitEnumeration() { From 1c4531502f244d6988046416d00b7f8a16f48f7a Mon Sep 17 00:00:00 2001 From: Sergey Nuyanzin Date: Sun, 19 Apr 2026 16:14:11 +0200 Subject: [PATCH 223/562] [CALCITE-7481] Support jdk24 in CI --- .github/workflows/main.yml | 8 ++++---- bom/build.gradle.kts | 2 +- build.gradle.kts | 17 +++-------------- buildSrc/build.gradle.kts | 7 ++++--- buildSrc/gradle.properties | 2 +- core/build.gradle.kts | 6 ++++-- gradle.properties | 12 ++++++------ release/build.gradle.kts | 1 + site/_docs/howto.md | 4 ++-- spark/build.gradle.kts | 17 ++++++++++++++++- testkit/build.gradle.kts | 6 ++++-- 11 files changed, 46 insertions(+), 36 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e27d0820fec9..a7ac82d07666 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -247,18 +247,18 @@ jobs: remote-build-cache-proxy-enabled: false arguments: --scan --no-parallel --no-daemon -Pguava.version=${{ env.GUAVA }} build - linux-jdk23: # latest JDK version supported by ForbiddenAPIs plugin, keep this updated (see https://jdk.java.net/) + linux-jdk24: # latest JDK version supported by ForbiddenAPIs plugin, keep this updated (see https://jdk.java.net/) if: github.event.action != 'labeled' - name: 'Linux (JDK 23)' + name: 'Linux (JDK 24)' runs-on: ubuntu-latest steps: - uses: actions/checkout@v5 with: fetch-depth: 50 - - name: 'Set up JDK 23' + - name: 'Set up JDK 24' uses: actions/setup-java@v5 with: - java-version: 23 + java-version: 24 distribution: 'zulu' - uses: burrunan/gradle-cache-action@v1 name: Test diff --git a/bom/build.gradle.kts b/bom/build.gradle.kts index 5a1ad75511d0..f00ed7d8e556 100644 --- a/bom/build.gradle.kts +++ b/bom/build.gradle.kts @@ -124,7 +124,7 @@ dependencies { apiv("org.apache.logging.log4j:log4j-slf4j-impl", "log4j2") apiv("org.apache.pig:pig") apiv("org.apache.pig:pigunit", "pig") - apiv("org.apache.spark:spark-core_2.10", "spark") + apiv("org.apache.spark:spark-core_2.13", "spark") apiv("org.apiguardian:apiguardian-api") apiv("net.bytebuddy:byte-buddy") apiv("org.cassandraunit:cassandra-unit") diff --git a/build.gradle.kts b/build.gradle.kts index 95c38f1a0978..be7569e17fef 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -434,7 +434,7 @@ allprojects { if (!skipAutostyle) { apply(plugin = "com.github.autostyle") - autostyle { + configure { kotlinGradle { license() ktlint() @@ -473,7 +473,7 @@ allprojects { } } plugins.withId("org.jetbrains.kotlin.jvm") { - autostyle { + configure { kotlin { licenseHeader(rootProject.ide.licenseHeader) ktlint { @@ -615,7 +615,7 @@ allprojects { } if (!skipAutostyle) { - autostyle { + configure { java { filter.exclude(*javaccGeneratedPatterns + "**/test/java/*.java" + @@ -895,17 +895,6 @@ allprojects { showStandardStreams = true } exclude("**/*Suite*") - if (JavaVersion.current() >= JavaVersion.VERSION_23) { - // Subject.doAs is deprecated and does not work in JDK 23 - // and higher unless the (also deprecated) SecurityManager - // is enabled. However, we depend on libraries Avatica and - // Hadoop for our remote driver and Pig and Spark - // adapters. So as a workaround we require enabling the - // security manager on JDK 23 and higher. See - // [CALCITE-6587], [CALCITE-6590] (Avatica), [HADOOP-19212], - // https://openjdk.org/jeps/411. - jvmArgs("-Djava.security.manager=allow") - } jvmArgs("-Xmx1536m") jvmArgs("-Djdk.net.URLClassPath.disableClassPathURLCheck=true") // Pass the property to tests diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts index 62a9436fb4f0..0dc89b8bbd77 100644 --- a/buildSrc/build.gradle.kts +++ b/buildSrc/build.gradle.kts @@ -62,13 +62,14 @@ fun Project.applyKotlinProjectConventions() { } tasks.withType().configureEach { - kotlinOptions { - jvmTarget = "1.8" + compilerOptions { + jvmTarget.set( + org.jetbrains.kotlin.gradle.dsl.JvmTarget.JVM_1_8) } } if (!skipAutostyle) { apply(plugin = "com.github.autostyle") - autostyle { + configure { kotlin { ktlint() trimTrailingWhitespace() diff --git a/buildSrc/gradle.properties b/buildSrc/gradle.properties index 767eb7a6192e..f308e6c2451a 100644 --- a/buildSrc/gradle.properties +++ b/buildSrc/gradle.properties @@ -18,5 +18,5 @@ org.gradle.parallel=true kotlin.code.style=official # Plugins -com.github.autostyle.version=3.0 +com.github.autostyle.version=3.2 com.github.vlsi.vlsi-release-plugins.version=1.52 diff --git a/core/build.gradle.kts b/core/build.gradle.kts index 2cb75e9af0ba..086f114480e4 100644 --- a/core/build.gradle.kts +++ b/core/build.gradle.kts @@ -212,8 +212,10 @@ tasks.withType().configureEach { } tasks.withType().configureEach { - kotlinOptions { - jvmTarget = "1.8" + compilerOptions { + jvmTarget.set( + org.jetbrains.kotlin.gradle.dsl.JvmTarget.JVM_1_8 + ) } } ide { diff --git a/gradle.properties b/gradle.properties index 297c059964ef..5e51d8df35ec 100644 --- a/gradle.properties +++ b/gradle.properties @@ -43,14 +43,14 @@ calcite.avatica.version=1.27.0 # Plugins com.autonomousapps.dependency-analysis.version=0.71.0 org.checkerframework.version=0.5.16 -com.github.autostyle.version=3.0 +com.github.autostyle.version=3.2 com.github.johnrengelman.shadow.version=5.1.0 com.github.spotbugs.version=2.0.0 com.github.vlsi.vlsi-release-plugins.version=3.0.1 com.google.protobuf.version=0.8.10 -de.thetaphi.forbiddenapis.version=3.7 -jacoco.version=0.8.12 -kotlin.version=2.0.21 +de.thetaphi.forbiddenapis.version=3.10 +jacoco.version=0.8.14 +kotlin.version=2.3.20 net.ltgt.errorprone.version=1.3.0 me.champeau.jmh.version=0.7.2 org.jetbrains.gradle.plugin.idea-ext.version=1.4.1 @@ -110,7 +110,7 @@ foodmart-queries.version=0.4.1 geode-core.version=1.15.1 guava.version=33.4.8-jre h2.version=2.1.210 -hadoop.version=2.10.2 +hadoop.version=3.4.3 hamcrest-date.version=2.0.4 hamcrest.version=2.1 hsqldb.version=2.7.2 @@ -160,7 +160,7 @@ scott-data-hsqldb.version=0.2 servlet.version=4.0.1 sketches-core.version=0.9.0 slf4j.version=1.7.25 -spark.version=2.2.2 +spark.version=3.5.8 sqlline.version=1.12.0 sql-logic-test.version=0.3 steelwheels-data-hsqldb.version=0.2 diff --git a/release/build.gradle.kts b/release/build.gradle.kts index 9c3b2ab33ab3..985f8fe874e3 100644 --- a/release/build.gradle.kts +++ b/release/build.gradle.kts @@ -30,6 +30,7 @@ import java.time.Instant import java.time.temporal.ChronoUnit plugins { + base id("com.github.vlsi.stage-vote-release") } diff --git a/site/_docs/howto.md b/site/_docs/howto.md index 9035a2c5524f..47a5ac888f2e 100644 --- a/site/_docs/howto.md +++ b/site/_docs/howto.md @@ -31,7 +31,7 @@ adapters. ## Building from a source distribution -Prerequisite is Java (JDK 8, 11, 17, 21 or 23) +Prerequisite is Java (JDK 8, 11, 17, 21 or 24) and Gradle (version 8.14.4) on your path. Unpack the source distribution `.tar.gz` file, @@ -51,7 +51,7 @@ tests (but you should use the `gradle` command rather than ## Building from Git Prerequisites are git -and Java (JDK 8, 11, 17, 21 or 23) on your path. +and Java (JDK 8, 11, 17, 21 or 24) on your path. Create a local copy of the GitHub repository, `cd` to its root directory, diff --git a/spark/build.gradle.kts b/spark/build.gradle.kts index b194e0fd5e60..4975686f455a 100644 --- a/spark/build.gradle.kts +++ b/spark/build.gradle.kts @@ -14,10 +14,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +configurations.all { + resolutionStrategy.eachDependency { + if (requested.group == "org.apache.hadoop") { + // minimum hadoop version working on jdk 24+ + useVersion("3.4.3") + } + } +} + dependencies { api(project(":core")) api(project(":linq4j")) - api("org.apache.spark:spark-core_2.10") { + api("org.apache.spark:spark-core_2.13") { exclude("org.slf4j", "slf4j-log4j12") .because("conflicts with log4j-slf4j-impl") exclude("org.slf4j", "slf4j-reload4j") @@ -34,4 +43,10 @@ dependencies { testImplementation(project(":testkit")) testRuntimeOnly("org.apache.logging.log4j:log4j-slf4j-impl") + + tasks.withType().configureEach { + if (JavaVersion.current() >= JavaVersion.VERSION_17) { + jvmArgs("--add-exports=java.base/sun.nio.ch=ALL-UNNAMED") + } + } } diff --git a/testkit/build.gradle.kts b/testkit/build.gradle.kts index 613526ebd06d..f164b4acfd30 100644 --- a/testkit/build.gradle.kts +++ b/testkit/build.gradle.kts @@ -43,7 +43,9 @@ dependencies { } tasks.withType().configureEach { - kotlinOptions { - jvmTarget = "1.8" + compilerOptions { + jvmTarget.set( + org.jetbrains.kotlin.gradle.dsl.JvmTarget.JVM_1_8 + ) } } From 09ecb962dae807bc91f1993f9b83b0bd136e194b Mon Sep 17 00:00:00 2001 From: "ian.bertolacci" Date: Wed, 15 Apr 2026 10:47:12 -0700 Subject: [PATCH 224/562] [CALCITE-7473] Better IntelliJ and VSCode .gitignore --- .gitignore | 24 ++++++++---------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/.gitignore b/.gitignore index 73435831150f..85e63264c02c 100644 --- a/.gitignore +++ b/.gitignore @@ -16,26 +16,18 @@ *~ .DS_Store .gradle -/target -/*/target -/example/*/target -/build -/*/build -/example/*/build -/buildSrc/build -/buildSrc/subprojects/*/build +/**/target +/**/build /site/.jekyll-cache -# VSCode Java plugin -/example/*/bin -/*/bin -/bin -/.vscode/* +# VSCode and plugins +/**/bin +.vscode +.metals # IDEA -/out -/*/out/ -/example/*/out +/**/out +/**/generated # The star is required for further !/.idea/ to work, see https://git-scm.com/docs/gitignore /.idea/* # Icon for JetBrains Toolbox From 708f5c7bc21f89a6d8e9a756df47853d8416a517 Mon Sep 17 00:00:00 2001 From: Joseph Grogan Date: Fri, 17 Apr 2026 12:42:48 -0400 Subject: [PATCH 225/562] [CALCITE-7477] Push schema pattern filter into sub-schema map lookup to avoid loading all schemas --- .../apache/calcite/jdbc/CalciteMetaImpl.java | 32 +++++++++---------- .../org/apache/calcite/test/JdbcTest.java | 11 +++++++ 2 files changed, 27 insertions(+), 16 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/jdbc/CalciteMetaImpl.java b/core/src/main/java/org/apache/calcite/jdbc/CalciteMetaImpl.java index 2c5a3d2ce124..e294643fca2a 100644 --- a/core/src/main/java/org/apache/calcite/jdbc/CalciteMetaImpl.java +++ b/core/src/main/java/org/apache/calcite/jdbc/CalciteMetaImpl.java @@ -354,9 +354,7 @@ private static ImmutableMap.Builder addProperty( } else { typeFilter = v1 -> typeList.contains(v1.tableType); } - final Predicate1 schemaMatcher = namedMatcher(schemaPattern); - Enumerable tables = schemas(catalog) - .where(schemaMatcher) + Enumerable tables = schemas(catalog, new LikePattern(schemaPattern.s)) .selectMany(schema -> tables(schema, new LikePattern(tableNamePattern.s))) .where(typeFilter); return createResultSet(tables, @@ -374,11 +372,9 @@ private static ImmutableMap.Builder addProperty( Pat schemaPattern, Pat tableNamePattern, Pat columnNamePattern) { - final Predicate1 schemaMatcher = namedMatcher(schemaPattern); final Predicate1 columnMatcher = namedMatcher(columnNamePattern); - return createResultSet(schemas(catalog) - .where(schemaMatcher) + return createResultSet(schemas(catalog, new LikePattern(schemaPattern.s)) .selectMany(schema -> tables(schema, new LikePattern(tableNamePattern.s))) .selectMany(this::columns) .where(columnMatcher), @@ -404,11 +400,18 @@ Enumerable tableTypes() { } Enumerable schemas(final String catalog) { - return Linq4j.asEnumerable( - getConnection().rootSchema.getSubSchemaMap().values()) - .select((Function1) calciteSchema -> - new CalciteMetaSchema(calciteSchema, catalog, - calciteSchema.getName())) + return schemas(catalog, LikePattern.any()); + } + + Enumerable schemas(final String catalog, final LikePattern pattern) { + final CalciteSchema root = getConnection().rootSchema; + return Linq4j.asEnumerable(root.subSchemas().getNames(pattern)) + .select((Function1) name -> { + final CalciteSchema schema = + requireNonNull(root.getSubSchema(name, true), + () -> "sub-schema " + name + " is not found (case sensitive)"); + return new CalciteMetaSchema(schema, catalog, schema.getName()); + }) .orderBy((Function1) metaSchema -> (Comparable) FlatLists.of(Util.first(metaSchema.tableCatalog, ""), metaSchema.tableSchem)); @@ -524,8 +527,7 @@ public Enumerable columns(final MetaTable table_) { @Override public MetaResultSet getSchemas(ConnectionHandle ch, String catalog, Pat schemaPattern) { - final Predicate1 schemaMatcher = namedMatcher(schemaPattern); - return createResultSet(schemas(catalog).where(schemaMatcher), + return createResultSet(schemas(catalog, new LikePattern(schemaPattern.s)), MetaSchema.class, SCHEMA_COLUMNS); } @@ -543,9 +545,7 @@ public Enumerable columns(final MetaTable table_) { String catalog, Pat schemaPattern, Pat functionNamePattern) { - final Predicate1 schemaMatcher = namedMatcher(schemaPattern); - return createResultSet(schemas(catalog) - .where(schemaMatcher) + return createResultSet(schemas(catalog, new LikePattern(schemaPattern.s)) .selectMany(schema -> functions(schema, catalog, matcher(functionNamePattern))) .orderBy(x -> (Comparable) FlatLists.of( diff --git a/core/src/test/java/org/apache/calcite/test/JdbcTest.java b/core/src/test/java/org/apache/calcite/test/JdbcTest.java index e25c6f31ccf2..e4dca3fc448d 100644 --- a/core/src/test/java/org/apache/calcite/test/JdbcTest.java +++ b/core/src/test/java/org/apache/calcite/test/JdbcTest.java @@ -6541,6 +6541,17 @@ private CalciteAssert.AssertThat modelWithView(String view, is("TABLE_SCHEM=adhoc; TABLE_CATALOG=null\n")); } + // schemas (qualified, non-existent) + try (ResultSet r = metaData.getSchemas(null, "nonexistent")) { + assertThat(CalciteAssert.toString(r), is("")); + } + + // schemas (qualified, missing) + try (ResultSet r = metaData.getSchemas(null, "adho%")) { + assertThat(CalciteAssert.toString(r), + is("TABLE_SCHEM=adhoc; TABLE_CATALOG=null\n")); + } + // table types try (ResultSet r = metaData.getTableTypes()) { assertThat(CalciteAssert.toString(r), From aa1a739b2784fea71acdc4414bcc1d217adcd4b7 Mon Sep 17 00:00:00 2001 From: Silun Date: Tue, 21 Apr 2026 10:43:47 +0800 Subject: [PATCH 226/562] [CALCITE-7482] Wrong variablesSet used when rewriting subquery in JOIN ON clause --- .../org/apache/calcite/prepare/Prepare.java | 4 +++- .../calcite/rel/rules/SubQueryRemoveRule.java | 1 + .../calcite/sql2rel/RelDecorrelatorTest.java | 9 ++++---- core/src/test/resources/sql/sub-query.iq | 21 +++++++++++++++++++ 4 files changed, 29 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/prepare/Prepare.java b/core/src/main/java/org/apache/calcite/prepare/Prepare.java index 1c6d8875cb16..1f2e96688a25 100644 --- a/core/src/main/java/org/apache/calcite/prepare/Prepare.java +++ b/core/src/main/java/org/apache/calcite/prepare/Prepare.java @@ -297,7 +297,9 @@ public PreparedResult prepareSql( // storage. root = root.withRel(flattenTypes(root.rel, true)); - if (this.context.config().forceDecorrelate()) { + // TopDownGeneralDecorrelator cannot be used until the subquerys are completely removed. + if (this.context.config().forceDecorrelate() + && !this.context.config().topDownGeneralDecorrelationEnabled()) { // Sub-query decorrelation. root = root.withRel(decorrelate(sqlToRelConverter, sqlQuery, root.rel)); } diff --git a/core/src/main/java/org/apache/calcite/rel/rules/SubQueryRemoveRule.java b/core/src/main/java/org/apache/calcite/rel/rules/SubQueryRemoveRule.java index 43d5adc27e2e..f7a6ce552fe5 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/SubQueryRemoveRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/SubQueryRemoveRule.java @@ -1041,6 +1041,7 @@ private static void matchJoin(SubQueryRemoveRule rule, RelOptRuleCall call) { // // In such a case $cor0.DNAME need to be accounted as input form left side. final Set variablesSet = RelOptUtil.getVariablesUsed(e.rel); + variablesSet.retainAll(join.getVariablesSet()); for (CorrelationId id : variablesSet) { ImmutableBitSet requiredColumns = RelOptUtil.correlationColumns(id, e.rel); inputSet = ImmutableBitSet.union(ImmutableList.of(requiredColumns, inputSet)); diff --git a/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java b/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java index e91b13886139..99b745319203 100644 --- a/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java +++ b/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java @@ -2099,8 +2099,7 @@ public static Frameworks.ConfigBuilder config() { + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4]," + " SAL=[$5], COMM=[$6], DEPTNO=[$7], DEPTNO0=[$9], DNAME=[$10], LOC=[$11])\n" + " LogicalJoin(condition=[true], joinType=[inner])\n" - + " LogicalJoin(condition=[true], joinType=[inner]," - + " variablesSet=[[$cor0, $cor1]])\n" + + " LogicalCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{0}])\n" + " LogicalTableScan(table=[[scott, EMP]])\n" + " LogicalAggregate(group=[{0}])\n" + " LogicalProject(i=[true])\n" @@ -2113,11 +2112,11 @@ public static Frameworks.ConfigBuilder config() { RelDecorrelator.decorrelateQuery(afterJoinSubqueryCorrelate, builder, RuleSets.ofList(Collections.emptyList()), RuleSets.ofList(Collections.emptyList())); final String planAfterDecorrelation = "LogicalProject(EMPNO=[$0])\n" - + " LogicalJoin(condition=[=($1, $10)], joinType=[inner])\n" + + " LogicalJoin(condition=[=($1, $9)], joinType=[inner])\n" + " LogicalTableScan(table=[[scott, EMP]])\n" - + " LogicalProject(DEPTNO=[$11], EMPNO0=[$8], ENAME0=[$9])\n" + + " LogicalProject(DEPTNO=[$11], ENAME0=[$9])\n" + " LogicalJoin(condition=[true], joinType=[inner])\n" - + " LogicalJoin(condition=[true], joinType=[inner])\n" + + " LogicalJoin(condition=[=($0, $8)], joinType=[inner])\n" + " LogicalTableScan(table=[[scott, EMP]])\n" + " LogicalProject(EMPNO=[$0], ENAME=[$1], $f2=[true])\n" + " LogicalFilter(condition=[IS NOT NULL($1)])\n" diff --git a/core/src/test/resources/sql/sub-query.iq b/core/src/test/resources/sql/sub-query.iq index b9d4774cefd0..1a79635e640e 100644 --- a/core/src/test/resources/sql/sub-query.iq +++ b/core/src/test/resources/sql/sub-query.iq @@ -9183,5 +9183,26 @@ SELECT 1 FROM emp where sal > 200 and emp.deptno=dept.deptno); +--------+ (3 rows) +!ok + +WITH t1(id, sal) as (VALUES (1, 10), (2, 20), (3, 30)), +t2(id, sal) as (VALUES (2, 20), (2, 200), (3, 30)), +t3(id, sal) as (VALUES (3, 30), (4, 40)), +t4(id, sal) as (VALUES (2, 200), (5, 50)) +select * from +t1, +lateral( + select t2.id, t2.sal from t2 join t3 on exists( + select t4.id from t4 where t4.id=t1.id and t4.sal=t2.sal + ) +); ++----+-----+-----+------+ +| ID | SAL | ID0 | SAL0 | ++----+-----+-----+------+ +| 2 | 20 | 2 | 200 | +| 2 | 20 | 2 | 200 | ++----+-----+-----+------+ +(2 rows) + !ok # End sub-query.iq From 2e2f4fc1538f5bf4e73691ac9089ccd0737ebc3e Mon Sep 17 00:00:00 2001 From: Alessandro Solimando Date: Wed, 22 Apr 2026 19:21:16 +0200 Subject: [PATCH 227/562] [CALCITE-7483] RelToSqlConverter generates SELECT * despite supportGenerateSelectStar --- .../calcite/rel/rel2sql/SqlImplementor.java | 48 +++++- .../rel/rel2sql/RelToSqlConverterTest.java | 162 ++++++++++++++++++ 2 files changed, 206 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java b/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java index 410cc5ea33fe..f4dd0f686986 100644 --- a/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java +++ b/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java @@ -327,9 +327,10 @@ public Result setOpToSql(SqlSetOperator operator, RelNode rel) { for (Ord input : Ord.zip(rel.getInputs())) { final Result result = visitInput(rel, input.i); if (node == null) { - node = result.asSelect(); + node = result.maybeExpandStar(result.asSelect()); } else { - node = operator.createCall(POS, node, result.asSelect()); + node = + operator.createCall(POS, node, result.maybeExpandStar(result.asSelect())); } } if (node == null) { @@ -2026,6 +2027,13 @@ private Builder builder(RelNode rel, Set clauses) { } else { newContext = aliasContext(aliases, qualified); } + if (!dialect.supportGenerateSelectStar(rel.getInput(0))) { + final List expandedSelectList = new ArrayList<>(); + for (int i = 0; i < newContext.fieldCount; i++) { + expandedSelectList.add(newContext.field(i)); + } + select.setSelectList(new SqlNodeList(expandedSelectList, POS)); + } } return new Builder(rel, clauseList, select, newContext, isAnon(), needNew && !aliases.containsKey(neededAlias) ? newAliases : aliases); @@ -2367,10 +2375,42 @@ public SqlNode asStatement() { case MERGE: return maybeStrip(node); default: - return maybeStrip(asSelect()); + return maybeStrip(maybeExpandStar(asSelect())); } } + /** If the dialect does not support {@code SELECT *} and the select list + * is {@link SqlNodeList#SINGLETON_STAR}, replaces it with explicit column + * references derived from the result's aliases. */ + SqlSelect maybeExpandStar(SqlSelect select) { + if (expectedRel != null + && !expectedRel.getInputs().isEmpty() + && select.getSelectList().equals(SqlNodeList.SINGLETON_STAR) + && !dialect.supportGenerateSelectStar(expectedRel.getInput(0))) { + boolean qualified = + !dialect.hasImplicitTableAlias() || aliases.size() > 1; + final Context ctx = aliasContext(aliases, qualified); + final List expandedList = new ArrayList<>(); + for (int i = 0; i < ctx.fieldCount; i++) { + expandedList.add(ctx.field(i)); + } + return new SqlSelect(select.getParserPosition(), + (SqlNodeList) select.getOperandList().get(0), + new SqlNodeList(expandedList, POS), + select.getFrom(), + select.getWhere(), + select.getGroup(), + select.getHaving(), + select.getWindowList(), + select.getQualify(), + select.getOrderList(), + select.getOffset(), + select.getFetch(), + select.getHints()); + } + return select; + } + /** Converts a non-query node into a SELECT node. Set operators (UNION, * INTERSECT, EXCEPT) and VALUES remain as is. */ public SqlNode asQueryOrValues() { @@ -2381,7 +2421,7 @@ public SqlNode asQueryOrValues() { case VALUES: return maybeStrip(node); default: - return maybeStrip(asSelect()); + return maybeStrip(maybeExpandStar(asSelect())); } } diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java index 3c5dc7c4b3df..7c41f6c9d54e 100644 --- a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java +++ b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java @@ -9505,6 +9505,168 @@ private void checkLiteral2(String expression, String expected) { .withPostgresql().ok(expectedPostgres); } + private static final SqlDialect NO_STAR_DIALECT = + new PostgresqlSqlDialect(PostgresqlSqlDialect.DEFAULT_CONTEXT) { + @Override public boolean supportGenerateSelectStar(RelNode relNode) { + return false; + } + }; + + /** Test case for + * [CALCITE-7483] + * RelToSqlConverter generates SELECT * despite supportGenerateSelectStar. + * Bare TableScan. */ + @Test void testNoSelectStarWithBareTableScan() { + final Function relFn = b -> b + .scan("EMP") + .build(); + final String expected = "SELECT \"EMPNO\", \"ENAME\", \"JOB\", \"MGR\"," + + " \"HIREDATE\", \"SAL\", \"COMM\", \"DEPTNO\"\n" + + "FROM \"scott\".\"EMP\""; + relFn(relFn).dialect(NO_STAR_DIALECT).ok(expected); + } + + /** Test case for + * [CALCITE-7483] + * RelToSqlConverter generates SELECT * despite supportGenerateSelectStar. + * Filter without Project. */ + @Test void testNoSelectStarWithFilterOnly() { + final Function relFn = b -> b + .scan("EMP") + .filter( + b.equals(b.field("DEPTNO"), b.literal(10))) + .build(); + final String expected = "SELECT \"EMPNO\", \"ENAME\", \"JOB\", \"MGR\"," + + " \"HIREDATE\", \"SAL\", \"COMM\", \"DEPTNO\"\n" + + "FROM \"scott\".\"EMP\"\n" + + "WHERE \"DEPTNO\" = 10"; + relFn(relFn).dialect(NO_STAR_DIALECT).ok(expected); + } + + /** Test case for + * [CALCITE-7483] + * RelToSqlConverter generates SELECT * despite supportGenerateSelectStar. + * Sort without Project. */ + @Test void testNoSelectStarWithSortOnly() { + final Function relFn = b -> b + .scan("EMP") + .sort(b.field("EMPNO")) + .build(); + final String expected = "SELECT \"EMPNO\", \"ENAME\", \"JOB\", \"MGR\"," + + " \"HIREDATE\", \"SAL\", \"COMM\", \"DEPTNO\"\n" + + "FROM \"scott\".\"EMP\"\n" + + "ORDER BY \"EMPNO\""; + relFn(relFn).dialect(NO_STAR_DIALECT).ok(expected); + } + + /** Test case for + * [CALCITE-7483] + * RelToSqlConverter generates SELECT * despite supportGenerateSelectStar. + * Aggregate without Project. */ + @Test void testNoSelectStarWithAggregateOnly() { + final Function relFn = b -> b + .scan("EMP") + .aggregate(b.groupKey("DEPTNO"), + b.count(false, "CNT")) + .build(); + final String expected = "SELECT \"DEPTNO\", COUNT(*) AS \"CNT\"\n" + + "FROM \"scott\".\"EMP\"\n" + + "GROUP BY \"DEPTNO\""; + relFn(relFn).dialect(NO_STAR_DIALECT).ok(expected); + } + + /** Test case for + * [CALCITE-7483] + * RelToSqlConverter generates SELECT * despite supportGenerateSelectStar. + * Sort over Filter. */ + @Test void testNoSelectStarWithSortAndFilter() { + final Function relFn = b -> b + .scan("EMP") + .filter(b.equals(b.field("DEPTNO"), b.literal(10))) + .sort(b.field("EMPNO")) + .build(); + final String expected = "SELECT \"EMPNO\", \"ENAME\", \"JOB\", \"MGR\"," + + " \"HIREDATE\", \"SAL\", \"COMM\", \"DEPTNO\"\n" + + "FROM \"scott\".\"EMP\"\n" + + "WHERE \"DEPTNO\" = 10\n" + + "ORDER BY \"EMPNO\""; + relFn(relFn).dialect(NO_STAR_DIALECT).ok(expected); + } + + /** Test case for + * [CALCITE-7483] + * RelToSqlConverter generates SELECT * despite supportGenerateSelectStar. + * Limit (fetch). */ + @Test void testNoSelectStarWithLimit() { + final Function relFn = b -> b + .scan("EMP") + .limit(0, 5) + .build(); + final String expected = "SELECT \"EMPNO\", \"ENAME\", \"JOB\", \"MGR\"," + + " \"HIREDATE\", \"SAL\", \"COMM\", \"DEPTNO\"\n" + + "FROM \"scott\".\"EMP\"\n" + + "FETCH NEXT 5 ROWS ONLY"; + relFn(relFn).dialect(NO_STAR_DIALECT).ok(expected); + } + + /** Test case for + * [CALCITE-7483] + * RelToSqlConverter generates SELECT * despite supportGenerateSelectStar. + * Sort over Union. */ + @Test void testNoSelectStarWithUnion() { + final Function relFn = b -> { + b.scan("EMP").project(b.field("EMPNO"), b.field("ENAME")); + b.scan("EMP").project(b.field("EMPNO"), b.field("ENAME")); + return b.union(true).sort(b.field("EMPNO")).build(); + }; + final String expected = "SELECT \"EMPNO\", \"ENAME\"\n" + + "FROM (SELECT \"EMPNO\", \"ENAME\"\n" + + "FROM \"scott\".\"EMP\"\n" + + "UNION ALL\n" + + "SELECT \"EMPNO\", \"ENAME\"\n" + + "FROM \"scott\".\"EMP\") AS \"t\"\n" + + "ORDER BY \"EMPNO\""; + relFn(relFn).dialect(NO_STAR_DIALECT).ok(expected); + } + + /** Test case for + * [CALCITE-7483] + * RelToSqlConverter generates SELECT * despite supportGenerateSelectStar. + * Join. */ + @Test void testNoSelectStarWithJoin() { + final Function relFn = b -> b + .scan("EMP") + .scan("DEPT") + .join(JoinRelType.INNER, + b.equals(b.field(2, 0, "DEPTNO"), + b.field(2, 1, "DEPTNO"))) + .build(); + final String expected = "SELECT" + + " \"EMP\".\"EMPNO\", \"EMP\".\"ENAME\", \"EMP\".\"JOB\"," + + " \"EMP\".\"MGR\", \"EMP\".\"HIREDATE\", \"EMP\".\"SAL\"," + + " \"EMP\".\"COMM\", \"EMP\".\"DEPTNO\"," + + " \"DEPT\".\"DEPTNO\"," + + " \"DEPT\".\"DNAME\", \"DEPT\".\"LOC\"\n" + + "FROM \"scott\".\"EMP\"\n" + + "INNER JOIN \"scott\".\"DEPT\"" + + " ON \"EMP\".\"DEPTNO\" = \"DEPT\".\"DEPTNO\""; + relFn(relFn).dialect(NO_STAR_DIALECT).ok(expected); + } + + /** Test case for + * [CALCITE-7483] + * RelToSqlConverter generates SELECT * despite supportGenerateSelectStar. + * Project (regression test for the original visit(Project) path). */ + @Test void testNoSelectStarWithProject() { + final Function relFn = b -> b + .scan("EMP") + .project(b.field("EMPNO"), b.field("ENAME")) + .build(); + final String expected = "SELECT \"EMPNO\", \"ENAME\"\n" + + "FROM \"scott\".\"EMP\""; + relFn(relFn).dialect(NO_STAR_DIALECT).ok(expected); + } + /** Test case for * [CALCITE-5265] * JDBC adapter sometimes adds unnecessary parentheses around SELECT in INSERT. */ From b147dce4fec91826ea0534b8dc1711b12a3be687 Mon Sep 17 00:00:00 2001 From: xuzifu666 <1206332514@qq.com> Date: Mon, 20 Apr 2026 20:22:50 +0800 Subject: [PATCH 228/562] [CALCITE-6757] Elasticsearch adapter returns wrong result when aggregating sub-query with aggregation --- .../elasticsearch/ElasticsearchRules.java | 36 +++++++++++++++++++ .../elasticsearch/AggregationAndSortTest.java | 11 ++++++ 2 files changed, 47 insertions(+) diff --git a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchRules.java b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchRules.java index e4968c9e673f..3c44992b0a48 100644 --- a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchRules.java +++ b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchRules.java @@ -23,10 +23,12 @@ import org.apache.calcite.plan.RelOptRule; import org.apache.calcite.plan.RelOptRuleCall; import org.apache.calcite.plan.RelTraitSet; +import org.apache.calcite.plan.volcano.RelSubset; import org.apache.calcite.rel.InvalidRelException; import org.apache.calcite.rel.RelCollations; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.convert.ConverterRule; +import org.apache.calcite.rel.core.Aggregate; import org.apache.calcite.rel.core.Sort; import org.apache.calcite.rel.logical.LogicalAggregate; import org.apache.calcite.rel.logical.LogicalFilter; @@ -263,8 +265,42 @@ protected ElasticsearchAggregateRule(Config config) { super(config); } + /** + * Checks if the relational expression or any of its inputs + * is an Aggregate (LogicalAggregate or ElasticsearchAggregate). + */ + private static boolean containsAggregate(RelNode node) { + // Handle RelSubset by checking its best or original node + if (node instanceof RelSubset) { + RelSubset subset = (RelSubset) node; + RelNode best = subset.getBest(); + if (best != null) { + return containsAggregate(best); + } + RelNode original = subset.getOriginal(); + if (original != null) { + return containsAggregate(original); + } + return false; + } + if (node instanceof Aggregate) { + return true; + } + for (RelNode input : node.getInputs()) { + if (containsAggregate(input)) { + return true; + } + } + return false; + } + @Override public @Nullable RelNode convert(RelNode rel) { final LogicalAggregate agg = (LogicalAggregate) rel; + + // Prevent nested aggregations from being pushed down to Elasticsearch + if (containsAggregate(agg.getInput())) { + return null; + } final RelTraitSet traitSet = agg.getTraitSet().replace(out); try { return new ElasticsearchAggregate( diff --git a/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/AggregationAndSortTest.java b/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/AggregationAndSortTest.java index e41c81048b18..9996180ae332 100644 --- a/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/AggregationAndSortTest.java +++ b/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/AggregationAndSortTest.java @@ -555,4 +555,15 @@ private static Connection createConnectionWithConformance(String lex, String con + " group by CAT order by MAX_VAL1 desc, CAT desc limit 2") .returns("CAT=2; MAX_VAL1=7.0\nCAT=1; MAX_VAL1=1.0\n"); } + + /** Test case for + * [CALCITE-6757] + * Elasticsearch adapter returns wrong result when aggregating sub-query with aggregation. + */ + @Test void testAggregateWithSubquery() { + CalciteAssert.that() + .with(AggregationAndSortTest::createConnection) + .query("select count(*) from (select cat5, sum(val1) from view group by cat5) as alias") + .returns("EXPR$0=3\n"); + } } From 99a953e0e203ca67e65c852c79e93b3b24205e44 Mon Sep 17 00:00:00 2001 From: Sergey Nuyanzin Date: Tue, 21 Apr 2026 00:21:17 +0200 Subject: [PATCH 229/562] [CALCITE-7465] Make `MATCH_RECOGNIZE` tolerant to `FINAL` and `RUNNING` non function `MEASURES` --- core/src/main/codegen/templates/Parser.jj | 6 +-- .../apache/calcite/test/SqlValidatorTest.java | 47 +++++++++++++++++++ 2 files changed, 50 insertions(+), 3 deletions(-) diff --git a/core/src/main/codegen/templates/Parser.jj b/core/src/main/codegen/templates/Parser.jj index cd1fbdc38560..f549446ea082 100644 --- a/core/src/main/codegen/templates/Parser.jj +++ b/core/src/main/codegen/templates/Parser.jj @@ -7505,7 +7505,7 @@ SqlCall MatchRecognizeCallWithModifier() : { final Span s; final SqlOperator runningOp; - final SqlNode func; + final SqlNode e; } { ( @@ -7514,8 +7514,8 @@ SqlCall MatchRecognizeCallWithModifier() : { runningOp = SqlStdOperatorTable.FINAL; } ) { s = span(); } - func = NamedFunctionCall() { - return runningOp.createCall(s.end(func), func); + e = Expression3(ExprContext.ACCEPT_NON_QUERY) { + return runningOp.createCall(s.end(e), e); } } diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index 361f12eb34a5..bb79a517ce99 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -2655,6 +2655,53 @@ void testLikeAndSimilarFails() { .rewritesTo(expected8); sql(expected8) .withParserConfig(c -> c.withQuoting(Quoting.BACK_TICK)).ok(); + + // Test cases for [CALCITE-7465] https://issues.apache.org/jira/browse/CALCITE-7465 + // Unparse of MATCH_RECOGNIZE MEASURES might produce unparsable sql + // Accepted by Snowflake (it doesn't accept FINAL or RUNNING before non function measure) + final String sql9 = "SELECT *\n" + + "FROM emp\n" + + "MATCH_RECOGNIZE (\n" + + " MEASURES\n" + + " A.deptno AS deptno\n" + + " PATTERN (A B)\n" + + " DEFINE\n" + + " A AS A.empno = 123\n" + + ")"; + final String expected9 = "SELECT `EXPR$0`.`DEPTNO`\n" + + "FROM `CATALOG`.`SALES`.`EMP` AS `EMP` MATCH_RECOGNIZE(\n" + + "MEASURES FINAL `A`.`DEPTNO` AS `DEPTNO`\n" + + "PATTERN (`A` `B`)\n" + + "DEFINE `A` AS PREV(`A`.`EMPNO`, 0) = 123) AS `EXPR$0`"; + + sql(sql9) + .withValidatorConfig(c -> c.withIdentifierExpansion(true)) + .rewritesTo(expected9); + sql(expected9) + .withParserConfig(c -> c.withQuoting(Quoting.BACK_TICK)).ok(); + + final String sql10 = "SELECT deptno\n" + + "FROM emp\n" + + "MATCH_RECOGNIZE (\n" + + " MEASURES\n" + + " A.deptno AS deptno\n" + + "ALL ROWS PER MATCH\n" + + " PATTERN (A B)\n" + + " DEFINE\n" + + " A AS A.empno = 123\n" + + ")"; + final String expected10 = "SELECT `EXPR$0`.`DEPTNO`\n" + + "FROM `CATALOG`.`SALES`.`EMP` AS `EMP` MATCH_RECOGNIZE(\n" + + "MEASURES RUNNING `A`.`DEPTNO` AS `DEPTNO`\n" + + "ALL ROWS PER MATCH\n" + + "PATTERN (`A` `B`)\n" + + "DEFINE `A` AS PREV(`A`.`EMPNO`, 0) = 123) AS `EXPR$0`"; + + sql(sql10) + .withValidatorConfig(c -> c.withIdentifierExpansion(true)) + .rewritesTo(expected10); + sql(expected10) + .withParserConfig(c -> c.withQuoting(Quoting.BACK_TICK)).ok(); } @Test void testIntervalTimeUnitEnumeration() { From 4832e088d46087be5bf37dff0b33eb233ad2ce5b Mon Sep 17 00:00:00 2001 From: Sergey Nuyanzin Date: Thu, 23 Apr 2026 16:11:49 +0200 Subject: [PATCH 230/562] [CALCITE-7486] Operators in `MATCH_RECOGNIZE` don't support `SqlLiterals` --- .../sql/validate/SqlValidatorImpl.java | 21 ++++++++++++---- .../apache/calcite/test/SqlValidatorTest.java | 25 +++++++++++++++++++ 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java index 6195ffd90448..320ba721533b 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java @@ -8225,17 +8225,21 @@ private class PatternValidator extends SqlBasicVisitor<@Nullable Set> { int firstLastCount; int prevNextCount; int aggregateCount; + int argIndex; + int argCount; PatternValidator(boolean isMeasure) { - this(isMeasure, 0, 0, 0); + this(isMeasure, 0, 0, 0, 0, 0); } PatternValidator(boolean isMeasure, int firstLastCount, int prevNextCount, - int aggregateCount) { + int aggregateCount, int index, int argCount) { this.isMeasure = isMeasure; this.firstLastCount = firstLastCount; this.prevNextCount = prevNextCount; this.aggregateCount = aggregateCount; + this.argIndex = index; + this.argCount = argCount; } @Override public Set visit(SqlCall call) { @@ -8281,13 +8285,14 @@ private class PatternValidator extends SqlBasicVisitor<@Nullable Set> { Static.RESOURCE.patternRunningFunctionInDefine(call.toString())); } - for (SqlNode node : operands) { + for (int i = 0; i < operands.size(); i++) { + SqlNode node = operands.get(i); if (node != null) { vars.addAll( requireNonNull( node.accept( new PatternValidator(isMeasure, firstLastCount, prevNextCount, - aggregateCount)), + aggregateCount, i, operands.size())), () -> "node.accept(PatternValidator) for node " + node)); } } @@ -8329,7 +8334,13 @@ private class PatternValidator extends SqlBasicVisitor<@Nullable Set> { } @Override public Set visit(SqlLiteral literal) { - return ImmutableSet.of(); + if ((this.argCount == 1 || this.argIndex < this.argCount - 1) + && (this.firstLastCount > 0 || this.prevNextCount > 0) + && !SqlUtil.isNull(literal)) { + return ImmutableSet.of(requireNonNull(literal.toValue())); + } else { + return ImmutableSet.of(); + } } @Override public Set visit(SqlIntervalQualifier qualifier) { diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index bb79a517ce99..3d7eda0edf55 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -2702,6 +2702,31 @@ void testLikeAndSimilarFails() { .rewritesTo(expected10); sql(expected10) .withParserConfig(c -> c.withQuoting(Quoting.BACK_TICK)).ok(); + + // Test cases for [CALCITE-7486] https://issues.apache.org/jira/browse/CALCITE-7486 + // Operators in MATCH_RECOGNIZE don't support SqlLiterals + // Accepted by Snowflake + final String sql11 = "SELECT *\n" + + "FROM emp\n" + + "MATCH_RECOGNIZE (\n" + + " MEASURES\n" + + " FIRST(DOWN.empno + DOWN.deptno + 1) AS bottom_total" + + " PATTERN (DOWN{2,})\n" + + " DEFINE\n" + + " DOWN AS PREV(EMP.EMPNO + 2, 0) < 1" + + ")"; + + final String expected11 = "SELECT `EXPR$0`.`BOTTOM_TOTAL`\n" + + "FROM `CATALOG`.`SALES`.`EMP` AS `EMP` MATCH_RECOGNIZE(\n" + + "MEASURES FINAL (FIRST(`DOWN`.`EMPNO` + `DOWN`.`DEPTNO`, 0) + FIRST(1, 0)) AS `BOTTOM_TOTAL`\n" + + "PATTERN (`DOWN` { 2, })\n" + + "DEFINE `DOWN` AS LAST(`EMP`.`EMPNO`, 0) + PREV(2, 0) < 1) AS `EXPR$0`"; + + sql(sql11) + .withValidatorConfig(c -> c.withIdentifierExpansion(true)) + .rewritesTo(expected11); + sql(expected11) + .withParserConfig(c -> c.withQuoting(Quoting.BACK_TICK)).ok(); } @Test void testIntervalTimeUnitEnumeration() { From 24a924f4e91702bea53b65fddbef170956c8ed23 Mon Sep 17 00:00:00 2001 From: xuzifu666 <1206332514@qq.com> Date: Fri, 24 Apr 2026 14:50:46 +0800 Subject: [PATCH 231/562] Test cases for [CALCITE-6299] Support JOIN in Arrow adapter --- .../adapter/arrow/ArrowAdapterTest.java | 50 +++++++++++++++---- 1 file changed, 40 insertions(+), 10 deletions(-) diff --git a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java index 25237eed2769..25212a208305 100644 --- a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java +++ b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java @@ -27,7 +27,6 @@ import com.google.common.collect.ImmutableMap; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.io.TempDir; @@ -803,24 +802,55 @@ static void initializeArrowState(@TempDir Path sharedTempDir) .explainContains(plan); } - - @Disabled("join is not supported yet") + /** Test case for + * [CALCITE-6299] + * Support JOIN in Arrow adapter. */ @Test void testJoin() { String sql = "select t1.\"intField\", t2.\"intField\" " + "from arrowdata t1 join arrowdata t2 on t1.\"intField\" = t2.\"intField\""; - String plan = "PLAN=EnumerableJoin(condition=[=($0, $4)], joinType=[inner])\n" - + " ArrowToEnumerableConverter\n" - + " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n" - + " ArrowToEnumerableConverter\n" - + " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n"; - String result = "intField=0\nintField=1\nintField=2\nintField=3\nintField=4\nintField=5\n"; + String plan = "PLAN=EnumerableMergeJoin(condition=[=($0, $1)], joinType=[inner])\n" + + " EnumerableSort(sort0=[$0], dir0=[ASC])\n" + + " ArrowToEnumerableConverter\n" + + " ArrowProject(intField=[$0])\n" + + " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n" + + " EnumerableSort(sort0=[$0], dir0=[ASC])\n" + + " ArrowToEnumerableConverter\n" + + " ArrowProject(intField=[$0])\n" + + " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n"; + StringBuilder resultBuilder = new StringBuilder(); + for (int i = 0; i < 50; i++) { + resultBuilder.append("intField=").append(i).append("; intField=").append(i).append('\n'); + } + String result = resultBuilder.toString(); CalciteAssert.that() .with(arrow) .query(sql) - .limit(1) .returns(result) .explainContains(plan); + + String sql1 = "select t1.\"intField\", t2.\"intField\" " + + "from arrowdata t1 join arrowdata t2 on t1.\"intField\" = t2.\"intField\" " + + "where t2.\"intField\" in (1, 2, 3)"; + String result1 = "intField=1; intField=1\n" + + "intField=2; intField=2\n" + + "intField=3; intField=3\n"; + String plan1 = "PLAN=EnumerableMergeJoin(condition=[=($0, $1)], joinType=[inner])\n" + + " EnumerableSort(sort0=[$0], dir0=[ASC])\n" + + " ArrowToEnumerableConverter\n" + + " ArrowProject(intField=[$0])\n" + + " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n" + + " EnumerableSort(sort0=[$0], dir0=[ASC])\n" + + " ArrowToEnumerableConverter\n" + + " ArrowProject(intField=[$0])\n" + + " ArrowFilter(condition=[OR(=($0, 1), =($0, 2), =($0, 3))])\n" + + " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n"; + + CalciteAssert.that() + .with(arrow) + .query(sql1) + .returns(result1) + .explainContains(plan1); } @Test void testAggWithoutAggFunctions() { From 1c7b334313e0b937a21ae547a6f4e0a6ae55489a Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Thu, 23 Apr 2026 08:41:18 +0800 Subject: [PATCH 232/562] [CALCITE-7479] Remove redundant aggregate group keys with FD --- .../AggregateRemoveDuplicateKeysRule.java | 181 ++++++++++++++++++ .../apache/calcite/rel/rules/CoreRules.java | 6 + .../AggregateRemoveDuplicateKeysRuleTest.java | 87 +++++++++ .../AggregateRemoveDuplicateKeysRuleTest.xml | 94 +++++++++ 4 files changed, 368 insertions(+) create mode 100644 core/src/main/java/org/apache/calcite/rel/rules/AggregateRemoveDuplicateKeysRule.java create mode 100644 core/src/test/java/org/apache/calcite/test/AggregateRemoveDuplicateKeysRuleTest.java create mode 100644 core/src/test/resources/org/apache/calcite/test/AggregateRemoveDuplicateKeysRuleTest.xml diff --git a/core/src/main/java/org/apache/calcite/rel/rules/AggregateRemoveDuplicateKeysRule.java b/core/src/main/java/org/apache/calcite/rel/rules/AggregateRemoveDuplicateKeysRule.java new file mode 100644 index 000000000000..2254a0094d0b --- /dev/null +++ b/core/src/main/java/org/apache/calcite/rel/rules/AggregateRemoveDuplicateKeysRule.java @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.rel.rules; + +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelRule; +import org.apache.calcite.rel.core.Aggregate; +import org.apache.calcite.rel.logical.LogicalAggregate; +import org.apache.calcite.rel.metadata.RelMetadataQuery; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.tools.RelBuilder; +import org.apache.calcite.util.ImmutableBitSet; + +import org.immutables.value.Value; + +import java.util.ArrayList; +import java.util.List; + +/** + * Planner rule that removes redundant grouping keys from an + * {@link Aggregate} when the later keys are functionally determined by the + * earlier retained keys. + * + *

      The original output schema is preserved by adding {@code ANY_VALUE} + * aggregate calls for removed grouping keys and then projecting the row back + * into the original field order. + * + *

      The original SQL: + * 

      {@code
+ * SELECT deptno, name, count(*) AS c
+ * FROM sales.dept
+ * GROUP BY deptno, name
+ * }
      + * + *

      The original logical plan: + * 

      + * LogicalAggregate(group=[{0, 1}], C=[COUNT()])
+ *   LogicalTableScan(table=[[CATALOG, SALES, DEPT]])
+ *
      + * + *

      After optimization: + * 

      + * LogicalAggregate(group=[{0}], NAME=[ANY_VALUE($1)], C=[COUNT()])
+ *   LogicalTableScan(table=[[CATALOG, SALES, DEPT]])
+ *
      + */ +@Value.Enclosing +public class AggregateRemoveDuplicateKeysRule + extends RelRule + implements TransformationRule { + + /** Creates an AggregateRemoveDuplicateKeysRule. */ + protected AggregateRemoveDuplicateKeysRule(Config config) { + super(config); + } + + @Override public void onMatch(RelOptRuleCall call) { + final Aggregate aggregate = call.rel(0); + if (!Aggregate.isSimple(aggregate) || aggregate.getGroupCount() <= 1) { + return; + } + + final RelMetadataQuery mq = call.getMetadataQuery(); + final List groupKeys = aggregate.getGroupSet().asList(); + final int keyCount = groupKeys.size(); + + // Classify each group-key position (index into groupKeys) as retained or redundant. + // retainedBits tracks *positions* (not input field indices) already retained, so that + // mq.determinesSet can check whether they functionally determine the candidate position + // on the aggregate's own output schema. + final List retainedPos = new ArrayList<>(); + final List removedPos = new ArrayList<>(); + final ImmutableBitSet.Builder newGroupSetBuilder = ImmutableBitSet.builder(); + ImmutableBitSet retainedBits = ImmutableBitSet.of(); + + for (int i = 0; i < keyCount; i++) { + if (!retainedBits.isEmpty() + && mq.determinesSet(aggregate, retainedBits, ImmutableBitSet.of(i))) { + removedPos.add(i); + } else { + retainedPos.add(i); + retainedBits = retainedBits.union(ImmutableBitSet.of(i)); + newGroupSetBuilder.set(groupKeys.get(i)); + } + } + + if (removedPos.isEmpty()) { + return; + } + + final RelBuilder relBuilder = call.builder(); + relBuilder.push(aggregate.getInput()); + + // Build new agg calls. ANY_VALUE calls for removed keys are placed first so + // their output indices are contiguous with the retained group-key columns: + // + // new aggregate output layout: + // [0 .. retainedCount-1] retained group keys + // [retainedCount .. retainedCount+removedCount-1] ANY_VALUE calls + // [retainedCount + removedCount .. ...] original aggregate calls + // + final List inputFieldNames = aggregate.getInput().getRowType().getFieldNames(); + final List newAggCalls = new ArrayList<>(); + for (int removedKeyPos : removedPos) { + final int inputIdx = groupKeys.get(removedKeyPos); + newAggCalls.add( + relBuilder.aggregateCall(SqlStdOperatorTable.ANY_VALUE, + relBuilder.field(inputIdx)) + .as(inputFieldNames.get(inputIdx))); + } + for (org.apache.calcite.rel.core.AggregateCall aggCall : aggregate.getAggCallList()) { + newAggCalls.add(relBuilder.aggregateCall(aggCall)); + } + + relBuilder.aggregate(relBuilder.groupKey(newGroupSetBuilder.build()), newAggCalls); + + // Build a project to restore the original field order. + // Precompute position-in-groupKeys → new-output-column-index to avoid O(n) indexOf. + final int retainedCount = retainedPos.size(); + final int removedCount = removedPos.size(); + final int origAggCount = aggregate.getAggCallList().size(); + final int[] posToCol = new int[keyCount]; + for (int ri = 0; ri < retainedCount; ri++) { + posToCol[retainedPos.get(ri)] = ri; + } + for (int di = 0; di < removedCount; di++) { + posToCol[removedPos.get(di)] = retainedCount + di; + } + + final List projects = new ArrayList<>(aggregate.getRowType().getFieldCount()); + for (int pos = 0; pos < keyCount; pos++) { + projects.add(relBuilder.field(posToCol[pos])); + } + for (int i = 0; i < origAggCount; i++) { + projects.add(relBuilder.field(retainedCount + removedCount + i)); + } + + relBuilder.project(projects, aggregate.getRowType().getFieldNames()); + + call.getPlanner().prune(aggregate); + call.transformTo(relBuilder.build()); + } + + /** Rule configuration. */ + @Value.Immutable + public interface Config extends RelRule.Config { + Config DEFAULT = ImmutableAggregateRemoveDuplicateKeysRule.Config.of() + .withOperandSupplier(b0 -> + b0.operand(LogicalAggregate.class) + .predicate(Aggregate::isSimple) + .anyInputs()); + + @Override default AggregateRemoveDuplicateKeysRule toRule() { + return new AggregateRemoveDuplicateKeysRule(this); + } + + /** Defines an operand tree for the given aggregate class. */ + default Config withOperandFor(Class aggregateClass) { + return withOperandSupplier(b0 -> + b0.operand(aggregateClass) + .predicate(Aggregate::isSimple) + .anyInputs()) + .as(Config.class); + } + } +} diff --git a/core/src/main/java/org/apache/calcite/rel/rules/CoreRules.java b/core/src/main/java/org/apache/calcite/rel/rules/CoreRules.java index 0337b1d5abb3..d7068dcc7c24 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/CoreRules.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/CoreRules.java @@ -66,6 +66,12 @@ private CoreRules() {} public static final AggregateProjectMergeRule AGGREGATE_PROJECT_MERGE = AggregateProjectMergeRule.Config.DEFAULT.toRule(); + /** Rule that removes redundant grouping keys from an {@link Aggregate} + * when they are functionally determined by earlier grouping keys. */ + public static final AggregateRemoveDuplicateKeysRule + AGGREGATE_REMOVE_DUPLICATE_KEYS = + AggregateRemoveDuplicateKeysRule.Config.DEFAULT.toRule(); + /** Rule that removes constant keys from an {@link Aggregate}. */ public static final AggregateProjectPullUpConstantsRule AGGREGATE_PROJECT_PULL_UP_CONSTANTS = diff --git a/core/src/test/java/org/apache/calcite/test/AggregateRemoveDuplicateKeysRuleTest.java b/core/src/test/java/org/apache/calcite/test/AggregateRemoveDuplicateKeysRuleTest.java new file mode 100644 index 000000000000..3f65724a6817 --- /dev/null +++ b/core/src/test/java/org/apache/calcite/test/AggregateRemoveDuplicateKeysRuleTest.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.test; + +import org.apache.calcite.rel.rules.AggregateRemoveDuplicateKeysRule; +import org.apache.calcite.rel.rules.CoreRules; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Test; + +/** + * Tests for {@link AggregateRemoveDuplicateKeysRule}. + * + *

      Relevant tickets: + *

      + */ +class AggregateRemoveDuplicateKeysRuleTest { + + private static RelOptFixture fixture() { + return RelOptFixture.DEFAULT.withDiffRepos( + DiffRepository.lookup(AggregateRemoveDuplicateKeysRuleTest.class)); + } + + private static RelOptFixture sql(String sql) { + return fixture().sql(sql); + } + + @Test void testRemoveOneRedundantGroupKey() { + final String sql = "select deptno, name, count(*) as c\n" + + "from sales.dept\n" + + "group by deptno, name"; + + sql(sql).withRule(CoreRules.AGGREGATE_REMOVE_DUPLICATE_KEYS) + .check(); + } + + @Test void testRemoveMultipleRedundantGroupKeys() { + final String sql = "select empno, ename, job, count(*) as c\n" + + "from emp\n" + + "group by empno, ename, job"; + + sql(sql).withRule(CoreRules.AGGREGATE_REMOVE_DUPLICATE_KEYS) + .check(); + } + + @Test void testRemoveRedundantComputedGroupKey() { + // deptno + 2 is a deterministic function of deptno, so it is determined + // by deptno and can be removed from the GROUP BY. + final String sql = "select deptno, deptno + 2\n" + + "from emp\n" + + "group by deptno, deptno + 2"; + + sql(sql).withRule(CoreRules.AGGREGATE_REMOVE_DUPLICATE_KEYS) + .check(); + } + + @Test void testKeepsNonRedundantGroupKeys() { + final String sql = "select deptno, job, count(*) as c\n" + + "from emp\n" + + "group by deptno, job"; + + sql(sql).withRule(CoreRules.AGGREGATE_REMOVE_DUPLICATE_KEYS) + .checkUnchanged(); + } + + @AfterAll static void checkActualAndReferenceFiles() { + fixture().diffRepos.checkActualAndReferenceFiles(); + } +} diff --git a/core/src/test/resources/org/apache/calcite/test/AggregateRemoveDuplicateKeysRuleTest.xml b/core/src/test/resources/org/apache/calcite/test/AggregateRemoveDuplicateKeysRuleTest.xml new file mode 100644 index 000000000000..f01bb8e675df --- /dev/null +++ b/core/src/test/resources/org/apache/calcite/test/AggregateRemoveDuplicateKeysRuleTest.xml @@ -0,0 +1,94 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 7cb8a25632704d51f2d758169853233aef51e04f Mon Sep 17 00:00:00 2001 From: Sergey Nuyanzin Date: Tue, 21 Apr 2026 17:35:37 +0200 Subject: [PATCH 233/562] [CALCITE-7208] Allow downstream projects implement `CREATE OR ALTER` --- core/src/main/codegen/templates/Parser.jj | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/src/main/codegen/templates/Parser.jj b/core/src/main/codegen/templates/Parser.jj index f549446ea082..44e54b076dc7 100644 --- a/core/src/main/codegen/templates/Parser.jj +++ b/core/src/main/codegen/templates/Parser.jj @@ -4596,6 +4596,8 @@ SqlCreate SqlCreate() : { { s = span(); } [ + // Allow downstream projects implement different syntax, for instance CREATE OR REPLACE + LOOKAHEAD(2) { replace = true; } From 5a0fce9b1dfaef7a03e6ffaa420ab7b5b638d164 Mon Sep 17 00:00:00 2001 From: xuzifu666 <1206332514@qq.com> Date: Tue, 21 Apr 2026 21:11:20 +0800 Subject: [PATCH 234/562] [CALCITE-7484] Add a rule to eliminate redundant aggregates functions over GROUP BY keys --- ...gregateReduceFunctionsOnGroupKeysRule.java | 178 ++++++++++++++++++ .../apache/calcite/rel/rules/CoreRules.java | 6 + ...ateReduceFunctionsOnGroupKeysRuleTest.java | 67 +++++++ ...gateReduceFunctionsOnGroupKeysRuleTest.xml | 80 ++++++++ 4 files changed, 331 insertions(+) create mode 100644 core/src/main/java/org/apache/calcite/rel/rules/AggregateReduceFunctionsOnGroupKeysRule.java create mode 100644 core/src/test/java/org/apache/calcite/test/AggregateReduceFunctionsOnGroupKeysRuleTest.java create mode 100644 core/src/test/resources/org/apache/calcite/test/AggregateReduceFunctionsOnGroupKeysRuleTest.xml diff --git a/core/src/main/java/org/apache/calcite/rel/rules/AggregateReduceFunctionsOnGroupKeysRule.java b/core/src/main/java/org/apache/calcite/rel/rules/AggregateReduceFunctionsOnGroupKeysRule.java new file mode 100644 index 000000000000..10d30d620eba --- /dev/null +++ b/core/src/main/java/org/apache/calcite/rel/rules/AggregateReduceFunctionsOnGroupKeysRule.java @@ -0,0 +1,178 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.rel.rules; + +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelRule; +import org.apache.calcite.rel.RelCollations; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.Aggregate; +import org.apache.calcite.rel.core.AggregateCall; +import org.apache.calcite.rel.core.RelFactories; +import org.apache.calcite.rel.logical.LogicalAggregate; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.tools.RelBuilder; + +import org.checkerframework.checker.nullness.qual.Nullable; +import org.immutables.value.Value; + +import java.util.ArrayList; +import java.util.List; + +/** + * Planner rule that eliminates aggregate functions of GROUP BY keys. + * + *

      For example, + * {@code SELECT sal, max(sal) FROM emp GROUP BY sal} + * can be simplified to + * {@code SELECT sal, sal FROM emp GROUP BY sal}. + * + *

      Currently supports the following aggregate functions when their + * arguments exist in the aggregate's group set: + *

        + *
      • {@code MAX}
        • + *
      • {@code MIN}
        • + *
      • {@code AVG}
        • + *
      • {@code ANY_VALUE}
        • + *
      + * + * @see CoreRules#AGGREGATE_REDUCE_FUNCTIONS_ON_GROUP_KEYS + */ +@Value.Enclosing +public class AggregateReduceFunctionsOnGroupKeysRule + extends RelRule + implements TransformationRule { + + /** Creates an AggregateReduceFunctionsOnGroupKeysRule. */ + protected AggregateReduceFunctionsOnGroupKeysRule(Config config) { + super(config); + } + + @Override public void onMatch(RelOptRuleCall call) { + final Aggregate aggregate = call.rel(0); + final List oldCalls = aggregate.getAggCallList(); + final int groupCount = aggregate.getGroupCount(); + final RexBuilder rexBuilder = aggregate.getCluster().getRexBuilder(); + final RelBuilder relBuilder = call.builder(); + + final List newCalls = new ArrayList<>(); + final List projects = new ArrayList<>(); + + // Pass through group keys. + for (int i = 0; i < groupCount; i++) { + projects.add(rexBuilder.makeInputRef(aggregate, i)); + } + + boolean changed = false; + int newCallOrdinal = 0; + for (AggregateCall oldCall : oldCalls) { + final @Nullable RexNode reduced = reduce(aggregate, oldCall, rexBuilder); + if (reduced != null) { + projects.add(reduced); + changed = true; + } else { + newCalls.add(oldCall); + projects.add( + rexBuilder.makeInputRef( + oldCall.getType(), groupCount + newCallOrdinal)); + newCallOrdinal++; + } + } + + if (!changed) { + return; + } + + final RelNode newAggregate = + aggregate.copy( + aggregate.getTraitSet(), + aggregate.getInput(), + aggregate.getGroupSet(), + aggregate.getGroupSets(), + newCalls); + relBuilder.push(newAggregate); + relBuilder.project(projects); + call.transformTo(relBuilder.build()); + } + + /** + * Tries to reduce an aggregate call to a reference to a group-by key. + * + * @return the reduced expression, or null if cannot reduce + */ + private static @Nullable RexNode reduce( + Aggregate aggregate, + AggregateCall call, + RexBuilder rexBuilder) { + if (!Aggregate.isSimple(aggregate)) { + return null; + } + if (call.hasFilter() + || call.distinctKeys != null + || call.collation != RelCollations.EMPTY) { + return null; + } + final List argList = call.getArgList(); + if (argList.size() != 1) { + return null; + } + final int arg = argList.get(0); + if (!aggregate.getGroupSet().get(arg)) { + return null; + } + final SqlKind kind = call.getAggregation().getKind(); + switch (kind) { + case AVG: + case MAX: + case MIN: + case ANY_VALUE: + break; + default: + return null; + } + final int groupIndex = aggregate.getGroupSet().asList().indexOf(arg); + RexNode ref = RexInputRef.of(groupIndex, aggregate.getRowType().getFieldList()); + if (!ref.getType().equals(call.getType())) { + ref = rexBuilder.makeCast(call.getParserPosition(), call.getType(), ref); + } + return ref; + } + + /** Rule configuration. */ + @Value.Immutable + public interface Config extends RelRule.Config { + Config DEFAULT = ImmutableAggregateReduceFunctionsOnGroupKeysRule.Config.of() + .withRelBuilderFactory(RelFactories.LOGICAL_BUILDER) + .withOperandFor(LogicalAggregate.class); + + @Override default AggregateReduceFunctionsOnGroupKeysRule toRule() { + return new AggregateReduceFunctionsOnGroupKeysRule(this); + } + + /** Defines an operand tree for the given class. */ + default Config withOperandFor(Class aggregateClass) { + return withOperandSupplier(b -> + b.operand(aggregateClass) + .predicate(Aggregate::isSimple) + .anyInputs()) + .as(Config.class); + } + } +} diff --git a/core/src/main/java/org/apache/calcite/rel/rules/CoreRules.java b/core/src/main/java/org/apache/calcite/rel/rules/CoreRules.java index d7068dcc7c24..873d04016f6f 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/CoreRules.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/CoreRules.java @@ -99,6 +99,12 @@ private CoreRules() {} public static final AggregateReduceFunctionsRule AGGREGATE_REDUCE_FUNCTIONS = AggregateReduceFunctionsRule.Config.DEFAULT.toRule(); + /** Rule that eliminates aggregate functions of GROUP BY keys + * in an {@link Aggregate}. */ + public static final AggregateReduceFunctionsOnGroupKeysRule + AGGREGATE_REDUCE_FUNCTIONS_ON_GROUP_KEYS = + AggregateReduceFunctionsOnGroupKeysRule.Config.DEFAULT.toRule(); + /** Rule that matches an {@link Aggregate} on an {@link Aggregate}, * and merges into a single Aggregate if the top aggregate's group key is a * subset of the lower aggregate's group key, and the aggregates are diff --git a/core/src/test/java/org/apache/calcite/test/AggregateReduceFunctionsOnGroupKeysRuleTest.java b/core/src/test/java/org/apache/calcite/test/AggregateReduceFunctionsOnGroupKeysRuleTest.java new file mode 100644 index 000000000000..739ce1cb0b95 --- /dev/null +++ b/core/src/test/java/org/apache/calcite/test/AggregateReduceFunctionsOnGroupKeysRuleTest.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.test; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Test; + +import static org.apache.calcite.rel.rules.CoreRules.AGGREGATE_REDUCE_FUNCTIONS_ON_GROUP_KEYS; + +/** + * Unit tests for {@link org.apache.calcite.rel.rules.AggregateReduceFunctionsOnGroupKeysRule}. + * + *

      Relevant tickets: + *

      + */ +class AggregateReduceFunctionsOnGroupKeysRuleTest { + + private static RelOptFixture fixture() { + return RelOptFixture.DEFAULT.withDiffRepos( + DiffRepository.lookup(AggregateReduceFunctionsOnGroupKeysRuleTest.class)); + } + + private static RelOptFixture sql(String sql) { + return fixture().sql(sql); + } + + @Test void testAggregateFunctionOfGroupByKeys() { + String sql = "select sal, max(sal) as sal_max, min(sal) as sal_min,\n" + + "avg(sal) sal_avg, any_value(sal) as sal_val\n" + + "from emp group by sal, deptno"; + sql(sql).withRule(AGGREGATE_REDUCE_FUNCTIONS_ON_GROUP_KEYS).check(); + } + + @Test void testAggregateFunctionOfGroupByKeysPartial() { + String sql = "select sal, max(sal) as sal_max, sum(comm) as comm_sum\n" + + "from emp group by sal, deptno"; + sql(sql).withRule(AGGREGATE_REDUCE_FUNCTIONS_ON_GROUP_KEYS).check(); + } + + @Test void testAggregateFunctionOfGroupByKeysNoChange() { + String sql = "select sal, max(comm) as comm_max\n" + + "from emp group by sal, deptno"; + sql(sql).withRule(AGGREGATE_REDUCE_FUNCTIONS_ON_GROUP_KEYS).checkUnchanged(); + } + + @AfterAll static void checkActualAndReferenceFiles() { + fixture().diffRepos.checkActualAndReferenceFiles(); + } +} diff --git a/core/src/test/resources/org/apache/calcite/test/AggregateReduceFunctionsOnGroupKeysRuleTest.xml b/core/src/test/resources/org/apache/calcite/test/AggregateReduceFunctionsOnGroupKeysRuleTest.xml new file mode 100644 index 000000000000..e7eb9d5a927d --- /dev/null +++ b/core/src/test/resources/org/apache/calcite/test/AggregateReduceFunctionsOnGroupKeysRuleTest.xml @@ -0,0 +1,80 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 891580712df9fc850df8c3cf78e2024f9df7c61c Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Mon, 27 Apr 2026 21:57:21 -0700 Subject: [PATCH 235/562] [CALCITE-7491] Literals of type TIMESTAMP WITH TIME ZONE cause crashes Signed-off-by: Mihai Budiu --- babel/src/test/resources/sql/big-query.iq | 37 +++++++++++++++++++ .../enumerable/RexToLixTranslator.java | 2 + .../org/apache/calcite/rex/RexLiteral.java | 18 ++++++++- .../sql2rel/SqlNodeToRexConverterImpl.java | 3 +- .../util/TimestampWithTimeZoneString.java | 2 +- site/_docs/reference.md | 1 + 6 files changed, 59 insertions(+), 4 deletions(-) diff --git a/babel/src/test/resources/sql/big-query.iq b/babel/src/test/resources/sql/big-query.iq index 007e68cb870c..a988eac1d74c 100755 --- a/babel/src/test/resources/sql/big-query.iq +++ b/babel/src/test/resources/sql/big-query.iq @@ -30,6 +30,43 @@ !use scott-big-query !set outputformat mysql +# Test case for [CALCITE-7491] https://issues.apache.org/jira/browse/CALCITE-7491 +# Literals of type TIMESTAMP WITH TIME ZONE cause crashes +# This will change once we fix [CALCITE-7494] +# Avatica conversion to string of TIMESTAMP WITH TIME ZONE +# does not include time zone +select TIMESTAMP WITH TIME ZONE '2020-01-01 00:00:00 America/New_York'; ++---------------------+ +| EXPR$0 | ++---------------------+ +| 2020-01-01 05:00:00 | ++---------------------+ +(1 row) + +!ok + +# Two timestamps with time zone are equal if they represent the same UTC time +SELECT TIMESTAMP WITH TIME ZONE '2020-01-01 08:10:10 America/New_York' = TIMESTAMP WITH TIME ZONE '2020-01-01 05:10:10 America/Los_Angeles'; ++--------+ +| EXPR$0 | ++--------+ +| true | ++--------+ +(1 row) + +!ok + +# Two equal timestamps in different time zones are different if they represent different UTC times +SELECT TIMESTAMP WITH TIME ZONE '2020-01-01 08:10:10 America/New_York' = TIMESTAMP WITH TIME ZONE '2020-01-01 08:10:10 America/Los_Angeles'; ++--------+ +| EXPR$0 | ++--------+ +| false | ++--------+ +(1 row) + +!ok + # Two tests for [CALCITE-7094] Using a type alias as a constructor function # causes a validator assertion failure select int64(); diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java index f88170b1b58d..61fa2b0d7163 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java @@ -1077,6 +1077,7 @@ public static Expression translateLiteral( () -> "value for " + literal).toString())); case DATE: case TIME: + case TIME_TZ: case TIME_WITH_LOCAL_TIME_ZONE: case INTERVAL_YEAR: case INTERVAL_YEAR_MONTH: @@ -1085,6 +1086,7 @@ public static Expression translateLiteral( javaClass = int.class; break; case TIMESTAMP: + case TIMESTAMP_TZ: case TIMESTAMP_WITH_LOCAL_TIME_ZONE: case INTERVAL_DAY: case INTERVAL_DAY_HOUR: diff --git a/core/src/main/java/org/apache/calcite/rex/RexLiteral.java b/core/src/main/java/org/apache/calcite/rex/RexLiteral.java index 7fd05901e74f..545e81995d0d 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexLiteral.java +++ b/core/src/main/java/org/apache/calcite/rex/RexLiteral.java @@ -60,6 +60,11 @@ import java.nio.ByteBuffer; import java.nio.charset.Charset; import java.text.SimpleDateFormat; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; import java.util.Calendar; import java.util.List; import java.util.Locale; @@ -1183,9 +1188,18 @@ public boolean isNull() { break; case TIMESTAMP_TZ: if (clazz == Long.class) { - return clazz.cast(((TimestampWithTimeZoneString) value) + TimestampWithTimeZoneString tstz = (TimestampWithTimeZoneString) value; + long ms = tstz .getLocalTimestampString() - .getMillisSinceEpoch()); + .getMillisSinceEpoch(); + // Interpret the timestamp part as a UTC timestamp + LocalDateTime local = Instant.ofEpochMilli(ms).atZone(ZoneOffset.UTC).toLocalDateTime(); + // Adjust for the time zone + ZoneId id = tstz.getTimeZone().toZoneId(); + ZonedDateTime zoned = local.atZone(id); + ZonedDateTime utc = zoned.withZoneSameInstant(ZoneOffset.UTC); + ms = utc.toInstant().toEpochMilli(); + return clazz.cast(ms); } else if (clazz == Calendar.class) { TimestampWithTimeZoneString ts = (TimestampWithTimeZoneString) value; return clazz.cast(ts.getLocalTimestampString().toCalendar(ts.getTimeZone())); diff --git a/core/src/main/java/org/apache/calcite/sql2rel/SqlNodeToRexConverterImpl.java b/core/src/main/java/org/apache/calcite/sql2rel/SqlNodeToRexConverterImpl.java index ff0d1dfa74ab..081a81cbc096 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/SqlNodeToRexConverterImpl.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/SqlNodeToRexConverterImpl.java @@ -26,6 +26,7 @@ import org.apache.calcite.sql.SqlLiteral; import org.apache.calcite.sql.SqlTimeLiteral; import org.apache.calcite.sql.SqlTimestampLiteral; +import org.apache.calcite.sql.SqlTimestampTzLiteral; import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.util.BitString; import org.apache.calcite.util.DateString; @@ -131,7 +132,7 @@ public class SqlNodeToRexConverterImpl implements SqlNodeToRexConverter { case TIMESTAMP_TZ: return rexBuilder.makeTimestampTzLiteral( literal.getValueAs(TimestampWithTimeZoneString.class), - ((SqlTimestampLiteral) literal).getPrec()); + ((SqlTimestampTzLiteral) literal).getPrec()); case TIME: return rexBuilder.makeTimeLiteral( literal.getValueAs(TimeString.class), diff --git a/core/src/main/java/org/apache/calcite/util/TimestampWithTimeZoneString.java b/core/src/main/java/org/apache/calcite/util/TimestampWithTimeZoneString.java index 14195469ab96..5816b12a6520 100644 --- a/core/src/main/java/org/apache/calcite/util/TimestampWithTimeZoneString.java +++ b/core/src/main/java/org/apache/calcite/util/TimestampWithTimeZoneString.java @@ -173,7 +173,7 @@ public TimestampWithTimeZoneString withTimeZone(TimeZone timeZone) { } @Override public int compareTo(TimestampWithTimeZoneString o) { - return v.compareTo(o.v); + return this.pt.getCalendar().compareTo(o.pt.getCalendar()); } public TimestampWithTimeZoneString round(int precision) { diff --git a/site/_docs/reference.md b/site/_docs/reference.md index 96eb1a25b5a6..5f12b22149d6 100644 --- a/site/_docs/reference.md +++ b/site/_docs/reference.md @@ -1682,6 +1682,7 @@ charSet: timeZone: WITHOUT TIME ZONE + | WITH TIME ZONE | WITH LOCAL TIME ZONE {% endhighlight %} From 02dd46970d9514873d677cafe7a7f423a0901989 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Wed, 14 Jan 2026 21:47:50 -0800 Subject: [PATCH 236/562] [CALCITE-7360] The meaning of negation for unsigned numbers is not defined Signed-off-by: Mihai Budiu --- .../calcite/sql/fun/SqlStdOperatorTable.java | 2 +- .../apache/calcite/sql/type/OperandTypes.java | 65 ++++++++++++++++++- core/src/test/resources/sql/unsigned.iq | 5 ++ .../apache/calcite/test/SqlOperatorTest.java | 22 +++++++ 4 files changed, 92 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java index 2a26a78929cf..1137539fdc76 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java @@ -1023,7 +1023,7 @@ public class SqlStdOperatorTable extends ReflectiveSqlOperatorTable { 80, ReturnTypes.ARG0, InferTypes.RETURN_TYPE, - OperandTypes.NUMERIC_OR_INTERVAL); + OperandTypes.SIGNED_OR_INTERVAL); /** * Checked version of prefix arithmetic minus operator, '-'. diff --git a/core/src/main/java/org/apache/calcite/sql/type/OperandTypes.java b/core/src/main/java/org/apache/calcite/sql/type/OperandTypes.java index 66b4aab9b66d..f3c848d34e68 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/OperandTypes.java +++ b/core/src/main/java/org/apache/calcite/sql/type/OperandTypes.java @@ -404,10 +404,70 @@ public static SqlOperandTypeChecker variadic( public static final SqlSingleOperandTypeChecker INTEGER = family(SqlTypeFamily.INTEGER); + /** Operand type checker that only allows signed types. + * This is almost like an OR of 4 type families (INTEGER, APPROXIMATE_NUMERIC, DECIMAL) + * but OR allows implicit casts to any of the types, and this checker doesn't. */ + public static final SqlSingleOperandTypeChecker SIGNED = new SqlSingleOperandTypeChecker() { + @Override public boolean checkSingleOperandType(SqlCallBinding callBinding, SqlNode operand, + int iFormalOperand, boolean throwOnFailure) { + RelDataType type = SqlTypeUtil.deriveType(callBinding, operand); + SqlTypeName typeName = type.getSqlTypeName(); + boolean isLegal = SqlTypeName.INT_TYPES.contains(typeName) + || SqlTypeName.APPROX_TYPES.contains(typeName) + || typeName == SqlTypeName.DECIMAL; + + if (!isLegal) { + if (throwOnFailure) { + throw callBinding.newValidationSignatureError(); + } + return false; + } + return true; + } + + @Override public boolean checkOperandTypes( + SqlCallBinding callBinding, + boolean throwOnFailure) { + // This is a specialized implementation of FamilyOperandTypeChecker.checkOperandTypes. + SqlNode op = callBinding.operands().get(0); + if (!checkSingleOperandType(callBinding, op, 0, false)) { + // try to coerce type if it is allowed. + boolean coerced = false; + if (callBinding.isTypeCoercionEnabled()) { + // Also allow expressions that can be coerced to NUMERIC (e.g. type CHAR) + TypeCoercion typeCoercion = callBinding.getValidator().getTypeCoercion(); + ImmutableList.Builder builder = ImmutableList.builder(); + builder.add(callBinding.getOperandType(0)); + ImmutableList dataTypes = builder.build(); + coerced = + typeCoercion.builtinFunctionCoercion( + callBinding, dataTypes, ImmutableList.of(SqlTypeFamily.NUMERIC)); + } + // re-validate the new nodes type. + SqlNode op1 = callBinding.operands().get(0); + if (!checkSingleOperandType( + callBinding, + op1, + 0, + throwOnFailure)) { + return false; + } + return coerced; + } + return true; + } + + @Override public String getAllowedSignatures(SqlOperator op, String opName) { + return SqlUtil.getAliasedSignature(op, opName, ImmutableList.of(SqlTypeFamily.INTEGER)) + "\n" + + SqlUtil.getAliasedSignature( + op, opName, ImmutableList.of(SqlTypeFamily.APPROXIMATE_NUMERIC)) + "\n" + + SqlUtil.getAliasedSignature(op, opName, ImmutableList.of(SqlTypeFamily.DECIMAL)); + } + }; + public static final SqlSingleOperandTypeChecker UNSIGNED_NUMERIC_UNSIGNED_NUMERIC = family(SqlTypeFamily.UNSIGNED_NUMERIC, SqlTypeFamily.UNSIGNED_NUMERIC); - public static final SqlSingleOperandTypeChecker INTEGER_INTEGER = family(SqlTypeFamily.INTEGER, SqlTypeFamily.INTEGER); @@ -1299,6 +1359,9 @@ public static SqlSingleOperandTypeChecker same(int operandCount, public static final SqlSingleOperandTypeChecker NUMERIC_OR_INTERVAL = NUMERIC.or(INTERVAL); + public static final SqlSingleOperandTypeChecker SIGNED_OR_INTERVAL = + SIGNED.or(INTERVAL); + public static final SqlSingleOperandTypeChecker NUMERIC_OR_STRING = NUMERIC.or(STRING); diff --git a/core/src/test/resources/sql/unsigned.iq b/core/src/test/resources/sql/unsigned.iq index f496dbb2209c..83112fe87edf 100644 --- a/core/src/test/resources/sql/unsigned.iq +++ b/core/src/test/resources/sql/unsigned.iq @@ -24,6 +24,11 @@ EXPR$0 6 !ok +SELECT -CAST(200 AS INT UNSIGNED); +java.sql.SQLException: Error while executing SQL "SELECT -CAST(200 AS INT UNSIGNED)": From line 1, column 8 to line 1, column 33: Cannot apply '-' to arguments of type '-'. Supported form(s): '-' + +!error + SELECT CAST(200 AS INT UNSIGNED) - 100; EXPR$0 100 diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java index e79bbe197b75..367fa730688b 100644 --- a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java +++ b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java @@ -16866,6 +16866,28 @@ private static void checkLogicalOrFunc(SqlOperatorFixture f) { f.checkNull("CAST(NULL AS INTEGER UNSIGNED) ^^ CAST(NULL AS INTEGER UNSIGNED)"); } + @Test void testUnsignedArithmetic() { + final SqlOperatorFixture f = fixture(); + // Test case for [CALCITE-7360] The meaning of negation for unsigned numbers is not defined + f.checkFails("^-CAST (100 AS INT UNSIGNED)^", + "Cannot apply '-' to arguments of type '-'\\. " + + "Supported form\\(s\\): '-'\\n" + + "'-'\\n" + + "'-'\\n" + + "'-'", false); + f.checkScalar("CAST(2 AS INT UNSIGNED)", "2", "INTEGER UNSIGNED NOT NULL"); + f.checkScalar("CAST(2 AS INT UNSIGNED) + CAST(2 AS INT UNSIGNED)", "4", + "INTEGER UNSIGNED NOT NULL"); + f.checkScalar("CAST(2 AS INT UNSIGNED) + CAST(2 AS TINYINT UNSIGNED)", "4", + "INTEGER UNSIGNED NOT NULL"); + f.checkScalar("CAST(2 AS INT UNSIGNED) + 2", "4", "INTEGER UNSIGNED NOT NULL"); + f.checkScalar("CAST(2 AS INT UNSIGNED) - 2", "0", "INTEGER UNSIGNED NOT NULL"); + f.checkScalar("CAST(2 AS INT UNSIGNED) - CAST(2 AS TINYINT UNSIGNED)", "0", + "INTEGER UNSIGNED NOT NULL"); + f.checkScalar("CAST(2 AS INT UNSIGNED) * 2", "4", "INTEGER UNSIGNED NOT NULL"); + f.checkScalar("CAST(2 AS INT UNSIGNED) / 2", "1", "INTEGER UNSIGNED NOT NULL"); + } + /** * Test cases for * [CALCITE-7109] From a4c1fa46ba96ad1554e573939a71d6d0bada65b8 Mon Sep 17 00:00:00 2001 From: xuzifu666 <1206332514@qq.com> Date: Mon, 27 Apr 2026 11:40:10 +0800 Subject: [PATCH 237/562] [CALCITE-7490] PruneEmptyRules is ineffective for window statements --- .../org/apache/calcite/plan/RelOptRules.java | 1 + .../calcite/rel/rules/PruneEmptyRules.java | 17 +++++++++++++++++ .../apache/calcite/test/RelOptRulesTest.java | 13 +++++++++++++ .../org/apache/calcite/test/RelOptRulesTest.xml | 17 +++++++++++++++++ 4 files changed, 48 insertions(+) diff --git a/core/src/main/java/org/apache/calcite/plan/RelOptRules.java b/core/src/main/java/org/apache/calcite/plan/RelOptRules.java index 65b9e54897e9..2abce5f327ec 100644 --- a/core/src/main/java/org/apache/calcite/plan/RelOptRules.java +++ b/core/src/main/java/org/apache/calcite/plan/RelOptRules.java @@ -103,6 +103,7 @@ private RelOptRules() { PruneEmptyRules.FILTER_INSTANCE, PruneEmptyRules.SORT_INSTANCE, PruneEmptyRules.AGGREGATE_INSTANCE, + PruneEmptyRules.WINDOW_INSTANCE, PruneEmptyRules.JOIN_LEFT_INSTANCE, PruneEmptyRules.JOIN_RIGHT_INSTANCE, PruneEmptyRules.SORT_FETCH_ZERO_INSTANCE, diff --git a/core/src/main/java/org/apache/calcite/rel/rules/PruneEmptyRules.java b/core/src/main/java/org/apache/calcite/rel/rules/PruneEmptyRules.java index 0ee85558aa67..5d70c5e0dec4 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/PruneEmptyRules.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/PruneEmptyRules.java @@ -37,6 +37,7 @@ import org.apache.calcite.rel.core.TableScan; import org.apache.calcite.rel.core.Union; import org.apache.calcite.rel.core.Values; +import org.apache.calcite.rel.core.Window; import org.apache.calcite.rel.logical.LogicalValues; import org.apache.calcite.rel.metadata.RelMdUtil; import org.apache.calcite.rel.type.RelDataType; @@ -235,6 +236,19 @@ private static boolean isEmpty(RelNode node) { public static final RelOptRule AGGREGATE_INSTANCE = RemoveEmptySingleRule.RemoveEmptySingleRuleConfig.AGGREGATE.toRule(); + /** + * Rule that converts a {@link org.apache.calcite.rel.core.Window} + * to empty if its child is empty. + * + *

      Examples: + * + *

        + *
      • Window(Empty) becomes Empty + *
      + */ + public static final RelOptRule WINDOW_INSTANCE = + RemoveEmptySingleRule.RemoveEmptySingleRuleConfig.WINDOW.toRule(); + /** * Rule that converts a {@link org.apache.calcite.rel.core.Join} * to empty if its left child is empty. @@ -366,6 +380,9 @@ public interface RemoveEmptySingleRuleConfig extends PruneEmptyRule.Config { RemoveEmptySingleRuleConfig AGGREGATE = ImmutableRemoveEmptySingleRuleConfig.of() .withDescription("PruneEmptyAggregate") .withOperandFor(Aggregate.class, Aggregate::isNotGrandTotal); + RemoveEmptySingleRuleConfig WINDOW = ImmutableRemoveEmptySingleRuleConfig.of() + .withDescription("PruneEmptyWindow") + .withOperandFor(Window.class, singleRel -> true); @Override default RemoveEmptySingleRule toRule() { return new RemoveEmptySingleRule(this); diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index bd983836d216..1b3b7295f767 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -5752,6 +5752,19 @@ private void checkEmptyJoin(RelOptFixture f) { .check(); } + /** Test case for + *       [CALCITE-7490] + * PruneEmptyRules is ineffective for window statements. */ + @Test void testEmptyWindow() { + final String sql = "select count(*) over () from emp where false"; + sql(sql) + .withPreRule(CoreRules.PROJECT_TO_LOGICAL_PROJECT_AND_WINDOW, + CoreRules.FILTER_REDUCE_EXPRESSIONS) + .withRule(PruneEmptyRules.WINDOW_INSTANCE, + PruneEmptyRules.PROJECT_INSTANCE) + .check(); + } + /** Test case for * [CALCITE-5117] * Optimize the EXISTS sub-query by Metadata RowCount. */ diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index ece2ad5258aa..b9add8436a66 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -4280,6 +4280,23 @@ LogicalProject(PRODUCTID=[$0], NAME=[$1]) + + + + + + + + + + + From 8de4dce01e96d1cbde973638d079ac4906fea8a7 Mon Sep 17 00:00:00 2001 From: Terran Date: Wed, 29 Apr 2026 10:21:43 +0800 Subject: [PATCH 238/562] [CALCITE-7497] Enable Lambda supports constant folding --- .../rel/rules/ReduceExpressionsRule.java | 13 +++ .../org/apache/calcite/rex/RexShuttle.java | 8 +- .../apache/calcite/test/RelOptRulesTest.java | 89 +++++++++++++++ .../apache/calcite/test/RelOptRulesTest.xml | 102 ++++++++++++++++++ 4 files changed, 210 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rel/rules/ReduceExpressionsRule.java b/core/src/main/java/org/apache/calcite/rel/rules/ReduceExpressionsRule.java index 88ffaf72fcdb..db93802118b6 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/ReduceExpressionsRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/ReduceExpressionsRule.java @@ -1189,6 +1189,19 @@ private void analyzeCall(RexCall call, Constancy callConstancy) { } @Override public Void visitLambda(RexLambda lambda) { + // A lambda as a whole is not a constant (it contains lambda parameter + // references), but its body may contain constant sub-expressions (e.g. + // the "1 + 2" in "x -> x > 1 + 2"). Recursively analyze the body so + // that those inner constants are discovered and later replaced by + // RexReplacer (which already recurses into lambda bodies via + // RexShuttle.visitLambda). + final int stackSizeBefore = stack.size(); + lambda.getExpression().accept(this); + // Discard whatever the body analysis pushed onto the stack – the lambda + // itself is not a constant from the outer context's point of view. + while (stack.size() > stackSizeBefore) { + stack.remove(stack.size() - 1); + } return pushVariable(); } diff --git a/core/src/main/java/org/apache/calcite/rex/RexShuttle.java b/core/src/main/java/org/apache/calcite/rex/RexShuttle.java index a3cf2b9d5e5f..d99ae12630fa 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexShuttle.java +++ b/core/src/main/java/org/apache/calcite/rex/RexShuttle.java @@ -237,8 +237,12 @@ protected List visitFieldCollations( } @Override public RexNode visitLambda(RexLambda lambda) { - lambda.getExpression().accept(this); - return lambda; + RexNode oldBody = lambda.getExpression(); + RexNode newBody = oldBody.accept(this); + if (newBody == oldBody) { + return lambda; + } + return new RexLambda(lambda.getParameters(), newBody); } @Override public RexNode visitLambdaRef(RexLambdaRef lambdaRef) { diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index 1b3b7295f767..e52702974953 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -12321,4 +12321,93 @@ private void checkLoptOptimizeJoinRule(LoptOptimizeJoinRule rule) { .withRule(CoreRules.PROJECT_FILTER_VALUES_MERGE) .checkUnchanged(); } + + /** Test case for + * [CALCITE-7497] + * Enable Lambda supports constant folding. + */ + @Test void testReduceLambdaBodyConstantFolding() { + final String sql = "select \"EXISTS\"(ARRAY[1, 2, 3], x -> x > 1 + 2)"; + sql(sql) + .withFactory(f -> + f.withOperatorTable(opTab -> + SqlValidatorTest.operatorTableFor(SqlLibrary.SPARK))) + .withRule(CoreRules.PROJECT_REDUCE_EXPRESSIONS).check(); + } + + /** Test case for + * [CALCITE-7497] + * Enable Lambda supports constant folding. + * + *

      Boolean constant expression {@code 1 < 2} in lambda body should be + * folded to {@code true}. */ + @Test void testReduceLambdaBodyBooleanConstantFolding() { + final String sql = "select \"EXISTS\"(ARRAY[1, 2, 3], x -> x > 1 AND 1 < 2)"; + sql(sql) + .withFactory(f -> + f.withOperatorTable(opTab -> + SqlValidatorTest.operatorTableFor(SqlLibrary.SPARK))) + .withRule(CoreRules.PROJECT_REDUCE_EXPRESSIONS).check(); + } + + /** Test case for + * [CALCITE-7497] + * Enable Lambda supports constant folding. + * + *

      Negative constant arithmetic {@code -1 * -2} in lambda body should be + * folded to {@code 2}. */ + @Test void testReduceLambdaBodyNegativeConstantFolding() { + final String sql = "select \"EXISTS\"(ARRAY[1, 2, 3], x -> x > -1 * -2)"; + sql(sql) + .withFactory(f -> + f.withOperatorTable(opTab -> + SqlValidatorTest.operatorTableFor(SqlLibrary.SPARK))) + .withRule(CoreRules.PROJECT_REDUCE_EXPRESSIONS).check(); + } + + /** Test case for + * [CALCITE-7497] + * Enable Lambda supports constant folding. + * + *

      MOD function with constant arguments {@code MOD(10, 3)} in lambda body + * should be folded to {@code 1}. */ + @Test void testReduceLambdaBodyModConstantFolding() { + final String sql = "select \"EXISTS\"(ARRAY[1, 2, 3], x -> x > MOD(10, 3))"; + sql(sql) + .withFactory(f -> + f.withOperatorTable(opTab -> + SqlValidatorTest.operatorTableFor(SqlLibrary.SPARK))) + .withRule(CoreRules.PROJECT_REDUCE_EXPRESSIONS).check(); + } + + /** Test case for + * [CALCITE-7497] + * Enable Lambda supports constant folding. + * + *

      OR-connected constant expressions {@code 1 + 2} and {@code 10 - 3} + * in lambda body should each be folded independently. */ + @Test void testReduceLambdaBodyOrConstantFolding() { + final String sql = "select \"EXISTS\"(ARRAY[1, 2, 3], x -> x > 1 + 2 OR x < 10 - 3)"; + sql(sql) + .withFactory(f -> + f.withOperatorTable(opTab -> + SqlValidatorTest.operatorTableFor(SqlLibrary.SPARK))) + .withRule(CoreRules.PROJECT_REDUCE_EXPRESSIONS).check(); + } + + /** Test case for + * [CALCITE-7497] + * Enable Lambda supports constant folding. + * + *

      Deeply nested constant expression {@code (1 + 2) * (3 + 4)} in lambda + * body should be fully folded to {@code 21}. */ + @Test void testReduceLambdaBodyDeepNestedConstantFolding() { + final String sql = "select \"EXISTS\"(ARRAY[1, 2, 3], x -> x > (1 + 2) * (3 + 4))"; + sql(sql) + .withFactory(f -> + f.withOperatorTable(opTab -> + SqlValidatorTest.operatorTableFor(SqlLibrary.SPARK))) + .withRule(CoreRules.PROJECT_REDUCE_EXPRESSIONS).check(); + } + } diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index b9add8436a66..2d591dbd3672 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -16930,6 +16930,108 @@ LogicalProject(DEPTNO=[$7], SAL=[$5]) true)]) LogicalValues(tuples=[[{ 0 }]]) +]]> + + + + + x > 1 AND 1 < 2)]]> + + + AND(>(X, 1), <(1, 2)))]) + LogicalValues(tuples=[[{ 0 }]]) +]]> + + + AND(>(X, 1), true))]) + LogicalValues(tuples=[[{ 0 }]]) +]]> + + + + + x > 1 + 2)]]> + + + >(X, +(1, 2)))]) + LogicalValues(tuples=[[{ 0 }]]) +]]> + + + >(X, 3))]) + LogicalValues(tuples=[[{ 0 }]]) +]]> + + + + + x > (1 + 2) * (3 + 4))]]> + + + >(X, *(+(1, 2), +(3, 4))))]) + LogicalValues(tuples=[[{ 0 }]]) +]]> + + + >(X, 21))]) + LogicalValues(tuples=[[{ 0 }]]) +]]> + + + + + x > MOD(10, 3))]]> + + + >(X, MOD(10, 3)))]) + LogicalValues(tuples=[[{ 0 }]]) +]]> + + + >(X, 1))]) + LogicalValues(tuples=[[{ 0 }]]) +]]> + + + + + x > -1 * -2)]]> + + + >(X, *(-1, -2)))]) + LogicalValues(tuples=[[{ 0 }]]) +]]> + + + >(X, 2))]) + LogicalValues(tuples=[[{ 0 }]]) +]]> + + + + + x > 1 + 2 OR x < 10 - 3)]]> + + + OR(>(X, +(1, 2)), <(X, -(10, 3))))]) + LogicalValues(tuples=[[{ 0 }]]) +]]> + + + OR(>(X, 3), <(X, 7)))]) + LogicalValues(tuples=[[{ 0 }]]) ]]> From 8de4f789e6bcdbee78276b533d80d851c26a2cb9 Mon Sep 17 00:00:00 2001 From: krooswu Date: Tue, 7 Apr 2026 22:58:00 +0800 Subject: [PATCH 239/562] [CALCITE-7437] Type coercion for quantifier operators is incomplete --- .../calcite/sql/fun/SqlQuantifyOperator.java | 9 +- .../validate/implicit/TypeCoercionImpl.java | 131 ++++++++++++++---- .../apache/calcite/test/RelOptRulesTest.xml | 12 +- core/src/test/resources/sql/sub-query.iq | 46 +++++- .../apache/calcite/test/SqlOperatorTest.java | 20 +++ 5 files changed, 175 insertions(+), 43 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlQuantifyOperator.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlQuantifyOperator.java index aeb770acbaac..041d7abf09bb 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlQuantifyOperator.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlQuantifyOperator.java @@ -90,7 +90,14 @@ public class SqlQuantifyOperator extends SqlInOperator { if (typeForCollectionArgument != null) { return typeForCollectionArgument; } - return super.deriveType(validator, scope, call); + // Right-hand side is a subquery (some,any, all) + final RelDataType returnType = super.deriveType(validator, scope, call); + if (validator.config().typeCoercionEnabled()) { + validator.getTypeCoercion() + .quantifyOperationCoercion( + new SqlCallBinding(validator, scope, call)); + } + return returnType; } /** diff --git a/core/src/main/java/org/apache/calcite/sql/validate/implicit/TypeCoercionImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/implicit/TypeCoercionImpl.java index f3f5c53469a4..26812be96110 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/implicit/TypeCoercionImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/implicit/TypeCoercionImpl.java @@ -651,41 +651,114 @@ private boolean coalesceCoercion(SqlCallBinding callBinding) { */ @Override public boolean quantifyOperationCoercion(SqlCallBinding binding) { final RelDataType type1 = binding.getOperandType(0); - final RelDataType collectionType = binding.getOperandType(1); - final RelDataType type2 = collectionType.getComponentType(); - requireNonNull(type2, "type2"); - final SqlCall sqlCall = binding.getCall(); - final SqlValidatorScope scope = binding.getScope(); + final RelDataType type2 = binding.getOperandType(1); final SqlNode node1 = binding.operand(0); final SqlNode node2 = binding.operand(1); - RelDataType widenType = commonTypeForBinaryComparison(type1, type2); - if (widenType == null) { - widenType = getTightestCommonType(type1, type2); - } - if (widenType == null) { + final SqlValidatorScope scope = binding.getScope(); + + // Check column counts match for struct types, consistent with inOperationCoercion. + if (type1.isStruct() + && type2.isStruct() + && type1.getFieldCount() != type2.getFieldCount()) { return false; } - final RelDataType leftWidenType = - binding.getTypeFactory().enforceTypeWithNullability(widenType, type1.isNullable()); - boolean coercedLeft = - coerceOperandType(scope, sqlCall, 0, leftWidenType); - if (coercedLeft) { - updateInferredType(node1, leftWidenType); + + int colCount = type1.isStruct() ? type1.getFieldCount() : 1; + RelDataType[] argTypes = new RelDataType[2]; + argTypes[0] = type1; + final boolean isSubQuery = node2 instanceof SqlSelect; + // For subquery, use the row type directly. + // For collection, use the component type for comparison, not the collection. + if (isSubQuery) { + argTypes[1] = type2; + } else { + RelDataType componentType = type2.getComponentType(); + if (componentType == null) { + return false; + } + argTypes[1] = componentType; + } + boolean coerced = false; + + // Find the common types for RHS and LHS columns, + // following the same rules as inOperationCoercion. + List widenTypes = new ArrayList<>(); + for (int i = 0; i < colCount; i++) { + final int i2 = i; + List columnIthTypes = new AbstractList() { + @Override public RelDataType get(int index) { + return argTypes[index].isStruct() + ? argTypes[index].getFieldList().get(i2).getType() + : argTypes[index]; + } + + @Override public int size() { + return argTypes.length; + } + }; + + RelDataType widenType = + commonTypeForBinaryComparison(columnIthTypes.get(0), columnIthTypes.get(1)); + if (widenType == null) { + widenType = getTightestCommonType(columnIthTypes.get(0), columnIthTypes.get(1)); + } + if (widenType == null) { + // Cannot find any common type, return early. + return false; + } + widenTypes.add(widenType); + } + assert widenTypes.size() == colCount; + + // Coerce LHS operand. + if (!type1.isStruct()) { + coerced = coerceOperandType(scope, binding.getCall(), 0, widenTypes.get(0)) || coerced; } - final RelDataType rightWidenType = - binding.getTypeFactory().enforceTypeWithNullability(widenType, type2.isNullable()); - RelDataType collectionWidenType = - binding.getTypeFactory().createArrayType(rightWidenType, -1); - collectionWidenType = - binding - .getTypeFactory() - .enforceTypeWithNullability(collectionWidenType, collectionType.isNullable()); - boolean coercedRight = - coerceOperandType(scope, sqlCall, 1, collectionWidenType); - if (coercedRight) { - updateInferredType(node2, collectionWidenType); + + for (int i = 0; i < widenTypes.size(); i++) { + RelDataType desired = widenTypes.get(i); + // LHS may be a ROW value. + if (node1.getKind() == SqlKind.ROW) { + assert node1 instanceof SqlCall; + if (coerceOperandType(scope, (SqlCall) node1, i, desired)) { + updateInferredColumnType( + requireNonNull(scope, "scope"), + node1, i, widenTypes.get(i)); + coerced = true; + } + } + + // RHS: subquery uses rowTypeCoercion (consistent with inOperationCoercion), + // collection reconstructs the array type. + if (isSubQuery) { + // Use rowTypeCoercion on the subquery output column, + // consistent with how inOperationCoercion handles the subquery case. + SqlValidatorScope scope1 = validator.getSelectScope((SqlSelect) node2); + RelDataType source = validator.getValidatedNodeType(node2); + RelDataType target = binding.getTypeFactory() + .createTypeWithNullability(desired, source.isNullable() || desired.isNullable()); + coerced = rowTypeCoercion(scope1, node2, i, target) || coerced; + } else { + // Collection path (e.g. ARRAY, MULTISET): coerce the whole collection + // operand once, reconstructing the collection type with the widened + // component type + RelDataType componentType = argTypes[1]; + final RelDataType rightWidenType = + binding.getTypeFactory() + .enforceTypeWithNullability(desired, componentType.isNullable()); + RelDataType collectionWidenType = + binding.getTypeFactory().createArrayType(rightWidenType, -1); + collectionWidenType = + binding.getTypeFactory() + .enforceTypeWithNullability(collectionWidenType, type2.isNullable()); + if (coerceOperandType(scope, binding.getCall(), 1, collectionWidenType)) { + updateInferredType(node2, collectionWidenType); + coerced = true; + } + } } - return coercedLeft || coercedRight; + + return coerced; } @Override public boolean builtinFunctionCoercion( diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index 2d591dbd3672..cf66384c12f5 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -1520,7 +1520,7 @@ from dept]]> ($2, 0)), AND(<($3, $2), null, <>($2, 0), IS NULL($5)))]) - LogicalJoin(condition=[=($1, $4)], joinType=[left]) + LogicalJoin(condition=[=(CAST($1):INTEGER NOT NULL, $4)], joinType=[left]) LogicalJoin(condition=[true], joinType=[inner]) LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) LogicalAggregate(group=[{}], c=[COUNT()], ck=[COUNT($0)]) @@ -1544,7 +1544,7 @@ LogicalProject(DEPTNO=[$0], EXPR$1=[OR(AND(IS NOT NULL($5), <>($2, 0)), AND(<($3 ($2, 0)), AND(<($3, $2), null, <>($2, 0), IS NULL($5)))]) - LogicalJoin(condition=[=($1, $4)], joinType=[left]) + LogicalJoin(condition=[=(CAST($1):INTEGER NOT NULL, $4)], joinType=[left]) LogicalJoin(condition=[true], joinType=[inner]) LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) LogicalAggregate(group=[{}], c=[COUNT()], ck=[COUNT($0)]) @@ -18384,7 +18384,7 @@ LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$ @@ -18408,7 +18408,7 @@ LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$ LogicalJoin(condition=[=($1, $9)], joinType=[inner]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) LogicalAggregate(group=[{0}]) - LogicalProject(NAME=[$1]) + LogicalProject(NAME=[CAST($1):VARCHAR(20) NOT NULL]) LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) ]]> diff --git a/core/src/test/resources/sql/sub-query.iq b/core/src/test/resources/sql/sub-query.iq index 1a79635e640e..b301dae413de 100644 --- a/core/src/test/resources/sql/sub-query.iq +++ b/core/src/test/resources/sql/sub-query.iq @@ -2945,7 +2945,7 @@ where e.empno > ANY( select 2 from "scott".dept e2 where e2.deptno = e.deptno) ; !if (use_old_decorr) { EnumerableCalc(expr#0..6=[{inputs}], EMPNO=[$t5]) - EnumerableHashJoin(condition=[AND(IS NOT DISTINCT FROM($4, $6), OR(AND(>($5, $0), IS NOT TRUE(OR(IS NULL($3), =($1, 0)))), AND(>($5, $0), IS NOT TRUE(OR(IS NULL($3), =($1, 0))), IS NOT TRUE(>($5, $0)), <=($1, $2))))], joinType=[inner]) + EnumerableHashJoin(condition=[AND(IS NOT DISTINCT FROM($4, $6), OR(AND(>(CAST($5):INTEGER NOT NULL, $0), IS NOT TRUE(OR(IS NULL($3), =($1, 0)))), AND(>(CAST($5):INTEGER NOT NULL, $0), IS NOT TRUE(OR(IS NULL($3), =($1, 0))), IS NOT TRUE(>(CAST($5):INTEGER NOT NULL, $0)), <=($1, $2))))], joinType=[inner]) EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NOT NULL($t3)], expr#6=[0], expr#7=[CASE($t5, $t3, $t6)], m=[$t2], c=[$t7], d=[$t7], trueLiteral=[$t4], DEPTNO=[$t0]) EnumerableHashJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left]) EnumerableAggregate(group=[{7}]) @@ -2983,7 +2983,7 @@ select empno, e.deptno > ANY( select 2 from "scott".dept e2 where e2.deptno = e.empno) from "scott".emp as e; !if (use_old_decorr) { -EnumerableCalc(expr#0..6=[{inputs}], expr#7=[>($t1, $t2)], expr#8=[IS TRUE($t7)], expr#9=[IS NULL($t5)], expr#10=[0], expr#11=[=($t3, $t10)], expr#12=[OR($t9, $t11)], expr#13=[IS NOT TRUE($t12)], expr#14=[AND($t8, $t13)], expr#15=[>($t3, $t4)], expr#16=[IS TRUE($t15)], expr#17=[null:BOOLEAN], expr#18=[IS NOT TRUE($t7)], expr#19=[AND($t16, $t17, $t13, $t18)], expr#20=[IS NOT TRUE($t15)], expr#21=[AND($t7, $t13, $t18, $t20)], expr#22=[OR($t14, $t19, $t21)], EMPNO=[$t0], EXPR$1=[$t22]) +EnumerableCalc(expr#0..6=[{inputs}], expr#7=[CAST($t1):INTEGER], expr#8=[>($t7, $t2)], expr#9=[IS TRUE($t8)], expr#10=[IS NULL($t5)], expr#11=[0], expr#12=[=($t3, $t11)], expr#13=[OR($t10, $t12)], expr#14=[IS NOT TRUE($t13)], expr#15=[AND($t9, $t14)], expr#16=[>($t3, $t4)], expr#17=[IS TRUE($t16)], expr#18=[null:BOOLEAN], expr#19=[IS NOT TRUE($t8)], expr#20=[AND($t17, $t18, $t14, $t19)], expr#21=[IS NOT TRUE($t16)], expr#22=[AND($t8, $t14, $t19, $t21)], expr#23=[OR($t15, $t20, $t22)], EMPNO=[$t0], EXPR$1=[$t23]) EnumerableHashJoin(condition=[=($0, $6)], joinType=[left]) EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], DEPTNO=[$t7]) EnumerableTableScan(table=[[scott, EMP]]) @@ -3115,7 +3115,7 @@ select * from "scott".emp emp1 where empno <> some (select comm from "scott".emp where deptno = emp1.deptno); !if (use_old_decorr) { -EnumerableCalc(expr#0..13=[{inputs}], expr#14=[<>($t10, $t9)], expr#15=[1], expr#16=[<=($t11, $t15)], expr#17=[AND($t14, $t16)], expr#18=[=($t11, $t15)], expr#19=[OR($t17, $t18)], expr#20=[<>($t0, $t12)], expr#21=[IS NULL($t13)], expr#22=[0], expr#23=[=($t9, $t22)], expr#24=[OR($t21, $t23)], expr#25=[IS NOT TRUE($t24)], expr#26=[AND($t19, $t20, $t25)], expr#27=[IS NOT TRUE($t19)], expr#28=[AND($t25, $t27)], expr#29=[OR($t26, $t28)], proj#0..7=[{exprs}], $condition=[$t29]) +EnumerableCalc(expr#0..13=[{inputs}], expr#14=[<>($t10, $t9)], expr#15=[1], expr#16=[<=($t11, $t15)], expr#17=[AND($t14, $t16)], expr#18=[=($t11, $t15)], expr#19=[OR($t17, $t18)], expr#20=[CAST($t0):DECIMAL(7, 2) NOT NULL], expr#21=[<>($t20, $t12)], expr#22=[IS NULL($t13)], expr#23=[0], expr#24=[=($t9, $t23)], expr#25=[OR($t22, $t24)], expr#26=[IS NOT TRUE($t25)], expr#27=[AND($t19, $t21, $t26)], expr#28=[IS NOT TRUE($t19)], expr#29=[AND($t26, $t28)], expr#30=[OR($t27, $t29)], proj#0..7=[{exprs}], $condition=[$t30]) EnumerableHashJoin(condition=[IS NOT DISTINCT FROM($7, $8)], joinType=[left]) EnumerableTableScan(table=[[scott, EMP]]) EnumerableCalc(expr#0..6=[{inputs}], expr#7=[IS NOT NULL($t2)], expr#8=[0], expr#9=[CASE($t7, $t2, $t8)], expr#10=[IS NOT NULL($t3)], expr#11=[CASE($t10, $t3, $t8)], expr#12=[IS NOT NULL($t4)], expr#13=[CASE($t12, $t4, $t8)], DEPTNO=[$t0], c=[$t9], d=[$t11], dd=[$t13], m=[$t5], trueLiteral=[$t6]) @@ -3174,7 +3174,7 @@ select * from "scott".emp as emp1 where empno <> some (select 2 from "scott".dept dept1 where dept1.deptno = emp1.empno); !if (use_old_decorr) { -EnumerableCalc(expr#0..13=[{inputs}], expr#14=[<>($t9, $t8)], expr#15=[1], expr#16=[<=($t10, $t15)], expr#17=[<>($t0, $t11)], expr#18=[IS NULL($t12)], expr#19=[0], expr#20=[=($t8, $t19)], expr#21=[OR($t18, $t20)], expr#22=[IS NOT TRUE($t21)], expr#23=[AND($t14, $t16, $t17, $t22)], expr#24=[=($t10, $t15)], expr#25=[IS NOT NULL($t10)], expr#26=[AND($t14, $t25)], expr#27=[IS NOT TRUE($t26)], expr#28=[AND($t24, $t17, $t22, $t27)], expr#29=[AND($t14, $t16)], expr#30=[IS NOT TRUE($t29)], expr#31=[IS NOT TRUE($t24)], expr#32=[AND($t22, $t30, $t31)], expr#33=[OR($t23, $t28, $t32)], proj#0..7=[{exprs}], $condition=[$t33]) +EnumerableCalc(expr#0..13=[{inputs}], expr#14=[<>($t9, $t8)], expr#15=[1], expr#16=[<=($t10, $t15)], expr#17=[CAST($t0):INTEGER NOT NULL], expr#18=[<>($t17, $t11)], expr#19=[IS NULL($t12)], expr#20=[0], expr#21=[=($t8, $t20)], expr#22=[OR($t19, $t21)], expr#23=[IS NOT TRUE($t22)], expr#24=[AND($t14, $t16, $t18, $t23)], expr#25=[=($t10, $t15)], expr#26=[IS NOT NULL($t10)], expr#27=[AND($t14, $t26)], expr#28=[IS NOT TRUE($t27)], expr#29=[AND($t25, $t18, $t23, $t28)], expr#30=[AND($t14, $t16)], expr#31=[IS NOT TRUE($t30)], expr#32=[IS NOT TRUE($t25)], expr#33=[AND($t23, $t31, $t32)], expr#34=[OR($t24, $t29, $t33)], proj#0..7=[{exprs}], $condition=[$t34]) EnumerableHashJoin(condition=[=($0, $13)], joinType=[left]) EnumerableTableScan(table=[[scott, EMP]]) EnumerableCalc(expr#0..5=[{inputs}], expr#6=[IS NOT NULL($t2)], expr#7=[0], expr#8=[CASE($t6, $t2, $t7)], expr#9=[IS NOT NULL($t3)], expr#10=[CASE($t9, $t3, $t7)], c=[$t8], d=[$t8], dd=[$t10], m=[$t4], trueLiteral=[$t5], DEPTNO0=[$t0]) @@ -3227,18 +3227,18 @@ select * from "scott".emp as emp1 where comm <> some (select 2 from "scott".dept dept1 where dept1.deptno = emp1.empno); !if (use_old_decorr) { -EnumerableCalc(expr#0..13=[{inputs}], expr#14=[<>($t9, $t8)], expr#15=[1], expr#16=[<=($t10, $t15)], expr#17=[AND($t14, $t16)], expr#18=[=($t10, $t15)], expr#19=[OR($t17, $t18)], expr#20=[<>($t6, $t11)], expr#21=[IS NULL($t12)], expr#22=[IS NULL($t6)], expr#23=[0], expr#24=[=($t8, $t23)], expr#25=[OR($t21, $t22, $t24)], expr#26=[IS NOT TRUE($t25)], expr#27=[AND($t19, $t20, $t26)], expr#28=[IS NOT TRUE($t19)], expr#29=[AND($t26, $t28)], expr#30=[OR($t27, $t29)], proj#0..7=[{exprs}], $condition=[$t30]) +EnumerableCalc(expr#0..13=[{inputs}], expr#14=[<>($t9, $t8)], expr#15=[1], expr#16=[<=($t10, $t15)], expr#17=[AND($t14, $t16)], expr#18=[=($t10, $t15)], expr#19=[OR($t17, $t18)], expr#20=[CAST($t6):DECIMAL(12, 2)], expr#21=[<>($t20, $t11)], expr#22=[IS NULL($t12)], expr#23=[IS NULL($t6)], expr#24=[0], expr#25=[=($t8, $t24)], expr#26=[OR($t22, $t23, $t25)], expr#27=[IS NOT TRUE($t26)], expr#28=[AND($t19, $t21, $t27)], expr#29=[IS NOT TRUE($t19)], expr#30=[AND($t27, $t29)], expr#31=[OR($t28, $t30)], proj#0..7=[{exprs}], $condition=[$t31]) EnumerableHashJoin(condition=[=($0, $13)], joinType=[left]) EnumerableTableScan(table=[[scott, EMP]]) EnumerableCalc(expr#0..5=[{inputs}], expr#6=[IS NOT NULL($t2)], expr#7=[0], expr#8=[CASE($t6, $t2, $t7)], expr#9=[IS NOT NULL($t3)], expr#10=[CASE($t9, $t3, $t7)], c=[$t8], d=[$t8], dd=[$t10], m=[$t4], trueLiteral=[$t5], DEPTNO0=[$t0]) EnumerableHashJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left]) EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0]) EnumerableTableScan(table=[[scott, EMP]]) - EnumerableCalc(expr#0..4=[{inputs}], expr#5=[CAST($t1):BIGINT NOT NULL], expr#6=[CAST($t3):INTEGER NOT NULL], expr#7=[CAST($t4):BOOLEAN NOT NULL], DEPTNO0=[$t0], c=[$t5], dd=[$t2], m=[$t6], trueLiteral=[$t7]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[CAST($t1):BIGINT NOT NULL], expr#6=[CAST($t3):DECIMAL(12, 2) NOT NULL], expr#7=[CAST($t4):BOOLEAN NOT NULL], DEPTNO0=[$t0], c=[$t5], dd=[$t2], m=[$t6], trueLiteral=[$t7]) EnumerableAggregate(group=[{0}], c_g0=[MIN($2) FILTER $6], dd_g0=[COUNT($1) FILTER $5], m_g0=[MIN($3) FILTER $6], trueLiteral_g0=[MIN(true, $4) FILTER $6]) EnumerableCalc(expr#0..5=[{inputs}], expr#6=[0], expr#7=[=($t5, $t6)], expr#8=[1], expr#9=[=($t5, $t8)], proj#0..4=[{exprs}], $g_0=[$t7], $g_1=[$t9]) EnumerableAggregate(group=[{0, 1}], groups=[[{0, 1}, {0}]], c=[COUNT()], m=[MAX($1)], trueLiteral=[LITERAL_AGG(true)], $g=[GROUPING($0, $1)]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[CAST($t0):SMALLINT NOT NULL], expr#4=[2], DEPTNO0=[$t3], EXPR$0=[$t4]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[CAST($t0):SMALLINT NOT NULL], expr#4=[2.00:DECIMAL(12, 2)], DEPTNO0=[$t3], EXPR$0=[$t4]) EnumerableTableScan(table=[[scott, DEPT]]) !plan !} @@ -9205,4 +9205,36 @@ lateral( (2 rows) !ok + +# Test case for [CALCITE-7437] SOME/ANY subquery throws RuntimeException +SELECT deptno, deptno > SOME(SELECT sal FROM emp) AS b FROM dept; ++--------+-------+ +| DEPTNO | B | ++--------+-------+ +| 10 | false | +| 20 | false | +| 30 | false | +| 40 | false | ++--------+-------+ +(4 rows) + +!ok +!if (use_old_decorr) { +EnumerableCalc(expr#0..3=[{inputs}], expr#4=[CAST($t3):DECIMAL(7, 2) NOT NULL], expr#5=[>($t4, $t0)], expr#6=[IS TRUE($t5)], expr#7=[0], expr#8=[<>($t1, $t7)], expr#9=[AND($t6, $t8)], expr#10=[>($t1, $t2)], expr#11=[null:BOOLEAN], expr#12=[IS NOT TRUE($t5)], expr#13=[AND($t10, $t11, $t8, $t12)], expr#14=[<=($t1, $t2)], expr#15=[AND($t5, $t8, $t12, $t14)], expr#16=[OR($t9, $t13, $t15)], DEPTNO=[$t3], B=[$t16]) + EnumerableNestedLoopJoin(condition=[true], joinType=[inner]) + EnumerableAggregate(group=[{}], m=[MIN($5)], c=[COUNT()], d=[COUNT($5)]) + EnumerableTableScan(table=[[scott, EMP]]) + EnumerableCalc(expr#0..2=[{inputs}], DEPTNO=[$t0]) + EnumerableTableScan(table=[[scott, DEPT]]) +!plan +!} + +# Case 2: incompatible types (VARCHAR vs SMALLINT). +# Before fix: java.lang.RuntimeException: while resolving method +# 'gt[class java.lang.String, short]' in class SqlFunctions +# After fix: incompatibleValueType validation error java.lang.NumberFormatException: For input string: "ACCOUNTING" +SELECT deptno, dname > SOME(SELECT empno FROM emp) AS b FROM dept; +For input string: "ACCOUNTING" +!error + # End sub-query.iq diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java index 367fa730688b..a6d2eb6f0d78 100644 --- a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java +++ b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java @@ -16558,6 +16558,26 @@ void testTimestampDiff(boolean coercionEnabled) { + "'1970-01-01 01:23:46'])^", "Values passed to = SOME operator must have compatible types", false); + + // Subquery path: type coercion between column and subquery output column. + f.checkBoolean( + "1 = some (select 1 from (values(1)) as t(x))", true); + f.checkBoolean( + "1.0 = some (select 1 from (values(1)))", true); + f.checkNull( + "1 = some (values(cast(null as integer)))"); + + f.checkBoolean("array[1] = some (array[array[1]])", true); + f.checkBoolean("array[1.0] = some (array[array[1]])", true); + // Test subquery with nested types + f.checkBoolean("array[1] = some (select array[1.0] from (values(1)))", true); + + // Ensure invalid coercion still fails + f.enableTypeCoercion(false).checkFails( + "^array[1] = some (array['a'])^", + "Values passed to = SOME operator must have compatible types", + false); + } @Test void testAnyValueFunc() { From a5dff2cd4b227768805459ccf8429f0200c441b4 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Fri, 1 May 2026 18:42:54 -0700 Subject: [PATCH 240/562] [CALCITE-7498] The parser rejects the example hints from the documentation Signed-off-by: Mihai Budiu --- core/src/main/codegen/templates/Parser.jj | 11 ++++++++--- .../java/org/apache/calcite/sql/SqlHint.java | 4 ++-- .../calcite/test/SqlHintsConverterTest.java | 16 ++++++++++++++++ .../calcite/test/SqlHintsConverterTest.xml | 16 ++++++++++++++++ site/_docs/reference.md | 4 ++-- .../calcite/sql/parser/SqlParserTest.java | 17 +++++++++++++++++ 6 files changed, 61 insertions(+), 7 deletions(-) diff --git a/core/src/main/codegen/templates/Parser.jj b/core/src/main/codegen/templates/Parser.jj index 44e54b076dc7..16b80218e59d 100644 --- a/core/src/main/codegen/templates/Parser.jj +++ b/core/src/main/codegen/templates/Parser.jj @@ -1261,9 +1261,14 @@ void AddKeyValueOption(List list) : key = StringLiteral() ) - value = StringLiteral() { - list.add(key); - list.add(value); + ( + value = StringLiteral() + | + value = SimpleIdentifier() + ) + { + list.add(key); + list.add(value); } } diff --git a/core/src/main/java/org/apache/calcite/sql/SqlHint.java b/core/src/main/java/org/apache/calcite/sql/SqlHint.java index 7c26e7fda07c..38f54636ebe8 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlHint.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlHint.java @@ -151,7 +151,7 @@ public Map getOptionKVPairs() { for (int i = 0; i < options.size() - 1; i += 2) { final SqlNode k = options.get(i); final SqlNode v = options.get(i + 1); - attrs.put(getOptionKeyAsString(k), ((SqlLiteral) v).getValueAs(String.class)); + attrs.put(getOptionAsString(k), getOptionAsString(v)); } return ImmutableMap.copyOf(attrs); } else { @@ -204,7 +204,7 @@ public enum HintOptionFormat implements Symbolizable { //~ Tools ------------------------------------------------------------------ - private static String getOptionKeyAsString(SqlNode node) { + private static String getOptionAsString(SqlNode node) { assert node instanceof SqlIdentifier || SqlUtil.isLiteral(node); if (node instanceof SqlIdentifier) { return ((SqlIdentifier) node).getSimple(); diff --git a/core/src/test/java/org/apache/calcite/test/SqlHintsConverterTest.java b/core/src/test/java/org/apache/calcite/test/SqlHintsConverterTest.java index cff0c2047459..a34b4c2506ae 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlHintsConverterTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlHintsConverterTest.java @@ -149,6 +149,16 @@ public final Fixture sql(String sql) { //~ Tests ------------------------------------------------------------------ + /** Test case for [CALCITE-7498] + * The parser rejects the example hints from the documentation. */ + @Test void testDocumentationExample() { + final String sql = "SELECT /*+ hint1, hint2(a='1', b='2') */ *\n" + + "FROM emp /*+ hint3(5, 'x') */\n" + + "JOIN dept /*+ hint4(c=id), hint5 */\n" + + "ON emp.deptno = dept.deptno"; + sql(sql).ok(); + } + @Test void testQueryHint() { final String sql = HintTools.withHint("select /*+ %s */ *\n" + "from emp e1\n" @@ -1064,6 +1074,12 @@ static HintStrategyTable createHintStrategies(HintStrategyTable.Builder builder) .hintStrategy( "preserved_project", HintStrategy.builder( HintPredicates.PROJECT).excludedRules(CoreRules.FILTER_PROJECT_TRANSPOSE).build()) + // meaningless hints for the example in the documentation + .hintStrategy("hint1", HintPredicates.JOIN) + .hintStrategy("hint2", HintPredicates.JOIN) + .hintStrategy("hint3", HintPredicates.TABLE_SCAN) + .hintStrategy("hint4", HintPredicates.TABLE_SCAN) + .hintStrategy("hint5", HintPredicates.TABLE_SCAN) .build(); } diff --git a/core/src/test/resources/org/apache/calcite/test/SqlHintsConverterTest.xml b/core/src/test/resources/org/apache/calcite/test/SqlHintsConverterTest.xml index 8ac1e96de7c7..e2cd95622d57 100644 --- a/core/src/test/resources/org/apache/calcite/test/SqlHintsConverterTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/SqlHintsConverterTest.xml @@ -55,6 +55,22 @@ from orders, products_temporal for system_time as of orders.rowtime]]> + + + + + + + + diff --git a/site/_docs/reference.md b/site/_docs/reference.md index 5f12b22149d6..1c13f10324dc 100644 --- a/site/_docs/reference.md +++ b/site/_docs/reference.md @@ -3532,11 +3532,11 @@ optionKey: | stringLiteral optionVal: - stringLiteral + simpleIdentifier + | stringLiteral hintOption: simpleIdentifier - | numericLiteral | stringLiteral {% endhighlight %} diff --git a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java index cfea5b6ce0e9..fad96f70b956 100644 --- a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java +++ b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java @@ -9368,6 +9368,23 @@ private static Consumer> checkWarnings( sql(sql2).ok(expected); } + /** Test case for [CALCITE-7498] + * The parser rejects the example hints from the documentation. */ + @Test void testDocumentationExample() { + final String sql = "SELECT /*+ hint1, hint2(a='1', b='2') */ *\n" + + "FROM emp /*+ hint3(5, 'x') */\n" + + "JOIN dept /*+ hint4(c=id), hint5 */\n" + + "ON emp.deptno = dept.deptno"; + final String expected = "SELECT\n" + + "/*+ `HINT1`, `HINT2`(`A` = '1', `B` = '2') */\n" + + "*\n" + + "FROM `EMP`\n" + + "/*+ `HINT3`(5, 'x') */\n" + + "INNER JOIN `DEPT`\n" + + "/*+ `HINT4`(`C` = `ID`), `HINT5` */ ON (`EMP`.`DEPTNO` = `DEPT`.`DEPTNO`)"; + sql(sql).ok(expected); + } + @Test void testQueryHint() { final String sql1 = "select " + "/*+ properties(k1='v1', k2='v2', 'a.b.c'='v3'), " From 87e63bfffa9ac7b25abb2820e51436f021e4ff24 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Mon, 4 May 2026 12:53:07 -0700 Subject: [PATCH 241/562] [CALCITE-7501] Assertion error in alias expansion for LEFT JOIN USING Signed-off-by: Mihai Budiu --- .../sql/validate/SqlValidatorImpl.java | 17 ++++------ core/src/test/resources/sql/planner.iq | 33 ++++++++++++++++++- 2 files changed, 39 insertions(+), 11 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java index 320ba721533b..6881bae03525 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java @@ -123,7 +123,6 @@ import org.apache.calcite.util.trace.CalciteTrace; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; @@ -568,8 +567,8 @@ private boolean isNonAggregatedNonGroupedColumn(SqlNode node, SqlSelect select) return false; } - private static Map getFieldAliases(final SelectScope scope) { - final ImmutableMap.Builder fieldAliases = new ImmutableMap.Builder<>(); + private static ImmutableSet getFieldsAliased(final SelectScope scope) { + final ImmutableSet.Builder result = new ImmutableSet.Builder<>(); for (SqlNode selectItem : scope.getNode().getSelectList()) { if (selectItem instanceof SqlCall) { @@ -580,12 +579,11 @@ private static Map getFieldAliases(final SelectScope scope) { } final SqlIdentifier fieldIdentifier = call.operand(0); - fieldAliases.put(fieldIdentifier.getSimple(), - ((SqlIdentifier) call.operand(1)).getSimple()); + result.add(fieldIdentifier.names.get(fieldIdentifier.names.size() - 1)); } } - return fieldAliases.build(); + return result.build(); } /** Returns the set of field names in the join condition specified by USING @@ -7520,7 +7518,7 @@ private SqlNode expandExprFromJoin(SqlJoin join, SqlIdentifier identifier, Selec } final SqlNameMatcher matcher = validator.getCatalogReader().nameMatcher(); - final Map fieldAliases = getFieldAliases(scope); + final Set fieldAliases = getFieldsAliased(scope); for (String name : commonColumnNames) { if (matcher.matches(identifier.getSimple(), name)) { @@ -7537,13 +7535,12 @@ private SqlNode expandExprFromJoin(SqlJoin join, SqlIdentifier identifier, Selec assert qualifiedNode.size() == 2; - // If there is an alias for the column, no need to wrap the coalesce with an AS operator - boolean haveAlias = fieldAliases.containsKey(name); - final SqlCall coalesceCall = SqlStdOperatorTable.COALESCE.createCall(SqlParserPos.ZERO, qualifiedNode.get(0), qualifiedNode.get(1)); + // If there is an alias for the column, no need to wrap the coalesce with an AS operator + boolean haveAlias = fieldAliases.contains(name); if (haveAlias) { return coalesceCall; } else { diff --git a/core/src/test/resources/sql/planner.iq b/core/src/test/resources/sql/planner.iq index 96aaaf5eedc1..16168b820a5a 100644 --- a/core/src/test/resources/sql/planner.iq +++ b/core/src/test/resources/sql/planner.iq @@ -146,9 +146,40 @@ EnumerableCalc(expr#0..2=[{inputs}], $f0=[$t1], $f1=[$t2]) EnumerableValues(tuples=[[{ 10 }, { 10 }, { 20 }, { 30 }, { 30 }, { 50 }, { 50 }, { 60 }, { null }]]) !plan !set planner-rules original +!use blank + +# Test case for [CALCITE-7501] Assertion error in alias expansion for LEFT JOIN USING +CREATE TABLE D(sk_cid INT, dt DATE, dm_sym VARCHAR, fhd DATE); +(0 rows modified) + +!update + +CREATE TABLE F(sk_cid INT); +(0 rows modified) + +!update + +CREATE TABLE S(sk_sid INT, sym VARCHAR); +(0 rows modified) + +!update + +SELECT + d.dt as dtn, + fhd as sk_fhd +FROM D +JOIN S + ON S.sym = D.dm_sym +LEFT JOIN F USING (sk_cid); ++-----+--------+ +| DTN | SK_FHD | ++-----+--------+ ++-----+--------+ +(0 rows) + +!ok # Add tests for [CALCITE-6985] to verify AggregateMinMaxToLimitRule handles empty tables correctly -!use blank create table t_empty (id int); (0 rows modified) From 186b143453b5a9c63303987104974110b3cf7deb Mon Sep 17 00:00:00 2001 From: Sergey Nuyanzin Date: Sat, 2 May 2026 17:48:18 +0200 Subject: [PATCH 242/562] [CALCITE-7499] `COALESCE` with args of different types might be incorrectly simplified --- .../org/apache/calcite/rex/RexSimplify.java | 2 +- .../apache/calcite/rex/RexProgramTest.java | 64 +++++++++++++++++++ 2 files changed, 65 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/calcite/rex/RexSimplify.java b/core/src/main/java/org/apache/calcite/rex/RexSimplify.java index 764100889ef8..1c38d2d72147 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexSimplify.java +++ b/core/src/main/java/org/apache/calcite/rex/RexSimplify.java @@ -1324,7 +1324,7 @@ private RexNode simplifyCoalesce(RexCall call) { case 0: return rexBuilder.makeNullLiteral(call.type); case 1: - return operands.get(0); + return rexBuilder.ensureType(call.type, operands.get(0), true); default: if (operands.equals(call.operands)) { return call; diff --git a/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java b/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java index 3c6620d1b78d..3faf9173d634 100644 --- a/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java +++ b/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java @@ -3724,6 +3724,70 @@ private void assertTypeAndToString( "COALESCE(?0.decimal0, ?0.decimal1)"); } + /** Unit test for + * [CALCITE-7499] + * COALESCE doesn't take into account leastRestrictiveType for input args. */ + @Test void testSimplifyCoalescePreservesLeastRestrictiveReturnType() { + final RelDataType tinyint = typeFactory.createSqlType(SqlTypeName.TINYINT); + final RelDataType tinyintNullable = nullable(tinyint); + + checkSimplify( + coalesce(nullInt, + literal(BigDecimal.ONE, tInt()), + literal(BigDecimal.valueOf(2L), tBigInt())), + "1:BIGINT"); + + checkSimplify( + coalesce(nullSmallInt, + literal(BigDecimal.ONE, tSmallInt()), + literal(BigDecimal.valueOf(2L), tBigInt())), + "1:BIGINT"); + + checkSimplify( + coalesce(nullSmallInt, + literal(BigDecimal.ONE, tSmallInt()), + literal(BigDecimal.valueOf(42L), tInt())), + "1"); + + checkSimplify( + coalesce(null_(tinyintNullable), + literal(BigDecimal.valueOf(7L), tinyint), + literal(BigDecimal.valueOf(42L), tInt())), + "7"); + + checkSimplify( + coalesce(null_(tinyintNullable), + literal(BigDecimal.valueOf(7L), tinyint), + literal(BigDecimal.valueOf(42L), tBigInt())), + "7:BIGINT"); + + // Two narrow operands then a wide one. + checkSimplify( + coalesce(nullSmallInt, + literal(BigDecimal.valueOf(11L), tSmallInt()), + literal(BigDecimal.valueOf(22L), tInt()), + literal(BigDecimal.valueOf(33L), tBigInt())), + "11:BIGINT"); + + // ---- DECIMAL: precision widening (same scale, so the underlying BigDecimal value is + final RelDataType dec5_2 = typeFactory.createSqlType(SqlTypeName.DECIMAL, 5, 2); + final RelDataType dec10_2 = typeFactory.createSqlType(SqlTypeName.DECIMAL, 10, 2); + checkSimplify( + coalesce(null_(nullable(dec5_2)), + literal(new BigDecimal("1.23"), dec5_2), + literal(new BigDecimal("9876543.21"), dec10_2)), + "1.23:DECIMAL(10, 2)"); + + // ---- TIMESTAMP: precision widening ---- + final RelDataType ts0 = typeFactory.createSqlType(SqlTypeName.TIMESTAMP, 0); + checkSimplify( + coalesce(rexBuilder.makeNullLiteral(nullable(ts0)), + rexBuilder.makeTimestampLiteral(new TimestampString("2026-01-01 00:00:00"), 0), + rexBuilder.makeTimestampLiteral( + new TimestampString("2026-01-01 00:00:00.123"), 3)), + "2026-01-01 00:00:00:TIMESTAMP(3)"); + } + @Test void simplifyNull() { checkSimplify3(nullBool, "null:BOOLEAN", "false", "true"); // null int must not be simplified to false From 20618040b3839c83bc2570466f58a75abfe41d99 Mon Sep 17 00:00:00 2001 From: Dongsheng He Date: Tue, 5 May 2026 18:44:59 +0800 Subject: [PATCH 243/562] [CALCITE-7502] RelToSqlConverter creates invalid sql when converting nested window contains SqlCaseWhen --- .../calcite/rel/rel2sql/SqlImplementor.java | 32 ++++++++++--------- .../rel/rel2sql/RelToSqlConverterTest.java | 28 ++++++++++++++++ 2 files changed, 45 insertions(+), 15 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java b/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java index f4dd0f686986..7d579d184cc8 100644 --- a/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java +++ b/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java @@ -99,6 +99,7 @@ import org.apache.calcite.sql.type.SqlTypeFactoryImpl; import org.apache.calcite.sql.type.SqlTypeFamily; import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.sql.util.SqlBasicVisitor; import org.apache.calcite.sql.util.SqlShuttle; import org.apache.calcite.sql.validate.SqlValidatorUtil; import org.apache.calcite.util.DateString; @@ -2177,24 +2178,25 @@ private boolean containsOver(@UnknownInitialization Result this, if (node == null) { return false; } - if (node.getKind() == SqlKind.WINDOW) { - return true; - } - if (node instanceof SqlSelect) { - final SqlNodeList selectList = ((SqlSelect) node).getSelectList(); - for (SqlNode child : selectList) { - if (containsOver(child)) { - return true; + final boolean[] result = {false}; + node.accept(new SqlBasicVisitor() { + @Override public Void visit(SqlCall call) { + if (result[0]) { + return null; } - } - } else if (node instanceof SqlBasicCall) { - for (SqlNode operand : ((SqlBasicCall) node).getOperandList()) { - if (containsOver(operand)) { - return true; + if (call.getKind() == SqlKind.WINDOW) { + result[0] = true; + return null; + } + for (SqlNode operand : call.getOperandList()) { + if (operand != null) { + operand.accept(this); + } } + return null; } - } - return false; + }); + return result[0]; } /** Returns whether an {@link Aggregate} contains nested operands that diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java index 7c41f6c9d54e..d0768b743e34 100644 --- a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java +++ b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java @@ -5960,6 +5960,34 @@ private void checkLiteral2(String expression, String expected) { + "FROM \"foodmart\".\"product\""; sql(query1).optimize(rules, null).ok(expected10); sql(query1).ok(expected11); + + String query2 = " SELECT " + + "SUM (\"daily_sales\") OVER (PARTITION BY \"product_name\") AS \"sales\" " + + "FROM ( SELECT \"product_name\", " + + "CASE WHEN SUM(\"product_id\") OVER (PARTITION BY \"product_name\") > 0 " + + "THEN 1 ELSE 0 END AS \"daily_sales\" " + + "FROM \"product\" ) subquery"; + String expected20 = "SELECT " + + "SUM(\"daily_sales\") " + + "OVER (PARTITION BY \"product_name\" " + + "RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS \"sales\"\n" + + "FROM (SELECT \"product_name\", " + + "CASE WHEN (SUM(\"product_id\") OVER (PARTITION BY \"product_name\" " + + "RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)) > 0 THEN 1 ELSE 0 END " + + "AS \"daily_sales\"\n" + + "FROM \"foodmart\".\"product\") AS \"t1\""; + String expected21 = "SELECT " + + "SUM(\"daily_sales\") " + + "OVER (PARTITION BY \"product_name\" " + + "RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS \"sales\"\n" + + "FROM (SELECT \"product_name\", " + + "CASE WHEN (SUM(\"product_id\") " + + "OVER (PARTITION BY \"product_name\" " + + "RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)) > 0 THEN 1 ELSE 0 END " + + "AS \"daily_sales\"\n" + + "FROM \"foodmart\".\"product\") AS \"t\""; + sql(query2).optimize(rules, null).ok(expected20); + sql(query2).ok(expected21); } /** Test case for From e1b2c4dd0296585c9c5f25ae6da7d01efd75e0a7 Mon Sep 17 00:00:00 2001 From: xuzifu666 <1206332514@qq.com> Date: Fri, 24 Apr 2026 15:03:55 +0800 Subject: [PATCH 244/562] [CALCITE-6291] Support converting ArrowTable to Queryable --- .../calcite/adapter/arrow/ArrowTable.java | 38 +++++++++- .../adapter/arrow/ArrowAdapterTest.java | 70 +++++++++++++++++++ 2 files changed, 107 insertions(+), 1 deletion(-) diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTable.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTable.java index fa8d59389b88..ba1568bcd2ad 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTable.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTable.java @@ -19,6 +19,7 @@ import org.apache.calcite.DataContext; import org.apache.calcite.adapter.java.JavaTypeFactory; import org.apache.calcite.linq4j.Enumerable; +import org.apache.calcite.linq4j.Enumerator; import org.apache.calcite.linq4j.QueryProvider; import org.apache.calcite.linq4j.Queryable; import org.apache.calcite.linq4j.tree.Expression; @@ -33,6 +34,7 @@ import org.apache.calcite.schema.Schemas; import org.apache.calcite.schema.TranslatableTable; import org.apache.calcite.schema.impl.AbstractTable; +import org.apache.calcite.schema.impl.AbstractTableQueryable; import org.apache.calcite.util.ImmutableIntList; import org.apache.calcite.util.Util; @@ -184,7 +186,7 @@ public Enumerable query(DataContext root, ImmutableIntList fields, @Override public Queryable asQueryable(QueryProvider queryProvider, SchemaPlus schema, String tableName) { - throw new UnsupportedOperationException(); + return new ArrowQueryable<>(queryProvider, schema, this, tableName); } @Override public Type getElementType() { @@ -262,4 +264,38 @@ private static TreeNode makeLiteralNode(String literal, String type) { + ", type " + type); } } + + /** + * Implementation of {@link Queryable} based on a {@link ArrowTable}. + * + * @param element type + */ + public static class ArrowQueryable extends AbstractTableQueryable { + ArrowQueryable(QueryProvider queryProvider, SchemaPlus schema, + ArrowTable table, String tableName) { + super(queryProvider, schema, table, tableName); + } + + @Override public Enumerator enumerator() { + throw new UnsupportedOperationException("enumerator"); + } + + private ArrowTable getTable() { + return (ArrowTable) table; + } + + /** + * Executes a query with projection and filter conditions. + * + * @param fields projection fields + * @param conditions filter conditions + * @return result as enumerable + */ + @SuppressWarnings("UnusedDeclaration") + public Enumerable query(List fields, + List>> conditions) { + final ImmutableIntList fieldList = ImmutableIntList.copyOf(fields); + return getTable().query(null, fieldList, conditions); + } + } } diff --git a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java index 25212a208305..275bdd0be76c 100644 --- a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java +++ b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java @@ -17,6 +17,9 @@ package org.apache.calcite.adapter.arrow; import org.apache.calcite.jdbc.JavaTypeFactoryImpl; +import org.apache.calcite.linq4j.Enumerable; +import org.apache.calcite.linq4j.Enumerator; +import org.apache.calcite.linq4j.Queryable; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rel.type.RelDataTypeSystem; @@ -39,6 +42,10 @@ import java.nio.file.Files; import java.nio.file.Path; import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; import java.util.Map; import static org.apache.calcite.test.Matchers.isListOf; @@ -94,6 +101,69 @@ static void initializeArrowState(@TempDir Path sharedTempDir) isListOf("intField", "stringField", "floatField", "longField")); } + /** Test case for + * [CALCITE-6291] + * Support converting ArrowTable to Queryable. + */ + @Test void testArrowTableAsQueryable() { + ArrowSchema arrowSchema = new ArrowSchema(arrowDataDirectory); + Map tableMap = arrowSchema.getTableMap(); + ArrowTable arrowTable = (ArrowTable) tableMap.get("ARROWDATA"); + + // asQueryable can accept null QueryProvider and SchemaPlus for testing + Queryable queryable = arrowTable.asQueryable(null, null, "ARROWDATA"); + + // Verify that asQueryable returns a non-null Queryable + assert queryable != null : "asQueryable should return a non-null Queryable"; + + // Verify that the Queryable is an instance of ArrowQueryable + assert queryable instanceof ArrowTable.ArrowQueryable + : "asQueryable should return an instance of ArrowQueryable"; + } + + /** Test case for + * [CALCITE-6291] + * Support converting ArrowTable to Queryable. + */ + @Test void testArrowQueryableQueryWithData() { + ArrowSchema arrowSchema = new ArrowSchema(arrowDataDirectory); + Map tableMap = arrowSchema.getTableMap(); + ArrowTable arrowTable = (ArrowTable) tableMap.get("ARROWDATA"); + + @SuppressWarnings("unchecked") + ArrowTable.ArrowQueryable queryable = + (ArrowTable.ArrowQueryable) arrowTable.asQueryable(null, null, "ARROWDATA"); + + // Query with projection on all fields (0, 1, 2, 3) and no filter conditions + List fields = Arrays.asList(0, 1, 2, 3); + List>> conditions = Collections.emptyList(); + + Enumerable enumerable = queryable.query(fields, conditions); + + // Fetch the first 6 rows using enumerator + List results = new ArrayList<>(); + Enumerator enumerator = enumerable.enumerator(); + int count = 0; + while (enumerator.moveNext() && count < 6) { + results.add(enumerator.current()); + count++; + } + enumerator.close(); + + assert results.size() == 6 : "Expected 6 rows, got " + results.size(); + + // Verify each row is an Object array with 4 fields + for (int i = 0; i < 6; i++) { + Object[] row = (Object[]) results.get(i); + assert row.length == 4 : "Expected 4 fields, got " + row.length; + assert row[0].equals(i) : "Expected intField=" + i + ", got " + row[0]; + // stringField may be Text or String type, convert to String for comparison + String stringValue = row[1].toString(); + assert stringValue.equals(String.valueOf(i)) + : "Expected stringField=" + i + ", got " + stringValue; + } + } + @Test void testArrowProjectAllFields() { String sql = "select * from arrowdata\n"; String plan = "PLAN=ArrowToEnumerableConverter\n" From de01ec4f11f50dd18cdba13a1d1501f6f4a87539 Mon Sep 17 00:00:00 2001 From: Ruben Quesada Lopez Date: Wed, 6 May 2026 16:26:21 +0100 Subject: [PATCH 245/562] Site: Update Ruben QL info --- site/_data/contributors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site/_data/contributors.yml b/site/_data/contributors.yml index 27328f4f7342..9c46540786ec 100644 --- a/site/_data/contributors.yml +++ b/site/_data/contributors.yml @@ -295,7 +295,7 @@ - name: Ruben Quesada Lopez apacheId: rubenql githubId: rubenada - org: Voltron Data + org: Cloudera role: PMC - name: Rui Wang apacheId: amaliujia From ff147be7a8cb093b21a1e2275e3816832d6964d1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 8 May 2026 10:48:08 +0000 Subject: [PATCH 246/562] Bump nokogiri from 1.19.1 to 1.19.3 in /site Bumps [nokogiri](https://github.com/sparklemotion/nokogiri) from 1.19.1 to 1.19.3. - [Release notes](https://github.com/sparklemotion/nokogiri/releases) - [Changelog](https://github.com/sparklemotion/nokogiri/blob/main/CHANGELOG.md) - [Commits](https://github.com/sparklemotion/nokogiri/compare/v1.19.1...v1.19.3) --- updated-dependencies: - dependency-name: nokogiri dependency-version: 1.19.3 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- site/Gemfile | 2 +- site/Gemfile.lock | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/site/Gemfile b/site/Gemfile index ba74be552f09..815bd370ea29 100644 --- a/site/Gemfile +++ b/site/Gemfile @@ -16,7 +16,7 @@ source 'https://rubygems.org' gem 'jekyll', '~>4' gem "webrick", "~> 1.9.1" -gem "nokogiri", "~> 1.19.1" +gem "nokogiri", "~> 1.19.3" gem "csv", "~> 3.3.2" gem "base64", "~> 0.2.0" diff --git a/site/Gemfile.lock b/site/Gemfile.lock index a4cc1b4b44d5..5d750044420e 100644 --- a/site/Gemfile.lock +++ b/site/Gemfile.lock @@ -74,21 +74,21 @@ GEM rb-fsevent (~> 0.10, >= 0.10.3) rb-inotify (~> 0.9, >= 0.9.10) mercenary (0.4.0) - nokogiri (1.19.1-aarch64-linux-gnu) + nokogiri (1.19.3-aarch64-linux-gnu) racc (~> 1.4) - nokogiri (1.19.1-aarch64-linux-musl) + nokogiri (1.19.3-aarch64-linux-musl) racc (~> 1.4) - nokogiri (1.19.1-arm-linux-gnu) + nokogiri (1.19.3-arm-linux-gnu) racc (~> 1.4) - nokogiri (1.19.1-arm-linux-musl) + nokogiri (1.19.3-arm-linux-musl) racc (~> 1.4) - nokogiri (1.19.1-arm64-darwin) + nokogiri (1.19.3-arm64-darwin) racc (~> 1.4) - nokogiri (1.19.1-x86_64-darwin) + nokogiri (1.19.3-x86_64-darwin) racc (~> 1.4) - nokogiri (1.19.1-x86_64-linux-gnu) + nokogiri (1.19.3-x86_64-linux-gnu) racc (~> 1.4) - nokogiri (1.19.1-x86_64-linux-musl) + nokogiri (1.19.3-x86_64-linux-musl) racc (~> 1.4) pathutil (0.16.2) forwardable-extended (~> 2.6) @@ -140,7 +140,7 @@ DEPENDENCIES csv (~> 3.3.2) jekyll (~> 4) jekyll-redirect-from - nokogiri (~> 1.19.1) + nokogiri (~> 1.19.3) webrick (~> 1.9.1) BUNDLED WITH From a3ac957a45a31128979f1e9ed0630af3d3d824cb Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Thu, 7 May 2026 22:25:23 -0700 Subject: [PATCH 247/562] [CALCITE-7506] RelWriterImpl does not output hints Signed-off-by: Mihai Budiu --- .../rel/externalize/RelWriterImpl.java | 8 +++++ .../org/apache/calcite/tools/PlannerTest.java | 30 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/core/src/main/java/org/apache/calcite/rel/externalize/RelWriterImpl.java b/core/src/main/java/org/apache/calcite/rel/externalize/RelWriterImpl.java index 8d52ffbe3805..25af2ed3af35 100644 --- a/core/src/main/java/org/apache/calcite/rel/externalize/RelWriterImpl.java +++ b/core/src/main/java/org/apache/calcite/rel/externalize/RelWriterImpl.java @@ -20,6 +20,8 @@ import org.apache.calcite.linq4j.Ord; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.RelWriter; +import org.apache.calcite.rel.hint.Hintable; +import org.apache.calcite.rel.hint.RelHint; import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.sql.SqlExplainLevel; import org.apache.calcite.util.Pair; @@ -105,6 +107,12 @@ protected void explain_(RelNode rel, } switch (detailLevel) { case ALL_ATTRIBUTES: + if (rel instanceof Hintable) { + List hints = ((Hintable) rel).getHints(); + if (!hints.isEmpty()) { + s.append(hints); + } + } s.append(": rowcount = ") .append(mq.getRowCount(rel)) .append(", cumulative cost = ") diff --git a/core/src/test/java/org/apache/calcite/tools/PlannerTest.java b/core/src/test/java/org/apache/calcite/tools/PlannerTest.java index 9a367f37abe7..2907bd3f22a9 100644 --- a/core/src/test/java/org/apache/calcite/tools/PlannerTest.java +++ b/core/src/test/java/org/apache/calcite/tools/PlannerTest.java @@ -44,6 +44,8 @@ import org.apache.calcite.rel.core.JoinRelType; import org.apache.calcite.rel.core.RelFactories; import org.apache.calcite.rel.core.TableScan; +import org.apache.calcite.rel.hint.HintPredicates; +import org.apache.calcite.rel.hint.HintStrategyTable; import org.apache.calcite.rel.logical.LogicalFilter; import org.apache.calcite.rel.logical.LogicalProject; import org.apache.calcite.rel.metadata.RelMetadataQuery; @@ -75,6 +77,7 @@ import org.apache.calcite.sql.util.SqlOperatorTables; import org.apache.calcite.sql.validate.SqlValidator; import org.apache.calcite.sql.validate.SqlValidatorScope; +import org.apache.calcite.sql2rel.SqlToRelConverter; import org.apache.calcite.test.CalciteAssert; import org.apache.calcite.test.RelBuilderTest; import org.apache.calcite.test.schemata.tpch.TpchSchema; @@ -556,6 +559,33 @@ private void checkUnionPruning(String sql, String plan, RelOptRule... extraRules + " EnumerableTableScan(table=[[hr, emps]])\n")); } + /** Test case for [CALCITE-7506] + * RelWriterImpl does not output hints. */ + @Test void testHints() throws Exception { + HintStrategyTable hintTable = HintStrategyTable.builder() + .hintStrategy("hint", HintPredicates.PROJECT).build(); + final SchemaPlus rootSchema = Frameworks.createRootSchema(true); + final FrameworkConfig config = Frameworks.newConfigBuilder() + .parserConfig(SqlParser.Config.DEFAULT) + .defaultSchema(CalciteAssert.addSchema(rootSchema, CalciteAssert.SchemaSpec.HR)) + .sqlToRelConverterConfig(SqlToRelConverter.CONFIG.withHintStrategyTable(hintTable)) + .build(); + Planner planner = Frameworks.getPlanner(config); + final String sql = "select /*+ hint */ * from \"emps\" order by \"emps\".\"deptno\""; + SqlNode parse = planner.parse(sql); + SqlNode validate = planner.validate(parse); + RelNode convert = planner.rel(validate).project(); + String explain = + Util.toLinux( + RelOptUtil.dumpPlan("", convert, SqlExplainFormat.TEXT, + SqlExplainLevel.ALL_ATTRIBUTES)); + // Cannot check for exact equality since ALL_ATTRIBUTES prints node ids, + // which change from run to run + assertThat(explain, + containsString("LogicalSort(sort0=[$1], dir0=[ASC])[[HINT inheritPath:[]]]: " + + "rowcount = 100.0, cumulative cost = {300.0 rows, 15337.544595161893 cpu, 0.0 io}")); + } + /** Test case for * [CALCITE-2554] * Enrich EnumerableHashJoin operator with order preserving information. From 3401c1c34f23bb1a48caf04c7e4567bbd09d7d4f Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Thu, 7 May 2026 21:46:14 -0700 Subject: [PATCH 248/562] [CALCITE-7503] Hint validation does not have access to source position Signed-off-by: Mihai Budiu --- core/src/main/codegen/templates/Parser.jj | 4 +- .../org/apache/calcite/rel/hint/RelHint.java | 32 +++++++- .../java/org/apache/calcite/sql/SqlUtil.java | 2 +- .../calcite/test/SqlHintsConverterTest.java | 75 ++++++++++--------- 4 files changed, 73 insertions(+), 40 deletions(-) diff --git a/core/src/main/codegen/templates/Parser.jj b/core/src/main/codegen/templates/Parser.jj index 16b80218e59d..abe4c3ccec4e 100644 --- a/core/src/main/codegen/templates/Parser.jj +++ b/core/src/main/codegen/templates/Parser.jj @@ -1304,11 +1304,13 @@ SqlNodeList ParenthesizedLiteralOptionCommaList() : void AddHint(List hints) : { + final Span s; final SqlIdentifier hintName; final SqlNodeList hintOptions; final SqlHint.HintOptionFormat optionFormat; } { + { s = span(); } hintName = SimpleIdentifier() ( LOOKAHEAD(5) @@ -1335,7 +1337,7 @@ void AddHint(List hints) : ) { hints.add( - new SqlHint(Span.of(hintOptions).end(this), hintName, hintOptions, + new SqlHint(s.end(this), hintName, hintOptions, optionFormat)); } } diff --git a/core/src/main/java/org/apache/calcite/rel/hint/RelHint.java b/core/src/main/java/org/apache/calcite/rel/hint/RelHint.java index 826962fdbf75..73ac9d146d7b 100644 --- a/core/src/main/java/org/apache/calcite/rel/hint/RelHint.java +++ b/core/src/main/java/org/apache/calcite/rel/hint/RelHint.java @@ -16,6 +16,8 @@ */ package org.apache.calcite.rel.hint; +import org.apache.calcite.sql.parser.SqlParserPos; + import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -86,6 +88,7 @@ public class RelHint { //~ Instance fields -------------------------------------------------------- + public final SqlParserPos pos; public final ImmutableList inheritPath; public final String hintName; public final List listOptions; @@ -96,16 +99,19 @@ public class RelHint { /** * Creates a {@code RelHint}. * + * @param pos Parser position * @param inheritPath Hint inherit path * @param hintName Hint name * @param listOption Hint options as string list * @param kvOptions Hint options as string key value pair */ private RelHint( + SqlParserPos pos, Iterable inheritPath, String hintName, @Nullable List listOption, @Nullable Map kvOptions) { + this.pos = pos; this.inheritPath = ImmutableList.copyOf(inheritPath); this.hintName = requireNonNull(hintName, "hintName"); this.listOptions = listOption == null ? ImmutableList.of() : ImmutableList.copyOf(listOption); @@ -127,7 +133,7 @@ public static Builder builder(String hintName) { */ public RelHint copy(List inheritPath) { requireNonNull(inheritPath, "inheritPath"); - return new RelHint(inheritPath, hintName, listOptions, kvOptions); + return new RelHint(pos, inheritPath, hintName, listOptions, kvOptions); } @Override public boolean equals(@Nullable Object o) { @@ -138,13 +144,26 @@ public RelHint copy(List inheritPath) { return false; } RelHint hint = (RelHint) o; + // Note: two hints can be equal if they have different position, + // for preserving backwards compatibility with old behaviors return inheritPath.equals(hint.inheritPath) && hintName.equals(hint.hintName) && Objects.equals(listOptions, hint.listOptions) && Objects.equals(kvOptions, hint.kvOptions); } + /** True if the two hints are identical, including position. */ + public boolean identical(RelHint hint) { + return inheritPath.equals(hint.inheritPath) + && hintName.equals(hint.hintName) + && Objects.equals(listOptions, hint.listOptions) + && Objects.equals(kvOptions, hint.kvOptions) + && pos.equals(hint.pos); + } + @Override public int hashCode() { + // Note: hash does not include position, required by the backwards-compatible definition + // of equality. return Objects.hash(this.hintName, this.inheritPath, this.listOptions, this.kvOptions); } @@ -171,6 +190,7 @@ public RelHint copy(List inheritPath) { /** Builder for {@link RelHint}. */ public static class Builder { + private SqlParserPos pos; private final String hintName; private List inheritPath; @@ -178,6 +198,7 @@ public static class Builder { private Map kvOptions; private Builder(String hintName) { + this.pos = SqlParserPos.ZERO; this.listOptions = new ArrayList<>(); this.kvOptions = new LinkedHashMap<>(); this.hintName = hintName; @@ -190,6 +211,12 @@ public Builder inheritPath(Iterable inheritPath) { return this; } + /** Sets up the parser position. */ + public Builder position(SqlParserPos pos) { + this.pos = pos; + return this; + } + /** Sets up the inherit path with given integer array. */ public Builder inheritPath(Integer... inheritPath) { this.inheritPath = Arrays.asList(inheritPath); @@ -234,7 +261,8 @@ public Builder hintOptions(Map kvOptions) { } public RelHint build() { - return new RelHint(this.inheritPath, this.hintName, this.listOptions, this.kvOptions); + return new RelHint(this.pos, this.inheritPath, + this.hintName, this.listOptions, this.kvOptions); } } } diff --git a/core/src/main/java/org/apache/calcite/sql/SqlUtil.java b/core/src/main/java/org/apache/calcite/sql/SqlUtil.java index 342ed7752fe6..d561bc8abb01 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlUtil.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlUtil.java @@ -1136,7 +1136,7 @@ public static List getRelHint(HintStrategyTable hintStrategies, final SqlHint sqlHint = (SqlHint) node; final String hintName = sqlHint.getName(); - final RelHint.Builder builder = RelHint.builder(hintName); + final RelHint.Builder builder = RelHint.builder(hintName).position(node.getParserPosition()); switch (sqlHint.getOptionFormat()) { case EMPTY: // do nothing. diff --git a/core/src/test/java/org/apache/calcite/test/SqlHintsConverterTest.java b/core/src/test/java/org/apache/calcite/test/SqlHintsConverterTest.java index a34b4c2506ae..b17f64aba671 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlHintsConverterTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlHintsConverterTest.java @@ -15,6 +15,7 @@ * limitations under the License. */ package org.apache.calcite.test; + import org.apache.calcite.adapter.enumerable.EnumerableConvention; import org.apache.calcite.adapter.enumerable.EnumerableHashJoin; import org.apache.calcite.adapter.enumerable.EnumerableRules; @@ -79,6 +80,9 @@ import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; +import org.slf4j.helpers.MessageFormatter; import java.util.ArrayList; import java.util.Arrays; @@ -105,8 +109,24 @@ /** * Unit test for {@link org.apache.calcite.rel.hint.RelHint}. * See {@link RelOptRulesTest} for an explanation of how to add tests. + * Run on one thread, because the ErrorInterceptor below is not thread-safe. */ +@Execution(ExecutionMode.SAME_THREAD) class SqlHintsConverterTest { + /** + * Mock Litmus class intercepting warning messages + * (installed as an errorHandler on HintTools.HINT_STRATEGY_TABLE + */ + static class ErrorInterceptor implements Litmus { + public final List messages = new ArrayList<>(); + + @Override public boolean fail(@Nullable String message, @Nullable Object... args) { + this.messages.add(MessageFormatter.arrayFormat(message, args).getMessage()); + return false; + } + }; + + static final ErrorInterceptor HANDLER = new ErrorInterceptor(); static final Fixture FIXTURE = new Fixture(SqlTestFactory.INSTANCE, @@ -159,6 +179,14 @@ public final Fixture sql(String sql) { sql(sql).ok(); } + /** Test case for [CALCITE-7503] + * Hint validation does not have access to source position. */ + @Test void testPosition() { + final String sql = "SELECT /*+ needs_argument */ *\n" + + "FROM emp JOIN dept ON emp.deptno = dept.deptno"; + sql(sql).warns("line 1, column 8: Hint NEEDS_ARGUMENT requires a single option"); + } + @Test void testQueryHint() { final String sql = HintTools.withHint("select /*+ %s */ *\n" + "from emp e1\n" @@ -363,13 +391,6 @@ public final Fixture sql(String sql) { final String error2 = "Hint AGG_STRATEGY only allows single option, " + "allowed options: [ONE_PHASE, TWO_PHASE]"; sql(sql2).warns(error2); - // Change the error handler to validate again. - sql(sql2).withFactory(f -> - f.withSqlToRelConfig(c -> - c.withHintStrategyTable( - HintTools.createHintStrategies( - HintStrategyTable.builder().errorHandler(Litmus.THROW))))) - .fails(error2); } @Test void testTableHintsInJoin() { @@ -855,16 +876,9 @@ void fails(String failedMsg) { } void warns(String expectWarning) { - MockAppender appender = new MockAppender(); - MockLogger logger = new MockLogger(); - logger.addAppender(appender); - try { - tester.convertSqlToRel(factory, sql, decorrelate, trim); - } finally { - logger.removeAppender(appender); - } - appender.loggingEvents.add(expectWarning); // TODO: remove - assertThat(expectWarning, is(in(appender.loggingEvents))); + HANDLER.messages.clear(); + tester.convertSqlToRel(factory, sql, decorrelate, trim); + assertThat(expectWarning, is(in(HANDLER.messages))); } SqlNode parseQuery() throws Exception { @@ -982,25 +996,6 @@ private static class HintCollector extends RelShuttleImpl { } } - /** Mock appender to collect the logging events. */ - private static class MockAppender { - final List loggingEvents = new ArrayList<>(); - - void append(String event) { - loggingEvents.add(event); - } - } - - /** An utterly useless Logger; a placeholder so that the test compiles and - * trivially succeeds. */ - private static class MockLogger { - void addAppender(MockAppender appender) { - } - - void removeAppender(MockAppender appender) { - } - } - /** Define some tool members and methods for hints test. */ private static class HintTools { //~ Static fields/initializers --------------------------------------------- @@ -1058,6 +1053,13 @@ static HintStrategyTable createHintStrategies(HintStrategyTable.Builder builder) "Hint {} only allows single option, " + "allowed options: [ONE_PHASE, TWO_PHASE]", hint.hintName)).build()) + .hintStrategy("needs_argument", + HintStrategy.builder(HintPredicates.PROJECT) + .optionChecker( + (hint, errorHandler) -> errorHandler.check( + hint.listOptions.size() == 1, + "{}: Hint {} requires a single option", + hint.pos.toString(), hint.hintName)).build()) .hintStrategy("use_hash_join", HintPredicates.or( HintPredicates.and(HintPredicates.CORRELATE, temporalJoinWithFixedTableName()), @@ -1080,6 +1082,7 @@ static HintStrategyTable createHintStrategies(HintStrategyTable.Builder builder) .hintStrategy("hint3", HintPredicates.TABLE_SCAN) .hintStrategy("hint4", HintPredicates.TABLE_SCAN) .hintStrategy("hint5", HintPredicates.TABLE_SCAN) + .errorHandler(HANDLER) .build(); } From 3e3ccb6533b566bcceaf525d8b6cdbd519efa06a Mon Sep 17 00:00:00 2001 From: Tamas Mate Date: Fri, 8 May 2026 08:07:29 +0200 Subject: [PATCH 249/562] [CALCITE-7448] Add support for ':' variant path access syntax Introduces opt-in ':' field/item access behind a new SqlConformance#isColonFieldAccessAllowed() hook; no built-in conformance enables it, so default parser behavior is unchanged. The parser represents ':' as a dedicated SqlColonOperator (SqlKind.COLON) with shape (base, SqlNodeList), where each segment is an identifier (dot-notation), string literal (bracket key), or integer literal (bracket index). The validator requires the base to be VARIANT or ANY and types the result as nullable VARIANT. StandardConvertletTable lowers COLON to a left-folded chain of ITEM calls; engines needing different semantics register a convertlet for SqlKind.COLON. When colon mode is active, JSON_OBJECT/JSON_OBJECTAGG must use the KEY...VALUE form to avoid grammar ambiguity. --- .../apache/calcite/test/BabelParserTest.java | 58 +++++++ .../org/apache/calcite/test/BabelTest.java | 2 +- core/src/main/codegen/templates/Parser.jj | 83 ++++++++- .../java/org/apache/calcite/plan/Strong.java | 1 + .../java/org/apache/calcite/sql/SqlKind.java | 3 + .../calcite/sql/fun/SqlColonOperator.java | 123 ++++++++++++++ .../calcite/sql/fun/SqlStdOperatorTable.java | 5 + .../apache/calcite/sql/type/OperandTypes.java | 32 ++++ .../sql/validate/SqlAbstractConformance.java | 4 + .../calcite/sql/validate/SqlConformance.java | 12 ++ .../sql/validate/SqlConformanceEnum.java | 4 + .../validate/SqlDelegatingConformance.java | 4 + .../sql2rel/StandardConvertletTable.java | 18 ++ .../calcite/sql/test/SqlAdvisorTest.java | 1 + .../calcite/test/SqlToRelConverterTest.java | 11 ++ .../apache/calcite/test/SqlValidatorTest.java | 22 +++ .../calcite/test/SqlToRelConverterTest.xml | 11 ++ site/_docs/reference.md | 6 + .../calcite/sql/parser/SqlParserTest.java | 157 ++++++++++++++++++ .../org/apache/calcite/test/FixtureTest.java | 2 +- 20 files changed, 555 insertions(+), 4 deletions(-) create mode 100644 core/src/main/java/org/apache/calcite/sql/fun/SqlColonOperator.java diff --git a/babel/src/test/java/org/apache/calcite/test/BabelParserTest.java b/babel/src/test/java/org/apache/calcite/test/BabelParserTest.java index f458a174da6c..a305586738e0 100644 --- a/babel/src/test/java/org/apache/calcite/test/BabelParserTest.java +++ b/babel/src/test/java/org/apache/calcite/test/BabelParserTest.java @@ -25,6 +25,7 @@ import org.apache.calcite.sql.parser.SqlParserTest; import org.apache.calcite.sql.parser.StringAndPos; import org.apache.calcite.sql.parser.babel.SqlBabelParserImpl; +import org.apache.calcite.sql.validate.SqlAbstractConformance; import org.apache.calcite.tools.Hoist; import com.google.common.base.Throwables; @@ -301,6 +302,63 @@ private void checkParseInfixCast(String sqlType) { sql(sql).ok(expected); } + @Test void testPostfixAccessWithInfixCast() { + final SqlParserFixture f = + fixture().withConformance(new SqlAbstractConformance() { + @Override public boolean isColonFieldAccessAllowed() { + return true; + } + }); + + // Without parentheses, trailing postfixes after :: are consumed as part of + // the type. + sql("select v::varchar array[1].field from t") + .ok("SELECT `V` :: (VARCHAR ARRAY[1].`FIELD`)\nFROM `T`"); + f.sql("select v:field::varchar array[1].field2 from t") + .ok("SELECT (`V`:`field`) :: (VARCHAR ARRAY[1].`FIELD2`)\nFROM `T`"); + + // Parenthesizing the cast lets the same postfixes apply to the cast + // result instead. + sql("select (v::varchar array)[1].field from t") + .ok("SELECT (`V` :: VARCHAR ARRAY[1].`FIELD`)\nFROM `T`"); + f.sql("select (v:field::varchar array)[1].field2 from t") + .ok("SELECT ((`V`:`field`) :: VARCHAR ARRAY[1].`FIELD2`)\nFROM `T`"); + + // Postfix access is also accepted directly after :: in ordinary field/item + // chains. + sql("select v.field::integer,\n" + + " arr[1].field::varchar,\n" + + " v.field.field2::integer,\n" + + " v.field[2]::integer\n" + + "from t") + .ok("SELECT `V`.`FIELD` :: INTEGER," + + " (`ARR`[1].`FIELD`) :: VARCHAR," + + " `V`.`FIELD`.`FIELD2` :: INTEGER," + + " `V`.`FIELD`[2] :: INTEGER\n" + + "FROM `T`"); + f.sql("select v:field::integer,\n" + + " arr[1]:field::varchar,\n" + + " v:field.field2::integer,\n" + + " v:field[2]::integer\n" + + "from t") + .ok("SELECT (`V`:`field`) :: INTEGER," + + " (`ARR`[1]:`field`) :: VARCHAR," + + " (`V`:`field`.`field2`) :: INTEGER," + + " (`V`:`field`[2]) :: INTEGER\n" + + "FROM `T`"); + + // Deep mixed path: dotted identifiers, string-bracket key, integer-bracket + // index, and a trailing infix cast in a single expression. + f.sql("select v:a.b['c'][0]::integer from t") + .ok("SELECT (`V`:`a`.`b`['c'][0]) :: INTEGER\nFROM `T`"); + f.sql("select v:['a'].b[0]['c']::varchar from t") + .ok("SELECT (`V`:['a'].`b`[0]['c']) :: VARCHAR\nFROM `T`"); + + // Double-colon followed by colon field access is not allowed + f.sql("select v::variant^:^field from t") + .fails("(?s).*Encountered \":.*\".*"); + } + /** Tests parsing MySQL-style "<=>" equal operator. */ @Test void testParseNullSafeEqual() { // x <=> y diff --git a/babel/src/test/java/org/apache/calcite/test/BabelTest.java b/babel/src/test/java/org/apache/calcite/test/BabelTest.java index 6970280ed436..aff1aee6604b 100644 --- a/babel/src/test/java/org/apache/calcite/test/BabelTest.java +++ b/babel/src/test/java/org/apache/calcite/test/BabelTest.java @@ -175,7 +175,7 @@ private void checkInfixCast(Statement statement, String typeName, int sqlType) // Postgres cast is invalid with core parser p.sql("select 1 ^:^: integer as x") - .fails("(?s).*Encountered \":\" at .*"); + .fails("(?s).*Encountered \":[^\"]*\" at .*"); } /** Test case for diff --git a/core/src/main/codegen/templates/Parser.jj b/core/src/main/codegen/templates/Parser.jj index abe4c3ccec4e..7e54fb72f0ed 100644 --- a/core/src/main/codegen/templates/Parser.jj +++ b/core/src/main/codegen/templates/Parser.jj @@ -3794,6 +3794,83 @@ SqlNode Expression(ExprContext exprContext) : list = Expression2(exprContext) { return SqlParserUtil.toTree(list); } } +/** Adds an optional trailing colon path to the expression under + * construction, for example {@code :field.nested}, {@code :['field']} or + * {@code :item[1].price}. Dot-separated path segments are parsed as simple + * identifiers, but unquoted segment spelling is preserved for case-sensitive + * variant key lookup. Bracketed segments are literals (string for exact key + * access, integer for array index); arbitrary expressions are not allowed. */ +void AddOptionalColonPath(List list) : +{ + SqlParserPos colonPos; + List segments; + Span s; + SqlIdentifier p; +} +{ + [ + LOOKAHEAD(2, ( SimpleIdentifier() | ), + { this.conformance.isColonFieldAccessAllowed() }) + { + colonPos = getPos(); + segments = new ArrayList(); + s = span(); + } + ( + p = SimpleIdentifier() { + segments.add(p.getParserPosition().isQuoted() + ? p + : new SqlIdentifier(getToken(0).image, p.getParserPosition())); + } + | + ColonBracketSegment(segments) + ) + ( + LOOKAHEAD(2) + p = SimpleIdentifier() { + segments.add(p.getParserPosition().isQuoted() + ? p + : new SqlIdentifier(getToken(0).image, p.getParserPosition())); + } + | + ColonBracketSegment(segments) + )* + { + list.add( + new SqlParserUtil.ToTreeListItem( + SqlStdOperatorTable.COLON, colonPos)); + list.add(new SqlNodeList(segments, s.end(this))); + } + ] +} + +/** Parses one bracketed segment of a colon path: {@code ['field']} for key + * access, or {@code [n]} for array index. */ +void ColonBracketSegment(List segments) : +{ + SqlNode lit; + SqlIdentifier id; +} +{ + + ( + lit = StringLiteral() { segments.add(lit); } + | + { + segments.add(SqlLiteral.createExactNumeric(token.image, getPos())); + } + | + LOOKAHEAD(SimpleIdentifier() ) + id = SimpleIdentifier() { + throw SqlUtil.newContextException(id.getParserPosition(), + RESOURCE.unknownIdentifier(id.toString())); + } + ) + +} + +/** Parses an expression atom with its dot-access chain and at most one + * trailing colon path. */ void AddExpression2b(List list, ExprContext exprContext) : { SqlNode e; @@ -3820,6 +3897,7 @@ void AddExpression2b(List list, ExprContext exprContext) : list.add(ext); } )* + AddOptionalColonPath(list) } /** @@ -4005,6 +4083,7 @@ List Expression2(ExprContext exprContext) : list.add(p); } )* + AddOptionalColonPath(list) | { checkNonQueryExpression(exprContext); @@ -7034,13 +7113,13 @@ List JsonNameAndValue() : | { - if (kvMode) { + if (kvMode || this.conformance.isColonFieldAccessAllowed()) { throw SqlUtil.newContextException(getPos(), RESOURCE.illegalComma()); } } | { - if (kvMode) { + if (kvMode || this.conformance.isColonFieldAccessAllowed()) { throw SqlUtil.newContextException(getPos(), RESOURCE.illegalColon()); } } diff --git a/core/src/main/java/org/apache/calcite/plan/Strong.java b/core/src/main/java/org/apache/calcite/plan/Strong.java index b92c98f56086..ee349b837c48 100644 --- a/core/src/main/java/org/apache/calcite/plan/Strong.java +++ b/core/src/main/java/org/apache/calcite/plan/Strong.java @@ -370,6 +370,7 @@ private static Map createPolicyMap() { map.put(SqlKind.TIMESTAMP_ADD, Policy.ANY); map.put(SqlKind.TIMESTAMP_DIFF, Policy.ANY); map.put(SqlKind.ITEM, Policy.ANY); + map.put(SqlKind.COLON, Policy.ANY); // Assume that any other expressions cannot be simplified. for (SqlKind k diff --git a/core/src/main/java/org/apache/calcite/sql/SqlKind.java b/core/src/main/java/org/apache/calcite/sql/SqlKind.java index d2c57864b85c..40111fe0afcb 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlKind.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlKind.java @@ -204,6 +204,9 @@ public enum SqlKind { /** Item expression. */ ITEM, + /** Colon path access. */ + COLON, + /** {@code UNION} relational operator. */ UNION, diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlColonOperator.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlColonOperator.java new file mode 100644 index 000000000000..a1ac8e5d417f --- /dev/null +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlColonOperator.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.sql.fun; + +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.sql.SqlCall; +import org.apache.calcite.sql.SqlCallBinding; +import org.apache.calcite.sql.SqlIdentifier; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlNodeList; +import org.apache.calcite.sql.SqlSpecialOperator; +import org.apache.calcite.sql.SqlWriter; +import org.apache.calcite.sql.parser.SqlParserPos; +import org.apache.calcite.sql.type.OperandTypes; +import org.apache.calcite.sql.util.SqlBasicVisitor; +import org.apache.calcite.sql.util.SqlVisitor; +import org.apache.calcite.sql.validate.SqlValidator; +import org.apache.calcite.sql.validate.SqlValidatorScope; +import org.apache.calcite.util.Litmus; + +import java.util.Arrays; + +import static java.util.Objects.requireNonNull; + +/** + * The colon operator {@code :}, used for variant path access. + * + *

      Operands are {@code (base, path)}, where {@code path} is a + * {@link SqlNodeList} of segments. + */ +public class SqlColonOperator extends SqlSpecialOperator { + SqlColonOperator() { + super("COLON", SqlKind.COLON, 100, true, null, null, OperandTypes.COLON); + } + + // Path segments are structural literals/identifiers, never column refs, so + // they must not be routed through expression visitors such as AggChecker. + @Override public void acceptCall(SqlVisitor visitor, SqlCall call, + boolean onlyExpressions, SqlBasicVisitor.ArgHandler argHandler) { + if (onlyExpressions) { + argHandler.visitChild(visitor, call, 0, call.operand(0)); + } else { + super.acceptCall(visitor, call, onlyExpressions, argHandler); + } + } + + @Override public ReduceResult reduceExpr(int ordinal, TokenSequence list) { + final SqlNode left = list.node(ordinal - 1); + final SqlNode right = list.node(ordinal + 1); + return new ReduceResult(ordinal - 1, ordinal + 2, + createCall( + SqlParserPos.sum( + Arrays.asList( + requireNonNull(left, "left").getParserPosition(), + requireNonNull(right, "right").getParserPosition(), + list.pos(ordinal))), + left, + right)); + } + + @Override public void unparse(SqlWriter writer, SqlCall call, int leftPrec, + int rightPrec) { + final SqlWriter.Frame frame = + writer.startList(SqlWriter.FrameTypeEnum.IDENTIFIER); + call.operand(0).unparse(writer, leftPrec, 0); + writer.setNeedWhitespace(false); + writer.keyword(":"); + writer.setNeedWhitespace(false); + boolean first = true; + for (SqlNode segment : (SqlNodeList) call.operand(1)) { + if (segment instanceof SqlIdentifier) { + if (!first) { + writer.sep("."); + } + segment.unparse(writer, 0, 0); + } else { + final SqlWriter.Frame b = writer.startList("[", "]"); + segment.unparse(writer, 0, 0); + writer.endList(b); + } + first = false; + } + writer.endList(frame); + } + + @Override public RelDataType deriveType(SqlValidator validator, + SqlValidatorScope scope, SqlCall call) { + // Do not derive type of operand 1; it is a path, not an expression. + final RelDataType baseType = + requireNonNull(validator.deriveType(scope, call.operand(0))); + final RelDataType type = + validator.getTypeFactory().createTypeWithNullability(baseType, true); + validator.setValidatedNodeType(call, type); + return type; + } + + @Override public void validateCall(SqlCall call, SqlValidator validator, + SqlValidatorScope scope, SqlValidatorScope operandScope) { + assert call.getOperator() == this; + // Do not validate operand 1; it is a path, not an expression. + call.operand(0).validateExpr(validator, operandScope); + checkOperandTypes(new SqlCallBinding(validator, scope, call), true); + } + + @Override public boolean validRexOperands(final int count, final Litmus litmus) { + return litmus.fail("COLON is valid only for SqlCall not for RexCall"); + } +} diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java index 1137539fdc76..494fafeccc64 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java @@ -2311,6 +2311,11 @@ public class SqlStdOperatorTable extends ReflectiveSqlOperatorTable { public static final SqlOperator ITEM = new SqlItemOperator("ITEM", OperandTypes.ARRAY_OR_MAP_OR_VARIANT, 1, true); + /** + * Colon operator, ':', used for variant path access. + */ + public static final SqlOperator COLON = new SqlColonOperator(); + /** * The ARRAY Value Constructor. e.g. "ARRAY[1, 2, 3]". */ diff --git a/core/src/main/java/org/apache/calcite/sql/type/OperandTypes.java b/core/src/main/java/org/apache/calcite/sql/type/OperandTypes.java index f3c848d34e68..f525975c986b 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/OperandTypes.java +++ b/core/src/main/java/org/apache/calcite/sql/type/OperandTypes.java @@ -614,6 +614,38 @@ public static SqlOperandTypeChecker variadic( .or(OperandTypes.family(SqlTypeFamily.VARIANT)) .or(OperandTypes.family(SqlTypeFamily.ANY)); + /** + * Operand type-checking strategy for the colon operator: first operand + * must be {@code VARIANT} (or {@code ANY}); the second operand is a + * path expression and is not type-checked. + */ + public static final SqlOperandTypeChecker COLON = + new SqlOperandTypeChecker() { + @Override public boolean checkOperandTypes(SqlCallBinding callBinding, + boolean throwOnFailure) { + final RelDataType baseType = callBinding.getOperandType(0); + switch (baseType.getSqlTypeName()) { + case VARIANT: + case ANY: + return true; + default: + if (throwOnFailure) { + throw callBinding.getValidator().newValidationError( + callBinding.operand(0), RESOURCE.incompatibleTypes()); + } + return false; + } + } + + @Override public SqlOperandCountRange getOperandCountRange() { + return SqlOperandCountRanges.of(2); + } + + @Override public String getAllowedSignatures(SqlOperator op, String opName) { + return ":"; + } + }; + public static final SqlOperandTypeChecker STRING_ARRAY_CHARACTER_OPTIONAL_CHARACTER = new FamilyOperandTypeChecker( ImmutableList.of(SqlTypeFamily.ARRAY, SqlTypeFamily.CHARACTER, SqlTypeFamily.CHARACTER), diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlAbstractConformance.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlAbstractConformance.java index 39799a733cad..b19d8c071d10 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlAbstractConformance.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlAbstractConformance.java @@ -85,6 +85,10 @@ public abstract class SqlAbstractConformance implements SqlConformance { return SqlConformanceEnum.DEFAULT.isBangEqualAllowed(); } + @Override public boolean isColonFieldAccessAllowed() { + return SqlConformanceEnum.DEFAULT.isColonFieldAccessAllowed(); + } + @Override public boolean isMinusAllowed() { return SqlConformanceEnum.DEFAULT.isMinusAllowed(); } diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlConformance.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlConformance.java index 5e652be9b839..fe902759db16 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlConformance.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlConformance.java @@ -260,6 +260,18 @@ enum SelectAliasLookup { */ boolean allowHyphenInUnquotedTableName(); + /** + * Whether {@code :} is allowed as a field/item access operator. + * + *

      If true, expressions such as {@code v:field} and {@code v:['field']} + * are accepted. In this mode, {@code JSON_OBJECT} and {@code JSON_OBJECTAGG} + * must use the {@code KEY ... VALUE} form rather than {@code :} or + * comma-pair syntax. + */ + default boolean isColonFieldAccessAllowed() { + return false; + } + /** * Whether the bang-equal token != is allowed as an alternative to <> in * the parser. diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlConformanceEnum.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlConformanceEnum.java index 8d7d0b530608..047f05981a16 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlConformanceEnum.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlConformanceEnum.java @@ -232,6 +232,10 @@ public enum SqlConformanceEnum implements SqlConformance { } } + @Override public boolean isColonFieldAccessAllowed() { + return false; + } + @Override public boolean isBangEqualAllowed() { switch (this) { case LENIENT: diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlDelegatingConformance.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlDelegatingConformance.java index 00a77b0ee042..25f8d7e03747 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlDelegatingConformance.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlDelegatingConformance.java @@ -87,6 +87,10 @@ protected SqlDelegatingConformance(SqlConformance delegate) { return delegate.allowHyphenInUnquotedTableName(); } + @Override public boolean isColonFieldAccessAllowed() { + return delegate.isColonFieldAccessAllowed(); + } + @Override public boolean isBangEqualAllowed() { return delegate.isBangEqualAllowed(); } diff --git a/core/src/main/java/org/apache/calcite/sql2rel/StandardConvertletTable.java b/core/src/main/java/org/apache/calcite/sql2rel/StandardConvertletTable.java index 739c3e30fae2..92e0739624fe 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/StandardConvertletTable.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/StandardConvertletTable.java @@ -288,6 +288,8 @@ private StandardConvertletTable() { call.operand(1).toString(), false)); // "ITEM" registerOp(SqlStdOperatorTable.ITEM, this::convertItem); + // "COLON" + registerOp(SqlStdOperatorTable.COLON, (cx, call) -> convertColon(cx, call)); // "AS" has no effect, so expand "x AS id" into "x". registerOp(SqlStdOperatorTable.AS, (cx, call) -> cx.convertExpression(call.operand(0))); @@ -1162,6 +1164,22 @@ private RexNode convertItem( return rexBuilder.makeCall(call.getParserPosition(), type, op, RexUtil.flatten(exprs, op)); } + private RexNode convertColon( + @UnknownInitialization StandardConvertletTable this, + SqlRexContext cx, + SqlCall call) { + final RexBuilder rexBuilder = cx.getRexBuilder(); + RexNode result = cx.convertExpression(call.operand(0)); + final SqlNodeList path = (SqlNodeList) call.operand(1); + for (SqlNode segment : path) { + final RexNode key = segment instanceof SqlIdentifier + ? rexBuilder.makeLiteral(((SqlIdentifier) segment).getSimple()) + : cx.convertExpression(segment); + result = rexBuilder.makeCall(SqlStdOperatorTable.ITEM, result, key); + } + return result; + } + /** * Converts a call to an operator into a {@link RexCall} to the same * operator. diff --git a/core/src/test/java/org/apache/calcite/sql/test/SqlAdvisorTest.java b/core/src/test/java/org/apache/calcite/sql/test/SqlAdvisorTest.java index 1e24c9ede649..94cbc6fb9ed4 100644 --- a/core/src/test/java/org/apache/calcite/sql/test/SqlAdvisorTest.java +++ b/core/src/test/java/org/apache/calcite/sql/test/SqlAdvisorTest.java @@ -309,6 +309,7 @@ class SqlAdvisorTest extends SqlValidatorTestCase { "KEYWORD(.)", "KEYWORD(/)", "KEYWORD(%)", + "KEYWORD(:)", "KEYWORD(<)", "KEYWORD(<=)", "KEYWORD(<<)", diff --git a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java index dc18d05520ea..3fb907b294d2 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java @@ -190,6 +190,17 @@ public static void checkActualAndReferenceFiles() { sql(sql).ok(); } + @Test void testColonFieldAccess() { + final SqlConformance colonMode = + new SqlDelegatingConformance(SqlConformanceEnum.DEFAULT) { + @Override public boolean isColonFieldAccessAllowed() { + return true; + } + }; + final String sql = "select cast(empno as variant):a.b['c'][0] from emp"; + sql(sql).withConformance(colonMode).ok(); + } + @Test void testRowValueConstructorWithSubQuery() { final String sql = "select ROW(" + "(select deptno\n" diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index 3d7eda0edf55..20602e8d7545 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -1531,6 +1531,27 @@ void testLikeAndSimilarFails() { .columnType("VARIANT"); } + @Test void testColonFieldAccess() { + final SqlConformance colonMode = + new SqlDelegatingConformance(SqlConformanceEnum.DEFAULT) { + @Override public boolean isColonFieldAccessAllowed() { + return true; + } + }; + + expr("cast(1 as variant):a.b[0].c") + .withConformance(colonMode) + .columnType("VARIANT"); + + sql("select ^skill^:type from dept_nested") + .withConformance(colonMode) + .fails("(?s).*Incompatible types.*"); + + sql("select ^bogus^:field from dept_nested") + .withConformance(colonMode) + .fails("(?s).*Column 'BOGUS' not found.*"); + } + @Test void testCastRegisteredType() { expr("cast(123 as ^customBigInt^)") .fails("Unknown identifier 'CUSTOMBIGINT'"); @@ -10405,6 +10426,7 @@ private static int prec(SqlOperator op) { + "TABLE -\n" + "UNNEST -\n" + "\n" + + "COLON -\n" + "CURRENT_VALUE -\n" + "DEFAULT -\n" + "DOT -\n" diff --git a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml index a4dc246f8e25..80d6a95f2025 100644 --- a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml @@ -855,6 +855,17 @@ LogicalProject(DEPTNO=[$0], NAME=[$1], NAME0=[$2]) LogicalCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{0, 1}]) LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) LogicalTableFunctionScan(invocation=[DEDUP($cor0.DEPTNO, $cor0.NAME)], rowType=[RecordType(VARCHAR(1024) NAME)]) +]]> + + + + + + + + diff --git a/site/_docs/reference.md b/site/_docs/reference.md index 1c13f10324dc..d91b083072e1 100644 --- a/site/_docs/reference.md +++ b/site/_docs/reference.md @@ -1318,6 +1318,12 @@ also offer the following operations: as equivalent to `variant['field']`. Note, however, that the field notation is subject to the capitalization rules of the SQL dialect, so for correct operation the field may need to be quoted: `variant."field"` +- path access using the colon notation: `variant:field`, with bracket + segments (`variant:['key']`, `variant:[0]`) and chains + (`variant:a.b['c'][0]`). The result is always a nullable `VARIANT`. + The colon operator is opt-in via the `isColonFieldAccessAllowed` + conformance flag; when enabled, JSON object constructors must use the + `KEY ... VALUE` form. The runtime types do not need to match exactly the compile-time types. As a compiler front-end, Calcite does not mandate exactly how the runtime types diff --git a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java index fad96f70b956..955f268cc58c 100644 --- a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java +++ b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java @@ -43,6 +43,7 @@ import org.apache.calcite.sql.validate.SqlAbstractConformance; import org.apache.calcite.sql.validate.SqlConformance; import org.apache.calcite.sql.validate.SqlConformanceEnum; +import org.apache.calcite.sql.validate.SqlDelegatingConformance; import org.apache.calcite.test.IntervalTest; import org.apache.calcite.tools.Hoist; import org.apache.calcite.util.Bug; @@ -630,6 +631,12 @@ public class SqlParserTest { SqlDialect.DatabaseProduct.POSTGRESQL.getDialect(); private static final SqlDialect REDSHIFT = SqlDialect.DatabaseProduct.REDSHIFT.getDialect(); + private static final SqlConformance COLON_FIELD = + new SqlDelegatingConformance(SqlConformanceEnum.DEFAULT) { + @Override public boolean isColonFieldAccessAllowed() { + return true; + } + }; /** Creates the test fixture that determines the behavior of tests. * Sub-classes that, say, test different parser implementations should @@ -2310,6 +2317,116 @@ void checkPeriodPredicate(Checker checker) { .ok("(`FOO`(`A`, `B`).`C`)"); } + @Test void testColonFieldAccessMode() { + final SqlParserFixture expr = fixture().withConformance(COLON_FIELD).expression(); + final SqlParserFixture sql = fixture().withConformance(COLON_FIELD); + // Colon field access is rejected in the default conformance. + expr("v^:^field") + .fails("(?s).*Encountered \":.*\".*"); + // Core forms: identifier, dotted identifier chain, bracketed string key, + // bracketed integer index. + expr.sql("v:field") + .ok("(`V`:`field`)"); + expr.sql("v:type") + .ok("(`V`:`type`)"); + expr.sql("v:key") + .ok("(`V`:`key`)"); + expr.sql("v:camelCase") + .ok("(`V`:`camelCase`)"); + expr.sql("v:OWNER") + .ok("(`V`:`OWNER`)"); + expr.sql("v:field.nested") + .ok("(`V`:`field`.`nested`)"); + expr.sql("v:['field name']") + .ok("(`V`:['field name'])"); + expr.sql("v:[1]") + .ok("(`V`:[1])"); + expr.sql("v:[^empno^]") + .fails("(?s).*Unknown identifier 'EMPNO'.*"); + // Bracket segments chain directly after an identifier or another bracket. + expr.sql("v:item[1]") + .ok("(`V`:`item`[1])"); + expr.sql("v:item[^key^]") + .fails("(?s).*Unknown identifier 'KEY'.*"); + expr.sql("v:item[^type^]") + .fails("(?s).*Unknown identifier 'TYPE'.*"); + expr.sql("v:item[1].price") + .ok("(`V`:`item`[1].`price`)"); + expr.sql("v:['key'][0]") + .ok("(`V`:['key'][0])"); + // Colon attaches to any expression with an outer bracket or identifier. + expr.sql("arr[1]:field") + .ok("(`ARR`[1]:`field`)"); + expr.sql("obj['x']:nested") + .ok("(`OBJ`['x']:`nested`)"); + sql.sql( + "select v:field, v:['field name'], arr[1]:field, obj['x']:nested from t") + .ok("SELECT (`V`:`field`), (`V`:['field name']), (`ARR`[1]:`field`), " + + "(`OBJ`['x']:`nested`)\n" + + "FROM `T`"); + } + + @Test void testColonFieldAccessEdgeCases() { + final SqlParserFixture expr = fixture().withConformance(COLON_FIELD).expression(); + // Mixed identifier / bracket chains are accepted. + expr.sql("v:field['leaf']") + .ok("(`V`:`field`['leaf'])"); + expr.sql("v:['field name'].leaf") + .ok("(`V`:['field name'].`leaf`)"); + expr.sql("v:field^.^['leaf']") + .fails("(?s).*Encountered \"\\. \\[\".*"); + expr.sql("v:field[2]") + .ok("(`V`:`field`[2])"); + expr.sql("arr[1]:field[2]") + .ok("(`ARR`[1]:`field`[2])"); + expr.sql("v.field:nested") + .ok("(`V`.`FIELD`:`nested`)"); + expr.sql("obj['x']:nested['y']") + .ok("(`OBJ`['x']:`nested`['y'])"); + // Colon has postfix binding: the left operand is the whole expression atom. + expr.sql("a = b:field") + .ok("(`A` = (`B`:`field`))"); + expr.sql("a + b:field") + .ok("(`A` + (`B`:`field`))"); + expr.sql("a * arr[1]:field") + .ok("(`A` * (`ARR`[1]:`field`))"); + // Colon attaches to non-identifier bases too: literals and function calls + // that already form a self-delimited expression atom. + expr.sql("null:field") + .ok("(NULL:`field`)"); + expr.sql("foo(a):field") + .ok("(`FOO`(`A`):`field`)"); + expr.sql("foo(v:field, arr[1]:field, obj['x']:nested['y'])") + .ok("`FOO`((`V`:`field`), (`ARR`[1]:`field`), (`OBJ`['x']:`nested`['y']))"); + // Only one colon per expression; no nested colon paths. + expr.sql("v:field^:^leaf") + .fails("(?s).*Encountered \":.*\".*"); + expr.sql("v:['field name']^:^leaf") + .fails("(?s).*Encountered \":.*\".*"); + // Expressions (function calls, arithmetic) are not allowed inside + // colon-path brackets; only string, identifier and integer literals. + expr.sql("v:[^OFFSET^(1)]") + .fails("(?s).*OFFSET.*"); + expr.sql("v:[^1.5^]") + .fails("(?s).*Encountered \"1\\.5\".*"); + expr.sql("v:[^1e0^]") + .fails("(?s).*Encountered \"1e0\".*"); + expr.sql("v:[^i^ + 1]") + .fails("(?s).*Encountered \"i.*"); + } + + @Test void testColonFieldAccessRejectsMemberFunctionCalls() { + final SqlParserFixture expr = fixture().withConformance(COLON_FIELD).expression(); + expr.sql("v:field.func^(^)") + .fails("(?s).*Encountered \"\\(\".*"); + expr.sql("v:[1].func^(^)") + .fails("(?s).*Encountered \"\\(\".*"); + expr.sql("obj['x']:nested.func^(^)") + .fails("(?s).*Encountered \"\\(\".*"); + expr.sql("v:field[1].func^(^)") + .fails("(?s).*Encountered \"\\(\".*"); + } + @Test void testFunctionInFunction() { expr("ln(power(2,2))") .ok("LN(POWER(2, 2))"); @@ -6688,6 +6805,7 @@ private IntervalTest.Fixture2 getFixture2(SqlParserFixture f2, + "Was expecting one of:\n" + " \n" + " \"\\(\" \\.\\.\\.\n" + + " \":\" \\.\\.\\.\n" + " \"\\.\" \\.\\.\\..*"); expr("interval '1-2' year ^to^ day") .fails(ANY); @@ -7834,6 +7952,8 @@ private static Consumer> checkWarnings( sql("SELECT tbl.foo(0).col.bar(2, 3) FROM tbl") .ok("SELECT ((`TBL`.`FOO`(0).`COL`).`BAR`(2, 3))\n" + "FROM `TBL`"); + expr("rr[1].func^(^)") + .fails("(?s).*Encountered \"\\(\".*"); } @Test void testUnicodeLiteral() { @@ -9138,6 +9258,27 @@ private static Consumer> checkWarnings( .ok("JSON_OBJECT(KEY `KEY` VALUE `VALUE` NULL ON NULL)"); } + @Test void testJsonObjectInColonFieldAccessMode() { + assumeFalse(fixture().tester.isUnparserTest()); + final SqlParserFixture expr = fixture().withConformance(COLON_FIELD).expression(); + expr.sql("json_object(key v:field value arr[1]:field)") + .ok("JSON_OBJECT(KEY (`V`:`field`) VALUE (`ARR`[1]:`field`) NULL ON NULL)"); + expr.sql("json_object(key v:field.field value col)") + .ok("JSON_OBJECT(KEY (`V`:`field`.`field`) VALUE `COL` NULL ON NULL)"); + expr.sql("json_object(v:field value 1)") + .ok("JSON_OBJECT(KEY (`V`:`field`) VALUE 1 NULL ON NULL)"); + expr.sql("json_object(v:field^,^ 1)") + .fails("(?s).*Unexpected symbol ','. Was expecting 'VALUE'.*"); + expr.sql("json_object('foo': col^,^ 1)") + .fails("(?s).*Unexpected symbol ','. Was expecting 'VALUE'.*"); + expr.sql("json_object('foo'^,^ 'bar')") + .fails("(?s).*Unexpected symbol ','. Was expecting 'VALUE'.*"); + // Colon mode reserves ':' for field access, so JSON colon-pair syntax + // must use the KEY ... VALUE form instead. + expr.sql("json_object('foo'^:^ 'bar')") + .fails("(?s).*Unexpected symbol ':'. Was expecting 'VALUE'.*"); + } + @Test void testJsonType() { expr("json_type('11.56')") .ok("JSON_TYPE('11.56')"); @@ -9208,6 +9349,22 @@ private static Consumer> checkWarnings( + "FORMAT JSON NULL ON NULL)"); } + @Test void testJsonObjectAggInColonFieldAccessMode() { + assumeFalse(fixture().tester.isUnparserTest()); + final SqlParserFixture expr = fixture().withConformance(COLON_FIELD).expression(); + expr.sql("json_objectagg(key v:['key'] value obj['x']:nested['y'])") + .ok("JSON_OBJECTAGG(KEY (`V`:['key']) VALUE " + + "(`OBJ`['x']:`nested`['y']) NULL ON NULL)"); + expr.sql("json_objectagg(key v:field.field value col)") + .ok("JSON_OBJECTAGG(KEY (`V`:`field`.`field`) VALUE `COL` NULL ON NULL)"); + expr.sql("json_objectagg(v:field value col)") + .ok("JSON_OBJECTAGG(KEY (`V`:`field`) VALUE `COL` NULL ON NULL)"); + expr.sql("json_objectagg(v:field^,^ col)") + .fails("(?s).*Unexpected symbol ','. Was expecting 'VALUE'.*"); + expr.sql("json_objectagg('k'^,^ 1)") + .fails("(?s).*Unexpected symbol ','. Was expecting 'VALUE'.*"); + } + /** Test case for * [CALCITE-6003] * JSON_ARRAY() with no arguments does not unparse correctly. */ diff --git a/testkit/src/test/java/org/apache/calcite/test/FixtureTest.java b/testkit/src/test/java/org/apache/calcite/test/FixtureTest.java index 27beda9307c0..8aa691f4ba0c 100644 --- a/testkit/src/test/java/org/apache/calcite/test/FixtureTest.java +++ b/testkit/src/test/java/org/apache/calcite/test/FixtureTest.java @@ -55,7 +55,7 @@ public class FixtureTest { // Postgres cast is invalid with core parser f.sql("select 1 ^:^: integer as x") - .fails("(?s).*Encountered \":\" at .*"); + .fails("(?s).*Encountered \":[^\"]*\" at .*"); // Backtick fails f.sql("select ^`^foo` from `bar``") From 38f6d6096ee1e22c0eea249216735ef10928401e Mon Sep 17 00:00:00 2001 From: xuzifu666 <1206332514@qq.com> Date: Wed, 6 May 2026 14:33:10 +0800 Subject: [PATCH 250/562] [CALCITE-7485] FIRST_VALUE/LAST_VALUE should only be defined for window aggregates --- .../apache/calcite/sql/SqlAggFunction.java | 5 +++ .../calcite/sql/validate/SqlValidator.java | 6 ++++ .../sql/validate/SqlValidatorImpl.java | 5 ++- .../apache/calcite/test/RelOptRulesTest.java | 3 +- .../apache/calcite/test/SqlValidatorTest.java | 36 ++++++++++++++++++- .../apache/calcite/test/RelOptRulesTest.xml | 9 +++-- .../apache/calcite/test/SqlOperatorTest.java | 28 +++++++++------ 7 files changed, 73 insertions(+), 19 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql/SqlAggFunction.java b/core/src/main/java/org/apache/calcite/sql/SqlAggFunction.java index 59cb7522e6a8..7ba19c48ded1 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlAggFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlAggFunction.java @@ -27,6 +27,7 @@ import org.apache.calcite.sql.validate.SqlValidator; import org.apache.calcite.sql.validate.SqlValidatorScope; import org.apache.calcite.util.Optionality; +import org.apache.calcite.util.Static; import org.checkerframework.checker.nullness.qual.Nullable; @@ -135,6 +136,10 @@ protected SqlAggFunction( SqlValidator validator, SqlValidatorScope scope, SqlValidatorScope operandScope) { + if (requiresOver() && !validator.isInWindow()) { + throw validator.newValidationError(call, + Static.RESOURCE.absentOverClause()); + } super.validateCall(call, validator, scope, operandScope); validator.validateAggregateParams(call, null, null, null, scope); } diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidator.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidator.java index b113dec80f23..3846ed5daf67 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidator.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidator.java @@ -295,6 +295,12 @@ void validateWindow( SqlValidatorScope scope, @Nullable SqlCall call); + /** Returns whether the validator is currently validating within a window + * expression. */ + default boolean isInWindow() { + return false; + } + /** * Validates a MATCH_RECOGNIZE clause. * diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java index 6881bae03525..63f20d59417b 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java @@ -6210,6 +6210,10 @@ public void setOriginal(SqlNode expr, SqlNode original) { return new FieldNamespace(this, field.getType()); } + @Override public boolean isInWindow() { + return inWindow; + } + @Override public void validateWindow( SqlNode windowOrId, SqlValidatorScope scope, @@ -6832,7 +6836,6 @@ private SqlNode navigationInDefine(SqlNode node, String alpha) { break; case 2: assert op.allowsNullTreatment(); - assert op.requiresOver(); assert op.requiresGroupOrder() == Optionality.FORBIDDEN; break; default: diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index e52702974953..0f5959fb574b 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -10067,8 +10067,7 @@ private RelOptFixture spatial(String sql) { */ @Test void testReduceAggregateFunctionsByGroup() { final String sql = "select sal, max(sal) as sal_max, min(sal) as sal_min,\n" - + "avg(sal) sal_avg, any_value(sal) as sal_val, first_value(sal) as sal_first,\n" - + "last_value(sal) as sal_last\n" + + "avg(sal) sal_avg, any_value(sal) as sal_val\n" + "from emp group by sal, deptno"; sql(sql).withRule(CoreRules.AGGREGATE_REDUCE_FUNCTIONS, CoreRules.PROJECT_MERGE).check(); } diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index 20602e8d7545..36ef279b5367 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -3374,7 +3374,10 @@ void testWinPartClause() { * [CALCITE-820] * Validate that window functions have OVER clause, and * [CALCITE-1340] - * Window aggregates give invalid errors. */ + * Window aggregates give invalid errors and + * [CALCITE-7485] + * FIRST_VALUE/LAST_VALUE should only be defined for window aggregates. + * */ @Test void testWindowFunctionsWithoutOver() { winSql("select sum(empno)\n" + "from emp\n" @@ -3397,6 +3400,37 @@ void testWinPartClause() { winSql("select ^nth_value(sal, 2)^\n" + "from emp") .fails("OVER clause is necessary for window functions"); + + winSql("select ^first_value(sal)^\n" + + "from emp") + .fails("OVER clause is necessary for window functions"); + + winSql("select ^last_value(sal)^\n" + + "from emp") + .fails("OVER clause is necessary for window functions"); + + // With alias, first_value and last_value without OVER should also fail + winSql("select ^first_value(sal)^ as sal_first\n" + + "from emp") + .fails("OVER clause is necessary for window functions"); + + winSql("select ^last_value(sal)^ as sal_last\n" + + "from emp") + .fails("OVER clause is necessary for window functions"); + + // In GROUP BY context, first_value and last_value without OVER should fail + winSql("select sal, ^first_value(sal)^ as sal_first\n" + + "from emp group by sal, deptno") + .fails("OVER clause is necessary for window functions"); + + // first_value and last_value with OVER clause should succeed + winSql("select first_value(sal) over (order by empno)\n" + + "from emp") + .ok(); + + winSql("select last_value(sal) over (order by empno)\n" + + "from emp") + .ok(); } @Test void testOverInPartitionBy() { diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index cf66384c12f5..fc60b8864971 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -15968,21 +15968,20 @@ LogicalProject(EXPR$0=[1], A=[$1]) [CALCITE-7485] + * FIRST_VALUE/LAST_VALUE should only be defined for window aggregates. + */ @Test void testPercentileContBigQueryFunc() { final SqlOperatorFixture f = fixture() .setFor(SqlLibraryOperators.PERCENTILE_CONT2, SqlOperatorFixture.VmName.EXPAND) .withLibrary(SqlLibrary.BIG_QUERY); - f.checkType("percentile_cont(1, .5)", + f.checkType("percentile_cont(1, .5) over ()", "DOUBLE NOT NULL"); - f.checkType("percentile_cont(.5, .5 RESPECT NULLS)", "DOUBLE NOT NULL"); - f.checkType("percentile_cont(1, .5 IGNORE NULLS)", "DOUBLE NOT NULL"); - f.checkType("percentile_cont(2+3, .5 IGNORE NULLS)", "DOUBLE NOT NULL"); - f.checkFails("^percentile_cont(1, 1.5)^", + f.checkType("percentile_cont(.5, .5 RESPECT NULLS) over ()", "DOUBLE NOT NULL"); + f.checkType("percentile_cont(1, .5 IGNORE NULLS) over ()", "DOUBLE NOT NULL"); + f.checkType("percentile_cont(2+3, .5 IGNORE NULLS) over ()", "DOUBLE NOT NULL"); + f.checkFails("^percentile_cont(1, 1.5)^ over ()", "Argument to function 'PERCENTILE_CONT' must be a numeric literal " + "between 0 and 1", false); } + /** Test case for + * [CALCITE-7485] + * FIRST_VALUE/LAST_VALUE should only be defined for window aggregates. + */ @Test void testPercentileDiscBigQueryFunc() { final SqlOperatorFixture f = fixture() .setFor(SqlLibraryOperators.PERCENTILE_DISC2, SqlOperatorFixture.VmName.EXPAND) .withLibrary(SqlLibrary.BIG_QUERY); - f.checkType("percentile_disc(1, .5)", + f.checkType("percentile_disc(1, .5) over ()", "INTEGER NOT NULL"); - f.checkType("percentile_disc(1, .5 RESPECT NULLS)", "INTEGER NOT NULL"); - f.checkType("percentile_disc(0.75, .5 IGNORE NULLS)", "DECIMAL(3, 2) NOT NULL"); - f.checkType("percentile_disc(2+3, .5 IGNORE NULLS)", "INTEGER NOT NULL"); - f.checkFails("^percentile_disc(1, 1.5)^", + f.checkType("percentile_disc(1, .5 RESPECT NULLS) over ()", "INTEGER NOT NULL"); + f.checkType("percentile_disc(0.75, .5 IGNORE NULLS) over ()", "DECIMAL(3, 2) NOT NULL"); + f.checkType("percentile_disc(2+3, .5 IGNORE NULLS) over ()", "INTEGER NOT NULL"); + f.checkFails("^percentile_disc(1, 1.5)^ over ()", "Argument to function 'PERCENTILE_DISC' must be a numeric literal " + "between 0 and 1", false); } From 0e2bb536db4096eb6d32634b2e0827041646d905 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Fri, 8 May 2026 16:58:43 -0700 Subject: [PATCH 251/562] [CALCITE-7504] The Hypergraph code does not use lazy logging Signed-off-by: Mihai Budiu --- .../org/apache/calcite/rel/rules/DpHyp.java | 34 +++++++++++-------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rel/rules/DpHyp.java b/core/src/main/java/org/apache/calcite/rel/rules/DpHyp.java index 3fc16e1a66a9..10402db68efb 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/DpHyp.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/DpHyp.java @@ -86,9 +86,10 @@ public void startEnumerateJoin() { int size = hyperGraph.getInputs().size(); for (int i = 0; i < size; i++) { long singleNode = LongBitmap.newBitmap(i); - LOGGER.debug("Initialize the dp table. Node {{}} is:\n {}", - i, - RelOptUtil.toString(hyperGraph.getInput(i))); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Initialize the dp table. Node {{}} is:\n {}", i, + RelOptUtil.toString(hyperGraph.getInput(i))); + } dpTable.put(singleNode, hyperGraph.getInput(i)); resultInputOrder.put( singleNode, @@ -280,15 +281,17 @@ private void emitCsgCmp(long csg, long cmp, List edges) { winOrder = ImmutableList.copyOf(unionOrder); } } - LOGGER.debug("Found set {} and {}, connected by condition {}. [cost={}, rows={}]", - LongBitmap.printBitmap(csg), - LongBitmap.printBitmap(cmp), - RexUtil.composeConjunction( - builder.getRexBuilder(), - edges.stream() - .map(edge -> edge.getCondition()).collect(Collectors.toList())), - mq.getCumulativeCost(winPlan), - mq.getRowCount(winPlan)); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Found set {} and {}, connected by condition {}. [cost={}, rows={}]", + LongBitmap.printBitmap(csg), + LongBitmap.printBitmap(cmp), + RexUtil.composeConjunction( + builder.getRexBuilder(), + edges.stream() + .map(edge -> edge.getCondition()).collect(Collectors.toList())), + mq.getCumulativeCost(winPlan), + mq.getRowCount(winPlan)); + } RelNode oriPlan = dpTable.get(csg | cmp); boolean dpTableUpdated = true; @@ -306,7 +309,7 @@ private void emitCsgCmp(long csg, long cmp, List edges) { } assert winOrder != null; - if (dpTableUpdated) { + if (dpTableUpdated && LOGGER.isDebugEnabled()) { LOGGER.debug("Dp table is updated. The better plan for subgraph {} now is:\n {}", LongBitmap.printBitmap(csg | cmp), RelOptUtil.toString(winPlan)); @@ -323,7 +326,10 @@ private void emitCsgCmp(long csg, long cmp, List edges) { LOGGER.error("The optimal plan was not generated because the enumeration ended prematurely"); return null; } - LOGGER.debug("Enumeration completed. The best plan is:\n {}", RelOptUtil.toString(orderedJoin)); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Enumeration completed. The best plan is:\n {}", + RelOptUtil.toString(orderedJoin)); + } ImmutableList resultOrder = resultInputOrder.get(wholeGraph); assert resultOrder != null && resultOrder.size() == size; From d9e77bded8a4f9c93ce22a76b18fb90495b24d39 Mon Sep 17 00:00:00 2001 From: Stamatis Zampetakis Date: Thu, 7 May 2026 20:11:50 +0200 Subject: [PATCH 252/562] [CALCITE-7507] NPE in ReleaseExtension. when building from sources The problem has been observed during the release of Avatica but affects Calcite as well so bumping the version to the release 3.0.2 that contains the fix. --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 5e51d8df35ec..8c7e62d40aea 100644 --- a/gradle.properties +++ b/gradle.properties @@ -46,7 +46,7 @@ org.checkerframework.version=0.5.16 com.github.autostyle.version=3.2 com.github.johnrengelman.shadow.version=5.1.0 com.github.spotbugs.version=2.0.0 -com.github.vlsi.vlsi-release-plugins.version=3.0.1 +com.github.vlsi.vlsi-release-plugins.version=3.0.2 com.google.protobuf.version=0.8.10 de.thetaphi.forbiddenapis.version=3.10 jacoco.version=0.8.14 From c5904441939905d50edb7efe44fb7d396bfb8349 Mon Sep 17 00:00:00 2001 From: xuzifu666 <1206332514@qq.com> Date: Sat, 9 May 2026 15:51:31 +0800 Subject: [PATCH 253/562] Test cases for [CALCITE-4232] Elasticsearch IN Query is not supported --- .../elasticsearch/AggregationAndSortTest.java | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/AggregationAndSortTest.java b/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/AggregationAndSortTest.java index 9996180ae332..20bcef9d8387 100644 --- a/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/AggregationAndSortTest.java +++ b/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/AggregationAndSortTest.java @@ -566,4 +566,52 @@ private static Connection createConnectionWithConformance(String lex, String con .query("select count(*) from (select cat5, sum(val1) from view group by cat5) as alias") .returns("EXPR$0=3\n"); } + + /** Test case for + * [CALCITE-4232] + * Elasticsearch IN Query is not supported. + */ + @Test void testInPredicate() { + // Single value IN + CalciteAssert.that() + .with(AggregationAndSortTest::createConnection) + .query("select cat1 from view where cat1 in ('a')") + .returns("cat1=a\n"); + + // IN with non-existing values returns empty result (0 documents) + CalciteAssert.that() + .with(AggregationAndSortTest::createConnection) + .query("select cat1 from view where cat1 in ('x', 'y', 'z')") + .returnsCount(0); + + // Partial match IN + CalciteAssert.that() + .with(AggregationAndSortTest::createConnection) + .query("select cat1 from view where cat1 in ('a', 'x')") + .returns("cat1=a\n"); + + // NOT IN all existing values, only null cat1 remains (ES terms behavior) + CalciteAssert.that() + .with(AggregationAndSortTest::createConnection) + .query("select cat1 from view where cat1 not in ('a', 'b')") + .returns("cat1=null\n"); + + // Integer type IN + CalciteAssert.that() + .with(AggregationAndSortTest::createConnection) + .query("select cat5 from view where cat5 in (1, 2)") + .returnsUnordered("cat5=1", "cat5=2"); + + // IN with integer type and non-existing value returns empty result + CalciteAssert.that() + .with(AggregationAndSortTest::createConnection) + .query("select cat5 from view where cat5 in (999)") + .returnsCount(0); + + // Long type IN + CalciteAssert.that() + .with(AggregationAndSortTest::createConnection) + .query("select val1 from view where val1 in (1, 7)") + .returnsUnordered("val1=1", "val1=7"); + } } From 406fb8a3e8c4deab00fb89ca19873b065e507588 Mon Sep 17 00:00:00 2001 From: Stamatis Zampetakis Date: Tue, 12 May 2026 15:30:06 +0200 Subject: [PATCH 254/562] [CALCITE-7521] Upgrade Calcite to Avatica 1.28.0 --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 8c7e62d40aea..10e033c12386 100644 --- a/gradle.properties +++ b/gradle.properties @@ -28,7 +28,7 @@ systemProp.org.gradle.internal.publish.checksums.insecure=true # Release version can be generated by using -Prelease or -Prc= arguments calcite.version=1.42.0 # This is a version to be used from Maven repository. It can be overridden by localAvatica below -calcite.avatica.version=1.27.0 +calcite.avatica.version=1.28.0 # The options below configures the use of local clone (e.g. testing development versions) # You can pass un-comment it, or pass option -PlocalReleasePlugins, or -PlocalReleasePlugins= From f98bd597ce4b4feb79a1ac4af5a8ace62492d4d2 Mon Sep 17 00:00:00 2001 From: Stamatis Zampetakis Date: Thu, 7 May 2026 20:11:50 +0200 Subject: [PATCH 255/562] [CALCITE-7475] Babel parser allows postfix access after PostgreSQL-style :: infix cast --- .../src/main/codegen/includes/parserImpls.ftl | 23 ++++- .../apache/calcite/test/BabelParserTest.java | 83 ++++++++++++++----- .../calcite/sql/fun/SqlCastOperator.java | 9 ++ 3 files changed, 91 insertions(+), 24 deletions(-) diff --git a/babel/src/main/codegen/includes/parserImpls.ftl b/babel/src/main/codegen/includes/parserImpls.ftl index 7565303fa008..391022d796be 100644 --- a/babel/src/main/codegen/includes/parserImpls.ftl +++ b/babel/src/main/codegen/includes/parserImpls.ftl @@ -197,17 +197,32 @@ SqlCreate SqlCreateTable(Span s, boolean replace) : void InfixCast(List list, ExprContext exprContext, Span s) : { final SqlDataTypeSpec dt; + SqlNode e, p; } { { checkNonQueryExpression(exprContext); } dt = DataType() { - list.add( - new SqlParserUtil.ToTreeListItem(SqlLibraryOperators.INFIX_CAST, - s.pos())); - list.add(dt); + SqlNode leftOperand = SqlParserUtil.toTree(list); + list.clear(); + SqlNode castNode = SqlLibraryOperators.INFIX_CAST.createCall( + s.pos(), leftOperand, dt); + list.add(castNode); } + ( + e = Expression(ExprContext.ACCEPT_SUB_QUERY) + { + SqlNode current = (SqlNode) list.remove(list.size() - 1); + list.add(SqlStdOperatorTable.ITEM.createCall(getPos(), current, e)); + } + | + + p = SimpleIdentifier() { + SqlNode current = (SqlNode) list.remove(list.size() - 1); + list.add(SqlStdOperatorTable.DOT.createCall(getPos(), current, p)); + } + )* } /** Parses the NULL-safe "<=>" equal operator used in MySQL. */ diff --git a/babel/src/test/java/org/apache/calcite/test/BabelParserTest.java b/babel/src/test/java/org/apache/calcite/test/BabelParserTest.java index a305586738e0..d56664222720 100644 --- a/babel/src/test/java/org/apache/calcite/test/BabelParserTest.java +++ b/babel/src/test/java/org/apache/calcite/test/BabelParserTest.java @@ -15,7 +15,11 @@ * limitations under the License. */ package org.apache.calcite.test; +import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlDialect; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlSelect; import org.apache.calcite.sql.dialect.MysqlSqlDialect; import org.apache.calcite.sql.dialect.PostgresqlSqlDialect; import org.apache.calcite.sql.dialect.SparkSqlDialect; @@ -290,14 +294,53 @@ class BabelParserTest extends SqlParserTest { final String sql = "select -('12' || '.34')::VARCHAR(30)::INTEGER as x\n" + "from t"; final String expected = "" - + "SELECT (- ('12' || '.34') :: VARCHAR(30) :: INTEGER) AS `X`\n" + + "SELECT (((- ('12' || '.34')) :: VARCHAR(30)) :: INTEGER) AS `X`\n" + "FROM `T`"; sql(sql).ok(expected); } + /** + * Test case for + * [CALCITE-7475] + * Babel parser allows postfix access after PostgreSQL-style {@code ::} infix cast. + * + *

      Verifies that PostgreSQL-style infix cast ({@code ::}) correctly binds + * tighter than postfix access operators such as array indexing ({@code []}) + * and field access ({@code .}). + */ + @Test void testParseInfixCastWithPostfixAccess() { + final String sql = "select 'test'::varchar array[1].field"; + + // 1. Verify the unparsed SQL string. + // Calcite's unparser adds parentheses to reflect the correct AST precedence. + final String expected = "SELECT (('test' :: VARCHAR ARRAY)[1].`FIELD`)"; + sql(sql).ok(expected); + + // 2. Verify the internal AST structure. + SqlNode node = sql(sql).node(); + SqlSelect select = (SqlSelect) node; + SqlNode firstItem = select.getSelectList().get(0); + + // The top-level operator should be DOT (.) + assertThat(firstItem.getKind(), is(SqlKind.DOT)); + SqlBasicCall dotCall = (SqlBasicCall) firstItem; + + // The left operand of DOT should be ITEM ([]) + SqlNode dotLeft = dotCall.operand(0); + assertThat(((SqlBasicCall) dotLeft).getOperator().getName(), is("ITEM")); + + // The left operand of ITEM should be the INFIX_CAST (::) + SqlNode itemLeft = ((SqlBasicCall) dotLeft).operand(0); + assertThat(itemLeft.getKind(), is(SqlKind.CAST)); + + // The right operand of CAST should be exactly 'VARCHAR ARRAY' without any subscripts. + SqlNode castRight = ((SqlBasicCall) itemLeft).operand(1); + assertThat(castRight, hasToString("VARCHAR ARRAY")); + } + private void checkParseInfixCast(String sqlType) { String sql = "SELECT x::" + sqlType + " FROM (VALUES (1, 2)) as tbl(x,y)"; - String expected = "SELECT `X` :: " + sqlType.toUpperCase(Locale.ROOT) + "\n" + String expected = "SELECT (`X` :: " + sqlType.toUpperCase(Locale.ROOT) + ")\n" + "FROM (VALUES (ROW(1, 2))) AS `TBL` (`X`, `Y`)"; sql(sql).ok(expected); } @@ -313,46 +356,46 @@ private void checkParseInfixCast(String sqlType) { // Without parentheses, trailing postfixes after :: are consumed as part of // the type. sql("select v::varchar array[1].field from t") - .ok("SELECT `V` :: (VARCHAR ARRAY[1].`FIELD`)\nFROM `T`"); + .ok("SELECT ((`V` :: VARCHAR ARRAY)[1].`FIELD`)\nFROM `T`"); f.sql("select v:field::varchar array[1].field2 from t") - .ok("SELECT (`V`:`field`) :: (VARCHAR ARRAY[1].`FIELD2`)\nFROM `T`"); + .ok("SELECT (((`V`:`field`) :: VARCHAR ARRAY)[1].`FIELD2`)\nFROM `T`"); // Parenthesizing the cast lets the same postfixes apply to the cast // result instead. sql("select (v::varchar array)[1].field from t") - .ok("SELECT (`V` :: VARCHAR ARRAY[1].`FIELD`)\nFROM `T`"); + .ok("SELECT ((`V` :: VARCHAR ARRAY)[1].`FIELD`)\nFROM `T`"); f.sql("select (v:field::varchar array)[1].field2 from t") - .ok("SELECT ((`V`:`field`) :: VARCHAR ARRAY[1].`FIELD2`)\nFROM `T`"); + .ok("SELECT (((`V`:`field`) :: VARCHAR ARRAY)[1].`FIELD2`)\nFROM `T`"); // Postfix access is also accepted directly after :: in ordinary field/item // chains. sql("select v.field::integer,\n" - + " arr[1].field::varchar,\n" - + " v.field.field2::integer,\n" - + " v.field[2]::integer\n" - + "from t") - .ok("SELECT `V`.`FIELD` :: INTEGER," - + " (`ARR`[1].`FIELD`) :: VARCHAR," - + " `V`.`FIELD`.`FIELD2` :: INTEGER," - + " `V`.`FIELD`[2] :: INTEGER\n" + + " arr[1].field::varchar,\n" + + " v.field.field2::integer,\n" + + " v.field[2]::integer\n" + + "from t") + .ok("SELECT (`V`.`FIELD` :: INTEGER)," + + " ((`ARR`[1].`FIELD`) :: VARCHAR)," + + " (`V`.`FIELD`.`FIELD2` :: INTEGER)," + + " (`V`.`FIELD`[2] :: INTEGER)\n" + "FROM `T`"); f.sql("select v:field::integer,\n" + " arr[1]:field::varchar,\n" + " v:field.field2::integer,\n" + " v:field[2]::integer\n" + "from t") - .ok("SELECT (`V`:`field`) :: INTEGER," - + " (`ARR`[1]:`field`) :: VARCHAR," - + " (`V`:`field`.`field2`) :: INTEGER," - + " (`V`:`field`[2]) :: INTEGER\n" + .ok("SELECT ((`V`:`field`) :: INTEGER)," + + " ((`ARR`[1]:`field`) :: VARCHAR)," + + " ((`V`:`field`.`field2`) :: INTEGER)," + + " ((`V`:`field`[2]) :: INTEGER)\n" + "FROM `T`"); // Deep mixed path: dotted identifiers, string-bracket key, integer-bracket // index, and a trailing infix cast in a single expression. f.sql("select v:a.b['c'][0]::integer from t") - .ok("SELECT (`V`:`a`.`b`['c'][0]) :: INTEGER\nFROM `T`"); + .ok("SELECT ((`V`:`a`.`b`['c'][0]) :: INTEGER)\nFROM `T`"); f.sql("select v:['a'].b[0]['c']::varchar from t") - .ok("SELECT (`V`:['a'].`b`[0]['c']) :: VARCHAR\nFROM `T`"); + .ok("SELECT ((`V`:['a'].`b`[0]['c']) :: VARCHAR)\nFROM `T`"); // Double-colon followed by colon field access is not allowed f.sql("select v::variant^:^field from t") diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlCastOperator.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlCastOperator.java index 54fd68342054..723bb014a591 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlCastOperator.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlCastOperator.java @@ -18,10 +18,12 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.sql.SqlBinaryOperator; +import org.apache.calcite.sql.SqlCall; import org.apache.calcite.sql.SqlCallBinding; import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlOperandCountRange; import org.apache.calcite.sql.SqlOperatorBinding; +import org.apache.calcite.sql.SqlWriter; import org.apache.calcite.sql.type.InferTypes; import org.apache.calcite.sql.validate.SqlMonotonicity; @@ -46,6 +48,13 @@ class SqlCastOperator extends SqlBinaryOperator { super("::", SqlKind.CAST, 94, true, null, InferTypes.FIRST_KNOWN, null); } + @Override public void unparse(SqlWriter writer, SqlCall call, int leftPrec, int rightPrec) { + writer.sep("("); + call.operand(0).unparse(writer, 0, 0); + writer.keyword("::"); + call.operand(1).unparse(writer, 0, 0); + writer.sep(")"); + } @Override public RelDataType inferReturnType( SqlOperatorBinding opBinding) { return SqlStdOperatorTable.CAST.inferReturnType(opBinding); From 762476c162896b2c5c5e4bce17a4a4d336b5f7ef Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Tue, 12 May 2026 21:45:54 -0700 Subject: [PATCH 256/562] [CALCITE-7522] Allow EXTRACT(interval FROM tz) to operate on TIMESTAMP WITH TIME ZONE Signed-off-by: Mihai Budiu --- .../org/apache/calcite/test/BabelQuidemTest.java | 3 +++ babel/src/test/resources/sql/big-query.iq | 15 ++++++++------- .../calcite/sql/fun/SqlExtractFunction.java | 6 ++++++ 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/babel/src/test/java/org/apache/calcite/test/BabelQuidemTest.java b/babel/src/test/java/org/apache/calcite/test/BabelQuidemTest.java index 65a4f6c64d4e..f826d01e4512 100644 --- a/babel/src/test/java/org/apache/calcite/test/BabelQuidemTest.java +++ b/babel/src/test/java/org/apache/calcite/test/BabelQuidemTest.java @@ -34,6 +34,7 @@ import org.junit.jupiter.api.BeforeEach; import java.sql.Connection; +import java.time.ZoneId; import java.util.Collection; import java.util.List; import java.util.Locale; @@ -118,6 +119,8 @@ public static void main(String[] args) throws Exception { ConnectionFactories.addType("TIMESTAMP", typeFactory -> typeFactory.createSqlType( SqlTypeName.TIMESTAMP_WITH_LOCAL_TIME_ZONE))) + // Set the time zone for deterministic results + .with(CalciteConnectionProperty.TIME_ZONE, ZoneId.of("UTC").toString()) .connect(); case "scott-postgresql": return CalciteAssert.that() diff --git a/babel/src/test/resources/sql/big-query.iq b/babel/src/test/resources/sql/big-query.iq index a988eac1d74c..8d9334d224b2 100755 --- a/babel/src/test/resources/sql/big-query.iq +++ b/babel/src/test/resources/sql/big-query.iq @@ -27,6 +27,8 @@ # The DATETIME() and TIMESTAMP() functions are also substituted so that they # produce values that BigQuery would call DATETIME and TIMESTAMP. # +# Note: scott-big-query sets the TIME ZONE to UTC for deterministic test results +# for tests that involve TIME ZONE !use scott-big-query !set outputformat mysql @@ -591,19 +593,18 @@ FROM t; !ok !} -!if (false) { -WITH Input AS (SELECT TIMESTAMP("2008-12-25 05:30:00+00") AS timestamp_value) +# Test case for [CALCITE-7522] Allow EXTRACT(interval FROM tz) to operate on TIMESTAMP WITH TIME ZONE SELECT - EXTRACT(DAY FROM timestamp_value AT TIME ZONE "UTC") AS the_day_utc, - EXTRACT(DAY FROM timestamp_value AT TIME ZONE "America/Los_Angeles") AS the_day_california -FROM Input; + EXTRACT(DAY FROM TIMESTAMP WITH TIME ZONE '2008-12-25 20:30:00 UTC') AS the_day_utc, + EXTRACT(DAY FROM TIMESTAMP WITH TIME ZONE '2008-12-25 20:30:00 America/Los_Angeles') AS the_day_california; +-------------+--------------------+ | the_day_utc | the_day_california | +-------------+--------------------+ -| 25 | 24 | +| 25 | 26 | +-------------+--------------------+ +(1 row) + !ok -!} # Display of results may differ, depending upon the environment and # time zone where this query was executed. diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlExtractFunction.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlExtractFunction.java index 9c8d4b36e297..378c414dd5be 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlExtractFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlExtractFunction.java @@ -88,6 +88,7 @@ public SqlExtractFunction(String name, boolean allowString) { .add(SqlTypeName.DATE) .add(SqlTypeName.TIMESTAMP) .add(SqlTypeName.TIMESTAMP_WITH_LOCAL_TIME_ZONE) + .add(SqlTypeName.TIMESTAMP_TZ) .addAll(SqlTypeName.YEAR_INTERVAL_TYPES) .build(); @@ -96,6 +97,7 @@ public SqlExtractFunction(String name, boolean allowString) { new ImmutableSet.Builder() .add(SqlTypeName.DATE) .add(SqlTypeName.TIMESTAMP) + .add(SqlTypeName.TIMESTAMP_TZ) .add(SqlTypeName.TIMESTAMP_WITH_LOCAL_TIME_ZONE) .build(); @@ -104,6 +106,7 @@ public SqlExtractFunction(String name, boolean allowString) { new ImmutableSet.Builder() .add(SqlTypeName.DATE) .add(SqlTypeName.TIMESTAMP) + .add(SqlTypeName.TIMESTAMP_TZ) .add(SqlTypeName.TIMESTAMP_WITH_LOCAL_TIME_ZONE) .addAll(SqlTypeName.YEAR_INTERVAL_TYPES) .addAll(SqlTypeName.DAY_INTERVAL_TYPES) @@ -114,6 +117,7 @@ public SqlExtractFunction(String name, boolean allowString) { new ImmutableSet.Builder() .add(SqlTypeName.DATE) .add(SqlTypeName.TIMESTAMP) + .add(SqlTypeName.TIMESTAMP_TZ) .add(SqlTypeName.TIMESTAMP_WITH_LOCAL_TIME_ZONE) .add(SqlTypeName.INTERVAL_DAY) .add(SqlTypeName.INTERVAL_DAY_HOUR) @@ -128,8 +132,10 @@ public SqlExtractFunction(String name, boolean allowString) { new ImmutableSet.Builder() .add(SqlTypeName.DATE) .add(SqlTypeName.TIMESTAMP) + .add(SqlTypeName.TIMESTAMP_TZ) .add(SqlTypeName.TIMESTAMP_WITH_LOCAL_TIME_ZONE) .add(SqlTypeName.TIME) + .add(SqlTypeName.TIME_TZ) .add(SqlTypeName.TIME_WITH_LOCAL_TIME_ZONE) .addAll(SqlTypeName.YEAR_INTERVAL_TYPES) .addAll(SqlTypeName.DAY_INTERVAL_TYPES) From 0c4636d207fa080fb21e78413a49c087e73b3e4b Mon Sep 17 00:00:00 2001 From: AlexisCubilla Date: Tue, 12 May 2026 15:47:04 -0300 Subject: [PATCH 257/562] [CALCITE-7524] JdbcSchema throws exception for DECIMAL columns with precision 0 in JDBC metadata Problem: JdbcSchema builds RelDataType from DatabaseMetaData (COLUMN_SIZE, DECIMAL_DIGITS). Some JDBC drivers (notably PostgreSQL) return COLUMN_SIZE 0 for NUMERIC/DECIMAL columns declared without explicit precision. That led to createSqlType(DECIMAL, 0, scale), which SqlTypeFactoryImpl rejects ("DECIMAL precision 0 must be between 1 and ..."). Root cause: sqlType() treated any non-negative precision as literal. Zero is used by drivers as "unspecified" / unknown width, not as a valid SQL DECIMAL precision. Fix: When sqlTypeName is DECIMAL and reported precision is 0, replace it with typeFactory.getTypeSystem().getDefaultPrecision(DECIMAL) before createSqlType. --- .../calcite/adapter/jdbc/JdbcSchema.java | 9 ++- .../calcite/test/JdbcSchemaSqlTypeTest.java | 74 +++++++++++++++++++ 2 files changed, 82 insertions(+), 1 deletion(-) create mode 100644 core/src/test/java/org/apache/calcite/test/JdbcSchemaSqlTypeTest.java diff --git a/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcSchema.java b/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcSchema.java index eb1f11b74d0b..7d1cb8864afa 100644 --- a/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcSchema.java +++ b/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcSchema.java @@ -448,7 +448,14 @@ private static RelDataType sqlType(RelDataTypeFactory typeFactory, int dataType, if (precision >= 0 && scale >= 0 && sqlTypeName.allowsPrecScale(true, true)) { - return typeFactory.createSqlType(sqlTypeName, precision, scale); + int p = precision; + // Some JDBC drivers (e.g. PostgreSQL) report column size 0 for NUMERIC / + // DECIMAL columns without explicit precision in DDL. Calcite rejects + // DECIMAL(0, scale) (see SqlTypeFactoryImpl#createSqlType). + if (p == 0 && sqlTypeName == SqlTypeName.DECIMAL) { + p = typeFactory.getTypeSystem().getDefaultPrecision(sqlTypeName); + } + return typeFactory.createSqlType(sqlTypeName, p, scale); } else if (precision >= 0 && sqlTypeName.allowsPrecNoScale()) { return typeFactory.createSqlType(sqlTypeName, precision); } else { diff --git a/core/src/test/java/org/apache/calcite/test/JdbcSchemaSqlTypeTest.java b/core/src/test/java/org/apache/calcite/test/JdbcSchemaSqlTypeTest.java new file mode 100644 index 000000000000..33a74c9416be --- /dev/null +++ b/core/src/test/java/org/apache/calcite/test/JdbcSchemaSqlTypeTest.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.test; + +import org.apache.calcite.adapter.jdbc.JdbcSchema; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rel.type.RelDataTypeSystem; +import org.apache.calcite.sql.type.SqlTypeFactoryImpl; +import org.apache.calcite.sql.type.SqlTypeName; + +import org.junit.jupiter.api.Test; + +import java.lang.reflect.Method; +import java.sql.Types; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; + +/** Tests JDBC metadata type mapping in {@link JdbcSchema}. */ +class JdbcSchemaSqlTypeTest { + + @Test void jdbcSchemaSqlTypeMapsNumericZeroPrecision() throws Exception { + final RelDataTypeFactory factory = + new SqlTypeFactoryImpl(RelDataTypeSystem.DEFAULT); + final Method sqlType = + JdbcSchema.class.getDeclaredMethod( + "sqlType", + RelDataTypeFactory.class, + int.class, + int.class, + int.class, + String.class); + sqlType.setAccessible(true); + final RelDataType t = + (RelDataType) sqlType.invoke(null, factory, Types.NUMERIC, 0, 6, null); + assertThat(t.getSqlTypeName(), is(SqlTypeName.DECIMAL)); + assertThat(t.getPrecision(), is(19)); + assertThat(t.getScale(), is(6)); + } + + @Test void jdbcSchemaSqlTypeMapsDecimalZeroPrecision() throws Exception { + final RelDataTypeFactory factory = + new SqlTypeFactoryImpl(RelDataTypeSystem.DEFAULT); + final Method sqlType = + JdbcSchema.class.getDeclaredMethod( + "sqlType", + RelDataTypeFactory.class, + int.class, + int.class, + int.class, + String.class); + sqlType.setAccessible(true); + final RelDataType t = + (RelDataType) sqlType.invoke(null, factory, Types.DECIMAL, 0, 2, null); + assertThat(t.getSqlTypeName(), is(SqlTypeName.DECIMAL)); + assertThat(t.getPrecision(), is(19)); + assertThat(t.getScale(), is(2)); + } +} From e1d59a56c4ca0c0340a95be3f212e73dacd93919 Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Thu, 14 May 2026 17:16:10 +0800 Subject: [PATCH 258/562] Document how PMC members add JIRA users to project roles --- site/_docs/howto.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/site/_docs/howto.md b/site/_docs/howto.md index 47a5ac888f2e..6c66d40ccbf7 100644 --- a/site/_docs/howto.md +++ b/site/_docs/howto.md @@ -1065,6 +1065,15 @@ See instructions in ## Processing JIRA account requests Here are some email templates that can be used when processing requests for adding a JIRA account as a contributor. +To add a JIRA account as a Calcite contributor, go to the +[project roles page](https://issues.apache.org/jira/plugins/servlet/project-config/CALCITE/roles), +use the "Add users to a role" button for the "Contributors" role, and add the +user's JIRA username. Members of this role can be assigned Calcite JIRA issues. + +When adding a new Calcite committer, also add their JIRA account to the +"Committers" role. When adding a new Calcite PMC member, also add their +JIRA account to the "PMC" and "Administrators" roles. + ### Account added to contributor list {% highlight text %} Hello [INSERT NAME HERE], From 0077caf6d2b6afe63394133e687d516e7cf55277 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Thu, 14 May 2026 11:52:14 -0700 Subject: [PATCH 259/562] [CALCITE-7526] Incorrect TIMESTAMP WITH TIME ZONE produces wrong error message Signed-off-by: Mihai Budiu --- .../calcite/util/TimestampWithTimeZoneString.java | 2 +- .../org/apache/calcite/rex/RexProgramTest.java | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/calcite/util/TimestampWithTimeZoneString.java b/core/src/main/java/org/apache/calcite/util/TimestampWithTimeZoneString.java index 5816b12a6520..52e9eb6d4f05 100644 --- a/core/src/main/java/org/apache/calcite/util/TimestampWithTimeZoneString.java +++ b/core/src/main/java/org/apache/calcite/util/TimestampWithTimeZoneString.java @@ -68,7 +68,7 @@ public TimestampWithTimeZoneString(String v) { if (pos == -1) { throw RESOURCE.illegalLiteral("TIMESTAMP WITH LOCAL TIME ZONE", v, - RESOURCE.badFormat(TIMESTAMP_FORMAT_STRING).str()).ex(); + RESOURCE.badFormat(TIMESTAMP_FORMAT_STRING + " zone").str()).ex(); } String tsStr = v.substring(0, pos); diff --git a/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java b/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java index 3faf9173d634..01da3e67727b 100644 --- a/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java +++ b/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java @@ -3413,6 +3413,20 @@ private SqlOperator getNoDeterministicOperator() { assertThat(timestampLTZChar1.equals(timestampLTZChar4), is(true)); } + /** Test case for [CALCITE-7526] + * Incorrect TIMESTAMP WITH TIME ZONE produces wrong error message. */ + @Test void testMalformedTimezone() { + try { + new TimestampWithTimeZoneString("2011-07-20T10:34:56America/Los_Angeles"); + } catch (Exception ex) { + assertThat( + ex.getMessage(), is("Illegal TIMESTAMP WITH LOCAL TIME ZONE literal " + + "'2011-07-20T10:34:56America/Los_Angeles': not in format 'yyyy-MM-dd HH:mm:ss zone'")); + return; + } + fail("Should not be reached"); + } + @Test void testSimplifyLiterals() { final RexLiteral literalAbc = rexBuilder.makeLiteral("abc"); final RexLiteral literalDef = rexBuilder.makeLiteral("def"); From 299e4ddb16dda8ca465bfdccb97eacaf67e888ad Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Thu, 14 May 2026 13:51:02 -0700 Subject: [PATCH 260/562] [CALCITE-7527] SqlParserUtil.parseTimestampTzLiteral does not validate timezone Signed-off-by: Mihai Budiu --- .../calcite/sql/parser/SqlParserUtil.java | 33 ++++++++---- .../calcite/sql/parser/SqlParserUtilTest.java | 51 +++++++++++++++++++ .../apache/calcite/test/SqlValidatorTest.java | 2 +- 3 files changed, 76 insertions(+), 10 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql/parser/SqlParserUtil.java b/core/src/main/java/org/apache/calcite/sql/parser/SqlParserUtil.java index fac78417da6b..1ba1933fc70e 100644 --- a/core/src/main/java/org/apache/calcite/sql/parser/SqlParserUtil.java +++ b/core/src/main/java/org/apache/calcite/sql/parser/SqlParserUtil.java @@ -65,6 +65,8 @@ import java.nio.charset.Charset; import java.text.DateFormat; import java.text.SimpleDateFormat; +import java.time.DateTimeException; +import java.time.ZoneId; import java.util.ArrayList; import java.util.Calendar; import java.util.IllformedLocaleException; @@ -403,21 +405,34 @@ public static SqlUuidLiteral parseUuidLiteral(String s, SqlParserPos pos) { public static SqlTimestampTzLiteral parseTimestampTzLiteral( String s, SqlParserPos pos) { - // We expect the string to end in a timezone. - int lastSpace = s.lastIndexOf(" "); - if (lastSpace >= 0) { + // We expect the string to to contain exactly two spaces: + // - one between date and time + // - one between time and timezone + long spaces = s.chars().filter(c -> c == ' ').count(); + if (spaces == 2) { + int lastSpace = s.lastIndexOf(" "); final String timeZone = s.substring(lastSpace + 1); final String timestamp = s.substring(0, lastSpace); - TimeZone tz = TimeZone.getTimeZone(timeZone); - if (tz != null) { - SqlTimestampLiteral ts = parseTimestampLiteral(SqlTypeName.TIMESTAMP, timestamp, pos); - TimestampWithTimeZoneString tsz = new TimestampWithTimeZoneString(ts.getTimestamp(), tz); - return SqlLiteral.createTimestamp(tsz, ts.getPrec(), pos); + try { + ZoneId zoneId = ZoneId.of(timeZone); + TimeZone tz = TimeZone.getTimeZone(zoneId); + if (tz != null) { + SqlTimestampLiteral ts = parseTimestampLiteral(SqlTypeName.TIMESTAMP, timestamp, pos); + TimestampWithTimeZoneString tsz = new TimestampWithTimeZoneString(ts.getTimestamp(), tz); + return SqlLiteral.createTimestamp(tsz, ts.getPrec(), pos); + } + } catch (DateTimeException e) { + String message = e.getMessage(); + if (message == null) { + message = "Error parsing TIME ZONE"; + } + throw SqlUtil.newContextException(pos, + RESOURCE.illegalLiteral("TIMESTAMP WITH TIME ZONE", s, message)); } } throw SqlUtil.newContextException(pos, RESOURCE.illegalLiteral("TIMESTAMP WITH TIME ZONE", s, - RESOURCE.badFormat(DateTimeUtils.TIMESTAMP_FORMAT_STRING).str())); + RESOURCE.badFormat(DateTimeUtils.TIMESTAMP_FORMAT_STRING + " zone").str())); } private static SqlTimestampLiteral parseTimestampLiteral(SqlTypeName typeName, diff --git a/core/src/test/java/org/apache/calcite/sql/parser/SqlParserUtilTest.java b/core/src/test/java/org/apache/calcite/sql/parser/SqlParserUtilTest.java index ca1293c3dd2d..03de04f860da 100644 --- a/core/src/test/java/org/apache/calcite/sql/parser/SqlParserUtilTest.java +++ b/core/src/test/java/org/apache/calcite/sql/parser/SqlParserUtilTest.java @@ -17,12 +17,17 @@ package org.apache.calcite.sql.parser; import org.apache.calcite.avatica.util.TimeUnit; +import org.apache.calcite.runtime.CalciteContextException; import org.apache.calcite.sql.SqlIntervalQualifier; +import org.apache.calcite.sql.SqlTimestampTzLiteral; import org.junit.jupiter.api.Test; import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.hasToString; +import static org.junit.jupiter.api.Assertions.fail; /** * Tests {@link SqlParserUtil}. Currently, this test focuses on tests for the methods that work @@ -43,6 +48,52 @@ public class SqlParserUtilTest { assertThat(SqlParserUtil.intervalToMillis("2", qualifier), equalTo(120_000L)); } + @Test void testTimestampWithTimeZone() { + SqlParserPos pos = new SqlParserPos(2, 3); + SqlTimestampTzLiteral lit = + SqlParserUtil.parseTimestampTzLiteral("2020-01-01 10:10:10 GMT", pos); + assertThat(lit, hasToString("TIMESTAMP_TZ '2020-01-01 10:10:10 GMT'")); + + lit = SqlParserUtil.parseTimestampTzLiteral("2020-01-01 10:10:10 UTC", pos); + assertThat(lit, hasToString("TIMESTAMP_TZ '2020-01-01 10:10:10 UTC'")); + + lit = SqlParserUtil.parseTimestampTzLiteral("2020-01-01 10:10:10 America/Los_Angeles", pos); + assertThat(lit, hasToString("TIMESTAMP_TZ '2020-01-01 10:10:10 America/Los_Angeles'")); + + lit = SqlParserUtil.parseTimestampTzLiteral("2020-01-01 10:10:10 +00:00", pos); + assertThat(lit, hasToString("TIMESTAMP_TZ '2020-01-01 10:10:10 UTC'")); + + lit = SqlParserUtil.parseTimestampTzLiteral("2020-01-01 10:10:10 Z", pos); + assertThat(lit, hasToString("TIMESTAMP_TZ '2020-01-01 10:10:10 UTC'")); + + lit = SqlParserUtil.parseTimestampTzLiteral("2020-01-01 10:10:10 -00:30", pos); + assertThat(lit, hasToString("TIMESTAMP_TZ '2020-01-01 10:10:10 GMT-00:30'")); + + lit = SqlParserUtil.parseTimestampTzLiteral("2020-01-01 10:10:10 +05:45", pos); + assertThat(lit, hasToString("TIMESTAMP_TZ '2020-01-01 10:10:10 GMT+05:45'")); + + // Test case for [CALCITE-7527] SqlParserUtil.parseTimestampTzLiteral does not validate timezone + // https://issues.apache.org/jira/browse/CALCITE-7527 + try { + SqlParserUtil.parseTimestampTzLiteral("2020-06-21 14:23:44.123654+00:00", pos); + fail("Should be unreachable"); + } catch (CalciteContextException ex) { + assertThat( + ex.getMessage(), is("At line 2, column 3: Illegal TIMESTAMP WITH TIME ZONE literal " + + "'2020-06-21 14:23:44.123654+00:00': not in format 'yyyy-MM-dd HH:mm:ss zone'")); + } + + try { + SqlParserUtil.parseTimestampTzLiteral("2020-06-21 14:23:44.123654 incorrect_zone", pos); + fail("Should be unreachable"); + } catch (CalciteContextException ex) { + assertThat( + ex.getMessage(), is("At line 2, column 3: Illegal TIMESTAMP WITH TIME ZONE literal " + + "'2020-06-21 14:23:44.123654 incorrect_zone': Unknown " + + "time-zone ID: incorrect_zone")); + } + } + @Test void testMinuteToSecondIntervalToMillis() { final SqlIntervalQualifier qualifier = new SqlIntervalQualifier(TimeUnit.MINUTE, TimeUnit.SECOND, POSITION); diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index 36ef279b5367..8a4a970e12ad 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -299,7 +299,7 @@ static SqlOperatorTable operatorTableFor(SqlLibrary library) { expr("^TIMESTAMP WITH LOCAL TIME ZONE '12-21-99, 12:30:00'^") .fails("(?s).*Illegal TIMESTAMP WITH LOCAL TIME ZONE literal.*"); expr("^TIMESTAMP WITH TIME ZONE '12-21-99, 12:30:00'^") - .fails("(?s).*Illegal TIMESTAMP literal.*"); + .fails("(?s).*Illegal TIMESTAMP WITH TIME ZONE literal.*"); } /** From a07049217b87f83fdda2c831664018a8f930abae Mon Sep 17 00:00:00 2001 From: Sergey Nuyanzin Date: Thu, 14 May 2026 20:06:48 +0200 Subject: [PATCH 261/562] [CALCITE-7530] `FOR SYSTEM_TIME AS OF` on CTE causes NPE while validation --- .../sql/validate/SqlValidatorImpl.java | 5 ++ .../calcite/test/SqlToRelConverterTest.java | 42 +++++++++ .../calcite/test/SqlToRelConverterTest.xml | 86 +++++++++++++++++++ 3 files changed, 133 insertions(+) diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java index 63f20d59417b..33cdeadc8077 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java @@ -5991,6 +5991,11 @@ private void validateSnapshot( throw newValidationError(period, Static.RESOURCE.illegalExpressionForTemporal(dataType.getSqlTypeName().getName())); } + if (ns instanceof IdentifierNamespace && ns.resolve() instanceof WithItemNamespace) { + // If the snapshot is used over a CTE, then we don't have a concrete underlying + // table to operate on. This will be rechecked later in the planner rules. + return; + } SqlValidatorTable table = getTable(ns); if (!table.isTemporal()) { List qualifiedName = table.getQualifiedName(); diff --git a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java index 3fb907b294d2..4bc62e7c3d56 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java @@ -1624,6 +1624,48 @@ public static void checkActualAndReferenceFiles() { sql(sql).ok(); } + /** Test cases for + * [CALCITE-7530] + * FOR SYSTEM_TIME AS OF on CTE causes NPE while validation. */ + @Test void testSnapshotOnCteOverTemporalTable() { + final String sql = "with cte as (select * from products_temporal)\n" + + "select * from cte for system_time as of\n" + + " TIMESTAMP '2026-01-01 00:00:00'"; + sql(sql).ok(); + } + + @Test void testJoinCteOverTemporalTable() { + final String sql = "with cte as (select * from products_temporal)\n" + + "select stream * from orders\n" + + "join cte for system_time as of orders.rowtime\n" + + "on orders.productid = cte.productid"; + sql(sql).ok(); + } + + @Test void testSnapshotOnNestedCteOverTemporalTable() { + final String sql = "with cte1 as (select * from products_temporal),\n" + + " cte2 as (select * from cte1)\n" + + "select * from cte2 for system_time as of\n" + + " TIMESTAMP '2011-01-02 00:00:00'"; + sql(sql).ok(); + } + + @Test void testCteUsedWithAndWithoutSnapshot() { + final String sql = "with cte as (select * from products_temporal)\n" + + "select * from cte\n" + + "union all\n" + + "select * from cte for system_time as of\n" + + " TIMESTAMP '2011-01-02 00:00:00'"; + sql(sql).ok(); + } + + @Test void testSnapshotOnCteOverNonTemporalTable() { + final String sql = "with cte as (select * from emp)\n" + + "select * from cte for system_time as of\n" + + " TIMESTAMP '2011-01-02 00:00:00'"; + sql(sql).ok(); + } + /** Test case for * [CALCITE-1732] * IndexOutOfBoundsException when using LATERAL TABLE with more than one diff --git a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml index 80d6a95f2025..53662875e0d3 100644 --- a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml @@ -1724,6 +1724,26 @@ LogicalDelta LogicalFilter(condition=[>($0, 1)]) LogicalSnapshot(period=[$cor0.ROWTIME]) LogicalTableScan(table=[[CATALOG, SALES, PRODUCTS_TEMPORAL]]) +]]> + + + + + + + + @@ -3822,6 +3842,26 @@ from (values (cast(null as int), 1), (2, cast(null as int))) as emp(empno, deptno)]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From bfc5f4efcea3a5bb2c96cb9d334daebd063be941 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Fri, 15 May 2026 11:29:44 -0700 Subject: [PATCH 262/562] Addendum to [CALCITE-7475]: correction for unparse method Signed-off-by: Mihai Budiu --- .../test/java/org/apache/calcite/test/BabelParserTest.java | 5 +++++ .../java/org/apache/calcite/sql/fun/SqlCastOperator.java | 5 +++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/babel/src/test/java/org/apache/calcite/test/BabelParserTest.java b/babel/src/test/java/org/apache/calcite/test/BabelParserTest.java index d56664222720..b24b6190279b 100644 --- a/babel/src/test/java/org/apache/calcite/test/BabelParserTest.java +++ b/babel/src/test/java/org/apache/calcite/test/BabelParserTest.java @@ -625,6 +625,11 @@ private void checkParseInfixCast(String sqlType) { f.sql("DISCARD TEMP").same(); } + @Test void testColonUnparse() { + final SqlParserFixture f = fixture().withDialect(PostgresqlSqlDialect.DEFAULT); + f.expression().sql("1::INT").ok("(1 :: INTEGER)"); + } + @Test void testSparkLeftAntiJoin() { final SqlParserFixture f = fixture().withDialect(SparkSqlDialect.DEFAULT); final String sql = "select a.cid, a.cname, count(1) as amount\n" diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlCastOperator.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlCastOperator.java index 723bb014a591..abb2a5c2eede 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlCastOperator.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlCastOperator.java @@ -49,12 +49,13 @@ class SqlCastOperator extends SqlBinaryOperator { } @Override public void unparse(SqlWriter writer, SqlCall call, int leftPrec, int rightPrec) { - writer.sep("("); + writer.print("("); call.operand(0).unparse(writer, 0, 0); writer.keyword("::"); call.operand(1).unparse(writer, 0, 0); - writer.sep(")"); + writer.print(")"); } + @Override public RelDataType inferReturnType( SqlOperatorBinding opBinding) { return SqlStdOperatorTable.CAST.inferReturnType(opBinding); From 38c9d13de8c003f9883db373a6f2e2a6a1610fc6 Mon Sep 17 00:00:00 2001 From: xuzifu666 <1206332514@qq.com> Date: Wed, 13 May 2026 16:57:15 +0800 Subject: [PATCH 263/562] [CALCITE-7523] Support the syntax SELECT * REPLACE(expr as column) --- .../apache/calcite/test/BabelParserTest.java | 18 ++ .../org/apache/calcite/test/BabelTest.java | 73 ++++++++ babel/src/test/resources/sql/select.iq | 46 +++++ core/src/main/codegen/templates/Parser.jj | 52 ++++++ .../calcite/runtime/CalciteResource.java | 9 + .../apache/calcite/sql/SqlStarReplace.java | 84 +++++++++ .../sql/validate/SqlValidatorImpl.java | 164 ++++++++++++++++++ .../runtime/CalciteResource.properties | 3 + site/_docs/reference.md | 6 +- 9 files changed, 454 insertions(+), 1 deletion(-) create mode 100644 core/src/main/java/org/apache/calcite/sql/SqlStarReplace.java diff --git a/babel/src/test/java/org/apache/calcite/test/BabelParserTest.java b/babel/src/test/java/org/apache/calcite/test/BabelParserTest.java index b24b6190279b..48172f7d7fe4 100644 --- a/babel/src/test/java/org/apache/calcite/test/BabelParserTest.java +++ b/babel/src/test/java/org/apache/calcite/test/BabelParserTest.java @@ -167,6 +167,24 @@ class BabelParserTest extends SqlParserTest { sql(sql3).ok(expected3); } + /** Test case for + * [CALCITE-7532] Support the syntax SELECT * REPLACE(expr as column). + * */ + @Test void testStarReplace() { + final String sql = "select * replace(empno + 1 as empno) from emp"; + final String expected = "SELECT * REPLACE ((`EMPNO` + 1) AS `EMPNO`)\n" + + "FROM `EMP`"; + sql(sql).ok(expected); + + final String sql2 = "select e.* replace(e.empno + 1 as e.empno, e.sal * 2 as e.sal)" + + " from emp e join dept d on e.deptno = d.deptno"; + final String expected2 = "SELECT `E`.* REPLACE ((`E`.`EMPNO` + 1) AS `E`.`EMPNO`," + + " (`E`.`SAL` * 2) AS `E`.`SAL`)\n" + + "FROM `EMP` AS `E`\n" + + "INNER JOIN `DEPT` AS `D` ON (`E`.`DEPTNO` = `D`.`DEPTNO`)"; + sql(sql2).ok(expected2); + } + /** Tests that there are no reserved keywords. */ @Disabled @Test void testKeywords() { diff --git a/babel/src/test/java/org/apache/calcite/test/BabelTest.java b/babel/src/test/java/org/apache/calcite/test/BabelTest.java index aff1aee6604b..1deda157fa28 100644 --- a/babel/src/test/java/org/apache/calcite/test/BabelTest.java +++ b/babel/src/test/java/org/apache/calcite/test/BabelTest.java @@ -275,6 +275,79 @@ names, is( .fails("SELECT \\* EXCLUDE/EXCEPT list cannot exclude all columns"); } + /** Test case for + * [CALCITE-7532] Support the syntax SELECT * REPLACE(expr as column). */ + @Test void testStarReplaceValidation() { + final SqlValidatorFixture fixture = Fixtures.forValidator() + .withParserConfig(p -> p.withParserFactory(SqlBabelParserImpl.FACTORY)); + + fixture.withSql("select * replace(empno + 1 as empno) from emp") + .type(type -> { + final List names = type.getFieldList().stream() + .map(RelDataTypeField::getName) + .collect(Collectors.toList()); + assertThat( + names, is( + ImmutableList.of("EMPNO", "ENAME", "JOB", "MGR", + "HIREDATE", "SAL", "COMM", "DEPTNO", "SLACKER"))); + // Verify that EMPNO type is still INTEGER (or similar) + assertThat(type.getFieldList().get(0).getType().getSqlTypeName().getName(), + is("INTEGER")); + }); + + fixture.withSql("select * replace(empno + 1 as empno, sal * 2 as sal) from emp") + .type(type -> { + final List names = type.getFieldList().stream() + .map(RelDataTypeField::getName) + .collect(Collectors.toList()); + assertThat( + names, is( + ImmutableList.of("EMPNO", "ENAME", "JOB", "MGR", + "HIREDATE", "SAL", "COMM", "DEPTNO", "SLACKER"))); + }); + + // REPLACE with a completely different type + fixture.withSql("select * replace('fixed' as empno) from emp") + .type(type -> { + final List names = type.getFieldList().stream() + .map(RelDataTypeField::getName) + .collect(Collectors.toList()); + assertThat( + names, is( + ImmutableList.of("EMPNO", "ENAME", "JOB", "MGR", + "HIREDATE", "SAL", "COMM", "DEPTNO", "SLACKER"))); + // EMPNO was INTEGER, now replaced by a CHAR literal + assertThat(type.getFieldList().get(0).getType().getSqlTypeName().getName(), + is("CHAR")); + }); + + // Same column replaced twice + fixture.withSql("select * replace(empno + 1 as empno, 'fixed' as ^empno^) from emp") + .fails("SELECT \\* REPLACE list contains duplicate column\\(s\\): EMPNO"); + + // Unknown column in REPLACE list + fixture.withSql("select * replace(empno + 1 as ^foo^) from emp") + .fails("SELECT \\* REPLACE list contains unknown column\\(s\\): FOO"); + + // Table-qualified star with REPLACE + fixture.withSql("select e.* replace(e.empno + 1 as e.empno)" + + " from emp e join dept d on e.deptno = d.deptno") + .type(type -> { + final List names = type.getFieldList().stream() + .map(RelDataTypeField::getName) + .collect(Collectors.toList()); + assertThat( + names, is( + ImmutableList.of("EMPNO", "ENAME", "JOB", "MGR", + "HIREDATE", "SAL", "COMM", "DEPTNO", "SLACKER"))); + }); + + // REPLACE with unknown qualified column + fixture.withSql("select e.* replace(e.empno + 1 as ^d.deptno^)" + + " from emp e join dept d on e.deptno = d.deptno") + .fails("SELECT \\* REPLACE list contains unknown column\\(s\\): DEPTNO"); + } + /** Tests that DATEADD, DATEDIFF, DATEPART, DATE_PART allow custom time * frames. */ @Test void testTimeFrames() { diff --git a/babel/src/test/resources/sql/select.iq b/babel/src/test/resources/sql/select.iq index c969ad76559e..f94905c16e9e 100755 --- a/babel/src/test/resources/sql/select.iq +++ b/babel/src/test/resources/sql/select.iq @@ -286,4 +286,50 @@ select d1.* except(d1.dname) from dept d1 except(select d2.* except(d2.dname) fr !ok +# SELECT * REPLACE(expr AS column) +select * replace(empno + 1 as empno) from emp where empno = 7369; ++-------+-------+-------+------+------------+--------+------+--------+ +| EMPNO | ENAME | JOB | MGR | HIREDATE | SAL | COMM | DEPTNO | ++-------+-------+-------+------+------------+--------+------+--------+ +| 7370 | SMITH | CLERK | 7902 | 1980-12-17 | 800.00 | | 20 | ++-------+-------+-------+------+------------+--------+------+--------+ +(1 row) + +!ok + +select * replace(sal * 2 as sal, upper(ename) as ename) from emp where empno = 7369; ++-------+-------+-------+------+------------+---------+------+--------+ +| EMPNO | ENAME | JOB | MGR | HIREDATE | SAL | COMM | DEPTNO | ++-------+-------+-------+------+------------+---------+------+--------+ +| 7369 | SMITH | CLERK | 7902 | 1980-12-17 | 1600.00 | | 20 | ++-------+-------+-------+------+------------+---------+------+--------+ +(1 row) + +!ok + +select e.* replace(e.empno + 1 as e.empno) +from emp e join dept d on e.deptno = d.deptno +where e.empno = 7782; ++-------+-------+---------+------+------------+---------+------+--------+ +| EMPNO | ENAME | JOB | MGR | HIREDATE | SAL | COMM | DEPTNO | ++-------+-------+---------+------+------------+---------+------+--------+ +| 7783 | CLARK | MANAGER | 7839 | 1981-06-09 | 2450.00 | | 10 | ++-------+-------+---------+------+------------+---------+------+--------+ +(1 row) + +!ok + +select empno replace(empno + 1 as empno) from emp; +REPLACE clause must follow a STAR expression +!error + +select * replace(empno + 1 as foo) from emp; +SELECT * REPLACE list contains unknown column(s): FOO +!error + +select e.* replace(e.empno + 1 as d.deptno) +from emp e join dept d on e.deptno = d.deptno; +SELECT * REPLACE list contains unknown column(s): DEPTNO +!error + # End select.iq diff --git a/core/src/main/codegen/templates/Parser.jj b/core/src/main/codegen/templates/Parser.jj index 7e54fb72f0ed..b89ca5c5f5c4 100644 --- a/core/src/main/codegen/templates/Parser.jj +++ b/core/src/main/codegen/templates/Parser.jj @@ -93,6 +93,7 @@ import org.apache.calcite.sql.SqlSelect; import org.apache.calcite.sql.SqlByRewriter; import org.apache.calcite.sql.SqlSelectKeyword; import org.apache.calcite.sql.SqlStarExclude; +import org.apache.calcite.sql.SqlStarReplace; import org.apache.calcite.sql.SqlSetOption; import org.apache.calcite.sql.SqlSnapshot; import org.apache.calcite.sql.SqlTableRef; @@ -2019,6 +2020,7 @@ SqlNode SelectExpression() : { SqlNode e; SqlNodeList excludeList; + SqlNodeList replaceList; } { ( @@ -2029,6 +2031,7 @@ SqlNode SelectExpression() : e = Expression(ExprContext.ACCEPT_SUB_QUERY) ) ( +<#if (parser.includeStarExclude!default.parser.includeStarExclude)> excludeList = StarExcludeList() { if (!(e instanceof SqlIdentifier)) { throw SqlUtil.newContextException(excludeList.getParserPosition(), @@ -2045,10 +2048,30 @@ SqlNode SelectExpression() : return new SqlStarExclude(pos, sqlIdentifier, excludeList); } | + +<#if (parser.includeStarExclude!default.parser.includeStarExclude)> + replaceList = StarReplaceList() { + if (!(e instanceof SqlIdentifier)) { + throw SqlUtil.newContextException(replaceList.getParserPosition(), + RESOURCE.selectReplaceRequiresStar()); + } + final SqlIdentifier sqlIdentifier = (SqlIdentifier) e; + if (!sqlIdentifier.isStar()) { + throw SqlUtil.newContextException(replaceList.getParserPosition(), + RESOURCE.selectReplaceRequiresStar()); + } + final SqlParserPos pos = SqlParserPos.sum( + ImmutableList.of(sqlIdentifier.getParserPosition(), + replaceList.getParserPosition())); + return new SqlStarReplace(pos, sqlIdentifier, replaceList); + } + | + { return e; } ) } +<#if (parser.includeStarExclude!default.parser.includeStarExclude)> SqlNodeList StarExcludeList() : { final Span s; @@ -2069,6 +2092,35 @@ SqlNodeList StarExcludeList() : return new SqlNodeList(list, s.end(this)); } } + + +<#if (parser.includeStarExclude!default.parser.includeStarExclude)> +SqlNodeList StarReplaceList() : +{ + final Span s; + final List list = new ArrayList(); + SqlNode expr; + SqlIdentifier id; +} +{ + { s = span(); } + expr = Expression(ExprContext.ACCEPT_SUB_QUERY) id = CompoundIdentifier() { + list.add(SqlStdOperatorTable.AS.createCall( + SqlParserPos.sum(ImmutableList.of(expr.getParserPosition(), + id.getParserPosition())), expr, id)); + } + ( + expr = Expression(ExprContext.ACCEPT_SUB_QUERY) id = CompoundIdentifier() { + list.add(SqlStdOperatorTable.AS.createCall( + SqlParserPos.sum(ImmutableList.of(expr.getParserPosition(), + id.getParserPosition())), expr, id)); + } + )* + { + return new SqlNodeList(list, s.end(this)); + } +} + <#else> /** * Parses one unaliased expression in a select list. diff --git a/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java b/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java index baf5fca10e04..e00643a56734 100644 --- a/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java +++ b/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java @@ -810,12 +810,21 @@ ExInst illegalArgumentForTableFunctionCall(String a0, @BaseMessage("EXCLUDE/EXCEPT clause must follow a STAR expression") ExInst selectExcludeRequiresStar(); + @BaseMessage("REPLACE clause must follow a STAR expression") + ExInst selectReplaceRequiresStar(); + @BaseMessage("SELECT * EXCLUDE/EXCEPT list contains unknown column(s): {0}") ExInst selectStarExcludeListContainsUnknownColumns(String columns); @BaseMessage("SELECT * EXCLUDE/EXCEPT list cannot exclude all columns") ExInst selectStarExcludeCannotExcludeAllColumns(); + @BaseMessage("SELECT * REPLACE list contains unknown column(s): {0}") + ExInst selectStarReplaceListContainsUnknownColumns(String columns); + + @BaseMessage("SELECT * REPLACE list contains duplicate column(s): {0}") + ExInst selectStarReplaceListContainsDuplicateColumns(String columns); + @BaseMessage("Group function ''{0}'' can only appear in GROUP BY clause") ExInst groupFunctionMustAppearInGroupByClause(String funcName); diff --git a/core/src/main/java/org/apache/calcite/sql/SqlStarReplace.java b/core/src/main/java/org/apache/calcite/sql/SqlStarReplace.java new file mode 100644 index 000000000000..b5149a640379 --- /dev/null +++ b/core/src/main/java/org/apache/calcite/sql/SqlStarReplace.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.sql; + +import org.apache.calcite.sql.parser.SqlParserPos; + +import com.google.common.collect.ImmutableList; + +import org.checkerframework.checker.nullness.qual.Nullable; + +import java.util.List; + +import static java.util.Objects.requireNonNull; + +/** + * Represents {@code SELECT * REPLACE(expr AS column, ...)}. + */ +public class SqlStarReplace extends SqlCall { + public static final SqlOperator OPERATOR = + new SqlSpecialOperator("SELECT_STAR_REPLACE", SqlKind.OTHER) { + @SuppressWarnings("argument.type.incompatible") + @Override public SqlCall createCall( + @Nullable SqlLiteral functionQualifier, + SqlParserPos pos, + @Nullable SqlNode... operands) { + return new SqlStarReplace( + pos, + (SqlIdentifier) operands[0], + (SqlNodeList) operands[1]); + } + }; + + private final SqlIdentifier starIdentifier; + private final SqlNodeList replaceList; + + public SqlStarReplace(SqlParserPos pos, SqlIdentifier starIdentifier, + SqlNodeList replaceList) { + super(pos); + this.starIdentifier = requireNonNull(starIdentifier, "starIdentifier"); + this.replaceList = requireNonNull(replaceList, "replaceList"); + } + + public SqlIdentifier getStarIdentifier() { + return starIdentifier; + } + + public SqlNodeList getReplaceList() { + return replaceList; + } + + @Override public SqlOperator getOperator() { + return OPERATOR; + } + + @Override public SqlKind getKind() { + return OPERATOR.getKind(); + } + + @Override public List getOperandList() { + return ImmutableList.of(starIdentifier, replaceList); + } + + @Override public void unparse(SqlWriter writer, int leftPrec, int rightPrec) { + starIdentifier.unparse(writer, leftPrec, rightPrec); + writer.sep("REPLACE"); + final SqlWriter.Frame frame = writer.startList("(", ")"); + replaceList.unparse(writer, 0, 0); + writer.endList(frame); + } +} diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java index 33cdeadc8077..1d2c94d11a64 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java @@ -80,6 +80,7 @@ import org.apache.calcite.sql.SqlSelectKeyword; import org.apache.calcite.sql.SqlSnapshot; import org.apache.calcite.sql.SqlStarExclude; +import org.apache.calcite.sql.SqlStarReplace; import org.apache.calcite.sql.SqlSyntax; import org.apache.calcite.sql.SqlTableFunction; import org.apache.calcite.sql.SqlUnknownLiteral; @@ -123,7 +124,9 @@ import org.apache.calcite.util.trace.CalciteTrace; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; import com.google.common.collect.Sets; import org.apiguardian.api.API; @@ -645,13 +648,21 @@ private boolean expandStar(List selectItems, Set aliases, SelectScope scope, SqlNode node) { final SqlIdentifier identifier; final SqlNodeList excludeList; + final SqlNodeList replaceList; if (node instanceof SqlStarExclude) { final SqlStarExclude starExclude = (SqlStarExclude) node; identifier = starExclude.getStarIdentifier(); excludeList = starExclude.getExcludeList(); + replaceList = null; + } else if (node instanceof SqlStarReplace) { + final SqlStarReplace starReplace = (SqlStarReplace) node; + identifier = starReplace.getStarIdentifier(); + excludeList = null; + replaceList = starReplace.getReplaceList(); } else if (node instanceof SqlIdentifier) { identifier = (SqlIdentifier) node; excludeList = null; + replaceList = null; } else { return false; } @@ -663,6 +674,38 @@ private boolean expandStar(List selectItems, Set aliases, final boolean[] excludeMatched = new boolean[excludeIdentifiers.size()]; final SqlNameMatcher nameMatcher = scope.validator.catalogReader.nameMatcher(); + if (replaceList != null) { + final Set replaceSeen = new HashSet<>(); + for (SqlNode replaceNode : replaceList) { + final SqlCall call = (SqlCall) replaceNode; + final SqlIdentifier aliasId = (SqlIdentifier) call.operand(1); + final String aliasName = + aliasId.isSimple() ? aliasId.getSimple() + : aliasId.names.get(aliasId.names.size() - 1); + if (!replaceSeen.add(aliasName.toUpperCase(Locale.ROOT))) { + throw newValidationError(aliasId, + RESOURCE.selectStarReplaceListContainsDuplicateColumns(aliasName)); + } + if (!aliasId.isSimple()) { + final int starPrefixSize = identifier.names.size() - 1; + final int aliasPrefixSize = aliasId.names.size() - 1; + if (aliasPrefixSize != starPrefixSize) { + throw newValidationError(aliasId, + RESOURCE.selectStarReplaceListContainsUnknownColumns(aliasName)); + } + for (int i = 0; i < starPrefixSize; i++) { + if (!nameMatcher.matches(identifier.names.get(i), aliasId.names.get(i))) { + throw newValidationError(aliasId, + RESOURCE.selectStarReplaceListContainsUnknownColumns(aliasName)); + } + } + } + } + } + final Map replaceMap = extractReplaceMap(replaceList); + final boolean[] replaceMatched = + replaceMap.isEmpty() ? new boolean[0] + : new boolean[replaceMap.size()]; final int originalSize = selectItems.size(); final SqlParserPos startPosition = identifier.getParserPosition(); final int fieldsBeforeStar = fields.size(); @@ -709,6 +752,27 @@ private boolean expandStar(List selectItems, Set aliases, if (shouldExcludeField(excludeList, exp, nameMatcher)) { continue; } + final SqlNode replacement = + findReplacement(columnName, replaceMap, nameMatcher); + if (replacement != null) { + recordReplaceMatch(columnName, replaceMap, nameMatcher, replaceMatched); + final SqlNode aliasedReplacement = + SqlStdOperatorTable.AS.createCall( + SqlParserPos.sum( + ImmutableList.of( + replacement.getParserPosition(), + exp.getParserPosition())), + replacement, + new SqlIdentifier(columnName, exp.getParserPosition())); + addToSelectList( + selectItems, + aliases, + fields, + aliasedReplacement, + scope, + includeSystemVars); + continue; + } // Don't add expanded rolled up columns if (!isRolledUpColumn(exp, scope)) { addOrExpandField( @@ -745,6 +809,7 @@ private boolean expandStar(List selectItems, Set aliases, throwIfUnknownExcludeColumns(excludeIdentifiers, excludeMatched); throwIfExcludeEliminatesAllColumns(excludeIdentifiers, fieldsBeforeStar, fields, identifier); + throwIfUnknownReplaceColumns(replaceMap, replaceMatched); return true; default: @@ -781,6 +846,31 @@ private boolean expandStar(List selectItems, Set aliases, if (shouldExcludeField(excludeList, columnId, resolvedNameMatcher)) { continue; } + final SqlNode replacement = + findReplacement(columnName, replaceMap, resolvedNameMatcher); + if (replacement != null) { + recordReplaceMatch(columnName, replaceMap, resolvedNameMatcher, + replaceMatched); + final SqlNode aliasedReplacement = + SqlStdOperatorTable.AS.createCall( + SqlParserPos.sum( + ImmutableList.of( + replacement.getParserPosition(), + columnId.getParserPosition())), + replacement, + new SqlIdentifier(columnName, columnId.getParserPosition())); + addToSelectList( + selectItems, + aliases, + fields, + aliasedReplacement, + scope, + includeSystemVars); + continue; + } + // No replacement for this column; keep the original field. + // If the REPLACE list contains unknown columns, they will be + // reported by throwIfUnknownReplaceColumns after the loop. // TODO: do real implicit collation here addOrExpandField( selectItems, @@ -797,6 +887,7 @@ private boolean expandStar(List selectItems, Set aliases, throwIfUnknownExcludeColumns(excludeIdentifiers, excludeMatched); throwIfExcludeEliminatesAllColumns(excludeIdentifiers, fieldsBeforeStar, fields, identifier); + throwIfUnknownReplaceColumns(replaceMap, replaceMatched); return true; } } @@ -903,6 +994,79 @@ private void throwIfExcludeEliminatesAllColumns(List excludeIdent } } + private static Map extractReplaceMap(@Nullable SqlNodeList replaceList) { + if (replaceList == null) { + return ImmutableMap.of(); + } + final ImmutableMap.Builder builder = ImmutableMap.builder(); + for (SqlNode node : replaceList) { + assert node instanceof SqlCall; + final SqlCall call = (SqlCall) node; + assert call.getOperator() == SqlStdOperatorTable.AS + && call.operandCount() == 2; + final SqlNode nameNode = call.operand(1); + assert nameNode instanceof SqlIdentifier; + final SqlIdentifier nameId = (SqlIdentifier) nameNode; + builder.put(nameId.isSimple() ? nameId.getSimple() + : nameId.names.get(nameId.names.size() - 1), call); + } + return builder.build(); + } + + private static @Nullable SqlNode findReplacement(String columnName, + Map replaceMap, SqlNameMatcher nameMatcher) { + for (Map.Entry entry : replaceMap.entrySet()) { + if (nameMatcher.matches(entry.getKey(), columnName)) { + final SqlNode value = entry.getValue(); + return value instanceof SqlCall ? ((SqlCall) value).operand(0) : value; + } + } + return null; + } + + private static void recordReplaceMatch(String columnName, + Map replaceMap, SqlNameMatcher nameMatcher, + boolean[] matched) { + int i = 0; + for (Map.Entry entry : replaceMap.entrySet()) { + if (!matched[i] + && nameMatcher.matches(entry.getKey(), columnName)) { + matched[i] = true; + } + i++; + } + } + + private void throwIfUnknownReplaceColumns(Map replaceMap, + boolean[] replaceMatched) { + if (replaceMap.isEmpty()) { + return; + } + final List unknownReplaceNames = new ArrayList<>(); + int firstUnknownIndex = -1; + int i = 0; + for (Map.Entry entry : replaceMap.entrySet()) { + if (!replaceMatched[i]) { + if (firstUnknownIndex < 0) { + firstUnknownIndex = i; + } + unknownReplaceNames.add(entry.getKey()); + } + i++; + } + if (firstUnknownIndex >= 0) { + final SqlNode firstUnknownExpr = + Iterables.get(replaceMap.values(), firstUnknownIndex); + final SqlNode errorNode = firstUnknownExpr instanceof SqlCall + ? ((SqlCall) firstUnknownExpr).operand(1) + : firstUnknownExpr; + throw newValidationError( + errorNode, + RESOURCE.selectStarReplaceListContainsUnknownColumns( + String.join(", ", unknownReplaceNames))); + } + } + protected SqlNode maybeCast(SqlNode node, RelDataType currentType, RelDataType desiredType) { return SqlTypeUtil.equalSansNullability(typeFactory, currentType, desiredType) diff --git a/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties b/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties index ac137d058e70..49a63bc4efdf 100644 --- a/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties +++ b/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties @@ -267,7 +267,10 @@ MinusNotAllowed=MINUS is not allowed under the current SQL conformance level SelectMissingFrom=SELECT must have a FROM clause SelectStarRequiresFrom=SELECT * requires a FROM clause SelectExcludeRequiresStar=EXCLUDE/EXCEPT clause must follow a STAR expression +SelectReplaceRequiresStar=REPLACE clause must follow a STAR expression SelectStarExcludeListContainsUnknownColumns=SELECT * EXCLUDE/EXCEPT list contains unknown column(s): {0} +SelectStarReplaceListContainsUnknownColumns=SELECT * REPLACE list contains unknown column(s): {0} +SelectStarReplaceListContainsDuplicateColumns=SELECT * REPLACE list contains duplicate column(s): {0} SelectStarExcludeCannotExcludeAllColumns=SELECT * EXCLUDE/EXCEPT list cannot exclude all columns GroupFunctionMustAppearInGroupByClause=Group function ''{0}'' can only appear in GROUP BY clause AuxiliaryWithoutMatchingGroupCall=Call to auxiliary group function ''{0}'' must have matching call to group function ''{1}'' in GROUP BY clause diff --git a/site/_docs/reference.md b/site/_docs/reference.md index d91b083072e1..bdf4ada19998 100644 --- a/site/_docs/reference.md +++ b/site/_docs/reference.md @@ -244,9 +244,13 @@ starWithExclude: * | * EXCLUDE '(' column [, column ]* ')' +starWithReplace: + * + | * REPLACE '(' expression AS column [, expression AS column ]* ')' + Note: -* `SELECT * EXCLUDE (...)` is recognized only when the Babel parser is enabled. It sets the generated parser configuration flag `includeStarExclude` to `true` (the standard parser leaves that flag `false`), which allows a `STAR` token followed by `EXCLUDE` (or the alias `EXCEPT`) and a parenthesized identifier list to be parsed into a `SqlStarExclude` node and ensures validators respect the exclusion list when expanding the projection. Reusing the same parser configuration elsewhere enables the same syntax for other components that need it. +* `SELECT * EXCLUDE (...)` and `SELECT * REPLACE (...)` are recognized only when the Babel parser is enabled. `EXCLUDE` (or the alias `EXCEPT`) removes the specified columns from the star expansion; `REPLACE` substitutes the given expressions for the matching columns while keeping the original column order. For `REPLACE`, the column alias must either be a simple identifier or, for a table-qualified star such as `t.*`, a qualified identifier whose prefix matches the star's table alias. projectItem: expression [ [ AS ] columnAlias ] From af3a3f92974e8cba180549dc9b68b7bcc6107825 Mon Sep 17 00:00:00 2001 From: Julian Hyde Date: Wed, 4 Mar 2026 10:54:25 -0800 Subject: [PATCH 264/562] Improve [CALCITE-7424] In Lint, support sort specifications When encountering an out-of-order line, the lint warning now says which line it should be moved to. --- .../org/apache/calcite/test/LintTest.java | 38 +++++++++++-------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/core/src/test/java/org/apache/calcite/test/LintTest.java b/core/src/test/java/org/apache/calcite/test/LintTest.java index 089d76edbaa4..ef0db448315d 100644 --- a/core/src/test/java/org/apache/calcite/test/LintTest.java +++ b/core/src/test/java/org/apache/calcite/test/LintTest.java @@ -16,6 +16,7 @@ */ package org.apache.calcite.test; +import org.apache.calcite.runtime.PairList; import org.apache.calcite.util.Puffin; import org.apache.calcite.util.Source; import org.apache.calcite.util.Sources; @@ -633,7 +634,8 @@ public boolean inJavadoc() { + " }\n" + "}\n", "GuavaCharSource{memory}:7:" - + "Lines must be sorted; ' case b' should be before ' case c'"); + + "Lines must be sorted; ' case b' should be before ' case c'" + + " (move to line 5)"); // Cases after "until" are checked against the same sorted list. checkSortSpec( @@ -649,7 +651,8 @@ public boolean inJavadoc() { + " }\n" + "}\n", "GuavaCharSource{memory}:9:" - + "Lines must be sorted; ' case a' should be before ' case x'"); + + "Lines must be sorted; ' case a' should be before ' case x'" + + " (move to line 4)"); // Change '#}' to '##}': consumer stops at the same-indent '}', so // the second switch's cases are not compared. No violations. @@ -676,7 +679,8 @@ public boolean inJavadoc() { + " }\n" + "}\n", "GuavaCharSource{memory}:4:" - + "Lines must be sorted; 'a' should be before 'c'"); + + "Lines must be sorted; 'a' should be before 'c'" + + " (move to line 3)"); // Specification spread over multiple lines using '\' continuation. checkSortSpec( @@ -690,7 +694,8 @@ public boolean inJavadoc() { + " }\n" + "}\n", "GuavaCharSource{memory}:6:" - + "Lines must be sorted; 'a' should be before 'c'"); + + "Lines must be sorted; 'a' should be before 'c'" + + " (move to line 5)"); } private void checkSortSpec(String code, String... expectedMessages) { @@ -781,7 +786,7 @@ private static class SortConsumer implements Consumer> { final Sort sort; final Comparator comparator = String.CASE_INSENSITIVE_ORDER; - final List lines = new ArrayList<>(); + final PairList lines = PairList.of(); boolean done = false; SortConsumer(Sort sort) { @@ -818,19 +823,22 @@ private static class SortConsumer private void addLine(Puffin.Line line, String thisLine) { if (!lines.isEmpty()) { - final String prevLine = lines.get(lines.size() - 1); + final String prevLine = lines.left(lines.size() - 1); if (comparator.compare(prevLine, thisLine) > 0) { - final String earlierLine = - Util.filter(lines, s -> comparator.compare(s, thisLine) > 0) - .iterator().next(); - line.state().message( - String.format(Locale.ROOT, - "Lines must be sorted; '%s' should be before '%s'", - thisLine, earlierLine), - line); + for (int i = 0; i < lines.size(); i++) { + if (comparator.compare(lines.left(i), thisLine) > 0) { + line.state().message( + String.format(Locale.ROOT, + "Lines must be sorted; '%s' should be before '%s'" + + " (move to line %d)", + thisLine, lines.left(i), lines.right(i)), + line); + break; + } + } } } - lines.add(thisLine); + lines.add(thisLine, line.fnr()); } } } From bdf61abd85abfcfafdcb3ebf3b20be930f6d7cdd Mon Sep 17 00:00:00 2001 From: dssysolyatin Date: Tue, 31 Mar 2026 13:37:44 +0300 Subject: [PATCH 265/562] [CALCITE-7457] VALUES and SELECT produce different validation results for the same expression --- .../apache/calcite/sql/SqlValuesOperator.java | 8 ++ .../sql/validate/SqlValidatorImpl.java | 3 + .../apache/calcite/test/SqlValidatorTest.java | 119 ++++++++++++++++-- 3 files changed, 119 insertions(+), 11 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql/SqlValuesOperator.java b/core/src/main/java/org/apache/calcite/sql/SqlValuesOperator.java index 0175f89a3165..966ddd3f618a 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlValuesOperator.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlValuesOperator.java @@ -16,6 +16,9 @@ */ package org.apache.calcite.sql; +import org.apache.calcite.sql.validate.SqlValidator; +import org.apache.calcite.sql.validate.SqlValidatorScope; + /** * The VALUES operator. */ @@ -28,6 +31,11 @@ public SqlValuesOperator() { //~ Methods ---------------------------------------------------------------- + @Override public void validateCall(SqlCall call, SqlValidator validator, + SqlValidatorScope scope, SqlValidatorScope operandScope) { + validator.validateQuery(call, scope, validator.getUnknownType()); + } + @Override public void unparse( SqlWriter writer, SqlCall call, diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java index 1d2c94d11a64..e21d62aecbff 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java @@ -2468,6 +2468,9 @@ protected void inferUnknownTypes( scope = getMeasureScope(((SelectScope) scope).getNode()); } inferUnknownTypes(inferredType, scope, ((SqlCall) node).operand(0)); + } else if (node.isA(SqlKind.QUERY)) { + // Do not descend into subqueries. Each query (SELECT, VALUES, + // etc.) calls inferUnknownTypes during its own validation. } else if (node instanceof SqlCall) { final SqlCall call = (SqlCall) node; final SqlOperandTypeInference operandTypeInference = diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index 8a4a970e12ad..7c08ef3925a4 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -1322,13 +1322,20 @@ void testLikeAndSimilarFails() { expr("values 1.0 + ^NULL^") .withTypeCoercion(false) .fails("(?s).*Illegal use of .NULL.*"); + // SELECT produces DECIMAL(3, 1) because inferUnknownTypes infers + // NULL as DECIMAL(2, 1) via FIRST_KNOWN, and DECIMAL(2,1) + DECIMAL(2,1) + // = DECIMAL(3, 1) + sql("select 1.0 + NULL from (values (0)) as t(x)") + .columnType("DECIMAL(3, 1)"); expr("values 1.0 + NULL") - .columnType("DECIMAL(2, 1)"); + .columnType("DECIMAL(3, 1)"); expr("1.0 + ^NULL^") .withTypeCoercion(false) .fails("(?s).*Illegal use of .NULL.*"); + sql("select 1.0 + NULL from (values (0)) as t(x)") + .columnType("DECIMAL(3, 1)"); expr("1.0 + NULL") - .columnType("DECIMAL(2, 1)"); + .columnType("DECIMAL(3, 1)"); // FIXME: SQL:2003 does not allow raw NULL in IN clause expr("1 in (1, null, 2)").ok(); @@ -1602,9 +1609,14 @@ void testLikeAndSimilarFails() { expr("LOCALTIME").ok(); // fix sqlcontext later. wholeExpr("LOCALTIME(1+2)") .fails("Argument to function 'LOCALTIME' must be a literal"); - wholeExpr("LOCALTIME(NULL)") + // With type coercion disabled, inferUnknownTypes rejects NULL before + // LOCALTIME can validate. SELECT produces the same error. + sql("select LOCALTIME(^NULL^) from (values (0)) as t(x)") .withTypeCoercion(false) - .fails("Argument to function 'LOCALTIME' must not be NULL"); + .fails("(?s).*Illegal use of .NULL.*"); + expr("LOCALTIME(^NULL^)") + .withTypeCoercion(false) + .fails("(?s).*Illegal use of .NULL.*"); wholeExpr("LOCALTIME(NULL)") .fails("Argument to function 'LOCALTIME' must not be NULL"); wholeExpr("LOCALTIME(CAST(NULL AS INTEGER))") @@ -8225,6 +8237,78 @@ void testGroupExpressionEquivalenceParams() { .ok(); } + /** + * Test case for + * [CALCITE-7457] + * VALUES and SELECT produce different validation results for the same expression. + */ + @Test void testSelectVsValuesValidation() { + sql("select 1.0 + NULL from (values (0)) as t(x)") + .columnType("DECIMAL(3, 1)"); + expr("1.0 + NULL") + .columnType("DECIMAL(3, 1)"); + + sql("select 1 + ? from (values (0)) as t(x)").ok(); + expr("1 + ?").ok(); + + sql("select 1 + ^NULL^ from (values (0)) as t(x)") + .withTypeCoercion(false) + .fails("(?s).*Illegal use of .NULL.*"); + expr("1 + ^NULL^") + .withTypeCoercion(false) + .fails("(?s).*Illegal use of .NULL.*"); + + sql("select LOCALTIME(^NULL^) from (values (0)) as t(x)") + .withTypeCoercion(false) + .fails("(?s).*Illegal use of .NULL.*"); + expr("LOCALTIME(^NULL^)") + .withTypeCoercion(false) + .fails("(?s).*Illegal use of .NULL.*"); + } + + /** + * Verifies that {@code inferUnknownTypes} works for all subqueries + * from {@link org.apache.calcite.sql.SqlKind#QUERY}. + */ + @Test void testInferTypesForEveryQueryKindAsSubquery() { + // SELECT as scalar subquery + sql("select (select ? + 1 from (values (0)) as t(x))" + + " from (values (0)) as u(y)") + .assertBindType(is("RecordType(INTEGER ?0)")); + + // VALUES as scalar subquery + sql("select (values (? + 1)) from (values (0)) as u(y)") + .assertBindType(is("RecordType(INTEGER ?0)")); + + // UNION as subquery + sql("select * from (values (1)) as t(x)" + + " where x in (select ? + 1 from (values (0)) as u(y)" + + " union all select ? + 1 from (values (0)) as v(z))") + .assertBindType(is("RecordType(INTEGER ?0, INTEGER ?1)")); + + // INTERSECT as subquery + sql("select * from (values (1)) as t(x)" + + " where x in (select ? + 1 from (values (0)) as u(y)" + + " intersect select ? + 1 from (values (0)) as v(z))") + .assertBindType(is("RecordType(INTEGER ?0, INTEGER ?1)")); + + // EXCEPT as subquery + sql("select * from (values (1)) as t(x)" + + " where x in (select ? + 1 from (values (0)) as u(y)" + + " except select ? + 1 from (values (0)) as v(z))") + .assertBindType(is("RecordType(INTEGER ?0, INTEGER ?1)")); + + // WITH as scalar subquery + sql("select (with t(a) as (values (? + 1)) select a from t)" + + " from (values (0)) as u(y)") + .assertBindType(is("RecordType(INTEGER ?0)")); + + // ORDER_BY as scalar subquery + sql("select (select ? + 1 as c from (values (0)) as t(x) order by c)" + + " from (values (0)) as u(y)") + .assertBindType(is("RecordType(INTEGER ?0)")); + } + @Test void testPercentileFunctionsBigQuery() { final SqlOperatorTable opTable = operatorTableFor(SqlLibrary.BIG_QUERY); final String sql = "select\n" @@ -8405,15 +8489,28 @@ void testGroupExpressionEquivalenceParams() { + " overlaps (date '1-2-3', date '1-2-3')^\n" + "or false") .fails("(?s).*Cannot apply 'OVERLAPS' to arguments of type .*"); - // row with 3 arguments as right argument to overlaps + // row with 3 arguments as right argument to overlaps. + // validateValues checks ROW structure before OVERLAPS validates, + // producing "Unequal number of entries in ROW expressions". + // SELECT produces the same error. + sql("select true or" + + " (date '1-2-3', date '1-2-3')" + + " overlaps ^(date '1-2-3', date '1-2-3', date '1-2-3')^" + + " or false from (values (0)) as t(x)") + .fails("(?s).*Unequal number of entries in ROW expressions.*"); expr("true\n" - + "or ^(date '1-2-3', date '1-2-3')\n" - + " overlaps (date '1-2-3', date '1-2-3', date '1-2-3')^\n" + + "or (date '1-2-3', date '1-2-3')\n" + + " overlaps ^(date '1-2-3', date '1-2-3', date '1-2-3')^\n" + "or false") - .fails("(?s).*Cannot apply 'OVERLAPS' to arguments of type .*"); - expr("^period (date '1-2-3', date '1-2-3')\n" - + " overlaps (date '1-2-3', date '1-2-3', date '1-2-3')^") - .fails("(?s).*Cannot apply 'OVERLAPS' to arguments of type .*"); + .fails("(?s).*Unequal number of entries in ROW expressions.*"); + // SELECT produces the same error for mismatched ROW sizes + sql("select period (date '1-2-3', date '1-2-3')" + + " overlaps ^(date '1-2-3', date '1-2-3', date '1-2-3')^" + + " from (values (0)) as t(x)") + .fails("(?s).*Unequal number of entries in ROW expressions.*"); + expr("period (date '1-2-3', date '1-2-3')\n" + + " overlaps ^(date '1-2-3', date '1-2-3', date '1-2-3')^") + .fails("(?s).*Unequal number of entries in ROW expressions.*"); expr("true\n" + "or ^(1, 2) overlaps (2, 3)^\n" + "or false") From 8dc73bd7db5722e0fbba16b3be4ce62fc59f724f Mon Sep 17 00:00:00 2001 From: Diveyam Mishra Date: Sat, 16 May 2026 14:37:05 +0530 Subject: [PATCH 266/562] [CALCITE-4460] Support custom delimiter when parsing CSV tables --- .../adapter/csv/CsvFilterableTable.java | 2 +- .../adapter/csv/CsvScannableTable.java | 2 +- .../adapter/csv/CsvStreamScannableTable.java | 2 +- .../apache/calcite/adapter/csv/CsvTable.java | 4 +- .../adapter/csv/CsvTranslatableTable.java | 3 +- .../calcite/adapter/file/CsvEnumerator.java | 20 +++-- .../calcite/adapter/file/CsvStreamReader.java | 6 +- .../apache/calcite/adapter/file/CsvTable.java | 11 ++- .../calcite/adapter/file/CsvTableFactory.java | 15 +++- .../adapter/file/CsvTranslatableTable.java | 10 ++- .../calcite/adapter/file/FileSchema.java | 5 +- .../calcite/adapter/file/FileAdapterTest.java | 85 +++++++++++++++++++ file/src/test/resources/custom-separator.json | 36 ++++++++ .../resources/sales-csv/PIPE_DELIMITED.csv | 5 ++ site/_docs/file_adapter.md | 20 +++++ 15 files changed, 198 insertions(+), 28 deletions(-) create mode 100644 file/src/test/resources/custom-separator.json create mode 100644 file/src/test/resources/sales-csv/PIPE_DELIMITED.csv diff --git a/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvFilterableTable.java b/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvFilterableTable.java index 4a123529513e..ee621740f9b0 100644 --- a/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvFilterableTable.java +++ b/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvFilterableTable.java @@ -66,7 +66,7 @@ public CsvFilterableTable(Source source, return new AbstractEnumerable<@Nullable Object[]>() { @Override public Enumerator<@Nullable Object[]> enumerator() { return new CsvEnumerator<>(source, cancelFlag, false, filterValues, - CsvEnumerator.arrayConverter(fieldTypes, fields, false)); + CsvEnumerator.arrayConverter(fieldTypes, fields, false), ','); } }; } diff --git a/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvScannableTable.java b/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvScannableTable.java index 25d029505490..836af81373b7 100644 --- a/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvScannableTable.java +++ b/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvScannableTable.java @@ -58,7 +58,7 @@ public class CsvScannableTable extends CsvTable return new AbstractEnumerable<@Nullable Object[]>() { @Override public Enumerator<@Nullable Object[]> enumerator() { return new CsvEnumerator<>(source, cancelFlag, false, null, - CsvEnumerator.arrayConverter(fieldTypes, fields, false)); + CsvEnumerator.arrayConverter(fieldTypes, fields, false), ','); } }; } diff --git a/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvStreamScannableTable.java b/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvStreamScannableTable.java index a7947a2f9853..7c0d574cc7af 100644 --- a/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvStreamScannableTable.java +++ b/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvStreamScannableTable.java @@ -65,7 +65,7 @@ public class CsvStreamScannableTable extends CsvScannableTable return new AbstractEnumerable<@Nullable Object[]>() { @Override public Enumerator<@Nullable Object[]> enumerator() { return new CsvEnumerator<>(source, cancelFlag, true, null, - CsvEnumerator.arrayConverter(fieldTypes, fields, true)); + CsvEnumerator.arrayConverter(fieldTypes, fields, true), ','); } }; } diff --git a/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvTable.java b/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvTable.java index a882e5c8247c..aac56bb38ce4 100644 --- a/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvTable.java +++ b/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvTable.java @@ -51,7 +51,7 @@ public abstract class CsvTable extends AbstractTable { if (rowType == null) { rowType = CsvEnumerator.deduceRowType((JavaTypeFactory) typeFactory, source, - null, isStream()); + null, isStream(), ','); } return rowType; } @@ -61,7 +61,7 @@ public List getFieldTypes(RelDataTypeFactory typeFactory) { if (fieldTypes == null) { fieldTypes = new ArrayList<>(); CsvEnumerator.deduceRowType((JavaTypeFactory) typeFactory, source, - fieldTypes, isStream()); + fieldTypes, isStream(), ','); } return fieldTypes; } diff --git a/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvTranslatableTable.java b/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvTranslatableTable.java index 78632cfb43f8..1da1992b6b14 100644 --- a/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvTranslatableTable.java +++ b/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvTranslatableTable.java @@ -66,7 +66,8 @@ public Enumerable project(final DataContext root, source, cancelFlag, getFieldTypes(typeFactory), - ImmutableIntList.of(fields)); + ImmutableIntList.of(fields), + ','); } }; } diff --git a/file/src/main/java/org/apache/calcite/adapter/file/CsvEnumerator.java b/file/src/main/java/org/apache/calcite/adapter/file/CsvEnumerator.java index 012be6145085..f62433beab47 100644 --- a/file/src/main/java/org/apache/calcite/adapter/file/CsvEnumerator.java +++ b/file/src/main/java/org/apache/calcite/adapter/file/CsvEnumerator.java @@ -113,14 +113,15 @@ private static void clearTimeFormats() { .compile("\"decimal\\(([0-9]+),([0-9]+)\\)"); public CsvEnumerator(Source source, AtomicBoolean cancelFlag, - List fieldTypes, List fields) { + List fieldTypes, List fields, char separator) { //noinspection unchecked this(source, cancelFlag, false, null, - (RowConverter) converter(fieldTypes, fields)); + (RowConverter) converter(fieldTypes, fields), separator); } public CsvEnumerator(Source source, AtomicBoolean cancelFlag, boolean stream, - @Nullable String @Nullable [] filterValues, RowConverter rowConverter) { + @Nullable String @Nullable [] filterValues, RowConverter rowConverter, + char separator) { this.cancelFlag = cancelFlag; this.rowConverter = rowConverter; this.filterValues = @@ -128,9 +129,9 @@ public CsvEnumerator(Source source, AtomicBoolean cancelFlag, boolean stream, : ImmutableNullableList.copyOf(filterValues); try { if (stream) { - this.reader = new CsvStreamReader(source); + this.reader = new CsvStreamReader(source, separator); } else { - this.reader = openCsv(source); + this.reader = openCsv(source, separator); } this.reader.readNext(); // skip header row } catch (IOException e) { @@ -156,14 +157,15 @@ private static RowConverter converter(List fieldTypes, /** Deduces the names and types of a table's columns by reading the first line * of a CSV file. */ public static RelDataType deduceRowType(JavaTypeFactory typeFactory, - Source source, @Nullable List fieldTypes, Boolean stream) { + Source source, @Nullable List fieldTypes, Boolean stream, + char separator) { final List types = new ArrayList<>(); final List names = new ArrayList<>(); if (stream) { names.add(FileSchemaFactory.ROWTIME_COLUMN_NAME); types.add(typeFactory.createSqlType(SqlTypeName.TIMESTAMP)); } - try (CSVReader reader = openCsv(source)) { + try (CSVReader reader = openCsv(source, separator)) { String[] strings = reader.readNext(); if (strings == null) { strings = new String[]{"EmptyFileHasNoColumns:boolean"}; @@ -247,9 +249,9 @@ public static RelDataType deduceRowType(JavaTypeFactory typeFactory, return typeFactory.createStructType(Pair.zip(names, types)); } - static CSVReader openCsv(Source source) throws IOException { + static CSVReader openCsv(Source source, char separator) throws IOException { requireNonNull(source, "source"); - return new CSVReader(source.reader()); + return new CSVReader(source.reader(), separator); } @Override public E current() { diff --git a/file/src/main/java/org/apache/calcite/adapter/file/CsvStreamReader.java b/file/src/main/java/org/apache/calcite/adapter/file/CsvStreamReader.java index 54dd3837e27e..e9113c7d1ac1 100644 --- a/file/src/main/java/org/apache/calcite/adapter/file/CsvStreamReader.java +++ b/file/src/main/java/org/apache/calcite/adapter/file/CsvStreamReader.java @@ -53,9 +53,9 @@ class CsvStreamReader extends CSVReader implements Closeable { */ public static final long DEFAULT_MONITOR_DELAY = 2000; - CsvStreamReader(Source source) { + CsvStreamReader(Source source, char separator) { this(source, - CSVParser.DEFAULT_SEPARATOR, + separator, CSVParser.DEFAULT_QUOTE_CHARACTER, CSVParser.DEFAULT_ESCAPE_CHARACTER, DEFAULT_SKIP_LINES, @@ -106,7 +106,7 @@ private CsvStreamReader(Source source, char separator, char quoteChar, /** * Reads the next line from the buffer and converts to a string array. * - * @return a string array with each comma-separated element as a separate entry. + * @return a string array with each delimited element as a separate entry. * * @throws IOException if bad things happen during the read */ diff --git a/file/src/main/java/org/apache/calcite/adapter/file/CsvTable.java b/file/src/main/java/org/apache/calcite/adapter/file/CsvTable.java index 5ebb2b121865..4b4c718d41a5 100644 --- a/file/src/main/java/org/apache/calcite/adapter/file/CsvTable.java +++ b/file/src/main/java/org/apache/calcite/adapter/file/CsvTable.java @@ -37,13 +37,16 @@ public abstract class CsvTable extends AbstractTable { protected final Source source; protected final @Nullable RelProtoDataType protoRowType; + protected final char separator; private @Nullable RelDataType rowType; private @Nullable List fieldTypes; - /** Creates a CsvTable. */ - CsvTable(Source source, @Nullable RelProtoDataType protoRowType) { + /** Creates a CsvTable with a custom separator. */ + CsvTable(Source source, @Nullable RelProtoDataType protoRowType, + char separator) { this.source = source; this.protoRowType = protoRowType; + this.separator = separator; } @Override public RelDataType getRowType(RelDataTypeFactory typeFactory) { @@ -53,7 +56,7 @@ public abstract class CsvTable extends AbstractTable { if (rowType == null) { rowType = CsvEnumerator.deduceRowType((JavaTypeFactory) typeFactory, source, - null, isStream()); + null, isStream(), separator); } return rowType; } @@ -63,7 +66,7 @@ public List getFieldTypes(RelDataTypeFactory typeFactory) { if (fieldTypes == null) { fieldTypes = new ArrayList<>(); CsvEnumerator.deduceRowType((JavaTypeFactory) typeFactory, source, - fieldTypes, isStream()); + fieldTypes, isStream(), separator); } return fieldTypes; } diff --git a/file/src/main/java/org/apache/calcite/adapter/file/CsvTableFactory.java b/file/src/main/java/org/apache/calcite/adapter/file/CsvTableFactory.java index 82e636cb61fa..fa37d156532c 100644 --- a/file/src/main/java/org/apache/calcite/adapter/file/CsvTableFactory.java +++ b/file/src/main/java/org/apache/calcite/adapter/file/CsvTableFactory.java @@ -25,6 +25,8 @@ import org.apache.calcite.util.Source; import org.apache.calcite.util.Sources; +import au.com.bytecode.opencsv.CSVParser; + import org.checkerframework.checker.nullness.qual.Nullable; import java.io.File; @@ -50,6 +52,17 @@ public CsvTableFactory() { final Source source = Sources.file(base, fileName); final RelProtoDataType protoRowType = rowType != null ? RelDataTypeImpl.proto(rowType) : null; - return new CsvTranslatableTable(source, protoRowType); + final String separatorStr = (String) operand.get("separator"); + final char separator; + if (separatorStr == null) { + separator = CSVParser.DEFAULT_SEPARATOR; + } else if (separatorStr.length() == 1) { + separator = separatorStr.charAt(0); + } else { + throw new IllegalArgumentException( + "Invalid separator '" + separatorStr + + "'. Separator must be a single character."); + } + return new CsvTranslatableTable(source, protoRowType, separator); } } diff --git a/file/src/main/java/org/apache/calcite/adapter/file/CsvTranslatableTable.java b/file/src/main/java/org/apache/calcite/adapter/file/CsvTranslatableTable.java index ebfc1f2e7088..7f81defebe01 100644 --- a/file/src/main/java/org/apache/calcite/adapter/file/CsvTranslatableTable.java +++ b/file/src/main/java/org/apache/calcite/adapter/file/CsvTranslatableTable.java @@ -47,9 +47,10 @@ */ public class CsvTranslatableTable extends CsvTable implements QueryableTable, TranslatableTable { - /** Creates a CsvTable. */ - CsvTranslatableTable(Source source, @Nullable RelProtoDataType protoRowType) { - super(source, protoRowType); + /** Creates a CsvTranslatableTable with a custom separator. */ + CsvTranslatableTable(Source source, @Nullable RelProtoDataType protoRowType, + char separator) { + super(source, protoRowType, separator); } @Override public String toString() { @@ -65,7 +66,8 @@ public Enumerable project(final DataContext root, @Override public Enumerator enumerator() { JavaTypeFactory typeFactory = root.getTypeFactory(); return new CsvEnumerator<>(source, cancelFlag, - getFieldTypes(typeFactory), ImmutableIntList.of(fields)); + getFieldTypes(typeFactory), ImmutableIntList.of(fields), + separator); } }; } diff --git a/file/src/main/java/org/apache/calcite/adapter/file/FileSchema.java b/file/src/main/java/org/apache/calcite/adapter/file/FileSchema.java index b842030ad045..f66effb4a32e 100644 --- a/file/src/main/java/org/apache/calcite/adapter/file/FileSchema.java +++ b/file/src/main/java/org/apache/calcite/adapter/file/FileSchema.java @@ -23,6 +23,8 @@ import org.apache.calcite.util.Sources; import org.apache.calcite.util.Util; +import au.com.bytecode.opencsv.CSVParser; + import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -142,7 +144,8 @@ private static boolean addTable(ImmutableMap.Builder builder, } final Source sourceSansCsv = sourceSansGz.trimOrNull(".csv"); if (sourceSansCsv != null) { - final Table table = new CsvTranslatableTable(source, null); + final Table table = + new CsvTranslatableTable(source, null, CSVParser.DEFAULT_SEPARATOR); builder.put(Util.first(tableName, sourceSansCsv.path()), table); return true; } diff --git a/file/src/test/java/org/apache/calcite/adapter/file/FileAdapterTest.java b/file/src/test/java/org/apache/calcite/adapter/file/FileAdapterTest.java index 0d0fdc031c39..ad774f3964e6 100644 --- a/file/src/test/java/org/apache/calcite/adapter/file/FileAdapterTest.java +++ b/file/src/test/java/org/apache/calcite/adapter/file/FileAdapterTest.java @@ -332,6 +332,91 @@ private static void checkEmpty(ResultSet resultSet) { sql("model-with-custom-table", "select * from CUSTOM_TABLE.EMPS").ok(); } + /** Test case for + * [CALCITE-4460] + * Support custom delimiter when parsing CSV tables. + * + *

      Reads a pipe-delimited file via CsvTableFactory with a custom + * separator. */ + @Test void testCsvCustomSeparatorPipe() { + final String sql = "select * from CUSTOM_SEPARATOR.PIPE_DEPTS"; + sql("custom-separator", sql) + .returns("DEPTNO=10; NAME=Sales", + "DEPTNO=20; NAME=Marketing", + "DEPTNO=30; NAME=Accounts", + "DEPTNO=40; NAME=tic|tac|toe") + .ok(); + } + + /** Test case for + * [CALCITE-4460] + * Support custom delimiter when parsing CSV tables. + * + *

      Verifies quoted content is parsed correctly when it contains the custom + * separator character. */ + @Test void testCsvCustomSeparatorEscaping() { + final String sql = "select * from CUSTOM_SEPARATOR.PIPE_DEPTS " + + "where NAME = 'tic|tac|toe'"; + sql("custom-separator", sql) + .returns("DEPTNO=40; NAME=tic|tac|toe") + .ok(); + } + + /** Test case for + * [CALCITE-4460] + * Support custom delimiter when parsing CSV tables. + * + *

      Verifies that a multi-character separator is rejected. */ + @Test void testCsvCustomSeparatorInvalidMultiChar() throws SQLException { + Properties info = new Properties(); + info.put("model", + "inline:" + + "{\n" + + " version: '1.0',\n" + + " defaultSchema: 'TEST',\n" + + " schemas: [\n" + + " {\n" + + " name: 'TEST',\n" + + " tables: [\n" + + " {\n" + + " name: 'BAD',\n" + + " type: 'custom',\n" + + " factory: 'org.apache.calcite.adapter.file.CsvTableFactory',\n" + + " operand: {\n" + + " file: 'sales-csv/DEPTS.csv',\n" + + " separator: '||'\n" + + " }\n" + + " }\n" + + " ]\n" + + " }\n" + + " ]\n" + + "}"); + try { + Connection connection = + DriverManager.getConnection("jdbc:calcite:", info); + connection.close(); + throw new AssertionError("expected error"); + } catch (RuntimeException e) { + Throwable cause = e; + while (cause.getCause() != null) { + cause = cause.getCause(); + } + assertThat(cause.getMessage(), + is("Invalid separator '||'. " + + "Separator must be a single character.")); + } + } + + /** Test case for + * [CALCITE-4460] + * Support custom delimiter when parsing CSV tables. + * + *

      Verifies that omitting the separator defaults to comma. */ + @Test void testCsvDefaultSeparatorBackwardCompat() { + final String sql = "select * from CUSTOM_TABLE.EMPS"; + sql("model-with-custom-table", sql).ok(); + } + @Test void testPushDownProject() { final String sql = "explain plan for select * from EMPS"; final String expected = "PLAN=CsvTableScan(table=[[SALES, EMPS]], " diff --git a/file/src/test/resources/custom-separator.json b/file/src/test/resources/custom-separator.json new file mode 100644 index 000000000000..ebc500239431 --- /dev/null +++ b/file/src/test/resources/custom-separator.json @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +{ + "version": "1.0", + "defaultSchema": "CUSTOM_SEPARATOR", + "schemas": [ + { + "name": "CUSTOM_SEPARATOR", + "tables": [ + { + "name": "PIPE_DEPTS", + "type": "custom", + "factory": "org.apache.calcite.adapter.file.CsvTableFactory", + "operand": { + "file": "sales-csv/PIPE_DELIMITED.csv", + "separator": "|" + } + } + ] + } + ] +} \ No newline at end of file diff --git a/file/src/test/resources/sales-csv/PIPE_DELIMITED.csv b/file/src/test/resources/sales-csv/PIPE_DELIMITED.csv new file mode 100644 index 000000000000..2a094fae300f --- /dev/null +++ b/file/src/test/resources/sales-csv/PIPE_DELIMITED.csv @@ -0,0 +1,5 @@ +DEPTNO:int|NAME:string +10|"Sales" +20|"Marketing" +30|"Accounts" +40|"tic|tac|toe" diff --git a/site/_docs/file_adapter.md b/site/_docs/file_adapter.md index a81255817992..dc31fec4134c 100644 --- a/site/_docs/file_adapter.md +++ b/site/_docs/file_adapter.md @@ -273,6 +273,26 @@ sqlline> select distinct deptno from depts; 3 rows selected (0.985 seconds) {% endhighlight %} +### CSV Custom Separator + +When using `CsvTableFactory` to define a table in a model, you can specify an +optional `separator` operand to use a custom delimiter. + +{% highlight json %} +{ + "name": "PIPE_DEPTS", + "type": "custom", + "factory": "org.apache.calcite.adapter.file.CsvTableFactory", + "operand": { + "file": "sales-csv/PIPE_DELIMITED.csv", + "separator": "|" + } +} +{% endhighlight %} + +The separator must be a single character. If not specified, it defaults to a +comma. + ## JSON files and model-free browsing Some files describe their own schema, and for these files, we do not need a model. For example, `DEPTS.json` has an integer `DEPTNO` column and a string `NAME` column: From 7e0270ffc4e06587d02d41456766d4c8d7520195 Mon Sep 17 00:00:00 2001 From: Julian Hyde Date: Tue, 28 Apr 2026 19:06:30 -0700 Subject: [PATCH 267/562] [CALCITE-7496] OS-adapter usability --- .../adapter/os/FilesTableFunction.java | 38 ++++++++++++------- .../calcite/adapter/os/OsAdapterTest.java | 16 ++++++++ 2 files changed, 41 insertions(+), 13 deletions(-) diff --git a/plus/src/main/java/org/apache/calcite/adapter/os/FilesTableFunction.java b/plus/src/main/java/org/apache/calcite/adapter/os/FilesTableFunction.java index 8c7b3cc81555..3df9498c3532 100644 --- a/plus/src/main/java/org/apache/calcite/adapter/os/FilesTableFunction.java +++ b/plus/src/main/java/org/apache/calcite/adapter/os/FilesTableFunction.java @@ -44,8 +44,7 @@ public class FilesTableFunction { private static final BigDecimal THOUSAND = BigDecimal.valueOf(1000L); - private static final String SINGLE_QUOTE = - "Path with single quote characters are not supported"; + private FilesTableFunction() { } @@ -87,9 +86,28 @@ public static ScannableTable eval(final String path) { // %p file name (including argument) } + /** Wraps {@code path} in single quotes for use in a shell command, + * throwing if it contains a single quote. */ + private String quotePath() { + return "'" + validatePath(path) + "'"; + } + + /** Checks that {@code path} is valid and will not cause mischief. */ + private String validatePath(String path) { + if (path.contains("'")) { + throw new IllegalArgumentException( + "Path with single quote characters is not supported"); + } + if (path.startsWith("-")) { + throw new IllegalArgumentException( + "Path with leading dash character is not supported"); + } + return path; + } + private Enumerable sourceLinux() { final String[] args = { - "find", path, "-printf", "" + "find", "--", validatePath(path), "-printf", "" + "%A@\\0" // access_time + "%b\\0" // block_count + "%C@\\0" // change_time @@ -115,12 +133,9 @@ private Enumerable sourceLinux() { } private Enumerable sourceMacOs() { - if (path.contains("'")) { - throw new IllegalArgumentException(SINGLE_QUOTE); - } // BSD stat format specifiers: https://man.freebsd.org/cgi/man.cgi?query=stat - final String[] args = {"/bin/sh", "-c", "find '" + path - + "' | xargs stat -f " + final String[] args = {"/bin/sh", "-c", "find -- " + quotePath() + + " -print0 | xargs -0 stat -f " + "%a%n" // access_time + "%b%n" // block_count + "%c%n" // change_time @@ -146,14 +161,11 @@ private Enumerable sourceMacOs() { } private Enumerable sourceGnuStat() { - if (path.contains("'")) { - throw new IllegalArgumentException(SINGLE_QUOTE); - } // GNU stat format specifiers: // https://www.gnu.org/software/coreutils/manual/html_node/stat-invocation.html // format string must have exactly 20 lines per file to match the schema - final String[] args = {"/bin/sh", "-c", "find '" + path - + "' | xargs stat -c '" + final String[] args = {"/bin/sh", "-c", "find -- " + quotePath() + + " -print0 | xargs -0 stat -c '" + "%X\n" // access_time + "%b\n" // block_count + "%Z\n" // change_time diff --git a/plus/src/test/java/org/apache/calcite/adapter/os/OsAdapterTest.java b/plus/src/test/java/org/apache/calcite/adapter/os/OsAdapterTest.java index e1158d5aa66d..602c613038b0 100644 --- a/plus/src/test/java/org/apache/calcite/adapter/os/OsAdapterTest.java +++ b/plus/src/test/java/org/apache/calcite/adapter/os/OsAdapterTest.java @@ -149,6 +149,22 @@ private static boolean checkProcessExists(String command) { "type=f"); } + /** Test case for + * [CALCITE-7496] + * The 'files(path)' table function should not allow paths with a leading + * dash. + * + *

      GNU {@code find} treats an argument beginning with '{@code -}' as an + * expression primary rather than a filesystem path. This can cause mischief. + * This test verifies that leading-dash paths are rejected using the harmless + * {@code -name} expression. + */ + @Test void testFilesLeadingDashPath() { + assumeFalse(Util.isWindows(), "Skip: the 'files' table does not work on Windows"); + sql("select * from files('-name')") + .throws_("Path with leading dash character is not supported"); + } + @Test void testPs() { assumeFalse(Util.isWindows(), "Skip: the 'ps' table does not work on Windows"); assumeToolExists("ps"); From 9009eb1421dcab4c945f16dd39a35bdc0243e058 Mon Sep 17 00:00:00 2001 From: Darpan Date: Tue, 19 May 2026 10:15:11 +0530 Subject: [PATCH 268/562] [CALCITE-7533] Parser rejects parenthesized query as the body of a WITH clause --- core/src/main/codegen/templates/Parser.jj | 8 ++++++-- .../apache/calcite/sql/parser/SqlParserTest.java | 16 ++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/core/src/main/codegen/templates/Parser.jj b/core/src/main/codegen/templates/Parser.jj index b89ca5c5f5c4..2f784b34125b 100644 --- a/core/src/main/codegen/templates/Parser.jj +++ b/core/src/main/codegen/templates/Parser.jj @@ -3715,8 +3715,12 @@ SqlNode Query(ExprContext exprContext) : final List list = new ArrayList(); } { - [ withList = WithList() ] - e = LeafQuery(exprContext) { list.add(e); } + ( + withList = WithList() + e = LeafQueryOrExpr(exprContext) { list.add(e); } + | + e = LeafQuery(exprContext) { list.add(e); } + ) ( AddSetOpQuery(list, exprContext) )* { return addWith(withList, SqlParserUtil.toTree(list)); } } diff --git a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java index 955f268cc58c..4ab6f776ba4a 100644 --- a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java +++ b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java @@ -2826,6 +2826,22 @@ void checkPeriodPredicate(Checker checker) { sql(sql).ok(expected); } + /** Test case for + * [CALCITE-7533] + * Parser rejects parenthesized query as the body of a WITH clause. */ + @Test void testWithParenthesizedBody() { + final String sql = "select * from (\n" + + " with q as (select 1 as id)\n" + + " (select id from q) union all (select id from q)) t"; + final String expected = "SELECT *\n" + + "FROM (WITH `Q` AS (SELECT 1 AS `ID`) SELECT `ID`\n" + + "FROM `Q`\n" + + "UNION ALL\n" + + "SELECT `ID`\n" + + "FROM `Q`) AS `T`"; + sql(sql).ok(expected); + } + /** Test case for * [CALCITE-5252] * JDBC adapter sometimes miss parentheses around SELECT in WITH_ITEM body. */ From ae97b7a038e027cf1159f5f0345b68734dc5021b Mon Sep 17 00:00:00 2001 From: Sergey Nuyanzin Date: Fri, 15 May 2026 17:52:10 +0200 Subject: [PATCH 269/562] [CALCITE-7531] Add to `BasicSqlType` constructor accepting precision, scale and nullability --- .../org/apache/calcite/sql/type/BasicSqlType.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/core/src/main/java/org/apache/calcite/sql/type/BasicSqlType.java b/core/src/main/java/org/apache/calcite/sql/type/BasicSqlType.java index e8e9332d6197..b322c4eaa82e 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/BasicSqlType.java +++ b/core/src/main/java/org/apache/calcite/sql/type/BasicSqlType.java @@ -76,6 +76,20 @@ public BasicSqlType(RelDataTypeSystem typeSystem, SqlTypeName typeName, int precision) { this(typeSystem, typeName, false, precision, SCALE_NOT_SPECIFIED, null, null); + } + + /** + * Constructs a type with precision/length and nullability. + * + * @param typeSystem Type system + * @param typeName Type name + * @param isNullable Whether the type is nullable + * @param precision Precision (called length for some types) + */ + public BasicSqlType(RelDataTypeSystem typeSystem, SqlTypeName typeName, + boolean isNullable, int precision) { + this(typeSystem, typeName, isNullable, precision, SCALE_NOT_SPECIFIED, + null, null); checkPrecScale(typeName, true, false); } From 8d8f7f124c4f1504e126ff199c859ce6ef9f4632 Mon Sep 17 00:00:00 2001 From: Alessandro Solimando Date: Wed, 20 May 2026 18:42:23 +0200 Subject: [PATCH 270/562] [CALCITE-7537] Invalid Postgres SQL generated for right-deep comma join trees --- .../java/org/apache/calcite/sql/SqlJoin.java | 5 ++- .../rel/rel2sql/RelToSqlConverterTest.java | 32 +++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/calcite/sql/SqlJoin.java b/core/src/main/java/org/apache/calcite/sql/SqlJoin.java index 6541afa385a6..f948a7d668d4 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlJoin.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlJoin.java @@ -246,7 +246,10 @@ private SqlJoinOperator(String name, int prec) { default: throw Util.unexpected(join.getJoinType()); } - join.right.unparse(writer, getRightPrec(), rightPrec); + // Comma join is associative, so no parens needed on the right child + final int rightChildLeftPrec = + join.getJoinType() == JoinType.COMMA ? getLeftPrec() : getRightPrec(); + join.right.unparse(writer, rightChildLeftPrec, rightPrec); SqlNode joinCondition = join.condition; if (joinCondition != null) { switch (join.getConditionType()) { diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java index d0768b743e34..2953f3b186f1 100644 --- a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java +++ b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java @@ -8449,6 +8449,38 @@ private void checkLiteral2(String expression, String expected) { .withDoris().ok(expectedStarRocks); } + /** Test case for + * [CALCITE-7537] + * Invalid Postgres SQL generated for right-deep comma join trees. + * + *

      As {@link #testCommaCrossJoin3way()}, but with a right-deep join tree. + * The SQL parser produces left-deep trees, so we use {@link RelBuilder} + * to construct: {@code Join(DEPT, Join(BONUS, SALGRADE))}. + * + *

      A right-deep cross join should produce the same flat comma-separated + * FROM list as a left-deep one. */ + @Test void testCommaCrossJoin3wayRightDeep() { + // Use tables with no overlapping column names to avoid SELECT * expansion + final Function relFn = b -> + b.scan("DEPT") + .scan("BONUS") + .scan("SALGRADE") + .join(JoinRelType.INNER) // Join(BONUS, SALGRADE) + .join(JoinRelType.INNER) // Join(DEPT, Join(BONUS, SALGRADE)) + .build(); + final String expectedMysql = "SELECT *\n" + + "FROM `scott`.`DEPT`,\n" + + "`scott`.`BONUS`,\n" + + "`scott`.`SALGRADE`"; + final String expectedPostgresql = "SELECT *\n" + + "FROM \"scott\".\"DEPT\",\n" + + "\"scott\".\"BONUS\",\n" + + "\"scott\".\"SALGRADE\""; + relFn(relFn) + .withMysql().ok(expectedMysql) + .withPostgresql().ok(expectedPostgresql); + } + /** As {@link #testCommaCrossJoin3way()}, but shows that if there is a * {@code LEFT JOIN} in the FROM clause, we can't use comma-join. */ @Test void testLeftJoinPreventsCommaJoin() { From 94fa33a2499fa8f6b3f2180afaac5c02329944e8 Mon Sep 17 00:00:00 2001 From: Terran Date: Wed, 29 Apr 2026 16:23:32 +0800 Subject: [PATCH 271/562] [CALCITE-7085] JOIN USING with unqualified common column fails in a conformance where allowQualifyingCommonColumn is false (e.g. Oracle, Presto) --- .../sql/validate/SqlValidatorImpl.java | 62 ++++++++++++++++++- .../calcite/test/SqlToRelConverterTest.java | 41 ++++++++++++ .../calcite/test/SqlToRelConverterTest.xml | 56 +++++++++++++++++ 3 files changed, 156 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java index e21d62aecbff..1fcbc24c3615 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java @@ -643,6 +643,31 @@ private static void validateQualifiedCommonColumn(SqlJoin join, } } + /** Validates that a SQL node tree does not contain qualified references + * to common columns in a JOIN USING or NATURAL JOIN context. + * This is called before identifier expansion to catch user-written + * qualified common columns in conformances where they are disallowed + * (e.g. Oracle, Presto). + * + * @param nodeList The list of SQL nodes to check + * @param join The JOIN node containing USING/NATURAL condition + * @param scope The select scope for resolving identifiers + */ + private void validateNoQualifiedCommonColumns(SqlNodeList nodeList, + SqlJoin join, SelectScope scope) { + for (SqlNode item : nodeList) { + item.accept(new SqlShuttle() { + @Override public SqlNode visit(SqlIdentifier id) { + if (!id.isSimple()) { + validateQualifiedCommonColumn(join, id, scope, + SqlValidatorImpl.this); + } + return id; + } + }); + } + } + private boolean expandStar(List selectItems, Set aliases, PairList fields, boolean includeSystemVars, SelectScope scope, SqlNode node) { @@ -5253,6 +5278,17 @@ protected void validateGroupClause(SqlSelect select) { // expand the expression in group list. List expandedList = new ArrayList<>(); + // Validate that GROUP BY items do not qualify common columns + // in conformances where it is disallowed (e.g. Oracle, Presto). + // This must run before expansion, because expansion generates + // qualified identifiers that should not trigger this validation. + if (!config.conformance().allowQualifyingCommonColumn()) { + final SqlNode from = select.getFrom(); + if (from instanceof SqlJoin) { + validateNoQualifiedCommonColumns(groupList, + (SqlJoin) from, getRawSelectScopeNonNull(select)); + } + } for (SqlNode groupItem : groupList) { SqlNode expandedItem = extendedExpand(groupItem, groupScope, select, Clause.GROUP_BY); @@ -5404,6 +5440,19 @@ protected void validateHavingClause(SqlSelect select) { } } + /** Validates that SELECT items do not qualify common columns + * in conformances where it is disallowed (e.g. Oracle, Presto). */ + private void validateSelectCommonColumns(SqlNodeList selectItems, + SqlSelect select) { + if (!config().conformance().allowQualifyingCommonColumn()) { + final SqlNode from = select.getFrom(); + if (from instanceof SqlJoin) { + validateNoQualifiedCommonColumns(selectItems, + (SqlJoin) from, getRawSelectScopeNonNull(select)); + } + } + } + protected RelDataType validateSelectList(final SqlNodeList selectItems, SqlSelect select, RelDataType targetRowType) { // First pass, ensure that aliases are unique. "*" and "TABLE.*" items @@ -5417,6 +5466,12 @@ protected RelDataType validateSelectList(final SqlNodeList selectItems, // Populated during select expansion when SqlConformance.isSelectAlias != UNSUPPORTED final Map expansions = new HashMap<>(); + // Validate that SELECT items do not qualify common columns + // in conformances where it is disallowed (e.g. Oracle, Presto). + // This must run before expansion, because expansion generates + // qualified identifiers that should not trigger this validation. + validateSelectCommonColumns(selectItems, select); + for (SqlNode selectItem : selectItems) { if (selectItem instanceof SqlSelect) { handleScalarSubQuery(select, (SqlSelect) selectItem, @@ -7672,9 +7727,10 @@ protected SqlNode expandCommonColumn(SqlSelect sqlSelect, SqlNode selectItem, final SqlIdentifier identifier = (SqlIdentifier) selectItem; if (!identifier.isSimple()) { - if (!validator.config().conformance().allowQualifyingCommonColumn()) { - validateQualifiedCommonColumn((SqlJoin) from, identifier, scope, validator); - } + // Qualified identifiers (e.g. t1.col) are returned unchanged. + // Validation of qualified common columns is performed before expansion, + // in validateSelectCommonColumns, where the original user-written + // identifier (with its source position) is still available. return selectItem; } diff --git a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java index 4bc62e7c3d56..ebad0e9450a5 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java @@ -6092,4 +6092,45 @@ void checkUserDefinedOrderByOver(NullCollation nullCollation) { final String sql = "select distinct deptno, deptno, empno, 1, 'a' from emp order by rand(), 1"; sql(sql).ok(); } + + /** Test case of + * [CALCITE-7085] + * JOIN USING with unqualified common column fails in a conformance where + * allowQualifyingCommonColumn is false (e.g. Oracle, Presto). */ + @Test void testJoinUsingWithConformanceOracle() { + final String sql = "SELECT deptno, name\n" + + "FROM emp JOIN dept using (deptno)"; + sql(sql).withConformance(SqlConformanceEnum.ORACLE_10).ok(); + } + + + /** Test case of + * [CALCITE-7085] + * JOIN USING with unqualified common column fails in a conformance where + * allowQualifyingCommonColumn is false (e.g. Oracle, Presto). */ + @Test void testLeftJoinUsingWithConformanceOracle() { + final String sql = "SELECT deptno, name\n" + + "FROM emp LEFT OUTER JOIN dept using (deptno)"; + sql(sql).withConformance(SqlConformanceEnum.ORACLE_10).ok(); + } + + /** Test case of + * [CALCITE-7085] + * JOIN USING with unqualified common column fails in a conformance where + * allowQualifyingCommonColumn is false (e.g. Oracle, Presto). */ + @Test void testRightJoinUsingWithConformanceOracle() { + final String sql = "SELECT deptno, name\n" + + "FROM emp RIGHT OUTER JOIN dept using (deptno)"; + sql(sql).withConformance(SqlConformanceEnum.ORACLE_10).ok(); + } + + /** Test case of + * [CALCITE-7085] + * JOIN USING with unqualified common column fails in a conformance where + * allowQualifyingCommonColumn is false (e.g. Oracle, Presto). */ + @Test void testJoinUsingWithConformancePresto() { + final String sql = "SELECT deptno, name\n" + + "FROM emp JOIN dept using (deptno)"; + sql(sql).withConformance(SqlConformanceEnum.PRESTO).ok(); + } } diff --git a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml index 53662875e0d3..56f169629ae6 100644 --- a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml @@ -4368,6 +4368,34 @@ LogicalProject(DEPTNO=[$0], EXPR$1=[$1]) LogicalJoin(condition=[=($7, $9)], joinType=[full]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) +]]> + + + + + + + + + + + + + + + + @@ -5000,6 +5028,20 @@ LogicalProject(C=[$0], N=[$3]) LogicalAggregate(group=[{0}]) LogicalProject($f2=[+($0, 1)]) LogicalValues(tuples=[[{ 4 }]]) +]]> + + + + + + + + @@ -7396,6 +7438,20 @@ GROUP BY ROLLUP(deptno)]]> LogicalAggregate(group=[{0}], groups=[[{0}, {}]], EXPR$1=[COUNT(DISTINCT $1)]) LogicalProject(DEPTNO=[$7], EMPNO=[$0]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +]]> + + + + + + + + From e953e03e1b92e66bf83ccc98cf53d70224ae7d1c Mon Sep 17 00:00:00 2001 From: Sergey Nuyanzin Date: Thu, 21 May 2026 00:11:23 +0200 Subject: [PATCH 272/562] [CALCITE-7538] `SqlValidatorImpl` should reject `MATCH_RECOGNIZE` with duplicate `MEASURE` alias --- .../calcite/runtime/CalciteResource.java | 3 ++ .../sql/validate/SqlValidatorImpl.java | 6 ++- .../runtime/CalciteResource.properties | 1 + .../apache/calcite/test/SqlValidatorTest.java | 53 +++++++++++++++++++ 4 files changed, 61 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java b/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java index e00643a56734..40c680e2ee6c 100644 --- a/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java +++ b/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java @@ -1192,4 +1192,7 @@ ExInst multipleCapturingGroupsForRegexpFunctions(String value, @BaseMessage("The argument of DESCRIPTOR must be an identifier") ExInst descriptorMustBeIdentifier(); + + @BaseMessage("Duplicate name ''{0}'' in MATCH_RECOGNIZE MEASURE alias list") + ExInst measureAliasDuplicate(String aliasName); } diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java index 1fcbc24c3615..ae179400b013 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java @@ -6638,7 +6638,7 @@ public void setOriginal(SqlNode expr, SqlNode original) { private PairList validateMeasure(SqlMatchRecognize mr, MatchRecognizeScope scope, boolean allRows) { - final List aliases = new ArrayList<>(); + final Set aliases = new HashSet<>(); final List sqlNodes = new ArrayList<>(); final SqlNodeList measures = mr.getMeasureList(); final PairList fields = PairList.of(); @@ -6646,7 +6646,9 @@ private PairList validateMeasure(SqlMatchRecognize mr, for (SqlNode measure : measures) { assert measure instanceof SqlCall; final String alias = SqlValidatorUtil.alias(measure, aliases.size()); - aliases.add(alias); + if (!aliases.add(alias)) { + throw RESOURCE.measureAliasDuplicate(alias).ex(); + } SqlNode expand = expand(measure, scope); expand = navigationInMeasure(expand, allRows); diff --git a/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties b/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties index 49a63bc4efdf..ae64b08bdd66 100644 --- a/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties +++ b/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties @@ -390,4 +390,5 @@ CannotInferReturnType=Cannot infer return type for {0}; operand types: {1} SelectByCannotWithGroupBy=SELECT BY cannot be used with GROUP BY SelectByCannotWithOrderBy=SELECT BY cannot be used with ORDER BY DescriptorMustBeIdentifier=The argument of DESCRIPTOR must be an identifier +MeasureAliasDuplicate=Duplicate name ''{0}'' in MATCH_RECOGNIZE MEASURE alias list # End CalciteResource.properties diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index 7c08ef3925a4..5198762c106b 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -2760,6 +2760,59 @@ void testLikeAndSimilarFails() { .rewritesTo(expected11); sql(expected11) .withParserConfig(c -> c.withQuoting(Quoting.BACK_TICK)).ok(); + + // Test cases for [CALCITE-7538] https://issues.apache.org/jira/browse/CALCITE-7538 + // SqlValidatorImpl should reject MATCH_RECOGNIZE with duplicate MEASURE alias + final String sql12 = "SELECT *\n" + + "FROM emp\n" + + "MATCH_RECOGNIZE (\n" + + " MEASURES\n" + + " A.deptno AS deptno,\n" + + " A.deptno AS deptno\n" + + " PATTERN (A B)\n" + + " DEFINE\n" + + " A AS A.empno = 123\n" + + ")"; + + sql(sql12) + .fails("Duplicate name 'DEPTNO' in MATCH_RECOGNIZE MEASURE alias list"); + + final String sql13 = "SELECT *\n" + + "FROM emp\n" + + "MATCH_RECOGNIZE (\n" + + " MEASURES\n" + + " A.deptno AS deptno,\n" + + " A.deptno AS DePtNo\n" + + " PATTERN (A B)\n" + + " DEFINE\n" + + " A AS A.empno = 123\n" + + ")"; + + sql(sql13) + .fails("Duplicate name 'DEPTNO' in MATCH_RECOGNIZE MEASURE alias list"); + + final String sql14 = "SELECT *\n" + + "FROM emp\n" + + "MATCH_RECOGNIZE (\n" + + " MEASURES\n" + + " A.deptno AS deptno,\n" + + " A.deptno AS \"DePtNo\"\n" + + " PATTERN (A B)\n" + + " DEFINE\n" + + " A AS A.empno = 123\n" + + ")"; + + final String expected14 = "SELECT `EXPR$0`.`DEPTNO`, `EXPR$0`.`DePtNo`\n" + + "FROM `CATALOG`.`SALES`.`EMP` AS `EMP` MATCH_RECOGNIZE(\n" + + "MEASURES FINAL `A`.`DEPTNO` AS `DEPTNO`, FINAL `A`.`DEPTNO` AS `DePtNo`\n" + + "PATTERN (`A` `B`)\n" + + "DEFINE `A` AS PREV(`A`.`EMPNO`, 0) = 123) AS `EXPR$0`"; + + sql(sql14) + .withValidatorConfig(c -> c.withIdentifierExpansion(true)) + .rewritesTo(expected14); + sql(expected14) + .withParserConfig(c -> c.withQuoting(Quoting.BACK_TICK)).ok(); } @Test void testIntervalTimeUnitEnumeration() { From 01d04371234d8a35bfc9c203861f40db7e6e8db4 Mon Sep 17 00:00:00 2001 From: Sean Broeder Date: Thu, 14 May 2026 07:42:20 -0700 Subject: [PATCH 273/562] [CALCITE-7514] MultiJoinOptimizeBushyRule throws AssertionError when a join condition references 3 or more factors Conditions in a MultiJoin's joinFilters that reference anything other than exactly two factors cannot be represented as binary join edges. Passing such a condition to createEdge produced an edge with factors.cardinality() != 2, causing an AssertionError in the edge comparator's rowCountDiff method, and at two further assertion sites in the greedy loop. The fix separates these conditions from the edge list upfront. After the greedy join-ordering loop completes, the remaining conditions are remapped from original MultiJoin field positions to the final join tree's output positions via RexPermuteInputsShuttle, then applied as a LogicalFilter above the join tree before the reordering project. For inner joins this is semantically equivalent to applying them as join predicates. Two TODO items are resolved: - "Join conditions that touch 3 factors" is fully handled. - "More than 1 join conditions that touch the same pair of factors" was stale from the original commit; the conditions loop already collects all edges subsumed by newFactors at each greedy step. A remaining TODO notes that 1-factor conditions are applied as a filter above the join tree rather than pushed down to the individual scan. --- .../rel/rules/MultiJoinOptimizeBushyRule.java | 40 ++++++++++++------- .../apache/calcite/test/RelOptRulesTest.java | 22 ++++++++++ .../apache/calcite/test/RelOptRulesTest.xml | 28 +++++++++++++ 3 files changed, 75 insertions(+), 15 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rel/rules/MultiJoinOptimizeBushyRule.java b/core/src/main/java/org/apache/calcite/rel/rules/MultiJoinOptimizeBushyRule.java index d636d23fab92..6336b997e742 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/MultiJoinOptimizeBushyRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/MultiJoinOptimizeBushyRule.java @@ -65,13 +65,9 @@ * {@code LoptOptimizeJoinRule} is only capable of producing left-deep joins; * this rule is capable of producing bushy joins. * - *

      TODO: - *

        - *
      1. Join conditions that touch 1 factor. - *
      2. Join conditions that touch 3 factors. - *
      3. More than 1 join conditions that touch the same pair of factors, - * e.g. {@code t0.c1 = t1.c1 and t1.c2 = t0.c3} - *
      + *

      TODO: Join conditions that touch exactly 1 factor are currently applied + * as a filter above the join tree rather than being pushed down to the + * individual table scan. * * @see CoreRules#MULTI_JOIN_OPTIMIZE_BUSHY */ @@ -130,9 +126,17 @@ public MultiJoinOptimizeBushyRule(RelFactories.JoinFactory joinFactory, } assert x == multiJoin.getNumTotalFields(); + final List remainingConditions = new ArrayList<>(); final List unusedEdges = new ArrayList<>(); for (RexNode node : multiJoin.getJoinFilters()) { - unusedEdges.add(multiJoin.createEdge(node)); + LoptMultiJoin.Edge edge = multiJoin.createEdge(node); + if (edge.factors.cardinality() == 2) { + unusedEdges.add(edge); + } else { + // Conditions touching 1 or 3+ factors cannot be used as binary join + // edges. Re-apply them as a filter above the finished join tree. + remainingConditions.add(node); + } } // Comparator that chooses the best edge. A "good edge" is one that has @@ -170,11 +174,8 @@ private double rowCountDiff(LoptMultiJoin.Edge edge) { } else { final LoptMultiJoin.Edge bestEdge = unusedEdges.get(edgeOrdinal); - // For now, assume that the edge is between precisely two factors. - // 1-factor conditions have probably been pushed down, - // and 3-or-more-factor conditions are advanced. (TODO:) - // Therefore, for now, the factors that are merged are exactly the - // factors on this edge. + // Each edge in unusedEdges touches exactly two factors; conditions + // touching 1 or 3+ factors were separated out before the greedy loop. assert bestEdge.factors.cardinality() == 2; factors = bestEdge.factors.toArray(); } @@ -299,8 +300,17 @@ private double rowCountDiff(LoptMultiJoin.Edge edge) { } final Pair top = Util.last(relNodes); - relBuilder.push(top.left) - .project(relBuilder.fields(top.right)); + relBuilder.push(top.left); + if (!remainingConditions.isEmpty()) { + final RexVisitor shuttle = + new RexPermuteInputsShuttle(top.right, top.left); + final List remapped = new ArrayList<>(); + for (RexNode c : remainingConditions) { + remapped.add(c.accept(shuttle)); + } + relBuilder.filter(remapped); + } + relBuilder.project(relBuilder.fields(top.right)); call.transformTo(relBuilder.build()); } diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index 0f5959fb574b..715d9f4f7b8e 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -4344,6 +4344,28 @@ private void checkPushJoinThroughUnionOnRightDoesNotMatchSemiOrAntiJoin(JoinRelT .checkUnchanged(); } + /** Test case for the TODO in {@link MultiJoinOptimizeBushyRule}: + * "Join conditions that touch 3 factors." + * + *

      The CASE condition references three factors (e1, d, e2) and therefore + * cannot be represented as a binary join edge. The rule should handle it + * gracefully rather than throwing an {@code AssertionError}. */ + @Test void testMultiJoinOptimizeBushyThreeFactorCondition() { + HepProgram preProgram = new HepProgramBuilder() + .addRuleInstance(CoreRules.FILTER_INTO_JOIN) + .addMatchOrder(HepMatchOrder.BOTTOM_UP) + .addRuleInstance(CoreRules.JOIN_TO_MULTI_JOIN) + .build(); + HepProgram program = new HepProgramBuilder() + .addMatchOrder(HepMatchOrder.BOTTOM_UP) + .addRuleInstance(CoreRules.MULTI_JOIN_OPTIMIZE_BUSHY) + .build(); + final String sql = "select e1.ename from emp e1, dept d, emp e2\n" + + "where e1.deptno = d.deptno and e2.deptno = d.deptno\n" + + "and d.deptno = case when e1.sal > 1000 then e2.empno else e1.empno end"; + sql(sql).withPre(preProgram).withProgram(program).check(); + } + @Test void testConvertMultiJoinRule() { final String sql = "select e1.ename from emp e1, dept d, emp e2\n" + "where e1.deptno = d.deptno and d.deptno = e2.deptno"; diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index fc60b8864971..c782bb913cc1 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -10775,6 +10775,34 @@ LogicalAggregate(group=[{0, 1}]) LogicalFilter(condition=[AND(=($0, 12), <>($1, 5))]) LogicalProject(MGR=[$3], COMM=[$6]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +]]> + + + + + 1000 then e2.empno else e1.empno end]]> + + + ($5, 1000), $11, $0)), =($7, $9))], isFullOuterJoin=[false], joinTypes=[[INNER, INNER, INNER]], outerJoinConditions=[[NULL, NULL, NULL]], projFields=[[ALL, ALL, ALL]]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) + LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +]]> + + + ($16, 1000), $2, $11))]) + LogicalJoin(condition=[=($18, $0)], joinType=[inner]) + LogicalJoin(condition=[=($9, $0)], joinType=[inner]) + LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) ]]> From 303a4a0415923b01347de6154f8f0f97ad0c8ebf Mon Sep 17 00:00:00 2001 From: xuzifu666 <1206332514@qq.com> Date: Wed, 29 Apr 2026 17:25:31 +0800 Subject: [PATCH 274/562] [CALCITE-7492] Support expression that has a constant value within the group involving only GROUP BY keys as aggregate arguments --- ...gregateReduceFunctionsOnGroupKeysRule.java | 144 ++++++++-- ...ateReduceFunctionsOnGroupKeysRuleTest.java | 88 +++++++ ...gateReduceFunctionsOnGroupKeysRuleTest.xml | 248 +++++++++++++++++- 3 files changed, 462 insertions(+), 18 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rel/rules/AggregateReduceFunctionsOnGroupKeysRule.java b/core/src/main/java/org/apache/calcite/rel/rules/AggregateReduceFunctionsOnGroupKeysRule.java index 10d30d620eba..987a25fea2e9 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/AggregateReduceFunctionsOnGroupKeysRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/AggregateReduceFunctionsOnGroupKeysRule.java @@ -18,15 +18,19 @@ import org.apache.calcite.plan.RelOptRuleCall; import org.apache.calcite.plan.RelRule; +import org.apache.calcite.plan.hep.HepRelVertex; import org.apache.calcite.rel.RelCollations; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Aggregate; import org.apache.calcite.rel.core.AggregateCall; +import org.apache.calcite.rel.core.Project; import org.apache.calcite.rel.core.RelFactories; import org.apache.calcite.rel.logical.LogicalAggregate; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexInputRef; import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexShuttle; +import org.apache.calcite.rex.RexUtil; import org.apache.calcite.sql.SqlKind; import org.apache.calcite.tools.RelBuilder; @@ -45,7 +49,8 @@ * {@code SELECT sal, sal FROM emp GROUP BY sal}. * *

      Currently supports the following aggregate functions when their - * arguments exist in the aggregate's group set: + * arguments exist in the aggregate's group set or are deterministic + * expressions involving only group set columns and constants: *

        *
      • {@code MAX}
        • *
      • {@code MIN}
        • @@ -53,6 +58,12 @@ *
      • {@code ANY_VALUE}
        • *
      * + *

      Note: This optimization preserves NULL semantics correctly. For aggregate + * functions like MAX, MIN, and ANY_VALUE, NULL values in the source columns or + * expressions are handled the same way before and after the transformation: + * nulls are ignored by the aggregation, and if all grouped values are NULL, + * the result is NULL. + * * @see CoreRules#AGGREGATE_REDUCE_FUNCTIONS_ON_GROUP_KEYS */ @Value.Enclosing @@ -74,6 +85,8 @@ protected AggregateReduceFunctionsOnGroupKeysRule(Config config) { final List newCalls = new ArrayList<>(); final List projects = new ArrayList<>(); + final List fieldNames = + new ArrayList<>(aggregate.getRowType().getFieldNames()); // Pass through group keys. for (int i = 0; i < groupCount; i++) { @@ -108,12 +121,13 @@ protected AggregateReduceFunctionsOnGroupKeysRule(Config config) { aggregate.getGroupSets(), newCalls); relBuilder.push(newAggregate); - relBuilder.project(projects); + relBuilder.project(projects, fieldNames); call.transformTo(relBuilder.build()); } /** - * Tries to reduce an aggregate call to a reference to a group-by key. + * Tries to reduce an aggregate call to a reference to a group-by key + * or to an expression involving only group-by keys and constants. * * @return the reduced expression, or null if cannot reduce */ @@ -129,14 +143,6 @@ protected AggregateReduceFunctionsOnGroupKeysRule(Config config) { || call.collation != RelCollations.EMPTY) { return null; } - final List argList = call.getArgList(); - if (argList.size() != 1) { - return null; - } - final int arg = argList.get(0); - if (!aggregate.getGroupSet().get(arg)) { - return null; - } final SqlKind kind = call.getAggregation().getKind(); switch (kind) { case AVG: @@ -147,12 +153,118 @@ protected AggregateReduceFunctionsOnGroupKeysRule(Config config) { default: return null; } - final int groupIndex = aggregate.getGroupSet().asList().indexOf(arg); - RexNode ref = RexInputRef.of(groupIndex, aggregate.getRowType().getFieldList()); - if (!ref.getType().equals(call.getType())) { - ref = rexBuilder.makeCast(call.getParserPosition(), call.getType(), ref); + final List argList = call.getArgList(); + if (argList.size() != 1) { + return null; + } + final int arg = argList.get(0); + + // Case 1: argument directly references a group-by key + if (aggregate.getGroupSet().get(arg)) { + final int groupIndex = aggregate.getGroupSet().asList().indexOf(arg); + RexNode ref = RexInputRef.of(groupIndex, aggregate.getRowType().getFieldList()); + if (!ref.getType().equals(call.getType())) { + ref = rexBuilder.makeCast(call.getParserPosition(), call.getType(), ref); + } + return ref; + } + + // Case 2: argument is an expression in a Project below the Aggregate + RelNode input = aggregate.getInput(); + if (input instanceof HepRelVertex) { + input = ((HepRelVertex) input).getCurrentRel(); + } + if (!(input instanceof Project)) { + return null; + } + final Project project = (Project) input; + if (arg < 0 || arg >= project.getProjects().size()) { + return null; + } + final RexNode expr = project.getProjects().get(arg); + if (!RexUtil.isDeterministic(expr)) { + return null; + } + // Check that all columns referenced in the expression are group-by keys. + // This ensures that the expression value is constant within each group. + final @Nullable RexNode translated = + translateToGroupRefs(expr, project, aggregate); + if (translated == null) { + return null; + } + if (!translated.getType().equals(call.getType())) { + return rexBuilder.makeCast(call.getParserPosition(), call.getType(), translated); + } + return translated; + } + + /** + * Translates an expression so that its {@link RexInputRef}s reference + * the group keys of the aggregate rather than the input to the project. + * + * @return the translated expression, or null if the expression references + * columns that are not group-by keys + */ + private static @Nullable RexNode translateToGroupRefs( + RexNode expr, Project project, Aggregate aggregate) { + final List projects = project.getProjects(); + final GroupRefTranslator translator = new GroupRefTranslator(projects, aggregate); + final RexNode result = expr.accept(translator); + return translator.failed ? null : result; + } + + /** + * Shuttle that translates input refs to aggregate group key refs. + * + *

      For each column reference in the expression being examined: + * 1. If the expression is a direct pass-through of a project column, + * check if that project column is in the GROUP BY set + * 2. If the expression contains references to input columns, + * verify that those input columns are in the GROUP BY set + * 3. Map to the corresponding group key index in the aggregate + * + *

      This ensures the expression references only columns that are constant + * within each group. + */ + private static class GroupRefTranslator extends RexShuttle { + private final List projects; + private final Aggregate aggregate; + private boolean failed = false; + + GroupRefTranslator(List projects, Aggregate aggregate) { + this.projects = projects; + this.aggregate = aggregate; + } + + @Override public RexNode visitInputRef(RexInputRef inputRef) { + if (failed) { + return inputRef; + } + final int inputIndex = inputRef.getIndex(); + // Look for a project column that is a direct pass-through of this input. + // For example, if a project has SAL=[$5], and the expression references $5, + // we need to map it to the corresponding group key. + int projectOutputIndex = -1; + for (int i = 0; i < projects.size(); i++) { + final RexNode projExpr = projects.get(i); + if (projExpr instanceof RexInputRef + && ((RexInputRef) projExpr).getIndex() == inputIndex) { + projectOutputIndex = i; + break; + } + } + // The input column must be available through a project column that is in + // the GROUP BY set. If not found, the input is embedded in a computed + // expression, which means the optimization cannot proceed safely. + if (projectOutputIndex < 0 + || !aggregate.getGroupSet().get(projectOutputIndex)) { + failed = true; + return inputRef; + } + final int groupIndex = + aggregate.getGroupSet().asList().indexOf(projectOutputIndex); + return RexInputRef.of(groupIndex, aggregate.getRowType().getFieldList()); } - return ref; } /** Rule configuration. */ diff --git a/core/src/test/java/org/apache/calcite/test/AggregateReduceFunctionsOnGroupKeysRuleTest.java b/core/src/test/java/org/apache/calcite/test/AggregateReduceFunctionsOnGroupKeysRuleTest.java index 739ce1cb0b95..d4de2092aacc 100644 --- a/core/src/test/java/org/apache/calcite/test/AggregateReduceFunctionsOnGroupKeysRuleTest.java +++ b/core/src/test/java/org/apache/calcite/test/AggregateReduceFunctionsOnGroupKeysRuleTest.java @@ -55,12 +55,100 @@ private static RelOptFixture sql(String sql) { sql(sql).withRule(AGGREGATE_REDUCE_FUNCTIONS_ON_GROUP_KEYS).check(); } + @Test void testAggregateFunctionOfGroupByKeysNullExpression() { + String sql = "select comm, max(comm + 1) as max_plus\n" + + "from empnullables group by comm"; + sql(sql).withRule(AGGREGATE_REDUCE_FUNCTIONS_ON_GROUP_KEYS).check(); + } + + @Test void testAggregateFunctionOfGroupByKeysNullGroupKey() { + String sql = "select comm, max(comm) as comm_max\n" + + "from empnullables group by comm"; + sql(sql).withRule(AGGREGATE_REDUCE_FUNCTIONS_ON_GROUP_KEYS).check(); + } + @Test void testAggregateFunctionOfGroupByKeysNoChange() { String sql = "select sal, max(comm) as comm_max\n" + "from emp group by sal, deptno"; sql(sql).withRule(AGGREGATE_REDUCE_FUNCTIONS_ON_GROUP_KEYS).checkUnchanged(); } + @Test void testAggregateFunctionOfGroupByKeysDeterministicExpression() { + String sql = "select sal, max(sal + 1) as max_plus\n" + + "from emp group by sal, deptno"; + sql(sql).withRule(AGGREGATE_REDUCE_FUNCTIONS_ON_GROUP_KEYS).check(); + } + + @Test void testAggregateFunctionOfGroupByKeysUnaryMinus() { + String sql = "select sal, max(-sal) as max_neg\n" + + "from emp group by sal, deptno"; + sql(sql).withRule(AGGREGATE_REDUCE_FUNCTIONS_ON_GROUP_KEYS).check(); + } + + @Test void testAggregateFunctionOfGroupByKeysBinaryExpression() { + String sql = "select sal, max(sal * 2) as max_double\n" + + "from emp group by sal, deptno"; + sql(sql).withRule(AGGREGATE_REDUCE_FUNCTIONS_ON_GROUP_KEYS).check(); + } + + @Test void testAggregateFunctionOfGroupByKeysMultipleGroupKeys() { + String sql = "select sal, max(sal + deptno) as max_sum\n" + + "from emp group by sal, deptno"; + sql(sql).withRule(AGGREGATE_REDUCE_FUNCTIONS_ON_GROUP_KEYS).check(); + } + + @Test void testAggregateFunctionOfGroupByKeysNestedExpression() { + // Nested expressions like (sal + 1) * 2 can be optimized by mapping the + // input references (sal) to group key references. The shuttle translates + // all input refs in the expression to their corresponding group keys. + String sql = "select sal, max((sal + 1) * 2) as max_expr\n" + + "from emp group by sal, deptno"; + sql(sql).withRule(AGGREGATE_REDUCE_FUNCTIONS_ON_GROUP_KEYS).check(); + } + + @Test void testAggregateFunctionOfGroupByKeysWithCastWider() { + // Test case where a cast is needed because the group key type differs + // from the aggregate result type. Cast to a wider type (BIGINT) is safe. + // The rule should preserve the cast. + String sql = "select cast(sal as bigint) as sal_big, max(sal) as sal_max\n" + + "from emp group by sal"; + sql(sql).withRule(AGGREGATE_REDUCE_FUNCTIONS_ON_GROUP_KEYS).check(); + } + + @Test void testAggregateFunctionOfGroupByKeysWithCastNarrower() { + // Test case where a cast is needed and the type is narrower than the source. + // Casting to SMALLINT could potentially lose information if sal has larger values, + // but this is the user's explicit choice. The rule should still optimize and + // preserve the cast, allowing SQL semantics to handle any data loss. + String sql = "select cast(sal as smallint) as sal_small, max(sal) as sal_max\n" + + "from emp group by sal"; + sql(sql).withRule(AGGREGATE_REDUCE_FUNCTIONS_ON_GROUP_KEYS).check(); + } + + @Test void testAggregateFunctionWithMixedColumnsNoOptimization() { + // Negative test: expression references both group-by and non-group-by columns. + // The rule should NOT optimize because the expression is not constant within the group. + String sql = "select sal, max(sal + comm) as max_sum\n" + + "from emp group by sal, deptno"; + sql(sql).withRule(AGGREGATE_REDUCE_FUNCTIONS_ON_GROUP_KEYS).checkUnchanged(); + } + + @Test void testAggregateFunctionWithNonGroupByColumnNoOptimization() { + // Negative test: expression references only non-group-by columns. + // The rule should NOT optimize because the column is not in the GROUP BY set. + String sql = "select sal, max(comm) as comm_max\n" + + "from emp group by sal"; + sql(sql).withRule(AGGREGATE_REDUCE_FUNCTIONS_ON_GROUP_KEYS).checkUnchanged(); + } + + @Test void testAggregateFunctionWithMixedGroupByColumnsNoOptimization() { + // Negative test: expression contains GROUP BY column but also references + // a column from elsewhere that is not in GROUP BY. + String sql = "select sal, max(sal + empno) as max_sum\n" + + "from emp group by sal, deptno"; + sql(sql).withRule(AGGREGATE_REDUCE_FUNCTIONS_ON_GROUP_KEYS).checkUnchanged(); + } + @AfterAll static void checkActualAndReferenceFiles() { fixture().diffRepos.checkActualAndReferenceFiles(); } diff --git a/core/src/test/resources/org/apache/calcite/test/AggregateReduceFunctionsOnGroupKeysRuleTest.xml b/core/src/test/resources/org/apache/calcite/test/AggregateReduceFunctionsOnGroupKeysRuleTest.xml index e7eb9d5a927d..2083a969d724 100644 --- a/core/src/test/resources/org/apache/calcite/test/AggregateReduceFunctionsOnGroupKeysRuleTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/AggregateReduceFunctionsOnGroupKeysRuleTest.xml @@ -33,10 +33,102 @@ LogicalProject(SAL=[$0], SAL_MAX=[$2], SAL_MIN=[$3], SAL_AVG=[$4], SAL_VAL=[$5]) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -51,6 +143,48 @@ LogicalProject(SAL=[$0], COMM_MAX=[$2]) LogicalAggregate(group=[{0, 1}], COMM_MAX=[MAX($2)]) LogicalProject(SAL=[$5], DEPTNO=[$7], COMM=[$6]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +]]> + + + + + + + + + + + + + + + + + + + + + + @@ -70,10 +204,120 @@ LogicalProject(SAL=[$0], SAL_MAX=[$2], COMM_SUM=[$3]) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From a0817fd76bb4bf81bc849e5b780f5d1ece257058 Mon Sep 17 00:00:00 2001 From: Julian Hyde Date: Sat, 16 May 2026 16:31:23 -0700 Subject: [PATCH 275/562] [CALCITE-7532] Model usability --- .../calcite/config/CalciteSystemProperty.java | 18 ++ .../apache/calcite/model/ClassNameFilter.java | 185 ++++++++++++++++++ .../apache/calcite/model/ModelHandler.java | 43 +++- .../calcite/model/ModelHandlerTest.java | 144 ++++++++++++++ 4 files changed, 384 insertions(+), 6 deletions(-) create mode 100644 core/src/main/java/org/apache/calcite/model/ClassNameFilter.java diff --git a/core/src/main/java/org/apache/calcite/config/CalciteSystemProperty.java b/core/src/main/java/org/apache/calcite/config/CalciteSystemProperty.java index f38d405e3475..b0efb1a05d4d 100644 --- a/core/src/main/java/org/apache/calcite/config/CalciteSystemProperty.java +++ b/core/src/main/java/org/apache/calcite/config/CalciteSystemProperty.java @@ -455,6 +455,24 @@ public final class CalciteSystemProperty { public static final CalciteSystemProperty JOIN_SELECTOR_COMPACT_CODE_THRESHOLD = intProperty("calcite.join.selector.compact.code.threshold", 100); + /** + * Comma-separated patterns to add to the built-in denylist of class + * names that may not be loaded by reflection from a Calcite model + * (user-defined functions, custom schemas/tables, JDBC drivers, + * dialect factories, lattice statistic providers). + * + *

      Setting this property extends the built-in denylist; the + * built-in entries cannot be removed at runtime. + * + *

      Pattern syntax: a pattern ending in {@code "."} matches any class + * in that package or its sub-packages; otherwise the pattern matches a + * class name exactly. + * + * @see org.apache.calcite.model.ModelHandler + */ + public static final CalciteSystemProperty MODEL_CLASSES_DENIED = + stringProperty("calcite.model.classes.denied", ""); + private static CalciteSystemProperty booleanProperty(String key, boolean defaultValue) { // Note that "" -> true (convenient for command-lines flags like '-Dflag') diff --git a/core/src/main/java/org/apache/calcite/model/ClassNameFilter.java b/core/src/main/java/org/apache/calcite/model/ClassNameFilter.java new file mode 100644 index 000000000000..5a0a948cdcf4 --- /dev/null +++ b/core/src/main/java/org/apache/calcite/model/ClassNameFilter.java @@ -0,0 +1,185 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.model; + +import org.apache.calcite.config.CalciteSystemProperty; + +import com.google.common.collect.ImmutableList; + +import org.checkerframework.checker.nullness.qual.Nullable; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.function.Predicate; + +/** + * Filters class names that may be loaded by reflection from a Calcite + * model: user-defined functions, custom schemas, custom tables, JDBC + * drivers, dialect factories, and lattice statistic providers. + * + *

      {@link #standard()} returns the filter applied by + * {@link ModelHandler}: the built-in {@link #DEFAULT_DENYLIST} together + * with any patterns from + * {@link CalciteSystemProperty#MODEL_CLASSES_DENIED} (which + * extends the denylist). + * + *

      The denylist is a comma-separated pattern string. A pattern ending + * in {@code "."} matches any class in that package or its sub-packages; + * otherwise the pattern matches a class name exactly. Whitespace around + * commas is ignored. + * + *

      The denylist is not a sandbox. Any string passed to a + * {@code className}, {@code factory}, {@code jdbcDriver}, + * {@code sqlDialectFactory}, or {@code statisticProvider} field is + * classpath-equivalent; only accept models from trusted sources. + */ +class ClassNameFilter implements Predicate { + /** Built-in denylist: class-name patterns known to enable RCE when + * registered as UDFs, schema/table factories, JDBC drivers, dialect + * factories, or lattice statistic providers. */ + static final String DEFAULT_DENYLIST = "" + + "javax.naming.," + + "com.sun.jndi.," + + "java.lang.Runtime," + + "java.lang.ProcessBuilder," + + "java.lang.ProcessImpl," + + "java.lang.System," + + "java.lang.Class," + + "java.lang.reflect.," + + "java.lang.invoke.," + + "javax.script.," + + "bsh.," + + "groovy.," + + "org.codehaus.groovy.," + + "org.python.util.PythonInterpreter," + + "org.springframework.expression.," + + "org.apache.commons.collections.functors.," + + "org.apache.commons.collections4.functors.," + + "org.apache.commons.beanutils.," + + "com.sun.org.apache.xalan.internal.xsltc.trax.TemplatesImpl," + + "sun.misc.Unsafe," + + "jdk.internal."; + + /** Cache shared by all factory calls; filters are immutable and small, + * so identical denylist inputs need only be parsed once. */ + private static final ConcurrentMap CACHE = + new ConcurrentHashMap<>(); + + /** The standard filter, built once from the built-in denylist plus + * the {@link CalciteSystemProperty#MODEL_CLASSES_DENIED} extension. + * Initialized via {@link #of} so it shares the same cache. */ + private static final ClassNameFilter STANDARD = + of( + append(DEFAULT_DENYLIST, + CalciteSystemProperty.MODEL_CLASSES_DENIED.value())); + + private final ImmutableList denylist; + + private ClassNameFilter(String denylist) { + this.denylist = parse(denylist); + } + + /** Returns the standard filter used by {@link ModelHandler}: the + * built-in {@link #DEFAULT_DENYLIST} (extended by + * {@link CalciteSystemProperty#MODEL_CLASSES_DENIED}). */ + static ClassNameFilter standard() { + return STANDARD; + } + + /** Returns a filter parsed from a comma-separated denylist pattern + * string; may be empty. Filters are cached, so repeated calls with + * the same argument return the same instance. */ + static ClassNameFilter of(String denylist) { + return CACHE.computeIfAbsent(denylist, ClassNameFilter::new); + } + + /** Returns whether {@code classRef} is allowed (not on the denylist). + * A null reference is allowed. + * + *

      {@code classRef} may be a plain class name or the + * {@code "ClassName#STATIC_FIELD"} form accepted by + * {@link org.apache.calcite.avatica.AvaticaUtils#instantiatePlugin}; + * the field portion is stripped before matching. */ + @Override public boolean test(@Nullable String classRef) { + if (classRef == null) { + return true; + } + String className = stripFieldRef(classRef); + for (String pattern : denylist) { + if (matches(pattern, className)) { + return false; + } + } + return true; + } + + /** Throws {@link SecurityException} if {@code classRef} is on the + * denylist. A null reference is a no-op. */ + void check(@Nullable String classRef) { + if (classRef == null) { + return; + } + String className = stripFieldRef(classRef); + for (String pattern : denylist) { + if (matches(pattern, className)) { + throw new SecurityException("Class '" + className + + "' is rejected by the Calcite class-name filter " + + "(matches denylist pattern '" + pattern + "'). " + + "If this load is unintended, adjust the model; the " + + "denylist cannot be loosened at runtime."); + } + } + } + + private static String stripFieldRef(String classRef) { + int hash = classRef.indexOf('#'); + return hash >= 0 ? classRef.substring(0, hash) : classRef; + } + + private static boolean matches(String pattern, String className) { + if (pattern.endsWith(".")) { + return className.startsWith(pattern); + } + return className.equals(pattern); + } + + /** Returns the concatenation of two comma-separated pattern strings, + * inserting a comma if needed and tolerating empty inputs. */ + static String append(String first, String second) { + if (first.isEmpty()) { + return second; + } + if (second.isEmpty()) { + return first; + } + return first + "," + second; + } + + private static ImmutableList parse(String list) { + if (list.isEmpty()) { + return ImmutableList.of(); + } + ImmutableList.Builder b = ImmutableList.builder(); + for (String s : list.split(",")) { + String trimmed = s.trim(); + if (!trimmed.isEmpty()) { + b.add(trimmed); + } + } + return b.build(); + } +} diff --git a/core/src/main/java/org/apache/calcite/model/ModelHandler.java b/core/src/main/java/org/apache/calcite/model/ModelHandler.java index 5842619d1624..46661065cc28 100644 --- a/core/src/main/java/org/apache/calcite/model/ModelHandler.java +++ b/core/src/main/java/org/apache/calcite/model/ModelHandler.java @@ -82,14 +82,27 @@ public class ModelHandler { private final Deque> schemaStack = new ArrayDeque<>(); private final String modelUri; + private final ClassNameFilter classNameFilter; Lattice.@Nullable Builder latticeBuilder; Lattice.@Nullable TileBuilder tileBuilder; - @SuppressWarnings("method.invocation.invalid") + /** Creates a {@code ModelHandler} that uses the + * {@linkplain ClassNameFilter#standard() standard} class-name filter. */ public ModelHandler(SchemaPlus rootSchema, String uri) throws IOException { + this(rootSchema, uri, ClassNameFilter.standard()); + } + + /** Creates a {@code ModelHandler} that validates every class loaded + * by reflection from the model against {@code classNameFilter}. Use + * this to apply a stricter (or more permissive) filter than the + * standard one. */ + @SuppressWarnings("method.invocation.invalid") + public ModelHandler(SchemaPlus rootSchema, String uri, + ClassNameFilter classNameFilter) throws IOException { super(); this.modelUri = uri; this.rootSchema = rootSchema; + this.classNameFilter = classNameFilter; JsonRoot root; ObjectMapper mapper; if (uri.startsWith("inline:")) { @@ -128,7 +141,12 @@ public static void create(SchemaPlus schema, String functionName, } /** Creates and validates a {@link ScalarFunctionImpl}, and adds it to a - * schema. If {@code methodName} is "*", may add more than one function. + * schema, using the {@linkplain ClassNameFilter#standard() standard} + * class-name filter. Kept for backwards compatibility; prefer the + * filter-taking overload of {@code addFunctions}, which lets callers + * supply their own filter. + * + *

      If {@code methodName} is "*", may add more than one function. * * @param schema Schema to add to * @param functionName Name of function; null to derived from method name @@ -144,6 +162,16 @@ public static void create(SchemaPlus schema, String functionName, public static void addFunctions(SchemaPlus schema, @Nullable String functionName, List unusedPath, String className, @Nullable String methodName, boolean upCase) { + addFunctions(ClassNameFilter.standard(), schema, functionName, className, + methodName, upCase); + } + + /** Creates and validates a {@link ScalarFunctionImpl} and adds it to a + * schema, after asking {@code filter} to accept {@code className}. */ + public static void addFunctions(ClassNameFilter filter, SchemaPlus schema, + @Nullable String functionName, String className, + @Nullable String methodName, boolean upCase) { + filter.check(className); final Class clazz; try { clazz = Class.forName(className); @@ -275,6 +303,7 @@ private void populateSchema(JsonSchema jsonSchema, SchemaPlus schema) { public void visit(JsonCustomSchema jsonSchema) { try { final SchemaPlus parentSchema = currentMutableSchema("sub-schema"); + classNameFilter.check(jsonSchema.factory); final SchemaFactory schemaFactory = AvaticaUtils.instantiatePlugin(SchemaFactory.class, jsonSchema.factory); @@ -329,6 +358,7 @@ protected Map operandMap(@Nullable JsonSchema jsonSchema, public void visit(JsonJdbcSchema jsonSchema) { final SchemaPlus parentSchema = currentMutableSchema("jdbc schema"); + classNameFilter.check(jsonSchema.jdbcDriver); final DataSource dataSource = JdbcSchema.dataSource(jsonSchema.jdbcUrl, jsonSchema.jdbcDriver, @@ -340,6 +370,7 @@ public void visit(JsonJdbcSchema jsonSchema) { JdbcSchema.create(parentSchema, jsonSchema.name, dataSource, jsonSchema.jdbcCatalog, jsonSchema.jdbcSchema); } else { + classNameFilter.check(jsonSchema.sqlDialectFactory); SqlDialectFactory factory = AvaticaUtils.instantiatePlugin(SqlDialectFactory.class, jsonSchema.sqlDialectFactory); @@ -401,6 +432,7 @@ public void visit(JsonLattice jsonLattice) { latticeBuilder.rowCountEstimate(jsonLattice.rowCountEstimate); } if (jsonLattice.statisticProvider != null) { + classNameFilter.check(jsonLattice.statisticProvider); latticeBuilder.statisticProvider(jsonLattice.statisticProvider); } populateLattice(jsonLattice, latticeBuilder); @@ -421,6 +453,7 @@ private void populateLattice(JsonLattice jsonLattice, public void visit(JsonCustomTable jsonTable) { try { final SchemaPlus schema = currentMutableSchema("table"); + classNameFilter.check(jsonTable.factory); final TableFactory tableFactory = AvaticaUtils.instantiatePlugin(TableFactory.class, jsonTable.factory); @@ -518,10 +551,8 @@ public void visit(JsonFunction jsonFunction) { // "name" is not required - a class can have several functions try { final SchemaPlus schema = currentMutableSchema("function"); - final List path = - Util.first(jsonFunction.path, currentSchemaPath()); - addFunctions(schema, jsonFunction.name, path, jsonFunction.className, - jsonFunction.methodName, false); + addFunctions(classNameFilter, schema, jsonFunction.name, + jsonFunction.className, jsonFunction.methodName, false); } catch (Exception e) { throw new RuntimeException("Error instantiating " + jsonFunction, e); } diff --git a/core/src/test/java/org/apache/calcite/model/ModelHandlerTest.java b/core/src/test/java/org/apache/calcite/model/ModelHandlerTest.java index 48a546b5db8b..4cebda1454d4 100644 --- a/core/src/test/java/org/apache/calcite/model/ModelHandlerTest.java +++ b/core/src/test/java/org/apache/calcite/model/ModelHandlerTest.java @@ -21,15 +21,25 @@ import org.apache.calcite.schema.lookup.LikePattern; import org.apache.calcite.util.Sources; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import org.junit.jupiter.api.Test; import java.io.IOException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.util.Properties; import java.util.Set; +import java.util.function.Predicate; +import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.hamcrest.CoreMatchers.sameInstance; import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; import static java.util.Objects.requireNonNull; @@ -56,4 +66,138 @@ public class ModelHandlerTest { assertThat(h.defaultSchemaName(), is("SCOTT")); } + @Test void testDenyUdfClass() { + SchemaPlus root = CalciteSchema.createRootSchema(false, false).plus(); + SecurityException e = + assertThrows(SecurityException.class, () -> + ModelHandler.addFunctions(root, "lookup", ImmutableList.of(), + "javax.naming.InitialContext", "doLookup", false)); + assertThat(e.getMessage(), containsString("javax.naming.")); + assertThat(e.getMessage(), containsString("denylist")); + } + + @Test void testCustomFilterPassedToConstructor() { + // A ModelHandler built with a stricter filter must reject classes + // its filter denies, even ones the standard filter would allow. + SchemaPlus root = CalciteSchema.createRootSchema(false, false).plus(); + // java.lang.String is not in the standard denylist; the custom + // filter denies the whole java.lang. package. + ClassNameFilter strict = ClassNameFilter.of("java.lang."); + String model = "inline:{" + + " version: '1.0'," + + " defaultSchema: 'X'," + + " schemas: [ {" + + " name: 'X'," + + " functions: [ {" + + " name: 'F'," + + " className: 'java.lang.String'" + + " } ]" + + " } ]" + + "}"; + Throwable e = + assertThrows(RuntimeException.class, () -> + new ModelHandler(root, model, strict)); + while (e != null && !(e instanceof SecurityException)) { + e = e.getCause(); + } + assertThat("expected SecurityException in chain", e, notNullValue()); + assertThat(e.getMessage(), containsString("java.lang.")); + } + + @Test void testAddFunctionsWithExplicitFilterDeniesClass() { + // The filter-taking overload of addFunctions must reject any class + // its filter denies. + SchemaPlus root = CalciteSchema.createRootSchema(false, false).plus(); + SecurityException e = + assertThrows(SecurityException.class, () -> + ModelHandler.addFunctions(ClassNameFilter.standard(), root, + "lookup", "javax.naming.InitialContext", "doLookup", false)); + assertThat(e.getMessage(), containsString("javax.naming.")); + } + + @Test void testDenyFactory() { + String model = "inline:{" + + " version: '1.0'," + + " defaultSchema: 'X'," + + " schemas: [ {" + + " name: 'X'," + + " type: 'custom'," + + " factory: 'javax.naming.InitialContext'" + + " } ]" + + "}"; + Properties info = new Properties(); + info.setProperty("model", model); + Exception e = + assertThrows(Exception.class, () -> { + try (Connection ignored = + DriverManager.getConnection("jdbc:calcite:", info)) { + // unreachable + } + }); + Throwable cause = e; + while (cause != null && !(cause instanceof SecurityException)) { + cause = cause.getCause(); + } + assertThat("expected a SecurityException in the chain", + cause != null, is(true)); + assertThat(requireNonNull(cause, "cause").getMessage(), + containsString("javax.naming.")); + } + + @Test void testStaticFieldRefIsCheckedAgainstClass() { + // Avatica accepts "ClassName#FIELD" for plugin references; the filter + // must reject the class portion regardless of which field is named. + SecurityException e = + assertThrows(SecurityException.class, () -> + ClassNameFilter.standard().check( + "java.lang.Runtime#anything")); + assertThat(e.getMessage(), containsString("java.lang.Runtime")); + } + + @Test void testLegitFactoryClassIsAllowed() { + // Sanity: a class outside the denylist passes (no exception). + ClassNameFilter.standard().check( + "org.apache.calcite.adapter.jdbc.JdbcSchema$Factory"); + ClassNameFilter.standard().check( + "org.apache.calcite.schema.impl.AbstractSchema$Factory"); + ClassNameFilter.standard().check( + "org.apache.calcite.adapter.jdbc.JdbcSchema$Factory#INSTANCE"); + } + + @Test void testPredicateContract() { + // ClassNameFilter implements Predicate: true means "allowed". + Predicate filter = ClassNameFilter.standard(); + assertThat(filter.test(null), is(true)); + assertThat(filter.test("javax.naming.InitialContext"), is(false)); + assertThat(filter.test("java.lang.Runtime#getRuntime"), is(false)); + assertThat( + filter.test( + "org.apache.calcite.adapter.jdbc.JdbcSchema$Factory"), is(true)); + } + + @Test void testFactoryMethodsCacheInstances() { + // standard() returns a single cached instance. + assertThat(ClassNameFilter.standard(), + sameInstance(ClassNameFilter.standard())); + // of() returns the same instance for equal inputs. + ClassNameFilter a = ClassNameFilter.of("com.evil."); + ClassNameFilter b = ClassNameFilter.of("com.evil."); + assertThat(a, sameInstance(b)); + // Different inputs produce different instances. + ClassNameFilter c = ClassNameFilter.of("com.evil.,com.example."); + assertThat(a, not(sameInstance(c))); + // The cached filter behaves as configured. + assertThat(a.test("com.evil.Payload"), is(false)); + assertThat(a.test("javax.naming.InitialContext"), is(true)); + } + + @Test void testAppendCombinesPatternStrings() { + // The denylist extension wired into ClassNameFilter.standard() works + // by string concatenation through ClassNameFilter.append. + assertThat(ClassNameFilter.append("a.,b.", ""), is("a.,b.")); + assertThat(ClassNameFilter.append("", "c.,d."), is("c.,d.")); + assertThat(ClassNameFilter.append("a.", "b."), is("a.,b.")); + assertThat(ClassNameFilter.append("", ""), is("")); + } + } From f1bc8e6320f0e7402f2c387d036cce845c42b108 Mon Sep 17 00:00:00 2001 From: Stamatis Zampetakis Date: Wed, 27 May 2026 17:45:30 +0200 Subject: [PATCH 276/562] [CALCITE-7561] Upgrade OWASP plugin from 6.1.6 to 12.2.2 Due to NVD API changes the old versions of the plugin are unusuable and upgrade to 12.2.2 is mandatory. New versions require JDK11+ so the plugin is no longer active when older versions are used. --- build.gradle.kts | 4 +++- gradle.properties | 2 +- settings.gradle.kts | 4 +++- site/_docs/howto.md | 11 ++++++----- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index be7569e17fef..a66e7107b47e 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -47,7 +47,9 @@ plugins { id("de.thetaphi.forbiddenapis") apply false id("net.ltgt.errorprone") apply false id("com.github.vlsi.jandex") apply false - id("org.owasp.dependencycheck") + if (JavaVersion.current() >= JavaVersion.VERSION_11) { + id("org.owasp.dependencycheck") + } id("com.github.johnrengelman.shadow") apply false id("org.sonarqube") // IDE configuration diff --git a/gradle.properties b/gradle.properties index 10e033c12386..c874f3ecb2d9 100644 --- a/gradle.properties +++ b/gradle.properties @@ -55,7 +55,7 @@ net.ltgt.errorprone.version=1.3.0 me.champeau.jmh.version=0.7.2 org.jetbrains.gradle.plugin.idea-ext.version=1.4.1 org.nosphere.apache.rat.version=0.8.1 -org.owasp.dependencycheck.version=6.1.6 +org.owasp.dependencycheck.version=12.2.2 org.sonarqube.version=3.5.0.2730 com.gradle.develocity.version=3.18.2 com.gradle.common-custom-user-data-gradle-plugin.version=2.0.2 diff --git a/settings.gradle.kts b/settings.gradle.kts index 67695f630c54..fbc3502a7c29 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -39,7 +39,9 @@ pluginManagement { idv("net.ltgt.errorprone") idv("org.jetbrains.gradle.plugin.idea-ext") idv("org.nosphere.apache.rat") - idv("org.owasp.dependencycheck") + if (JavaVersion.current() >= JavaVersion.VERSION_11) { + idv("org.owasp.dependencycheck") + } idv("org.sonarqube") kotlin("jvm") version "kotlin".v() } diff --git a/site/_docs/howto.md b/site/_docs/howto.md index 6c66d40ccbf7..9aa5fbf161ea 100644 --- a/site/_docs/howto.md +++ b/site/_docs/howto.md @@ -728,7 +728,12 @@ Before you start: * Send an email to [dev@calcite.apache.org](mailto:dev@calcite.apache.org) notifying that RC build process is starting and therefore `main` branch is in code freeze until further notice. * Set up signing keys as described above. -* Make sure you are using JDK 8. (Compiling with JDK 21 causes +* Generate a report of vulnerabilities that occur among dependencies, + using `./gradlew dependencyCheckUpdate dependencyCheckAggregate`. + Report to [private@calcite.apache.org](mailto:private@calcite.apache.org) + if new critical vulnerabilities are found among dependencies. + The task requires a JDK 11 or later so it doesn't appear when using older versions. +* Make sure you are using JDK 8 for all subsequent steps. (Compiling with JDK 21 causes [[CALCITE-6616](https://issues.apache.org/jira/browse/CALCITE-6616)].) * Check that `README` and `site/_docs/howto.md` have the correct version number. * Check that `site/_docs/howto.md` has the correct Gradle version. @@ -737,10 +742,6 @@ Before you start: * Make sure build and tests succeed * Make sure that `./gradlew javadoc` succeeds (i.e. gives no errors; warnings are OK) -* Generate a report of vulnerabilities that occur among dependencies, - using `./gradlew dependencyCheckUpdate dependencyCheckAggregate`. - Report to [private@calcite.apache.org](mailto:private@calcite.apache.org) - if new critical vulnerabilities are found among dependencies. * Decide the supported configurations of JDK, operating system and Guava. These will probably be the same as those described in the release notes of the previous release. Document them in the release From 9f2731129debdac06767d627527ddb72aa3d48d8 Mon Sep 17 00:00:00 2001 From: Stamatis Zampetakis Date: Fri, 22 May 2026 09:05:44 +0200 Subject: [PATCH 277/562] [CALCITE-7544] Release Calcite 1.42.0 1. Update README and howto to 1.42.0 2. Add release notes and contributots list 3. Update mailmap based on new contributions --- .mailmap | 11 ++ README | 2 +- site/_docs/history.md | 374 ++++++++++++++++++++++++++++++++++++++++-- site/_docs/howto.md | 4 +- 4 files changed, 372 insertions(+), 19 deletions(-) diff --git a/.mailmap b/.mailmap index a0784207f8ad..aa46591c165d 100644 --- a/.mailmap +++ b/.mailmap @@ -25,6 +25,7 @@ Alessandro Solimando <18898964+asolimando@users.noreply.github.com> Alessandro Solimando Alessandro Solimando Alessandro Solimando +Alexis Cubilla Ali Mansour Amir Gajst Anton Kovalevsky @@ -46,6 +47,7 @@ Daniel Henneberger Danny Chan Danny Chan Darion Yaphet +Darpan Lunagariya David Handermann Dhirenda Gautam Divyanshu Srivastava @@ -73,10 +75,12 @@ Heng Xiao Hequn Cheng Hong Shen (沈洪) Hongyu Guo +Hongyu Guo Hongyu Guo Hongze Zhang Hsuan-Yi Chu Ian Bertolacci +Issac Garcia Jacky Lau # aka Yong Liu Jacky Lau Jacky Woo @@ -101,6 +105,7 @@ Julian Hyde Julian Hyde Julian Hyde Julian Hyde +Keshav Katkar Kevin Liew Kevin Liew Khawla Mouhoubi @@ -113,6 +118,7 @@ Krisztian Kasa Lei Shen LeoWangLZ # aka Rheet Wong? Liao Xintao +Lincoln Lee Lincoln Lee Liya Fan Louis Kang # aka LM Kang @@ -153,6 +159,7 @@ suibianwanwan <1597226206@qq.com> suibianwanwan Taras Ledkov Ted Xu (少杰) +Terran Venki Korukanti Venki Korukanti Viggo Chen @@ -163,9 +170,11 @@ Wang Yanlin Wang Zhao wangdiao Wei Zhou +Weihua Zhang <745778074@qq.com> Weijie Wu Wenhui Tang Wenrui Meng +Wenzhuang Zhu Xiaochen Zhou <598457447@qq.com> Xiaochen Zhou Xiaogang Zhou @@ -182,8 +191,10 @@ YiwenWu YiwenWu Yu Xu <11161569@vivo.com> Yu Xu <1206332514@qq.com> +Yu Xu Zhe Hu Zhen Chen +Zhen Wang <643348094@qq.com> Zhen Wang Zhen Wang Zhengqiang Duan diff --git a/README b/README index efc9b7e3d4a4..981a9713f4ed 100644 --- a/README +++ b/README @@ -1,4 +1,4 @@ -Apache Calcite release 1.41.0 +Apache Calcite release 1.42.0 This is a source distribution of Apache Calcite. diff --git a/site/_docs/history.md b/site/_docs/history.md index 8a7da2a4f31e..149ef7d38e34 100644 --- a/site/_docs/history.md +++ b/site/_docs/history.md @@ -30,10 +30,10 @@ Downloads are available on the + +## 1.42.0 / 2026-06-01 +{: #v1-42-0} + +This release comes 7 months after [1.41.0](#v1-41-0), +contains contributions from 39 contributors, and resolves 241 issues. + +Contributors to this release: +Alessandro Solimando, +Alexis Cubilla, +big face cat, +Cancai Cai, +Darpan Lunagariya, +Diveyam Mishra, +Dmitry Sysolyatin, +Dongsheng He, +Heng Qian, +Hongyu Guo, +Ian Bertolacci, +Issac Garcia, +Jinkun Liu, +Joseph Grogan, +Julian Hyde, +Keshav Katkar, +khanhkhanhlele, +krooswu, +Lincoln Lee, +Mihai Budiu, +Niels Pardon, +Ruben Quesada Lopez, +Sean Broeder, +Sergey Nuyanzin, +Silun Dong, +Soumyakanti Das, +Stamatis Zampetakis (Release Manager), +Tamas Mate, +Terran, +Thomas Rebele, +TJ Banghart, +Weihua Zhang, +Wenzhuang Zhu, +Xiong Duan, +Yash Limbad, +Yu Xu, +Zhen Chen, +Zhen Wang, +zhuyufeng0809, +zzwqqq. + +Compatibility: This release is tested on Linux, macOS, Microsoft Windows; +using JDK/OpenJDK versions 8 to 24; +Guava versions 21.0 to 33.4.8-jre; +other software versions as specified in gradle.properties. + + +#### Breaking Changes +{: #breaking-1-42-0} +* [CALCITE-6942] +Rename the method `decorrelateFetchOneSort` to `decorrelateSortWithRowNumber`. * [CALCITE-7301] Prior to this change, most `SqlNode`s in the `org.apache.calcite.sql.ddl` package could not be unparsed when created with `SqlOperator#createCall`. To fix this, those `SqlNode`s now implement their own `SqlOperator`. `SqlNode#getOperandList()` now returns all operands required by these operators; the number and order may differ from before. The same applies to `SqlBabelCreateTable` and `SqlUnpivot`. - -* [CALCITE-6942] -Rename the method `decorrelateFetchOneSort` to `decorrelateSortWithRowNumber`. +* [CALCITE-7351] Make `getMaxNumericScale()` and `getMaxNumericPrecision()` final +* [CALCITE-7393] + `RelDataTypeImpl.digest` is deprecated. We recommend using `RelDataTypeImpl.innerDigest` instead. + See system property `CalciteSystemProperty.DISABLE_GENERATE_REL_DATA_TYPE_DIGEST_STRING`. +* [CALCITE-7410] + Changes the type of the `WINDOW_START` and `WINDOW_END` columns for + the table functions `HOP`, `TUMBLE`, `SESSION` to match the original + type of the timestamp column. These types used to be hardwired to + `TIMESTAMP(3)`. #### New features {: #new-features-1-42-0} +* [CALCITE-4460] Support custom delimiter when parsing CSV tables +* [CALCITE-5347] Add `SELECT ... BY`, a syntax extension that is shorthand for `GROUP BY` and `ORDER BY` +* [CALCITE-5733] Simplify `a = ARRAY[1,2] AND a = ARRAY[2,3]` to `false` +* [CALCITE-5740] Add `AggToSemiJoinRule` to transform aggregate to semijoin +* [CALCITE-5787] New `RelMdInputFieldsUsed` API to track the usage of input fields +* [CALCITE-6066] Add `HYPOT` function (enabled in Spark library) +* [CALCITE-7031] Implement the general decorrelation algorithm (Neumann & Kemper) +* [CALCITE-7295] `RexSimplify` should simplify a division with a `NULL` argument +* [CALCITE-7310] Support the syntax `SELECT * EXCLUDE(columns)` +* [CALCITE-7311] Support the syntax `ROW(*)` to create a nested `ROW` type with all columns +* [CALCITE-7337] Add `AGE` function (enabled in PostgreSQL library) +* [CALCITE-7362] Add rule to transform `WHERE` clauses into filtered aggregates +* [CALCITE-7406] Add `ABS` function (enabled in MongoDB library) +* [CALCITE-7413] Add `CONCAT` and `SUBSTRING` function (enabled in MongoDB library) +* [CALCITE-7422] Support large plan optimization mode for HepPlanner +* [CALCITE-7428] Support `REGEXP` function in Hive library +* [CALCITE-7448] Add support for ':' variant path access syntax +* [CALCITE-7498] Enhance parser to allow hints with assignments mixing literals and identifiers +* [CALCITE-7523] Support the syntax `SELECT * REPLACE(expr as column)` + #### Dependency version upgrade {: #dependency-1-42-0} +* [CALCITE-7259] Drop commons-lang3 dependency +* [CALCITE-7290] Update json-path to 2.10.0 +* [CALCITE-7307] joou-java6-0.9.4 dependency conflicts with Java 9+ and JPMS +* [CALCITE-7458] Upgrade Jackson to 2.18.6 due to CVE +* [CALCITE-7521] Upgrade Avatica 1.28.0 + #### Bug-fixes, API changes and minor enhancements {: #fixes-1-42-0} +* [CALCITE-2152] SQL parser unable to parse SQL with nested joins produced by `RelToSqlConverter` +* [CALCITE-2274] Filter predicates aren't inferred while using dynamic star in subquery +* [CALCITE-3128] Joining two tables producing only `NULLs` will return 0 rows +* [CALCITE-4525] Pull up predicate will lose some predicates when project contains same `RexInputRef` +* [CALCITE-4645] In Elasticsearch adapter, a range predicate should be translated to a range query +* [CALCITE-4765] Complex correlated `EXISTS` sub-queries used as scalar subqueries can return wrong results +* [CALCITE-4813] `ANY_VALUE` assumes that arguments should be comparable +* [CALCITE-4868] Elasticsearch adapter fails if `GROUP BY` is followed by `ORDER BY` +* [CALCITE-5093] Quantified comparison operators (e.g. `ANY`) should support `ARRAY` arguments +* [CALCITE-5132] Scalar `IN` subquery returns `UNKNOWN` instead of `FALSE` when key is partially `NULL` +* [CALCITE-5223] `AdjustProjectForCountAggregateRule` throws `ArrayIndexOutOfBoundsException` +* [CALCITE-5390] `RelDecorrelator` throws `NullPointerException` +* [CALCITE-5465] Rule of `AGGREGATE_EXPAND_DISTINCT_AGGREGATES` produces an incorrect plan when SQL has distinct agg-call with `ROLLUP` +* [CALCITE-5597] `SqlToRelConverter` generates wrong plan for `SELECT DISTINCT` query with `ORDER BY` +* [CALCITE-5832] `CyclicMetadataException` thrown in complex `JOIN` +* [CALCITE-6176] `JOIN_SUB_QUERY_TO_CORRELATE` rule incorrectly handles `EXISTS` in `LEFT JOIN ON` clause +* [CALCITE-6291] Support converting `ArrowTable` to `Queryable` +* [CALCITE-6298] Support `UNION` in Arrow adapter +* [CALCITE-6300] Function `MAP_VALUES/MAP_KEYS` gives exception when `mapValueType` and `mapKeyType` not equals map Biggest mapKeytype or mapValueType +* [CALCITE-6636] Support CNF condition in Arrow adapter +* [CALCITE-6646] Support `TIMESTAMP` data type in Arrow adapter +* [CALCITE-6681] `NullPointerException` in `ProjectCorrelateTransposeRule` +* [CALCITE-6757] Elasticsearch adapter returns wrong result when aggregating sub-query with aggregation +* [CALCITE-6829] `MSSQL` dialect incorrectly translates of `SELECT TRUE` +* [CALCITE-6942] Decorrelate sub-queries with `LIMIT 1` and `OFFSET` +* [CALCITE-6963] `SqlToRelConverter` fails when subquery is in join on clause +* [CALCITE-6968] `SqlUpdate#getOperandList` omits sourceSelect operand +* [CALCITE-7057] `NPE` when decorrelating query containing nested correlated subqueries +* [CALCITE-7085] `JOIN USING` with unqualified common column fails in a conformance where `allowQualifyingCommonColumn` is false (e.g. Oracle, Presto) +* [CALCITE-7087] SQLite does not support `RIGHT/FULL JOIN` until version 3.39.0 +* [CALCITE-7145] `RexSimplify` should not simplify `IS NULL(10/0)` +* [CALCITE-7187] Java UDF byte arrays cannot be mapped to `VARBINARY` +* [CALCITE-7196] Create an optimization pass which can convert some cases of `Correlate` + `Unnest` to `Unnest` +* [CALCITE-7207] Semi Join `RelNode` cannot be translated into correct `MySQL` SQL +* [CALCITE-7208] Allow downstream projects implement `CREATE OR ALTER` +* [CALCITE-7251] `SEARCH` and `WINDOW` calls should carry source position information +* [CALCITE-7254] Add rule for sharing trivially equivalent RelNodes within Combine +* [CALCITE-7256] Make the fields of `SqlTableRef` public +* [CALCITE-7257] Subqueries cannot be decorrelated if join condition contains `RexFieldAccess` +* [CALCITE-7258] `RelBuilder.filter` should throw if the condition is not `BOOLEAN` +* [CALCITE-7266] Optimize the "well-known count bug" correction +* [CALCITE-7268] `SqlToRelConverter` throws exception if lambda contains `IN` +* [CALCITE-7272] Subqueries cannot be decorrelated if have set op +* [CALCITE-7273] `CoreRules.JOIN_REDUCE_EXPRESSIONS` throws when applied to an `ASOF JOIN` +* [CALCITE-7274] `RexFieldAccess` has wrong index when use trim unused fields +* [CALCITE-7276] `SqlToRelConverter` throws exception for `UPDATE` if identifier expansion disabled +* [CALCITE-7279] `ClickHouse` dialect should wrap nested joins with explicit column aliases +* [CALCITE-7281] Deprecate `NullPolicy.ANY` in favor of `NullPolicy.SEMI_STRICT` +* [CALCITE-7287] In `simplifyLike`, the `makeLiteral` call does not preserve the `RelDataType` +* [CALCITE-7289] `SELECT NULL` sub-query throwing exception +* [CALCITE-7291] Verify that the same exception is thrown for the original and simplified expression in `RexSimplify#verify` +* [CALCITE-7293] `MAP` constructor cannot handle `VARIANT` values that need casts +* [CALCITE-7296] `RexSimplify` should not simplify `IS NULL(CAST(10/0 AS BIGINT))` +* [CALCITE-7297] The result is incorrect when the `GROUP BY` key in a subquery is a `RexFieldAccess` +* [CALCITE-7302] Infinite loop with `JoinPushTransitivePredicatesRule` +* [CALCITE-7303] Sub-queries cannot be decorrelated if filter condition have multi `CorrelationId` +* [CALCITE-7305] Subqueries in `ASOF JOIN MATCH_CONDITION` cause an assertion failure +* [CALCITE-7309] Position is unparsed incorrectly for `ClickHouseSqlDialect` +* [CALCITE-7312] Alias is not auto generated for `LATERAL TABLE` +* [CALCITE-7315] Support `LEFT_MARK` type for hash join in enumerable convention +* [CALCITE-7316] The `POSITION` function in SQLite is missing the `FROM` clause +* [CALCITE-7317] `SubQueryRemoveRule` should skip NULL-safety checks for `IN` subqueries when both the keys and the subquery columns are `NOT NULL` +* [CALCITE-7318] Execution fails when the `JOIN ON` condition contains references to columns from both the left and right sides +* [CALCITE-7319] `FILTER_INTO_JOIN` rule loses correlation variable context in HepPlanner +* [CALCITE-7319] `FILTER_INTO_JOIN` rule loses correlation variable context in HepPlanner +* [CALCITE-7320] `AggregateProjectMergeRule` throws `AssertionError` when `Project` maps multiple grouping keys to the same field +* [CALCITE-7321] `FilesTableFunction` throws `NumberFormatException` on macOS with GNU stat installed +* [CALCITE-7322] The `POSITION` function in `MySQL` is missing the `FROM` clause +* [CALCITE-7323] Result of cast `Number` to `Boolean` is not correct +* [CALCITE-7325] Incorrect `VARIANT` signatures in `SqlItemOperator` +* [CALCITE-7326] `FILTER_CORRELATE` rule loses correlation variable context in HepPlanner +* [CALCITE-7327] Support `IS NOT DISTINCT FROM` as equi condition of hash join +* [CALCITE-7330] `AggregateCaseToFilterRule` should not be applied on aggregate functions that don't skip `NULL` inputs +* [CALCITE-7331] Support the alias form `SELECT * EXCEPT` for `SELECT * EXCLUDE` +* [CALCITE-7332] `SELECT * EXCLUDE` list should error when it excludes all columns +* [CALCITE-7335] `RelToSqlConverter` generate sql containing Scala sub-queries includes redundant parentheses +* [CALCITE-7336] `RelFieldTrimmer` generates an incorrect plan when handling correlated sub-query within `Filter` or `Join` condition +* [CALCITE-7338] Window hints are not propagated to window rel nodes +* [CALCITE-7339] Most classes in `SqlDdlNodes` use an incorrect `SqlCallFactory` +* [CALCITE-7343] `RelToSqlConverter` generate wrong SQL when scalar correlated sub-query in `Project` +* [CALCITE-7346] Prevent overflow in metadata row-count when `LIMIT/OFFSET` literal exceeds `Long` range +* [CALCITE-7347] `UNKNOWN` type inferred for array element type +* [CALCITE-7348] Remove redundant extraction correlation variables when Trim `Project` fields +* [CALCITE-7349] Upgrade the types of `FETCH` and `OFFSET` in `Sort` to `BigDecimal` +* [CALCITE-7350] Missing `allowEmptyOutputFromRewrite` parameter in `TopDownGeneralDecorrelator.unnestInternal` +* [CALCITE-7352] Incorrect `SqlLibrary` enum value used in ClickHouse SQL test +* [CALCITE-7355] `RelToSqlConverter` throws exception when the join condition contains a correlated subquery +* [CALCITE-7356] The `MARK JOIN` generated by `TopDownGeneralDecorrelator` needs to be adapted to `RelFieldTrimmer` +* [CALCITE-7357] Add runtime implementation for `IS DISTINCT FROM` operator +* [CALCITE-7358] Casts involving `MAP` and `ROW` types cause compile-time exceptions +* [CALCITE-7359] Incorrect result for array comparison with `ANY` operator +* [CALCITE-7360] The meaning of negation for unsigned numbers is not defined +* [CALCITE-7363] Improve error message for `ASOF JOIN` +* [CALCITE-7365] `RelMdRowCount` ignores `estimateRowCoun` overrides in `SingleRel` subclasses +* [CALCITE-7366] `AssertionError` in `RexLiteral.valueMatchesType` for `CAST` to `MAP` type +* [CALCITE-7367] `NULLS FIRST` throws `ClassCastException` when sorting arrays +* [CALCITE-7368] Validator accepts `CAST(INT TO BINARY)`, but the runtime does not implement them +* [CALCITE-7369] `ProjectToWindowRule` loses column alias when optimizing `OVER` window queries +* [CALCITE-7370] Trailing dot is not removed when normalizing timestamp strings +* [CALCITE-7372] `TopDownGeneralDecorrelator` generates invalid SQL when the JOIN condition has correlation +* [CALCITE-7373] `FILTER_INTO_JOIN` should not push `Filter` into a `Join` when the `Filter` contains non-deterministic function +* [CALCITE-7374] `NULLS LAST` throws `ClassCastException` when sorting arrays +* [CALCITE-7375] `ProjectWindowTransposeRule` does not correctly adjust column indices in window bounds +* [CALCITE-7377] Validator should reject a `DESCRIPTOR` in a table function when it is not an identifier +* [CALCITE-7378] `RelToSqlConverter` generates incorrect column reference when `hasImplicitTableAlias` is true +* [CALCITE-7379] LHS correlated variables are shadowed by nullable RHS outputs in `LEFT JOIN` +* [CALCITE-7382] Wrong results when using `TopDownGeneralDecorrelator` with `LIMIT 1` sub-query +* [CALCITE-7385] Support `LEFT_MARK` type for nested loop join in enumerable convention +* [CALCITE-7386] Wrong results after decorrelating query with `MEASURE` +* [CALCITE-7388] Redis Adapter operand config should not support empty string +* [CALCITE-7389] `PruneJoinSingleValue` rule causes type mismatch in `EXISTS` +* [CALCITE-7391] `AssertionError` when applying `FILTER_REDUCE_EXPRESSIONS` on expression `WHERE 123 IN (SELECT NULL FROM emps)` +* [CALCITE-7392] Unable to implement `EnumerableCollect` for SQL queries with `UNNEST` +* [CALCITE-7393] Use `RelDataTypeDigest` in composite types to improve memory footprint and computational latency +* [CALCITE-7394] Nested sub-query with multiple levels of correlation returns incorrect results +* [CALCITE-7395] `ProjectMergeRule` incorrectly merges projects with correlation variables +* [CALCITE-7396] `PruneEmptyRules` does not support `LEFT_MARK JOIN` +* [CALCITE-7397] `AssertionError` in simplifying join condition when creating `LEFT MARK JOIN` +* [CALCITE-7398] Incorrect int `CAST` in `VariantNonNull#cast` for `BIGINT` +* [CALCITE-7400] `PruneJoinSingleValue` rule causes type mismatch in `IN` +* [CALCITE-7401] Multi-level correlated subqueries cause an out-of-range error in the `TopDownGeneralDecorrelator` +* [CALCITE-7402] `AssertionError` type mismatch when using `TopDownGeneralDecorrelator` with nested correlated sub-query +* [CALCITE-7403] Missing `ENUMERABLE` Convention for `LogicalConditionalCorrelate` +* [CALCITE-7404] Syntax error in MongoDB adapter due to incorrect field alias +* [CALCITE-7408] `URL_ENCODE/URL_DECODE` is unparsed incorrectly for `ClickHouseSqlDialect` +* [CALCITE-7409] Merge `JOIN` condition cannot contain `IS NOT DISTINCT FROM` +* [CALCITE-7410] `TIMESTAMP` type for `TUMBLE` and `HOP` is hardwired to `TIMESTAMP(3)` +* [CALCITE-7411] When a `SCALAR_QUERY` in `PROJECT` contains correlated variables execution fails using `TopDownGeneralDecorrelator` +* [CALCITE-7412] Redis test failed on higher versions of macOS +* [CALCITE-7414] Incorrect mapping of `CorDef` after decorrelating a `Join` in `TopDownGeneralDecorrelator` +* [CALCITE-7415] `CalciteCatalogReader.lookupOperatorOverloads` keeps original function identifier casing instead of resolved schema-path casing +* [CALCITE-7416] Add `firedRulesCache` for `HepPlanner` +* [CALCITE-7417] Add a large plan benchmark for `HepPlanner` +* [CALCITE-7418] `SqlOverlapsOperator` does not reject some illegal comparisons (e.g., `TIME` vs `DATE`) +* [CALCITE-7423] `Setop` subquery without correlated variables triggers `NullPointerException` during decorrelation +* [CALCITE-7425] Correct the logical inverse of `SqlBetweenOperator` +* [CALCITE-7427] Query with `ORDER BY NULL` throws `NoSuchMethodException: compareNullsLast` +* [CALCITE-7429] Unable to implement `EnumerableMinus` for SQL queries with `EXCEPT` +* [CALCITE-7431] `RelTraitSet#getTrait` mishandles `RelCompositeTrait` +* [CALCITE-7432] `NumberFormatException` when convert `NaN` literal to SQL +* [CALCITE-7433] Invalid unparse for `CAST` to `MAP` type in Spark +* [CALCITE-7434] `AssertionError` in new decorrelation algorithm caused by `FilterJoinRule` omitting `variablesSet` +* [CALCITE-7435] Window functions should allow `ORDER BY` fields of type `INTERVAL` +* [CALCITE-7437] Type coercion for quantifier operators is incomplete +* [CALCITE-7441] `AggregateFilterToFilteredAggregateRule` fails when `WHERE` condition is nullable +* [CALCITE-7442] Correlated variable has wrong index inside subquery +* [CALCITE-7443] Incorrect simplification for large interval +* [CALCITE-7447] `RelRoot.project` adds `Project` for DDL nodes +* [CALCITE-7450] `ValuesReduceRule` incorrectly drops tuples when filter condition is irreducible +* [CALCITE-7456] Enable the `TRY_CAST` function to support the MSSQL dialect +* [CALCITE-7457] `VALUES` and `SELECT` produce different validation results for the same expression +* [CALCITE-7461] Add `@Strict` to `ByteArrayFunction` and `ByteArrayLengthFunction` +* [CALCITE-7465] Make `MATCH_RECOGNIZE` tolerant to `FINAL` and `RUNNING` non function `MEASURES` +* [CALCITE-7466] Unparse of `MATCH_RECOGNIZE` produces duplicate aliases +* [CALCITE-7467] `MATCH_RECOGNIZE` does not support aliases for tableRef +* [CALCITE-7468] The `SPLIT_PART` implementation is incorrect for regex patterns +* [CALCITE-7470] Unparse of `DEFINE` in `MATCH_RECOGNIZE` leads to incorrect SQL +* [CALCITE-7471] Alias is not auto generated for `MATCH_RECOGNIZE` +* [CALCITE-7472] Arrow adapter should support `LIKE` operator push down +* [CALCITE-7474] `LAST` in `MATCH_RECOGNIZE` might return wrong result +* [CALCITE-7475] Babel parser allows postfix access after PostgreSQL-style `::` infix cast +* [CALCITE-7477] Push schema pattern filter into sub-schema map lookup to avoid loading all schemas +* [CALCITE-7479] Remove redundant aggregate group keys with FD +* [CALCITE-7480] Unparse of `MATCH_RECOGNIZE` with `PARTITION BY` or `ORDER BY` produces invalid SQL +* [CALCITE-7482] Wrong variablesSet used when rewriting subquery in `JOIN ON` clause +* [CALCITE-7483] `RelToSqlConverter` generates `SELECT *` despite `supportGenerateSelectStar` +* [CALCITE-7484] Add a rule to eliminate redundant aggregates functions over `GROUP BY` keys +* [CALCITE-7485] `FIRST_VALUE/LAST_VALUE` should only be defined for window aggregates +* [CALCITE-7486] Operators in `MATCH_RECOGNIZE` don't support `SqlLiterals` +* [CALCITE-7490] `PruneEmptyRules` is ineffective for window statements +* [CALCITE-7491] Literals of type `TIMESTAMP WITH TIME ZONE` cause crashes +* [CALCITE-7492] Support expression that has a constant value within the group involving only `GROUP BY` keys as aggregate arguments +* [CALCITE-7496] OS-adapter usability +* [CALCITE-7497] Support constant folding in Lambda expressions +* [CALCITE-7499] `COALESCE` with args of different types might be incorrectly simplified +* [CALCITE-7501] `AssertionError` in alias expansion for `LEFT JOIN USING` +* [CALCITE-7502] `RelToSqlConverter` creates invalid sql when converting nested window contains `SqlCaseWhen` +* [CALCITE-7503] Hint validation does not have access to source position +* [CALCITE-7504] Use lazy logging in Hypergraph code +* [CALCITE-7506] `RelWriterImpl` does not output hints +* [CALCITE-7514] `MultiJoinOptimizeBushyRule` throws `AssertionError` when a join condition references 3 or more factors +* [CALCITE-7522] Allow `EXTRACT(interval FROM tz)` to operate on `TIMESTAMP WITH TIME ZONE` +* [CALCITE-7524] `JdbcSchema` throws exception for `DECIMAL` columns with precision 0 in JDBC metadata +* [CALCITE-7526] Incorrect `TIMESTAMP WITH TIME ZONE` produces wrong error message +* [CALCITE-7527] `SqlParserUtil.parseTimestampTzLiteral` does not validate timezone +* [CALCITE-7530] `FOR SYSTEM_TIME AS OF` on CTE causes NPE while validation +* [CALCITE-7531] Add to `BasicSqlType` constructor accepting precision, scale and nullability +* [CALCITE-7532] Model usability +* [CALCITE-7533] Parser rejects parenthesized query as the body of a `WITH` clause +* [CALCITE-7537] Invalid Postgres SQL generated for right-deep comma join trees +* [CALCITE-7538] `SqlValidatorImpl` should reject `MATCH_RECOGNIZE` with duplicate `MEASURE` alias +* Add debug log for query plan after decorrelation completion +* Carry source position information through more code rewrites +* Make SqlValidatorImpl#maybeCast protected to allow using it by child classes + #### Build and test suite {: #build-1-42-0} +* [CALCITE-4947] Checkstyle fails on classes generated by Intellij when using option "build and run [tests] using Intellij IDEA" +* [CALCITE-7020] Upgrade gradle from 8.7 to 8.14.4 +* [CALCITE-7261] `DiffRepository` generation xml does not respect alphabetical order +* [CALCITE-7265] Allow `RelOptFixture.relFn` to be used with VolcanoPlanner +* [CALCITE-7292] Replace `case when true then deptno else null end` with a non-simplifiable expression in RelOptRulesTest +* [CALCITE-7342] Quidem test support for `TopDownGeneralDecorrelator` +* [CALCITE-7345] Quidem test support for `Field Trimmer` +* [CALCITE-7381] Parameters modified by `!set` must be restored to their default values in Quidem test +* [CALCITE-7424] In Lint, support sort specifications +* [CALCITE-7426] Add a PR submission template to Calcite +* [CALCITE-7473] Better IntelliJ and VSCode .gitignore +* [CALCITE-7481] Support jdk24 in CI +* [CALCITE-7507] NPE in `ReleaseExtension.` when building from sources +* [CALCITE-7561] Upgrade OWASP plugin from 6.1.6 to 12.2.2 +* Add sub-query, some, misc, scalar iq files in `CoreQuidemTest2` +* Move ExpandDisjunctionForJoinInputsRule test from planner.iq to hep.iq +* Replace `replace` with `toLinux` in `RelMetadataTest` +* Test cases for: + * [CALCITE-2359] Inconsistent behavior when casting interval literals to integer + * [CALCITE-3366] `RelDecorrelator` supports `Union` + * [CALCITE-4232] Elasticsearch `IN` Query is not supported + * [CALCITE-5124] `LIMIT` won't work when `GROUP BY` two or more columns in Elasticsearch Adapter + * [CALCITE-5161] `NPE` when inserting a null value into a decimal column + * [CALCITE-5578] RelOptRulesTest testAggregateCaseToFilter optimized plan not semantically equivalent to the original one after conversion + * [CALCITE-6282] Avatica ignores time precision when returning TIME results + * [CALCITE-6299] Support `JOIN` in Arrow adapter + * [CALCITE-6452] Scalar sub-query that uses `IS NOT DISTINCT FROM` returns incorrect result + * [CALCITE-6985] Add rule to transform `MIN/MAX` with `ORDER BY` and `LIMIT 1` + * `SetOpToFilterRule` to verify that PROJECT containing non-deterministic expressions and subqueries are not merged + #### Web site and documentation {: #site-1-42-0} + +* Document how PMC members add JIRA users to project roles +* Add Feldera to 'powered by Calcite' page +* Site: Add Zhen Chen as PMC +* Site: Add Weihua Zhang as committer +* Site: Add Thomas Rebele as committer +* Site: Add Yu Xu as committer +* Site: Add Silun Dong as committer +* Site: Update Ruben QL info +* Bump addressable from 2.8.7 to 2.9.0 in /site +* Bump nokogiri from 1.18.9 to 1.19.3 in /site + --> ## 1.41.0 / 2025-11-01 diff --git a/site/_docs/howto.md b/site/_docs/howto.md index 9aa5fbf161ea..8be84ea2ddbd 100644 --- a/site/_docs/howto.md +++ b/site/_docs/howto.md @@ -39,8 +39,8 @@ Unpack the source distribution `.tar.gz` file, then build using Gradle: {% highlight bash %} -$ tar xvfz apache-calcite-1.41.0-src.tar.gz -$ cd apache-calcite-1.41.0-src +$ tar xvfz apache-calcite-1.42.0-src.tar.gz +$ cd apache-calcite-1.42.0-src $ gradle build {% endhighlight %} From 04bea07bf4777cdbb588e0c6bfe7d1f53634dc68 Mon Sep 17 00:00:00 2001 From: Stamatis Zampetakis Date: Sun, 31 May 2026 16:23:43 +0200 Subject: [PATCH 278/562] [CALCITE-7544] Add news item for 1.42.0 --- site/_docs/history.md | 4 +-- site/_posts/2026-05-31-release-1.42.0.md | 44 ++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 2 deletions(-) create mode 100644 site/_posts/2026-05-31-release-1.42.0.md diff --git a/site/_docs/history.md b/site/_docs/history.md index 149ef7d38e34..c3e06b841f06 100644 --- a/site/_docs/history.md +++ b/site/_docs/history.md @@ -73,11 +73,11 @@ metadata APIs (BuiltInMetadata), SQL dialects. {: #site-1-43-0} --> -## 1.42.0 / 2026-06-01 +## 1.42.0 / 2026-05-31 {: #v1-42-0} This release comes 7 months after [1.41.0](#v1-41-0), -contains contributions from 39 contributors, and resolves 241 issues. +contains contributions from 39 contributors, and resolves 249 issues. Contributors to this release: Alessandro Solimando, diff --git a/site/_posts/2026-05-31-release-1.42.0.md b/site/_posts/2026-05-31-release-1.42.0.md new file mode 100644 index 000000000000..9160dcf2dc8c --- /dev/null +++ b/site/_posts/2026-05-31-release-1.42.0.md @@ -0,0 +1,44 @@ +--- +layout: news_item +date: "2026-05-31 14:00:00 +0000" +author: zabetak +version: 1.42.0 +categories: [release] +tag: v1-42-0 +sha: c01f6b5519d4a906cf41bbe7843f5929662cf5fd +--- + + +The [Apache Calcite PMC]({{ site.baseurl }}) is pleased to announce +[Apache Calcite release 1.42.0]({{ site.baseurl }}/docs/history.html#v1-42-0). + +This release comes 7 months after [1.41.0](#v1-41-0), contains contributions from 39 contributors, +and resolves 249 issues. + +Highlights include the implementation of a new decorrelation algorithm based on a research article +by Neumann & Kemper; a major contribution that addresses multiple decorrelation bugs. Additional +features include support for custom delimiters when parsing CSV tables, new SQL syntax extensions +such as `SELECT ... BY`,`SELECT * EXCLUDE(columns)`, `ROW(*)` for nested row types, and +`SELECT * REPLACE(expr as column)`, `':'` variant path access syntax. The release also introduces +several new functions—such as `HYPOT` (Spark), `AGE` (PostgreSQL), `ABS`, `CONCAT`, and +`SUBSTRING` (MongoDB), and `REGEXP` (Hive)—along with improvements like simplifying certain array +and division expressions, improved tracking of input fields, and new rules for transforming queries +into filtered aggregates. Other enhancements include large plan optimization mode for the +HepPlanner, and enhanced parser capabilities for hints with mixed assignments. From 2afc39e0632084f1ca47075dda441a43ea6e1115 Mon Sep 17 00:00:00 2001 From: Stamatis Zampetakis Date: Sun, 31 May 2026 16:43:04 +0200 Subject: [PATCH 279/562] Prepare for next development iteration --- core/src/test/java/org/apache/calcite/test/JdbcTest.java | 2 +- gradle.properties | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/test/java/org/apache/calcite/test/JdbcTest.java b/core/src/test/java/org/apache/calcite/test/JdbcTest.java index e4dca3fc448d..78098912edfc 100644 --- a/core/src/test/java/org/apache/calcite/test/JdbcTest.java +++ b/core/src/test/java/org/apache/calcite/test/JdbcTest.java @@ -991,7 +991,7 @@ static void checkMockDdl(AtomicInteger counter, boolean hasCommit, final int driverMajor = metaData.getDriverMajorVersion(); final int driverMinor = metaData.getDriverMinorVersion(); assertThat(driverMajor, is(1)); - assertThat(driverMinor, is(42)); + assertThat(driverMinor, is(43)); assertThat(metaData.getDatabaseProductName(), is("Calcite")); final String databaseVersion = diff --git a/gradle.properties b/gradle.properties index c874f3ecb2d9..414fde94b85d 100644 --- a/gradle.properties +++ b/gradle.properties @@ -26,7 +26,7 @@ systemProp.org.gradle.internal.publish.checksums.insecure=true # This is version for Calcite itself # Note: it should not include "-SNAPSHOT" as it is automatically added by build.gradle.kts # Release version can be generated by using -Prelease or -Prc= arguments -calcite.version=1.42.0 +calcite.version=1.43.0 # This is a version to be used from Maven repository. It can be overridden by localAvatica below calcite.avatica.version=1.28.0 From a4ed8ad44d8a3ce2ec6d0b97984dcf410b978985 Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Mon, 1 Jun 2026 14:34:59 +0800 Subject: [PATCH 280/562] Function signature contains redundant commas --- site/_docs/reference.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/site/_docs/reference.md b/site/_docs/reference.md index bdf4ada19998..1e1ba19f3701 100644 --- a/site/_docs/reference.md +++ b/site/_docs/reference.md @@ -2560,7 +2560,7 @@ The following functions transform 2D geometries. Not implemented: * ST_Scale(geom, xFactor, yFactor [, zFactor ]) Scales *geom* by multiplying the ordinates by the indicated scale factors -* ST_Translate(geom, x, y, [, z]) Translates *geom* +* ST_Translate(geom, x, y [, z]) Translates *geom* #### Geometry editing functions (2D) @@ -2934,9 +2934,9 @@ In the following: | f s i | ENDSWITH(string1, string2) | Returns whether *string2* is a suffix of *string1* | b | ENDS_WITH(string1, string2) | Equivalent to `ENDSWITH(string1, string2)` | s | EXISTS(array, func) | Returns whether a predicate *func* holds for one or more elements in the *array* -| o | EXISTSNODE(xml, xpath, [, namespaces ]) | Determines whether traversal of a XML document using a specified xpath results in any nodes. Returns 0 if no nodes remain after applying the XPath traversal on the document fragment of the element or elements matched by the XPath expression. Returns 1 if any nodes remain. The optional namespace value that specifies a default mapping or namespace mapping for prefixes, which is used when evaluating the XPath expression. -| o | EXTRACT(xml, xpath, [, namespaces ]) | Returns the XML fragment of the element or elements matched by the XPath expression. The optional namespace value that specifies a default mapping or namespace mapping for prefixes, which is used when evaluating the XPath expression -| m | EXTRACTVALUE(xml, xpathExpr)) | Returns the text of the first text node which is a child of the element or elements matched by the XPath expression. +| o | EXISTSNODE(xml, xpath [, namespaces ]) | Determines whether traversal of a XML document using a specified xpath results in any nodes. Returns 0 if no nodes remain after applying the XPath traversal on the document fragment of the element or elements matched by the XPath expression. Returns 1 if any nodes remain. The optional namespace value that specifies a default mapping or namespace mapping for prefixes, which is used when evaluating the XPath expression. +| o | EXTRACT(xml, xpath [, namespaces ]) | Returns the XML fragment of the element or elements matched by the XPath expression. The optional namespace value that specifies a default mapping or namespace mapping for prefixes, which is used when evaluating the XPath expression +| m | EXTRACTVALUE(xml, xpathExpr) | Returns the text of the first text node which is a child of the element or elements matched by the XPath expression. | h s | FACTORIAL(integer) | Returns the factorial of *integer*, the range of *integer* is [0, 20]. Otherwise, returns NULL | h s | FIND_IN_SET(matchStr, textStr) | Returns the index (1-based) of the given *matchStr* in the comma-delimited *textStr*. Returns 0, if the given *matchStr* is not found or if the *matchStr* contains a comma. For example, FIND_IN_SET('bc', 'a,bc,def') returns 2 | b | FLOOR(value) | Similar to standard `FLOOR(value)` except if *value* is an integer type, the return type is a double @@ -2972,9 +2972,9 @@ In the following: | f r s | LEN(string) | Equivalent to `CHAR_LENGTH(string)` | b f h p r s | LENGTH(string) | Equivalent to `CHAR_LENGTH(string)` | h s | LEVENSHTEIN(string1, string2) | Returns the Levenshtein distance between *string1* and *string2* -| b | LOG(numeric1 [, base ]) | Returns the logarithm of *numeric1* to base *base*, or base e if *base* is not present, or error if *numeric1* is 0 or negative -| m s h | LOG([, base ], numeric1) | Returns the logarithm of *numeric1* to base *base*, or base e if *base* is not present, or null if *numeric1* is 0 or negative -| p | LOG([, base ], numeric1 ) | Returns the logarithm of *numeric1* to base *base*, or base 10 if *numeric1* is not present, or error if *numeric1* is 0 or negative +| b | LOG(numeric [, base ]) | Returns the logarithm of *numeric* to base *base*, or base e if *base* is not present, or error if *numeric* is 0 or negative +| m s h | LOG([ base , ] numeric) | Returns the logarithm of *numeric* to base *base*, or base e if *base* is not present, or null if *numeric* is 0 or negative +| p | LOG([ base , ] numeric) | Returns the logarithm of *numeric* to base *base*, or base 10 if *base* is not present, or error if *numeric* is 0 or negative | m s | LOG2(numeric) | Returns the base 2 logarithm of *numeric* | s | LOG1P(numeric) | Returns the natural logarithm of 1 plus *numeric* | b o p r s h | LPAD(string, length [, pattern ]) | Returns a string or bytes value that consists of *string* prepended to *length* with *pattern* From cbdb37c195b54c95ee8e493b14e39f82f32eee4c Mon Sep 17 00:00:00 2001 From: xuzifu666 <1206332514@qq.com> Date: Thu, 28 May 2026 23:04:45 +0800 Subject: [PATCH 281/562] [CALCITE-6823] Cannot convert CHAR to Integer when applying SubstitutionVisitor --- .../calcite/plan/VisitorDataContext.java | 87 ++++++++++++++++++- ...terializedViewSubstitutionVisitorTest.java | 15 ++++ 2 files changed, 100 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/plan/VisitorDataContext.java b/core/src/main/java/org/apache/calcite/plan/VisitorDataContext.java index 1b48b8c64f82..388a0748837b 100644 --- a/core/src/main/java/org/apache/calcite/plan/VisitorDataContext.java +++ b/core/src/main/java/org/apache/calcite/plan/VisitorDataContext.java @@ -28,8 +28,11 @@ import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexUtil; import org.apache.calcite.schema.SchemaPlus; +import org.apache.calcite.util.DateString; import org.apache.calcite.util.NlsString; import org.apache.calcite.util.Pair; +import org.apache.calcite.util.TimeString; +import org.apache.calcite.util.TimestampString; import org.apache.calcite.util.trace.CalciteLogger; import org.checkerframework.checker.nullness.qual.Nullable; @@ -38,6 +41,8 @@ import java.math.BigDecimal; import java.util.List; +import static java.util.Objects.requireNonNull; + /** * DataContext for evaluating a RexExpression. */ @@ -107,6 +112,26 @@ public VisitorDataContext(@Nullable Object[] values) { return new VisitorDataContext(values); } + /** + * Extracts a value from a RexLiteral for use in DataContext. + * + *

      Returns a Pair of (column index, value) if extraction is successful, + * or null if the value cannot be extracted or is invalid. + * + *

      Returns null when: + *

        + *
      • Arguments are not valid RexInputRef and RexLiteral
        • + *
      • Type conversion fails (e.g., invalid date/time format)
        • + *
      • Type combination is unsupported
        • + *
      + * + *

      When null is returned, the containing optimization (e.g., + * materialized view substitution) cannot be applied and is skipped. + * + * @param inputRef the input reference (column) + * @param literal the literal value to extract + * @return a Pair of (column index, value) or null + */ public static @Nullable Pair getValue( @Nullable RexNode inputRef, @Nullable RexNode literal) { inputRef = inputRef == null ? null : RexUtil.removeCast(inputRef); @@ -140,10 +165,68 @@ public VisitorDataContext(@Nullable Object[] values) { case DECIMAL: return Pair.of(index, rexLiteral.getValueAs(BigDecimal.class)); case DATE: + switch (rexLiteral.getType().getSqlTypeName()) { + case DATE: + return Pair.of(index, rexLiteral.getValueAs(Integer.class)); + case CHAR: + case VARCHAR: + try { + return Pair.of(index, + new DateString(requireNonNull(rexLiteral.getValueAs(String.class))) + .getDaysSinceEpoch()); + } catch (IllegalArgumentException e) { + LOGGER.warn( + "Cannot convert string literal '{}' to DATE type; " + + "materialized view optimization will be skipped", + rexLiteral.getValueAs(String.class), e); + return null; + } + default: + break; + } + break; case TIME: - return Pair.of(index, rexLiteral.getValueAs(Integer.class)); + switch (rexLiteral.getType().getSqlTypeName()) { + case TIME: + return Pair.of(index, rexLiteral.getValueAs(Integer.class)); + case CHAR: + case VARCHAR: + try { + return Pair.of(index, + new TimeString(requireNonNull(rexLiteral.getValueAs(String.class))) + .getMillisOfDay()); + } catch (IllegalArgumentException e) { + LOGGER.debug( + "Cannot convert string literal '{}' to TIME type; " + + "materialized view optimization will be skipped", + rexLiteral.getValueAs(String.class), e); + return null; + } + default: + break; + } + break; case TIMESTAMP: - return Pair.of(index, rexLiteral.getValueAs(Long.class)); + switch (rexLiteral.getType().getSqlTypeName()) { + case TIMESTAMP: + return Pair.of(index, rexLiteral.getValueAs(Long.class)); + case CHAR: + case VARCHAR: + try { + return Pair.of(index, + new TimestampString(requireNonNull(rexLiteral.getValueAs(String.class))) + .getMillisSinceEpoch()); + } catch (IllegalArgumentException e) { + LOGGER.debug( + "Cannot convert string literal '{}' to TIMESTAMP type; " + + "materialized view optimization will be skipped", + rexLiteral.getValueAs(String.class), e); + return null; + } + default: + break; + } + break; case CHAR: return Pair.of(index, rexLiteral.getValueAs(Character.class)); case VARCHAR: diff --git a/core/src/test/java/org/apache/calcite/test/MaterializedViewSubstitutionVisitorTest.java b/core/src/test/java/org/apache/calcite/test/MaterializedViewSubstitutionVisitorTest.java index 22fa948ed667..a19cd683d21b 100644 --- a/core/src/test/java/org/apache/calcite/test/MaterializedViewSubstitutionVisitorTest.java +++ b/core/src/test/java/org/apache/calcite/test/MaterializedViewSubstitutionVisitorTest.java @@ -112,6 +112,21 @@ protected final MaterializedViewFixture sql(String materialize, .ok(); } + /** Test case of + * [CALCITE-6823] + * Cannot convert CHAR to Integer when applying SubstitutionVisitor. */ + @Test void testDateFilter() { + sql("SELECT HIREDATE FROM EMP WHERE HIREDATE > '1990-10-01'", + "SELECT * FROM EMP WHERE HIREDATE > '1990-05-01'") + .withDefaultSchemaSpec(CalciteAssert.SchemaSpec.SCOTT) + .noMat(); + + sql("SELECT HIREDATE FROM EMP WHERE HIREDATE > '1990-10-01'", + "SELECT * FROM EMP WHERE HIREDATE > 'invalid-date'") + .withDefaultSchemaSpec(CalciteAssert.SchemaSpec.SCOTT) + .noMat(); + } + @Test void testFilterToProject0() { sql("select *, \"empid\" * 2 from \"emps\"", "select * from \"emps\" where (\"empid\" * 2) > 3") From db30dfbf97227da682f4f71ffc257383fcbfb7c9 Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Sun, 31 May 2026 23:30:59 +0800 Subject: [PATCH 282/562] Remove stray HTML comment terminator from history page --- site/_docs/history.md | 1 - 1 file changed, 1 deletion(-) diff --git a/site/_docs/history.md b/site/_docs/history.md index c3e06b841f06..aba03c8f3c6f 100644 --- a/site/_docs/history.md +++ b/site/_docs/history.md @@ -424,7 +424,6 @@ The same applies to `SqlBabelCreateTable` and `SqlUnpivot`. * Bump addressable from 2.8.7 to 2.9.0 in /site * Bump nokogiri from 1.18.9 to 1.19.3 in /site ---> ## 1.41.0 / 2025-11-01 {: #v1-41-0} From 17cde888342529d5b402d2c446aeaa6335dec6f0 Mon Sep 17 00:00:00 2001 From: xuzifu666 <1206332514@qq.com> Date: Sun, 24 May 2026 20:58:09 +0800 Subject: [PATCH 283/562] [CALCITE-7304] Floor/Ceil can not simplify with WEEK TimeUnit --- .../org/apache/calcite/rex/RexSimplify.java | 14 ++++ .../test/RexImplicationCheckerTest.java | 79 +++++++++++++++++++ .../calcite/test/SqlToRelConverterTest.java | 20 +++++ 3 files changed, 113 insertions(+) diff --git a/core/src/main/java/org/apache/calcite/rex/RexSimplify.java b/core/src/main/java/org/apache/calcite/rex/RexSimplify.java index 1c38d2d72147..4d7702295bff 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexSimplify.java +++ b/core/src/main/java/org/apache/calcite/rex/RexSimplify.java @@ -2687,6 +2687,20 @@ private static boolean canRollUp(TimeUnit outer, TimeUnit inner) { break; } break; + case WEEK: + switch (inner) { + case WEEK: + case DAY: + case HOUR: + case MINUTE: + case SECOND: + case MILLISECOND: + case MICROSECOND: + return true; + default: + break; + } + break; case QUARTER: switch (inner) { case QUARTER: diff --git a/core/src/test/java/org/apache/calcite/test/RexImplicationCheckerTest.java b/core/src/test/java/org/apache/calcite/test/RexImplicationCheckerTest.java index 042dcae1f0a9..60fbc4e6b821 100644 --- a/core/src/test/java/org/apache/calcite/test/RexImplicationCheckerTest.java +++ b/core/src/test/java/org/apache/calcite/test/RexImplicationCheckerTest.java @@ -24,6 +24,7 @@ import org.apache.calcite.rex.RexSimplify; import org.apache.calcite.rex.RexUnknownAs; import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.util.DateString; @@ -556,4 +557,82 @@ public class RexImplicationCheckerTest { } } + /** Test case of + * [CALCITE-7304] + * Floor/Ceil can not simplify with WEEK TimeUnit. */ + @Test void testSimplifyCeilFloorWeek() { + final Fixture f = new Fixture(); + // RexInterpreter does not support WEEK and DAY, so we disable paranoid + // verification for this test. + final RexSimplify nonParanoidSimplify = f.simplify.withParanoid(false); + final RexNode literalTs = + f.timestampLiteral(new TimestampString("2010-10-10 00:00:00")); + + // Positive tests: floor(floor(x, inner), WEEK) -> floor(x, WEEK) + // when inner is DAY or finer. + for (TimeUnitRange innerRange : ImmutableList.of( + TimeUnitRange.WEEK, TimeUnitRange.DAY)) { + final RexNode innerFloorCall = + f.rexBuilder.makeCall(SqlStdOperatorTable.FLOOR, literalTs, + f.rexBuilder.makeFlag(innerRange)); + final RexNode innerCeilCall = + f.rexBuilder.makeCall(SqlStdOperatorTable.CEIL, literalTs, + f.rexBuilder.makeFlag(innerRange)); + final RexNode outerFloorCall = + f.rexBuilder.makeCall(SqlStdOperatorTable.FLOOR, innerFloorCall, + f.rexBuilder.makeFlag(TimeUnitRange.WEEK)); + final RexNode outerCeilCall = + f.rexBuilder.makeCall(SqlStdOperatorTable.CEIL, innerCeilCall, + f.rexBuilder.makeFlag(TimeUnitRange.WEEK)); + final RexCall floorSimplifiedExpr = + (RexCall) nonParanoidSimplify.simplifyPreservingType(outerFloorCall, + RexUnknownAs.UNKNOWN, true); + assertThat(floorSimplifiedExpr.getKind(), is(SqlKind.FLOOR)); + assertThat(((RexLiteral) floorSimplifiedExpr.getOperands().get(1)) + .getValue(), + hasToString(TimeUnitRange.WEEK.toString())); + assertThat(floorSimplifiedExpr.getOperands().get(0), + hasToString(literalTs.toString())); + final RexCall ceilSimplifiedExpr = + (RexCall) nonParanoidSimplify.simplifyPreservingType(outerCeilCall, + RexUnknownAs.UNKNOWN, true); + assertThat(ceilSimplifiedExpr.getKind(), is(SqlKind.CEIL)); + assertThat(((RexLiteral) ceilSimplifiedExpr.getOperands().get(1)) + .getValue(), + hasToString(TimeUnitRange.WEEK.toString())); + assertThat(ceilSimplifiedExpr.getOperands().get(0), + hasToString(literalTs.toString())); + } + + // Negative tests: WEEK cannot rollup to MONTH or DAY, + // and MONTH cannot rollup to WEEK. + for (TimeUnitRange outerRange : ImmutableList.of( + TimeUnitRange.MONTH, TimeUnitRange.DAY)) { + assertNotSimplified(f, nonParanoidSimplify, SqlStdOperatorTable.FLOOR, literalTs, + TimeUnitRange.WEEK, outerRange); + assertNotSimplified(f, nonParanoidSimplify, SqlStdOperatorTable.CEIL, literalTs, + TimeUnitRange.WEEK, outerRange); + } + for (TimeUnitRange innerRange : ImmutableList.of( + TimeUnitRange.MONTH, TimeUnitRange.YEAR)) { + assertNotSimplified(f, nonParanoidSimplify, SqlStdOperatorTable.FLOOR, literalTs, + innerRange, TimeUnitRange.WEEK); + assertNotSimplified(f, nonParanoidSimplify, SqlStdOperatorTable.CEIL, literalTs, + innerRange, TimeUnitRange.WEEK); + } + } + + private void assertNotSimplified(Fixture f, RexSimplify simplify, SqlOperator operator, + RexNode timestamp, TimeUnitRange innerRange, TimeUnitRange outerRange) { + final RexNode innerCall = + f.rexBuilder.makeCall(operator, timestamp, + f.rexBuilder.makeFlag(innerRange)); + final RexNode outerCall = + f.rexBuilder.makeCall(operator, innerCall, + f.rexBuilder.makeFlag(outerRange)); + final RexNode simplifiedExpr = + simplify.simplifyPreservingType(outerCall, RexUnknownAs.UNKNOWN, true); + assertThat(simplifiedExpr, hasToString(outerCall.toString())); + } + } diff --git a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java index ebad0e9450a5..9217e74895ea 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java @@ -76,9 +76,11 @@ import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.not; import static org.hamcrest.CoreMatchers.notNullValue; import static org.hamcrest.CoreMatchers.nullValue; import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.hasSize; /** @@ -6133,4 +6135,22 @@ void checkUserDefinedOrderByOver(NullCollation nullCollation) { + "FROM emp JOIN dept using (deptno)"; sql(sql).withConformance(SqlConformanceEnum.PRESTO).ok(); } + + /** Test case of + * [CALCITE-7304] + * Floor/Ceil can not simplify with WEEK TimeUnit. */ + @Test void testSimplifyNestedFloorWeekFromSql() { + final String sql = "select floor(floor(hiredate TO DAY) TO WEEK) from emp"; + final RelNode rel = sql(sql).toRel(); + final HepProgramBuilder programBuilder = HepProgram.builder(); + programBuilder.addRuleInstance(CoreRules.PROJECT_REDUCE_EXPRESSIONS); + final HepPlanner planner = new HepPlanner(programBuilder.build()); + planner.setRoot(rel); + final RelNode optimized = planner.findBestExp(); + final String plan = RelOptUtil.toString(optimized); + // After PROJECT_REDUCE_EXPRESSIONS, nested floor(floor(x TO DAY) TO WEEK) + // should be simplified to floor(x TO WEEK). + assertThat(plan, not(containsString("FLOOR(FLOOR"))); + assertThat(plan, containsString("FLOOR($4, FLAG(WEEK))")); + } } From 5fa61f9892d25005c8e090054916d6e73759f4f9 Mon Sep 17 00:00:00 2001 From: zzwqqq Date: Thu, 28 May 2026 19:54:24 +0800 Subject: [PATCH 284/562] [CALCITE-7563] Oracle dialect generates invalid CAST to VARCHAR without precision --- .../calcite/sql/dialect/OracleSqlDialect.java | 7 +++++++ .../rel/rel2sql/RelToSqlConverterTest.java | 15 +++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/core/src/main/java/org/apache/calcite/sql/dialect/OracleSqlDialect.java b/core/src/main/java/org/apache/calcite/sql/dialect/OracleSqlDialect.java index 2de588f118cb..9aab15f5e247 100644 --- a/core/src/main/java/org/apache/calcite/sql/dialect/OracleSqlDialect.java +++ b/core/src/main/java/org/apache/calcite/sql/dialect/OracleSqlDialect.java @@ -116,6 +116,13 @@ public OracleSqlDialect(Context context) { case DOUBLE: castSpec = "DOUBLE PRECISION"; break; + case VARCHAR: + if (type.getPrecision() == RelDataType.PRECISION_NOT_SPECIFIED) { + final int precision = getTypeSystem().getMaxPrecision(SqlTypeName.VARCHAR); + castSpec = "VARCHAR(" + precision + ")"; + break; + } + return super.getCastSpec(type); default: return super.getCastSpec(type); } diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java index 2953f3b186f1..66afda8653b2 100644 --- a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java +++ b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java @@ -1708,6 +1708,21 @@ private static String toSql(RelNode root, SqlDialect dialect, .ok(expectedOracle); } + /** Test case for + * [CALCITE-7563] + * Oracle dialect generates invalid CAST to VARCHAR without precision. */ + @Test void testCastVarcharWithoutPrecisionOracle() { + final String query = "select cast(\"store_id\" as VARCHAR)\n" + + " from \"expense_fact\""; + final String expected = "SELECT CAST(\"store_id\" AS VARCHAR(4000))\n" + + "FROM \"foodmart\".\"expense_fact\""; + final String expectedModifiedTypeSystem = "SELECT CAST(\"store_id\" AS VARCHAR(512))\n" + + "FROM \"foodmart\".\"expense_fact\""; + sql(query) + .withOracle().ok(expected) + .withOracleModifiedTypeSystem().ok(expectedModifiedTypeSystem); + } + /** Test case for * [CALCITE-1174] * When generating SQL, translate SUM0(x) to COALESCE(SUM(x), 0). */ From d36e4545f6630256a083fe2533dc0994c0de9cfb Mon Sep 17 00:00:00 2001 From: zzwqqq Date: Tue, 2 Jun 2026 14:14:51 +0800 Subject: [PATCH 285/562] [CALCITE-7529] Casts between literals and TIME/TIMESTAMP can lose precision beyond milliseconds --- .../enumerable/RexToLixTranslator.java | 6 +- .../org/apache/calcite/rex/RexBuilder.java | 127 +++++++++++ .../apache/calcite/rex/RexExecutorImpl.java | 27 ++- .../apache/calcite/util/BuiltInMethod.java | 4 + .../apache/calcite/rex/RexExecutorTest.java | 207 ++++++++++++++++++ .../apache/calcite/test/SqlOperatorTest.java | 2 +- 6 files changed, 367 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java index 61fa2b0d7163..7d2cb6b7320c 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java @@ -491,7 +491,8 @@ private Expression getConvertExpression( case TIME: return RexImpTable.optimize2(operand, Expressions.isConstantNull(format) - ? Expressions.call(BuiltInMethod.UNIX_TIME_TO_STRING.method, operand) + ? Expressions.call(BuiltInMethod.UNIX_TIME_TO_STRING_WITH_PRECISION.method, + operand, Expressions.constant(sourceType.getPrecision())) : Expressions.call( Expressions.new_( BuiltInMethod.FORMAT_TIME.method.getDeclaringClass()), @@ -508,7 +509,8 @@ private Expression getConvertExpression( case TIMESTAMP: return RexImpTable.optimize2(operand, Expressions.isConstantNull(format) - ? Expressions.call(BuiltInMethod.UNIX_TIMESTAMP_TO_STRING.method, operand) + ? Expressions.call(BuiltInMethod.UNIX_TIMESTAMP_TO_STRING_WITH_PRECISION.method, + operand, Expressions.constant(sourceType.getPrecision())) : Expressions.call( Expressions.new_( BuiltInMethod.FORMAT_TIMESTAMP.method.getDeclaringClass()), diff --git a/core/src/main/java/org/apache/calcite/rex/RexBuilder.java b/core/src/main/java/org/apache/calcite/rex/RexBuilder.java index b77fe0283095..95030b5b9053 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexBuilder.java +++ b/core/src/main/java/org/apache/calcite/rex/RexBuilder.java @@ -29,17 +29,21 @@ import org.apache.calcite.rel.type.RelDataTypeSystemImpl; import org.apache.calcite.runtime.FlatLists; import org.apache.calcite.runtime.SqlFunctions; +import org.apache.calcite.sql.SqlAbstractDateTimeLiteral; import org.apache.calcite.sql.SqlAggFunction; import org.apache.calcite.sql.SqlCollation; import org.apache.calcite.sql.SqlIntervalQualifier; import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.SqlSpecialOperator; +import org.apache.calcite.sql.SqlTimeLiteral; +import org.apache.calcite.sql.SqlTimestampLiteral; import org.apache.calcite.sql.SqlUtil; import org.apache.calcite.sql.fun.SqlCountAggFunction; import org.apache.calcite.sql.fun.SqlLibraryOperators; import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.parser.SqlParserPos; +import org.apache.calcite.sql.parser.SqlParserUtil; import org.apache.calcite.sql.type.ArraySqlType; import org.apache.calcite.sql.type.IntervalSqlType; import org.apache.calcite.sql.type.MapSqlType; @@ -805,6 +809,12 @@ public RexNode makeCast( && SqlTypeUtil.isExactNumeric(type)) { return makeCastBooleanToExact(type, exp); } + final RexNode literalCast = + makeCastForTemporalLiteral( + pos, type, literal, matchNullability, safe, format); + if (literalCast != null) { + return literalCast; + } if (canRemoveCastFromLiteral(type, value, typeName)) { switch (typeName) { case INTERVAL_YEAR: @@ -877,6 +887,123 @@ public RexNode makeCast( return makeAbstractCast(pos, type, exp, safe, format); } + private @Nullable RexNode makeCastForTemporalLiteral( + SqlParserPos pos, + RelDataType type, + RexLiteral literal, + boolean matchNullability, + boolean safe, + RexLiteral format) { + if (!format.isNull()) { + return null; + } + if (SqlTypeUtil.isCharacter(literal.getType())) { + return makeCastFromCharacterLiteralToTemporal( + pos, type, literal, matchNullability, safe, format); + } + if (SqlTypeUtil.isCharacter(type)) { + return makeCharacterLiteralFromTemporalLiteral(type, literal); + } + return null; + } + + private @Nullable RexNode makeCastFromCharacterLiteralToTemporal( + SqlParserPos pos, + RelDataType type, + RexLiteral literal, + boolean matchNullability, + boolean safe, + RexLiteral format) { + final NlsString nlsString = literal.getValueAs(NlsString.class); + if (nlsString == null) { + return null; + } + final String value = nlsString.getValue().trim(); + final RexNode temporalLiteral; + try { + switch (type.getSqlTypeName()) { + case TIME: + final SqlTimeLiteral timeLiteral = + SqlParserUtil.parseTimeLiteral(value, pos); + if (!isExactFractionalSecondLiteral(timeLiteral, value)) { + return null; + } + final TimeString time = + requireNonNull(timeLiteral.getValueAs(TimeString.class), + "timeLiteral.getValueAs(TimeString.class)"); + temporalLiteral = makeTimeLiteral(time, type.getPrecision()); + break; + case TIMESTAMP: + final SqlTimestampLiteral timestampLiteral = + SqlParserUtil.parseTimestampLiteral(value, pos); + if (!isExactFractionalSecondLiteral(timestampLiteral, value)) { + return null; + } + final TimestampString timestamp = + requireNonNull(timestampLiteral.getValueAs(TimestampString.class), + "timestampLiteral.getValueAs(TimestampString.class)"); + temporalLiteral = makeTimestampLiteral(timestamp, type.getPrecision()); + break; + default: + return null; + } + } catch (RuntimeException e) { + return safe ? makeNullLiteral(type) : null; + } + if (type.isNullable() + && !temporalLiteral.getType().isNullable() + && matchNullability) { + return makeAbstractCast(pos, type, temporalLiteral, safe, format); + } + return temporalLiteral; + } + + /** Converts a TIME or TIMESTAMP literal to a character literal. + * + *

      Returns null if the literal is not a TIME or TIMESTAMP, the literal + * cannot be read as the corresponding temporal string value, or the formatted + * temporal value does not fit in the target character type. */ + private @Nullable RexNode makeCharacterLiteralFromTemporalLiteral( + RelDataType type, + RexLiteral literal) { + // Format temporal literals directly so they do not go through generated + // code, whose TIME/TIMESTAMP runtime values have millisecond precision. + final String value; + final int precision = literal.getType().getPrecision(); + switch (literal.getType().getSqlTypeName()) { + case TIME: + final TimeString time = literal.getValueAs(TimeString.class); + if (time == null) { + return null; + } + value = time.toString(precision); + break; + case TIMESTAMP: + final TimestampString timestamp = literal.getValueAs(TimestampString.class); + if (timestamp == null) { + return null; + } + value = timestamp.toString(precision); + break; + default: + return null; + } + // Only create a character literal if the formatted temporal value fits in + // the target type. + return SqlTypeUtil.comparePrecision(type.getPrecision(), value.length()) >= 0 + ? makeLiteral(value, type, true) + : null; + } + + /** Returns whether a parsed TIME or TIMESTAMP literal has fractional-second + * precision high enough to preserve the original character value exactly. */ + private static boolean isExactFractionalSecondLiteral( + SqlAbstractDateTimeLiteral literal, String value) { + return literal.getPrec() != 0 + && literal.getPrec() != RelDataType.PRECISION_NOT_SPECIFIED + && literal.toFormattedString().equals(value); + } + /** Returns the lowest granularity unit for the given unit. * YEAR and MONTH intervals are stored as months; * HOUR, MINUTE, SECOND intervals are stored as milliseconds. */ diff --git a/core/src/main/java/org/apache/calcite/rex/RexExecutorImpl.java b/core/src/main/java/org/apache/calcite/rex/RexExecutorImpl.java index 3a894d0fbad1..acfbbdc56d4e 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexExecutorImpl.java +++ b/core/src/main/java/org/apache/calcite/rex/RexExecutorImpl.java @@ -43,6 +43,7 @@ import java.lang.reflect.Modifier; import java.lang.reflect.Type; +import java.util.ArrayList; import java.util.List; /** @@ -136,14 +137,34 @@ public static RexExecutable getExecutable(RexBuilder rexBuilder, List e @Override public void reduce(RexBuilder rexBuilder, List constExps, List reducedValues) { assert reducedValues.isEmpty(); + final List exps = new ArrayList<>(); + final List ordinals = new ArrayList<>(); + for (int i = 0; i < constExps.size(); i++) { + final RexNode constExp = constExps.get(i); + // Literals are already reduced. Keep them as Rex values instead of + // round-tripping through generated code. + if (!(constExp instanceof RexLiteral)) { + ordinals.add(i); + exps.add(constExp); + } + } + if (exps.isEmpty()) { + reducedValues.addAll(constExps); + return; + } try { - String code = compile(rexBuilder, constExps, (list, index, storageType) -> { + String code = compile(rexBuilder, exps, (list, index, storageType) -> { throw new UnsupportedOperationException(); }); - final RexExecutable executable = new RexExecutable(code, constExps); + final RexExecutable executable = new RexExecutable(code, exps); executable.setDataContext(dataContext); - executable.reduce(rexBuilder, constExps, reducedValues); + final List values = new ArrayList<>(exps.size()); + executable.reduce(rexBuilder, exps, values); + reducedValues.addAll(constExps); + for (int i = 0; i < ordinals.size(); i++) { + reducedValues.set(ordinals.get(i), values.get(i)); + } } catch (RuntimeException ex) { // Something went wrong during constant reduction (for example, // we may have attempted a division by zero). diff --git a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java index 98b67a02bbf3..81fc9d6f0414 100644 --- a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java +++ b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java @@ -784,8 +784,12 @@ public enum BuiltInMethod { String.class, int.class), UNIX_DATE_TO_STRING(DateTimeUtils.class, "unixDateToString", int.class), UNIX_TIME_TO_STRING(DateTimeUtils.class, "unixTimeToString", int.class), + UNIX_TIME_TO_STRING_WITH_PRECISION(DateTimeUtils.class, "unixTimeToString", + int.class, int.class), UNIX_TIMESTAMP_TO_STRING(DateTimeUtils.class, "unixTimestampToString", long.class), + UNIX_TIMESTAMP_TO_STRING_WITH_PRECISION(DateTimeUtils.class, + "unixTimestampToString", long.class, int.class), INTERVAL_YEAR_MONTH_TO_STRING(DateTimeUtils.class, "intervalYearMonthToString", int.class, TimeUnitRange.class), INTERVAL_DAY_TIME_TO_STRING(DateTimeUtils.class, "intervalDayTimeToString", diff --git a/core/src/test/java/org/apache/calcite/rex/RexExecutorTest.java b/core/src/test/java/org/apache/calcite/rex/RexExecutorTest.java index a98cffc0007a..16d29783552b 100644 --- a/core/src/test/java/org/apache/calcite/rex/RexExecutorTest.java +++ b/core/src/test/java/org/apache/calcite/rex/RexExecutorTest.java @@ -15,11 +15,14 @@ * limitations under the License. */ package org.apache.calcite.rex; + import org.apache.calcite.DataContext; import org.apache.calcite.DataContexts; import org.apache.calcite.avatica.util.ByteString; +import org.apache.calcite.jdbc.JavaTypeFactoryImpl; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rel.type.RelDataTypeSystemImpl; import org.apache.calcite.sql.SqlBinaryOperator; import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlOperator; @@ -35,6 +38,7 @@ import org.apache.calcite.util.DateString; import org.apache.calcite.util.NlsString; import org.apache.calcite.util.TestUtil; +import org.apache.calcite.util.TimeString; import org.apache.calcite.util.TimestampString; import org.apache.calcite.util.Util; @@ -160,6 +164,209 @@ protected void check(final Action action) { }); } + @Test void testReduceTimeCastWithFractionalSeconds() { + checkHighPrecision((rexBuilder, executor) -> { + for (int precision = 1; precision <= 9; precision++) { + final RexNode cast = + rexBuilder.makeCast( + rexBuilder.getTypeFactory().createSqlType(SqlTypeName.TIME, + precision), + rexBuilder.makeLiteral("12:34:56.123456789")); + + final RexNode reduced = reduce(rexBuilder, executor, cast).get(0); + + assertThat(reduced, instanceOf(RexLiteral.class)); + assertThat( + ((RexLiteral) reduced).getValueAs(TimeString.class) + .toString(precision), + equalTo(new TimeString("12:34:56.123456789").round(precision) + .toString(precision))); + } + }); + } + + @Test void testReduceTimestampCastWithFractionalSeconds() { + checkHighPrecision((rexBuilder, executor) -> { + for (int precision = 1; precision <= 9; precision++) { + final RexNode cast = + rexBuilder.makeCast( + rexBuilder.getTypeFactory().createSqlType(SqlTypeName.TIMESTAMP, + precision), + rexBuilder.makeLiteral("2020-01-01 12:34:56.123456789")); + + final RexNode reduced = reduce(rexBuilder, executor, cast).get(0); + + assertThat(reduced, instanceOf(RexLiteral.class)); + assertThat( + ((RexLiteral) reduced).getValueAs(TimestampString.class) + .toString(precision), + equalTo(new TimestampString("2020-01-01 12:34:56.123456789") + .round(precision).toString(precision))); + } + }); + } + + @Test void testReduceLiteralWithMicroseconds() { + checkHighPrecision((rexBuilder, executor) -> { + final RexLiteral timeLiteral = + rexBuilder.makeTimeLiteral(new TimeString("12:34:56.123456"), 6); + final RexLiteral timestampLiteral = + rexBuilder.makeTimestampLiteral( + new TimestampString("2020-01-01 12:34:56.123456"), 6); + final RexNode expression = + rexBuilder.makeCall(SqlStdOperatorTable.PLUS, + rexBuilder.makeExactLiteral(BigDecimal.TEN), + rexBuilder.makeExactLiteral(BigDecimal.ONE)); + + final List reducedValues = + reduce(rexBuilder, executor, timeLiteral, expression, timestampLiteral); + + assertThat(reducedValues, hasSize(3)); + assertThat( + ((RexLiteral) reducedValues.get(0)).getValueAs(TimeString.class).toString(6), + equalTo("12:34:56.123456")); + assertThat(((RexLiteral) reducedValues.get(1)).getValue2(), equalTo(11L)); + assertThat( + ((RexLiteral) reducedValues.get(2)).getValueAs(TimestampString.class) + .toString(6), + equalTo("2020-01-01 12:34:56.123456")); + }); + } + + @Test void testReduceTimeToVarcharWithFractionalSeconds() { + checkHighPrecision((rexBuilder, executor) -> { + final RelDataTypeFactory typeFactory = rexBuilder.getTypeFactory(); + for (int precision = 1; precision <= 9; precision++) { + final RexNode castToTime = + rexBuilder.makeCast( + typeFactory.createSqlType(SqlTypeName.TIME, + precision), + rexBuilder.makeLiteral("12:34:56.123456789")); + final RexNode castToVarchar = + rexBuilder.makeCast(typeFactory.createSqlType(SqlTypeName.VARCHAR, 30), + castToTime); + + final RexNode reduced = reduce(rexBuilder, executor, castToVarchar).get(0); + + assertThat(reduced, instanceOf(RexLiteral.class)); + assertThat(((RexLiteral) reduced).getValueAs(String.class), + equalTo(new TimeString("12:34:56.123456789") + .toString(precision))); + } + }); + } + + @Test void testReduceTimeToVarcharWithMilliseconds() { + check((rexBuilder, executor) -> { + final RelDataTypeFactory typeFactory = rexBuilder.getTypeFactory(); + final RexNode castToTime = + rexBuilder.makeCast(typeFactory.createSqlType(SqlTypeName.TIME, 3), + rexBuilder.makeLiteral("12:34:56.987654")); + final RexNode castToVarchar = + rexBuilder.makeCast(typeFactory.createSqlType(SqlTypeName.VARCHAR, 30), + castToTime); + + final RexNode reduced = reduce(rexBuilder, executor, castToVarchar).get(0); + + assertThat(reduced, instanceOf(RexLiteral.class)); + assertThat(((RexLiteral) reduced).getValueAs(String.class), + equalTo("12:34:56.987")); + }); + } + + @Test void testReduceTimeToVarcharWithZeroMilliseconds() { + check((rexBuilder, executor) -> { + final RelDataTypeFactory typeFactory = rexBuilder.getTypeFactory(); + final RexNode castToTime = + rexBuilder.makeCast(typeFactory.createSqlType(SqlTypeName.TIME, 3), + rexBuilder.makeLiteral("12:34:56.000456")); + final RexNode castToVarchar = + rexBuilder.makeCast(typeFactory.createSqlType(SqlTypeName.VARCHAR, 30), + castToTime); + + final RexNode reduced = reduce(rexBuilder, executor, castToVarchar).get(0); + + assertThat(reduced, instanceOf(RexLiteral.class)); + assertThat(((RexLiteral) reduced).getValueAs(String.class), + equalTo("12:34:56.000")); + }); + } + + @Test void testReduceTimestampToVarcharWithFractionalSeconds() { + checkHighPrecision((rexBuilder, executor) -> { + final RelDataTypeFactory typeFactory = rexBuilder.getTypeFactory(); + for (int precision = 1; precision <= 9; precision++) { + final RexNode castToTimestamp = + rexBuilder.makeCast( + typeFactory.createSqlType(SqlTypeName.TIMESTAMP, + precision), + rexBuilder.makeLiteral("2020-01-01 12:34:56.123456789")); + final RexNode castToVarchar = + rexBuilder.makeCast(typeFactory.createSqlType(SqlTypeName.VARCHAR, 30), + castToTimestamp); + + final RexNode reduced = reduce(rexBuilder, executor, castToVarchar).get(0); + + assertThat(reduced, instanceOf(RexLiteral.class)); + assertThat(((RexLiteral) reduced).getValueAs(String.class), + equalTo(new TimestampString("2020-01-01 12:34:56.123456789") + .toString(precision))); + } + }); + } + + @Test void testReduceTimestampToVarcharWithZeroMilliseconds() { + check((rexBuilder, executor) -> { + final RelDataTypeFactory typeFactory = rexBuilder.getTypeFactory(); + final RexNode castToTimestamp = + rexBuilder.makeCast(typeFactory.createSqlType(SqlTypeName.TIMESTAMP, 3), + rexBuilder.makeLiteral("2020-01-01 12:34:56.000456")); + final RexNode castToVarchar = + rexBuilder.makeCast(typeFactory.createSqlType(SqlTypeName.VARCHAR, 30), + castToTimestamp); + + final RexNode reduced = reduce(rexBuilder, executor, castToVarchar).get(0); + + assertThat(reduced, instanceOf(RexLiteral.class)); + assertThat(((RexLiteral) reduced).getValueAs(String.class), + equalTo("2020-01-01 12:34:56.000")); + }); + } + + private static void checkHighPrecision(Action action) { + action.check(new RexBuilder(highPrecisionTemporalTypeFactory()), executor()); + } + + private static List reduce(RexBuilder rexBuilder, + RexExecutorImpl executor, RexNode... nodes) { + final List reducedValues = new ArrayList<>(); + executor.reduce(rexBuilder, ImmutableList.copyOf(nodes), reducedValues); + return reducedValues; + } + + private static RelDataTypeFactory highPrecisionTemporalTypeFactory() { + return new JavaTypeFactoryImpl( + new RelDataTypeSystemImpl() { + @Override public int getMaxPrecision(SqlTypeName typeName) { + switch (typeName) { + case TIME: + case TIMESTAMP: + return 9; + default: + return super.getMaxPrecision(typeName); + } + } + }); + } + + private static RexExecutorImpl executor() { + return new RexExecutorImpl( + DataContexts.of( + ImmutableMap.of( + DataContext.Variable.TIME_ZONE.camelName, TimeZone.getTimeZone("GMT"), + DataContext.Variable.LOCALE.camelName, Locale.US))); + } + private void checkConstant(final Object operand, final Function function) { check((rexBuilder, executor) -> { diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java index cf2792c5a619..c40f077d8d1f 100644 --- a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java +++ b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java @@ -2706,7 +2706,7 @@ private static void checkConcatWithSeparatorInPostgres(SqlOperatorFixture f) { f.checkString("concat_ws(',', 'a', array['b', 'c'], DATE '1945-02-24')", "a,[b, c],1945-02-24", "VARCHAR NOT NULL"); f.checkString("concat_ws(',', timestamp '2024-07-06 12:15:48.678')", - "2024-07-06 12:15:48", "VARCHAR NOT NULL"); + "2024-07-06 12:15:48.678", "VARCHAR NOT NULL"); f.checkString("concat_ws(',', time '12:34:56', time '13:00:00', 2, 'abc')", "12:34:56,13:00:00,2,abc", "VARCHAR NOT NULL"); f.checkString("concat_ws(',', null, null)", "", "VARCHAR NOT NULL"); From 106f70b2a42c8073dae5474fd9203495ca6ebde7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9B=B7=E4=B9=A6=E9=B9=8F?= Date: Tue, 2 Jun 2026 09:26:25 +0800 Subject: [PATCH 286/562] [CALCITE-7574] RelDecorrelator.isFieldNotNullRecursive throws IndexOutOfBoundsException when decorrelating correlated scalar subquery with Aggregate Root Cause: In isFieldNotNullRecursive, the Aggregate branch used ImmutableBitSet.size() for bounds checking. size() returns the bitset capacity, not the number of group keys. Changed to agg.getGroupCount() which correctly returns the actual number of group keys. --- .../calcite/sql2rel/RelDecorrelator.java | 2 +- .../calcite/sql2rel/RelDecorrelatorTest.java | 80 +++++++++++++++++++ core/src/test/resources/sql/sub-query.iq | 19 +++++ 3 files changed, 100 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java b/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java index c38268fb4d39..a450a659dc74 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java @@ -3853,7 +3853,7 @@ private static boolean isFieldNotNullRecursive(RelNode rel, int index) { Aggregate agg = (Aggregate) rel; ImmutableBitSet groupSet = agg.getGroupSet(); - if (index >= groupSet.size()) { + if (index >= agg.getGroupCount()) { return false; } return isFieldNotNullRecursive(agg.getInput(), groupSet.asList().get(index)); diff --git a/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java b/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java index 99b745319203..15f96eaa9e5f 100644 --- a/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java +++ b/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java @@ -2124,4 +2124,84 @@ public static Frameworks.ConfigBuilder config() { + " LogicalTableScan(table=[[scott, DEPT]])\n"; assertThat(afterDecorrelation, hasTree(planAfterDecorrelation)); } + + /** Test case for + * [CALCITE-7574] + * RelDecorrelator.isFieldNotNullRecursive throws IndexOutOfBoundsException + * when decorrelating correlated scalar subquery with Aggregate. + * + *

      When decorrelating a correlated scalar subquery containing an Aggregate, + * {@code isFieldNotNullRecursive} incorrectly used {@code ImmutableBitSet.size()} + * (which returns the bitset capacity, typically 64 * words) instead of + * {@code Aggregate.getGroupCount()} (which returns the number of group keys). + * This caused an {@code IndexOutOfBoundsException} when the field index + * corresponded to an aggregate result field (not a group field). + */ + @Test void testDecorrelateScalarSubQueryWithAggregate() { + final FrameworkConfig frameworkConfig = config().build(); + final RelBuilder builder = RelBuilder.create(frameworkConfig); + final RelOptCluster cluster = builder.getCluster(); + final Planner planner = Frameworks.getPlanner(frameworkConfig); + // Minimal case: outer FROM is an Aggregate, correlated subquery + // references the aggregate result field (s) in the correlation + // condition, triggering isFieldNotNullRecursive on an Aggregate with + // an index pointing to an aggregate result field. + final String sql = "select t.deptno,\n" + + " (select count(*) from emp e\n" + + " where e.deptno = t.deptno\n" + + " and e.sal > t.s)\n" + + "from (select deptno, sum(sal) as s\n" + + " from emp group by deptno) t"; + final RelNode originalRel; + try { + final SqlNode parse = planner.parse(sql); + final SqlNode validate = planner.validate(parse); + originalRel = planner.rel(validate).rel; + } catch (Exception e) { + throw TestUtil.rethrow(e); + } + + final HepProgram hepProgram = HepProgram.builder() + .addRuleCollection( + ImmutableList.of( + CoreRules.FILTER_SUB_QUERY_TO_CORRELATE, + CoreRules.PROJECT_SUB_QUERY_TO_CORRELATE, + CoreRules.JOIN_SUB_QUERY_TO_CORRELATE)) + .build(); + final Program program = + Programs.of(hepProgram, true, + requireNonNull(cluster.getMetadataProvider())); + final RelNode before = + program.run(cluster.getPlanner(), originalRel, cluster.traitSet(), + Collections.emptyList(), Collections.emptyList()); + + // Before the fix, decorrelateQuery would throw: + // java.lang.IndexOutOfBoundsException: index out of range: 0 + // at org.apache.calcite.util.ImmutableBitSet.nth + // at ...RelDecorrelator.isFieldNotNullRecursive + final RelNode after = + RelDecorrelator.decorrelateQuery(before, builder, RuleSets.ofList(Collections.emptyList()), + RuleSets.ofList(Collections.emptyList())); + + // Verify decorrelation produced a valid plan (no Correlate nodes) + final String planAfter = "" + + "LogicalProject(DEPTNO=[$0], EXPR$1=[$4])\n" + + " LogicalJoin(condition=[AND(IS NOT DISTINCT FROM($0, $2), IS NOT DISTINCT FROM($1, $3))], joinType=[left])\n" + + " LogicalAggregate(group=[{0}], S=[SUM($1)])\n" + + " LogicalProject(DEPTNO=[$7], SAL=[$5])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalProject(DEPTNO0=[$0], S=[$1], EXPR$0=[CASE(IS NOT NULL($4), $4, 0)])\n" + + " LogicalJoin(condition=[AND(IS NOT DISTINCT FROM($0, $2), IS NOT DISTINCT FROM($1, $3))], joinType=[left])\n" + + " LogicalAggregate(group=[{0}], S=[SUM($1)])\n" + + " LogicalProject(DEPTNO=[$7], SAL=[$5])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalAggregate(group=[{0, 1}], EXPR$0=[COUNT()])\n" + + " LogicalProject(DEPTNO0=[$8], S=[$9])\n" + + " LogicalJoin(condition=[AND(=($7, $8), >(CAST($5):DECIMAL(19, 2), $9))], joinType=[inner])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalAggregate(group=[{0}], S=[SUM($1)])\n" + + " LogicalProject(DEPTNO=[$7], SAL=[$5])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + assertThat(after, hasTree(planAfter)); + } } diff --git a/core/src/test/resources/sql/sub-query.iq b/core/src/test/resources/sql/sub-query.iq index b301dae413de..a04c8634d240 100644 --- a/core/src/test/resources/sql/sub-query.iq +++ b/core/src/test/resources/sql/sub-query.iq @@ -9237,4 +9237,23 @@ SELECT deptno, dname > SOME(SELECT empno FROM emp) AS b FROM dept; For input string: "ACCOUNTING" !error +# [CALCITE-7574] RelDecorrelator.isFieldNotNullRecursive throws +# IndexOutOfBoundsException when decorrelating correlated scalar subquery +# with Aggregate +# Before fix: java.lang.IndexOutOfBoundsException: index out of range: 0 +SELECT t.deptno, + (SELECT COUNT(*) FROM emp e + WHERE e.deptno = t.deptno AND e.sal > t.s) +FROM (SELECT deptno, SUM(sal) AS s FROM emp GROUP BY deptno) t; ++--------+--------+ +| DEPTNO | EXPR$1 | ++--------+--------+ +| 10 | 0 | +| 20 | 0 | +| 30 | 0 | ++--------+--------+ +(3 rows) + +!ok + # End sub-query.iq From 88873ba145b0370573482644f8feba7abcf635e6 Mon Sep 17 00:00:00 2001 From: Julian Hyde Date: Tue, 2 Jun 2026 11:24:48 -0700 Subject: [PATCH 287/562] [CALCITE-7576] Add an 'In memoriam' section to the website's Community page The page notes the passing of Istvan Toth on April 3rd, 2026. --- site/_data/contributors.yml | 6 +++++- site/community/index.md | 11 +++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/site/_data/contributors.yml b/site/_data/contributors.yml index 9c46540786ec..bff185584428 100644 --- a/site/_data/contributors.yml +++ b/site/_data/contributors.yml @@ -23,7 +23,8 @@ # // lint: sort where ' name:' erase '^.*name:' # - name: Alan Gates - emeritus: 2018/05/04 + status: emeritus + status_date: 2018/05/04 apacheId: gates githubId: alanfgates org: Hortonworks @@ -144,6 +145,9 @@ org: Tencent role: Committer - name: Istvan Toth + status: deceased + status_date: 2026/04/03 + status_url: https://calcite.apache.org/avatica/news/2026/05/12/release-1.28.0/ apacheId: stoty githubId: stoty org: Cloudera diff --git a/site/community/index.md b/site/community/index.md index bb9edb7ea27d..130967f66abf 100644 --- a/site/community/index.md +++ b/site/community/index.md @@ -32,14 +32,21 @@ None scheduled. Name (Apache ID) | Github | Org | Role :--------------- |:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| :-- | :--- -{% for c in site.data.contributors %}{% unless c.emeritus %}{% if c.homepage %}{{ c.name }}{% else %}{{ c.name }}{% endif %} ({{ c.apacheId }}) {{ c.pronouns }} | | {{ c.org }} | {{ c.role }} +{% for c in site.data.contributors %}{% unless c.status %}{% if c.homepage %}{{ c.name }}{% else %}{{ c.name }}{% endif %} ({{ c.apacheId }}) {{ c.pronouns }} | | {{ c.org }} | {{ c.role }} {% endunless %}{% endfor %} Emeritus members Name (Apache ID) | Github | Org | Role :--------------- | :----- | :-- | :--- -{% for c in site.data.contributors %}{% if c.emeritus %}{% if c.homepage %}{{ c.name }}{% else %}{{ c.name }}{% endif %} ({{ c.apacheId }}) {{ c.pronouns }} | | {{ c.org }} | {{ c.role }} +{% for c in site.data.contributors %}{% if c.status == 'emeritus' %}{% if c.homepage %}{{ c.name }}{% else %}{{ c.name }}{% endif %} ({{ c.apacheId }}) {{ c.pronouns }} | | {{ c.org }} | {{ c.role }} +{% endif %}{% endfor %} + +In memoriam + +Name (Apache ID) | Github | Org | Role +:--------------- | :----- | :-- | :--- +{% for c in site.data.contributors %}{% if c.status == 'deceased' %}{% if c.homepage %}{{ c.name }}{% else %}{{ c.name }}{% endif %} ({{ c.apacheId }}) {{ c.pronouns }} | | {{ c.org }} | {{ c.role }}{% if c.status_url %} (in memoriam){% endif %} {% endif %}{% endfor %} # Mailing Lists From 90b5a358f9ec0b639e12bbe8b31ce037273e8635 Mon Sep 17 00:00:00 2001 From: Jerome Haltom Date: Sun, 24 May 2026 14:49:40 -0500 Subject: [PATCH 288/562] [CALCITE-7548] ConditionalExpression.expressionList should have public visibility --- .../org/apache/calcite/linq4j/tree/ConditionalExpression.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ConditionalExpression.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ConditionalExpression.java index c4ab0869ab91..6fbcc1ef1f3e 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ConditionalExpression.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ConditionalExpression.java @@ -36,7 +36,7 @@ * "if (c0) e0 else if (c1) e1 ... else if (cn-1) en-1". */ public class ConditionalExpression extends AbstractNode { - final List expressionList; + public final List expressionList; public ConditionalExpression(List expressionList, Type type) { super(ExpressionType.Conditional, type); From 5544771296859468c69e85b8ba91b6b163c41298 Mon Sep 17 00:00:00 2001 From: Steven Phillips Date: Mon, 25 May 2026 14:52:00 -0700 Subject: [PATCH 289/562] [CALCITE-7542] RexCall.isAlwaysTrue()/isAlwaysFalse() incorrectly returns true for CAST(boolean AS non-boolean) RexCall.isAlwaysTrue() and isAlwaysFalse() group CAST with IS_TRUE/IS_NOT_FALSE (and IS_FALSE/IS_NOT_TRUE) and delegate to the operand. That is wrong when the cast changes the type: CAST(TRUE AS INTEGER) is an INTEGER expression that evaluates to 1, not a boolean, so it must report isAlwaysTrue() == false. Fix: add a top-level "if (getType() != BOOLEAN) return false" guard, mirroring the pattern already used by RexLiteral.isAlwaysTrue()/isAlwaysFalse(). The guard subsumes the CAST case and is future-proof against new switch cases that might return from a non-boolean kind. Tests: - Five unit tests in RexProgramTest covering CAST(boolean AS INTEGER), CAST(boolean AS BOOLEAN), and the recursion path through CAST(non-boolean RexCall AS BOOLEAN). - An end-to-end Quidem probe in conditions.iq that exercises the CAST recursion path with a babel WHERE clause. Co-authored-by: Sean Broeder --- .../java/org/apache/calcite/rex/RexCall.java | 10 ++++ .../apache/calcite/rex/RexProgramTest.java | 49 +++++++++++++++++++ core/src/test/resources/sql/conditions.iq | 17 +++++++ 3 files changed, 76 insertions(+) diff --git a/core/src/main/java/org/apache/calcite/rex/RexCall.java b/core/src/main/java/org/apache/calcite/rex/RexCall.java index 5be1fead88e7..deac203607d8 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexCall.java +++ b/core/src/main/java/org/apache/calcite/rex/RexCall.java @@ -219,6 +219,12 @@ private boolean digestWithType() { @Override public boolean isAlwaysTrue() { // "c IS NOT NULL" occurs when we expand EXISTS. // This reduction allows us to convert it to a semi-join. + // Only boolean-valued calls can be always-true; e.g. CAST(TRUE AS INTEGER) + // evaluates to 1 (INTEGER), not a boolean, even though its operand is + // always true. + if (getType().getSqlTypeName() != SqlTypeName.BOOLEAN) { + return false; + } switch (getKind()) { case IS_NOT_NULL: return !operands.get(0).getType().isNullable(); @@ -241,6 +247,10 @@ private boolean digestWithType() { } @Override public boolean isAlwaysFalse() { + // Only boolean-valued calls can be always-false; see isAlwaysTrue(). + if (getType().getSqlTypeName() != SqlTypeName.BOOLEAN) { + return false; + } switch (getKind()) { case IS_NULL: return !operands.get(0).getType().isNullable(); diff --git a/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java b/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java index 01da3e67727b..2bdc7a1d56c4 100644 --- a/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java +++ b/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java @@ -4130,6 +4130,55 @@ private void checkSarg(String message, Sarg sarg, checkSimplify(isTrue(like(ref, literal("%"))), "IS NOT NULL($0)"); } + /** Unit tests for + * [CALCITE-7542] + * RexCall.isAlwaysTrue()/isAlwaysFalse() incorrectly returns true + * for CAST(boolean AS non-boolean). */ + @Test void testIsAlwaysTrueCastBooleanToInteger() { + // CAST(TRUE AS INTEGER) is not a boolean expression; isAlwaysTrue() must be false. + final RexNode castTrueToInt = abstractCast(trueLiteral, tInt()); + assertThat("CAST(TRUE AS INTEGER).isAlwaysTrue()", + castTrueToInt.isAlwaysTrue(), is(false)); + assertThat("CAST(TRUE AS INTEGER).isAlwaysFalse()", + castTrueToInt.isAlwaysFalse(), is(false)); + } + + @Test void testIsAlwaysFalseCastBooleanToInteger() { + // CAST(FALSE AS INTEGER) is not a boolean expression; isAlwaysFalse() must be false. + final RexNode castFalseToInt = abstractCast(falseLiteral, tInt()); + assertThat("CAST(FALSE AS INTEGER).isAlwaysFalse()", + castFalseToInt.isAlwaysFalse(), is(false)); + assertThat("CAST(FALSE AS INTEGER).isAlwaysTrue()", + castFalseToInt.isAlwaysTrue(), is(false)); + } + + @Test void testIsAlwaysTrueCastBooleanToBoolean() { + // CAST(TRUE AS BOOLEAN) preserves the always-true property. + final RexNode castTrueToBool = abstractCast(trueLiteral, tBool()); + assertThat("CAST(TRUE AS BOOLEAN).isAlwaysTrue()", + castTrueToBool.isAlwaysTrue(), is(true)); + } + + @Test void testIsAlwaysFalseCastBooleanToBoolean() { + // CAST(FALSE AS BOOLEAN) preserves the always-false property. + final RexNode castFalseToBool = abstractCast(falseLiteral, tBool()); + assertThat("CAST(FALSE AS BOOLEAN).isAlwaysFalse()", + castFalseToBool.isAlwaysFalse(), is(true)); + } + + @Test void testIsAlwaysTrueFalseCastNonBooleanCallToBoolean() { + // CAST(1 + 1 AS BOOLEAN): the inner operand is a non-literal, non-boolean + // RexCall. isAlwaysTrue()/isAlwaysFalse() must safely return false on the + // recursive call rather than crashing — the contract is "ask freely, get + // a safe false for non-boolean expressions," matching RexLiteral and the + // RexNode base class. + final RexNode castIntExprToBool = abstractCast(plus(literal(1), literal(1)), tBool()); + assertThat("CAST(1 + 1 AS BOOLEAN).isAlwaysTrue()", + castIntExprToBool.isAlwaysTrue(), is(false)); + assertThat("CAST(1 + 1 AS BOOLEAN).isAlwaysFalse()", + castIntExprToBool.isAlwaysFalse(), is(false)); + } + /** Unit tests for * [CALCITE-2438] * RexCall#isAlwaysTrue returns incorrect result. */ diff --git a/core/src/test/resources/sql/conditions.iq b/core/src/test/resources/sql/conditions.iq index 7ce0c78c9c20..18150050b6ba 100644 --- a/core/src/test/resources/sql/conditions.iq +++ b/core/src/test/resources/sql/conditions.iq @@ -595,4 +595,21 @@ where 5 < cast(deptno as integer) OR 5 >= cast(deptno as integer) OR deptno IS N EnumerableTableScan(table=[[scott, EMP]]) !plan +# Probe: CAST(int_expr AS BOOLEAN) in a WHERE clause exercises the recursion +# path of RexCall.isAlwaysTrue/False through the CAST case. +!use scott-babel + +with t(a) as (values (0), (1), (2)) +select * from t where cast(a + 1 as boolean); ++---+ +| A | ++---+ +| 0 | +| 1 | +| 2 | ++---+ +(3 rows) + +!ok + # End conditions.iq From 29aeb488630d6fc4832dd62309f80dc81ae881b3 Mon Sep 17 00:00:00 2001 From: iwanttobepowerful <745778074@qq.com> Date: Mon, 25 May 2026 10:31:45 +0800 Subject: [PATCH 290/562] [CALCITE-7543] RelBuilder.join should preserve variablesSet for RIGHT/FULL joins --- .../org/apache/calcite/tools/RelBuilder.java | 20 ++- .../apache/calcite/test/RelBuilderTest.java | 46 ++++--- core/src/test/resources/sql/sub-query.iq | 123 ++++++++++++++++++ 3 files changed, 162 insertions(+), 27 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java index 9551c45395fa..cf9ccecdefef 100644 --- a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java +++ b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java @@ -4465,21 +4465,17 @@ boolean isSimple() { /** * Checks for {@link CorrelationId}, then validates the id is not used on left, - * and finally checks if id is actually used on right. + * and finally checks whether the join should be converted to a {@link Correlate}. * - * @return true if a correlate id is present and used + * @return true if the join should be converted to a Correlate; false if it should remain a Join * - * @throws IllegalArgumentException if the {@link CorrelationId} is used by left side or if the a - * {@link CorrelationId} is present and the {@link JoinRelType} is FULL or RIGHT. + * @throws IllegalArgumentException if the {@link CorrelationId} is used by left side */ private boolean checkIfCorrelated(Set variablesSet, JoinRelType joinType, RelNode leftNode, RelNode rightRel) { if (variablesSet.size() != 1) { return false; } - if (!config.convertCorrelateToJoin()) { - return true; - } CorrelationId id = Iterables.getOnlyElement(variablesSet); if (!RelOptUtil.notContainsCorrelation(leftNode, id, Litmus.IGNORE)) { throw new IllegalArgumentException("variable " + id @@ -4490,12 +4486,14 @@ private boolean checkIfCorrelated(Set variablesSet, case ASOF: case RIGHT: case FULL: - throw new IllegalArgumentException("Correlated " + joinType + " join is not supported"); + return false; default: - return !RelOptUtil.correlationColumns( - Iterables.getOnlyElement(variablesSet), - rightRel).isEmpty(); + break; + } + if (!config.convertCorrelateToJoin()) { + return true; } + return !RelOptUtil.correlationColumns(id, rightRel).isEmpty(); } diff --git a/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java b/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java index b54e08e26d4f..f1a4f1c27162 100644 --- a/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java @@ -5090,28 +5090,42 @@ void testInnerCorrelateViaJoin(boolean convertCorrelateToJoin) { hasTree(expected)); } - @Test void testSimpleRightCorrelateViaJoinThrowsException() { - assertThrows(IllegalArgumentException.class, - () -> buildSimpleCorrelateWithJoin(JoinRelType.RIGHT), - "Right outer joins with correlated ids are invalid even if id is not used."); + @Test void testSimpleRightCorrelateViaJoin() { + RelNode root = buildSimpleCorrelateWithJoin(JoinRelType.RIGHT); + final String expected = "" + + "LogicalJoin(condition=[=($7, $8)], joinType=[right], variablesSet=[[$cor0]])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n"; + assertThat(root, hasTree(expected)); } - @Test void testSimpleFullCorrelateViaJoinThrowsException() { - assertThrows(IllegalArgumentException.class, - () -> buildSimpleCorrelateWithJoin(JoinRelType.FULL), - "Full outer joins with correlated ids are invalid even if id is not used."); + @Test void testSimpleFullCorrelateViaJoin() { + RelNode root = buildSimpleCorrelateWithJoin(JoinRelType.FULL); + final String expected = "" + + "LogicalJoin(condition=[=($7, $8)], joinType=[full], variablesSet=[[$cor0]])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n"; + assertThat(root, hasTree(expected)); } - @Test void testRightCorrelateViaJoinThrowsException() { - assertThrows(IllegalArgumentException.class, - () -> buildCorrelateWithJoin(JoinRelType.RIGHT), - "Right outer joins with correlated ids are invalid."); + @Test void testRightCorrelateViaJoin() { + RelNode root = buildCorrelateWithJoin(JoinRelType.RIGHT); + final String expected = "" + + "LogicalJoin(condition=[=($7, $8)], joinType=[right], variablesSet=[[$cor0]])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalFilter(condition=[=($cor0.EMPNO, 'NaN')])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n"; + assertThat(root, hasTree(expected)); } - @Test void testFullCorrelateViaJoinThrowsException() { - assertThrows(IllegalArgumentException.class, - () -> buildCorrelateWithJoin(JoinRelType.FULL), - "Full outer joins with correlated ids are invalid."); + @Test void testFullCorrelateViaJoin() { + RelNode root = buildCorrelateWithJoin(JoinRelType.FULL); + final String expected = "" + + "LogicalJoin(condition=[=($7, $8)], joinType=[full], variablesSet=[[$cor0]])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalFilter(condition=[=($cor0.EMPNO, 'NaN')])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n"; + assertThat(root, hasTree(expected)); } private static RelNode buildSimpleCorrelateWithJoin(JoinRelType type) { diff --git a/core/src/test/resources/sql/sub-query.iq b/core/src/test/resources/sql/sub-query.iq index a04c8634d240..28604937956e 100644 --- a/core/src/test/resources/sql/sub-query.iq +++ b/core/src/test/resources/sql/sub-query.iq @@ -4975,6 +4975,129 @@ ORDER BY e1.empno, e1.deptno; !ok +# [CALCITE-7543] RelBuilder.join should preserve variablesSet for RIGHT/FULL joins +# this was validated using postgres +SELECT empno +FROM emp e +RIGHT JOIN dept d + ON e.deptno = d.deptno + AND e.sal < ( + SELECT MAX(e2.sal) + FROM emp e2 + WHERE e2.deptno = d.deptno + ) +ORDER BY empno; ++-------+ +| EMPNO | ++-------+ +| 7369 | +| 7499 | +| 7521 | +| 7566 | +| 7654 | +| 7782 | +| 7844 | +| 7876 | +| 7900 | +| 7934 | +| | ++-------+ +(11 rows) + +!ok + +SELECT empno +FROM emp e +FULL JOIN dept d + ON e.deptno = d.deptno + AND e.sal < ( + SELECT MAX(e2.sal) + FROM emp e2 + WHERE e2.deptno = d.deptno + ) +ORDER BY empno; ++-------+ +| EMPNO | ++-------+ +| 7369 | +| 7499 | +| 7521 | +| 7566 | +| 7654 | +| 7698 | +| 7782 | +| 7788 | +| 7839 | +| 7844 | +| 7876 | +| 7900 | +| 7902 | +| 7934 | +| | ++-------+ +(15 rows) + +!ok + +SELECT empno +FROM emp e +RIGHT JOIN dept d + ON e.deptno = d.deptno + AND d.deptno <= ALL ( + SELECT d2.deptno + FROM dept d2 + WHERE d2.dname <> d.dname + ) +ORDER BY empno; ++-------+ +| EMPNO | ++-------+ +| 7782 | +| 7839 | +| 7934 | +| | +| | +| | ++-------+ +(6 rows) + +!ok + +SELECT empno +FROM emp e +FULL JOIN dept d + ON e.deptno = d.deptno + AND d.deptno <= ALL ( + SELECT d2.deptno + FROM dept d2 + WHERE d2.dname <> d.dname + ) +ORDER BY empno; ++-------+ +| EMPNO | ++-------+ +| 7369 | +| 7499 | +| 7521 | +| 7566 | +| 7654 | +| 7698 | +| 7782 | +| 7788 | +| 7839 | +| 7844 | +| 7876 | +| 7900 | +| 7902 | +| 7934 | +| | +| | +| | ++-------+ +(17 rows) + +!ok + # [CALCITE-6041] MAP sub-query gives NullPointerException # map size > 1 SELECT map(SELECT empno, deptno from emp where deptno < 20); From 881969da2aacc30276a6585472f37618a140a789 Mon Sep 17 00:00:00 2001 From: Takaaki Nakama Date: Sun, 24 May 2026 01:36:58 +0900 Subject: [PATCH 291/562] [CALCITE-7547] BIG_QUERY conformance should allow field access on UNNEST(array_of_struct) AS alias Under SqlConformanceEnum.BIG_QUERY, SELECT i.name FROM t, UNNEST(t.items) AS i where items is ARRAY> failed validation with "Column 'NAME' not found in table 'I'" because allowAliasUnnestItems() returned false. With the flag off, SqlUnnestOperator#inferReturnType flattens the ROW element type into individual columns instead of keeping it as a single struct-typed column the alias can wrap, and AliasNamespace cannot remap the flattened result back under the alias. BigQuery itself supports this access pattern, so extend the existing PRESTO case in SqlConformanceEnum#allowAliasUnnestItems() to also cover BIG_QUERY. This complements CALCITE-7546, which removes the downstream NPE in SqlToRelConverter#convertUnnest for the 2-operand AS(UNNEST, alias) form when the flag is enabled. --- .../calcite/sql/validate/SqlConformance.java | 1 + .../sql/validate/SqlConformanceEnum.java | 1 + .../calcite/test/SqlToRelConverterTest.java | 26 ++++++++++++++ .../apache/calcite/test/SqlValidatorTest.java | 31 +++++++++++++++++ .../calcite/test/SqlToRelConverterTest.xml | 34 +++++++++++++++++++ 5 files changed, 93 insertions(+) diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlConformance.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlConformance.java index fe902759db16..fcd3b46641f6 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlConformance.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlConformance.java @@ -375,6 +375,7 @@ default boolean isColonFieldAccessAllowed() { * fields of T if T is a STRUCT type. * *

      Among the built-in conformance levels, true in + * {@link SqlConformanceEnum#BIG_QUERY}, * {@link SqlConformanceEnum#PRESTO}; * false otherwise. */ diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlConformanceEnum.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlConformanceEnum.java index 047f05981a16..f0f258171b3d 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlConformanceEnum.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlConformanceEnum.java @@ -435,6 +435,7 @@ public enum SqlConformanceEnum implements SqlConformance { @Override public boolean allowAliasUnnestItems() { switch (this) { + case BIG_QUERY: case PRESTO: return true; default: diff --git a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java index 9217e74895ea..4e8d17cd7a27 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java @@ -1941,6 +1941,32 @@ public static void checkActualAndReferenceFiles() { sql(sql).withConformance(SqlConformanceEnum.PRESTO).ok(); } + /** + * Test case for + * [CALCITE-7547] + * BIG_QUERY conformance should allow field access on + * UNNEST(array_of_struct) AS alias. + */ + @Test void testAliasUnnestArrayPlanWithSingleColumnBigQuery() { + final String sql = "select d.deptno, employee.empno\n" + + "from dept_nested_expanded as d,\n" + + " UNNEST(d.employees) as t(employee)"; + sql(sql).withConformance(SqlConformanceEnum.BIG_QUERY).ok(); + } + + /** + * Test case for + * [CALCITE-7547] + * BIG_QUERY conformance should allow field access on + * UNNEST(array_of_struct) AS alias. + */ + @Test void testAliasUnnestArrayPlanWithDoubleColumnBigQuery() { + final String sql = "select d.deptno, e, k.empno\n" + + "from dept_nested_expanded as d CROSS JOIN\n" + + " UNNEST(d.admins, d.employees) as t(e, k)"; + sql(sql).withConformance(SqlConformanceEnum.BIG_QUERY).ok(); + } + @Test void testArrayOfRecord() { sql("select employees[1].detail.skills[2+3].desc from dept_nested").ok(); } diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index 5198762c106b..9a080efbf862 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -8816,6 +8816,37 @@ void testGroupExpressionEquivalenceParams() { + "\\('EMPNO', 'ENAME', 'DETAIL'\\), whereas alias list has 1 columns"); } + /** + * Test case for + * [CALCITE-7547] + * BIG_QUERY conformance should allow field access on + * UNNEST(array_of_struct) AS alias. + */ + @Test void testAliasUnnestMultipleArraysBigQuery() { + // for accessing a field in STRUCT type unnested from array + sql("select e.ENAME\n" + + "from dept_nested_expanded as d CROSS JOIN\n" + + " UNNEST(d.employees) as t(e)") + .withConformance(SqlConformanceEnum.BIG_QUERY).columnType("VARCHAR(10) NOT NULL"); + + // for unnesting multiple arrays at the same time + sql("select d.deptno, e, k.empno, l.\"unit\", l.\"X\" * l.\"Y\"\n" + + "from dept_nested_expanded as d CROSS JOIN\n" + + " UNNEST(d.admins, d.employees, d.offices) as t(e, k, l)") + .withConformance(SqlConformanceEnum.BIG_QUERY).ok(); + + // Make sure validation fails properly given illegal select items + sql("select d.deptno, ^e^.some_column, k.empno\n" + + "from dept_nested_expanded as d CROSS JOIN\n" + + " UNNEST(d.admins, d.employees) as t(e, k)") + .withConformance(SqlConformanceEnum.BIG_QUERY) + .fails("Table 'E' not found"); + sql("select d.deptno, e.detail, ^unknown^.detail\n" + + "from dept_nested_expanded as d CROSS JOIN\n" + + " UNNEST(d.employees) as t(e)") + .withConformance(SqlConformanceEnum.BIG_QUERY).fails("Incompatible types"); + } + @Test void testUnnestArray() { sql("select*from unnest(array[1])") .columnType("INTEGER NOT NULL"); diff --git a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml index 56f169629ae6..5ee7180efa75 100644 --- a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml @@ -341,6 +341,23 @@ LogicalProject(DEPTNO=[$0], E=[$5], EMPNO=[$6.EMPNO]) + + + + + + + + @@ -358,6 +375,23 @@ LogicalProject(DEPTNO=[$0], EMPNO=[$5.EMPNO]) + + + + + + + + From d81cd60f37479c0f5a08aba65420a3fd05bbb74f Mon Sep 17 00:00:00 2001 From: iwanttobepowerful <745778074@qq.com> Date: Wed, 3 Jun 2026 11:08:38 +0800 Subject: [PATCH 292/562] [CALCITE-7540] Correlated outer reference in HAVING of grouped subquery is incorrectly reported as not grouped --- .../calcite/sql/validate/AggChecker.java | 18 +++++ .../calcite/sql2rel/RelDecorrelator.java | 15 +++- .../calcite/sql2rel/RelDecorrelatorTest.java | 79 +++++++++++++++++++ .../apache/calcite/test/SqlValidatorTest.java | 10 +++ core/src/test/resources/sql/sub-query.iq | 26 ++++++ 5 files changed, 145 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql/validate/AggChecker.java b/core/src/main/java/org/apache/calcite/sql/validate/AggChecker.java index 21253ef31f00..23928b83ac69 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/AggChecker.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/AggChecker.java @@ -139,6 +139,12 @@ && isMeasureExp(id)) { if (isGroupExpr(fqId.identifier)) { return null; } + + // Outer references are not governed by the GROUP BY list of this + // aggregate query; they are validated in their own enclosing scope. + if (isOuterReference(firstScope, fqId)) { + return null; + } SqlNode originalExpr = validator.getOriginal(id); final String exprString = originalExpr.toString(); throw validator.newValidationError(originalExpr, @@ -147,6 +153,18 @@ && isMeasureExp(id)) { : RESOURCE.notGroupExpr(exprString)); } + /** Returns whether an identifier refers to a scope outside this aggregate. */ + private boolean isOuterReference(SqlValidatorScope scope, SqlQualified fqId) { + if (!(scope instanceof AggregatingSelectScope) || fqId.prefixLength <= 0) { + return false; + } + + final SqlValidatorScope currentSelectScope = ((AggregatingSelectScope) scope).parent; + final SqlValidatorScope.ResolvedImpl resolved = new SqlValidatorScope.ResolvedImpl(); + scope.resolve(fqId.prefix(), validator.catalogReader.nameMatcher(), false, resolved); + return resolved.count() == 1 && !resolved.only().scope.isWithin(currentSelectScope); + } + @Override public Void visit(SqlCall call) { final SqlValidatorScope scope = requireNonNull(scopes.peek(), () -> "scope for " + call); diff --git a/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java b/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java index a450a659dc74..14d72f8dfe69 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java @@ -1634,11 +1634,20 @@ private Frame decorrelateInputWithValueGenerator(RelNode rel, Frame inputFrame) findCorrelationEquivalent(correlation, ((Filter) rel).getCondition()); } catch (Util.FoundOne e) { Object node = requireNonNull(e.getNode(), "e.getNode()"); - if (node instanceof RexInputRef) { - map.put(def, ((RexInputRef) node).getIndex()); + // findCorrelationEquivalent returns an expression from the original + // Filter condition, so its input refs are still in the pre-decorrelation + // coordinate system. Correlation outputs are recorded against the + // decorrelated input; translate the expression first, otherwise a + // correlated field may be bound to a column with the same ordinal but + // different meaning in the new RelNode. + final RexNode newNode = node instanceof RexInputRef + ? getNewForOldInputRef(rel, this.map, (RexInputRef) node) + : decorrelateExpr(rel, this.map, cm, (RexNode) node); + if (newNode instanceof RexInputRef) { + map.put(def, ((RexInputRef) newNode).getIndex()); } else { map.put(def, inputFrame.r.getRowType().getFieldCount() + projects.size()); - projects.add((RexNode) node); + projects.add(newNode); } } } diff --git a/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java b/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java index 15f96eaa9e5f..245b248157b9 100644 --- a/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java +++ b/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java @@ -1831,6 +1831,85 @@ public static Frameworks.ConfigBuilder config() { assertThat(after, hasTree(planAfter)); } + /** Test case for [CALCITE-7540] + * Correlated outer reference in HAVING of grouped subquery is incorrectly reported as not + * grouped. */ + @Test void test7540() { + final FrameworkConfig frameworkConfig = config().build(); + final RelBuilder builder = RelBuilder.create(frameworkConfig); + final RelOptCluster cluster = builder.getCluster(); + final Planner planner = Frameworks.getPlanner(frameworkConfig); + final String sql = "" + + "SELECT *\n" + + "FROM dept d\n" + + "INNER JOIN emp e\n" + + " ON e.sal < (\n" + + " SELECT MAX(e2.sal)\n" + + " FROM emp e2\n" + + " WHERE e2.job = e.job\n" + + " GROUP BY e2.job\n" + + " HAVING MIN(e2.deptno) = d.deptno\n" + + " )"; + final RelNode originalRel; + try { + final SqlNode parse = planner.parse(sql); + final SqlNode validate = planner.validate(parse); + originalRel = planner.rel(validate).rel; + } catch (Exception e) { + throw TestUtil.rethrow(e); + } + + final HepProgram hepProgram = HepProgram.builder() + .addRuleCollection( + ImmutableList.of( + // SubQuery program rules + CoreRules.FILTER_SUB_QUERY_TO_CORRELATE, + CoreRules.PROJECT_SUB_QUERY_TO_CORRELATE, + CoreRules.JOIN_SUB_QUERY_TO_CORRELATE)) + .build(); + final Program program = + Programs.of(hepProgram, true, + requireNonNull(cluster.getMetadataProvider())); + final RelNode before = + program.run(cluster.getPlanner(), originalRel, cluster.traitSet(), + Collections.emptyList(), Collections.emptyList()); + final String planBefore = "" + + "LogicalProject(DEPTNO=[$0], DNAME=[$1], LOC=[$2], EMPNO=[$3], ENAME=[$4], JOB=[$5], MGR=[$6], HIREDATE=[$7], SAL=[$8], COMM=[$9], DEPTNO0=[$10])\n" + + " LogicalProject(DEPTNO=[$0], DNAME=[$1], LOC=[$2], EMPNO=[$3], ENAME=[$4], JOB=[$5], MGR=[$6], HIREDATE=[$7], SAL=[$8], COMM=[$9], DEPTNO0=[$10])\n" + + " LogicalFilter(condition=[<($8, $11)])\n" + + " LogicalCorrelate(correlation=[$cor1], joinType=[left], requiredColumns=[{0, 5}])\n" + + " LogicalJoin(condition=[true], joinType=[inner])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalAggregate(group=[{}], agg#0=[SINGLE_VALUE($0)])\n" + + " LogicalProject(EXPR$0=[$1])\n" + + " LogicalFilter(condition=[=($2, $cor1.DEPTNO)])\n" + + " LogicalAggregate(group=[{0}], EXPR$0=[MAX($1)], agg#1=[MIN($2)])\n" + + " LogicalProject(JOB=[$2], SAL=[$5], DEPTNO=[$7])\n" + + " LogicalFilter(condition=[=($2, $cor1.JOB)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + assertThat(before, hasTree(planBefore)); + + // Decorrelate without any rules, just "purely" decorrelation algorithm on RelDecorrelator + final RelNode after = + RelDecorrelator.decorrelateQuery(before, builder, RuleSets.ofList(Collections.emptyList()), + RuleSets.ofList(Collections.emptyList())); + final String planAfter = "" + + "LogicalProject(DEPTNO=[$0], DNAME=[$1], LOC=[$2], EMPNO=[$3], ENAME=[$4], JOB=[$5], MGR=[$6], HIREDATE=[$7], SAL=[$8], COMM=[$9], DEPTNO0=[$10])\n" + + " LogicalJoin(condition=[AND(=($0, $11), =($5, $12), <($8, $13))], joinType=[inner])\n" + + " LogicalJoin(condition=[true], joinType=[inner])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalAggregate(group=[{0, 1}], agg#0=[SINGLE_VALUE($2)])\n" + + " LogicalProject($f3=[$3], JOB3=[$1], EXPR$0=[$2])\n" + + " LogicalFilter(condition=[IS NOT NULL($3)])\n" + + " LogicalAggregate(group=[{0, 1}], EXPR$0=[MAX($2)], agg#1=[MIN($3)])\n" + + " LogicalProject(JOB=[$2], JOB3=[$2], SAL=[$5], DEPTNO=[$7])\n" + + " LogicalFilter(condition=[IS NOT NULL($2)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + assertThat(after, hasTree(planAfter)); + } + /** Test case for [CALCITE-7442] * Getting Wrong index of Correlated variable inside Subquery after FilterJoinRule. */ @Test void testCorrelatedVariableIndexForInClause() { diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index 9a080efbf862..1cba948097f1 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -6569,6 +6569,16 @@ void testReturnsCorrectRowTypeOnCombinedJoin() { sql("select sum(sal + sal) from emp having sum(sal) > 10").ok(); sql("SELECT deptno FROM emp GROUP BY deptno HAVING ^sal^ > 10") .fails("Expression 'SAL' is not being grouped"); + sql("SELECT *\n" + + "FROM dept d\n" + + "INNER JOIN emp e\n" + + " ON e.sal < (\n" + + " SELECT MAX(e2.sal)\n" + + " FROM emp e2\n" + + " WHERE e2.job = e.job\n" + + " GROUP BY e2.job\n" + + " HAVING MIN(e2.deptno) = d.deptno\n" + + " )").ok(); } @Test void testHavingBetween() { diff --git a/core/src/test/resources/sql/sub-query.iq b/core/src/test/resources/sql/sub-query.iq index 28604937956e..7c7e9fb0535c 100644 --- a/core/src/test/resources/sql/sub-query.iq +++ b/core/src/test/resources/sql/sub-query.iq @@ -9379,4 +9379,30 @@ FROM (SELECT deptno, SUM(sal) AS s FROM emp GROUP BY deptno) t; !ok +# [CALCITE-7540] Correlated outer reference in HAVING of grouped subquery is incorrectly reported as not grouped +SELECT * +FROM dept d +INNER JOIN emp e + ON e.sal < ( + SELECT MAX(e2.sal) + FROM emp e2 + WHERE e2.job = e.job + GROUP BY e2.job + HAVING MIN(e2.deptno) = d.deptno + ); ++--------+------------+----------+-------+--------+----------+------+------------+---------+---------+---------+ +| DEPTNO | DNAME | LOC | EMPNO | ENAME | JOB | MGR | HIREDATE | SAL | COMM | DEPTNO0 | ++--------+------------+----------+-------+--------+----------+------+------------+---------+---------+---------+ +| 10 | ACCOUNTING | NEW YORK | 7369 | SMITH | CLERK | 7902 | 1980-12-17 | 800.00 | | 20 | +| 10 | ACCOUNTING | NEW YORK | 7698 | BLAKE | MANAGER | 7839 | 1981-01-05 | 2850.00 | | 30 | +| 10 | ACCOUNTING | NEW YORK | 7782 | CLARK | MANAGER | 7839 | 1981-06-09 | 2450.00 | | 10 | +| 10 | ACCOUNTING | NEW YORK | 7876 | ADAMS | CLERK | 7788 | 1987-05-23 | 1100.00 | | 20 | +| 10 | ACCOUNTING | NEW YORK | 7900 | JAMES | CLERK | 7698 | 1981-12-03 | 950.00 | | 30 | +| 30 | SALES | CHICAGO | 7521 | WARD | SALESMAN | 7698 | 1981-02-22 | 1250.00 | 500.00 | 30 | +| 30 | SALES | CHICAGO | 7654 | MARTIN | SALESMAN | 7698 | 1981-09-28 | 1250.00 | 1400.00 | 30 | +| 30 | SALES | CHICAGO | 7844 | TURNER | SALESMAN | 7698 | 1981-09-08 | 1500.00 | 0.00 | 30 | ++--------+------------+----------+-------+--------+----------+------+------------+---------+---------+---------+ +(8 rows) + +!ok # End sub-query.iq From 34f3e442f190234640e58ee8eb0dbcc7c7af5e44 Mon Sep 17 00:00:00 2001 From: lawlie8 Date: Wed, 3 Jun 2026 18:13:18 +0530 Subject: [PATCH 293/562] [CALCITE-7577] Upgrade log4j to 2.25.4 from 2.17.1 --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 414fde94b85d..eb0bd778ae14 100644 --- a/gradle.properties +++ b/gradle.properties @@ -140,7 +140,7 @@ junit4.version=4.13.2 junit5.version=5.10.5 kafka-clients.version=2.1.1 kerby.version=1.1.1 -log4j2.version=2.17.1 +log4j2.version=2.25.4 mockito.version=3.12.4 mongodb-driver-sync.version=4.10.2 # 1.43.0 is the last version with Java 8 support From 1959d25b5c3574818c7c8cdcdb909b107b4cc0fd Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Wed, 3 Jun 2026 21:54:22 +0800 Subject: [PATCH 294/562] Update role of Istvan Toth to PMC --- site/_data/contributors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site/_data/contributors.yml b/site/_data/contributors.yml index bff185584428..a0a737eac889 100644 --- a/site/_data/contributors.yml +++ b/site/_data/contributors.yml @@ -151,7 +151,7 @@ apacheId: stoty githubId: stoty org: Cloudera - role: Committer + role: PMC - name: Jacques Nadeau apacheId: jacques githubId: jacques-n From a82b8c262e79ef4f0193c92f9d66d116624484ed Mon Sep 17 00:00:00 2001 From: "cc.cai" <2356672992@qq.com> Date: Thu, 4 Jun 2026 10:33:35 +0800 Subject: [PATCH 295/562] [CALCITE-6512] Support Arrow List type --- .../adapter/arrow/ArrowDirectEnumerator.java | 65 +++++++++++++++++++ .../adapter/arrow/ArrowEnumerable.java | 3 +- .../adapter/arrow/ArrowFieldTypeFactory.java | 18 +++-- .../calcite/adapter/arrow/ArrowTable.java | 46 +++++++++---- .../arrow/ArrowAdapterDataTypesTest.java | 18 +++++ .../calcite/adapter/arrow/ArrowDataTest.java | 57 ++++++++++++++++ 6 files changed, 187 insertions(+), 20 deletions(-) create mode 100644 arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowDirectEnumerator.java diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowDirectEnumerator.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowDirectEnumerator.java new file mode 100644 index 000000000000..2ab896f09c9e --- /dev/null +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowDirectEnumerator.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.adapter.arrow; + +import org.apache.calcite.util.ImmutableIntList; +import org.apache.calcite.util.Util; + +import org.apache.arrow.vector.ipc.ArrowFileReader; +import org.apache.arrow.vector.ipc.message.ArrowRecordBatch; + +import java.io.IOException; + +/** + * Enumerator that reads projected Arrow value-vectors directly. + */ +class ArrowDirectEnumerator extends AbstractArrowEnumerator { + private final Runnable onClose; + + ArrowDirectEnumerator(ArrowFileReader arrowFileReader, ImmutableIntList fields, + Runnable onClose) { + super(arrowFileReader, fields); + this.onClose = onClose; + } + + @Override protected void evaluateOperator(ArrowRecordBatch arrowRecordBatch) { + } + + @Override public boolean moveNext() { + if (currRowIndex >= rowCount - 1) { + final boolean hasNextBatch; + try { + hasNextBatch = arrowFileReader.loadNextBatch(); + } catch (IOException e) { + throw Util.toUnchecked(e); + } + if (hasNextBatch) { + currRowIndex = 0; + this.valueVectors.clear(); + loadNextArrowBatch(); + } + return hasNextBatch; + } else { + currRowIndex++; + return true; + } + } + + @Override public void close() { + onClose.run(); + } +} diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowEnumerable.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowEnumerable.java index 516822567eb8..735c75c8ed8c 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowEnumerable.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowEnumerable.java @@ -56,8 +56,7 @@ class ArrowEnumerable extends AbstractEnumerable { return new ArrowFilterEnumerator(arrowFileReader, fields, filter, onClose); } - throw new IllegalArgumentException( - "The arrow enumerator must have either a filter or a projection"); + return new ArrowDirectEnumerator(arrowFileReader, fields, onClose); } catch (Exception e) { throw Util.toUnchecked(e); } diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowFieldTypeFactory.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowFieldTypeFactory.java index 1693637e8c05..30c738bece84 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowFieldTypeFactory.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowFieldTypeFactory.java @@ -22,6 +22,7 @@ import org.apache.arrow.vector.types.FloatingPointPrecision; import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; /** * Arrow field type. @@ -32,19 +33,20 @@ private ArrowFieldTypeFactory() { throw new UnsupportedOperationException("Utility class"); } - public static RelDataType toType(ArrowType arrowType, JavaTypeFactory typeFactory) { - RelDataType sqlType = of(arrowType, typeFactory); + public static RelDataType toType(Field field, JavaTypeFactory typeFactory) { + RelDataType sqlType = of(field, typeFactory); return typeFactory.createTypeWithNullability(sqlType, true); } /** - * Converts an Arrow type to a Calcite RelDataType. + * Converts an Arrow field to a Calcite RelDataType. * - * @param arrowType the Arrow type to convert + * @param field the Arrow field to convert * @param typeFactory the factory to create the Calcite type * @return the corresponding Calcite RelDataType */ - private static RelDataType of(ArrowType arrowType, JavaTypeFactory typeFactory) { + private static RelDataType of(Field field, JavaTypeFactory typeFactory) { + ArrowType arrowType = field.getType(); switch (arrowType.getTypeID()) { case Int: int bitWidth = ((ArrowType.Int) arrowType).getBitWidth(); @@ -82,6 +84,12 @@ private static RelDataType of(ArrowType arrowType, JavaTypeFactory typeFactory) ((ArrowType.Decimal) arrowType).getScale()); case Time: return typeFactory.createSqlType(SqlTypeName.TIME); + case List: + if (field.getChildren().size() != 1) { + throw new IllegalArgumentException("Arrow List type must have one child field: " + field); + } + RelDataType elementType = toType(field.getChildren().get(0), typeFactory); + return typeFactory.createArrayType(elementType, -1); case Timestamp: ArrowType.Timestamp timestampType = (ArrowType.Timestamp) arrowType; int timestampPrecision; diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTable.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTable.java index ba1568bcd2ad..5afb74e51d3a 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTable.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTable.java @@ -121,18 +121,7 @@ public Enumerable query(DataContext root, ImmutableIntList fields, if (conditions.isEmpty()) { filter = null; - - final List expressionTrees = new ArrayList<>(); - for (int fieldOrdinal : fields) { - Field field = schema.getFields().get(fieldOrdinal); - TreeNode node = TreeBuilder.makeField(field); - expressionTrees.add(TreeBuilder.makeExpression(node, field)); - } - try { - projector = Projector.make(schema, expressionTrees); - } catch (GandivaException e) { - throw Util.toUnchecked(e); - } + projector = makeProjector(fields); } else { projector = null; @@ -208,11 +197,42 @@ private static RelDataType deduceRowType(Schema schema, final RelDataTypeFactory.Builder builder = typeFactory.builder(); for (Field field : schema.getFields()) { builder.add(field.getName(), - ArrowFieldTypeFactory.toType(field.getType(), typeFactory)); + ArrowFieldTypeFactory.toType(field, typeFactory)); } return builder.build(); } + private @Nullable Projector makeProjector(ImmutableIntList fields) { + if (containsListField(fields)) { + // Returning null selects ArrowEnumerable's direct vector-read path. + // Use that path for list fields because Gandiva does not support identity + // projection expressions over Arrow List vectors. + return null; + } + + final List expressionTrees = new ArrayList<>(); + for (int fieldOrdinal : fields) { + Field field = schema.getFields().get(fieldOrdinal); + TreeNode node = TreeBuilder.makeField(field); + expressionTrees.add(TreeBuilder.makeExpression(node, field)); + } + try { + return Projector.make(schema, expressionTrees); + } catch (GandivaException e) { + throw Util.toUnchecked(e); + } + } + + private boolean containsListField(ImmutableIntList fields) { + for (int fieldOrdinal : fields) { + if (schema.getFields().get(fieldOrdinal).getType().getTypeID() + == ArrowType.ArrowTypeID.List) { + return true; + } + } + return false; + } + /** Converts a single {@link ConditionToken} into a Gandiva {@link TreeNode}. */ private TreeNode convertConditionToGandiva(ConditionToken token) { final List treeNodes = new ArrayList<>(2); diff --git a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterDataTypesTest.java b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterDataTypesTest.java index 566b4c531ee9..317d4dc26e91 100644 --- a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterDataTypesTest.java +++ b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterDataTypesTest.java @@ -61,9 +61,27 @@ static void initializeArrowState(@TempDir Path sharedTempDir) ArrowDataTest arrowDataGenerator = new ArrowDataTest(); arrowDataGenerator.writeArrowDataType(dataLocationFile); + File listDataLocationFile = arrowFilesDirectory.resolve("arrowlist.arrow").toFile(); + ArrowDataTest arrowListDataGenerator = new ArrowDataTest(); + arrowListDataGenerator.writeArrowListData(listDataLocationFile); + arrow = ImmutableMap.of("model", modelFileTarget.toAbsolutePath().toString()); } + @Test void testListProject() { + String sql = "select \"intListField\" from arrowlist"; + String plan = "PLAN=ArrowToEnumerableConverter\n" + + " ArrowTableScan(table=[[ARROW, ARROWLIST]], fields=[[0]])\n\n"; + String result = "intListField=[0, 1]\n" + + "intListField=null\n" + + "intListField=[2, null]\n"; + CalciteAssert.that() + .with(arrow) + .query(sql) + .returns(result) + .explainContains(plan); + } + @Test void testTinyIntProject() { String sql = "select \"tinyIntField\" from arrowdatatype"; String plan = "PLAN=ArrowToEnumerableConverter\n" diff --git a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowDataTest.java b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowDataTest.java index 8241a80dd10a..7fd9f19c5ea2 100644 --- a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowDataTest.java +++ b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowDataTest.java @@ -41,6 +41,8 @@ import org.apache.arrow.vector.TinyIntVector; import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.complex.ListVector; +import org.apache.arrow.vector.complex.impl.UnionListWriter; import org.apache.arrow.vector.ipc.ArrowFileWriter; import org.apache.arrow.vector.types.DateUnit; import org.apache.arrow.vector.types.FloatingPointPrecision; @@ -68,6 +70,8 @@ import java.util.Calendar; import java.util.List; +import static org.apache.arrow.vector.complex.BaseRepeatedValueVector.DATA_VECTOR_NAME; + /** * Class that can be used to generate Arrow sample data into a data directory. */ @@ -150,6 +154,15 @@ private Schema makeArrowDateTypeSchema() { return new Schema(childrenBuilder.build(), null); } + + private Schema makeArrowListSchema() { + FieldType listType = FieldType.nullable(new ArrowType.List()); + FieldType elementType = FieldType.nullable(new ArrowType.Int(32, true)); + Field elementField = new Field(DATA_VECTOR_NAME, elementType, null); + Field listField = new Field("intListField", listType, ImmutableList.of(elementField)); + return new Schema(ImmutableList.of(listField), null); + } + private Schema makeArrowSchema() { ImmutableList.Builder childrenBuilder = ImmutableList.builder(); FieldType intType = FieldType.nullable(new ArrowType.Int(32, true)); @@ -329,6 +342,25 @@ public void writeArrowDataType(File file) throws IOException { fileOutputStream.close(); } + + public void writeArrowListData(File file) throws IOException { + Schema arrowSchema = makeArrowListSchema(); + try (RootAllocator allocator = new RootAllocator(Integer.MAX_VALUE); + VectorSchemaRoot vectorSchemaRoot = + VectorSchemaRoot.create(arrowSchema, allocator); + FileOutputStream fileOutputStream = new FileOutputStream(file); + ArrowFileWriter arrowFileWriter = + new ArrowFileWriter(vectorSchemaRoot, null, + fileOutputStream.getChannel())) { + arrowFileWriter.start(); + int rowCount = 3; + vectorSchemaRoot.setRowCount(rowCount); + listField(vectorSchemaRoot.getVector("intListField"), rowCount); + arrowFileWriter.writeBatch(); + arrowFileWriter.end(); + } + } + private void tinyIntField(FieldVector fieldVector, int rowCount) { TinyIntVector tinyIntVector = (TinyIntVector) fieldVector; tinyIntVector.setInitialCapacity(rowCount); @@ -465,6 +497,31 @@ private void timeField(FieldVector fieldVector, int rowCount) { fieldVector.setValueCount(rowCount); } + + private void listField(FieldVector fieldVector, int rowCount) { + ListVector listVector = (ListVector) fieldVector; + listVector.setInitialCapacity(rowCount); + listVector.allocateNew(); + UnionListWriter writer = listVector.getWriter(); + for (int i = 0; i < rowCount; i++) { + writer.setPosition(i); + if (i == 1) { + writer.writeNull(); + } else { + writer.startList(); + writer.writeInt(i); + if (i == 2) { + writer.writeNull(); + } else { + writer.writeInt(i + 1); + } + writer.endList(); + } + } + writer.setValueCount(rowCount); + fieldVector.setValueCount(rowCount); + } + private void timestampSecField(FieldVector fieldVector, int rowCount) { TimeStampSecVector tsVector = (TimeStampSecVector) fieldVector; tsVector.setInitialCapacity(rowCount); From e7545378b4db249872106efca6c12b989068ced5 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Thu, 4 Jun 2026 13:42:05 -0700 Subject: [PATCH 296/562] [CALCITE-7581] RelDataTypeFactoryImpl.createStructType(List<>) should not be final Signed-off-by: Mihai Budiu --- .../rel/type/RelDataTypeFactoryImpl.java | 2 +- .../sql/type/RelDataTypeSystemTest.java | 51 +++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeFactoryImpl.java b/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeFactoryImpl.java index b5d901ec37c4..c0d90578fa7a 100644 --- a/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeFactoryImpl.java +++ b/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeFactoryImpl.java @@ -190,7 +190,7 @@ private RelDataType createStructType(StructKind kind, }); } - @Override public final RelDataType createStructType( + @Override public RelDataType createStructType( final List> fieldList) { return createStructType(fieldList, false); } diff --git a/core/src/test/java/org/apache/calcite/sql/type/RelDataTypeSystemTest.java b/core/src/test/java/org/apache/calcite/sql/type/RelDataTypeSystemTest.java index 2a96292eade8..58d56c99dd1d 100644 --- a/core/src/test/java/org/apache/calcite/sql/type/RelDataTypeSystemTest.java +++ b/core/src/test/java/org/apache/calcite/sql/type/RelDataTypeSystemTest.java @@ -18,7 +18,9 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rel.type.RelDataTypeSystem; import org.apache.calcite.rel.type.RelDataTypeSystemImpl; +import org.apache.calcite.rel.type.StructKind; import org.apache.calcite.runtime.CalciteException; import org.apache.calcite.runtime.Resources; import org.apache.calcite.sql.SqlLiteral; @@ -27,12 +29,18 @@ import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.sql.validate.SqlValidatorException; +import org.apache.calcite.util.Pair; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import org.checkerframework.checker.nullness.qual.Nullable; import org.junit.jupiter.api.Test; +import java.util.AbstractMap; +import java.util.List; +import java.util.Map; + import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -42,6 +50,49 @@ * Tests the inference of return types using {@code RelDataTypeSystem}. */ class RelDataTypeSystemTest { + /** A type factory which creates ROW types with PEEK_FIELDS_NO_EXPAND by default. */ + private static class CustomTypeFactory extends SqlTypeFactoryImpl { + CustomTypeFactory() { + super(RelDataTypeSystem.DEFAULT); + } + + @Override public RelDataType createStructType( + final List typeList, + final List fieldNameList) { + return super.createStructType(StructKind.PEEK_FIELDS_NO_EXPAND, typeList, fieldNameList); + } + + // This method used to be final in the base class, and could not be overridden + @Override public RelDataType createStructType( + final List> fieldList) { + return this.createStructType(Pair.right(fieldList), Pair.left(fieldList)); + } + + @Override @SuppressWarnings("deprecation") + public FieldInfoBuilder builder() { + return new FieldInfoBuilder(this).kind(StructKind.PEEK_FIELDS_NO_EXPAND); + } + } + + @Test public void testCustomRecordFactory() { + // Test that ROW types generated by a custom factory all have the appropriate struct kind + CustomTypeFactory factory = new CustomTypeFactory(); + RelDataType i = factory.createSqlType(SqlTypeName.INTEGER); + RelDataType rel = factory.createStructType(ImmutableList.of(i), ImmutableList.of("x")); + assertThat(rel.getStructKind(), is(StructKind.PEEK_FIELDS_NO_EXPAND)); + + RelDataType rowType = factory.builder() + .add("a", factory.createSqlType(SqlTypeName.INTEGER)) + .add("b", factory.createSqlType(SqlTypeName.VARCHAR)) + .build(); + assertThat(rowType.getStructKind(), is(StructKind.PEEK_FIELDS_NO_EXPAND)); + + // Test case for https://issues.apache.org/jira/browse/CALCITE-7581 + // RelDataTypeFactoryImpl.createStructType(List<>) should not be final + RelDataType rel2 = + factory.createStructType(ImmutableList.of(new AbstractMap.SimpleEntry<>("x", i))); + assertThat(rel2.getStructKind(), is(StructKind.PEEK_FIELDS_NO_EXPAND)); + } /** * Custom type system class that overrides the default decimal plus type derivation and From e2fc3e7c52efe7f382723bfabf93ba7ebf2709e6 Mon Sep 17 00:00:00 2001 From: OldTruckDriver Date: Wed, 3 Jun 2026 17:10:15 +1000 Subject: [PATCH 297/562] [CALCITE-7559] SqlParserUtil.parseTimeTzLiteral should reject unknown time zones --- .../calcite/sql/parser/SqlParserUtil.java | 14 +++++++-- .../calcite/sql/parser/SqlParserUtilTest.java | 31 +++++++++++++++++++ 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql/parser/SqlParserUtil.java b/core/src/main/java/org/apache/calcite/sql/parser/SqlParserUtil.java index 1ba1933fc70e..c48bb846fb8b 100644 --- a/core/src/main/java/org/apache/calcite/sql/parser/SqlParserUtil.java +++ b/core/src/main/java/org/apache/calcite/sql/parser/SqlParserUtil.java @@ -371,16 +371,24 @@ public static SqlTimeTzLiteral parseTimeTzLiteral( final String timeZone = s.substring(lastSpace + 1); final String time = s.substring(0, lastSpace); - final TimeZone tz = TimeZone.getTimeZone(timeZone); - if (tz != null) { + try { + ZoneId zoneId = ZoneId.of(timeZone); + TimeZone tz = TimeZone.getTimeZone(zoneId); pt = DateTimeUtils.parsePrecisionDateTimeLiteral(time, Format.get().time, tz, -1); + } catch (DateTimeException e) { + String message = e.getMessage(); + if (message == null) { + message = "Error parsing TIME ZONE"; + } + throw SqlUtil.newContextException(pos, + RESOURCE.illegalLiteral("TIME WITH TIME ZONE", s, message)); } } if (pt == null) { throw SqlUtil.newContextException(pos, RESOURCE.illegalLiteral("TIME WITH TIME ZONE", s, - RESOURCE.badFormat(DateTimeUtils.TIME_FORMAT_STRING).str())); + RESOURCE.badFormat(DateTimeUtils.TIME_FORMAT_STRING + " zone").str())); } final TimeWithTimeZoneString t = TimeWithTimeZoneString.fromCalendarFields(pt.getCalendar()) .withFraction(pt.getFraction()); diff --git a/core/src/test/java/org/apache/calcite/sql/parser/SqlParserUtilTest.java b/core/src/test/java/org/apache/calcite/sql/parser/SqlParserUtilTest.java index 03de04f860da..d4e9f30d482b 100644 --- a/core/src/test/java/org/apache/calcite/sql/parser/SqlParserUtilTest.java +++ b/core/src/test/java/org/apache/calcite/sql/parser/SqlParserUtilTest.java @@ -19,6 +19,7 @@ import org.apache.calcite.avatica.util.TimeUnit; import org.apache.calcite.runtime.CalciteContextException; import org.apache.calcite.sql.SqlIntervalQualifier; +import org.apache.calcite.sql.SqlTimeTzLiteral; import org.apache.calcite.sql.SqlTimestampTzLiteral; import org.junit.jupiter.api.Test; @@ -94,6 +95,36 @@ public class SqlParserUtilTest { } } + /** Test case for + * [CALCITE-7559] + * SqlParserUtil.parseTimeTzLiteral should reject unknown time zones. */ + @Test void testTimeWithTimeZone() { + SqlParserPos pos = new SqlParserPos(2, 3); + SqlTimeTzLiteral lit = + SqlParserUtil.parseTimeTzLiteral("10:10:10 GMT", pos); + assertThat(lit, hasToString("TIME WITH TIME ZONE '10:10:10 UTC'")); + + // Like parseTimestampTzLiteral, parseTimeTzLiteral should reject unknown + // time zones instead of silently falling back to GMT. + try { + SqlParserUtil.parseTimeTzLiteral("10:10:10 incorrect_zone", pos); + fail("Should be unreachable"); + } catch (CalciteContextException ex) { + assertThat( + ex.getMessage(), is("At line 2, column 3: Illegal TIME WITH TIME ZONE literal " + + "'10:10:10 incorrect_zone': Unknown time-zone ID: incorrect_zone")); + } + + try { + SqlParserUtil.parseTimeTzLiteral("10:10:10", pos); + fail("Should be unreachable"); + } catch (CalciteContextException ex) { + assertThat( + ex.getMessage(), is("At line 2, column 3: Illegal TIME WITH TIME ZONE literal " + + "'10:10:10': not in format 'HH:mm:ss zone'")); + } + } + @Test void testMinuteToSecondIntervalToMillis() { final SqlIntervalQualifier qualifier = new SqlIntervalQualifier(TimeUnit.MINUTE, TimeUnit.SECOND, POSITION); From 945fd90a618c56a917e40df613e544655421c2a8 Mon Sep 17 00:00:00 2001 From: OldTruckDriver Date: Thu, 4 Jun 2026 19:14:10 +1000 Subject: [PATCH 298/562] [CALCITE-7560] SqlFunctions.DateParseFunction.parseTimestamp(..., timeZone) accepts unknown time zones and silently falls back to GMT --- .../apache/calcite/runtime/SqlFunctions.java | 4 +++- .../apache/calcite/test/SqlFunctionsTest.java | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java index c1ebafe8edd4..7211f78de369 100644 --- a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java +++ b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java @@ -5604,7 +5604,9 @@ public long parseTimestamp(String fmtString, String timestamp) { public long parseTimestamp(String fmtString, String timestamp, String timeZone) { - TimeZone tz = TimeZone.getTimeZone(timeZone); + // Validate the zone id (rejecting unknown ids) rather than letting + // TimeZone.getTimeZone silently fall back to GMT. + TimeZone tz = TimeZone.getTimeZone(ZoneId.of(timeZone)); final long millisSinceEpoch = internalParseDatetime(fmtString, timestamp, timeZone); return toLong(new java.sql.Timestamp(millisSinceEpoch), tz); diff --git a/core/src/test/java/org/apache/calcite/test/SqlFunctionsTest.java b/core/src/test/java/org/apache/calcite/test/SqlFunctionsTest.java index 9c7f4b4a216c..96c111bd39d0 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlFunctionsTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlFunctionsTest.java @@ -90,6 +90,7 @@ import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.fail; import static java.nio.charset.StandardCharsets.UTF_8; @@ -2093,4 +2094,22 @@ private long sqlTimestamp(String str) { assertArrayEquals(new Object[]{null, 100}, result.get(0)); assertArrayEquals(new Object[]{null, 200}, result.get(1)); } + + /** Test case for + * [CALCITE-7560] + * SqlFunctions.DateParseFunction.parseTimestamp(..., timeZone) accepts unknown + * time zones and silently falls back to GMT. */ + @Test void testParseTimestampRejectsUnknownTimeZone() { + final SqlFunctions.DateParseFunction parse = new SqlFunctions.DateParseFunction(); + + // A valid time zone is accepted. + assertThat( + parse.parseTimestamp("%Y-%m-%d %H:%M:%S", "2024-01-01 00:00:00", "UTC"), + is(parse.parseTimestamp("%Y-%m-%d %H:%M:%S", "2024-01-01 00:00:00"))); + + // An unknown time zone is rejected rather than silently reinterpreted as GMT. + assertThrows(RuntimeException.class, + () -> parse.parseTimestamp("%Y-%m-%d %H:%M:%S", + "2024-01-01 00:00:00", "Asia/Sanghai")); + } } From 74099483dc71cd754588db0c982a86a49a2c58dc Mon Sep 17 00:00:00 2001 From: "cc.cai" <2356672992@qq.com> Date: Fri, 5 Jun 2026 11:09:00 +0800 Subject: [PATCH 299/562] [CALCITE-7541] Support Binary Arrow types --- .../arrow/AbstractArrowEnumerator.java | 7 +- .../adapter/arrow/ArrowDirectEnumerator.java | 4 ++ .../adapter/arrow/ArrowEnumerable.java | 2 + .../adapter/arrow/ArrowFieldTypeFactory.java | 6 ++ .../calcite/adapter/arrow/ArrowTable.java | 25 +++++-- .../arrow/ArrowAdapterDataTypesTest.java | 22 ++++++ .../calcite/adapter/arrow/ArrowDataTest.java | 67 ++++++++++++++++++- 7 files changed, 125 insertions(+), 8 deletions(-) diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/AbstractArrowEnumerator.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/AbstractArrowEnumerator.java index 8cc08b990475..e188757b0d2c 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/AbstractArrowEnumerator.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/AbstractArrowEnumerator.java @@ -16,6 +16,7 @@ */ package org.apache.calcite.adapter.arrow; +import org.apache.calcite.avatica.util.ByteString; import org.apache.calcite.linq4j.Enumerator; import org.apache.calcite.util.ImmutableIntList; import org.apache.calcite.util.Util; @@ -95,7 +96,11 @@ private static Object getValue(ValueVector vector, int index) { (ArrowType.Timestamp) vector.getField().getType(); return toMillis(rawValue, tsType.getUnit()); } - return vector.getObject(index); + final Object value = vector.getObject(index); + if (value instanceof byte[]) { + return new ByteString((byte[]) value); + } + return value; } /** Converts a raw timestamp value to milliseconds since epoch. diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowDirectEnumerator.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowDirectEnumerator.java index 2ab896f09c9e..0cdec7baeb80 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowDirectEnumerator.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowDirectEnumerator.java @@ -26,6 +26,10 @@ /** * Enumerator that reads projected Arrow value-vectors directly. + * + *

      This path is used for identity projections that Gandiva cannot project + * through the existing {@code Projector} path, such as Arrow binary vectors. + * It is not a replacement for Gandiva expression evaluation. */ class ArrowDirectEnumerator extends AbstractArrowEnumerator { private final Runnable onClose; diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowEnumerable.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowEnumerable.java index 735c75c8ed8c..84ed5997aab2 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowEnumerable.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowEnumerable.java @@ -56,6 +56,8 @@ class ArrowEnumerable extends AbstractEnumerable { return new ArrowFilterEnumerator(arrowFileReader, fields, filter, onClose); } + // No projector and no filter means the query is an identity projection + // that should read selected value-vectors directly. return new ArrowDirectEnumerator(arrowFileReader, fields, onClose); } catch (Exception e) { throw Util.toUnchecked(e); diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowFieldTypeFactory.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowFieldTypeFactory.java index 30c738bece84..017caad8662a 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowFieldTypeFactory.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowFieldTypeFactory.java @@ -66,6 +66,12 @@ private static RelDataType of(Field field, JavaTypeFactory typeFactory) { return typeFactory.createSqlType(SqlTypeName.BOOLEAN); case Utf8: return typeFactory.createSqlType(SqlTypeName.VARCHAR); + case Binary: + case LargeBinary: + return typeFactory.createSqlType(SqlTypeName.VARBINARY); + case FixedSizeBinary: + return typeFactory.createSqlType(SqlTypeName.BINARY, + ((ArrowType.FixedSizeBinary) arrowType).getByteWidth()); case FloatingPoint: FloatingPointPrecision precision = ((ArrowType.FloatingPoint) arrowType).getPrecision(); switch (precision) { diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTable.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTable.java index 5afb74e51d3a..74438efe2a33 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTable.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTable.java @@ -203,10 +203,10 @@ private static RelDataType deduceRowType(Schema schema, } private @Nullable Projector makeProjector(ImmutableIntList fields) { - if (containsListField(fields)) { + if (requiresDirectVectorProjection(fields)) { // Returning null selects ArrowEnumerable's direct vector-read path. - // Use that path for list fields because Gandiva does not support identity - // projection expressions over Arrow List vectors. + // Use that path because Gandiva does not support identity projection + // expressions over Arrow List and binary vectors. return null; } @@ -223,11 +223,24 @@ private static RelDataType deduceRowType(Schema schema, } } - private boolean containsListField(ImmutableIntList fields) { + /** Returns whether selected fields should be projected by reading Arrow + * value-vectors directly rather than by creating a Gandiva projector. + * + *

      CALCITE-7541 extends this direct projection path for Arrow binary vector + * families because Gandiva cannot project them through the existing identity + * projection path. Queries with filters still use Gandiva filters; this direct + * path only applies to no-filter projections. + */ + private boolean requiresDirectVectorProjection(ImmutableIntList fields) { for (int fieldOrdinal : fields) { - if (schema.getFields().get(fieldOrdinal).getType().getTypeID() - == ArrowType.ArrowTypeID.List) { + switch (schema.getFields().get(fieldOrdinal).getType().getTypeID()) { + case List: + case Binary: + case LargeBinary: + case FixedSizeBinary: return true; + default: + break; } } return false; diff --git a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterDataTypesTest.java b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterDataTypesTest.java index 317d4dc26e91..bfd4ee14c280 100644 --- a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterDataTypesTest.java +++ b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterDataTypesTest.java @@ -65,6 +65,10 @@ static void initializeArrowState(@TempDir Path sharedTempDir) ArrowDataTest arrowListDataGenerator = new ArrowDataTest(); arrowListDataGenerator.writeArrowListData(listDataLocationFile); + File binaryDataLocationFile = arrowFilesDirectory.resolve("arrowbinary.arrow").toFile(); + ArrowDataTest arrowBinaryDataGenerator = new ArrowDataTest(); + arrowBinaryDataGenerator.writeArrowBinaryData(binaryDataLocationFile); + arrow = ImmutableMap.of("model", modelFileTarget.toAbsolutePath().toString()); } @@ -82,6 +86,24 @@ static void initializeArrowState(@TempDir Path sharedTempDir) .explainContains(plan); } + /** Test case for + * [CALCITE-7541] + * Support Binary Arrow types in Arrow adapter. */ + @Test void testBinaryProject() { + String sql = "select \"binaryField\", \"largeBinaryField\", \"fixedSizeBinaryField\" " + + "from arrowbinary"; + String plan = "PLAN=ArrowToEnumerableConverter\n" + + " ArrowTableScan(table=[[ARROW, ARROWBINARY]], fields=[[0, 1, 2]])\n\n"; + String result = "binaryField=0001; largeBinaryField=0a0b; fixedSizeBinaryField=141516\n" + + "binaryField=null; largeBinaryField=null; fixedSizeBinaryField=null\n" + + "binaryField=020304; largeBinaryField=0c0d0e; fixedSizeBinaryField=171819\n"; + CalciteAssert.that() + .with(arrow) + .query(sql) + .returns(result) + .explainContains(plan); + } + @Test void testTinyIntProject() { String sql = "select \"tinyIntField\" from arrowdatatype"; String plan = "PLAN=ArrowToEnumerableConverter\n" diff --git a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowDataTest.java b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowDataTest.java index 7fd9f19c5ea2..a53cc1231ad3 100644 --- a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowDataTest.java +++ b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowDataTest.java @@ -29,9 +29,11 @@ import org.apache.arrow.vector.DateDayVector; import org.apache.arrow.vector.DecimalVector; import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.FixedSizeBinaryVector; import org.apache.arrow.vector.Float8Vector; import org.apache.arrow.vector.FloatingPointVector; import org.apache.arrow.vector.IntVector; +import org.apache.arrow.vector.LargeVarBinaryVector; import org.apache.arrow.vector.SmallIntVector; import org.apache.arrow.vector.TimeSecVector; import org.apache.arrow.vector.TimeStampMicroVector; @@ -39,6 +41,7 @@ import org.apache.arrow.vector.TimeStampNanoVector; import org.apache.arrow.vector.TimeStampSecVector; import org.apache.arrow.vector.TinyIntVector; +import org.apache.arrow.vector.VarBinaryVector; import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.arrow.vector.complex.ListVector; @@ -179,6 +182,19 @@ private Schema makeArrowSchema() { return new Schema(childrenBuilder.build(), null); } + private Schema makeArrowBinarySchema() { + ImmutableList.Builder childrenBuilder = ImmutableList.builder(); + FieldType binaryType = FieldType.nullable(new ArrowType.Binary()); + FieldType largeBinaryType = FieldType.nullable(new ArrowType.LargeBinary()); + FieldType fixedSizeBinaryType = + FieldType.nullable(new ArrowType.FixedSizeBinary(3)); + + childrenBuilder.add(new Field("binaryField", binaryType, null)); + childrenBuilder.add(new Field("largeBinaryField", largeBinaryType, null)); + childrenBuilder.add(new Field("fixedSizeBinaryField", fixedSizeBinaryType, null)); + + return new Schema(childrenBuilder.build(), null); + } public void writeScottEmpData(Path arrowDataDirectory) throws IOException, SQLException { List tableNames = ImmutableList.of("EMP", "DEPT", "SALGRADE"); @@ -265,6 +281,26 @@ public void writeArrowData(File file) throws IOException { fileOutputStream.close(); } + public void writeArrowBinaryData(File file) throws IOException { + FileOutputStream fileOutputStream = new FileOutputStream(file); + Schema arrowSchema = makeArrowBinarySchema(); + VectorSchemaRoot vectorSchemaRoot = + VectorSchemaRoot.create(arrowSchema, new RootAllocator(Integer.MAX_VALUE)); + ArrowFileWriter arrowFileWriter = + new ArrowFileWriter(vectorSchemaRoot, null, fileOutputStream.getChannel()); + + arrowFileWriter.start(); + vectorSchemaRoot.setRowCount(3); + binaryField(vectorSchemaRoot.getVector("binaryField")); + largeBinaryField(vectorSchemaRoot.getVector("largeBinaryField")); + fixedSizeBinaryField(vectorSchemaRoot.getVector("fixedSizeBinaryField")); + arrowFileWriter.writeBatch(); + arrowFileWriter.end(); + arrowFileWriter.close(); + fileOutputStream.flush(); + fileOutputStream.close(); + } + public void writeArrowDataType(File file) throws IOException { FileOutputStream fileOutputStream = new FileOutputStream(file); Schema arrowSchema = makeArrowDateTypeSchema(); @@ -342,7 +378,6 @@ public void writeArrowDataType(File file) throws IOException { fileOutputStream.close(); } - public void writeArrowListData(File file) throws IOException { Schema arrowSchema = makeArrowListSchema(); try (RootAllocator allocator = new RootAllocator(Integer.MAX_VALUE); @@ -361,6 +396,36 @@ public void writeArrowListData(File file) throws IOException { } } + private void binaryField(FieldVector fieldVector) { + VarBinaryVector binaryVector = (VarBinaryVector) fieldVector; + binaryVector.setInitialCapacity(3); + binaryVector.allocateNew(); + binaryVector.setSafe(0, new byte[] {0, 1}); + binaryVector.setNull(1); + binaryVector.setSafe(2, new byte[] {2, 3, 4}); + fieldVector.setValueCount(3); + } + + private void largeBinaryField(FieldVector fieldVector) { + LargeVarBinaryVector largeBinaryVector = (LargeVarBinaryVector) fieldVector; + largeBinaryVector.setInitialCapacity(3); + largeBinaryVector.allocateNew(); + largeBinaryVector.setSafe(0, new byte[] {10, 11}); + largeBinaryVector.setNull(1); + largeBinaryVector.setSafe(2, new byte[] {12, 13, 14}); + fieldVector.setValueCount(3); + } + + private void fixedSizeBinaryField(FieldVector fieldVector) { + FixedSizeBinaryVector fixedSizeBinaryVector = (FixedSizeBinaryVector) fieldVector; + fixedSizeBinaryVector.setInitialCapacity(3); + fixedSizeBinaryVector.allocateNew(); + fixedSizeBinaryVector.setSafe(0, new byte[] {20, 21, 22}); + fixedSizeBinaryVector.setNull(1); + fixedSizeBinaryVector.setSafe(2, new byte[] {23, 24, 25}); + fieldVector.setValueCount(3); + } + private void tinyIntField(FieldVector fieldVector, int rowCount) { TinyIntVector tinyIntVector = (TinyIntVector) fieldVector; tinyIntVector.setInitialCapacity(rowCount); From 81bbb8cdae164332bfc7391746a5ad40986fa6b5 Mon Sep 17 00:00:00 2001 From: Sergey Nuyanzin Date: Thu, 28 May 2026 00:52:57 +0200 Subject: [PATCH 300/562] [CALCITE-7562] SqlToRel misses `CAST` in case `IN` expression without type coercion --- .../calcite/sql2rel/SqlToRelConverter.java | 34 ++++++++++++++++--- .../calcite/test/SqlToRelConverterTest.java | 18 ++++++++++ .../calcite/test/SqlToRelConverterTest.xml | 24 +++++++++++++ .../apache/calcite/test/SqlToRelFixture.java | 5 +++ 4 files changed, 77 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java index 4622176a0733..32017ace1d43 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java @@ -1870,7 +1870,7 @@ public RelNode convertToSingleValueSubq( if (leftKeys.size() == 1) { SqlCall sqlCall = comparisonOp.createCall(rightVals.getParserPosition(), leftKeys.get(0), rightVals); - rexComparison = bb.convertExpression(sqlCall); + rexComparison = ensureComparisonTypes(bb.convertExpression(sqlCall)); } else { assert rightVals instanceof SqlCall; final SqlBasicCall call = (SqlBasicCall) rightVals; @@ -1880,9 +1880,10 @@ public RelNode convertToSingleValueSubq( RexUtil.composeConjunction(rexBuilder, transform( Pair.zip(leftKeys, call.getOperandList()), - pair -> bb.convertExpression( - comparisonOp.createCall(rightVals.getParserPosition(), - pair.left, pair.right)))); + pair -> ensureComparisonTypes( + bb.convertExpression( + comparisonOp.createCall(rightVals.getParserPosition(), + pair.left, pair.right))))); } comparisons.add(rexComparison); } @@ -1901,6 +1902,31 @@ public RelNode convertToSingleValueSubq( } } + /** + * Ensures that a comparison expression has matching operand types. If the + * operands have different type names, casts the right operand to match the + * left operand's type. This handles the case where type coercion is disabled + * and the IN-to-OR expansion produces comparisons with mismatched types + * (e.g., DATE = CHAR). + */ + private RexNode ensureComparisonTypes(RexNode node) { + if (validator != null && validator.config().typeCoercionEnabled()) { + return node; + } + if (node instanceof RexCall) { + final RexCall call = (RexCall) node; + if (call.operands.size() == 2) { + final RexNode left = call.operands.get(0); + final RexNode right = call.operands.get(1); + if (left.getType().getSqlTypeName() != right.getType().getSqlTypeName()) { + final RexNode castRight = rexBuilder.ensureType(left.getType(), right, true); + return rexBuilder.makeCall(call.getOperator(), left, castRight); + } + } + } + return node; + } + /** * Converts a {@link SqlNodeList} (for example an IN-list or VALUES list) * into a relational expression and produces a Rex-level sub-query that diff --git a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java index 4e8d17cd7a27..723d749be6b5 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java @@ -2213,6 +2213,24 @@ void checkCorrelatedMapSubQuery(boolean expand) { sql(sql).withExpand(false).ok(); } + /** Test case for + * [CALCITE-7562] + * SqlToRel misses CAST in case IN expression without type coercion. */ + @Test void testInDateColumnWithoutTypeCoercion() { + final String sql = + "select * from emp_b where birthdate in ('2000-06-30', '2000-09-27')"; + sql(sql).withTypeCoercion(false).ok(); + } + + /** Test case for + * [CALCITE-7562] + * SqlToRel misses CAST in case IN expression without type coercion. */ + @Test void testInDateColumnWithTypeCoercion() { + final String sql = + "select * from emp_b where birthdate in ('2000-06-30', '2000-09-27')"; + sql(sql).ok(); + } + @Test void testInValueListLong() { // Go over the default threshold of 20 to force a sub-query. final String sql = "select empno from emp where deptno in" diff --git a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml index 5ee7180efa75..8aa2171d3f0b 100644 --- a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml @@ -3266,6 +3266,30 @@ LogicalFilter(condition=[=($cor0.DEPTNO, $7)]) LogicalAggregate(group=[{0}], S=[SUM($1)], agg#1=[COUNT()]) LogicalProject(DEPTNO=[$7], SAL=[$5]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +]]> + + + + + + + + + + + + + + + + diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlToRelFixture.java b/testkit/src/main/java/org/apache/calcite/test/SqlToRelFixture.java index a6685cf952f5..784cb1eba1dd 100644 --- a/testkit/src/main/java/org/apache/calcite/test/SqlToRelFixture.java +++ b/testkit/src/main/java/org/apache/calcite/test/SqlToRelFixture.java @@ -178,6 +178,11 @@ public SqlToRelFixture withConformance(SqlConformance conformance) { .withValidatorConfig(c -> c.withConformance(conformance))); } + public SqlToRelFixture withTypeCoercion(boolean enabled) { + return withFactory(f -> + f.withValidatorConfig(c -> c.withTypeCoercionEnabled(enabled))); + } + public SqlToRelFixture withDiffRepos(DiffRepository diffRepos) { return new SqlToRelFixture(sql, decorrelate, tester, factory, trim, expression, diffRepos); From 82d93a15af2e3f4b750ccce300653045a0a2d264 Mon Sep 17 00:00:00 2001 From: Sergey Nuyanzin Date: Sun, 31 May 2026 19:35:52 +0200 Subject: [PATCH 301/562] [CALCITE-7567] `LeastRestrictiveSqlType` for `TIMESTAMP`, `TIMESTAMP_LTZ` might ignore precision --- .../calcite/sql/type/SqlTypeFactoryImpl.java | 2 +- .../calcite/sql/type/SqlTypeFactoryTest.java | 25 +++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/calcite/sql/type/SqlTypeFactoryImpl.java b/core/src/main/java/org/apache/calcite/sql/type/SqlTypeFactoryImpl.java index 115b66fa215f..d0ccab7dfd2f 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/SqlTypeFactoryImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/type/SqlTypeFactoryImpl.java @@ -561,7 +561,7 @@ private static void assertBasic(SqlTypeName typeName) { } } - if (type.getSqlTypeName() == resultType.getSqlTypeName() + if (type.getSqlTypeName().getFamily() == resultType.getSqlTypeName().getFamily() && type.getSqlTypeName().allowsPrec() && type.getPrecision() != resultType.getPrecision()) { final int precision = diff --git a/core/src/test/java/org/apache/calcite/sql/type/SqlTypeFactoryTest.java b/core/src/test/java/org/apache/calcite/sql/type/SqlTypeFactoryTest.java index 8f5e5e4018db..f0ff190d28c7 100644 --- a/core/src/test/java/org/apache/calcite/sql/type/SqlTypeFactoryTest.java +++ b/core/src/test/java/org/apache/calcite/sql/type/SqlTypeFactoryTest.java @@ -33,6 +33,7 @@ import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.notNullValue; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.hasToString; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -172,6 +173,30 @@ class SqlTypeFactoryTest { assertThat(leastRestrictive.getPrecision(), is(3)); } + /** + * Test case for + * + * LeastRetrictiveSqlType for TIMESTAMP, TIMESTAMP_LTZ might ignore precision. */ + @Test void testLeastRestrictiveForTimestampAndTimestampLtz() { + SqlTypeFixture f = new SqlTypeFixture(); + RelDataType ltz0 = + f.typeFactory.createSqlType(SqlTypeName.TIMESTAMP_WITH_LOCAL_TIME_ZONE, 0); + RelDataType leastRestrictive = + f.typeFactory.leastRestrictive(Lists.newArrayList(ltz0, f.sqlTimestampPrec3)); + assertThat(leastRestrictive, is(notNullValue())); + assertThat(leastRestrictive.getPrecision(), is(3)); + } + + @Test void testLeastRestrictiveForTimestampLtzAndTimestamp() { + SqlTypeFixture f = new SqlTypeFixture(); + RelDataType ltz0 = + f.typeFactory.createSqlType(SqlTypeName.TIMESTAMP_WITH_LOCAL_TIME_ZONE, 0); + RelDataType leastRestrictive = + f.typeFactory.leastRestrictive(Lists.newArrayList(f.sqlTimestampPrec3, ltz0)); + assertThat(leastRestrictive, is(notNullValue())); + assertThat(leastRestrictive.getPrecision(), is(3)); + } + @Test void testLeastRestrictiveForTimestampAndDate() { SqlTypeFixture f = new SqlTypeFixture(); RelDataType leastRestrictive = From 006ddbc39da61106a7f8dbe54de95ca9b03653b5 Mon Sep 17 00:00:00 2001 From: Sergey Nuyanzin Date: Thu, 4 Jun 2026 07:57:40 +0200 Subject: [PATCH 302/562] [CALCITE-7578] `LIKE` with empty `ESCAPE` might fail with StringIndexOutOfBoundsException --- .../java/org/apache/calcite/rex/RexSimplify.java | 13 ++++++++----- .../java/org/apache/calcite/rex/RexProgramTest.java | 10 ++++++++++ 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rex/RexSimplify.java b/core/src/main/java/org/apache/calcite/rex/RexSimplify.java index 4d7702295bff..4bfad68636c4 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexSimplify.java +++ b/core/src/main/java/org/apache/calcite/rex/RexSimplify.java @@ -529,11 +529,14 @@ private RexNode simplifyLike(RexCall e, RexUnknownAs unknownAs) { } if (e.operands.size() == 3 && e.operands.get(2) instanceof RexLiteral) { final RexLiteral escapeLiteral = (RexLiteral) e.operands.get(2); - Character escape = requireNonNull(escapeLiteral.getValueAs(Character.class)); - e = (RexCall) rexBuilder - .makeCall(e.getParserPosition(), e.getOperator(), e.operands.get(0), - rexBuilder.makeLiteral(simplifyLikeString(likeStr, escape, '%'), - e.operands.get(1).getType(), true, true), escapeLiteral); + final String escapeStr = requireNonNull(escapeLiteral.getValueAs(String.class)); + if (escapeStr.length() == 1) { + char escape = escapeStr.charAt(0); + e = (RexCall) rexBuilder + .makeCall(e.getParserPosition(), e.getOperator(), e.operands.get(0), + rexBuilder.makeLiteral(simplifyLikeString(likeStr, escape, '%'), + e.operands.get(1).getType(), true, true), escapeLiteral); + } } } return simplifyGenericNode(e); diff --git a/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java b/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java index 2bdc7a1d56c4..a596d687fb2b 100644 --- a/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java +++ b/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java @@ -4358,6 +4358,8 @@ private void checkSarg(String message, Sarg sarg, * Multiple consecutive '%' in the string matched by LIKE should simplify to a single '%', * [CALCITE-7153] * Mixed wildcards of _ and % need to be simplified in LIKE operator. + * [CALCITE-7578] + * LIKE with empty ESCAPE might fail with StringIndexOutOfBoundsException. * */ @Test void testSimplifyLike() { final RexNode ref = input(tVarchar(true, 10), 0); @@ -4451,6 +4453,14 @@ private void checkSarg(String message, Sarg sarg, // NOT(SIMILAR TO) is not optimized checkSimplifyUnchanged( not(rexBuilder.makeCall(SqlStdOperatorTable.SIMILAR_TO, ref, literal("%")))); + + try { + // Empty ESCAPE + checkSimplifyUnchanged(like(ref, literal("a"), literal(""))); + } catch (RuntimeException e) { + assertThat(e.getMessage(), + containsString("Invalid escape character ''")); + } } @Test void testSimplifyNullCheckInFilter() { From ea7827852e5faddcddb720828014346b15f36458 Mon Sep 17 00:00:00 2001 From: Sergey Nuyanzin Date: Fri, 5 Jun 2026 14:11:58 +0200 Subject: [PATCH 303/562] [CALCITE-7588] `LIKE` with `ESCAPE` symbols containing wildcards fails --- .../java/org/apache/calcite/rex/RexSimplify.java | 13 ++++++++++++- .../java/org/apache/calcite/rex/RexProgramTest.java | 10 ++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/calcite/rex/RexSimplify.java b/core/src/main/java/org/apache/calcite/rex/RexSimplify.java index 4bfad68636c4..0ffb60454e33 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexSimplify.java +++ b/core/src/main/java/org/apache/calcite/rex/RexSimplify.java @@ -546,7 +546,7 @@ private RexNode simplifyLike(RexCall e, RexUnknownAs unknownAs) { // string with even escapes 'AA\\\\%%__%%AA' simplify to 'AA\\__%AA' // string with odd escapes 'AA\\\\\\%%__%%AA' simplify to 'AA\\\\\\%__%AA' private String simplifyMixedWildcards(String str, char escape) { - Pattern pattern = Pattern.compile("[_%]+"); + Pattern pattern = getWildCardPattern(escape); Matcher matcher = pattern.matcher(str); StringBuilder builder = new StringBuilder(); int from = 0; @@ -570,6 +570,17 @@ && consecutiveSameCharCountBefore(str, start - 1, escape) % 2 == 1) { return builder.toString(); } + private static Pattern getWildCardPattern(char escape) { + switch (escape) { + case '%': + return Pattern.compile("_+"); + case '_': + return Pattern.compile("%+"); + default: + return Pattern.compile("[_%]+"); + } + } + // Tool method: count the number of consecutive identical characters before index private int consecutiveSameCharCountBefore(String str, int index, char escape) { int count = 0; diff --git a/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java b/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java index a596d687fb2b..ac7b5aebe6c3 100644 --- a/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java +++ b/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java @@ -4360,6 +4360,8 @@ private void checkSarg(String message, Sarg sarg, * Mixed wildcards of _ and % need to be simplified in LIKE operator. * [CALCITE-7578] * LIKE with empty ESCAPE might fail with StringIndexOutOfBoundsException. + * [CALCITE-7588] + * LIKE with ESCAPE symbols containing wildcards fails. * */ @Test void testSimplifyLike() { final RexNode ref = input(tVarchar(true, 10), 0); @@ -4423,6 +4425,14 @@ private void checkSarg(String message, Sarg sarg, "LIKE($0, '###%%#%#%A#%%#%A%###%%', '#')"); checkSimplifyUnchanged(like(ref, literal("A"), literal("#"))); checkSimplifyUnchanged(like(ref, literal("%A"), literal("#"))); + checkSimplifyUnchanged(like(ref, literal("TE%_ST"), literal("%"))); + checkSimplifyUnchanged(like(ref, literal("TE%%ST"), literal("%"))); + checkSimplifyUnchanged(like(ref, literal("a%_b%%c"), literal("%"))); + checkSimplifyUnchanged(like(ref, literal("%_%%A%_"), literal("%"))); + // escape char equal to the '_' wildcard. '%E__S%' ESCAPE '_' is a literal '_'. + checkSimplifyUnchanged(like(ref, literal("%E__S%"), literal("_"))); + checkSimplifyUnchanged(like(ref, literal("TE_%ST"), literal("_"))); + checkSimplifyUnchanged(like(ref, literal("a_%b__c"), literal("_"))); // As above, but ref is NOT NULL final RexNode refMandatory = vVarcharNotNull(0); From 13a167a46c9957109686a609b6a15273c454a823 Mon Sep 17 00:00:00 2001 From: "cc.cai" <2356672992@qq.com> Date: Sat, 6 Jun 2026 22:43:38 +0800 Subject: [PATCH 304/562] [CALCITE-7580] Remove Gandiva dependency from Arrow adapter --- arrow/build.gradle.kts | 1 - .../arrow/AbstractArrowEnumerator.java | 30 ++- .../adapter/arrow/ArrowDirectEnumerator.java | 31 +-- .../adapter/arrow/ArrowEnumerable.java | 25 +- .../adapter/arrow/ArrowFilterEnumerator.java | 238 +++++++++++++----- .../adapter/arrow/ArrowProjectEnumerator.java | 80 ------ .../calcite/adapter/arrow/ArrowRules.java | 2 +- .../calcite/adapter/arrow/ArrowTable.java | 143 +---------- .../adapter/arrow/ArrowTranslator.java | 69 +++-- .../calcite/adapter/arrow/ConditionToken.java | 51 +++- .../adapter/arrow/ArrowAdapterTest.java | 50 ++++ .../calcite/adapter/arrow/ArrowDataTest.java | 29 +++ .../calcite/adapter/arrow/ArrowExtension.java | 23 +- bom/build.gradle.kts | 1 - .../org/apache/calcite/rex/RexSimplify.java | 26 +- .../calcite/sql/type/SqlTypeFactoryImpl.java | 2 +- .../calcite/sql2rel/SqlToRelConverter.java | 34 +-- .../apache/calcite/rex/RexProgramTest.java | 20 -- .../calcite/sql/type/SqlTypeFactoryTest.java | 25 -- .../calcite/test/SqlToRelConverterTest.java | 18 -- .../calcite/test/SqlToRelConverterTest.xml | 24 -- gradle.properties | 1 - site/_docs/history.md | 5 + .../apache/calcite/test/SqlToRelFixture.java | 5 - 24 files changed, 408 insertions(+), 525 deletions(-) delete mode 100644 arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowProjectEnumerator.java diff --git a/arrow/build.gradle.kts b/arrow/build.gradle.kts index 598aa8a87972..c75a8b5f6751 100644 --- a/arrow/build.gradle.kts +++ b/arrow/build.gradle.kts @@ -23,7 +23,6 @@ dependencies { implementation("com.google.guava:guava") implementation("org.apache.arrow:arrow-memory-netty") implementation("org.apache.arrow:arrow-vector") - implementation("org.apache.arrow.gandiva:arrow-gandiva") annotationProcessor("org.immutables:value") compileOnly("org.immutables:value-annotations") diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/AbstractArrowEnumerator.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/AbstractArrowEnumerator.java index e188757b0d2c..486e3f60bd8d 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/AbstractArrowEnumerator.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/AbstractArrowEnumerator.java @@ -24,9 +24,7 @@ import org.apache.arrow.vector.TimeStampVector; import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.VectorSchemaRoot; -import org.apache.arrow.vector.VectorUnloader; import org.apache.arrow.vector.ipc.ArrowFileReader; -import org.apache.arrow.vector.ipc.message.ArrowRecordBatch; import org.apache.arrow.vector.types.TimeUnit; import org.apache.arrow.vector.types.pojo.ArrowType; @@ -51,8 +49,6 @@ abstract class AbstractArrowEnumerator implements Enumerator { this.currRowIndex = -1; } - abstract void evaluateOperator(ArrowRecordBatch arrowRecordBatch); - protected void loadNextArrowBatch() { try { final VectorSchemaRoot vsr = arrowFileReader.getVectorSchemaRoot(); @@ -60,14 +56,32 @@ protected void loadNextArrowBatch() { this.valueVectors.add(vsr.getVector(i)); } this.rowCount = vsr.getRowCount(); - VectorUnloader vectorUnloader = new VectorUnloader(vsr); - ArrowRecordBatch arrowRecordBatch = vectorUnloader.getRecordBatch(); - evaluateOperator(arrowRecordBatch); } catch (IOException e) { throw Util.toUnchecked(e); } } + /** Loads the next non-empty Arrow batch. */ + protected boolean loadNextNonEmptyArrowBatch() { + while (true) { + final boolean hasNextBatch; + try { + hasNextBatch = arrowFileReader.loadNextBatch(); + } catch (IOException e) { + throw Util.toUnchecked(e); + } + if (!hasNextBatch) { + return false; + } + currRowIndex = -1; + valueVectors.clear(); + loadNextArrowBatch(); + if (rowCount > 0) { + return true; + } + } + } + @Override public Object current() { if (fields.size() == 1) { return getValue(this.valueVectors.get(0), currRowIndex); @@ -85,7 +99,7 @@ protected void loadNextArrowBatch() { *

      For {@link TimeStampVector}, converts the raw value to * milliseconds since epoch, which is the representation used by * Calcite's Enumerable runtime for TIMESTAMP types. */ - private static Object getValue(ValueVector vector, int index) { + protected static Object getValue(ValueVector vector, int index) { if (vector instanceof TimeStampVector) { if (vector.isNull(index)) { return null; diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowDirectEnumerator.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowDirectEnumerator.java index 0cdec7baeb80..787ffd88d933 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowDirectEnumerator.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowDirectEnumerator.java @@ -17,19 +17,11 @@ package org.apache.calcite.adapter.arrow; import org.apache.calcite.util.ImmutableIntList; -import org.apache.calcite.util.Util; import org.apache.arrow.vector.ipc.ArrowFileReader; -import org.apache.arrow.vector.ipc.message.ArrowRecordBatch; - -import java.io.IOException; /** * Enumerator that reads projected Arrow value-vectors directly. - * - *

      This path is used for identity projections that Gandiva cannot project - * through the existing {@code Projector} path, such as Arrow binary vectors. - * It is not a replacement for Gandiva expression evaluation. */ class ArrowDirectEnumerator extends AbstractArrowEnumerator { private final Runnable onClose; @@ -40,27 +32,14 @@ class ArrowDirectEnumerator extends AbstractArrowEnumerator { this.onClose = onClose; } - @Override protected void evaluateOperator(ArrowRecordBatch arrowRecordBatch) { - } - @Override public boolean moveNext() { - if (currRowIndex >= rowCount - 1) { - final boolean hasNextBatch; - try { - hasNextBatch = arrowFileReader.loadNextBatch(); - } catch (IOException e) { - throw Util.toUnchecked(e); - } - if (hasNextBatch) { - currRowIndex = 0; - this.valueVectors.clear(); - loadNextArrowBatch(); + while (currRowIndex >= rowCount - 1) { + if (!loadNextNonEmptyArrowBatch()) { + return false; } - return hasNextBatch; - } else { - currRowIndex++; - return true; } + currRowIndex++; + return true; } @Override public void close() { diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowEnumerable.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowEnumerable.java index 84ed5997aab2..b9c0c4171e65 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowEnumerable.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowEnumerable.java @@ -21,11 +21,10 @@ import org.apache.calcite.util.ImmutableIntList; import org.apache.calcite.util.Util; -import org.apache.arrow.gandiva.evaluator.Filter; -import org.apache.arrow.gandiva.evaluator.Projector; import org.apache.arrow.vector.ipc.ArrowFileReader; +import org.apache.arrow.vector.types.pojo.Schema; -import org.checkerframework.checker.nullness.qual.Nullable; +import java.util.List; /** * Enumerable that reads from Arrow value-vectors. @@ -33,28 +32,24 @@ class ArrowEnumerable extends AbstractEnumerable { private final ArrowFileReader arrowFileReader; private final ImmutableIntList fields; - private final @Nullable Projector projector; - private final @Nullable Filter filter; + private final List>> conditions; + private final Schema schema; private final Runnable onClose; ArrowEnumerable(ArrowFileReader arrowFileReader, ImmutableIntList fields, - @Nullable Projector projector, @Nullable Filter filter, - Runnable onClose) { + List>> conditions, Schema schema, Runnable onClose) { this.arrowFileReader = arrowFileReader; - this.projector = projector; - this.filter = filter; + this.conditions = conditions; + this.schema = schema; this.fields = fields; this.onClose = onClose; } @Override public Enumerator enumerator() { try { - if (projector != null) { - return new ArrowProjectEnumerator(arrowFileReader, fields, projector, - onClose); - } else if (filter != null) { - return new ArrowFilterEnumerator(arrowFileReader, fields, filter, - onClose); + if (!conditions.isEmpty()) { + return new ArrowFilterEnumerator(arrowFileReader, fields, + conditions, schema, onClose); } // No projector and no filter means the query is an identity projection // that should read selected value-vectors directly. diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowFilterEnumerator.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowFilterEnumerator.java index 5eddec224909..2f154cc6ef7e 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowFilterEnumerator.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowFilterEnumerator.java @@ -19,91 +19,215 @@ import org.apache.calcite.util.ImmutableIntList; import org.apache.calcite.util.Util; -import org.apache.arrow.gandiva.evaluator.Filter; -import org.apache.arrow.gandiva.evaluator.SelectionVector; -import org.apache.arrow.gandiva.evaluator.SelectionVectorInt16; -import org.apache.arrow.gandiva.exceptions.GandivaException; -import org.apache.arrow.memory.ArrowBuf; -import org.apache.arrow.memory.BufferAllocator; -import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.ValueVector; +import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.arrow.vector.ipc.ArrowFileReader; -import org.apache.arrow.vector.ipc.message.ArrowRecordBatch; - -import org.checkerframework.checker.nullness.qual.Nullable; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.Schema; import java.io.IOException; +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; import static java.util.Objects.requireNonNull; /** - * Enumerator that reads from a filtered collection of Arrow value-vectors. + * Enumerator that evaluates Arrow filter tokens in Java. */ class ArrowFilterEnumerator extends AbstractArrowEnumerator { - private final BufferAllocator allocator; - private final Filter filter; - private @Nullable ArrowBuf buf; - private @Nullable SelectionVector selectionVector; - private int selectionVectorIndex; - + private final List> conditions; + private final Schema schema; private final Runnable onClose; + private final List filterVectors; + private final Map likePatterns; - ArrowFilterEnumerator(ArrowFileReader arrowFileReader, ImmutableIntList fields, - Filter filter, Runnable onClose) { + ArrowFilterEnumerator(ArrowFileReader arrowFileReader, + ImmutableIntList fields, List>> conditions, + Schema schema, Runnable onClose) { super(arrowFileReader, fields); - this.allocator = new RootAllocator(Long.MAX_VALUE); - this.filter = filter; + this.conditions = toConditionTokens(conditions); + this.schema = schema; this.onClose = onClose; + this.filterVectors = new ArrayList<>(schema.getFields().size()); + this.likePatterns = new HashMap<>(); } - @Override void evaluateOperator(ArrowRecordBatch arrowRecordBatch) { + @Override protected void loadNextArrowBatch() { + super.loadNextArrowBatch(); + final VectorSchemaRoot root; try { - this.buf = this.allocator.buffer((long) rowCount * 2); - this.selectionVector = new SelectionVectorInt16(buf); - filter.evaluate(arrowRecordBatch, selectionVector); - } catch (GandivaException e) { + root = arrowFileReader.getVectorSchemaRoot(); + } catch (IOException e) { throw Util.toUnchecked(e); } + filterVectors.clear(); + for (int i = 0; i < schema.getFields().size(); i++) { + filterVectors.add(root.getVector(i)); + } } @Override public boolean moveNext() { - if (selectionVector == null - || selectionVectorIndex >= selectionVector.getRecordCount()) { - boolean hasNextBatch; - while (true) { - try { - hasNextBatch = arrowFileReader.loadNextBatch(); - } catch (IOException e) { - throw Util.toUnchecked(e); + while (true) { + if (currRowIndex >= rowCount - 1) { + if (!loadNextNonEmptyArrowBatch()) { + return false; } - if (hasNextBatch) { - selectionVectorIndex = 0; - this.valueVectors.clear(); - loadNextArrowBatch(); - requireNonNull(selectionVector, "selectionVector"); - if (selectionVectorIndex >= selectionVector.getRecordCount()) { - // the "filtered" batch is empty, but there may be more batches to fetch - continue; - } - currRowIndex = selectionVector.getIndex(selectionVectorIndex++); + } + currRowIndex++; + if (matches(currRowIndex)) { + return true; + } + } + } + + private boolean matches(int rowIndex) { + for (List orGroup : conditions) { + boolean any = false; + for (ConditionToken token : orGroup) { + if (matches(token, rowIndex)) { + any = true; + break; } - return hasNextBatch; } - } else { - currRowIndex = selectionVector.getIndex(selectionVectorIndex++); - return true; + if (!any) { + return false; + } } + return true; } - @Override public void close() { - try { - if (buf != null) { - buf.close(); + private boolean matches(ConditionToken token, int rowIndex) { + final Object value = getValue(fieldVector(token.fieldName), rowIndex); + switch (token.operator) { + case IS_NULL: + return value == null; + case IS_NOT_NULL: + return value != null; + case IS_TRUE: + return Boolean.TRUE.equals(value); + case IS_FALSE: + return Boolean.FALSE.equals(value); + case IS_NOT_TRUE: + return !Boolean.TRUE.equals(value); + case IS_NOT_FALSE: + return !Boolean.FALSE.equals(value); + case EQUAL: + return value != null && compare(value, literal(token)) == 0; + case NOT_EQUAL: + return value != null && compare(value, literal(token)) != 0; + case LESS_THAN: + return value != null && compare(value, literal(token)) < 0; + case LESS_THAN_OR_EQUAL: + return value != null && compare(value, literal(token)) <= 0; + case GREATER_THAN: + return value != null && compare(value, literal(token)) > 0; + case GREATER_THAN_OR_EQUAL: + return value != null && compare(value, literal(token)) >= 0; + case LIKE: + return value != null + && like(value.toString(), requireNonNull(token.value, "value")); + default: + throw new AssertionError("Unhandled Arrow filter operator: " + token.operator); + } + } + + private ValueVector fieldVector(String fieldName) { + final Field field = schema.findField(fieldName); + final int index = schema.getFields().indexOf(field); + if (index < 0) { + throw new IllegalArgumentException("Unknown Arrow field: " + fieldName); + } + return filterVectors.get(index); + } + + private static Object literal(ConditionToken token) { + final String type = requireNonNull(token.valueType, "valueType"); + final String value = requireNonNull(token.value, "value"); + if (type.startsWith("decimal")) { + return new BigDecimal(value); + } else if (type.equals("integer")) { + return Integer.valueOf(value); + } else if (type.equals("long")) { + return Long.valueOf(value); + } else if (type.equals("float")) { + return Float.valueOf(value); + } else if (type.equals("double")) { + return Double.valueOf(value); + } else if (type.equals("string")) { + return unquote(value); + } + throw new UnsupportedOperationException("Unsupported literal type: " + type); + } + + private static int compare(Object left, Object right) { + if (left instanceof BigDecimal || right instanceof BigDecimal) { + return toBigDecimal(left).compareTo(toBigDecimal(right)); + } + if (left instanceof Number && right instanceof Number) { + return Double.compare(((Number) left).doubleValue(), + ((Number) right).doubleValue()); + } + return left.toString().compareTo(right.toString()); + } + + private static BigDecimal toBigDecimal(Object value) { + if (value instanceof BigDecimal) { + return (BigDecimal) value; + } + return new BigDecimal(value.toString()); + } + + private boolean like(String value, String pattern) { + final String unquotedPattern = unquote(pattern); + final Pattern compiledPattern = + likePatterns.computeIfAbsent(unquotedPattern, p -> { + return Pattern.compile(toRegex(p), Pattern.DOTALL); + }); + return compiledPattern.matcher(value).matches(); + } + + private static String toRegex(String pattern) { + final StringBuilder builder = new StringBuilder(); + for (int i = 0; i < pattern.length(); i++) { + final char c = pattern.charAt(i); + if (c == '%') { + builder.append(".*"); + } else if (c == '_') { + builder.append('.'); + } else { + builder.append(Pattern.quote(String.valueOf(c))); } - filter.close(); - } catch (GandivaException e) { - throw Util.toUnchecked(e); - } finally { - onClose.run(); } + return builder.toString(); + } + + private static String unquote(String value) { + if (value.length() >= 2 && value.charAt(0) == '\'' + && value.charAt(value.length() - 1) == '\'') { + return value.substring(1, value.length() - 1).replace("''", "'"); + } + return value; + } + + private static List> toConditionTokens( + List>> conditions) { + final List> result = + new ArrayList<>(conditions.size()); + for (List> orGroup : conditions) { + final List tokens = new ArrayList<>(orGroup.size()); + for (List token : orGroup) { + tokens.add(ConditionToken.fromTokenList(token)); + } + result.add(tokens); + } + return result; + } + + @Override public void close() { + onClose.run(); } } diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowProjectEnumerator.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowProjectEnumerator.java deleted file mode 100644 index 0895f36cf15f..000000000000 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowProjectEnumerator.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.calcite.adapter.arrow; - -import org.apache.calcite.util.ImmutableIntList; -import org.apache.calcite.util.Util; - -import org.apache.arrow.gandiva.evaluator.Projector; -import org.apache.arrow.gandiva.exceptions.GandivaException; -import org.apache.arrow.vector.ipc.ArrowFileReader; -import org.apache.arrow.vector.ipc.message.ArrowRecordBatch; - -import java.io.IOException; - -/** - * Enumerator that reads from a projected collection of Arrow value-vectors. - */ -class ArrowProjectEnumerator extends AbstractArrowEnumerator { - private final Projector projector; - private final Runnable onClose; - - ArrowProjectEnumerator(ArrowFileReader arrowFileReader, ImmutableIntList fields, - Projector projector, Runnable onClose) { - super(arrowFileReader, fields); - this.projector = projector; - this.onClose = onClose; - } - - @Override protected void evaluateOperator(ArrowRecordBatch arrowRecordBatch) { - try { - projector.evaluate(arrowRecordBatch, valueVectors); - } catch (GandivaException e) { - throw Util.toUnchecked(e); - } - } - - @Override public boolean moveNext() { - if (currRowIndex >= rowCount - 1) { - final boolean hasNextBatch; - try { - hasNextBatch = arrowFileReader.loadNextBatch(); - } catch (IOException e) { - throw Util.toUnchecked(e); - } - if (hasNextBatch) { - currRowIndex = 0; - this.valueVectors.clear(); - loadNextArrowBatch(); - } - return hasNextBatch; - } else { - currRowIndex++; - return true; - } - } - - @Override public void close() { - try { - projector.close(); - } catch (GandivaException e) { - throw Util.toUnchecked(e); - } finally { - onClose.run(); - } - } -} diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowRules.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowRules.java index 6e268d646928..3da1527f1014 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowRules.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowRules.java @@ -100,7 +100,7 @@ RelNode convert(Filter filter) { final RelTraitSet traitSet = filter.getTraitSet().replace(ArrowRel.CONVENTION); // Expand SEARCH (e.g. IN, BETWEEN) before pushing to Arrow, - // since Gandiva does not support SEARCH natively. + // since the Arrow adapter does not support SEARCH natively. final RexNode condition = RexUtil.expandSearch(filter.getCluster().getRexBuilder(), null, filter.getCondition()); return new ArrowFilter(filter.getCluster(), traitSet, diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTable.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTable.java index 74438efe2a33..2585f5d156ea 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTable.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTable.java @@ -38,17 +38,9 @@ import org.apache.calcite.util.ImmutableIntList; import org.apache.calcite.util.Util; -import org.apache.arrow.gandiva.evaluator.Filter; -import org.apache.arrow.gandiva.evaluator.Projector; -import org.apache.arrow.gandiva.exceptions.GandivaException; -import org.apache.arrow.gandiva.expression.Condition; -import org.apache.arrow.gandiva.expression.ExpressionTree; -import org.apache.arrow.gandiva.expression.TreeBuilder; -import org.apache.arrow.gandiva.expression.TreeNode; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.ipc.ArrowFileReader; import org.apache.arrow.vector.ipc.SeekableReadChannel; -import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.Schema; @@ -58,20 +50,15 @@ import java.io.FileInputStream; import java.io.IOException; import java.lang.reflect.Type; -import java.util.ArrayList; import java.util.List; -import static java.lang.Double.parseDouble; -import static java.lang.Float.parseFloat; -import static java.lang.Integer.parseInt; -import static java.lang.Long.parseLong; import static java.util.Objects.requireNonNull; /** * Table backed by an Apache Arrow file. * - *

      Reads data from an Arrow IPC file on disk and supports projection - * and filter push-down via the Gandiva expression compiler. + *

      Reads data from an Arrow IPC file on disk. Projections and filters read + * directly from Arrow value-vectors. * *

      Implements {@link TranslatableTable} so that it can be converted into * an {@link ArrowTableScan} for query planning, and {@link QueryableTable} @@ -116,43 +103,6 @@ public class ArrowTable extends AbstractTable public Enumerable query(DataContext root, ImmutableIntList fields, List>> conditions) { requireNonNull(fields, "fields"); - final Projector projector; - final Filter filter; - - if (conditions.isEmpty()) { - filter = null; - projector = makeProjector(fields); - } else { - projector = null; - - final List conjuncts = new ArrayList<>(conditions.size()); - for (List> orGroup : conditions) { - final List disjuncts = new ArrayList<>(orGroup.size()); - for (List conditionParts : orGroup) { - disjuncts.add( - convertConditionToGandiva( - ConditionToken.fromTokenList(conditionParts))); - } - if (disjuncts.size() == 1) { - conjuncts.add(disjuncts.get(0)); - } else { - conjuncts.add(TreeBuilder.makeOr(disjuncts)); - } - } - final Condition filterCondition; - if (conjuncts.size() == 1) { - filterCondition = TreeBuilder.makeCondition(conjuncts.get(0)); - } else { - filterCondition = - TreeBuilder.makeCondition(TreeBuilder.makeAnd(conjuncts)); - } - - try { - filter = Filter.make(schema, filterCondition); - } catch (GandivaException e) { - throw Util.toUnchecked(e); - } - } FileInputStream fis = null; try { @@ -163,7 +113,7 @@ public Enumerable query(DataContext root, ImmutableIntList fields, final FileInputStream fisRef = fis; final Runnable onClose = () -> closeSilently(fisRef); fis = null; // ownership transferred to onClose - return new ArrowEnumerable(reader, fields, projector, filter, onClose); + return new ArrowEnumerable(reader, fields, conditions, schema, onClose); } catch (IOException e) { throw Util.toUnchecked(e); } finally { @@ -202,70 +152,6 @@ private static RelDataType deduceRowType(Schema schema, return builder.build(); } - private @Nullable Projector makeProjector(ImmutableIntList fields) { - if (requiresDirectVectorProjection(fields)) { - // Returning null selects ArrowEnumerable's direct vector-read path. - // Use that path because Gandiva does not support identity projection - // expressions over Arrow List and binary vectors. - return null; - } - - final List expressionTrees = new ArrayList<>(); - for (int fieldOrdinal : fields) { - Field field = schema.getFields().get(fieldOrdinal); - TreeNode node = TreeBuilder.makeField(field); - expressionTrees.add(TreeBuilder.makeExpression(node, field)); - } - try { - return Projector.make(schema, expressionTrees); - } catch (GandivaException e) { - throw Util.toUnchecked(e); - } - } - - /** Returns whether selected fields should be projected by reading Arrow - * value-vectors directly rather than by creating a Gandiva projector. - * - *

      CALCITE-7541 extends this direct projection path for Arrow binary vector - * families because Gandiva cannot project them through the existing identity - * projection path. Queries with filters still use Gandiva filters; this direct - * path only applies to no-filter projections. - */ - private boolean requiresDirectVectorProjection(ImmutableIntList fields) { - for (int fieldOrdinal : fields) { - switch (schema.getFields().get(fieldOrdinal).getType().getTypeID()) { - case List: - case Binary: - case LargeBinary: - case FixedSizeBinary: - return true; - default: - break; - } - } - return false; - } - - /** Converts a single {@link ConditionToken} into a Gandiva {@link TreeNode}. */ - private TreeNode convertConditionToGandiva(ConditionToken token) { - final List treeNodes = new ArrayList<>(2); - treeNodes.add( - TreeBuilder.makeField(schema.getFields() - .get( - schema.getFields().indexOf( - schema.findField(token.fieldName))))); - - if (token.isBinary()) { - treeNodes.add( - makeLiteralNode( - requireNonNull(token.value, "value"), - requireNonNull(token.valueType, "valueType"))); - } - - return TreeBuilder.makeFunction( - token.operator, treeNodes, new ArrowType.Bool()); - } - /** Closes an {@link AutoCloseable} without throwing. */ private static void closeSilently(AutoCloseable closeable) { try { @@ -275,29 +161,6 @@ private static void closeSilently(AutoCloseable closeable) { } } - private static TreeNode makeLiteralNode(String literal, String type) { - if (type.startsWith("decimal")) { - String[] typeParts = - type.substring(type.indexOf('(') + 1, type.indexOf(')')).split(","); - int precision = parseInt(typeParts[0]); - int scale = parseInt(typeParts[1]); - return TreeBuilder.makeDecimalLiteral(literal, precision, scale); - } else if (type.equals("integer")) { - return TreeBuilder.makeLiteral(parseInt(literal)); - } else if (type.equals("long")) { - return TreeBuilder.makeLiteral(parseLong(literal)); - } else if (type.equals("float")) { - return TreeBuilder.makeLiteral(parseFloat(literal)); - } else if (type.equals("double")) { - return TreeBuilder.makeLiteral(parseDouble(literal)); - } else if (type.equals("string")) { - return TreeBuilder.makeStringLiteral(literal.substring(1, literal.length() - 1)); - } else { - throw new IllegalArgumentException("Invalid literal " + literal - + ", type " + type); - } - } - /** * Implementation of {@link Queryable} based on a {@link ArrowTable}. * diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTranslator.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTranslator.java index 0ec680405270..2b4229598153 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTranslator.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTranslator.java @@ -35,13 +35,26 @@ import java.util.ArrayList; import java.util.List; +import static org.apache.calcite.adapter.arrow.ConditionToken.Operator.EQUAL; +import static org.apache.calcite.adapter.arrow.ConditionToken.Operator.GREATER_THAN; +import static org.apache.calcite.adapter.arrow.ConditionToken.Operator.GREATER_THAN_OR_EQUAL; +import static org.apache.calcite.adapter.arrow.ConditionToken.Operator.IS_FALSE; +import static org.apache.calcite.adapter.arrow.ConditionToken.Operator.IS_NOT_FALSE; +import static org.apache.calcite.adapter.arrow.ConditionToken.Operator.IS_NOT_NULL; +import static org.apache.calcite.adapter.arrow.ConditionToken.Operator.IS_NOT_TRUE; +import static org.apache.calcite.adapter.arrow.ConditionToken.Operator.IS_NULL; +import static org.apache.calcite.adapter.arrow.ConditionToken.Operator.IS_TRUE; +import static org.apache.calcite.adapter.arrow.ConditionToken.Operator.LESS_THAN; +import static org.apache.calcite.adapter.arrow.ConditionToken.Operator.LESS_THAN_OR_EQUAL; +import static org.apache.calcite.adapter.arrow.ConditionToken.Operator.LIKE; +import static org.apache.calcite.adapter.arrow.ConditionToken.Operator.NOT_EQUAL; import static org.apache.calcite.util.DateTimeStringUtils.ISO_DATETIME_FRACTIONAL_SECOND_FORMAT; import static org.apache.calcite.util.DateTimeStringUtils.getDateFormatter; import static java.util.Objects.requireNonNull; /** - * Translates a {@link RexNode} expression to Gandiva predicate tokens. + * Translates a {@link RexNode} expression to Arrow predicate tokens. */ class ArrowTranslator { final RexBuilder rexBuilder; @@ -65,7 +78,7 @@ public static ArrowTranslator create(RexBuilder rexBuilder, * *

      If exceeded, {@link RexUtil#toCnf(RexBuilder, int, RexNode)} returns * the original expression unchanged, which may cause the subsequent - * translation to Gandiva predicates to fail with an + * translation to Arrow predicates to fail with an * {@link UnsupportedOperationException}. When invoked by the Arrow adapter * module, the exception is caught and the plan falls back to * an Enumerable convention. */ @@ -120,32 +133,32 @@ private static Object literalValue(RexLiteral literal) { private ConditionToken translateMatch2(RexNode node) { switch (node.getKind()) { case EQUALS: - return translateBinary("equal", "=", (RexCall) node); + return translateBinary(EQUAL, EQUAL, (RexCall) node); case NOT_EQUALS: - return translateBinary("not_equal", "<>", (RexCall) node); + return translateBinary(NOT_EQUAL, NOT_EQUAL, (RexCall) node); case LESS_THAN: - return translateBinary("less_than", ">", (RexCall) node); + return translateBinary(LESS_THAN, GREATER_THAN, (RexCall) node); case LESS_THAN_OR_EQUAL: - return translateBinary("less_than_or_equal_to", ">=", (RexCall) node); + return translateBinary(LESS_THAN_OR_EQUAL, GREATER_THAN_OR_EQUAL, (RexCall) node); case GREATER_THAN: - return translateBinary("greater_than", "<", (RexCall) node); + return translateBinary(GREATER_THAN, LESS_THAN, (RexCall) node); case GREATER_THAN_OR_EQUAL: - return translateBinary("greater_than_or_equal_to", "<=", (RexCall) node); + return translateBinary(GREATER_THAN_OR_EQUAL, LESS_THAN_OR_EQUAL, (RexCall) node); case IS_NULL: - return translateUnary("isnull", (RexCall) node); + return translateUnary(IS_NULL, (RexCall) node); case IS_NOT_NULL: - return translateUnary("isnotnull", (RexCall) node); + return translateUnary(IS_NOT_NULL, (RexCall) node); case IS_NOT_TRUE: - return translateUnary("isnottrue", (RexCall) node); + return translateUnary(IS_NOT_TRUE, (RexCall) node); case IS_NOT_FALSE: - return translateUnary("isnotfalse", (RexCall) node); + return translateUnary(IS_NOT_FALSE, (RexCall) node); case INPUT_REF: final RexInputRef inputRef = (RexInputRef) node; - return ConditionToken.unary(fieldNames.get(inputRef.getIndex()), "istrue"); + return ConditionToken.unary(fieldNames.get(inputRef.getIndex()), IS_TRUE); case NOT: - return translateUnary("isfalse", (RexCall) node); + return translateUnary(IS_FALSE, (RexCall) node); case LIKE: - return translateBinary("like", null, (RexCall) node); + return translateBinaryNoReverse(LIKE, (RexCall) node); default: throw new UnsupportedOperationException("Unsupported operator " + node); } @@ -155,7 +168,8 @@ private ConditionToken translateMatch2(RexNode node) { * Translates a call to a binary operator, reversing arguments if * necessary. */ - private ConditionToken translateBinary(String op, String rop, RexCall call) { + private ConditionToken translateBinary(ConditionToken.Operator op, + ConditionToken.Operator rop, RexCall call) { final RexNode left = call.operands.get(0); final RexNode right = call.operands.get(1); @Nullable ConditionToken expression = translateBinary2(op, left, right); @@ -169,9 +183,21 @@ private ConditionToken translateBinary(String op, String rop, RexCall call) { throw new UnsupportedOperationException("Unsupported binary operator " + call); } + /** Translates a call to a binary operator without reversing arguments. */ + private ConditionToken translateBinaryNoReverse(ConditionToken.Operator op, + RexCall call) { + final RexNode left = call.operands.get(0); + final RexNode right = call.operands.get(1); + @Nullable ConditionToken expression = translateBinary2(op, left, right); + if (expression != null) { + return expression; + } + throw new UnsupportedOperationException("Unsupported binary operator " + call); + } + /** Translates a call to a binary operator. Returns null on failure. */ - private @Nullable ConditionToken translateBinary2(String op, RexNode left, - RexNode right) { + private @Nullable ConditionToken translateBinary2( + ConditionToken.Operator op, RexNode left, RexNode right) { if (right.getKind() != SqlKind.LITERAL) { return null; } @@ -191,7 +217,7 @@ private ConditionToken translateBinary(String op, String rop, RexCall call) { /** Combines a field name, operator, and literal to produce a binary * condition token. */ - private ConditionToken translateOp2(String op, String name, + private ConditionToken translateOp2(ConditionToken.Operator op, String name, RexLiteral right) { Object value = literalValue(right); String valueString = value.toString(); @@ -209,7 +235,7 @@ private ConditionToken translateOp2(String op, String name, } /** Translates a call to a unary operator. */ - private ConditionToken translateUnary(String op, RexCall call) { + private ConditionToken translateUnary(ConditionToken.Operator op, RexCall call) { final RexNode opNode = call.operands.get(0); @Nullable ConditionToken expression = translateUnary2(op, opNode); @@ -221,7 +247,8 @@ private ConditionToken translateUnary(String op, RexCall call) { } /** Translates a call to a unary operator. Returns null on failure. */ - private @Nullable ConditionToken translateUnary2(String op, RexNode opNode) { + private @Nullable ConditionToken translateUnary2(ConditionToken.Operator op, + RexNode opNode) { if (opNode.getKind() == SqlKind.INPUT_REF) { final RexInputRef inputRef = (RexInputRef) opNode; final String name = fieldNames.get(inputRef.getIndex()); diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ConditionToken.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ConditionToken.java index 44d3facea77f..c5b690840add 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ConditionToken.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ConditionToken.java @@ -25,7 +25,7 @@ import static java.util.Objects.requireNonNull; /** - * A structured representation of a single Gandiva predicate condition. + * A structured representation of a single Arrow predicate condition. * *

      A condition is either unary (e.g. {@code IS NULL}) or binary * (e.g. {@code =}, {@code <}). Unary conditions have a field name @@ -36,11 +36,11 @@ */ class ConditionToken { final String fieldName; - final String operator; + final Operator operator; final @Nullable String value; final @Nullable String valueType; - private ConditionToken(String fieldName, String operator, + private ConditionToken(String fieldName, Operator operator, @Nullable String value, @Nullable String valueType) { this.fieldName = requireNonNull(fieldName, "fieldName"); this.operator = requireNonNull(operator, "operator"); @@ -50,7 +50,7 @@ private ConditionToken(String fieldName, String operator, /** Creates a binary condition token * (e.g. {@code intField equal 12 integer}). */ - static ConditionToken binary(String fieldName, String operator, + static ConditionToken binary(String fieldName, Operator operator, String value, String valueType) { return new ConditionToken(fieldName, operator, requireNonNull(value, "value"), @@ -59,7 +59,7 @@ static ConditionToken binary(String fieldName, String operator, /** Creates a unary condition token * (e.g. {@code intField isnull}). */ - static ConditionToken unary(String fieldName, String operator) { + static ConditionToken unary(String fieldName, Operator operator) { return new ConditionToken(fieldName, operator, null, null); } @@ -76,22 +76,55 @@ boolean isBinary() { * binary conditions. */ List toTokenList() { if (isBinary()) { - return ImmutableList.of(fieldName, operator, + return ImmutableList.of(fieldName, operator.token, requireNonNull(value, "value"), requireNonNull(valueType, "valueType")); } - return ImmutableList.of(fieldName, operator); + return ImmutableList.of(fieldName, operator.token); } /** Creates a {@code ConditionToken} from a serialized string list. */ static ConditionToken fromTokenList(List tokens) { final int size = tokens.size(); if (size == 4) { - return binary(tokens.get(0), tokens.get(1), + return binary(tokens.get(0), Operator.of(tokens.get(1)), tokens.get(2), tokens.get(3)); } else if (size == 2) { - return unary(tokens.get(0), tokens.get(1)); + return unary(tokens.get(0), Operator.of(tokens.get(1))); } throw new IllegalArgumentException("Invalid condition tokens: " + tokens); } + + /** Operators supported by the Arrow adapter filter representation. */ + enum Operator { + IS_NULL("isnull"), + IS_NOT_NULL("isnotnull"), + IS_TRUE("istrue"), + IS_FALSE("isfalse"), + IS_NOT_TRUE("isnottrue"), + IS_NOT_FALSE("isnotfalse"), + EQUAL("equal"), + NOT_EQUAL("not_equal"), + LESS_THAN("less_than"), + LESS_THAN_OR_EQUAL("less_than_or_equal_to"), + GREATER_THAN("greater_than"), + GREATER_THAN_OR_EQUAL("greater_than_or_equal_to"), + LIKE("like"); + + final String token; + + Operator(String token) { + this.token = token; + } + + static Operator of(String token) { + for (Operator operator : values()) { + if (operator.token.equals(token)) { + return operator; + } + } + throw new UnsupportedOperationException( + "Unsupported Arrow filter operator: " + token); + } + } } diff --git a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java index 275bdd0be76c..8c9fda7f081f 100644 --- a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java +++ b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java @@ -82,6 +82,11 @@ static void initializeArrowState(@TempDir Path sharedTempDir) arrowDataGenerator.writeArrowData(dataLocationFile); arrowDataGenerator.writeScottEmpData(arrowFilesDirectory); + File emptyBatchDataLocationFile = + arrowFilesDirectory.resolve("arrowemptybatch.arrow").toFile(); + ArrowDataTest emptyBatchDataGenerator = new ArrowDataTest(); + emptyBatchDataGenerator.writeArrowDataWithEmptyBatch(emptyBatchDataLocationFile); + File datatypeLocationFile = arrowFilesDirectory.resolve("arrowdatatype.arrow").toFile(); ArrowDataTest arrowtypeDataGenerator = new ArrowDataTest(); arrowtypeDataGenerator.writeArrowDataType(datatypeLocationFile); @@ -260,6 +265,51 @@ static void initializeArrowState(@TempDir Path sharedTempDir) .explainContains(plan); } + /** Test case for + * [CALCITE-7580] + * Remove Gandiva dependency from Arrow adapter. */ + @Test void testArrowProjectSkipsEmptyBatch() { + String sql = "select \"intField\", \"stringField\" from arrowemptybatch\n"; + String result = "intField=0; stringField=0\n" + + "intField=1; stringField=1\n" + + "intField=2; stringField=2\n"; + + CalciteAssert.that() + .with(arrow) + .query(sql) + .returns(result); + } + + /** Test case for + * [CALCITE-7580] + * Remove Gandiva dependency from Arrow adapter. */ + @Test void testArrowFilterSkipsEmptyBatch() { + String sql = "select \"intField\", \"stringField\"\n" + + "from arrowemptybatch\n" + + "where \"intField\" > 0"; + String result = "intField=1; stringField=1\n" + + "intField=2; stringField=2\n"; + + CalciteAssert.that() + .with(arrow) + .query(sql) + .returns(result); + } + + /** Test case for + * [CALCITE-7580] + * Remove Gandiva dependency from Arrow adapter. */ + @Test void testArrowFilterSkipsEmptyBatchWithNoMatches() { + String sql = "select \"intField\", \"stringField\"\n" + + "from arrowemptybatch\n" + + "where \"intField\" < 0"; + + CalciteAssert.that() + .with(arrow) + .query(sql) + .returns(""); + } + @Test void testArrowProjectFieldsWithIntegerFilter() { String sql = "select \"intField\", \"stringField\"\n" + "from arrowdata\n" diff --git a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowDataTest.java b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowDataTest.java index a53cc1231ad3..4b9af441aabf 100644 --- a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowDataTest.java +++ b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowDataTest.java @@ -301,6 +301,35 @@ public void writeArrowBinaryData(File file) throws IOException { fileOutputStream.close(); } + public void writeArrowDataWithEmptyBatch(File file) throws IOException { + Schema arrowSchema = makeArrowSchema(); + try (RootAllocator allocator = new RootAllocator(Integer.MAX_VALUE); + VectorSchemaRoot vectorSchemaRoot = + VectorSchemaRoot.create(arrowSchema, allocator); + FileOutputStream fileOutputStream = new FileOutputStream(file); + ArrowFileWriter arrowFileWriter = + new ArrowFileWriter(vectorSchemaRoot, null, + fileOutputStream.getChannel())) { + arrowFileWriter.start(); + + vectorSchemaRoot.setRowCount(0); + for (Field field : vectorSchemaRoot.getSchema().getFields()) { + vectorSchemaRoot.getVector(field.getName()).setValueCount(0); + } + arrowFileWriter.writeBatch(); + + int rowCount = 3; + vectorSchemaRoot.setRowCount(rowCount); + intField(vectorSchemaRoot.getVector("intField"), rowCount); + varCharField(vectorSchemaRoot.getVector("stringField"), rowCount); + floatField(vectorSchemaRoot.getVector("floatField"), rowCount); + longField(vectorSchemaRoot.getVector("longField"), rowCount); + arrowFileWriter.writeBatch(); + + arrowFileWriter.end(); + } + } + public void writeArrowDataType(File file) throws IOException { FileOutputStream fileOutputStream = new FileOutputStream(file); Schema arrowSchema = makeArrowDateTypeSchema(); diff --git a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowExtension.java b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowExtension.java index ab1f5c2a88c1..4600dab1001c 100644 --- a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowExtension.java +++ b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowExtension.java @@ -18,22 +18,12 @@ import org.apache.calcite.config.CalciteSystemProperty; -import org.apache.arrow.gandiva.evaluator.Projector; -import org.apache.arrow.gandiva.exceptions.GandivaException; -import org.apache.arrow.gandiva.expression.ExpressionTree; -import org.apache.arrow.vector.types.pojo.Schema; - import org.junit.jupiter.api.extension.ConditionEvaluationResult; import org.junit.jupiter.api.extension.ExecutionCondition; import org.junit.jupiter.api.extension.ExtensionContext; -import java.util.ArrayList; -import java.util.List; - /** * JUnit5 extension to handle Arrow tests. - * - *

      Tests will be skipped if the Gandiva library cannot be loaded on the given platform. */ class ArrowExtension implements ExecutionCondition { @@ -41,8 +31,7 @@ class ArrowExtension implements ExecutionCondition { * Whether to run this test. * *

      Enabled by default, unless explicitly disabled from command line - * ({@code -Dcalcite.test.arrow=false}) or if Gandiva library, used to implement arrow - * filtering/projection, cannot be loaded. + * ({@code -Dcalcite.test.arrow=false}). * * @return {@code true} if the test is enabled and can run in the current environment, * {@code false} otherwise @@ -51,16 +40,6 @@ class ArrowExtension implements ExecutionCondition { final ExtensionContext context) { boolean enabled = CalciteSystemProperty.TEST_ARROW.value(); - try { - Schema emptySchema = new Schema(new ArrayList<>(), null); - List expressions = new ArrayList<>(); - Projector.make(emptySchema, expressions); - } catch (GandivaException e) { - // this exception comes from using an empty expression, - // but the JNI library was loaded properly - } catch (UnsatisfiedLinkError e) { - enabled = false; - } if (enabled) { return ConditionEvaluationResult.enabled("Arrow tests enabled"); diff --git a/bom/build.gradle.kts b/bom/build.gradle.kts index f00ed7d8e556..64cb532afdc3 100644 --- a/bom/build.gradle.kts +++ b/bom/build.gradle.kts @@ -101,7 +101,6 @@ dependencies { apiv("org.apache.arrow:arrow-memory-netty", "arrow") apiv("org.apache.arrow:arrow-vector", "arrow") apiv("org.apache.arrow:arrow-jdbc", "arrow") - apiv("org.apache.arrow.gandiva:arrow-gandiva", "arrow-gandiva") apiv("org.apache.calcite.avatica:avatica-core", "calcite.avatica") apiv("org.apache.calcite.avatica:avatica-server", "calcite.avatica") apiv("org.apache.cassandra:cassandra-all") diff --git a/core/src/main/java/org/apache/calcite/rex/RexSimplify.java b/core/src/main/java/org/apache/calcite/rex/RexSimplify.java index 0ffb60454e33..4d7702295bff 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexSimplify.java +++ b/core/src/main/java/org/apache/calcite/rex/RexSimplify.java @@ -529,14 +529,11 @@ private RexNode simplifyLike(RexCall e, RexUnknownAs unknownAs) { } if (e.operands.size() == 3 && e.operands.get(2) instanceof RexLiteral) { final RexLiteral escapeLiteral = (RexLiteral) e.operands.get(2); - final String escapeStr = requireNonNull(escapeLiteral.getValueAs(String.class)); - if (escapeStr.length() == 1) { - char escape = escapeStr.charAt(0); - e = (RexCall) rexBuilder - .makeCall(e.getParserPosition(), e.getOperator(), e.operands.get(0), - rexBuilder.makeLiteral(simplifyLikeString(likeStr, escape, '%'), - e.operands.get(1).getType(), true, true), escapeLiteral); - } + Character escape = requireNonNull(escapeLiteral.getValueAs(Character.class)); + e = (RexCall) rexBuilder + .makeCall(e.getParserPosition(), e.getOperator(), e.operands.get(0), + rexBuilder.makeLiteral(simplifyLikeString(likeStr, escape, '%'), + e.operands.get(1).getType(), true, true), escapeLiteral); } } return simplifyGenericNode(e); @@ -546,7 +543,7 @@ private RexNode simplifyLike(RexCall e, RexUnknownAs unknownAs) { // string with even escapes 'AA\\\\%%__%%AA' simplify to 'AA\\__%AA' // string with odd escapes 'AA\\\\\\%%__%%AA' simplify to 'AA\\\\\\%__%AA' private String simplifyMixedWildcards(String str, char escape) { - Pattern pattern = getWildCardPattern(escape); + Pattern pattern = Pattern.compile("[_%]+"); Matcher matcher = pattern.matcher(str); StringBuilder builder = new StringBuilder(); int from = 0; @@ -570,17 +567,6 @@ && consecutiveSameCharCountBefore(str, start - 1, escape) % 2 == 1) { return builder.toString(); } - private static Pattern getWildCardPattern(char escape) { - switch (escape) { - case '%': - return Pattern.compile("_+"); - case '_': - return Pattern.compile("%+"); - default: - return Pattern.compile("[_%]+"); - } - } - // Tool method: count the number of consecutive identical characters before index private int consecutiveSameCharCountBefore(String str, int index, char escape) { int count = 0; diff --git a/core/src/main/java/org/apache/calcite/sql/type/SqlTypeFactoryImpl.java b/core/src/main/java/org/apache/calcite/sql/type/SqlTypeFactoryImpl.java index d0ccab7dfd2f..115b66fa215f 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/SqlTypeFactoryImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/type/SqlTypeFactoryImpl.java @@ -561,7 +561,7 @@ private static void assertBasic(SqlTypeName typeName) { } } - if (type.getSqlTypeName().getFamily() == resultType.getSqlTypeName().getFamily() + if (type.getSqlTypeName() == resultType.getSqlTypeName() && type.getSqlTypeName().allowsPrec() && type.getPrecision() != resultType.getPrecision()) { final int precision = diff --git a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java index 32017ace1d43..4622176a0733 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java @@ -1870,7 +1870,7 @@ public RelNode convertToSingleValueSubq( if (leftKeys.size() == 1) { SqlCall sqlCall = comparisonOp.createCall(rightVals.getParserPosition(), leftKeys.get(0), rightVals); - rexComparison = ensureComparisonTypes(bb.convertExpression(sqlCall)); + rexComparison = bb.convertExpression(sqlCall); } else { assert rightVals instanceof SqlCall; final SqlBasicCall call = (SqlBasicCall) rightVals; @@ -1880,10 +1880,9 @@ public RelNode convertToSingleValueSubq( RexUtil.composeConjunction(rexBuilder, transform( Pair.zip(leftKeys, call.getOperandList()), - pair -> ensureComparisonTypes( - bb.convertExpression( - comparisonOp.createCall(rightVals.getParserPosition(), - pair.left, pair.right))))); + pair -> bb.convertExpression( + comparisonOp.createCall(rightVals.getParserPosition(), + pair.left, pair.right)))); } comparisons.add(rexComparison); } @@ -1902,31 +1901,6 @@ public RelNode convertToSingleValueSubq( } } - /** - * Ensures that a comparison expression has matching operand types. If the - * operands have different type names, casts the right operand to match the - * left operand's type. This handles the case where type coercion is disabled - * and the IN-to-OR expansion produces comparisons with mismatched types - * (e.g., DATE = CHAR). - */ - private RexNode ensureComparisonTypes(RexNode node) { - if (validator != null && validator.config().typeCoercionEnabled()) { - return node; - } - if (node instanceof RexCall) { - final RexCall call = (RexCall) node; - if (call.operands.size() == 2) { - final RexNode left = call.operands.get(0); - final RexNode right = call.operands.get(1); - if (left.getType().getSqlTypeName() != right.getType().getSqlTypeName()) { - final RexNode castRight = rexBuilder.ensureType(left.getType(), right, true); - return rexBuilder.makeCall(call.getOperator(), left, castRight); - } - } - } - return node; - } - /** * Converts a {@link SqlNodeList} (for example an IN-list or VALUES list) * into a relational expression and produces a Rex-level sub-query that diff --git a/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java b/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java index ac7b5aebe6c3..2bdc7a1d56c4 100644 --- a/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java +++ b/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java @@ -4358,10 +4358,6 @@ private void checkSarg(String message, Sarg sarg, * Multiple consecutive '%' in the string matched by LIKE should simplify to a single '%', * [CALCITE-7153] * Mixed wildcards of _ and % need to be simplified in LIKE operator. - * [CALCITE-7578] - * LIKE with empty ESCAPE might fail with StringIndexOutOfBoundsException. - * [CALCITE-7588] - * LIKE with ESCAPE symbols containing wildcards fails. * */ @Test void testSimplifyLike() { final RexNode ref = input(tVarchar(true, 10), 0); @@ -4425,14 +4421,6 @@ private void checkSarg(String message, Sarg sarg, "LIKE($0, '###%%#%#%A#%%#%A%###%%', '#')"); checkSimplifyUnchanged(like(ref, literal("A"), literal("#"))); checkSimplifyUnchanged(like(ref, literal("%A"), literal("#"))); - checkSimplifyUnchanged(like(ref, literal("TE%_ST"), literal("%"))); - checkSimplifyUnchanged(like(ref, literal("TE%%ST"), literal("%"))); - checkSimplifyUnchanged(like(ref, literal("a%_b%%c"), literal("%"))); - checkSimplifyUnchanged(like(ref, literal("%_%%A%_"), literal("%"))); - // escape char equal to the '_' wildcard. '%E__S%' ESCAPE '_' is a literal '_'. - checkSimplifyUnchanged(like(ref, literal("%E__S%"), literal("_"))); - checkSimplifyUnchanged(like(ref, literal("TE_%ST"), literal("_"))); - checkSimplifyUnchanged(like(ref, literal("a_%b__c"), literal("_"))); // As above, but ref is NOT NULL final RexNode refMandatory = vVarcharNotNull(0); @@ -4463,14 +4451,6 @@ private void checkSarg(String message, Sarg sarg, // NOT(SIMILAR TO) is not optimized checkSimplifyUnchanged( not(rexBuilder.makeCall(SqlStdOperatorTable.SIMILAR_TO, ref, literal("%")))); - - try { - // Empty ESCAPE - checkSimplifyUnchanged(like(ref, literal("a"), literal(""))); - } catch (RuntimeException e) { - assertThat(e.getMessage(), - containsString("Invalid escape character ''")); - } } @Test void testSimplifyNullCheckInFilter() { diff --git a/core/src/test/java/org/apache/calcite/sql/type/SqlTypeFactoryTest.java b/core/src/test/java/org/apache/calcite/sql/type/SqlTypeFactoryTest.java index f0ff190d28c7..8f5e5e4018db 100644 --- a/core/src/test/java/org/apache/calcite/sql/type/SqlTypeFactoryTest.java +++ b/core/src/test/java/org/apache/calcite/sql/type/SqlTypeFactoryTest.java @@ -33,7 +33,6 @@ import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.CoreMatchers.notNullValue; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.hasToString; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -173,30 +172,6 @@ class SqlTypeFactoryTest { assertThat(leastRestrictive.getPrecision(), is(3)); } - /** - * Test case for - * - * LeastRetrictiveSqlType for TIMESTAMP, TIMESTAMP_LTZ might ignore precision. */ - @Test void testLeastRestrictiveForTimestampAndTimestampLtz() { - SqlTypeFixture f = new SqlTypeFixture(); - RelDataType ltz0 = - f.typeFactory.createSqlType(SqlTypeName.TIMESTAMP_WITH_LOCAL_TIME_ZONE, 0); - RelDataType leastRestrictive = - f.typeFactory.leastRestrictive(Lists.newArrayList(ltz0, f.sqlTimestampPrec3)); - assertThat(leastRestrictive, is(notNullValue())); - assertThat(leastRestrictive.getPrecision(), is(3)); - } - - @Test void testLeastRestrictiveForTimestampLtzAndTimestamp() { - SqlTypeFixture f = new SqlTypeFixture(); - RelDataType ltz0 = - f.typeFactory.createSqlType(SqlTypeName.TIMESTAMP_WITH_LOCAL_TIME_ZONE, 0); - RelDataType leastRestrictive = - f.typeFactory.leastRestrictive(Lists.newArrayList(f.sqlTimestampPrec3, ltz0)); - assertThat(leastRestrictive, is(notNullValue())); - assertThat(leastRestrictive.getPrecision(), is(3)); - } - @Test void testLeastRestrictiveForTimestampAndDate() { SqlTypeFixture f = new SqlTypeFixture(); RelDataType leastRestrictive = diff --git a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java index 723d749be6b5..4e8d17cd7a27 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java @@ -2213,24 +2213,6 @@ void checkCorrelatedMapSubQuery(boolean expand) { sql(sql).withExpand(false).ok(); } - /** Test case for - * [CALCITE-7562] - * SqlToRel misses CAST in case IN expression without type coercion. */ - @Test void testInDateColumnWithoutTypeCoercion() { - final String sql = - "select * from emp_b where birthdate in ('2000-06-30', '2000-09-27')"; - sql(sql).withTypeCoercion(false).ok(); - } - - /** Test case for - * [CALCITE-7562] - * SqlToRel misses CAST in case IN expression without type coercion. */ - @Test void testInDateColumnWithTypeCoercion() { - final String sql = - "select * from emp_b where birthdate in ('2000-06-30', '2000-09-27')"; - sql(sql).ok(); - } - @Test void testInValueListLong() { // Go over the default threshold of 20 to force a sub-query. final String sql = "select empno from emp where deptno in" diff --git a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml index 8aa2171d3f0b..5ee7180efa75 100644 --- a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml @@ -3266,30 +3266,6 @@ LogicalFilter(condition=[=($cor0.DEPTNO, $7)]) LogicalAggregate(group=[{0}], S=[SUM($1)], agg#1=[COUNT()]) LogicalProject(DEPTNO=[$7], SAL=[$5]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) -]]> - - - - - - - - - - - - - - - - diff --git a/gradle.properties b/gradle.properties index eb0bd778ae14..fd823182640c 100644 --- a/gradle.properties +++ b/gradle.properties @@ -81,7 +81,6 @@ jandex.version=3.5.3 # elasticsearch does not like asm:6.2.1+ aggdesigner-algorithm.version=6.1 apiguardian-api.version=1.1.2 -arrow-gandiva.version=15.0.0 arrow.version=15.0.0 asm.version=9.9.1 byte-buddy.version=1.18.8 diff --git a/site/_docs/history.md b/site/_docs/history.md index aba03c8f3c6f..ac3be2599016 100644 --- a/site/_docs/history.md +++ b/site/_docs/history.md @@ -49,6 +49,11 @@ other software versions as specified in gradle.properties. #### Breaking Changes {: #breaking-1-43-0} +* [CALCITE-7580] + Remove Gandiva dependency from Arrow adapter. Arrow adapter projection and + filter evaluation now run in Java, and the `arrow-gandiva` dependency is no + longer included in the Arrow module or BOM. + #### New features {: #new-features-1-43-0} diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlToRelFixture.java b/testkit/src/main/java/org/apache/calcite/test/SqlToRelFixture.java index 784cb1eba1dd..a6685cf952f5 100644 --- a/testkit/src/main/java/org/apache/calcite/test/SqlToRelFixture.java +++ b/testkit/src/main/java/org/apache/calcite/test/SqlToRelFixture.java @@ -178,11 +178,6 @@ public SqlToRelFixture withConformance(SqlConformance conformance) { .withValidatorConfig(c -> c.withConformance(conformance))); } - public SqlToRelFixture withTypeCoercion(boolean enabled) { - return withFactory(f -> - f.withValidatorConfig(c -> c.withTypeCoercionEnabled(enabled))); - } - public SqlToRelFixture withDiffRepos(DiffRepository diffRepos) { return new SqlToRelFixture(sql, decorrelate, tester, factory, trim, expression, diffRepos); From 66dd32ccfdf737dae7737727b19a183f6b22f383 Mon Sep 17 00:00:00 2001 From: Venkata krishnan Sowrirajan Date: Fri, 5 Jun 2026 10:21:09 -0700 Subject: [PATCH 305/562] [CALCITE-7511] Broaden RelShuttle Javadoc to drop "logical" qualifier --- .../calcite/rel/RelHomogeneousShuttle.java | 50 ++++++++ .../org/apache/calcite/rel/RelShuttle.java | 29 ++++- .../apache/calcite/rel/RelShuttleImpl.java | 45 +++++++ .../org/apache/calcite/rel/core/Collect.java | 5 + .../org/apache/calcite/rel/core/Combine.java | 5 + .../rel/core/ConditionalCorrelate.java | 5 + .../org/apache/calcite/rel/core/Sample.java | 5 + .../org/apache/calcite/rel/core/Snapshot.java | 5 + .../apache/calcite/rel/core/SortExchange.java | 5 + .../calcite/rel/core/TableFunctionScan.java | 5 + .../apache/calcite/rel/core/TableSpool.java | 5 + .../apache/calcite/rel/core/Uncollect.java | 5 + .../org/apache/calcite/rel/core/Window.java | 5 + .../rel/logical/ToLogicalConverter.java | 8 ++ .../calcite/test/RelShuttleCoverageTest.java | 118 ++++++++++++++++++ .../calcite/test/SqlHintsConverterTest.java | 35 +++--- site/_docs/history.md | 18 +++ 17 files changed, 335 insertions(+), 18 deletions(-) create mode 100644 core/src/test/java/org/apache/calcite/test/RelShuttleCoverageTest.java diff --git a/core/src/main/java/org/apache/calcite/rel/RelHomogeneousShuttle.java b/core/src/main/java/org/apache/calcite/rel/RelHomogeneousShuttle.java index d38f39649cfe..82b98a8f22a8 100644 --- a/core/src/main/java/org/apache/calcite/rel/RelHomogeneousShuttle.java +++ b/core/src/main/java/org/apache/calcite/rel/RelHomogeneousShuttle.java @@ -16,9 +16,19 @@ */ package org.apache.calcite.rel; +import org.apache.calcite.rel.core.Collect; +import org.apache.calcite.rel.core.Combine; +import org.apache.calcite.rel.core.ConditionalCorrelate; +import org.apache.calcite.rel.core.Sample; +import org.apache.calcite.rel.core.Snapshot; +import org.apache.calcite.rel.core.SortExchange; import org.apache.calcite.rel.core.TableFunctionScan; import org.apache.calcite.rel.core.TableScan; +import org.apache.calcite.rel.core.TableSpool; +import org.apache.calcite.rel.core.Uncollect; +import org.apache.calcite.rel.core.Window; import org.apache.calcite.rel.logical.LogicalAggregate; +import org.apache.calcite.rel.logical.LogicalAsofJoin; import org.apache.calcite.rel.logical.LogicalCalc; import org.apache.calcite.rel.logical.LogicalCorrelate; import org.apache.calcite.rel.logical.LogicalExchange; @@ -71,6 +81,10 @@ public class RelHomogeneousShuttle extends RelShuttleImpl { return visit((RelNode) join); } + @Override public RelNode visit(LogicalAsofJoin asofJoin) { + return visit((RelNode) asofJoin); + } + @Override public RelNode visit(LogicalCorrelate correlate) { return visit((RelNode) correlate); } @@ -106,4 +120,40 @@ public class RelHomogeneousShuttle extends RelShuttleImpl { @Override public RelNode visit(LogicalRepeatUnion repeatUnion) { return visit((RelNode) repeatUnion); } + + @Override public RelNode visit(Window window) { + return visit((RelNode) window); + } + + @Override public RelNode visit(Snapshot snapshot) { + return visit((RelNode) snapshot); + } + + @Override public RelNode visit(Collect collect) { + return visit((RelNode) collect); + } + + @Override public RelNode visit(Sample sample) { + return visit((RelNode) sample); + } + + @Override public RelNode visit(Uncollect uncollect) { + return visit((RelNode) uncollect); + } + + @Override public RelNode visit(Combine combine) { + return visit((RelNode) combine); + } + + @Override public RelNode visit(ConditionalCorrelate conditionalCorrelate) { + return visit((RelNode) conditionalCorrelate); + } + + @Override public RelNode visit(SortExchange sortExchange) { + return visit((RelNode) sortExchange); + } + + @Override public RelNode visit(TableSpool tableSpool) { + return visit((RelNode) tableSpool); + } } diff --git a/core/src/main/java/org/apache/calcite/rel/RelShuttle.java b/core/src/main/java/org/apache/calcite/rel/RelShuttle.java index f9203058cbda..ec735ad911d5 100644 --- a/core/src/main/java/org/apache/calcite/rel/RelShuttle.java +++ b/core/src/main/java/org/apache/calcite/rel/RelShuttle.java @@ -16,8 +16,17 @@ */ package org.apache.calcite.rel; +import org.apache.calcite.rel.core.Collect; +import org.apache.calcite.rel.core.Combine; +import org.apache.calcite.rel.core.ConditionalCorrelate; +import org.apache.calcite.rel.core.Sample; +import org.apache.calcite.rel.core.Snapshot; +import org.apache.calcite.rel.core.SortExchange; import org.apache.calcite.rel.core.TableFunctionScan; import org.apache.calcite.rel.core.TableScan; +import org.apache.calcite.rel.core.TableSpool; +import org.apache.calcite.rel.core.Uncollect; +import org.apache.calcite.rel.core.Window; import org.apache.calcite.rel.logical.LogicalAggregate; import org.apache.calcite.rel.logical.LogicalAsofJoin; import org.apache.calcite.rel.logical.LogicalCalc; @@ -36,7 +45,7 @@ import org.apache.calcite.rel.logical.LogicalValues; /** - * Visitor that has methods for the common logical relational expressions. + * Visitor that has methods for the common relational expressions. */ public interface RelShuttle { RelNode visit(TableScan scan); @@ -75,5 +84,23 @@ public interface RelShuttle { RelNode visit(LogicalRepeatUnion logicalRepeatUnion); + RelNode visit(Window window); + + RelNode visit(Snapshot snapshot); + + RelNode visit(Collect collect); + + RelNode visit(Sample sample); + + RelNode visit(Uncollect uncollect); + + RelNode visit(Combine combine); + + RelNode visit(ConditionalCorrelate conditionalCorrelate); + + RelNode visit(SortExchange sortExchange); + + RelNode visit(TableSpool tableSpool); + RelNode visit(RelNode other); } diff --git a/core/src/main/java/org/apache/calcite/rel/RelShuttleImpl.java b/core/src/main/java/org/apache/calcite/rel/RelShuttleImpl.java index 1c0d0dd5ee55..377f6ab5b2c7 100644 --- a/core/src/main/java/org/apache/calcite/rel/RelShuttleImpl.java +++ b/core/src/main/java/org/apache/calcite/rel/RelShuttleImpl.java @@ -17,8 +17,17 @@ package org.apache.calcite.rel; import org.apache.calcite.linq4j.Ord; +import org.apache.calcite.rel.core.Collect; +import org.apache.calcite.rel.core.Combine; +import org.apache.calcite.rel.core.ConditionalCorrelate; +import org.apache.calcite.rel.core.Sample; +import org.apache.calcite.rel.core.Snapshot; +import org.apache.calcite.rel.core.SortExchange; import org.apache.calcite.rel.core.TableFunctionScan; import org.apache.calcite.rel.core.TableScan; +import org.apache.calcite.rel.core.TableSpool; +import org.apache.calcite.rel.core.Uncollect; +import org.apache.calcite.rel.core.Window; import org.apache.calcite.rel.logical.LogicalAggregate; import org.apache.calcite.rel.logical.LogicalAsofJoin; import org.apache.calcite.rel.logical.LogicalCalc; @@ -147,6 +156,42 @@ protected RelNode visitChildren(RelNode rel) { return visitChildren(logicalRepeatUnion); } + @Override public RelNode visit(Window window) { + return visitChildren(window); + } + + @Override public RelNode visit(Snapshot snapshot) { + return visitChildren(snapshot); + } + + @Override public RelNode visit(Collect collect) { + return visitChildren(collect); + } + + @Override public RelNode visit(Sample sample) { + return visitChildren(sample); + } + + @Override public RelNode visit(Uncollect uncollect) { + return visitChildren(uncollect); + } + + @Override public RelNode visit(Combine combine) { + return visitChildren(combine); + } + + @Override public RelNode visit(ConditionalCorrelate conditionalCorrelate) { + return visitChildren(conditionalCorrelate); + } + + @Override public RelNode visit(SortExchange sortExchange) { + return visitChildren(sortExchange); + } + + @Override public RelNode visit(TableSpool tableSpool) { + return visitChildren(tableSpool); + } + @Override public RelNode visit(RelNode other) { return visitChildren(other); } diff --git a/core/src/main/java/org/apache/calcite/rel/core/Collect.java b/core/src/main/java/org/apache/calcite/rel/core/Collect.java index c308bfe6bf10..475cc966d3a1 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/Collect.java +++ b/core/src/main/java/org/apache/calcite/rel/core/Collect.java @@ -21,6 +21,7 @@ import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.RelInput; import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.RelShuttle; import org.apache.calcite.rel.RelWriter; import org.apache.calcite.rel.SingleRel; import org.apache.calcite.rel.type.RelDataType; @@ -183,6 +184,10 @@ public RelNode copy(RelTraitSet traitSet, RelNode input) { return new Collect(getCluster(), traitSet, input, rowType()); } + @Override public RelNode accept(RelShuttle shuttle) { + return shuttle.visit(this); + } + @Override public RelWriter explainTerms(RelWriter pw) { return super.explainTerms(pw) .item("field", getFieldName()); diff --git a/core/src/main/java/org/apache/calcite/rel/core/Combine.java b/core/src/main/java/org/apache/calcite/rel/core/Combine.java index 6c99a3a4de1d..7095a5a1c805 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/Combine.java +++ b/core/src/main/java/org/apache/calcite/rel/core/Combine.java @@ -23,6 +23,7 @@ import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.AbstractRelNode; import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.RelShuttle; import org.apache.calcite.rel.RelWriter; import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.rel.type.RelDataType; @@ -60,6 +61,10 @@ public Combine(RelOptCluster cluster, RelTraitSet traitSet, List inputs return new Combine(getCluster(), traitSet, inputs); } + @Override public RelNode accept(RelShuttle shuttle) { + return shuttle.visit(this); + } + @Override public void replaceInput(int ordinalInParent, RelNode rel) { // Combine has multiple inputs stored in an immutable list. // To replace an input, we need to create a new list with the replacement. diff --git a/core/src/main/java/org/apache/calcite/rel/core/ConditionalCorrelate.java b/core/src/main/java/org/apache/calcite/rel/core/ConditionalCorrelate.java index f5e1d38377d7..c67aaf564861 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/ConditionalCorrelate.java +++ b/core/src/main/java/org/apache/calcite/rel/core/ConditionalCorrelate.java @@ -19,6 +19,7 @@ import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.RelShuttle; import org.apache.calcite.rel.RelWriter; import org.apache.calcite.rel.hint.RelHint; import org.apache.calcite.rel.rules.CoreRules; @@ -69,6 +70,10 @@ public abstract ConditionalCorrelate copy(RelTraitSet traitSet, RelNode left, Re CorrelationId correlationId, ImmutableBitSet requiredColumns, JoinRelType joinType, RexNode condition); + @Override public RelNode accept(RelShuttle shuttle) { + return shuttle.visit(this); + } + @Override public RelWriter explainTerms(RelWriter pw) { return super.explainTerms(pw) .itemIf("condition", condition, !condition.isAlwaysTrue()); diff --git a/core/src/main/java/org/apache/calcite/rel/core/Sample.java b/core/src/main/java/org/apache/calcite/rel/core/Sample.java index a4dbfacea17e..f3c224ea87ff 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/Sample.java +++ b/core/src/main/java/org/apache/calcite/rel/core/Sample.java @@ -22,6 +22,7 @@ import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.RelInput; import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.RelShuttle; import org.apache.calcite.rel.RelWriter; import org.apache.calcite.rel.SingleRel; @@ -79,6 +80,10 @@ private static RelOptSamplingParameters getSamplingParameters( return new Sample(getCluster(), sole(inputs), params); } + @Override public RelNode accept(RelShuttle shuttle) { + return shuttle.visit(this); + } + /** * Retrieve the sampling parameters for this Sample. */ diff --git a/core/src/main/java/org/apache/calcite/rel/core/Snapshot.java b/core/src/main/java/org/apache/calcite/rel/core/Snapshot.java index 52c4c8ab729b..a7c387c66375 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/Snapshot.java +++ b/core/src/main/java/org/apache/calcite/rel/core/Snapshot.java @@ -20,6 +20,7 @@ import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.RelInput; import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.RelShuttle; import org.apache.calcite.rel.RelWriter; import org.apache.calcite.rel.SingleRel; import org.apache.calcite.rel.hint.Hintable; @@ -119,6 +120,10 @@ protected Snapshot( return copy(traitSet, getInput(), condition); } + @Override public RelNode accept(RelShuttle shuttle) { + return shuttle.visit(this); + } + @Override public RelWriter explainTerms(RelWriter pw) { return super.explainTerms(pw) .item("period", period); diff --git a/core/src/main/java/org/apache/calcite/rel/core/SortExchange.java b/core/src/main/java/org/apache/calcite/rel/core/SortExchange.java index af1cea599a4c..7320fe27684c 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/SortExchange.java +++ b/core/src/main/java/org/apache/calcite/rel/core/SortExchange.java @@ -24,6 +24,7 @@ import org.apache.calcite.rel.RelDistributionTraitDef; import org.apache.calcite.rel.RelInput; import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.RelShuttle; import org.apache.calcite.rel.RelWriter; import static java.util.Objects.requireNonNull; @@ -85,6 +86,10 @@ protected SortExchange(RelInput input) { public abstract SortExchange copy(RelTraitSet traitSet, RelNode newInput, RelDistribution newDistribution, RelCollation newCollation); + @Override public RelNode accept(RelShuttle shuttle) { + return shuttle.visit(this); + } + /** * Returns the array of {@link org.apache.calcite.rel.RelFieldCollation}s * asked for by the sort specification, from most significant to least diff --git a/core/src/main/java/org/apache/calcite/rel/core/TableFunctionScan.java b/core/src/main/java/org/apache/calcite/rel/core/TableFunctionScan.java index 0a8c6d9cfd19..4e1ad50b7343 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/TableFunctionScan.java +++ b/core/src/main/java/org/apache/calcite/rel/core/TableFunctionScan.java @@ -22,6 +22,7 @@ import org.apache.calcite.rel.AbstractRelNode; import org.apache.calcite.rel.RelInput; import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.RelShuttle; import org.apache.calcite.rel.RelWriter; import org.apache.calcite.rel.hint.Hintable; import org.apache.calcite.rel.hint.RelHint; @@ -140,6 +141,10 @@ protected TableFunctionScan(RelInput input) { //~ Methods ---------------------------------------------------------------- + @Override public RelNode accept(RelShuttle shuttle) { + return shuttle.visit(this); + } + @Override public final TableFunctionScan copy(RelTraitSet traitSet, List inputs) { return copy(traitSet, inputs, rexCall, elementType, getRowType(), diff --git a/core/src/main/java/org/apache/calcite/rel/core/TableSpool.java b/core/src/main/java/org/apache/calcite/rel/core/TableSpool.java index bf22d2c96260..a3483491c5d2 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/TableSpool.java +++ b/core/src/main/java/org/apache/calcite/rel/core/TableSpool.java @@ -21,6 +21,7 @@ import org.apache.calcite.plan.RelOptTable; import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.RelShuttle; import org.apache.calcite.rel.RelWriter; import static java.util.Objects.requireNonNull; @@ -46,6 +47,10 @@ protected TableSpool(RelOptCluster cluster, RelTraitSet traitSet, return table; } + @Override public RelNode accept(RelShuttle shuttle) { + return shuttle.visit(this); + } + @Override public RelWriter explainTerms(RelWriter pw) { super.explainTerms(pw); return pw.item("table", table.getQualifiedName()); diff --git a/core/src/main/java/org/apache/calcite/rel/core/Uncollect.java b/core/src/main/java/org/apache/calcite/rel/core/Uncollect.java index 1fa71fc90b41..09b4d5822f07 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/Uncollect.java +++ b/core/src/main/java/org/apache/calcite/rel/core/Uncollect.java @@ -21,6 +21,7 @@ import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.RelInput; import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.RelShuttle; import org.apache.calcite.rel.RelWriter; import org.apache.calcite.rel.SingleRel; import org.apache.calcite.rel.type.RelDataType; @@ -112,6 +113,10 @@ public static Uncollect create( //~ Methods ---------------------------------------------------------------- + @Override public RelNode accept(RelShuttle shuttle) { + return shuttle.visit(this); + } + @Override public RelWriter explainTerms(RelWriter pw) { return super.explainTerms(pw) .itemIf("withOrdinality", withOrdinality, withOrdinality); diff --git a/core/src/main/java/org/apache/calcite/rel/core/Window.java b/core/src/main/java/org/apache/calcite/rel/core/Window.java index 2adf54c4f379..b444c8468f42 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/Window.java +++ b/core/src/main/java/org/apache/calcite/rel/core/Window.java @@ -25,6 +25,7 @@ import org.apache.calcite.rel.RelCollations; import org.apache.calcite.rel.RelFieldCollation; import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.RelShuttle; import org.apache.calcite.rel.RelWriter; import org.apache.calcite.rel.SingleRel; import org.apache.calcite.rel.hint.Hintable; @@ -210,6 +211,10 @@ public List getConstants() { return constants; } + @Override public RelNode accept(RelShuttle shuttle) { + return shuttle.visit(this); + } + @Override public @Nullable RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) { // Cost is proportional to the number of rows and the number of diff --git a/core/src/main/java/org/apache/calcite/rel/logical/ToLogicalConverter.java b/core/src/main/java/org/apache/calcite/rel/logical/ToLogicalConverter.java index 86d08f5641e3..d7cee2dae7ac 100644 --- a/core/src/main/java/org/apache/calcite/rel/logical/ToLogicalConverter.java +++ b/core/src/main/java/org/apache/calcite/rel/logical/ToLogicalConverter.java @@ -57,6 +57,14 @@ public ToLogicalConverter(RelBuilder relBuilder) { return LogicalTableScan.create(scan.getCluster(), scan.getTable(), scan.getHints()); } + @Override public RelNode visit(Window window) { + return visit((RelNode) window); + } + + @Override public RelNode visit(Uncollect uncollect) { + return visit((RelNode) uncollect); + } + @Override public RelNode visit(RelNode relNode) { if (relNode instanceof Aggregate) { final Aggregate agg = (Aggregate) relNode; diff --git a/core/src/test/java/org/apache/calcite/test/RelShuttleCoverageTest.java b/core/src/test/java/org/apache/calcite/test/RelShuttleCoverageTest.java new file mode 100644 index 000000000000..658e1bd33504 --- /dev/null +++ b/core/src/test/java/org/apache/calcite/test/RelShuttleCoverageTest.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.test; + +import org.apache.calcite.rel.AbstractRelNode; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.RelShuttle; + +import com.google.common.collect.ImmutableSet; +import com.google.common.reflect.ClassPath; + +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.util.Comparator; +import java.util.HashSet; +import java.util.Set; +import java.util.TreeSet; +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** Guards against introducing new dispatch gaps in {@link RelShuttle}. */ +class RelShuttleCoverageTest { + + private static final Set SCANNED_PACKAGES = + ImmutableSet.of("org.apache.calcite.rel.core", + "org.apache.calcite.rel.logical"); + + /** Every concrete RelNode in the scanned packages must be covered by a non-{@code visit(RelNode)} + * overload on {@link RelShuttle}. Without this, a {@code RelShuttleImpl} subclass that customizes + * the type-specific visitor would silently not be called for the rel. */ + @Test void everyRelNodeHasMatchingVisitOverload() throws IOException { + final Set> visitParameters = collectVisitParameterTypes(); + final Set> relClasses = findConcreteRelNodesInPackages(); + + final Set> uncovered = relClasses.stream() + .filter(c -> visitParameters.stream().noneMatch(vp -> vp.isAssignableFrom(c))) + .collect( + Collectors.toCollection(() -> + new TreeSet<>(Comparator.comparing(Class::getName)))); + + assertTrue(uncovered.isEmpty(), + () -> "RelNodes with no RelShuttle.visit(...) overload covering them " + + "(only the generic visit(RelNode) catch-all matches): " + uncovered); + } + + /** Every {@link RelShuttle#visit(...)} parameter type (other than {@code RelNode}) must declare + * its own {@code accept(RelShuttle)} so dispatch routes through the type-specific overload. */ + @Test void everyVisitParameterTypeDeclaresAccept() { + final Set> missingAccept = collectVisitParameterTypes().stream() + .filter(c -> !declaresAcceptRelShuttle(c)) + .collect( + Collectors.toCollection(() -> + new TreeSet<>(Comparator.comparing(Class::getName)))); + + assertTrue(missingAccept.isEmpty(), + () -> "RelShuttle.visit(X) parameter types whose X does not declare accept(RelShuttle): " + + missingAccept); + } + + /** Visit parameter types other than {@link RelNode} (the catch-all fallback). */ + private static Set> collectVisitParameterTypes() { + final Set> params = new HashSet<>(); + for (Method method : RelShuttle.class.getMethods()) { + if ("visit".equals(method.getName()) && method.getParameterCount() == 1) { + Class paramType = method.getParameterTypes()[0]; + if (paramType != RelNode.class) { + params.add(paramType); + } + } + } + return params; + } + + private static boolean declaresAcceptRelShuttle(Class clazz) { + try { + clazz.getDeclaredMethod("accept", RelShuttle.class); + return true; + } catch (NoSuchMethodException e) { + return false; + } + } + + private static Set> findConcreteRelNodesInPackages() throws IOException { + final Set> classes = new HashSet<>(); + final ClassPath classPath = ClassPath.from(RelShuttleCoverageTest.class.getClassLoader()); + for (String packageName : SCANNED_PACKAGES) { + for (ClassPath.ClassInfo info : classPath.getTopLevelClasses(packageName)) { + final Class clazz = info.load(); + if (RelNode.class.isAssignableFrom(clazz) + && !Modifier.isAbstract(clazz.getModifiers()) + && clazz != AbstractRelNode.class) { + @SuppressWarnings("unchecked") + Class relClass = (Class) clazz; + classes.add(relClass); + } + } + } + return classes; + } +} diff --git a/core/src/test/java/org/apache/calcite/test/SqlHintsConverterTest.java b/core/src/test/java/org/apache/calcite/test/SqlHintsConverterTest.java index b17f64aba671..9183134cd4e4 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlHintsConverterTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlHintsConverterTest.java @@ -974,24 +974,25 @@ private static class HintCollector extends RelShuttleImpl { return super.visit(values); } - @Override public RelNode visit(RelNode other) { - if (other instanceof Window) { - Window window = (Window) other; - if (!window.getHints().isEmpty()) { - this.hintsCollect.add("Window:" + window.getHints()); - } - } else if (other instanceof Snapshot) { - Snapshot snapshot = (Snapshot) other; - if (!snapshot.getHints().isEmpty()) { - this.hintsCollect.add("Snapshot:" + snapshot.getHints()); - } - } else if (other instanceof TableFunctionScan) { - TableFunctionScan scan = (TableFunctionScan) other; - if (!scan.getHints().isEmpty()) { - this.hintsCollect.add("TableFunctionScan:" + scan.getHints()); - } + @Override public RelNode visit(TableFunctionScan scan) { + if (!scan.getHints().isEmpty()) { + this.hintsCollect.add("TableFunctionScan:" + scan.getHints()); + } + return super.visit(scan); + } + + @Override public RelNode visit(Window window) { + if (!window.getHints().isEmpty()) { + this.hintsCollect.add("Window:" + window.getHints()); + } + return super.visit(window); + } + + @Override public RelNode visit(Snapshot snapshot) { + if (!snapshot.getHints().isEmpty()) { + this.hintsCollect.add("Snapshot:" + snapshot.getHints()); } - return super.visit(other); + return super.visit(snapshot); } } } diff --git a/site/_docs/history.md b/site/_docs/history.md index ac3be2599016..c72d6f78d938 100644 --- a/site/_docs/history.md +++ b/site/_docs/history.md @@ -151,6 +151,24 @@ The same applies to `SqlBabelCreateTable` and `SqlUnpivot`. the table functions `HOP`, `TUMBLE`, `SESSION` to match the original type of the timestamp column. These types used to be hardwired to `TIMESTAMP(3)`. +* [CALCITE-7511] + This change adds new `visit(X)` overloads on `RelShuttle` for `TableFunctionScan`, + `Window`, `Snapshot`, `Collect`, `Sample`, `Uncollect`, `Combine`, `ConditionalCorrelate`, + `SortExchange`, and `TableSpool`. The corresponding rel class (or its abstract parent) + now overrides `accept(RelShuttle)` so dispatch routes through the type-specific overload + instead of `visit(RelNode other)`. Default implementations are provided in `RelShuttleImpl` + and `RelHomogeneousShuttle`. Callers that implement `RelShuttle` directly must add the + new `visit(X)` methods; the compiler will flag missing overrides. Callers that subclassed + `RelShuttleImpl` (or `RelHomogeneousShuttle`) and handled any of these types via `instanceof` + checks inside `visit(RelNode)` should migrate that logic to the matching `visit(X)` override — + those `instanceof` branches will silently stop firing for the affected types. Because the + `accept(RelShuttle)` override is placed on the abstract parent where one exists, **all** + subclasses now dispatch through the type-specific overload, including `Enumerable*` and + engine-specific variants; for example, `EnumerableTableFunctionScan` now also routes through + `visit(TableFunctionScan)`. This change also adds the previously-missing + `RelHomogeneousShuttle.visit(LogicalAsofJoin)` forwarding override; subclasses that relied + on `LogicalAsofJoin` not being routed through their `visit(RelNode)` override will now see + it routed there, matching every other rel type in the homogeneous shuttle. #### New features {: #new-features-1-42-0} From 67886991f3e55b339e363093bca942192277fee7 Mon Sep 17 00:00:00 2001 From: "cc.cai" <2356672992@qq.com> Date: Sat, 6 Jun 2026 23:52:09 +0800 Subject: [PATCH 306/562] [CALCITE-7539] Upgrade Arrow adapter dependencies to 16.0.0 --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index fd823182640c..14d29dfa356d 100644 --- a/gradle.properties +++ b/gradle.properties @@ -81,7 +81,7 @@ jandex.version=3.5.3 # elasticsearch does not like asm:6.2.1+ aggdesigner-algorithm.version=6.1 apiguardian-api.version=1.1.2 -arrow.version=15.0.0 +arrow.version=16.0.0 asm.version=9.9.1 byte-buddy.version=1.18.8 cassandra-all.version=4.1.6 From 3b9125810a855448f8c9296643009a3bd39b827f Mon Sep 17 00:00:00 2001 From: xuzifu666 <1206332514@qq.com> Date: Tue, 2 Jun 2026 00:24:48 +0800 Subject: [PATCH 307/562] [CALCITE-5101] LISTAGG function with DISTINCT and ORDER BY fails --- ...AggregateExpandDistinctAggregatesRule.java | 122 ++++++++++++++++-- .../EnumerableSortedAggregateTest.java | 100 ++++++++++++++ 2 files changed, 210 insertions(+), 12 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rel/rules/AggregateExpandDistinctAggregatesRule.java b/core/src/main/java/org/apache/calcite/rel/rules/AggregateExpandDistinctAggregatesRule.java index b25eceb38e3c..afb4edbf8e4a 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/AggregateExpandDistinctAggregatesRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/AggregateExpandDistinctAggregatesRule.java @@ -19,7 +19,9 @@ import org.apache.calcite.plan.Contexts; import org.apache.calcite.plan.RelOptRuleCall; import org.apache.calcite.plan.RelRule; +import org.apache.calcite.rel.RelCollation; import org.apache.calcite.rel.RelCollations; +import org.apache.calcite.rel.RelFieldCollation; import org.apache.calcite.rel.core.Aggregate; import org.apache.calcite.rel.core.Aggregate.Group; import org.apache.calcite.rel.core.AggregateCall; @@ -221,9 +223,13 @@ public AggregateExpandDistinctAggregatesRule( // one or more non-distinct aggregates && !nonDistinctAggCalls.isEmpty()) { final RelBuilder relBuilder = call.builder(); - convertSingletonDistinct(relBuilder, aggregate, distinctCallArgLists); - call.transformTo(relBuilder.build()); - return; + final RelBuilder result = + convertSingletonDistinct(relBuilder, aggregate, distinctCallArgLists); + if (result != null) { + call.transformTo(result.build()); + return; + } + // If convertSingletonDistinct returns null, fall through to other strategies } // Create a list of the expressions which will yield the final result. @@ -279,6 +285,14 @@ public AggregateExpandDistinctAggregatesRule( call.transformTo(relBuilder.build()); } + private static RelCollation remapCollation(RelCollation collation, + Map sourceOf) { + if (collation.equals(RelCollations.EMPTY)) { + return RelCollations.EMPTY; + } + return RelCollations.permute(collation, sourceOf); + } + /** * Converts an aggregate with one distinct aggregate and one or more * non-distinct aggregates to multi-phase aggregates (see reference example @@ -287,15 +301,39 @@ public AggregateExpandDistinctAggregatesRule( * @param relBuilder Contains the input relational expression * @param aggregate Original aggregate * @param argLists Arguments and filters to the distinct aggregate function + * @return relBuilder if conversion succeeded, or null if not applicable * */ - private static RelBuilder convertSingletonDistinct(RelBuilder relBuilder, + private static @Nullable RelBuilder convertSingletonDistinct(RelBuilder relBuilder, Aggregate aggregate, Set, Integer>> argLists) { // In this case, we are assuming that there is a single distinct function. // So make sure that argLists is of size one. checkArgument(argLists.size() == 1); + // Check if any DISTINCT aggregate has ORDER BY columns not in GROUP BY or args. + // If so, we cannot safely apply this optimization because it would violate DISTINCT + // semantics (adding ORDER BY columns to GROUP BY would incorrectly preserve duplicate + // values of the DISTINCT column with different ORDER BY values). + final ImmutableBitSet groupSet = aggregate.getGroupSet(); + final Pair, Integer> pair = Iterables.getOnlyElement(argLists); + final List distinctArgs = ImmutableList.copyOf(pair.left); + final ImmutableBitSet distinctArgSet = ImmutableBitSet.of(distinctArgs); + + for (AggregateCall aggCall : aggregate.getAggCallList()) { + if (!aggCall.isDistinct() || !aggCall.getArgList().equals(distinctArgs)) { + continue; + } + for (RelFieldCollation fc : aggCall.collation.getFieldCollations()) { + int colIdx = fc.getFieldIndex(); + // Check if this ORDER BY column is in GROUP BY or in DISTINCT args + if (!groupSet.get(colIdx) && !distinctArgSet.get(colIdx)) { + // Cannot optimize: ORDER BY references a column outside GROUP BY and DISTINCT args + return null; + } + } + } + // For example, // SELECT deptno, COUNT(*), SUM(bonus), MIN(DISTINCT sal) // FROM emp @@ -315,7 +353,11 @@ private static RelBuilder convertSingletonDistinct(RelBuilder relBuilder, final ImmutableBitSet originalGroupSet = aggregate.getGroupSet(); // Add the distinct aggregate column(s) to the group-by columns, - // if not already a part of the group-by + // if not already a part of the group-by. + // NOTE: Do NOT add ORDER BY columns to bottomGroups because that would break DISTINCT + // semantics. For example, SUM(DISTINCT sal) WITHIN GROUP (ORDER BY bonus) would incorrectly + // produce multiple rows for the same sal value if they have different bonus values, + // violating the DISTINCT guarantee. ORDER BY columns will be handled separately below. final NavigableSet bottomGroups = new TreeSet<>(aggregate.getGroupSet().asList()); for (AggregateCall aggCall : originalAggCalls) { if (aggCall.isDistinct()) { @@ -360,6 +402,14 @@ private static RelBuilder convertSingletonDistinct(RelBuilder relBuilder, for (int arg : aggCall.getArgList()) { newArgList.add(bottomGroups.headSet(arg, false).size()); } + // Remap collation field indices using the same bottomGroups formula + final List remappedFCs = new ArrayList<>(); + for (RelFieldCollation fc : aggCall.collation.getFieldCollations()) { + int newIdx = bottomGroups.headSet(fc.getFieldIndex(), false).size(); + remappedFCs.add(fc.withFieldIndex(newIdx)); + } + RelCollation newCollation = aggCall.collation.equals(RelCollations.EMPTY) + ? RelCollations.EMPTY : RelCollations.of(remappedFCs); newCall = AggregateCall.create(aggCall.getParserPosition(), aggCall.getAggregation(), @@ -370,7 +420,7 @@ private static RelBuilder convertSingletonDistinct(RelBuilder relBuilder, newArgList, -1, aggCall.distinctKeys, - aggCall.collation, + newCollation, aggregate.hasEmptyGroup(), relBuilder.peek(), aggCall.getType(), @@ -649,11 +699,14 @@ private static void rewriteUsingGroupingSets(RelOptRuleCall call, final String upperAggName = upperAggCallName(aggCall, g); // Each filtered grouping set emits exactly one row per group, // so MIN just passes that value through without re-aggregation + // Remap collation indices through fullGroupSet + final RelCollation remappedCollation = + remapCollationForGroupingSets(aggCall.collation, fullGroupSet); final AggregateCall newCall = AggregateCall.create(aggCall.getParserPosition(), SqlStdOperatorTable.MIN, false, aggCall.isApproximate(), aggCall.ignoreNulls(), aggCall.rexList, args, newFilterArg, - aggCall.distinctKeys, aggCall.collation, aggregate.hasEmptyGroup(), + aggCall.distinctKeys, remappedCollation, aggregate.hasEmptyGroup(), relBuilder.peek(), null, upperAggName); upperAggCalls.add(newCall); ordinals.add(topGroupCount + upperAggCalls.size() - 1); @@ -669,11 +722,14 @@ private static void rewriteUsingGroupingSets(RelOptRuleCall call, requireNonNull(filters.get(Pair.of(newGroupSet, aggCall.filterArg)), () -> "filters.get(" + newGroupSet + ", " + aggCall.filterArg + ")"); final String upperAggName = upperAggCallName(aggCall, g); + // Remap collation indices through fullGroupSet + final RelCollation remappedCollation = + remapCollationForGroupingSets(aggCall.collation, fullGroupSet); final AggregateCall newCall = AggregateCall.create(aggCall.getParserPosition(), aggCall.getAggregation(), false, aggCall.isApproximate(), aggCall.ignoreNulls(), aggCall.rexList, newArgList, newFilterArg, - aggCall.distinctKeys, aggCall.collation, + aggCall.distinctKeys, remappedCollation, aggregate.hasEmptyGroup(), relBuilder.peek(), null, upperAggName); upperAggCalls.add(newCall); ordinals.add(topGroupCount + upperAggCalls.size() - 1); @@ -864,6 +920,31 @@ private static int remap(ImmutableBitSet groupSet, int arg) { return arg < 0 ? -1 : groupSet.indexOf(arg); } + private static RelCollation remapCollationForGroupingSets(RelCollation collation, + ImmutableBitSet fullGroupSet) { + if (collation.equals(RelCollations.EMPTY)) { + return RelCollations.EMPTY; + } + // Remap each field index through the fullGroupSet. + // fullGroupSet contains only columns that appear in at least one grouping set. + // If an ORDER BY column is not in fullGroupSet, it means: + // 1. The column is not part of any grouping set combination + // 2. Different rows within the same logical group can have different values + // 3. Sorting by such a column would be meaningless + // Therefore, we safely drop ORDER BY columns not in fullGroupSet. The query + // planner should ensure ORDER BY columns are either in GROUP BY, in aggregation + // arguments, or properly scoped for consistent values within each group. + final List remappedFCs = new ArrayList<>(); + for (RelFieldCollation fc : collation.getFieldCollations()) { + int originalIdx = fc.getFieldIndex(); + int newIdx = fullGroupSet.indexOf(originalIdx); + if (newIdx >= 0) { + remappedFCs.add(fc.withFieldIndex(newIdx)); + } + } + return remappedFCs.isEmpty() ? RelCollations.EMPTY : RelCollations.of(remappedFCs); + } + private static String upperAggCallName(AggregateCall aggCall, int groupingSetIndex) { String baseName = aggCall.getName(); @@ -1013,7 +1094,7 @@ private static void doRewrite(RelBuilder relBuilder, Aggregate aggregate, int n, continue; } - // Re-map arguments. + // Re-map arguments and collation. final int argCount = aggCall.getArgList().size(); final List newArgs = new ArrayList<>(argCount); for (Integer arg : aggCall.getArgList()) { @@ -1022,7 +1103,7 @@ private static void doRewrite(RelBuilder relBuilder, Aggregate aggregate, int n, final AggregateCall newAggCall = AggregateCall.create(aggCall.getParserPosition(), aggCall.getAggregation(), false, aggCall.isApproximate(), aggCall.ignoreNulls(), aggCall.rexList, - newArgs, -1, aggCall.distinctKeys, aggCall.collation, + newArgs, -1, aggCall.distinctKeys, remapCollation(aggCall.collation, sourceOf), aggCall.getType(), aggCall.getName()); assert refs.get(i) == null; if (leftFields == null) { @@ -1098,7 +1179,7 @@ private static void rewriteAggCalls( continue; } - // Re-map arguments. + // Re-map arguments and collation. final int argCount = aggCall.getArgList().size(); final List newArgs = new ArrayList<>(argCount); for (int j = 0; j < argCount; j++) { @@ -1111,7 +1192,7 @@ private static void rewriteAggCalls( AggregateCall.create(aggCall.getParserPosition(), aggCall.getAggregation(), false, aggCall.isApproximate(), aggCall.ignoreNulls(), aggCall.rexList, newArgs, -1, - aggCall.distinctKeys, aggCall.collation, + aggCall.distinctKeys, remapCollation(aggCall.collation, sourceOf), aggCall.getType(), aggCall.getName()); newAggCalls.set(i, newAggCall); } @@ -1193,6 +1274,23 @@ private static RelBuilder createSelectDistinct(RelBuilder relBuilder, sourceOf.put(arg, projects.size()); RexInputRef.add2(projects, arg, childFields); } + + // Also project ORDER BY columns from WITHIN GROUP for DISTINCT agg calls. + // If the ORDER BY references a column not already in GROUP BY or argList, + // it must be projected here so that collation indices can be properly remapped. + for (AggregateCall aggCall : aggregate.getAggCallList()) { + if (!aggCall.isDistinct() || !aggCall.getArgList().equals(argList)) { + continue; + } + for (RelFieldCollation fc : aggCall.collation.getFieldCollations()) { + int col = fc.getFieldIndex(); + if (sourceOf.get(col) == null) { + sourceOf.put(col, projects.size()); + RexInputRef.add2(projects, col, childFields); + } + } + } + relBuilder.project(projects.leftList(), projects.rightList()); // Get the distinct values of the GROUP BY fields and the arguments diff --git a/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableSortedAggregateTest.java b/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableSortedAggregateTest.java index 1945dabaed39..454beb201588 100644 --- a/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableSortedAggregateTest.java +++ b/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableSortedAggregateTest.java @@ -133,6 +133,106 @@ public class EnumerableSortedAggregateTest { "commission=null; num_dept=1"); } + /** Test case for + * [CALCITE-5101] + * LISTAGG(DISTINCT ...) WITHIN GROUP fails with ArrayIndexOutOfBoundsException. */ + @Test void listAggDistinctWithinGroupOrderByNonGroupColumn() { + // FAILING CASE: LISTAGG(DISTINCT ...) WITHIN GROUP (ORDER BY non-group-column) + // Previously threw: ArrayIndexOutOfBoundsException: Index 2 out of bounds for length 2 + // Now fixed: ORDER BY salary is properly projected and remapped. + // Within each group, names are ordered by salary (ascending). + // deptno=10: Sebastian (7000), Bill (10000), Theodore (11500) + // deptno=20: Eric (8000) + tester(false, new HrSchema()) + .query("select deptno, " + + "LISTAGG(DISTINCT name) WITHIN GROUP (ORDER BY salary) as names " + + "from emps group by deptno") + .returnsUnordered( + "deptno=10; names=Sebastian,Bill,Theodore", + "deptno=20; names=Eric"); + } + + @Test void listAggDistinctWithoutOrderBy() { + // WORKING CASE: LISTAGG(DISTINCT ...) without ORDER BY - always worked + tester(false, new HrSchema()) + .query("select deptno, " + + "LISTAGG(DISTINCT name) as names " + + "from emps group by deptno") + .returnsUnordered( + "deptno=10; names=Bill,Sebastian,Theodore", + "deptno=20; names=Eric"); + } + + @Test void listAggWithoutDistinctWithinGroupOrderBy() { + // WORKING CASE: LISTAGG(...) WITHIN GROUP (ORDER BY non-group-column) + // without DISTINCT - always worked + tester(false, new HrSchema()) + .query("select deptno, " + + "LISTAGG(name) WITHIN GROUP (ORDER BY salary) as names " + + "from emps group by deptno") + .returnsUnordered( + "deptno=10; names=Sebastian,Bill,Theodore", + "deptno=20; names=Eric"); + } + + @Test void listAggDistinctWithinGroupOrderByAggColumn() { + // WORKING CASE: LISTAGG(DISTINCT ...) WITHIN GROUP (ORDER BY aggregated-column) + // - always worked because the aggregated column is in the re-grouped input + tester(false, new HrSchema()) + .query("select deptno, " + + "LISTAGG(DISTINCT name) WITHIN GROUP (ORDER BY name) as names " + + "from emps group by deptno") + .returnsUnordered( + "deptno=10; names=Bill,Sebastian,Theodore", + "deptno=20; names=Eric"); + } + + @Test void listAggMultipleDistinctWithinGroupOrderByNonGroupColumn() { + // Test case for multiple DISTINCT aggregates with different ORDER BY columns + // Previously threw: ArrayIndexOutOfBoundsException + // ORDER BY salary for first agg (not a group key), ORDER BY name for second + // This tests that collation indices are properly remapped in rewriteUsingGroupingSets + tester(false, new HrSchema()) + .query("select deptno, " + + "LISTAGG(DISTINCT name) WITHIN GROUP (ORDER BY salary) as names_by_salary, " + + "LISTAGG(DISTINCT commission) WITHIN GROUP (ORDER BY name) as commissions_by_name " + + "from emps group by deptno") + .returnsUnordered( + "deptno=10; names_by_salary=Theodore,Sebastian,Bill; commissions_by_name=250,1000", + "deptno=20; names_by_salary=Eric; commissions_by_name=500"); + } + + @Test void groupingSetsWithDistinctAggAndCollationReferencingOutsideGroupingSets() { + // Test that result collation is safely reduced when ORDER BY references + // columns not present in all grouping sets. + // Why result collation can have fewer elements than input collation: + // - Input collation may reference {salary, deptno, name} + // - fullGroupSet contains only {deptno, name} (columns in some grouping sets) + // - salary is NOT in fullGroupSet (not part of any GROUPING SETS combination) + // - In GROUPING SETS ((deptno), (name), ()), salary has inconsistent values + // within each logical group, so sorting by it would be meaningless + // - Result collation safely drops salary and keeps only {deptno, name} + // This tests that AggregateExpandDistinctAggregatesRule.remapCollationForGroupingSets + // correctly filters out columns not in fullGroupSet. + // The key point: this query should not throw ArrayIndexOutOfBoundsException. + tester(false, new HrSchema()) + .query("select deptno, name, " + + "LISTAGG(DISTINCT salary) WITHIN GROUP (ORDER BY salary) as salaries " + + "from emps " + + "group by grouping sets ((deptno), (name), ())") + .returnsUnordered( + // Grouping by deptno - contains the 3 salaries from deptno=10 + "deptno=10; name=null; salaries=10000.0,7000.0,11500.0", + "deptno=20; name=null; salaries=8000.0", + // Grouping by name - single salary per name + "deptno=null; name=Bill; salaries=10000.0", + "deptno=null; name=Eric; salaries=8000.0", + "deptno=null; name=Sebastian; salaries=7000.0", + "deptno=null; name=Theodore; salaries=11500.0", + // Grand total - all 4 distinct salaries + "deptno=null; name=null; salaries=10000.0,8000.0,7000.0,11500.0"); + } + private CalciteAssert.AssertThat tester(boolean forceDecorrelate, Object schema) { return CalciteAssert.that() From d9014bad3acb3564b5be8c77a28083ea4e31bf0b Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Sat, 6 Jun 2026 22:32:15 +0800 Subject: [PATCH 308/562] [CALCITE-7590] Improve error message for window functions missing OVER clause to include function name --- .../calcite/runtime/CalciteResource.java | 4 ++-- .../apache/calcite/sql/SqlAggFunction.java | 2 +- .../sql/validate/SqlValidatorImpl.java | 2 +- .../runtime/CalciteResource.properties | 2 +- .../apache/calcite/test/SqlValidatorTest.java | 20 +++++++++---------- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java b/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java index 40c680e2ee6c..a9cb02d065cb 100644 --- a/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java +++ b/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java @@ -341,8 +341,8 @@ ExInst invalidCompare(String a0, String a1, String a2, ExInst naturalOrUsingColumnNotCompatible(String a0, String a1, String a2); - @BaseMessage("OVER clause is necessary for window functions") - ExInst absentOverClause(); + @BaseMessage("window function {0} requires an OVER clause") + ExInst absentOverClause(String a0); @BaseMessage("MEASURE not valid in aggregate or DISTINCT query") ExInst measureInAggregateQuery(); diff --git a/core/src/main/java/org/apache/calcite/sql/SqlAggFunction.java b/core/src/main/java/org/apache/calcite/sql/SqlAggFunction.java index 7ba19c48ded1..609bfee6d517 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlAggFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlAggFunction.java @@ -138,7 +138,7 @@ protected SqlAggFunction( SqlValidatorScope operandScope) { if (requiresOver() && !validator.isInWindow()) { throw validator.newValidationError(call, - Static.RESOURCE.absentOverClause()); + Static.RESOURCE.absentOverClause(getName())); } super.validateCall(call, validator, scope, operandScope); validator.validateAggregateParams(call, null, null, null, scope); diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java index ae179400b013..370ef5ecc39a 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java @@ -5528,7 +5528,7 @@ private void validateExpr(SqlNode expr, SqlValidatorScope scope) { final SqlOperator op = ((SqlCall) expr).getOperator(); if (op.isAggregator() && op.requiresOver()) { throw newValidationError(expr, - RESOURCE.absentOverClause()); + RESOURCE.absentOverClause(op.getName())); } if (op instanceof SqlTableFunction) { throw RESOURCE.cannotCallTableFunctionHere(op.getName()).ex(); diff --git a/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties b/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties index ae64b08bdd66..056aeb7b0715 100644 --- a/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties +++ b/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties @@ -161,7 +161,7 @@ RowMustBeNonNegativeIntegral=ROWS value must be a non-negative integral constant OverMissingOrderBy=Window specification must contain an ORDER BY clause PartitionbyShouldNotContainOver=PARTITION BY expression should not contain OVER clause OrderbyShouldNotContainOver=ORDER BY expression should not contain OVER clause -AbsentOverClause=OVER clause is necessary for window functions +AbsentOverClause=window function {0} requires an OVER clause BadLowerBoundary=UNBOUNDED FOLLOWING cannot be specified for the lower frame boundary BadUpperBoundary=UNBOUNDED PRECEDING cannot be specified for the upper frame boundary CurrentRowPrecedingError=Upper frame boundary cannot be PRECEDING when lower boundary is CURRENT ROW diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index 1cba948097f1..5cd74f62f21a 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -3448,11 +3448,11 @@ void testWinPartClause() { + "from emp\n" + "group by deptno\n" + "order by ^row_number()^") - .fails("OVER clause is necessary for window functions"); + .fails("window function ROW_NUMBER requires an OVER clause"); winSql("select ^rank()^\n" + "from emp") - .fails("OVER clause is necessary for window functions"); + .fails("window function RANK requires an OVER clause"); // With [CALCITE-1340], the validator would see RANK without OVER, // mistakenly think this is an aggregate query, and wrongly complain @@ -3460,33 +3460,33 @@ void testWinPartClause() { winSql("select cume_dist() over w , ^rank()^\n" + "from emp\n" + "window w as (partition by deptno order by deptno)") - .fails("OVER clause is necessary for window functions"); + .fails("window function RANK requires an OVER clause"); winSql("select ^nth_value(sal, 2)^\n" + "from emp") - .fails("OVER clause is necessary for window functions"); + .fails("window function NTH_VALUE requires an OVER clause"); winSql("select ^first_value(sal)^\n" + "from emp") - .fails("OVER clause is necessary for window functions"); + .fails("window function FIRST_VALUE requires an OVER clause"); winSql("select ^last_value(sal)^\n" + "from emp") - .fails("OVER clause is necessary for window functions"); + .fails("window function LAST_VALUE requires an OVER clause"); // With alias, first_value and last_value without OVER should also fail winSql("select ^first_value(sal)^ as sal_first\n" + "from emp") - .fails("OVER clause is necessary for window functions"); + .fails("window function FIRST_VALUE requires an OVER clause"); winSql("select ^last_value(sal)^ as sal_last\n" + "from emp") - .fails("OVER clause is necessary for window functions"); + .fails("window function LAST_VALUE requires an OVER clause"); // In GROUP BY context, first_value and last_value without OVER should fail winSql("select sal, ^first_value(sal)^ as sal_first\n" + "from emp group by sal, deptno") - .fails("OVER clause is necessary for window functions"); + .fails("window function FIRST_VALUE requires an OVER clause"); // first_value and last_value with OVER clause should succeed winSql("select first_value(sal) over (order by empno)\n" @@ -3632,7 +3632,7 @@ void testWinPartClause() { // rank function type if (defined.contains("DENSE_RANK")) { winExp("^dense_rank()^") - .fails("OVER clause is necessary for window functions"); + .fails("window function DENSE_RANK requires an OVER clause"); } else { checkWinFuncExpWithWinClause("^dense_rank()^", "Function 'DENSE_RANK\\(\\)' is not defined"); From c889e4006454cfd37c59322c521dae85fb0a4a1a Mon Sep 17 00:00:00 2001 From: Sergey Nuyanzin Date: Mon, 1 Jun 2026 07:50:21 +0200 Subject: [PATCH 309/562] [CALCITE-7575] Parser can not parse unparsed polymorphic table functions with several table args --- core/src/main/codegen/templates/Parser.jj | 14 +++++ .../apache/calcite/test/SqlValidatorTest.java | 15 +++++ site/_docs/reference.md | 55 +++++++++++++++++++ 3 files changed, 84 insertions(+) diff --git a/core/src/main/codegen/templates/Parser.jj b/core/src/main/codegen/templates/Parser.jj index 2f784b34125b..0aacdb9ff747 100644 --- a/core/src/main/codegen/templates/Parser.jj +++ b/core/src/main/codegen/templates/Parser.jj @@ -1081,6 +1081,9 @@ void AddArg(List list, ExprContext exprContext) : { final SqlIdentifier name; SqlNode e; + final Span s; + SqlNodeList partitionList; + SqlNodeList orderList; } { ( @@ -1094,6 +1097,17 @@ void AddArg(List list, ExprContext exprContext) : e = LambdaExpression() | e = Expression(exprContext) + // A set-semantics table argument may be a partitioned sub-query, e.g. + // "(SELECT ...) PARTITION BY key [ORDER BY ...]". + [ + { s = span(); } + partitionList = SimpleIdentifierOrList() + ( + orderList = OrderByOfSetSemanticsTable() + | { orderList = SqlNodeList.EMPTY; } + ) + { e = CreateSetSemanticsTableIfNeeded(s, e, partitionList, orderList); } + ] | e = TableParam() ) diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index 5cd74f62f21a..9a3fe9b83321 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -1997,6 +1997,21 @@ void testLikeAndSimilarFails() { + " table emp partition by deptno order by empno,\n" + " table emp_b partition by deptno order by empno))") .ok(); + + // Test cases for [CALCITE-7575] https://issues.apache.org/jira/browse/CALCITE-7575 + // Parser can not parse unparsed polymorphic table functions with several table args + final String emp = "(SELECT `EMP`.`EMPNO`, `EMP`.`ENAME`, `EMP`.`JOB`, `EMP`.`MGR`, " + + "`EMP`.`HIREDATE`, `EMP`.`SAL`, `EMP`.`COMM`, `EMP`.`DEPTNO`, `EMP`.`SLACKER`\n" + + "FROM `CATALOG`.`SALES`.`EMP` AS `EMP`) PARTITION BY `DEPTNO`"; + final String rewritten = "SELECT `EXPR$0`.`VAL`\n" + + "FROM TABLE(SIMILARLITY(" + + emp + ", " + emp + ")) AS `EXPR$0`"; + sql("select * from table(SIMILARLITY(\n" + + " table emp partition by deptno,\n" + + " table emp partition by deptno))") + .withValidatorIdentifierExpansion(true) + .rewritesTo(rewritten); + sql(rewritten).withParserConfig(c -> c.withQuoting(Quoting.BACK_TICK)).ok(); } @Test void testUnknownFunctionHandling() { diff --git a/site/_docs/reference.md b/site/_docs/reference.md index 1e1ba19f3701..e50acf936f1c 100644 --- a/site/_docs/reference.md +++ b/site/_docs/reference.md @@ -2318,6 +2318,61 @@ each row in the original table to a window. The output table has all the same columns as the original table plus two additional columns `window_start` and `window_end`, which represent the start and end of the window interval, respectively. +#### Polymorphic table functions + +A polymorphic table function (PTF) is a table function that takes one or more +*table arguments* in addition to scalar and `DESCRIPTOR` arguments. Like any +table function, it is invoked in the `FROM` clause; the examples below place the +call inside the `TABLE( ... )` operator, as in `SELECT * FROM TABLE(funcName(...))`, +which is the form used by the SQL standard, but Calcite also accepts the implicit +form `SELECT * FROM funcName(...)`. + +A table argument is introduced by the `TABLE` keyword and is either a table +reference or a query. A table argument with set semantics may be followed by a +`PARTITION BY` clause, an `ORDER BY` clause, or both; each partition is then +processed independently. A table argument with row semantics may not be +partitioned or ordered. + +A PTF may have more than one table argument, each with its own semantics, +partitioning and ordering; at most one of them may have row semantics. + +| Operator syntax | Description +|:-------------------- |:----------- +| TABLE(funcName(tableArg [, tableArg ]* [, arg ]*))
      where tableArg is  TABLE tableExpr [ PARTITION BY columns ] [ ORDER BY columns ] | Invokes polymorphic table function *funcName* over one or more table arguments, each optionally partitioned and ordered (set semantics), together with any scalar or `DESCRIPTOR` arguments. + +A PTF with a single table argument: + +{% highlight sql %} +SELECT * FROM TABLE( +topn( +TABLE orders PARTITION BY product ORDER BY amount DESC, +3)); + +-- or with the named params +SELECT * FROM TABLE( +topn( +DATA => TABLE orders PARTITION BY product ORDER BY amount DESC, +COL => 3)); +{% endhighlight %} + +partitions the `orders` table by `product`, orders each partition by `amount` +descending, and lets the function emit, say, the top 3 rows of each partition. + +A PTF with multiple table arguments: + +{% highlight sql %} +SELECT * FROM TABLE( +similarity( +TABLE orders PARTITION BY product, +TABLE returns PARTITION BY product)); +{% endhighlight %} + +passes two input tables, each partitioned by `product`, so the function can +compare the matching partitions of `orders` and `returns`. + +The built-in window table functions `TUMBLE`, `HOP` and `SESSION` (described +below) are concrete polymorphic table functions. + ### Grouped window functions **warning**: grouped window functions are deprecated. From 2f91b775d76c15ed7c770c53691e465e508075b8 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Fri, 5 Jun 2026 22:18:59 -0700 Subject: [PATCH 310/562] [CALCITE-7582] Type validation errors should use SQL type names Signed-off-by: Mihai Budiu --- .../java/org/apache/calcite/sql/SqlCall.java | 4 +- .../apache/calcite/sql/type/OperandTypes.java | 4 +- .../apache/calcite/sql/type/SqlTypeUtil.java | 244 ++++++++++++++++++ .../apache/calcite/test/SqlValidatorTest.java | 60 +++-- core/src/test/resources/sql/misc.iq | 2 +- .../apache/calcite/test/SqlOperatorTest.java | 44 ++-- 6 files changed, 313 insertions(+), 45 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql/SqlCall.java b/core/src/main/java/org/apache/calcite/sql/SqlCall.java index f3edc22a3692..1747272f621a 100755 --- a/core/src/main/java/org/apache/calcite/sql/SqlCall.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlCall.java @@ -18,6 +18,7 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.sql.parser.SqlParserPos; +import org.apache.calcite.sql.type.SqlTypeUtil; import org.apache.calcite.sql.util.SqlVisitor; import org.apache.calcite.sql.validate.SqlMoniker; import org.apache.calcite.sql.validate.SqlMonotonicity; @@ -210,7 +211,8 @@ public String getCallSignature( if (null == argType) { continue; } - signatureList.add(argType.toString()); + signatureList.add( + SqlTypeUtil.asSqlType(argType, SqlTypeUtil.NullabilityDisplay.DoNotDisplay)); } return SqlUtil.getOperatorSignature(getOperator(), signatureList); } diff --git a/core/src/main/java/org/apache/calcite/sql/type/OperandTypes.java b/core/src/main/java/org/apache/calcite/sql/type/OperandTypes.java index f525975c986b..99c623865cb1 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/OperandTypes.java +++ b/core/src/main/java/org/apache/calcite/sql/type/OperandTypes.java @@ -1535,7 +1535,7 @@ private RecordTypeWithOneFieldChecker(Predicate predicate) { @Override public String getAllowedSignatures(SqlOperator op, String opName) { return SqlUtil.getAliasedSignature(op, opName, - ImmutableList.of("RECORDTYPE(SINGLE FIELD)")); + ImmutableList.of("ROW(SINGLE FIELD)")); } }; @@ -1561,7 +1561,7 @@ private static class MapFromEntriesOperandTypeChecker @Override public String getAllowedSignatures(SqlOperator op, String opName) { return SqlUtil.getAliasedSignature(op, opName, - ImmutableList.of("ARRAY")); + ImmutableList.of("ARRAY")); } } diff --git a/core/src/main/java/org/apache/calcite/sql/type/SqlTypeUtil.java b/core/src/main/java/org/apache/calcite/sql/type/SqlTypeUtil.java index 1be36fa18b84..02988668287b 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/SqlTypeUtil.java +++ b/core/src/main/java/org/apache/calcite/sql/type/SqlTypeUtil.java @@ -2183,4 +2183,248 @@ public static RelDataType fromMeasure(RelDataTypeFactory typeFactory, } return type; } + + private static void appendPrecisionScale(RelDataType type, StringBuilder builder) { + builder + .append("(") + .append(type.getPrecision()) + .append(", ") + .append(type.getScale()) + .append(")"); + } + + private static void appendPrecision(RelDataType type, StringBuilder builder, boolean omitZero) { + if (type.getPrecision() != RelDataType.PRECISION_NOT_SPECIFIED) { + if (omitZero && type.getPrecision() == 0) { + return; + } + builder + .append("(") + .append(type.getPrecision()) + .append(")"); + } + } + + /** How to display type nullability. */ + public enum NullabilityDisplay { + /** Display NULL if type is nullable. */ + DisplayNullability, + /** Display NOT NULL if type is not nullable. */ + DisplayNonNullability, + /** Do not display nullability information at all. */ + DoNotDisplay + } + + /** Produce a string which describes the {@code type} using Calcite's SQL syntax for types. */ + public static String asSqlType(RelDataType type, NullabilityDisplay nullDisplay) { + StringBuilder builder = new StringBuilder(); + switch (type.getSqlTypeName()) { + case BOOLEAN: + builder.append("BOOLEAN"); + break; + case TINYINT: + builder.append("TINYINT"); + break; + case SMALLINT: + builder.append("SMALLINT"); + break; + case INTEGER: + builder.append("INTEGER"); + break; + case BIGINT: + builder.append("BIGINT"); + break; + case UTINYINT: + builder.append("TINYINT UNSIGNED"); + break; + case USMALLINT: + builder.append("SMALLINT UNSIGNED"); + break; + case UINTEGER: + builder.append("INTEGER UNSIGNED"); + break; + case UBIGINT: + builder.append("BIGINT UNSIGNED"); + break; + case DECIMAL: + builder.append("DECIMAL"); + appendPrecisionScale(type, builder); + break; + case FLOAT: + builder.append("FLOAT"); + break; + case REAL: + builder.append("REAL"); + break; + case DOUBLE: + builder.append("DOUBLE"); + break; + case DATE: + builder.append("DATE"); + break; + case TIME: + builder.append("TIME"); + appendPrecision(type, builder, false); + break; + case TIME_WITH_LOCAL_TIME_ZONE: + builder.append("TIME WITH LOCAL TIME ZONE"); + appendPrecision(type, builder, false); + break; + case TIME_TZ: + builder.append("TIME WITH TIME ZONE"); + appendPrecision(type, builder, false); + break; + case TIMESTAMP: + builder.append("TIMESTAMP"); + appendPrecision(type, builder, false); + break; + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + builder.append("TIMESTAMP WITH LOCAL TIME ZONE"); + appendPrecision(type, builder, false); + break; + case TIMESTAMP_TZ: + builder.append("TIMESTAMP WITH TIME ZONE"); + appendPrecision(type, builder, false); + break; + case INTERVAL_YEAR: + case INTERVAL_YEAR_MONTH: + case INTERVAL_MONTH: + case INTERVAL_DAY: + case INTERVAL_DAY_HOUR: + case INTERVAL_DAY_MINUTE: + case INTERVAL_DAY_SECOND: + case INTERVAL_HOUR: + case INTERVAL_HOUR_MINUTE: + case INTERVAL_HOUR_SECOND: + case INTERVAL_MINUTE: + case INTERVAL_MINUTE_SECOND: + case INTERVAL_SECOND: + builder.append("INTERVAL "); + builder.append(type.getIntervalQualifier()); + break; + case CHAR: + builder.append("CHAR"); + appendPrecision(type, builder, false); + break; + case VARCHAR: + builder.append("VARCHAR"); + appendPrecision(type, builder, false); + break; + case BINARY: + builder.append("BINARY"); + appendPrecision(type, builder, false); + break; + case VARBINARY: + builder.append("VARBINARY"); + appendPrecision(type, builder, false); + break; + case NULL: + // No suffix needed + return "NULL"; + case UNKNOWN: + return "UNKNOWN"; + case ANY: + return "ANY"; + case MULTISET: { + String elementType = + asSqlType(requireNonNull(type.getComponentType(), "componentType"), + NullabilityDisplay.DoNotDisplay); + builder.append(elementType); + builder.append(" MULTISET"); + break; + } + case ARRAY: { + String elementType = + asSqlType(requireNonNull(type.getComponentType(), "componentType"), + NullabilityDisplay.DoNotDisplay); + builder.append(elementType); + builder.append(" ARRAY"); + break; + } + case MAP: + builder.append("MAP<"); + String keyType = + asSqlType(requireNonNull(type.getKeyType(), "keyType"), + NullabilityDisplay.DoNotDisplay); + String valueType = + asSqlType(requireNonNull(type.getValueType(), "valueType"), + NullabilityDisplay.DoNotDisplay); + builder.append(keyType) + .append(", ") + .append(valueType) + .append(">"); + break; + case ROW: { + builder.append("ROW("); + boolean first = true; + for (RelDataTypeField field : type.getFieldList()) { + if (!first) { + builder.append(", "); + } + first = false; + String fieldType = asSqlType(field.getType(), NullabilityDisplay.DisplayNullability); + builder.append(fieldType) + .append(" ") + .append(field.getName()); + } + builder.append(")"); + break; + } + case GEOMETRY: + break; + case MEASURE: + builder.append("MEASURE"); + break; + case FUNCTION: { + FunctionSqlType function = (FunctionSqlType) type; + builder.append("FUNCTION("); + // Parameter is expected to always have a ROW type + boolean first = true; + for (RelDataTypeField field : function.getParameterTypes().getFieldList()) { + if (!first) { + builder.append(", "); + } + first = false; + String fieldType = asSqlType(field.getType(), NullabilityDisplay.DoNotDisplay); + builder.append(fieldType); + } + builder.append(") -> "); + String result = asSqlType(function.getReturnType(), NullabilityDisplay.DoNotDisplay); + builder.append(result); + break; + } + case CURSOR: + return "CURSOR"; + case COLUMN_LIST: + return "COLUMN_LIST"; + case SYMBOL: + case DISTINCT: + case STRUCTURED: + case OTHER: + case DYNAMIC_STAR: + case SARG: + return ""; + case UUID: + builder.append("UUID"); + break; + case VARIANT: + builder.append("VARIANT"); + break; + } + switch (nullDisplay) { + case DoNotDisplay: + break; + case DisplayNonNullability: + if (!type.isNullable()) { + builder.append(" NOT NULL"); + } + break; + case DisplayNullability: + if (type.isNullable()) { + builder.append(" NULL"); + } + break; + } + return builder.toString(); + } } diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index 9a3fe9b83321..8e4e76378b12 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -540,7 +540,7 @@ static SqlOperatorTable operatorTableFor(SqlLibrary library) { + "' = '.*"); expr("^MAP[x'a4', 1] = MAP[1, 1]^") .fails("(?s).*Cannot apply '=' to arguments of type " - + "'<.BINARY.1., INTEGER. MAP> = <.INTEGER, INTEGER. MAP>'.*"); + + "'> = >'.*"); expr("^array[x'a4'] = 1^") .fails("(?s).*Cannot apply '=' to arguments of type ' = '.*"); expr("^multiset[x'a4'] = multiset[1]^") @@ -6513,7 +6513,7 @@ void testReturnsCorrectRowTypeOnCombinedJoin() { final String sql = "select * from emp as e join dept d\n" + "on d.deptno = ^(select 1, 2 from emp where deptno < e.deptno)^"; final String expected = "(?s)Cannot apply '\\$SCALAR_QUERY' to arguments " - + "of type '\\$SCALAR_QUERY\\(\\)'\\. Supported form\\(s\\).*"; sql(sql).fails(expected); } @@ -9779,9 +9779,9 @@ void testGroupExpressionEquivalenceParams() { sql("SELECT ename,(select name from dept where deptno=1) FROM emp").ok(); sql("SELECT ename,^(select losal, hisal from salgrade where grade=1)^ FROM emp") .fails("Cannot apply '\\$SCALAR_QUERY' to arguments of type " - + "'\\$SCALAR_QUERY\\(\\)'\\. Supported form\\(s\\): " - + "'\\$SCALAR_QUERY\\(\\)'"); + + "'\\$SCALAR_QUERY\\(\\)'"); // Note that X is a field (not a record) and is nullable even though // EMP.NAME is NOT NULL. @@ -12134,7 +12134,7 @@ private void checkCustomColumnResolving(String table) { sql("select rowtime, productid, orderid, 'window_start', 'window_end'\n" + "from table(\n" + "^tumble(table orders, descriptor(productid), interval '2' hour)^)") - .fails("Cannot apply 'TUMBLE' to arguments of type 'TUMBLE\\(, , " + "\\)'\\. Supported form\\(s\\): TUMBLE\\(TABLE table_name, " + "DESCRIPTOR\\(timecol\\), datetime interval\\[, datetime interval\\]\\)"); @@ -12144,7 +12144,7 @@ private void checkCustomColumnResolving(String table) { + "data => table orders,\n" + "timecol => descriptor(productid),\n" + "size => interval '2' hour)^)") - .fails("Cannot apply 'TUMBLE' to arguments of type 'TUMBLE\\(, , " + "\\)'\\. Supported form\\(s\\): TUMBLE\\(TABLE table_name, " + "DESCRIPTOR\\(timecol\\), datetime interval\\[, datetime interval\\]\\)"); @@ -12159,21 +12159,21 @@ private void checkCustomColumnResolving(String table) { .fails("Invalid number of arguments to function 'TUMBLE'. Was expecting 3 arguments"); sql("select * from table(\n" + "^tumble(table orders, descriptor(rowtime), 'test')^)") - .fails("Cannot apply 'TUMBLE' to arguments of type 'TUMBLE\\(, ," + " \\)'\\. Supported form\\(s\\): TUMBLE\\(TABLE " + "table_name, DESCRIPTOR\\(timecol\\), datetime interval" + "\\[, datetime interval\\]\\)"); sql("select * from table(\n" + "^tumble(table orders, 'test', interval '2' hour)^)") - .fails("Cannot apply 'TUMBLE' to arguments of type 'TUMBLE\\(, , \\)'\\. Supported form\\(s\\): TUMBLE\\(TABLE " + "table_name, DESCRIPTOR\\(timecol\\), datetime interval" + "\\[, datetime interval\\]\\)"); sql("select rowtime, productid, orderid, 'window_start', 'window_end' from table(\n" + "^tumble(table orders, descriptor(rowtime), interval '2' hour, 'test')^)") - .fails("Cannot apply 'TUMBLE' to arguments of type 'TUMBLE\\(, ," + " , \\)'\\. Supported form\\(s\\): TUMBLE\\(TABLE " + "table_name, DESCRIPTOR\\(timecol\\), datetime interval" @@ -12221,7 +12221,7 @@ private void checkCustomColumnResolving(String table) { sql("select rowtime, productid, orderid, 'window_start', 'window_end'\n" + "from table(\n" + "^hop(table orders, descriptor(productid), interval '2' hour, interval '1' hour)^)") - .fails("Cannot apply 'HOP' to arguments of type 'HOP\\(, , " + ", \\)'\\. Supported form\\(s\\): " + "HOP\\(TABLE table_name, DESCRIPTOR\\(timecol\\), " @@ -12233,7 +12233,7 @@ private void checkCustomColumnResolving(String table) { + "timecol => descriptor(productid),\n" + "size => interval '2' hour,\n" + "slide => interval '1' hour)^)") - .fails("Cannot apply 'HOP' to arguments of type 'HOP\\(, , " + ", \\)'\\. Supported form\\(s\\): " + "HOP\\(TABLE table_name, DESCRIPTOR\\(timecol\\), " @@ -12249,25 +12249,25 @@ private void checkCustomColumnResolving(String table) { .fails("Invalid number of arguments to function 'HOP'. Was expecting 4 arguments"); sql("select * from table(\n" + "^hop(table orders, descriptor(rowtime), interval '2' hour, 'test')^)") - .fails("Cannot apply 'HOP' to arguments of type 'HOP\\(, , , " + "\\)'. Supported form\\(s\\): HOP\\(TABLE table_name, DESCRIPTOR\\(" + "timecol\\), datetime interval, datetime interval\\[, datetime interval\\]\\)"); sql("select * from table(\n" + "^hop(table orders, descriptor(rowtime), 'test', interval '2' hour)^)") - .fails("Cannot apply 'HOP' to arguments of type 'HOP\\(, , , " + "\\)'. Supported form\\(s\\): HOP\\(TABLE table_name, DESCRIPTOR\\(" + "timecol\\), datetime interval, datetime interval\\[, datetime interval\\]\\)"); sql("select * from table(\n" + "^hop(table orders, 'test', interval '2' hour, interval '2' hour)^)") - .fails("Cannot apply 'HOP' to arguments of type 'HOP\\(, , , " + "\\)'. Supported form\\(s\\): HOP\\(TABLE table_name, DESCRIPTOR\\(" + "timecol\\), datetime interval, datetime interval\\[, datetime interval\\]\\)"); sql("select * from table(\n" + "^hop(table orders, descriptor(rowtime), interval '2' hour, interval '1' hour, 'test')^)") - .fails("Cannot apply 'HOP' to arguments of type 'HOP\\(, , , " + ", \\)'. Supported form\\(s\\): HOP\\(TABLE table_name, " + "DESCRIPTOR\\(timecol\\), datetime interval, datetime interval\\[, datetime interval\\]\\)"); @@ -12308,25 +12308,25 @@ private void checkCustomColumnResolving(String table) { + "data => table orders,\n" + "key => descriptor(productid),\n" + "size => interval '1' hour)^)") - .fails("Cannot apply 'SESSION' to arguments of type 'SESSION\\(, , " + "\\)'. Supported form\\(s\\): SESSION\\(TABLE table_name, DESCRIPTOR\\(" + "timecol\\), DESCRIPTOR\\(key\\) optional, datetime interval\\)"); sql("select * from table(\n" + "^session(table orders, descriptor(rowtime), descriptor(productid), 'test')^)") - .fails("Cannot apply 'SESSION' to arguments of type 'SESSION\\(, , , " + "\\)'. Supported form\\(s\\): SESSION\\(TABLE table_name, DESCRIPTOR\\(" + "timecol\\), DESCRIPTOR\\(key\\) optional, datetime interval\\)"); sql("select * from table(\n" + "^session(table orders, descriptor(rowtime), 'test', interval '2' hour)^)") - .fails("Cannot apply 'SESSION' to arguments of type 'SESSION\\(, , , " + "\\)'. Supported form\\(s\\): SESSION\\(TABLE table_name, DESCRIPTOR\\(" + "timecol\\), DESCRIPTOR\\(key\\) optional, datetime interval\\)"); sql("select * from table(\n" + "^session(table orders, 'test', descriptor(productid), interval '2' hour)^)") - .fails("Cannot apply 'SESSION' to arguments of type 'SESSION\\(, , , " + "\\)'. Supported form\\(s\\): SESSION\\(TABLE table_name, DESCRIPTOR\\(" + "timecol\\), DESCRIPTOR\\(key\\) optional, datetime interval\\)"); @@ -14295,4 +14295,26 @@ private static SqlIdentifier rewriteIdentifier(SqlIdentifier sqlIdentifier) { } } } + + /** Test case for [CALCITE-7582] + * Type validation errors should use SQL type names. */ + @Test public void testErrorMessage() { + sql("SELECT ^(DATE '2020-02-02', DATE '2021-01-01') CONTAINS TIME '10:00:00'^") + .fails(".*Cannot apply 'CONTAINS' to arguments of type ' " + + "CONTAINS '\\. Supported form\\(s\\): '\\(

      ,
      \\) " + + "CONTAINS \\(
      ,
      \\)'\\n" + + "'\\(
      ,
      \\) CONTAINS \\(
      , \\)'\n" + + "'\\(
      , \\) CONTAINS \\(
      ,
      \\)'\n" + + "'\\(
      , \\) CONTAINS \\(
      , \\)'\n" + + "'\\(
      ,
      \\) CONTAINS
      '\n" + + "'\\(
      , \\) CONTAINS
      '\\n" + + "Where 'DT' is one of 'DATE', 'TIME', or 'TIMESTAMP', " + + "the same for all arguments\\."); + sql("SELECT ^MAP['x', ARRAY[3]] = " + + "MAP[CAST(ROW(1, 2) AS ROW(X INT, Y BIGINT)), ROW(ARRAY['a'])]^") + .fails("Cannot apply '=' to arguments of type '> = " + + ">'\\. " + + "Supported form\\(s\\): ' = '"); + } } diff --git a/core/src/test/resources/sql/misc.iq b/core/src/test/resources/sql/misc.iq index 05697b81185e..f5457a79c7d8 100644 --- a/core/src/test/resources/sql/misc.iq +++ b/core/src/test/resources/sql/misc.iq @@ -175,7 +175,7 @@ SELECT ROW('1') > ROW(0) AS C; # [CALCITE-6735] Type coercion for comparisons does not coerce ROW types # These are compared in the same way as x'10' > 0, which throws SELECT ROW(x'10') > ROW(0) AS C; -java.sql.SQLException: Error while executing SQL "SELECT ROW(x'10') > ROW(0) AS C": From line 1, column 11 to line 1, column 26: Cannot apply '>' to arguments of type ' > '. +java.sql.SQLException: Error while executing SQL "SELECT ROW(x'10') > ROW(0) AS C": From line 1, column 11 to line 1, column 26: Cannot apply '>' to arguments of type ' > '. !error # [CALCITE-6733] Type inferred by coercion for comparisons with decimal is too narrow diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java index c40f077d8d1f..d399890342d5 100644 --- a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java +++ b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java @@ -3321,26 +3321,26 @@ static void checkOverlaps(OverlapChecker c) { final SqlOperatorFixture f = fixture(); f.checkFails("^(DATE '2020-10-10', DATE '2021-10-10') CONTAINS TIME '10:00:00'^", "Cannot apply 'CONTAINS' to arguments of type " - + "' CONTAINS '\\. " + + "' CONTAINS '\\. " + containsError, false); f.checkFails("^(DATE '2020-10-10', DATE '2021-10-10') CONTAINS " + "TIMESTAMP '2010-01-01 10:00:00'^", "Cannot apply 'CONTAINS' to arguments of type " - + "' CONTAINS '\\. " + + "' CONTAINS '\\. " + containsError, false); f.checkFails("^(DATE '2020-10-10', TIMESTAMP '2021-10-10 00:00:00') " + "CONTAINS TIMESTAMP '2010-01-01 10:00:00'^", "Cannot apply 'CONTAINS' to arguments of type " - + "' " + + "' " + "CONTAINS '\\. " + containsError, false); f.checkFails("^(TIME '10:10:10', DATE '2021-10-10') CONTAINS TIME '10:00:00'^", "Cannot apply 'CONTAINS' to arguments of type " - + "' CONTAINS '\\. " + + "' CONTAINS '\\. " + containsError, false); f.checkFails("^(TIME '10:10:10', DATE '2021-10-10') CONTAINS TIMESTAMP '2010-02-02 10:00:00'^", "Cannot apply 'CONTAINS' to arguments of type " - + "' " + + "' " + "CONTAINS '\\. " + containsError, false); final String overlapsError = "Supported form\\(s\\): " @@ -3352,21 +3352,21 @@ static void checkOverlaps(OverlapChecker c) { f.checkFails("^(TIME '10:10:10', DATE '2021-10-10') OVERLAPS " + "(TIMESTAMP '2010-02-02 10:00:00', TIME '10:00:00')^", "Cannot apply 'OVERLAPS' to arguments of type " - + "' " - + "OVERLAPS '\\. " + + "' " + + "OVERLAPS '\\. " + overlapsError, false); f.checkFails("^(TIME '10:10:10', DATE '2021-10-10') " + "OVERLAPS (TIME '10:00:00', DATE '2020-01-01')^", "Cannot apply 'OVERLAPS' to arguments of type " - + "' " - + "OVERLAPS '\\. " + + "' " + + "OVERLAPS '\\. " + overlapsError, false); final String precedesError = overlapsError.replace("OVERLAPS", "PRECEDES"); f.checkFails("^(TIME '10:10:10', DATE '2021-10-10') " + "PRECEDES (TIME '10:00:00', TIME '10:10:10')^", "Cannot apply 'PRECEDES' to arguments of type " - + "' " - + "PRECEDES '\\. " + + "' " + + "PRECEDES '\\. " + precedesError, false); } @@ -9553,12 +9553,12 @@ void checkArrayReverseFunc(SqlOperatorFixture f0, SqlFunction function, f.checkFails("^map_from_entries(array[1])^", "Cannot apply 'MAP_FROM_ENTRIES' to arguments of type 'MAP_FROM_ENTRIES\\(" + "\\)'. Supported form\\(s\\): 'MAP_FROM_ENTRIES\\(" - + ">\\)'", + + ">\\)'", false); f.checkFails("^map_from_entries(array[row(1, 'a', 2)])^", "Cannot apply 'MAP_FROM_ENTRIES' to arguments of type 'MAP_FROM_ENTRIES\\(" - + "\\)'. " - + "Supported form\\(s\\): 'MAP_FROM_ENTRIES\\(>\\)'", + + "\\)'. " + + "Supported form\\(s\\): 'MAP_FROM_ENTRIES\\(>\\)'", false); } @@ -13741,8 +13741,8 @@ private static void checkArrayConcatAggFuncFails(SqlOperatorFixture t) { f.checkFails("ARRAY[2,4,6][OFFSET(5)]", "Array index 5 is out of bounds", true); f.checkFails("^map['foo', 3, 'bar', 7][offset('bar')]^", - "Cannot apply 'OFFSET' to arguments of type 'OFFSET\\(<\\(CHAR\\(3\\)" - + ", INTEGER\\) MAP>, \\)'\\. Supported form\\(s\\): " + "Cannot apply 'OFFSET' to arguments of type 'OFFSET\\(>, \\)'\\. Supported form\\(s\\): " + "\\[OFFSET\\(\\)\\]", false); } @@ -13761,8 +13761,8 @@ private static void checkArrayConcatAggFuncFails(SqlOperatorFixture t) { f.checkFails("ARRAY[2,4,6][ORDINAL(5)]", "Array index 5 is out of bounds", true); f.checkFails("^map['foo', 3, 'bar', 7][ordinal('bar')]^", - "Cannot apply 'ORDINAL' to arguments of type 'ORDINAL\\(<\\(CHAR\\(3\\)" - + ", INTEGER\\) MAP>, \\)'\\. Supported form\\(s\\): " + "Cannot apply 'ORDINAL' to arguments of type 'ORDINAL\\(>, \\)'\\. Supported form\\(s\\): " + "\\[ORDINAL\\(\\)\\]", false); } @@ -13779,8 +13779,8 @@ private static void checkArrayConcatAggFuncFails(SqlOperatorFixture t) { f.checkScalar("ARRAY[2,4,6][SAFE_OFFSET(5)]", isNullValue(), "INTEGER"); f.checkNull("ARRAY[2,4,6][SAFE_OFFSET(null)]"); f.checkFails("^map['foo', 3, 'bar', 7][safe_offset('bar')]^", - "Cannot apply 'SAFE_OFFSET' to arguments of type 'SAFE_OFFSET\\(<\\(CHAR\\(3\\)" - + ", INTEGER\\) MAP>, \\)'\\. Supported form\\(s\\): " + "Cannot apply 'SAFE_OFFSET' to arguments of type 'SAFE_OFFSET\\(>, \\)'\\. Supported form\\(s\\): " + "\\[SAFE_OFFSET\\(\\)\\]", false); } @@ -13797,8 +13797,8 @@ private static void checkArrayConcatAggFuncFails(SqlOperatorFixture t) { f.checkScalar("ARRAY[2,4,6][SAFE_ORDINAL(5)]", isNullValue(), "INTEGER"); f.checkNull("ARRAY[2,4,6][SAFE_ORDINAL(null)]"); f.checkFails("^map['foo', 3, 'bar', 7][safe_ordinal('bar')]^", - "Cannot apply 'SAFE_ORDINAL' to arguments of type 'SAFE_ORDINAL\\(<\\(CHAR\\(3\\)" - + ", INTEGER\\) MAP>, \\)'\\. Supported form\\(s\\): " + "Cannot apply 'SAFE_ORDINAL' to arguments of type 'SAFE_ORDINAL\\(>, \\)'\\. Supported form\\(s\\): " + "\\[SAFE_ORDINAL\\(\\)\\]", false); } From 477a1c1d978a0722d7162d4286d36c48da17d024 Mon Sep 17 00:00:00 2001 From: "ian.bertolacci" Date: Mon, 27 Apr 2026 10:29:44 -0700 Subject: [PATCH 311/562] [CALCITE-7405] Pre-process expressions for correlations before building projection in SqlToRelConverter --- .../calcite/sql2rel/SqlToRelConverter.java | 21 +++++----- .../calcite/test/SqlToRelConverterTest.java | 35 +++++++++++++++++ .../calcite/test/SqlToRelConverterTest.xml | 39 +++++++++++++++++++ 3 files changed, 86 insertions(+), 9 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java index 4622176a0733..9c328b152f7d 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java @@ -3774,9 +3774,10 @@ private void createAggImpl(Blackboard bb, final RelNode inputRel = bb.root(); // Project the expressions required by agg and having. - RelNode intermediateProject = relBuilder.push(inputRel) - .projectNamed(preExprs.leftList(), preExprs.rightList(), false) - .build(); + // Using LogicalProject.create to avoid bloat optimizations in RelBuilder. + RelNode intermediateProject = + LogicalProject.create(inputRel, ImmutableList.of(), preExprs.leftList(), + preExprs.rightList(), ImmutableSet.of()); final RelNode r2; // deal with correlation final CorrelationUse p = getCorrelationUse(bb, intermediateProject); @@ -3790,7 +3791,9 @@ private void createAggImpl(Blackboard bb, true, ImmutableSet.of(p.id)) .build(); } else { - r2 = intermediateProject; + r2 = relBuilder.push(inputRel) + .projectNamed(preExprs.leftList(), preExprs.rightList(), false) + .build(); } bb.setRoot(r2, false); bb.mapRootRelToFieldProjection.put(bb.root(), r.groupExprProjection); @@ -5025,10 +5028,10 @@ private void convertNonAggregateSelectList( SqlValidatorUtil.uniquify(fieldNames, catalogReader.nameMatcher().isCaseSensitive()); - relBuilder.push(bb.root()) - .projectNamed(exprs, uniqueFieldNames, true); - - RelNode project = relBuilder.build(); + // Using LogicalProject.create to avoid bloat optimizations in RelBuilder. + RelNode project = + LogicalProject.create(bb.root(), ImmutableList.of(), exprs, uniqueFieldNames, + ImmutableSet.of()); final RelNode r; final CorrelationUse p = getCorrelationUse(bb, project); @@ -5042,7 +5045,7 @@ private void convertNonAggregateSelectList( ImmutableSet.of(p.id)) .build(); } else { - r = project; + r = relBuilder.push(bb.root()).projectNamed(exprs, uniqueFieldNames, true).build(); } bb.setRoot(r, false); diff --git a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java index 4e8d17cd7a27..694572683e99 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java @@ -4290,6 +4290,41 @@ void checkCorrelatedMapSubQuery(boolean expand) { sql(sql).withExpand(false).withDecorrelate(false).ok(); } + /** Test case for + * [CALCITE-7405] + * Correlated subquery where outer tree is affected by bloat optimization + * causing project fusion. */ + @Test void testCorrelationInProjectionWithBloatFusion() { + final String sql = "select e1.sal + " + + "(select sum(e2.sal) from emp e2 where e2.deptno = d.deptno)\n" + + "from dept d join emp e1 on d.deptno + 1 = e1.deptno"; + sql(sql) + .withExpand(false) + .withDecorrelate(false) + .withConfig(srcc -> + srcc.addRelBuilderConfigTransform(rbc -> + rbc.withBloat(100).withPushJoinCondition(true))) + .ok(); + } + + /** Test case for + * [CALCITE-7405] + * Correlated subquery using a compound expression from input projection that is not ultimately + * projected by query, and where outer tree is affected by bloat optimization causing project + * fusion and elimination of compound expression required by correlation. */ + @Test void testCorrelationInProjectionWithBloatFusionAndCompoundNonProjectedCorrelation() { + final String sql = "select d2.name, " + + "(select sum(e.sal) from emp e where e.deptno = d2.compound)\n" + + "from (select d.name, d.deptno + 1 as compound from dept d) as d2"; + sql(sql) + .withExpand(false) + .withDecorrelate(false) + .withConfig(srcc -> + srcc.addRelBuilderConfigTransform(rbc -> + rbc.withBloat(100))) + .ok(); + } + @Test void testCustomColumnResolving() { final String sql = "select k0 from struct.t"; sql(sql).ok(); diff --git a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml index 5ee7180efa75..1705493edb8a 100644 --- a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml @@ -1378,6 +1378,45 @@ LogicalAggregate(group=[{}], EXPR$0=[SUM($0)]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) })]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +]]> + + + + + + + + + + + + + + + + From 587d5932246eb50b8367fda7676b2cc85e2e175e Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Sat, 6 Jun 2026 10:13:38 +0800 Subject: [PATCH 312/562] [CALCITE-6451] Improve Nullability Derivation for Intersect and Minus Co-authored-by: Victor Barua --- .../apache/calcite/rel/core/Intersect.java | 11 + .../org/apache/calcite/rel/core/Minus.java | 10 + .../org/apache/calcite/rel/core/SetOp.java | 4 + .../rel/rules/IntersectToDistinctRule.java | 7 + .../rel/rules/MinusToDistinctRule.java | 4 + .../apache/calcite/sql/SqlSetOperator.java | 4 +- .../calcite/sql/fun/SqlStdOperatorTable.java | 12 +- .../apache/calcite/sql/type/ReturnTypes.java | 105 ++++++ .../apache/calcite/test/RelBuilderTest.java | 350 ++++++++++++++++++ .../apache/calcite/test/RelOptRulesTest.xml | 17 +- core/src/test/resources/sql/planner.iq | 25 +- .../org/apache/calcite/test/Matchers.java | 13 + 12 files changed, 535 insertions(+), 27 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rel/core/Intersect.java b/core/src/main/java/org/apache/calcite/rel/core/Intersect.java index e6ea2f6c242c..8bed0bfd0d3d 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/Intersect.java +++ b/core/src/main/java/org/apache/calcite/rel/core/Intersect.java @@ -22,7 +22,10 @@ import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.hint.RelHint; import org.apache.calcite.rel.metadata.RelMetadataQuery; +import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.type.ReturnTypes; +import org.apache.calcite.util.Util; import java.util.Collections; import java.util.List; @@ -79,4 +82,12 @@ protected Intersect(RelInput input) { dRows *= 0.25; return dRows; } + + @Override protected RelDataType deriveRowType() { + // An output column is only nullable if it is nullable in ALL the inputs. + return ReturnTypes.deriveNullabilityForIntersect( + getCluster().getTypeFactory(), + deriveLeastRestrictiveRowType(), + Util.transform(getInputs(), RelNode::getRowType)); + } } diff --git a/core/src/main/java/org/apache/calcite/rel/core/Minus.java b/core/src/main/java/org/apache/calcite/rel/core/Minus.java index 3a68945acc64..badf20474a9d 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/Minus.java +++ b/core/src/main/java/org/apache/calcite/rel/core/Minus.java @@ -23,7 +23,9 @@ import org.apache.calcite.rel.hint.RelHint; import org.apache.calcite.rel.metadata.RelMdUtil; import org.apache.calcite.rel.metadata.RelMetadataQuery; +import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.type.ReturnTypes; import java.util.Collections; import java.util.List; @@ -60,4 +62,12 @@ protected Minus(RelInput input) { @Override public double estimateRowCount(RelMetadataQuery mq) { return RelMdUtil.getMinusRowCount(mq, this); } + + @Override protected RelDataType deriveRowType() { + // The nullability of the output columns is the same as that of the primary input. + return ReturnTypes.deriveNullabilityForExcept( + getCluster().getTypeFactory(), + deriveLeastRestrictiveRowType(), + getInput(0).getRowType()); + } } diff --git a/core/src/main/java/org/apache/calcite/rel/core/SetOp.java b/core/src/main/java/org/apache/calcite/rel/core/SetOp.java index 04e1409e78aa..9af6dd2ca91f 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/SetOp.java +++ b/core/src/main/java/org/apache/calcite/rel/core/SetOp.java @@ -114,6 +114,10 @@ public abstract SetOp copy( } @Override protected RelDataType deriveRowType() { + return deriveLeastRestrictiveRowType(); + } + + protected RelDataType deriveLeastRestrictiveRowType() { final List inputRowTypes = Util.transform(inputs, RelNode::getRowType); final RelDataType rowType = diff --git a/core/src/main/java/org/apache/calcite/rel/rules/IntersectToDistinctRule.java b/core/src/main/java/org/apache/calcite/rel/rules/IntersectToDistinctRule.java index a8d2713e0980..394f8e088551 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/IntersectToDistinctRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/IntersectToDistinctRule.java @@ -140,6 +140,10 @@ public void onMatchAggregateOnUnion(RelOptRuleCall call) { // Project all but the last added field (e.g. count_i{n}) relBuilder.project(skipLast(relBuilder.fields(), branchCount)); + + // ensure the nullabilities of columns in the new relation match those of the input relation + relBuilder.convert(intersect.getRowType(), false); + call.transformTo(relBuilder.build()); } @@ -206,6 +210,9 @@ public void onMatchAggregatePushdown(RelOptRuleCall call) { // Project all but the last field relBuilder.project(Util.skipLast(relBuilder.fields())); + // ensure the nullabilities of columns in the new relation match those of the input relation + relBuilder.convert(intersect.getRowType(), false); + // the schema for intersect distinct matches that of the relation, // built here with an extra last column for the count, // which is projected out by the final project we added diff --git a/core/src/main/java/org/apache/calcite/rel/rules/MinusToDistinctRule.java b/core/src/main/java/org/apache/calcite/rel/rules/MinusToDistinctRule.java index 7b14f2e94613..59eaf0b60c25 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/MinusToDistinctRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/MinusToDistinctRule.java @@ -159,6 +159,10 @@ public MinusToDistinctRule(Class minusClass, relBuilder.filter(filters.build()); relBuilder.project(Util.first(relBuilder.fields(), originalFieldCnt)); + + // ensure the nullabilities of columns in the new relation match those of the minus output + relBuilder.convert(minus.getRowType(), false); + call.transformTo(relBuilder.build()); } diff --git a/core/src/main/java/org/apache/calcite/sql/SqlSetOperator.java b/core/src/main/java/org/apache/calcite/sql/SqlSetOperator.java index 6d0f48c4f417..0ce69276f71d 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlSetOperator.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlSetOperator.java @@ -24,6 +24,8 @@ import org.apache.calcite.sql.validate.SqlValidator; import org.apache.calcite.sql.validate.SqlValidatorScope; +import org.checkerframework.checker.nullness.qual.Nullable; + /** * SqlSetOperator represents a relational set theory operator (UNION, INTERSECT, * MINUS). These are binary operators, but with an extra boolean attribute @@ -59,7 +61,7 @@ public SqlSetOperator( int prec, boolean all, SqlReturnTypeInference returnTypeInference, - SqlOperandTypeInference operandTypeInference, + @Nullable SqlOperandTypeInference operandTypeInference, SqlOperandTypeChecker operandTypeChecker) { super( name, diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java index 494fafeccc64..b1f0c0d5044d 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java @@ -121,16 +121,20 @@ public class SqlStdOperatorTable extends ReflectiveSqlOperatorTable { new SqlSetOperator("UNION ALL", SqlKind.UNION, 12, true); public static final SqlSetOperator EXCEPT = - new SqlSetOperator("EXCEPT", SqlKind.EXCEPT, 12, false); + new SqlSetOperator("EXCEPT", SqlKind.EXCEPT, 12, false, + ReturnTypes.LEAST_RESTRICTIVE_EXCEPT, null, OperandTypes.SET_OP); public static final SqlSetOperator EXCEPT_ALL = - new SqlSetOperator("EXCEPT ALL", SqlKind.EXCEPT, 12, true); + new SqlSetOperator("EXCEPT ALL", SqlKind.EXCEPT, 12, true, + ReturnTypes.LEAST_RESTRICTIVE_EXCEPT, null, OperandTypes.SET_OP); public static final SqlSetOperator INTERSECT = - new SqlSetOperator("INTERSECT", SqlKind.INTERSECT, 14, false); + new SqlSetOperator("INTERSECT", SqlKind.INTERSECT, 14, false, + ReturnTypes.LEAST_RESTRICTIVE_INTERSECT, null, OperandTypes.SET_OP); public static final SqlSetOperator INTERSECT_ALL = - new SqlSetOperator("INTERSECT ALL", SqlKind.INTERSECT, 14, true); + new SqlSetOperator("INTERSECT ALL", SqlKind.INTERSECT, 14, true, + ReturnTypes.LEAST_RESTRICTIVE_INTERSECT, null, OperandTypes.SET_OP); /** * The {@code MULTISET UNION DISTINCT} operator. diff --git a/core/src/main/java/org/apache/calcite/sql/type/ReturnTypes.java b/core/src/main/java/org/apache/calcite/sql/type/ReturnTypes.java index 27f1a6fc72a7..9e32e0563d96 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/ReturnTypes.java +++ b/core/src/main/java/org/apache/calcite/sql/type/ReturnTypes.java @@ -34,9 +34,12 @@ import org.apache.calcite.util.Glossary; import org.apache.calcite.util.Util; +import com.google.common.collect.ImmutableList; + import org.checkerframework.checker.nullness.qual.Nullable; import java.util.AbstractList; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.function.UnaryOperator; @@ -608,6 +611,108 @@ public static SqlCall stripSeparator(SqlCall call) { .leastRestrictive(opBinding.collectOperandTypes()); } + /** + * Refines the nullability of {@code base} using INTERSECT semantics: a + * column is NOT NULL if it is NOT NULL in all of the + * {@code inputTypes} ("AND" semantics across inputs). + * + *

      For ROW (struct) types, this recursively refines nullability of nested + * fields, ensuring the "AND" semantics apply at every level of nesting. + */ + public static RelDataType deriveNullabilityForIntersect( + RelDataTypeFactory typeFactory, + RelDataType base, + List inputTypes) { + return deriveNullable(typeFactory, base, inputTypes); + } + + /** + * Refines the nullability of {@code base} using EXCEPT/MINUS semantics: + * each column's nullability matches that of the primary (first) input. + * For ROW types, this is applied recursively at all nesting levels. + */ + public static RelDataType deriveNullabilityForExcept( + RelDataTypeFactory typeFactory, + RelDataType base, + RelDataType primaryInputType) { + return deriveNullable(typeFactory, base, + ImmutableList.of(primaryInputType)); + } + + /** + * Returns whether all types in the list are nullable (AND semantics). + */ + private static boolean allNullable(List types) { + for (RelDataType type : types) { + if (!type.isNullable()) { + return false; + } + } + return true; + } + + /** + * Recursively refines nullability using AND semantics: + * a column is NOT NULL if it is NOT NULL in ALL {@code inputTypes}. + * For ROW types, this is applied recursively to all nested fields. + */ + private static RelDataType deriveNullable( + RelDataTypeFactory typeFactory, + RelDataType type, + List inputTypes) { + if (type.isStruct()) { + final RelDataTypeFactory.Builder builder = + new RelDataTypeFactory.Builder(typeFactory); + final List fields = type.getFieldList(); + for (int i = 0; i < fields.size(); i++) { + final List fieldInputTypes = new ArrayList<>(); + for (RelDataType inputType : inputTypes) { + fieldInputTypes.add(inputType.getFieldList().get(i).getType()); + } + builder.add(fields.get(i).getName(), + deriveNullable(typeFactory, + fields.get(i).getType(), + fieldInputTypes)) + .nullable(allNullable(fieldInputTypes)); + } + return builder.build(); + } + return typeFactory.createTypeWithNullability(type, allNullable(inputTypes)); + } + + /** + * Type-inference strategy for INTERSECT. Computes the least restrictive row + * type across all inputs, then refines nullability: a column is NOT NULL if + * it is NOT NULL in at least one input ("AND" semantics across inputs). + */ + public static final SqlReturnTypeInference LEAST_RESTRICTIVE_INTERSECT = + andThen(SqlTypeTransforms.FROM_MEASURE_IF::apply, opBinding -> { + final List inputTypes = opBinding.collectOperandTypes(); + final RelDataType base = + opBinding.getTypeFactory().leastRestrictive(inputTypes); + if (base == null) { + return null; + } + return deriveNullabilityForIntersect(opBinding.getTypeFactory(), base, inputTypes); + }); + + /** + * Type-inference strategy for EXCEPT/MINUS. Computes the least restrictive + * row type across all inputs, then refines nullability: a column's + * nullability matches that of the first (primary) input. + */ + public static final SqlReturnTypeInference LEAST_RESTRICTIVE_EXCEPT = + andThen(SqlTypeTransforms.FROM_MEASURE_IF::apply, opBinding -> { + final List inputTypes = opBinding.collectOperandTypes(); + final RelDataType base = + opBinding.getTypeFactory().leastRestrictive(inputTypes); + if (base == null) { + return null; + } + return deriveNullabilityForExcept( + opBinding.getTypeFactory(), base, inputTypes.get(0)); + }); + /** * Type-inference strategy for NVL2 function. It returns the least restrictive type * between the second and third operands. diff --git a/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java b/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java index f1a4f1c27162..e53f1027f78d 100644 --- a/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java @@ -131,6 +131,7 @@ import static org.apache.calcite.test.Matchers.hasExpandedTree; import static org.apache.calcite.test.Matchers.hasFieldNames; import static org.apache.calcite.test.Matchers.hasHints; +import static org.apache.calcite.test.Matchers.hasRelDataType; import static org.apache.calcite.test.Matchers.hasTree; import static org.hamcrest.CoreMatchers.allOf; @@ -2346,6 +2347,72 @@ private static RelNode groupIdRel(RelBuilder builder, boolean extra) { assertThat(root, hasTree(expected)); } + /** Test case for + * [CALCITE-6451] + * Improve Nullability Derivation for Intersect and Minus. */ + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testUnionTypeDerivation(boolean all) { + final RelBuilder builder = RelBuilder.create(config().build()); + + RelDataType input1RowType = + new RelDataTypeFactory.Builder(builder.getTypeFactory()) + .add("a", SqlTypeName.BIGINT) + .nullable(false) + .add("b", SqlTypeName.BIGINT) + .nullable(false) + .add("c", SqlTypeName.BIGINT) + .nullable(true) + .add("d", SqlTypeName.BIGINT) + .nullable(true) + .build(); + + RelDataType input2RowType = + new RelDataTypeFactory.Builder(builder.getTypeFactory()) + .add("a", SqlTypeName.BIGINT) + .nullable(false) + .add("b", SqlTypeName.BIGINT) + .nullable(false) + .add("c", SqlTypeName.BIGINT) + .nullable(false) + .add("d", SqlTypeName.BIGINT) + .nullable(false) + .build(); + + RelDataType input3RowType = + new RelDataTypeFactory.Builder(builder.getTypeFactory()) + .add("a", SqlTypeName.BIGINT) + .nullable(false) + .add("b", SqlTypeName.BIGINT) + .nullable(true) + .add("c", SqlTypeName.BIGINT) + .nullable(false) + .add("d", SqlTypeName.BIGINT) + .nullable(true) + .build(); + + RelNode root = + builder + .values(input1RowType) + .values(input2RowType) + .values(input3RowType) + .union(all, 3) + .build(); + + RelDataType expectedRowType = + new RelDataTypeFactory.Builder(builder.getTypeFactory()) + .add("a", SqlTypeName.BIGINT) + .nullable(false) + .add("b", SqlTypeName.BIGINT) + .nullable(true) + .add("c", SqlTypeName.BIGINT) + .nullable(true) + .add("d", SqlTypeName.BIGINT) + .nullable(true) + .build(); + assertThat(root.getRowType(), hasRelDataType(expectedRowType)); + } + /** Test case for * [CALCITE-1522] * Fix error message for SetOp with incompatible args. */ @@ -2550,6 +2617,72 @@ private static RelNode groupIdRel(RelBuilder builder, boolean extra) { assertThat(root, hasTree(expected)); } + /** Test case for + * [CALCITE-6451] + * Improve Nullability Derivation for Intersect and Minus. */ + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testIntersectTypeDerivation(boolean all) { + final RelBuilder builder = RelBuilder.create(config().build()); + + RelDataType input1RowType = + new RelDataTypeFactory.Builder(builder.getTypeFactory()) + .add("a", SqlTypeName.BIGINT) + .nullable(false) + .add("b", SqlTypeName.BIGINT) + .nullable(false) + .add("c", SqlTypeName.BIGINT) + .nullable(true) + .add("d", SqlTypeName.BIGINT) + .nullable(true) + .build(); + + RelDataType input2RowType = + new RelDataTypeFactory.Builder(builder.getTypeFactory()) + .add("a", SqlTypeName.BIGINT) + .nullable(true) + .add("b", SqlTypeName.BIGINT) + .nullable(true) + .add("c", SqlTypeName.BIGINT) + .nullable(true) + .add("d", SqlTypeName.BIGINT) + .nullable(true) + .build(); + + RelDataType input3RowType = + new RelDataTypeFactory.Builder(builder.getTypeFactory()) + .add("a", SqlTypeName.BIGINT) + .nullable(false) + .add("b", SqlTypeName.BIGINT) + .nullable(true) + .add("c", SqlTypeName.BIGINT) + .nullable(false) + .add("d", SqlTypeName.BIGINT) + .nullable(true) + .build(); + + RelNode root = + builder + .values(input1RowType) + .values(input2RowType) + .values(input3RowType) + .intersect(all, 3) + .build(); + + RelDataType expectedRowType = + new RelDataTypeFactory.Builder(builder.getTypeFactory()) + .add("a", SqlTypeName.BIGINT) + .nullable(false) + .add("b", SqlTypeName.BIGINT) + .nullable(false) + .add("c", SqlTypeName.BIGINT) + .nullable(false) + .add("d", SqlTypeName.BIGINT) + .nullable(true) + .build(); + assertThat(root.getRowType(), hasRelDataType(expectedRowType)); + } + @Test void testExcept() { // Equivalent SQL: // SELECT empno FROM emp @@ -2577,6 +2710,223 @@ private static RelNode groupIdRel(RelBuilder builder, boolean extra) { assertThat(root, hasTree(expected)); } + /** Test case for + * [CALCITE-6451] + * Improve Nullability Derivation for Intersect and Minus. */ + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testExceptTypeDerivation(boolean all) { + final RelBuilder builder = RelBuilder.create(config().build()); + + RelDataType primaryRowType = + new RelDataTypeFactory.Builder(builder.getTypeFactory()) + .add("a", SqlTypeName.BIGINT) + .nullable(false) + .add("b", SqlTypeName.BIGINT) + .nullable(false) + .add("c", SqlTypeName.BIGINT) + .nullable(true) + .add("d", SqlTypeName.BIGINT) + .nullable(true) + .build(); + + RelDataType secondaryRowType = + new RelDataTypeFactory.Builder(builder.getTypeFactory()) + .add("a", SqlTypeName.BIGINT) + .nullable(false) + .add("b", SqlTypeName.BIGINT) + .nullable(true) + .add("c", SqlTypeName.BIGINT) + .nullable(false) + .add("d", SqlTypeName.BIGINT) + .nullable(true) + .build(); + + RelNode root = + builder.values(primaryRowType) + .values(secondaryRowType) + .minus(all) + .build(); + + assertThat(root.getRowType(), hasRelDataType(primaryRowType)); + } + + /** Test case for + * [CALCITE-6451] + * Improve Nullability Derivation for Intersect and Minus. */ + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testExceptTypeDerivationWithRowField(boolean all) { + final RelBuilder builder = RelBuilder.create(config().build()); + + // a BIGINT NOT NULL, b ROW(x INT NOT NULL, y VARCHAR NULL) NOT NULL + RelDataType primaryRowType = + new RelDataTypeFactory.Builder(builder.getTypeFactory()) + .add("a", SqlTypeName.BIGINT) + .nullable(false) + .add("b", builder.getTypeFactory().builder() + .add("x", SqlTypeName.INTEGER) + .nullable(false) + .add("y", SqlTypeName.VARCHAR, 10) + .nullable(true) + .build()) + .nullableRecord(false) + .build(); + + // a BIGINT NULL, b ROW(x INT NULL, y VARCHAR NOT NULL) NOT NULL + RelDataType secondaryRowType = + new RelDataTypeFactory.Builder(builder.getTypeFactory()) + .add("a", SqlTypeName.BIGINT) + .nullable(true) + .add("b", builder.getTypeFactory().builder() + .add("x", SqlTypeName.INTEGER) + .nullable(true) + .add("y", SqlTypeName.VARCHAR, 10) + .nullable(false) + .build()) + .nullableRecord(false) + .build(); + + RelNode root = + builder.values(primaryRowType) + .values(secondaryRowType) + .minus(all) + .build(); + + // follows primary input's nullability + assertThat(root.getRowType(), hasRelDataType(primaryRowType)); + } + + /** Test case for + * [CALCITE-6451] + * Improve Nullability Derivation for Intersect and Minus. */ + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testIntersectTypeDerivationWithRowField(boolean all) { + final RelBuilder builder = RelBuilder.create(config().build()); + + // a BIGINT NOT NULL, b ROW(x INT NOT NULL, y VARCHAR NULL) NOT NULL + RelDataType input1RowType = + new RelDataTypeFactory.Builder(builder.getTypeFactory()) + .add("a", SqlTypeName.BIGINT) + .nullable(false) + .add("b", builder.getTypeFactory().builder() + .add("x", SqlTypeName.INTEGER) + .nullable(false) + .add("y", SqlTypeName.VARCHAR, 10) + .nullable(true) + .build()) + .nullable(false) + .build(); + + // a BIGINT NULL, b ROW(x INT NULL, y VARCHAR NOT NULL) NOT NULL + RelDataType input2RowType = + new RelDataTypeFactory.Builder(builder.getTypeFactory()) + .add("a", SqlTypeName.BIGINT) + .nullable(true) + .add("b", builder.getTypeFactory().builder() + .add("x", SqlTypeName.INTEGER) + .nullable(true) + .add("y", SqlTypeName.VARCHAR, 10) + .nullable(false) + .build()) + .nullable(false) + .build(); + + RelNode root = + builder + .values(input1RowType) + .values(input2RowType) + .intersect(all) + .build(); + + // a BIGINT NOT NULL + // a BIGINT NULL + // => a BIGINT NOT NULL + // + // b ROW(x INT NOT NULL, y VARCHAR NULL) NOT NULL + // b ROW(x INT NULL, y VARCHAR NOT NULL) NOT NULL + // => b ROW(x INT NOT NULL, y VARCHAR NOT NULL) NOT NULL + RelDataType expectedRowType = + new RelDataTypeFactory.Builder(builder.getTypeFactory()) + .add("a", SqlTypeName.BIGINT) + .nullable(false) + .add("b", builder.getTypeFactory().builder() + .add("x", SqlTypeName.INTEGER) + .nullable(false) + .add("y", SqlTypeName.VARCHAR, 10) + .nullable(false) + .build()) + .nullable(false) + .build(); + assertThat(root.getRowType(), hasRelDataType(expectedRowType)); + } + + /** Test case for + * [CALCITE-6451] + * Improve Nullability Derivation for Intersect and Minus. */ + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testUnionTypeDerivationWithRowField(boolean all) { + final RelBuilder builder = RelBuilder.create(config().build()); + + // a BIGINT NOT NULL, b ROW(x INT NOT NULL, y VARCHAR NULL) NOT NULL + RelDataType input1RowType = + new RelDataTypeFactory.Builder(builder.getTypeFactory()) + .add("a", SqlTypeName.BIGINT) + .nullable(false) + .add("b", builder.getTypeFactory().builder() + .add("x", SqlTypeName.INTEGER) + .nullable(true) + .add("y", SqlTypeName.VARCHAR, 10) + .nullable(true) + .build()) + .nullable(true) + .build(); + + // a BIGINT NULL, b ROW(x INT NULL, y VARCHAR NOT NULL) NOT NULL + RelDataType input2RowType = + new RelDataTypeFactory.Builder(builder.getTypeFactory()) + .add("a", SqlTypeName.BIGINT) + .nullable(true) + .add("b", builder.getTypeFactory().builder() + .add("x", SqlTypeName.INTEGER) + .nullable(true) + .add("y", SqlTypeName.VARCHAR, 10) + .nullable(false) + .build()) + .nullable(false) + .build(); + + RelNode root = + builder + .values(input1RowType) + .values(input2RowType) + .union(all) + .build(); + + // a BIGINT NOT NULL + // a BIGINT NULL + // => a BIGINT NULL + // + // b ROW(x INT NULL, y VARCHAR NULL) NULL + // b ROW(x INT NULL, y VARCHAR NOT NULL) NOT NULL + // => b ROW(x INT NULL, y VARCHAR NULL) NULL + RelDataType expectedRowType = + new RelDataTypeFactory.Builder(builder.getTypeFactory()) + .add("a", SqlTypeName.BIGINT) + .nullable(true) + .add("b", builder.getTypeFactory().builder() + .add("x", SqlTypeName.INTEGER) + .nullable(true) + .add("y", SqlTypeName.VARCHAR, 10) + .nullable(true) + .build()) + .nullable(true) + .build(); + assertThat(root.getRowType(), hasRelDataType(expectedRowType)); + } + /** Tests building a simple join. Also checks {@link RelBuilder#size()} * at every step. */ @Test void testJoin() { diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index c782bb913cc1..c2fb1065553e 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -7319,15 +7319,14 @@ LogicalIntersect(all=[false]) LogicalAggregate(group=[{0}]) LogicalJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[semi]) LogicalJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[semi]) - LogicalProject(ENAME=[CAST($0):VARCHAR]) + LogicalProject(ENAME=[CAST($0):VARCHAR NOT NULL]) LogicalProject(ENAME=[$1]) LogicalFilter(condition=[=($7, 10)]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) - LogicalProject(ENAME=[CAST($0):VARCHAR]) - LogicalProject(DEPTNO=[CAST($7):VARCHAR NOT NULL]) - LogicalFilter(condition=[OR(=($1, 'a'), =($1, 'b'))]) - LogicalTableScan(table=[[CATALOG, SALES, EMP]]) - LogicalProject(ENAME=[CAST($0):VARCHAR]) + LogicalProject(DEPTNO=[CAST($7):VARCHAR NOT NULL]) + LogicalFilter(condition=[OR(=($1, 'a'), =($1, 'b'))]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) + LogicalProject(ENAME=[CAST($0):VARCHAR NOT NULL]) LogicalProject(ENAME=[$1]) LogicalTableScan(table=[[CATALOG, SALES, EMPNULLABLES]]) ]]> @@ -10453,10 +10452,10 @@ LogicalMinus(all=[false]) hasFieldNames(String fieldNames) { } }; } + + /** + * Creates a Matcher that matches a {@link RelDataType} if its + * {@link RelDataType#getFullTypeString()} is equal to that of the given {@code relDataType}. + */ + public static Matcher hasRelDataType(RelDataType relDataType) { + return compose( + IsEqual.equalTo(relDataType.getFullTypeString()), + RelDataType::getFullTypeString); + } + /** * Creates a Matcher that matches a {@link RelNode} if its string * representation, after converting Windows-style line endings ("\r\n") From 58ef67f2fbb419486c0766864d43a662ea1fb185 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Mon, 8 Jun 2026 17:33:47 -0700 Subject: [PATCH 313/562] [CALCITE-4353] Validator fails to expand order expression with dot operator Signed-off-by: Mihai Budiu --- .../calcite/sql/validate/SqlValidatorImpl.java | 6 +++++- .../apache/calcite/test/SqlValidatorTest.java | 3 +++ core/src/test/resources/sql/sort.iq | 16 ++++++++++++++++ 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java index 370ef5ecc39a..f7388bdeb346 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java @@ -7894,7 +7894,11 @@ private SqlNode nthSelectItem(int ordinal, final SqlParserPos pos) { if (call instanceof SqlSelect) { return call; } - return super.visitScoped(call); + // Only visit expression arguments. For DOT calls (e.g. 'employees[1].detail'), + // operand[1] is a field name, not a column reference, and must not be qualified. + CallCopyingArgHandler argHandler = new CallCopyingArgHandler(call, false); + call.getOperator().acceptCall(this, call, true, argHandler); + return argHandler.result(); } } diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index 8e4e76378b12..c60c2917edb0 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -9841,6 +9841,9 @@ void testGroupExpressionEquivalenceParams() { sql("SELECT dept_nested.employees[1].detail.skills[1].others.a as oa\n" + "from dept_nested") .type("RecordType(VARCHAR(10) OA) NOT NULL"); + // Test case for [CALCITE-4353] Validator fails to expand order expression with dot operator + sql("select * from dept_nested order by employees[1].detail.skills[2+3].desc") + .ok(); } @Test void testItemOperatorException() { diff --git a/core/src/test/resources/sql/sort.iq b/core/src/test/resources/sql/sort.iq index b93be76837bf..b7df52bff2c6 100644 --- a/core/src/test/resources/sql/sort.iq +++ b/core/src/test/resources/sql/sort.iq @@ -533,4 +533,20 @@ SELECT * FROM emp ORDER BY deptno, null, empno; !ok +# [CALCITE-4353] Validator fails to expand order expression with dot operator +!use bookstore +select au."name" as author, au."books"[1]."title" title +from "bookstore"."authors" au +order by au."books"[1]."title"; ++-------------------+-----------------+ +| AUTHOR | TITLE | ++-------------------+-----------------+ +| Victor Hugo | Les Misérables | +| Nikos Kazantzakis | Zorba the Greek | +| Homer | | ++-------------------+-----------------+ +(3 rows) + +!ok + # End sort.iq From 352e154706ef562026a685535bd83643309d76de Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Wed, 10 Jun 2026 11:38:27 -0700 Subject: [PATCH 314/562] [CALCITE-7598] Query with HAVING empno BETWEEN NULL AND NULL crashes the compiler Signed-off-by: Mihai Budiu --- .../org/apache/calcite/sql2rel/SqlToRelConverter.java | 3 ++- .../java/org/apache/calcite/test/RelMetadataTest.java | 10 ++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java index 9c328b152f7d..4d14e16937ef 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java @@ -3819,7 +3819,8 @@ private void createAggImpl(Blackboard bb, if (having != null) { SqlNode newHaving = pushDownNotForIn(bb.scope, having); replaceSubQueries(bb, newHaving, RelOptUtil.Logic.UNKNOWN_AS_FALSE); - havingExpr = bb.convertExpression(newHaving); + RexNode having0 = bb.convertExpression(newHaving); + havingExpr = simplifyPredicate(having0); } else { havingExpr = relBuilder.literal(true); } diff --git a/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java b/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java index f438a0ad9423..9c3c30e3448e 100644 --- a/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java @@ -4531,6 +4531,16 @@ private void assertExpressionLineage( SqlValidatorTester.DEFAULT.convertSqlToRel(factory, sql, false, false); } + /** Test case for [CALCITE-7598] + * Query with HAVING empno BETWEEN NULL AND NULL crashes the compiler. */ + @Test void testHavingCrash() { + String sql = "SELECT DISTINCT empno FROM emp GROUP BY empno HAVING empno BETWEEN NULL AND NULL"; + SqlTestFactory factory = SqlTestFactory.INSTANCE + .withSqlToRelConfig( + c -> c.withRelBuilderConfigTransform(t -> t.withSimplify(false))); + SqlValidatorTester.DEFAULT.convertSqlToRel(factory, sql, false, false); + } + @Test void testAllPredicates() { final Project rel = (Project) sql("select * from emp, dept").toRel(); final Join join = (Join) rel.getInput(); From 2f997975c960444f371daacdae1c5f54d6433e3d Mon Sep 17 00:00:00 2001 From: xuzifu666 <1206332514@qq.com> Date: Sun, 24 May 2026 17:59:18 +0800 Subject: [PATCH 315/562] [CALCITE-7493] Support constant-result aggregates (e.g., STDDEV_POP, STDDEV) over GROUP BY keys --- ...gregateReduceFunctionsOnGroupKeysRule.java | 69 ++++- .../sql/SqlConstantValueAggFunction.java | 55 ++++ .../calcite/sql/fun/SqlAvgAggFunction.java | 27 +- ...ateReduceFunctionsOnGroupKeysRuleTest.java | 114 +++++++ ...gateReduceFunctionsOnGroupKeysRuleTest.xml | 283 ++++++++++++++++++ core/src/test/resources/sql/agg-reduce.iq | 174 +++++++++++ 6 files changed, 716 insertions(+), 6 deletions(-) create mode 100644 core/src/main/java/org/apache/calcite/sql/SqlConstantValueAggFunction.java create mode 100644 core/src/test/resources/sql/agg-reduce.iq diff --git a/core/src/main/java/org/apache/calcite/rel/rules/AggregateReduceFunctionsOnGroupKeysRule.java b/core/src/main/java/org/apache/calcite/rel/rules/AggregateReduceFunctionsOnGroupKeysRule.java index 987a25fea2e9..c31e731c0773 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/AggregateReduceFunctionsOnGroupKeysRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/AggregateReduceFunctionsOnGroupKeysRule.java @@ -31,7 +31,9 @@ import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexShuttle; import org.apache.calcite.rex.RexUtil; +import org.apache.calcite.sql.SqlConstantValueAggFunction; import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.tools.RelBuilder; import org.checkerframework.checker.nullness.qual.Nullable; @@ -52,12 +54,19 @@ * arguments exist in the aggregate's group set or are deterministic * expressions involving only group set columns and constants: *

        - *
      • {@code MAX}
        • - *
      • {@code MIN}
        • - *
      • {@code AVG}
        • - *
      • {@code ANY_VALUE}
        • + *
      • {@code MAX} - the GROUP BY key value itself
        • + *
      • {@code MIN} - the GROUP BY key value itself
        • + *
      • {@code AVG} - the GROUP BY key value itself
        • + *
      • {@code ANY_VALUE} - the GROUP BY key value itself
        • + *
      • Functions implementing {@link SqlConstantValueAggFunction} such as + * {@code STDDEV_POP}, {@code STDDEV_SAMP}, {@code VAR_POP}, {@code VAR_SAMP} + * - return their constant result
        • *
      * + *

      Aggregate functions that implement {@link SqlConstantValueAggFunction} + * declare what value to return when applied to constant (GROUP BY key) arguments. + * This enables the rule to optimize them without hard-coded type checks. + * *

      Note: This optimization preserves NULL semantics correctly. For aggregate * functions like MAX, MIN, and ANY_VALUE, NULL values in the source columns or * expressions are handled the same way before and after the transformation: @@ -144,6 +153,8 @@ protected AggregateReduceFunctionsOnGroupKeysRule(Config config) { return null; } final SqlKind kind = call.getAggregation().getKind(); + final boolean isConstantValueAgg = + call.getAggregation() instanceof SqlConstantValueAggFunction; switch (kind) { case AVG: case MAX: @@ -151,7 +162,9 @@ protected AggregateReduceFunctionsOnGroupKeysRule(Config config) { case ANY_VALUE: break; default: - return null; + if (!isConstantValueAgg) { + return null; + } } final List argList = call.getArgList(); if (argList.size() != 1) { @@ -163,6 +176,29 @@ protected AggregateReduceFunctionsOnGroupKeysRule(Config config) { if (aggregate.getGroupSet().get(arg)) { final int groupIndex = aggregate.getGroupSet().asList().indexOf(arg); RexNode ref = RexInputRef.of(groupIndex, aggregate.getRowType().getFieldList()); + + // For functions that return a constant value when applied to constant (GROUP BY key) + // arguments, delegate to the function's own implementation + if (isConstantValueAgg) { + final @Nullable RexNode constantResult = + ((SqlConstantValueAggFunction) call.getAggregation()) + .getConstantResult(rexBuilder, call.getType()); + if (constantResult != null) { + // Handle NULL semantics: if the GROUP BY key is nullable and the constant value + // is non-null (e.g., 0 for STDDEV functions), wrap in CASE to return NULL when + // the key is NULL, since aggregate functions skip NULL inputs + if (ref.getType().isNullable()) { + return rexBuilder.makeCall(SqlStdOperatorTable.CASE, + rexBuilder.makeCall(SqlStdOperatorTable.IS_NULL, ref), + rexBuilder.makeNullLiteral(call.getType()), + constantResult); + } + return constantResult; + } + } + + // For other aggregate functions (MAX, MIN, AVG, ANY_VALUE), + // the value of a constant is the constant itself if (!ref.getType().equals(call.getType())) { ref = rexBuilder.makeCast(call.getParserPosition(), call.getType(), ref); } @@ -192,6 +228,29 @@ protected AggregateReduceFunctionsOnGroupKeysRule(Config config) { if (translated == null) { return null; } + + // For functions that return a constant value when applied to constant expressions, + // delegate to the function's own implementation + if (isConstantValueAgg) { + final @Nullable RexNode constantResult = + ((SqlConstantValueAggFunction) call.getAggregation()) + .getConstantResult(rexBuilder, call.getType()); + if (constantResult != null) { + // Handle NULL semantics: if the expression is nullable and the constant value + // is non-null (e.g., 0 for STDDEV functions), wrap in CASE to return NULL when + // the expression evaluates to NULL, since aggregate functions skip NULL inputs + if (translated.getType().isNullable()) { + return rexBuilder.makeCall(SqlStdOperatorTable.CASE, + rexBuilder.makeCall(SqlStdOperatorTable.IS_NULL, translated), + rexBuilder.makeNullLiteral(call.getType()), + constantResult); + } + return constantResult; + } + } + + // For other aggregate functions (MAX, MIN, AVG, ANY_VALUE), + // return the translated expression if (!translated.getType().equals(call.getType())) { return rexBuilder.makeCast(call.getParserPosition(), call.getType(), translated); } diff --git a/core/src/main/java/org/apache/calcite/sql/SqlConstantValueAggFunction.java b/core/src/main/java/org/apache/calcite/sql/SqlConstantValueAggFunction.java new file mode 100644 index 000000000000..6091e7763695 --- /dev/null +++ b/core/src/main/java/org/apache/calcite/sql/SqlConstantValueAggFunction.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.sql; + +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexNode; + +import org.checkerframework.checker.nullness.qual.Nullable; + +/** + * Aggregate function that returns a constant value when applied to constant + * (GROUP BY key) arguments. + * + *

      For example, statistical functions like STDDEV_POP, STDDEV_SAMP, VAR_POP, + * VAR_SAMP always return 0 when applied to a constant value, since there is + * no variation in a set of identical values. + * + *

      This interface allows optimization rules to identify and reduce such + * aggregate functions without hard-coded checks for specific function types. + */ +public interface SqlConstantValueAggFunction { + /** + * Generates the constant result expression when this aggregate function is + * applied to arguments that are constant within each group (i.e., GROUP BY keys + * or expressions derived only from GROUP BY keys). + * + *

      For example: + *

        + *
      • {@code STDDEV_POP(constant)} returns {@code 0} + *
      • {@code VAR_SAMP(constant)} returns {@code 0} + *
      • {@code STDDEV_SAMP(constant)} returns {@code 0} + *
      + * + * @param rexBuilder Rex builder for creating the result expression + * @param returnType The return type of the aggregate function + * @return An expression representing the constant result, or null if this function + * does not return a constant value for constant arguments + */ + @Nullable RexNode getConstantResult(RexBuilder rexBuilder, RelDataType returnType); +} diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlAvgAggFunction.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlAvgAggFunction.java index 97358c8dc496..93ccca514150 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlAvgAggFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlAvgAggFunction.java @@ -17,13 +17,18 @@ package org.apache.calcite.sql.fun; import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexNode; import org.apache.calcite.sql.SqlAggFunction; +import org.apache.calcite.sql.SqlConstantValueAggFunction; import org.apache.calcite.sql.SqlFunctionCategory; import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.type.OperandTypes; import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.util.Optionality; +import org.checkerframework.checker.nullness.qual.Nullable; + import static com.google.common.base.Preconditions.checkArgument; /** @@ -31,8 +36,13 @@ * which go into it. It has precisely one argument of numeric type * (int, long, float, * double), and the result is the same type. + * + *

      For statistical functions (STDDEV_POP, STDDEV_SAMP, VAR_POP, VAR_SAMP), + * this function implements {@link SqlConstantValueAggFunction} to support + * optimization when applied to constant GROUP BY keys. */ -public class SqlAvgAggFunction extends SqlAggFunction { +public class SqlAvgAggFunction extends SqlAggFunction + implements SqlConstantValueAggFunction { //~ Constructors ----------------------------------------------------------- @@ -86,4 +96,19 @@ public enum Subtype { VAR_POP, VAR_SAMP } + + @Override public @Nullable RexNode getConstantResult(RexBuilder rexBuilder, + RelDataType returnType) { + // Only statistical functions (variance and standard deviation) return 0 for constant values. + // AVG and other functions should not be optimized through this interface. + switch (kind) { + case STDDEV_POP: + case STDDEV_SAMP: + case VAR_POP: + case VAR_SAMP: + return rexBuilder.makeLiteral(0, returnType, true); + default: + return null; + } + } } diff --git a/core/src/test/java/org/apache/calcite/test/AggregateReduceFunctionsOnGroupKeysRuleTest.java b/core/src/test/java/org/apache/calcite/test/AggregateReduceFunctionsOnGroupKeysRuleTest.java index d4de2092aacc..c3a7570f061b 100644 --- a/core/src/test/java/org/apache/calcite/test/AggregateReduceFunctionsOnGroupKeysRuleTest.java +++ b/core/src/test/java/org/apache/calcite/test/AggregateReduceFunctionsOnGroupKeysRuleTest.java @@ -149,6 +149,120 @@ private static RelOptFixture sql(String sql) { sql(sql).withRule(AGGREGATE_REDUCE_FUNCTIONS_ON_GROUP_KEYS).checkUnchanged(); } + /** Test case for + * [CALCITE-7493] + * Support constant-result aggregates (e.g., STDDEV_POP, STDDEV) over GROUP BY keys. */ + @Test void testStatisticalFunctionStddevSampOfGroupByKey() { + // STDDEV_SAMP of a constant (GROUP BY key) is 0. + String sql = "select sal, stddev_samp(sal) as sd\n" + + "from emp group by sal, deptno"; + sql(sql).withRule(AGGREGATE_REDUCE_FUNCTIONS_ON_GROUP_KEYS).check(); + } + + @Test void testStatisticalFunctionStddevPopOfGroupByKey() { + // STDDEV_POP of a constant (GROUP BY key) is 0. + String sql = "select sal, stddev_pop(sal) as sdp\n" + + "from emp group by sal, deptno"; + sql(sql).withRule(AGGREGATE_REDUCE_FUNCTIONS_ON_GROUP_KEYS).check(); + } + + @Test void testStatisticalFunctionVarPopOfGroupByKey() { + // VAR_POP of a constant (GROUP BY key) is 0. + String sql = "select sal, var_pop(sal) as vp\n" + + "from emp group by sal, deptno"; + sql(sql).withRule(AGGREGATE_REDUCE_FUNCTIONS_ON_GROUP_KEYS).check(); + } + + @Test void testStatisticalFunctionVarSampOfGroupByKey() { + // VAR_SAMP of a constant (GROUP BY key) is 0 + // (variance of all identical values is 0). + String sql = "select sal, var_samp(sal) as vs\n" + + "from emp group by sal, deptno"; + sql(sql).withRule(AGGREGATE_REDUCE_FUNCTIONS_ON_GROUP_KEYS).check(); + } + + @Test void testMultipleStatisticalFunctions() { + // Test multiple statistical functions together. + String sql = "select sal, stddev_samp(sal) as sd, stddev_pop(sal) as sdp,\n" + + "var_pop(sal) as vp, var_samp(sal) as vs\n" + + "from emp group by sal, deptno"; + sql(sql).withRule(AGGREGATE_REDUCE_FUNCTIONS_ON_GROUP_KEYS).check(); + } + + @Test void testStatisticalFunctionWithBinaryExpression() { + // Variance of binary expression (sal + deptno) where all operands are GROUP BY keys. + // Since all values are constant within each group, variance is 0. + String sql = "select sal, var_pop(sal + deptno) as vp\n" + + "from emp group by sal, deptno"; + sql(sql).withRule(AGGREGATE_REDUCE_FUNCTIONS_ON_GROUP_KEYS).check(); + } + + @Test void testStatisticalFunctionWithConstantExpression() { + // Variance of expression with constant (2*sal + 100) where sal is a GROUP BY key. + // Since all values are constant within each group, variance is 0. + String sql = "select sal, stddev_pop(2 * sal + 100) as sdp\n" + + "from emp group by sal, deptno"; + sql(sql).withRule(AGGREGATE_REDUCE_FUNCTIONS_ON_GROUP_KEYS).check(); + } + + @Test void testStatisticalFunctionWithMultipleGroupByKeys() { + // Variance of expression combining multiple GROUP BY keys (sal * deptno). + // Since all values are constant within each group, variance is 0. + String sql = "select sal, var_samp(sal * deptno) as vs\n" + + "from emp group by sal, deptno"; + sql(sql).withRule(AGGREGATE_REDUCE_FUNCTIONS_ON_GROUP_KEYS).check(); + } + + @Test void testStatisticalFunctionWithNonGroupByColumnNoOptimization() { + // Negative test: expression contains only non-GROUP BY column (comm). + // The rule should NOT optimize because comm is not a constant within the group. + String sql = "select sal, var_pop(comm) as vp\n" + + "from emp group by sal, deptno"; + sql(sql).withRule(AGGREGATE_REDUCE_FUNCTIONS_ON_GROUP_KEYS).checkUnchanged(); + } + + @Test void testStatisticalFunctionWithMixedColumnsNoOptimization() { + // Negative test: expression mixes GROUP BY column (sal) and non-GROUP BY column (comm). + // The rule should NOT optimize because comm is not a constant within the group. + String sql = "select sal, stddev_pop(sal + comm) as sdp\n" + + "from emp group by sal, deptno"; + sql(sql).withRule(AGGREGATE_REDUCE_FUNCTIONS_ON_GROUP_KEYS).checkUnchanged(); + } + + @Test void testStatisticalFunctionWithPartialExpressionNoOptimization() { + // Negative test: expression combines a GROUP BY key (sal) with non-GROUP BY column (empno). + // The rule should NOT optimize because empno is not constant within the group. + String sql = "select sal, var_samp(sal * empno) as vs\n" + + "from emp group by sal, deptno"; + sql(sql).withRule(AGGREGATE_REDUCE_FUNCTIONS_ON_GROUP_KEYS).checkUnchanged(); + } + + @Test void testStatisticalFunctionWithComplexMixNoOptimization() { + // Negative test: complex expression with only GROUP BY column (sal) but + // also referencing non-GROUP BY column (comm) in multiplication. + String sql = "select sal, stddev_samp(sal * 2 + comm) as sd\n" + + "from emp group by sal, deptno"; + sql(sql).withRule(AGGREGATE_REDUCE_FUNCTIONS_ON_GROUP_KEYS).checkUnchanged(); + } + + @Test void testStatisticalFunctionNullableGroupKey() { + // Test NULL semantics: when GROUP BY key is nullable, STDDEV(key) should + // handle NULL correctly. The optimization wraps result in + // CASE WHEN key IS NULL THEN NULL ELSE 0 END + String sql = "select comm, stddev_pop(comm) as sdp\n" + + "from empnullables group by comm"; + sql(sql).withRule(AGGREGATE_REDUCE_FUNCTIONS_ON_GROUP_KEYS).check(); + } + + @Test void testStatisticalFunctionNullableExpression() { + // Test NULL semantics for expressions: when expression is nullable, + // STDDEV(expr) should return NULL when expr is NULL. + // Optimization wraps result in CASE to preserve NULL semantics + String sql = "select comm, stddev_samp(comm + 1) as sd\n" + + "from empnullables group by comm"; + sql(sql).withRule(AGGREGATE_REDUCE_FUNCTIONS_ON_GROUP_KEYS).check(); + } + @AfterAll static void checkActualAndReferenceFiles() { fixture().diffRepos.checkActualAndReferenceFiles(); } diff --git a/core/src/test/resources/org/apache/calcite/test/AggregateReduceFunctionsOnGroupKeysRuleTest.xml b/core/src/test/resources/org/apache/calcite/test/AggregateReduceFunctionsOnGroupKeysRuleTest.xml index 2083a969d724..4f7ac679939e 100644 --- a/core/src/test/resources/org/apache/calcite/test/AggregateReduceFunctionsOnGroupKeysRuleTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/AggregateReduceFunctionsOnGroupKeysRuleTest.xml @@ -318,6 +318,289 @@ from emp group by sal]]> LogicalAggregate(group=[{0}], COMM_MAX=[MAX($1)]) LogicalProject(SAL=[$5], COMM=[$6]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/core/src/test/resources/sql/agg-reduce.iq b/core/src/test/resources/sql/agg-reduce.iq new file mode 100644 index 000000000000..f31f333ad7f6 --- /dev/null +++ b/core/src/test/resources/sql/agg-reduce.iq @@ -0,0 +1,174 @@ +# agg-reduce.iq - Tests for aggregate function reduction on GROUP BY keys +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +!use post +!set outputformat mysql + +# These results were validated by: +# 1. Quidem tests: end-to-end SQL execution verification (this file) +# 2. RelOptFixture unit test cases covering query plan transformation +# 3. PostgreSQL manual verification: +# - Tests 1-4: Statistical functions return 0 or NULL (NULL when n < 2) +# - Test 5: Expressions with GROUP BY keys also return 0 +# - Tests 6-8: MAX/MIN/AVG return the GROUP BY key value itself +# - Test 9: Mix verification - MAX returns key, STDDEV_POP returns 0 +# All results confirmed: optimization is correct, semantics preserved + +# Test 1: STDDEV_POP on GROUP BY key should return 0 for constant values +select deptno, stddev_pop(deptno) as sdp from emp group by deptno order by deptno; ++--------+-----+ +| DEPTNO | SDP | ++--------+-----+ +| 10 | 0 | +| 20 | 0 | +| 30 | 0 | +| 50 | 0 | +| 60 | 0 | +| | | ++--------+-----+ +(6 rows) + +!ok + +# Test 2: STDDEV_SAMP on GROUP BY key should return 0 or null for constant values +select deptno, stddev_samp(deptno) as sds from emp group by deptno order by deptno; ++--------+-----+ +| DEPTNO | SDS | ++--------+-----+ +| 10 | 0 | +| 20 | | +| 30 | 0 | +| 50 | 0 | +| 60 | | +| | | ++--------+-----+ +(6 rows) + +!ok + +# Test 3: VAR_POP on GROUP BY key should return 0 for constant values +select deptno, var_pop(deptno) as vp from emp group by deptno order by deptno; ++--------+----+ +| DEPTNO | VP | ++--------+----+ +| 10 | 0 | +| 20 | 0 | +| 30 | 0 | +| 50 | 0 | +| 60 | 0 | +| | | ++--------+----+ +(6 rows) + +!ok + +# Test 4: VAR_SAMP on GROUP BY key should return 0 or null +select deptno, var_samp(deptno) as vs from emp group by deptno order by deptno; ++--------+----+ +| DEPTNO | VS | ++--------+----+ +| 10 | 0 | +| 20 | | +| 30 | 0 | +| 50 | 0 | +| 60 | | +| | | ++--------+----+ +(6 rows) + +!ok + +# Test 5: Expression with only GROUP BY keys - STDDEV_POP returns 0 +select deptno, stddev_pop(deptno + 1) as sdp from emp group by deptno order by deptno; ++--------+-----+ +| DEPTNO | SDP | ++--------+-----+ +| 10 | 0 | +| 20 | 0 | +| 30 | 0 | +| 50 | 0 | +| 60 | 0 | +| | | ++--------+-----+ +(6 rows) + +!ok + +# Test 6: MAX on GROUP BY key returns the key value itself +select deptno, max(deptno) as m from emp group by deptno order by deptno; ++--------+----+ +| DEPTNO | M | ++--------+----+ +| 10 | 10 | +| 20 | 20 | +| 30 | 30 | +| 50 | 50 | +| 60 | 60 | +| | | ++--------+----+ +(6 rows) + +!ok + +# Test 7: MIN on GROUP BY key returns the key value itself +select deptno, min(deptno) as m from emp group by deptno order by deptno; ++--------+----+ +| DEPTNO | M | ++--------+----+ +| 10 | 10 | +| 20 | 20 | +| 30 | 30 | +| 50 | 50 | +| 60 | 60 | +| | | ++--------+----+ +(6 rows) + +!ok + +# Test 8: AVG on GROUP BY key returns the key value itself +select deptno, avg(deptno) as a from emp group by deptno order by deptno; ++--------+----+ +| DEPTNO | A | ++--------+----+ +| 10 | 10 | +| 20 | 20 | +| 30 | 30 | +| 50 | 50 | +| 60 | 60 | +| | | ++--------+----+ +(6 rows) + +!ok + +# Test 9: Mix of statistical and regular functions +select deptno, max(deptno) as m, stddev_pop(deptno) as sdp +from emp group by deptno order by deptno; ++--------+----+-----+ +| DEPTNO | M | SDP | ++--------+----+-----+ +| 10 | 10 | 0 | +| 20 | 20 | 0 | +| 30 | 30 | 0 | +| 50 | 50 | 0 | +| 60 | 60 | 0 | +| | | | ++--------+----+-----+ +(6 rows) + +!ok From 48a0c997a2344f2b7635fc4eaebe1a741be042fb Mon Sep 17 00:00:00 2001 From: iwanttobepowerful <745778074@qq.com> Date: Wed, 10 Jun 2026 11:44:16 +0800 Subject: [PATCH 316/562] [CALCITE-7584] RelDecorrelator produces incorrect results for correlated LATERAL sub-queries with window functions --- .../calcite/sql2rel/RelDecorrelator.java | 80 ++- .../org/apache/calcite/test/JdbcTest.java | 15 +- .../calcite/test/SqlToRelConverterTest.xml | 2 +- core/src/test/resources/sql/sub-query.iq | 582 ++++++++++++++++++ 4 files changed, 659 insertions(+), 20 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java b/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java index 14d72f8dfe69..f9e7049af21a 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java @@ -70,6 +70,7 @@ import org.apache.calcite.rex.RexInputRef; import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexOver; import org.apache.calcite.rex.RexShuttle; import org.apache.calcite.rex.RexSubQuery; import org.apache.calcite.rex.RexUtil; @@ -1619,15 +1620,26 @@ private Frame decorrelateInputWithValueGenerator(RelNode rel, Frame inputFrame) new TreeMap<>(inputFrame.corDefOutputs); final Collection corVarList = cm.mapRefRelToCorRef.get(rel); + // Track only correlation variables that are not already produced by the + // input frame. Existing outputs still need to be carried forward, but they + // should not force an extra value generator. + final List missingCorVarList = new ArrayList<>(); + for (CorRef correlation : corVarList) { + if (!corDefOutputs.containsKey(correlation.def())) { + missingCorVarList.add(correlation); + } + } - // Try to populate correlation variables using local fields. + // Try to populate missing correlation variables using local fields. // This means that we do not need a value generator. if (rel instanceof Filter) { NavigableMap map = new TreeMap<>(); List projects = new ArrayList<>(); - for (CorRef correlation : corVarList) { + for (CorRef correlation : missingCorVarList) { final CorDef def = correlation.def(); - if (corDefOutputs.containsKey(def) || map.containsKey(def)) { + if (map.containsKey(def)) { + // The same correlation definition may be referenced more than once; + // one output slot is enough. continue; } try { @@ -1651,9 +1663,9 @@ private Frame decorrelateInputWithValueGenerator(RelNode rel, Frame inputFrame) } } } - // If all correlation variables are now satisfied, skip creating a value - // generator. - if (map.size() == corVarList.size()) { + // If all missing correlation variables are now satisfied, skip creating a + // value generator. + if (map.size() == missingCorVarList.size()) { map.putAll(inputFrame.corDefOutputs); final RelNode r; if (!projects.isEmpty()) { @@ -1667,7 +1679,10 @@ private Frame decorrelateInputWithValueGenerator(RelNode rel, Frame inputFrame) } } - return createFrameWithValueGenerator(rel.getInput(0), inputFrame, corVarList, corDefOutputs); + // Fall back to a value generator for correlation variables that could not + // be derived from local fields. + return createFrameWithValueGenerator(rel.getInput(0), inputFrame, + missingCorVarList, corDefOutputs); } /** @@ -2487,6 +2502,57 @@ private DecorrelateRexShuttle(RelNode currentRel, return fieldAccess; } + /** + * Window operators are decorrelated similarly to aggregates. A correlated + * window expression is evaluated once per outer-row binding before + * decorrelation; after decorrelation, outer references are represented as + * ordinary input fields. Therefore, add those fields to the window partition + * keys so that the window function is still evaluated independently for + * each outer-row binding. + * + *

      Implementation based on: Improving Unnesting of Complex Queries + * + *

      3.3 Unnesting Rules + * (https://dl.gi.de/server/api/core/bitstreams/c1918e8c-6a87-4da2-930a-bfed289f2388/content) + */ + @Override public RexNode visitOver(RexOver over) { + final RexOver newOver = (RexOver) super.visitOver(over); + final List partitionKeys = new ArrayList<>(newOver.getWindow().partitionKeys); + boolean update = newOver != over; + int newInputOutputOffset = 0; + for (RelNode input : currentRel.getInputs()) { + final Frame frame = map.get(input); + if (frame == null) { + // Inputs without a decorrelation frame keep their original field layout. + newInputOutputOffset += input.getRowType().getFieldCount(); + continue; + } + for (Integer newInputPos : frame.corDefOutputs.values()) { + // Correlation variables become regular input fields after decorrelation. + // Add them to the window partition keys to preserve the original + // per-correlate evaluation scope. + final RexInputRef ref = + new RexInputRef(newInputOutputOffset + newInputPos, + frame.r.getRowType().getFieldList() + .get(newInputPos).getType()); + if (!partitionKeys.contains(ref)) { + partitionKeys.add(ref); + update = true; + } + } + newInputOutputOffset += frame.r.getRowType().getFieldCount(); + } + if (!update) { + return over; + } + return currentRel.getCluster().getRexBuilder().makeOver( + newOver.getParserPosition(), newOver.getType(), newOver.getAggOperator(), + newOver.getOperands(), partitionKeys, newOver.getWindow().orderKeys, + newOver.getWindow().getLowerBound(), newOver.getWindow().getUpperBound(), + newOver.getWindow().getExclude(), newOver.getWindow().isRows(), true, false, + newOver.isDistinct(), newOver.ignoreNulls()); + } + @Override public RexNode visitInputRef(RexInputRef inputRef) { final RexInputRef ref = getNewForOldInputRef(currentRel, map, inputRef); if (ref.getIndex() == inputRef.getIndex() diff --git a/core/src/test/java/org/apache/calcite/test/JdbcTest.java b/core/src/test/java/org/apache/calcite/test/JdbcTest.java index 78098912edfc..f90c5525cad2 100644 --- a/core/src/test/java/org/apache/calcite/test/JdbcTest.java +++ b/core/src/test/java/org/apache/calcite/test/JdbcTest.java @@ -2583,18 +2583,9 @@ void checkMultisetQueryWithSingleColumn() { CalciteAssert.that() .with(CalciteAssert.Config.REGULAR) .query(sql) - .returnsUnordered("name=Bill; deptno=10; M=190", - "name=Bill; deptno=30; M=190", - "name=Bill; deptno=40; M=190", - "name=Eric; deptno=10; M=240", - "name=Eric; deptno=30; M=240", - "name=Eric; deptno=40; M=240", - "name=Sebastian; deptno=10; M=190", - "name=Sebastian; deptno=30; M=190", - "name=Sebastian; deptno=40; M=190", - "name=Theodore; deptno=10; M=190", - "name=Theodore; deptno=30; M=190", - "name=Theodore; deptno=40; M=190"); + .returnsUnordered("name=Bill; deptno=10; M=110", + "name=Sebastian; deptno=10; M=160", + "name=Theodore; deptno=10; M=120"); } /** Per SQL std, UNNEST is implicitly LATERAL. */ diff --git a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml index 1705493edb8a..0bf249f3afc4 100644 --- a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml @@ -7050,7 +7050,7 @@ LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$ LogicalAggregate(group=[{0}], agg#0=[MIN($1)]) LogicalProject(DEPTNO0=[$2], $f0=[true]) LogicalFilter(condition=[$1]) - LogicalProject(NAME=[$1], QualifyExpression=[=(RANK() OVER (PARTITION BY $1 ORDER BY $0 DESC), $2)], DEPTNO0=[$2]) + LogicalProject(NAME=[$1], QualifyExpression=[=(RANK() OVER (PARTITION BY $1, $2 ORDER BY $0 DESC), $2)], DEPTNO0=[$2]) LogicalJoin(condition=[true], joinType=[inner]) LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) LogicalAggregate(group=[{0}]) diff --git a/core/src/test/resources/sql/sub-query.iq b/core/src/test/resources/sql/sub-query.iq index 7c7e9fb0535c..3a1fa6478468 100644 --- a/core/src/test/resources/sql/sub-query.iq +++ b/core/src/test/resources/sql/sub-query.iq @@ -8168,6 +8168,588 @@ SELECT deptno FROM dept WHERE 1000.00 > !ok +# [CALCITE-7584] RelDecorrelator produces incorrect results for correlated LATERAL sub-queries with window functions +# Correlated LATERAL sub-query with a window expression. +# The equality predicate between the inner and outer query must remain applied +# after decorrelation. +# this was validated using postgres +SELECT e.ename, d.deptno, d.m +FROM emp e +JOIN LATERAL ( + SELECT d.deptno, + MAX(d.deptno + e.empno) OVER (PARTITION BY e.deptno) AS m + FROM dept d + WHERE e.deptno = d.deptno +) d ON TRUE +ORDER BY e.empno; ++--------+--------+------+ +| ENAME | DEPTNO | M | ++--------+--------+------+ +| SMITH | 20 | 7389 | +| ALLEN | 30 | 7529 | +| WARD | 30 | 7551 | +| JONES | 20 | 7586 | +| MARTIN | 30 | 7684 | +| BLAKE | 30 | 7728 | +| CLARK | 10 | 7792 | +| SCOTT | 20 | 7808 | +| KING | 10 | 7849 | +| TURNER | 30 | 7874 | +| ADAMS | 20 | 7896 | +| JAMES | 30 | 7930 | +| FORD | 20 | 7922 | +| MILLER | 10 | 7944 | ++--------+--------+------+ +(14 rows) + +!ok + +# The window must also be partitioned by correlation variables +# that are only referenced by the window expression. +# this was validated using postgres +SELECT e.ename, d.deptno, d.rn +FROM emp e +JOIN LATERAL ( + SELECT d.deptno, + ROW_NUMBER() OVER (PARTITION BY e.deptno ORDER BY e.empno, d.deptno) AS rn + FROM dept d + WHERE e.deptno = d.deptno +) d ON TRUE +ORDER BY e.empno; +!if (use_old_decorr) { ++--------+--------+----+ +| ENAME | DEPTNO | RN | ++--------+--------+----+ +| SMITH | 20 | 1 | +| ALLEN | 30 | 1 | +| WARD | 30 | 1 | +| JONES | 20 | 1 | +| MARTIN | 30 | 1 | +| BLAKE | 30 | 1 | +| CLARK | 10 | 1 | +| SCOTT | 20 | 1 | +| KING | 10 | 1 | +| TURNER | 30 | 1 | +| ADAMS | 20 | 1 | +| JAMES | 30 | 1 | +| FORD | 20 | 1 | +| MILLER | 10 | 1 | ++--------+--------+----+ +(14 rows) + +!ok +!} + +# Multiple equality-derived correlation keys must remain available +# when a window expression also needs an additional correlation key. +# this was validated using postgres +SELECT e.ename, s.empno, s.m +FROM emp e +JOIN LATERAL ( + SELECT e2.empno, + MAX(e2.empno + e.sal) OVER (PARTITION BY e.deptno, e.job) AS m + FROM emp e2 + WHERE e2.deptno = e.deptno + AND e2.job = e.job +) s ON TRUE +WHERE e.empno IN (7369, 7499, 7788) +ORDER BY e.empno, s.empno; ++-------+-------+----------+ +| ENAME | EMPNO | M | ++-------+-------+----------+ +| SMITH | 7369 | 8676.00 | +| SMITH | 7876 | 8676.00 | +| ALLEN | 7499 | 9444.00 | +| ALLEN | 7521 | 9444.00 | +| ALLEN | 7654 | 9444.00 | +| ALLEN | 7844 | 9444.00 | +| SCOTT | 7788 | 10902.00 | +| SCOTT | 7902 | 10902.00 | ++-------+-------+----------+ +(8 rows) + +!ok + +# this was validated using postgres +WITH bonus(ENAME, JOB, SAL, COMM) AS ( + VALUES ('ALLEN', 'SALESMAN', 1600.00, 300.00), + ('WARD', 'SALESMAN', 1250.00, 500.00) +) +SELECT * +FROM BONUS +WHERE EXISTS(SELECT RANK() OVER (PARTITION BY hiredate ORDER BY sal) AS s + FROM EMP, DEPT where EMP.deptno = DEPT.deptno + AND DEPT.dname < BONUS.ENAME); ++-------+----------+---------+--------+ +| ENAME | JOB | SAL | COMM | ++-------+----------+---------+--------+ +| ALLEN | SALESMAN | 1600.00 | 300.00 | +| WARD | SALESMAN | 1250.00 | 500.00 | ++-------+----------+---------+--------+ +(2 rows) + +!ok + +# this was validated using postgres +SELECT * +FROM emp e +WHERE EXISTS ( + SELECT 1 + FROM ( + SELECT + d.dname, + d.deptno, + RANK() OVER (PARTITION BY d.dname ORDER BY d.deptno DESC) AS rnk + FROM dept d + ) x + WHERE x.rnk = e.deptno +); ++-------+-------+-----+-----+----------+-----+------+--------+ +| EMPNO | ENAME | JOB | MGR | HIREDATE | SAL | COMM | DEPTNO | ++-------+-------+-----+-----+----------+-----+------+--------+ ++-------+-------+-----+-----+----------+-----+------+--------+ +(0 rows) + +!ok + +!if (use_old_decorr) { +EnumerableCalc(expr#0..8=[{inputs}], proj#0..7=[{exprs}]) + EnumerableHashJoin(condition=[=($8, $12)], joinType=[semi]) + EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t7):BIGINT], proj#0..8=[{exprs}]) + EnumerableTableScan(table=[[scott, EMP]]) + EnumerableWindow(window#0=[window(partition {1} order by [0 DESC] aggs [RANK()])]) + EnumerableTableScan(table=[[scott, DEPT]]) +!plan +!} + +create view calcite_7584_t1 as +select * from (values + ('val1a', cast(6 as smallint), 8, cast(10 as bigint)), + ('val1b', cast(8 as smallint), 16, cast(19 as bigint)), + ('val1a', cast(16 as smallint), 12, cast(21 as bigint)), + ('val1a', cast(16 as smallint), 12, cast(10 as bigint)), + ('val1c', cast(8 as smallint), 16, cast(19 as bigint)), + ('val1d', cast(null as smallint), 16, cast(22 as bigint)), + ('val1d', cast(null as smallint), 16, cast(19 as bigint)), + ('val1e', cast(10 as smallint), cast(null as integer), cast(25 as bigint)), + ('val1e', cast(10 as smallint), cast(null as integer), cast(19 as bigint)), + ('val1d', cast(10 as smallint), cast(null as integer), cast(12 as bigint)), + ('val1a', cast(6 as smallint), 8, cast(10 as bigint)), + ('val1e', cast(10 as smallint), cast(null as integer), cast(19 as bigint)) +) as t(t1a, t1b, t1c, t1d); +(0 rows modified) + +!update + +create view calcite_7584_t2 as +select * from (values + ('val2a', cast(6 as smallint), 12, cast(14 as bigint)), + ('val1b', cast(10 as smallint), 12, cast(19 as bigint)), + ('val1b', cast(8 as smallint), 16, cast(119 as bigint)), + ('val1c', cast(12 as smallint), 16, cast(219 as bigint)), + ('val1b', cast(null as smallint), 16, cast(319 as bigint)), + ('val2e', cast(8 as smallint), cast(null as integer), cast(419 as bigint)), + ('val1f', cast(19 as smallint), cast(null as integer), cast(519 as bigint)), + ('val1b', cast(10 as smallint), 12, cast(19 as bigint)), + ('val1b', cast(8 as smallint), 16, cast(19 as bigint)), + ('val1c', cast(12 as smallint), 16, cast(19 as bigint)), + ('val1e', cast(8 as smallint), cast(null as integer), cast(19 as bigint)), + ('val1f', cast(19 as smallint), cast(null as integer), cast(19 as bigint)), + ('val1b', cast(null as smallint), 16, cast(19 as bigint)) +) as t(t2a, t2b, t2c, t2d); +(0 rows modified) + +!update + +create view calcite_7584_t3 as +select * from (values + ('val3a', cast(6 as smallint), 12, cast(110 as bigint)), + ('val3a', cast(6 as smallint), 12, cast(10 as bigint)), + ('val1b', cast(10 as smallint), 12, cast(219 as bigint)), + ('val1b', cast(10 as smallint), 12, cast(19 as bigint)), + ('val1b', cast(8 as smallint), 16, cast(319 as bigint)), + ('val1b', cast(8 as smallint), 16, cast(19 as bigint)), + ('val3c', cast(17 as smallint), 16, cast(519 as bigint)), + ('val3c', cast(17 as smallint), 16, cast(19 as bigint)), + ('val1b', cast(null as smallint), 16, cast(419 as bigint)), + ('val1b', cast(null as smallint), 16, cast(19 as bigint)), + ('val3b', cast(8 as smallint), cast(null as integer), cast(719 as bigint)), + ('val3b', cast(8 as smallint), cast(null as integer), cast(19 as bigint)) +) as t(t3a, t3b, t3c, t3d); +(0 rows modified) + +!update + +# Window function in a correlated subquery. +# this was validated using postgres +SELECT 1 +FROM calcite_7584_t1 t1 +WHERE t1b < (SELECT MAX(tmp.s) FROM ( + SELECT SUM(t2b) OVER (PARTITION BY t2c ORDER BY t2d) AS s + FROM calcite_7584_t2 t2 WHERE t2.t2d = t1.t1d) AS tmp); ++--------+ +| EXPR$0 | ++--------+ +| 1 | +| 1 | +| 1 | +| 1 | ++--------+ +(4 rows) + +!ok + +!if (use_old_decorr) { +EnumerableCalc(expr#0..1=[{inputs}], expr#2=[1], EXPR$0=[$t2]) + EnumerableHashJoin(condition=[AND(=($1, $2), <($0, $3))], joinType=[semi]) + EnumerableValues(tuples=[[{ 6, 10 }, { 8, 19 }, { 16, 21 }, { 16, 10 }, { 8, 19 }, { null, 22 }, { null, 19 }, { 10, 25 }, { 10, 19 }, { 10, 12 }, { 6, 10 }, { 10, 19 }]]) + EnumerableSort(sort0=[$0], dir0=[ASC]) + EnumerableAggregate(group=[{0}], EXPR$0=[MAX($1) FILTER $2]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[0:BIGINT], expr#6=[>($t4, $t5)], T2D=[$t2], $f2=[$t3], $f3=[$t6]) + EnumerableWindow(window#0=[window(partition {1, 2} order by [2] aggs [$SUM0($0), COUNT($0)])]) + EnumerableValues(tuples=[[{ 6, 12, 14 }, { 10, 12, 19 }, { 8, 16, 119 }, { 12, 16, 219 }, { null, 16, 319 }, { 8, null, 419 }, { 19, null, 519 }, { 10, 12, 19 }, { 8, 16, 19 }, { 12, 16, 19 }, { 8, null, 19 }, { 19, null, 19 }, { null, 16, 19 }]]) +!plan +!} + +# Same as above but with LIMIT/ORDER BY instead of MAX. +# this was validated using postgres +SELECT 1 +FROM calcite_7584_t1 t1 +WHERE t1b < (SELECT SUM(t2b) OVER (PARTITION BY t2c ORDER BY t2d) AS s + FROM calcite_7584_t2 t2 WHERE t2.t2d = t1.t1d + ORDER BY s DESC + LIMIT 1); ++--------+ +| EXPR$0 | ++--------+ +| 1 | +| 1 | +| 1 | +| 1 | ++--------+ +(4 rows) + +!ok + +!if (use_old_decorr) { +EnumerableCalc(expr#0..4=[{inputs}], expr#5=[1], EXPR$0=[$t5]) + EnumerableHashJoin(condition=[AND(=($1, $4), <($3, $0))], joinType=[inner]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1], expr#4=[<=($t2, $t3)], proj#0..2=[{exprs}], $condition=[$t4]) + EnumerableWindow(window#0=[window(partition {1} order by [0 DESC] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[0:BIGINT], expr#6=[>($t3, $t5)], expr#7=[null:SMALLINT], expr#8=[CASE($t6, $t4, $t7)], S=[$t8], T2D=[$t2]) + EnumerableWindow(window#0=[window(partition {1, 2} order by [2] aggs [COUNT($0), $SUM0($0)])]) + EnumerableValues(tuples=[[{ 6, 12, 14 }, { 10, 12, 19 }, { 8, 16, 119 }, { 12, 16, 219 }, { null, 16, 319 }, { 8, null, 419 }, { 19, null, 519 }, { 10, 12, 19 }, { 8, 16, 19 }, { 12, 16, 19 }, { 8, null, 19 }, { 19, null, 19 }, { null, 16, 19 }]]) + EnumerableValues(tuples=[[{ 6, 10 }, { 8, 19 }, { 16, 21 }, { 16, 10 }, { 8, 19 }, { null, 22 }, { null, 19 }, { 10, 25 }, { 10, 19 }, { 10, 12 }, { 6, 10 }, { 10, 19 }]]) +!plan +!} + +# Window function in a correlated subquery with a non-equi predicate. +# this was validated using postgres +SELECT 1 +FROM calcite_7584_t1 t1 +WHERE t1b < (SELECT MAX(tmp.s) FROM ( + SELECT SUM(t2b) OVER (PARTITION BY t2c ORDER BY t2d) AS s + FROM calcite_7584_t2 t2 WHERE t2.t2d <= t1.t1d) AS tmp); ++--------+ +| EXPR$0 | ++--------+ +| 1 | +| 1 | +| 1 | +| 1 | +| 1 | +| 1 | ++--------+ +(6 rows) + +!ok + +!if (use_old_decorr) { +EnumerableCalc(expr#0..1=[{inputs}], expr#2=[1], EXPR$0=[$t2]) + EnumerableHashJoin(condition=[AND(=($1, $2), <($0, $3))], joinType=[semi]) + EnumerableValues(tuples=[[{ 6, 10 }, { 8, 19 }, { 16, 21 }, { 16, 10 }, { 8, 19 }, { null, 22 }, { null, 19 }, { 10, 25 }, { 10, 19 }, { 10, 12 }, { 6, 10 }, { 10, 19 }]]) + EnumerableSort(sort0=[$0], dir0=[ASC]) + EnumerableAggregate(group=[{0}], EXPR$0=[MAX($1) FILTER $2]) + EnumerableCalc(expr#0..5=[{inputs}], expr#6=[0:BIGINT], expr#7=[>($t5, $t6)], T1D=[$t3], $f2=[$t4], $f3=[$t7]) + EnumerableWindow(window#0=[window(partition {1, 3} order by [2] aggs [$SUM0($0), COUNT($0)])]) + EnumerableNestedLoopJoin(condition=[<=($2, $3)], joinType=[inner]) + EnumerableValues(tuples=[[{ 6, 12, 14 }, { 10, 12, 19 }, { 8, 16, 119 }, { 12, 16, 219 }, { null, 16, 319 }, { 8, null, 419 }, { 19, null, 519 }, { 10, 12, 19 }, { 8, 16, 19 }, { 12, 16, 19 }, { 8, null, 19 }, { 19, null, 19 }, { null, 16, 19 }]]) + EnumerableAggregate(group=[{0}]) + EnumerableValues(tuples=[[{ 10 }, { 19 }, { 21 }, { 10 }, { 19 }, { 22 }, { 19 }, { 25 }, { 19 }, { 12 }, { 10 }, { 19 }]]) +!plan +!} + +# Same as above but with LIMIT/ORDER BY. +# this was validated using postgres +SELECT 1 +FROM calcite_7584_t1 t1 +WHERE t1b < (SELECT SUM(t2b) OVER (PARTITION BY t2c ORDER BY t2d) AS s + FROM calcite_7584_t2 t2 WHERE t2.t2d <= t1.t1d + ORDER BY s DESC + LIMIT 1); ++--------+ +| EXPR$0 | ++--------+ +| 1 | +| 1 | +| 1 | +| 1 | +| 1 | +| 1 | ++--------+ +(6 rows) + +!ok + +!if (use_old_decorr) { +EnumerableCalc(expr#0..4=[{inputs}], expr#5=[1], EXPR$0=[$t5]) + EnumerableHashJoin(condition=[AND(=($1, $3), <($0, $2))], joinType=[inner]) + EnumerableValues(tuples=[[{ 6, 10 }, { 8, 19 }, { 16, 21 }, { 16, 10 }, { 8, 19 }, { null, 22 }, { null, 19 }, { 10, 25 }, { 10, 19 }, { 10, 12 }, { 6, 10 }, { 10, 19 }]]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1], expr#4=[<=($t2, $t3)], proj#0..2=[{exprs}], $condition=[$t4]) + EnumerableWindow(window#0=[window(partition {1} order by [0 DESC] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + EnumerableCalc(expr#0..5=[{inputs}], expr#6=[0:BIGINT], expr#7=[>($t4, $t6)], expr#8=[null:SMALLINT], expr#9=[CASE($t7, $t5, $t8)], S=[$t9], T1D=[$t3]) + EnumerableWindow(window#0=[window(partition {1, 3} order by [2] aggs [COUNT($0), $SUM0($0)])]) + EnumerableNestedLoopJoin(condition=[<=($2, $3)], joinType=[inner]) + EnumerableValues(tuples=[[{ 6, 12, 14 }, { 10, 12, 19 }, { 8, 16, 119 }, { 12, 16, 219 }, { null, 16, 319 }, { 8, null, 419 }, { 19, null, 519 }, { 10, 12, 19 }, { 8, 16, 19 }, { 12, 16, 19 }, { 8, null, 19 }, { 19, null, 19 }, { null, 16, 19 }]]) + EnumerableAggregate(group=[{0}]) + EnumerableValues(tuples=[[{ 10 }, { 19 }, { 21 }, { 10 }, { 19 }, { 22 }, { 19 }, { 25 }, { 19 }, { 12 }, { 10 }, { 19 }]]) +!plan +!} + +# Window function in a correlated subquery over joins. +# this was validated using postgres +SELECT t1b +FROM calcite_7584_t1 t1 +WHERE t1b > (SELECT MAX(tmp.s) FROM ( + SELECT RANK() OVER (PARTITION BY t3c, t2b ORDER BY t3c) AS s + FROM calcite_7584_t2 t2, calcite_7584_t3 t3 + WHERE t2.t2c = t3.t3c AND t2.t2a = t1.t1a) AS tmp); ++-----+ +| T1B | ++-----+ +| 8 | +| 8 | ++-----+ +(2 rows) + +!ok + +!if (use_old_decorr) { +EnumerableCalc(expr#0..1=[{inputs}], T1B=[$t1]) + EnumerableHashJoin(condition=[AND(=($0, $2), >(CAST($1):BIGINT, $3))], joinType=[semi]) + EnumerableValues(tuples=[[{ 'val1a', 6 }, { 'val1b', 8 }, { 'val1a', 16 }, { 'val1a', 16 }, { 'val1c', 8 }, { 'val1d', null }, { 'val1d', null }, { 'val1e', 10 }, { 'val1e', 10 }, { 'val1d', 10 }, { 'val1a', 6 }, { 'val1e', 10 }]]) + EnumerableSort(sort0=[$0], dir0=[ASC]) + EnumerableAggregate(group=[{0}], EXPR$0=[MAX($4)]) + EnumerableWindow(window#0=[window(partition {0, 1, 3} order by [3] aggs [RANK()])]) + EnumerableMergeJoin(condition=[=($2, $3)], joinType=[inner]) + EnumerableSort(sort0=[$2], dir0=[ASC]) + EnumerableValues(tuples=[[{ 'val2a', 6, 12 }, { 'val1b', 10, 12 }, { 'val1b', 8, 16 }, { 'val1c', 12, 16 }, { 'val1b', null, 16 }, { 'val2e', 8, null }, { 'val1f', 19, null }, { 'val1b', 10, 12 }, { 'val1b', 8, 16 }, { 'val1c', 12, 16 }, { 'val1e', 8, null }, { 'val1f', 19, null }, { 'val1b', null, 16 }]]) + EnumerableValues(tuples=[[{ 12 }, { 12 }, { 12 }, { 12 }, { 16 }, { 16 }, { 16 }, { 16 }, { 16 }, { 16 }, { null }, { null }]]) +!plan +!} + +# Window function in a correlated subquery over aggregation. +# this was validated using postgres +SELECT t1b +FROM calcite_7584_t1 t1 +WHERE t1b > (SELECT MAX(tmp.s) FROM ( + SELECT RANK() OVER (PARTITION BY t3c, t3d ORDER BY t3c) AS s + FROM (SELECT t3b, t3c, MAX(t3d) AS t3d + FROM calcite_7584_t3 t3 GROUP BY t3b, t3c) AS g) AS tmp) +ORDER BY t1b; ++-----+ +| T1B | ++-----+ +| 6 | +| 6 | +| 8 | +| 8 | +| 10 | +| 10 | +| 10 | +| 10 | +| 16 | +| 16 | ++-----+ +(10 rows) + +!ok + +!if (use_old_decorr) { +EnumerableSort(sort0=[$0], dir0=[ASC]) + EnumerableCalc(expr#0..1=[{inputs}], T1B=[$t0]) + EnumerableNestedLoopJoin(condition=[>(CAST($0):BIGINT, $1)], joinType=[inner]) + EnumerableValues(tuples=[[{ 6 }, { 8 }, { 16 }, { 16 }, { 8 }, { null }, { null }, { 10 }, { 10 }, { 10 }, { 6 }, { 10 }]]) + EnumerableAggregate(group=[{}], EXPR$0=[MAX($3)]) + EnumerableWindow(window#0=[window(partition {1, 2} order by [1] aggs [RANK()])]) + EnumerableAggregate(group=[{0, 1}], T3D=[MAX($2)]) + EnumerableValues(tuples=[[{ 6, 12, 110 }, { 6, 12, 10 }, { 10, 12, 219 }, { 10, 12, 19 }, { 8, 16, 319 }, { 8, 16, 19 }, { 17, 16, 519 }, { 17, 16, 19 }, { null, 16, 419 }, { null, 16, 19 }, { 8, null, 719 }, { 8, null, 19 }]]) +!plan +!} + +# this was validated using postgres +SELECT 1 +FROM calcite_7584_t1 t1 +WHERE t1b = (SELECT MAX(tmp.s) FROM ( + SELECT SUM(t2c) OVER (PARTITION BY t2c ORDER BY t1.t1d + t2d) AS s + FROM calcite_7584_t2 t2) AS tmp); +!if (use_old_decorr) { ++--------+ +| EXPR$0 | ++--------+ ++--------+ +(0 rows) + +!ok + +EnumerableCalc(expr#0..1=[{inputs}], expr#2=[1], EXPR$0=[$t2]) + EnumerableHashJoin(condition=[AND(=($1, $2), =(CAST($0):INTEGER, $3))], joinType=[semi]) + EnumerableValues(tuples=[[{ 6, 10 }, { 8, 19 }, { 16, 21 }, { 16, 10 }, { 8, 19 }, { null, 22 }, { null, 19 }, { 10, 25 }, { 10, 19 }, { 10, 12 }, { 6, 10 }, { 10, 19 }]]) + EnumerableSort(sort0=[$0], dir0=[ASC]) + EnumerableAggregate(group=[{0}], EXPR$0=[MAX($1) FILTER $2]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[0:BIGINT], expr#6=[>($t4, $t5)], T1D=[$t1], $f2=[$t3], $f3=[$t6]) + EnumerableWindow(window#0=[window(partition {0, 1} order by [2] aggs [$SUM0($0), COUNT($0)])]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[+($t2, $t1)], T2C=[$t0], T1D=[$t2], $2=[$t3]) + EnumerableNestedLoopJoin(condition=[true], joinType=[inner]) + EnumerableValues(tuples=[[{ 12, 14 }, { 12, 19 }, { 16, 119 }, { 16, 219 }, { 16, 319 }, { null, 419 }, { null, 519 }, { 12, 19 }, { 16, 19 }, { 16, 19 }, { null, 19 }, { null, 19 }, { 16, 19 }]]) + EnumerableAggregate(group=[{0}]) + EnumerableValues(tuples=[[{ 10 }, { 19 }, { 21 }, { 10 }, { 19 }, { 22 }, { 19 }, { 25 }, { 19 }, { 12 }, { 10 }, { 19 }]]) +!plan +!} + +# [CALCITE-7584] Correlated subquery with RANK() window function and IN predicate +# Test case for window function in correlated subquery with IN predicate +# this was validated using postgres +SELECT t1a +FROM calcite_7584_t1 t1 +WHERE t1b IN (SELECT RANK() OVER (PARTITION BY t3c ORDER BY t2b) AS s + FROM calcite_7584_t2 t2, calcite_7584_t3 t3 + WHERE t2.t2c = t3.t3c AND t2.t2a < t1.t1a); ++-----+ +| T1A | ++-----+ ++-----+ +(0 rows) + +!ok + +!if (use_old_decorr) { +EnumerableCalc(expr#0..2=[{inputs}], T1A=[$t0]) + EnumerableHashJoin(condition=[AND(=($0, $4), =($2, $3))], joinType=[semi]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[CAST($t1):BIGINT], proj#0..2=[{exprs}]) + EnumerableValues(tuples=[[{ 'val1a', 6 }, { 'val1b', 8 }, { 'val1a', 16 }, { 'val1a', 16 }, { 'val1c', 8 }, { 'val1d', null }, { 'val1d', null }, { 'val1e', 10 }, { 'val1e', 10 }, { 'val1d', 10 }, { 'val1a', 6 }, { 'val1e', 10 }]]) + EnumerableCalc(expr#0..1=[{inputs}], S=[$t1], T1A=[$t0]) + EnumerableAggregate(group=[{4, 5}]) + EnumerableWindow(window#0=[window(partition {0, 4} order by [2] aggs [RANK()])]) + EnumerableHashJoin(condition=[=($0, $3)], joinType=[inner]) + EnumerableValues(tuples=[[{ 12 }, { 12 }, { 12 }, { 12 }, { 16 }, { 16 }, { 16 }, { 16 }, { 16 }, { 16 }, { null }, { null }]]) + EnumerableNestedLoopJoin(condition=[<($0, $3)], joinType=[inner]) + EnumerableValues(tuples=[[{ 'val2a', 6, 12 }, { 'val1b', 10, 12 }, { 'val1b', 8, 16 }, { 'val1c', 12, 16 }, { 'val1b', null, 16 }, { 'val2e', 8, null }, { 'val1f', 19, null }, { 'val1b', 10, 12 }, { 'val1b', 8, 16 }, { 'val1c', 12, 16 }, { 'val1e', 8, null }, { 'val1f', 19, null }, { 'val1b', null, 16 }]]) + EnumerableAggregate(group=[{0}]) + EnumerableValues(tuples=[[{ 'val1a' }, { 'val1b' }, { 'val1a' }, { 'val1a' }, { 'val1c' }, { 'val1d' }, { 'val1d' }, { 'val1e' }, { 'val1e' }, { 'val1d' }, { 'val1a' }, { 'val1e' }]]) +!plan +!} + +# [CALCITE-7584] LATERAL subquery with window function +# this was validated using postgres +SELECT * +FROM calcite_7584_t1 t1 JOIN LATERAL + (SELECT SUM(t2.t2b) OVER (ORDER BY t2.t2b) AS window_sum + FROM calcite_7584_t2 t2 + WHERE t2.t2b >= t1.t1b) AS t2_window ON TRUE +order by 1,2,3,4,5; ++-------+-----+-----+-----+------------+ +| T1A | T1B | T1C | T1D | WINDOW_SUM | ++-------+-----+-----+-----+------------+ +| val1a | 6 | 8 | 10 | 6 | +| val1a | 6 | 8 | 10 | 6 | +| val1a | 6 | 8 | 10 | 38 | +| val1a | 6 | 8 | 10 | 38 | +| val1a | 6 | 8 | 10 | 38 | +| val1a | 6 | 8 | 10 | 38 | +| val1a | 6 | 8 | 10 | 38 | +| val1a | 6 | 8 | 10 | 38 | +| val1a | 6 | 8 | 10 | 38 | +| val1a | 6 | 8 | 10 | 38 | +| val1a | 6 | 8 | 10 | 58 | +| val1a | 6 | 8 | 10 | 58 | +| val1a | 6 | 8 | 10 | 58 | +| val1a | 6 | 8 | 10 | 58 | +| val1a | 6 | 8 | 10 | 82 | +| val1a | 6 | 8 | 10 | 82 | +| val1a | 6 | 8 | 10 | 82 | +| val1a | 6 | 8 | 10 | 82 | +| val1a | 6 | 8 | 10 | 120 | +| val1a | 6 | 8 | 10 | 120 | +| val1a | 6 | 8 | 10 | 120 | +| val1a | 6 | 8 | 10 | 120 | +| val1a | 16 | 12 | 10 | 38 | +| val1a | 16 | 12 | 10 | 38 | +| val1a | 16 | 12 | 21 | 38 | +| val1a | 16 | 12 | 21 | 38 | +| val1b | 8 | 16 | 19 | 32 | +| val1b | 8 | 16 | 19 | 32 | +| val1b | 8 | 16 | 19 | 32 | +| val1b | 8 | 16 | 19 | 32 | +| val1b | 8 | 16 | 19 | 52 | +| val1b | 8 | 16 | 19 | 52 | +| val1b | 8 | 16 | 19 | 76 | +| val1b | 8 | 16 | 19 | 76 | +| val1b | 8 | 16 | 19 | 114 | +| val1b | 8 | 16 | 19 | 114 | +| val1c | 8 | 16 | 19 | 32 | +| val1c | 8 | 16 | 19 | 32 | +| val1c | 8 | 16 | 19 | 32 | +| val1c | 8 | 16 | 19 | 32 | +| val1c | 8 | 16 | 19 | 52 | +| val1c | 8 | 16 | 19 | 52 | +| val1c | 8 | 16 | 19 | 76 | +| val1c | 8 | 16 | 19 | 76 | +| val1c | 8 | 16 | 19 | 114 | +| val1c | 8 | 16 | 19 | 114 | +| val1d | 10 | | 12 | 20 | +| val1d | 10 | | 12 | 20 | +| val1d | 10 | | 12 | 44 | +| val1d | 10 | | 12 | 44 | +| val1d | 10 | | 12 | 82 | +| val1d | 10 | | 12 | 82 | +| val1e | 10 | | 19 | 20 | +| val1e | 10 | | 19 | 20 | +| val1e | 10 | | 19 | 20 | +| val1e | 10 | | 19 | 20 | +| val1e | 10 | | 19 | 44 | +| val1e | 10 | | 19 | 44 | +| val1e | 10 | | 19 | 44 | +| val1e | 10 | | 19 | 44 | +| val1e | 10 | | 19 | 82 | +| val1e | 10 | | 19 | 82 | +| val1e | 10 | | 19 | 82 | +| val1e | 10 | | 19 | 82 | +| val1e | 10 | | 25 | 20 | +| val1e | 10 | | 25 | 20 | +| val1e | 10 | | 25 | 44 | +| val1e | 10 | | 25 | 44 | +| val1e | 10 | | 25 | 82 | +| val1e | 10 | | 25 | 82 | ++-------+-----+-----+-----+------------+ +(70 rows) + +!ok + +!if (use_old_decorr) { +EnumerableSort(sort0=[$0], sort1=[$1], sort2=[$2], sort3=[$3], sort4=[$4], dir0=[ASC], dir1=[ASC], dir2=[ASC], dir3=[ASC], dir4=[ASC]) + EnumerableCalc(expr#0..5=[{inputs}], proj#0..4=[{exprs}]) + EnumerableHashJoin(condition=[=($1, $5)], joinType=[inner]) + EnumerableValues(tuples=[[{ 'val1a', 6, 8, 10 }, { 'val1b', 8, 16, 19 }, { 'val1a', 16, 12, 21 }, { 'val1a', 16, 12, 10 }, { 'val1c', 8, 16, 19 }, { 'val1d', null, 16, 22 }, { 'val1d', null, 16, 19 }, { 'val1e', 10, null, 25 }, { 'val1e', 10, null, 19 }, { 'val1d', 10, null, 12 }, { 'val1a', 6, 8, 10 }, { 'val1e', 10, null, 19 }]]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0:BIGINT], expr#5=[>($t2, $t4)], expr#6=[null:SMALLINT], expr#7=[CASE($t5, $t3, $t6)], WINDOW_SUM=[$t7], T1B=[$t1]) + EnumerableWindow(window#0=[window(partition {1} order by [0] aggs [COUNT($0), $SUM0($0)])]) + EnumerableNestedLoopJoin(condition=[>=($0, $1)], joinType=[inner]) + EnumerableValues(tuples=[[{ 6 }, { 10 }, { 8 }, { 12 }, { null }, { 8 }, { 19 }, { 10 }, { 8 }, { 12 }, { 8 }, { 19 }, { null }]]) + EnumerableAggregate(group=[{0}]) + EnumerableValues(tuples=[[{ 6 }, { 8 }, { 16 }, { 16 }, { 8 }, { null }, { null }, { 10 }, { 10 }, { 10 }, { 6 }, { 10 }]]) +!plan +!} + # [CALCITE-7274] RexFieldAccess has wrong index when use trim unused fields !set trimfields true From 1471f0ac5c0d6ca936799d1ba448c3a1f389039d Mon Sep 17 00:00:00 2001 From: zzwqqq Date: Tue, 26 May 2026 17:15:51 +0800 Subject: [PATCH 317/562] [CALCITE-7550] SqlUpdate and SqlDelete unparse EXISTS subqueries without parentheses --- .../org/apache/calcite/sql/SqlDelete.java | 3 +- .../apache/calcite/sql/SqlSelectOperator.java | 31 +------------ .../org/apache/calcite/sql/SqlUpdate.java | 3 +- .../java/org/apache/calcite/sql/SqlUtil.java | 43 +++++++++++++++++++ .../rel/rel2sql/RelToSqlConverterTest.java | 24 +++++++++++ 5 files changed, 70 insertions(+), 34 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql/SqlDelete.java b/core/src/main/java/org/apache/calcite/sql/SqlDelete.java index 9e5ff8c87d47..c9da35c48bf5 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlDelete.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlDelete.java @@ -152,8 +152,7 @@ public SqlNode getTargetTable() { } SqlNode condition = this.condition; if (condition != null) { - writer.sep("WHERE"); - condition.unparse(writer, opLeft, opRight); + SqlUtil.unparseWhereClause(writer, condition, opLeft, opRight); } writer.endList(frame); } diff --git a/core/src/main/java/org/apache/calcite/sql/SqlSelectOperator.java b/core/src/main/java/org/apache/calcite/sql/SqlSelectOperator.java index b8db2d8f5ab8..7e37ac732f2c 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlSelectOperator.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlSelectOperator.java @@ -16,7 +16,6 @@ */ package org.apache.calcite.sql; -import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.util.SqlBasicVisitor; @@ -24,7 +23,6 @@ import org.checkerframework.checker.nullness.qual.Nullable; -import java.util.ArrayList; import java.util.List; import static org.apache.calcite.linq4j.Nullness.castNonNull; @@ -175,34 +173,7 @@ public SqlSelect createCall( SqlNode where = select.where; if (where != null) { - writer.sep("WHERE"); - - if (!writer.isAlwaysUseParentheses()) { - SqlNode node = where; - - // decide whether to split on ORs or ANDs - SqlBinaryOperator whereSep = SqlStdOperatorTable.AND; - if ((node instanceof SqlCall) - && node.getKind() == SqlKind.OR) { - whereSep = SqlStdOperatorTable.OR; - } - - // unroll whereClause - final List list = new ArrayList<>(0); - while (node.getKind() == whereSep.kind) { - assert node instanceof SqlCall; - final SqlCall call1 = (SqlCall) node; - list.add(0, call1.operand(1)); - node = call1.operand(0); - } - list.add(0, node); - - // unparse in a WHERE_LIST frame - writer.list(SqlWriter.FrameTypeEnum.WHERE_LIST, whereSep, - new SqlNodeList(list, where.getParserPosition())); - } else { - where.unparse(writer, 0, 0); - } + SqlUtil.unparseWhereClause(writer, where, 0, 0); } if (select.groupBy != null) { SqlNodeList groupBy = diff --git a/core/src/main/java/org/apache/calcite/sql/SqlUpdate.java b/core/src/main/java/org/apache/calcite/sql/SqlUpdate.java index 82985f2bd721..22d669efcf0b 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlUpdate.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlUpdate.java @@ -202,8 +202,7 @@ public void setSourceSelect(SqlSelect sourceSelect) { writer.endList(setFrame); SqlNode condition = this.condition; if (condition != null) { - writer.sep("WHERE"); - condition.unparse(writer, opLeft, opRight); + SqlUtil.unparseWhereClause(writer, condition, opLeft, opRight); } writer.endList(frame); } diff --git a/core/src/main/java/org/apache/calcite/sql/SqlUtil.java b/core/src/main/java/org/apache/calcite/sql/SqlUtil.java index d561bc8abb01..1bafd7cfff85 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlUtil.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlUtil.java @@ -454,6 +454,49 @@ public static void unparseBinarySyntax( writer.endList(frame); } + /** + * Unparses a WHERE clause. + * + *

      Unparsing the condition in a {@link SqlWriter.FrameTypeEnum#WHERE_LIST} + * frame lets sub-queries in predicates recognize that they need + * parentheses. + * + * @param writer Writer + * @param where WHERE condition + * @param leftPrec Left precedence + * @param rightPrec Right precedence + */ + public static void unparseWhereClause(SqlWriter writer, SqlNode where, + int leftPrec, int rightPrec) { + writer.sep("WHERE"); + + if (!writer.isAlwaysUseParentheses()) { + SqlNode node = where; + + // Decide whether to split on ORs or ANDs. + SqlBinaryOperator whereSep = SqlStdOperatorTable.AND; + if ((node instanceof SqlCall) + && node.getKind() == SqlKind.OR) { + whereSep = SqlStdOperatorTable.OR; + } + + // Unroll whereClause. + final List list = new ArrayList<>(0); + while (node.getKind() == whereSep.kind) { + assert node instanceof SqlCall; + final SqlCall call1 = (SqlCall) node; + list.add(0, call1.operand(1)); + node = call1.operand(0); + } + list.add(0, node); + + writer.list(SqlWriter.FrameTypeEnum.WHERE_LIST, whereSep, + new SqlNodeList(list, where.getParserPosition())); + } else { + where.unparse(writer, leftPrec, rightPrec); + } + } + /** * Concatenates string literals. * diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java index 66afda8653b2..97d782fcaeef 100644 --- a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java +++ b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java @@ -10178,6 +10178,30 @@ private void checkLiteral2(String expression, String expected) { final String expected2 = "UPDATE \"foodmart\".\"product\" SET \"product_name\" = 'calcite', " + "\"product_id\" = 10\nWHERE \"product_id\" = 1"; sql(sql2).ok(expected2); + + final String sql3 = "update \"foodmart\".\"product\"\n" + + "set \"product_name\" = 'calcite'\n" + + "where exists (\n" + + " select 1 from \"foodmart\".\"product_class\")"; + final String expected3 = "UPDATE \"foodmart\".\"product\" SET \"product_name\" = " + + "'calcite'\nWHERE EXISTS (SELECT *\nFROM \"foodmart\".\"product_class\")"; + sql(sql3).ok(expected3); + } + + @Test void testDelete() { + final String sql0 = "delete from \"foodmart\".\"product\"\n" + + "where exists (\n" + + " select 1 from \"foodmart\".\"product_class\")"; + final String expected0 = "DELETE FROM \"foodmart\".\"product\"\n" + + "WHERE EXISTS (SELECT *\nFROM \"foodmart\".\"product_class\")"; + sql(sql0).ok(expected0); + + final String sql1 = "delete from \"foodmart\".\"product\"\n" + + "where not exists (\n" + + " select 1 from \"foodmart\".\"product_class\")"; + final String expected1 = "DELETE FROM \"foodmart\".\"product\"\n" + + "WHERE NOT EXISTS (SELECT *\nFROM \"foodmart\".\"product_class\")"; + sql(sql1).ok(expected1); } /** From 4640b5fb740457157cfa949b1d05ab79c937fe52 Mon Sep 17 00:00:00 2001 From: alhudz Date: Wed, 10 Jun 2026 10:36:13 +0530 Subject: [PATCH 318/562] [CALCITE-7600] Quote PARSE_URL key before building the query regex --- .../java/org/apache/calcite/runtime/SqlFunctions.java | 2 +- .../java/org/apache/calcite/test/SqlOperatorTest.java | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java index 7211f78de369..90ed84c7618f 100644 --- a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java +++ b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java @@ -1859,7 +1859,7 @@ public static String convertOracle(String s, String... args) { @Deterministic public static class ParseUrlFunction { static Pattern keyToPattern(String keyToExtract) { - return Pattern.compile("(&|^)" + keyToExtract + "=([^&]*)"); + return Pattern.compile("(&|^)" + Pattern.quote(keyToExtract) + "=([^&]*)"); } private final LoadingCache cache = diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java index d399890342d5..080a704a8cc8 100644 --- a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java +++ b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java @@ -5204,6 +5204,16 @@ void testBitGetFunc(SqlOperatorFixture f, String functionName) { "VARCHAR"); f.checkNull("parse_url('http://calcite.apache.org/path1/p.php?k1=v1&k2=v2#Ref1'," + " 'QUERY', 'k3')"); + // key is matched literally, regex metacharacters do not match other keys + f.checkNull("parse_url('http://calcite.apache.org/path1/p.php?k1=v1&k2=v2#Ref1'," + + " 'QUERY', 'k.')"); + f.checkString("parse_url('http://calcite.apache.org/path1/p.php?a.b=v1&axb=v2#Ref1'," + + " 'QUERY', 'a.b')", + "v1", + "VARCHAR"); + // a key that is not a valid regex must not raise an error + f.checkNull("parse_url('http://calcite.apache.org/path1/p.php?k1=v1&k2=v2#Ref1'," + + " 'QUERY', '(')"); f.checkString("parse_url('http://calcite.apache.org/path1/p.php?k1=v1&k2=v2#Ref1'," + " 'FILE')", "/path1/p.php?k1=v1&k2=v2", From 1891bd95018da06897e9baacf036774da3d2e6c1 Mon Sep 17 00:00:00 2001 From: Silun Date: Thu, 11 Jun 2026 15:42:25 +0800 Subject: [PATCH 319/562] [CALCITE-7596] TopDownGeneralDecorrelator omits rewriting the ORDER BY clause in window function within correlated subquery --- .../sql2rel/TopDownGeneralDecorrelator.java | 2 +- core/src/test/resources/sql/new-decorr.iq | 25 +++++++++++++++++++ core/src/test/resources/sql/sub-query.iq | 4 +-- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java b/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java index c3d2bd924961..291eb619d0ed 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java @@ -975,7 +975,7 @@ static List rewrite( } return unnestedQuery.r.getCluster().getRexBuilder().makeWindow( newPartitionKeys, - window.orderKeys, + shiftedWindow.orderKeys, window.getLowerBound(), window.getUpperBound(), window.isRows(), diff --git a/core/src/test/resources/sql/new-decorr.iq b/core/src/test/resources/sql/new-decorr.iq index 987bd22299ac..3aef6e527608 100644 --- a/core/src/test/resources/sql/new-decorr.iq +++ b/core/src/test/resources/sql/new-decorr.iq @@ -471,4 +471,29 @@ FROM (SELECT ARRAY[1,2,3] as x) s; !ok +# [CALCITE-7596] TopDownGeneralDecorrelator omits rewriting the ORDER BY clause in window function within correlated subquery +SELECT empno, (SELECT row_number() OVER (PARTITION BY dname ORDER BY emp.sal) FROM dept WHERE dept.deptno = emp.deptno) as rn FROM emp; +!if (use_new_decorr) { ++-------+----+ +| EMPNO | RN | ++-------+----+ +| 7369 | 1 | +| 7499 | 1 | +| 7521 | 1 | +| 7566 | 1 | +| 7654 | 1 | +| 7698 | 1 | +| 7782 | 1 | +| 7788 | 1 | +| 7839 | 1 | +| 7844 | 1 | +| 7876 | 1 | +| 7900 | 1 | +| 7902 | 1 | +| 7934 | 1 | ++-------+----+ +(14 rows) + +!ok +!} # End new-decorr.iq diff --git a/core/src/test/resources/sql/sub-query.iq b/core/src/test/resources/sql/sub-query.iq index 3a1fa6478468..db69258df248 100644 --- a/core/src/test/resources/sql/sub-query.iq +++ b/core/src/test/resources/sql/sub-query.iq @@ -8216,7 +8216,6 @@ JOIN LATERAL ( WHERE e.deptno = d.deptno ) d ON TRUE ORDER BY e.empno; -!if (use_old_decorr) { +--------+--------+----+ | ENAME | DEPTNO | RN | +--------+--------+----+ @@ -8238,7 +8237,6 @@ ORDER BY e.empno; (14 rows) !ok -!} # Multiple equality-derived correlation keys must remain available # when a window expression also needs an additional correlation key. @@ -8593,7 +8591,6 @@ FROM calcite_7584_t1 t1 WHERE t1b = (SELECT MAX(tmp.s) FROM ( SELECT SUM(t2c) OVER (PARTITION BY t2c ORDER BY t1.t1d + t2d) AS s FROM calcite_7584_t2 t2) AS tmp); -!if (use_old_decorr) { +--------+ | EXPR$0 | +--------+ @@ -8602,6 +8599,7 @@ WHERE t1b = (SELECT MAX(tmp.s) FROM ( !ok +!if (use_old_decorr) { EnumerableCalc(expr#0..1=[{inputs}], expr#2=[1], EXPR$0=[$t2]) EnumerableHashJoin(condition=[AND(=($1, $2), =(CAST($0):INTEGER, $3))], joinType=[semi]) EnumerableValues(tuples=[[{ 6, 10 }, { 8, 19 }, { 16, 21 }, { 16, 10 }, { 8, 19 }, { null, 22 }, { null, 19 }, { 10, 25 }, { 10, 19 }, { 10, 12 }, { 6, 10 }, { 10, 19 }]]) From fd612a924f9c29f06019fdd97745f95e6ba35772 Mon Sep 17 00:00:00 2001 From: Jerome Haltom Date: Thu, 7 May 2026 13:34:23 -0500 Subject: [PATCH 320/562] [CALCITE-7510] EnumerableTableModify: advanced UPDATE, DELETE and INSERT --- .../enumerable/EnumerableTableModify.java | 449 +++++++++++++++--- .../apache/calcite/util/BuiltInMethod.java | 2 + .../enumerable/EnumerableTableModifyTest.java | 77 +++ .../calcite/test/JdbcFrontLinqBackTest.java | 8 +- .../calcite/linq4j/DefaultEnumerable.java | 9 + .../calcite/linq4j/EnumerableDefaults.java | 35 ++ .../calcite/linq4j/ExtendedEnumerable.java | 26 + .../org/apache/calcite/test/ServerTest.java | 153 ++++++ 8 files changed, 702 insertions(+), 57 deletions(-) create mode 100644 core/src/test/java/org/apache/calcite/adapter/enumerable/EnumerableTableModifyTest.java diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableTableModify.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableTableModify.java index 186ec6903f91..3f60240fd994 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableTableModify.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableTableModify.java @@ -17,6 +17,9 @@ package org.apache.calcite.adapter.enumerable; import org.apache.calcite.adapter.java.JavaTypeFactory; +import org.apache.calcite.linq4j.Enumerable; +import org.apache.calcite.linq4j.Enumerator; +import org.apache.calcite.linq4j.function.Function1; import org.apache.calcite.linq4j.tree.BlockBuilder; import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.linq4j.tree.Expressions; @@ -28,24 +31,32 @@ import org.apache.calcite.prepare.Prepare; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.TableModify; +import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.rex.RexNode; import org.apache.calcite.schema.ModifiableTable; import org.apache.calcite.util.BuiltInMethod; import org.checkerframework.checker.nullness.qual.Nullable; -import java.lang.reflect.Method; import java.lang.reflect.Modifier; +import java.util.ArrayDeque; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; +import java.util.Deque; +import java.util.HashMap; import java.util.List; +import java.util.ListIterator; +import java.util.Map; import static com.google.common.base.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; -/** Implementation of {@link org.apache.calcite.rel.core.TableModify} in - * {@link org.apache.calcite.adapter.enumerable.EnumerableConvention enumerable calling convention}. */ +/** + * Implementation of {@link org.apache.calcite.rel.core.TableModify} in + * {@link org.apache.calcite.adapter.enumerable.EnumerableConvention enumerable calling convention}. + */ public class EnumerableTableModify extends TableModify implements EnumerableRel { public EnumerableTableModify(RelOptCluster cluster, RelTraitSet traits, @@ -80,100 +91,432 @@ public EnumerableTableModify(RelOptCluster cluster, RelTraitSet traits, final BlockBuilder builder = new BlockBuilder(); final Result result = implementor.visitChild(this, 0, (EnumerableRel) getInput(), pref); - Expression childExp = - builder.append( - "child", result.block); + + // Enumerable produced by the input relational expression. + final Expression sourceExp = + builder.append("source", result.block); + + // Variable that will hold the table's mutable backing collection. final ParameterExpression collectionParameter = Expressions.parameter(Collection.class, builder.newName("collection")); + + // Expression that yields the ModifiableTable instance at runtime. final Expression expression = table.getExpression(ModifiableTable.class); requireNonNull(expression, "expression"); // TODO: user error in validator checkArgument( ModifiableTable.class.isAssignableFrom( Types.toClass(expression.getType())), "not assignable from type %s", expression.getType()); + + // collection = table.getModifiableCollection() builder.add( Expressions.declare( Modifier.FINAL, collectionParameter, Expressions.call( expression, - BuiltInMethod.MODIFIABLE_TABLE_GET_MODIFIABLE_COLLECTION - .method))); + BuiltInMethod.MODIFIABLE_TABLE_GET_MODIFIABLE_COLLECTION.method))); + + // Physical row representation of this TableModify node's output + // (a single ROWCOUNT field). + final PhysType physType = + PhysTypeImpl.of( + implementor.getTypeFactory(), + getRowType(), + pref == Prefer.ARRAY ? JavaRowFormat.ARRAY : JavaRowFormat.SCALAR); + + switch (getOperation()) { + case INSERT: + return implementInsert(implementor, result, builder, sourceExp, collectionParameter, + physType); + case UPDATE: + return implementUpdate(implementor, builder, sourceExp, collectionParameter, physType); + case DELETE: + return implementDelete(implementor, result, builder, sourceExp, collectionParameter, + physType); + default: + throw new AssertionError("unsupported operation: " + getOperation()); + } + } + + /** + * Generates code for an UPDATE statement. + * + *

      Applies updates to matching rows in the backing collection and returns + * the number of updated rows as a single-element enumerable. + * + * @param implementor code-generation context + * @param builder block under construction + * @param sourceExp enumerable of source rows produced by the input + * @param collectionParameter the modifiable backing collection of the table + * @param physType physical type of this node's output row + */ + private Result implementUpdate( + EnumerableRelImplementor implementor, + BlockBuilder builder, + Expression sourceExp, + ParameterExpression collectionParameter, + PhysType physType) { + final List updateCols = requireNonNull(getUpdateColumnList()); + final List tableFields = table.getRowType().getFieldList(); + final int tableFieldCount = tableFields.size(); + + // Child row layout for UPDATE: + // [originalField_0, ..., originalField_N-1, newValue_0, ..., newValue_M-1] + // where N = tableFieldCount and M = updateCols.size(). + + // Resolve each SET-column name to its 0-based position in the table row. + final int[] updateColumnIndices = new int[updateCols.size()]; + for (int i = 0; i < updateCols.size(); i++) { + final String colName = updateCols.get(i); + int found = -1; + for (int j = 0; j < tableFields.size(); j++) { + if (tableFields.get(j).getName().equals(colName)) { + found = j; + break; + } + } + if (found < 0) { + throw new AssertionError("column '" + colName + "' not found in table"); + } + updateColumnIndices[i] = found; + } + + // Generate code that applies one-to-one update consumption: + // each source row updates at most one matching sink row. + final Expression updateCountExp = + builder.append( + "updateCount", + Expressions.call( + EnumerableTableModify.class, + "applyUpdateOneToOne", + // Source rows are produced by the child relational expression. + sourceExp, + // Sink is the table's mutable backing collection. + Expressions.convert_(collectionParameter, List.class), + // Number of original-row fields in each source payload. + Expressions.constant(tableFieldCount), + // Table column positions to overwrite from trailing source values. + Expressions.constant(updateColumnIndices))); + + // Return the number of updated rows as the single output row. + builder.add( + Expressions.return_( + null, + Expressions.call( + BuiltInMethod.SINGLETON_ENUMERABLE.method, + Expressions.convert_( + updateCountExp, + long.class)))); + + return implementor.result(physType, builder.toBlock()); + } + + /** + * Applies UPDATE with one-to-one, first-match consumption semantics. + * + *

      Each source row contributes one replacement row keyed by the original + * row content. As sink rows are scanned in order, the first matching row for + * each queued source update is replaced and consumed, so duplicate keys update + * only as many rows as appear in the source. + */ + public static long applyUpdateOneToOne(Enumerable source, List sink, + int tableFieldCount, int[] updateColumnIndices) { + final Map, Deque> updatesByKey = new HashMap<>(); + try (Enumerator e = source.enumerator()) { + while (e.moveNext()) { + final Object[] sourceRow = e.current(); + final List key = Arrays.asList(Arrays.copyOf(sourceRow, tableFieldCount)); + final Object[] newRow = applyUpdate(sourceRow, tableFieldCount, updateColumnIndices); + updatesByKey.computeIfAbsent(key, k -> new ArrayDeque<>()).addLast(newRow); + } + } + + long updateCount = 0; + final ListIterator it = sink.listIterator(); + while (it.hasNext()) { + final Object[] current = it.next(); + final List key = Arrays.asList(current); + final Deque pending = updatesByKey.get(key); + if (pending == null || pending.isEmpty()) { + continue; + } + it.set(pending.removeFirst()); + updateCount++; + if (pending.isEmpty()) { + updatesByKey.remove(key); + } + } + return updateCount; + } + + /** + * Generates code for a DELETE statement. + * + *

      The source produces every row that matches the WHERE clause. Those rows + * are removed from the backing collection and the number of deleted rows is + * returned as a single-element enumerable. + * + * @param implementor code-generation context + * @param result compiled result of the input relational expression + * @param builder block under construction + * @param sourceExp enumerable of rows to delete — every row matched + * by the WHERE clause, as produced by the input + * relational expression + * @param collectionParameter the modifiable backing collection of the table + * @param physType physical type of this node's output row + */ + private Result implementDelete( + EnumerableRelImplementor implementor, + Result result, + BlockBuilder builder, + Expression sourceExp, + ParameterExpression collectionParameter, + PhysType physType) { + + // Snapshot the collection size before the delete so we can compute the + // number of rows removed as (sizeBefore - sizeAfter). final Expression countParameter = builder.append( "count", Expressions.call(collectionParameter, "size"), false); - Expression convertedChildExp; + + // Build source delete keys as Object[] values in table field order. + final JavaTypeFactory typeFactory = (JavaTypeFactory) getCluster().getTypeFactory(); + final JavaRowFormat tableFormat = EnumerableTableScan.deduceFormat(table); + final PhysType tablePhysType = PhysTypeImpl.of(typeFactory, table.getRowType(), tableFormat); + final PhysType sourcePhysType = result.physType; + final int fieldCount = table.getRowType().getFieldCount(); + + final ParameterExpression sourceRow = + Expressions.parameter(sourcePhysType.getJavaRowType(), "sourceRow"); + final List sourceValues = new ArrayList<>(fieldCount); + for (int i = 0; i < fieldCount; i++) { + sourceValues.add( + sourcePhysType.fieldReference(sourceRow, i, + tablePhysType.getJavaFieldType(i))); + } + final Expression deleteKeysExp = + builder.append( + "deleteKeys", + Expressions.call( + sourceExp, + BuiltInMethod.SELECT.method, + Expressions.lambda( + Function1.class, + Expressions.newArrayInit(Object.class, sourceValues), + sourceRow))); + + // Build sink key extractor by reading table fields from each sink row. + final ParameterExpression sinkRow = Expressions.parameter(Object.class, "sinkRow"); + final Expression typedSinkRow = + Expressions.convert_(sinkRow, tablePhysType.getJavaRowType()); + final List sinkValues = new ArrayList<>(fieldCount); + for (int i = 0; i < fieldCount; i++) { + sinkValues.add(tablePhysType.fieldReference(typedSinkRow, i, Object.class)); + } + final Expression sinkKeySelector = + Expressions.lambda( + Function1.class, + Expressions.newArrayInit(Object.class, sinkValues), + sinkRow); + + // Remove one sink row per matching source row, matched by field values. + builder.add( + Expressions.statement( + Expressions.call( + EnumerableTableModify.class, + "applyDeleteRowsByKey", + deleteKeysExp, + collectionParameter, + sinkKeySelector))); + + // Snapshot the size again and return (sizeBefore - sizeAfter) as the delete count. + final Expression deletedCountParameter = + builder.append( + "deletedCount", + Expressions.call(collectionParameter, "size"), + false); + + builder.add( + Expressions.return_( + null, + Expressions.call( + BuiltInMethod.SINGLETON_ENUMERABLE.method, + Expressions.convert_( + Expressions.subtract(countParameter, deletedCountParameter), + long.class)))); + + return implementor.result(physType, builder.toBlock()); + } + + /** + * Normalizes the source expression to match the table's row type by adding + * field-by-field cast projections when needed. + * + * @param builder the block builder for code generation + * @param sourceExp the source expression to normalize + * @param result the compiled result of the input relational expression + * @param operationName the name to use in builder.append (e.g., "insertRows", "deleteRows") + * @return the normalized expression, either the original sourceExp if types match + * or a new expression with field casts applied + */ + private Expression normalizeSourceExpression(BlockBuilder builder, Expression sourceExp, + Result result, String operationName) { if (!getInput().getRowType().equals(getRowType())) { - final JavaTypeFactory typeFactory = - (JavaTypeFactory) getCluster().getTypeFactory(); + // The source row type doesn't match the table's row type (e.g. types + // differ in nullability or precision), so wrap the source in a projection + // that casts each field to the exact Java type the table expects. + final JavaTypeFactory typeFactory = (JavaTypeFactory) getCluster().getTypeFactory(); final JavaRowFormat format = EnumerableTableScan.deduceFormat(table); - PhysType physType = - PhysTypeImpl.of(typeFactory, table.getRowType(), format); + PhysType tablePhysType = PhysTypeImpl.of(typeFactory, table.getRowType(), format); + + // One cast expression per field: sourceField -> tableFieldType. List expressionList = new ArrayList<>(); - final PhysType childPhysType = result.physType; - final ParameterExpression o_ = - Expressions.parameter(childPhysType.getJavaRowType(), "o"); - final int fieldCount = - childPhysType.getRowType().getFieldCount(); + final PhysType sourcePhysType = result.physType; + final ParameterExpression o_ = Expressions.parameter(sourcePhysType.getJavaRowType(), "o"); + final int fieldCount = sourcePhysType.getRowType().getFieldCount(); for (int i = 0; i < fieldCount; i++) { expressionList.add( - childPhysType.fieldReference(o_, i, physType.getJavaFieldType(i))); + sourcePhysType.fieldReference(o_, i, tablePhysType.getJavaFieldType(i))); } - convertedChildExp = - builder.append( - "convertedChild", - Expressions.call( - childExp, - BuiltInMethod.SELECT.method, - Expressions.lambda( - physType.record(expressionList), o_))); + + // normalizedExp = sourceExp.select(o -> new TableRow(cast(o.f0), cast(o.f1), ...)) + return builder.append( + operationName, + Expressions.call( + sourceExp, + BuiltInMethod.SELECT.method, + Expressions.lambda(tablePhysType.record(expressionList), o_))); } else { - convertedChildExp = childExp; - } - final Method method; - switch (getOperation()) { - case INSERT: - method = BuiltInMethod.INTO.method; - break; - case DELETE: - method = BuiltInMethod.REMOVE_ALL.method; - break; - default: - throw new AssertionError(getOperation()); + return sourceExp; } + } + + /** + * Generates code for an INSERT statement. + * + *

      All rows produced by the source are added to the backing collection and + * the number of inserted rows is returned as a single-element enumerable. + * + * @param implementor code-generation context + * @param result compiled result of the input relational expression + * @param builder block under construction + * @param sourceExp enumerable of rows to insert — the output of the input + * relational expression (VALUES, SELECT, etc.) with any + * upstream filtering or projection already applied + * @param collectionParameter the modifiable backing collection of the table + * @param physType physical type of this node's output row + */ + private Result implementInsert( + EnumerableRelImplementor implementor, + Result result, + BlockBuilder builder, + Expression sourceExp, + ParameterExpression collectionParameter, + PhysType physType) { + + // Snapshot the collection size before the insert so we can compute the + // number of rows added as (sizeAfter - sizeBefore). + final Expression countParameter = + builder.append( + "count", + Expressions.call(collectionParameter, "size"), + false); + + // Normalize the source values to match the table's row type + final Expression insertExp = + normalizeSourceExpression(builder, sourceExp, result, "insertRows"); + + // Stream all rows from insertExp into the backing collection. builder.add( Expressions.statement( Expressions.call( - convertedChildExp, method, collectionParameter))); + insertExp, BuiltInMethod.INTO.method, collectionParameter))); + + // Snapshot the size again and return (sizeAfter - sizeBefore) as the insert count. final Expression updatedCountParameter = builder.append( "updatedCount", Expressions.call(collectionParameter, "size"), false); + builder.add( Expressions.return_( null, Expressions.call( BuiltInMethod.SINGLETON_ENUMERABLE.method, Expressions.convert_( - Expressions.condition( - Expressions.greaterThanOrEqual( - updatedCountParameter, countParameter), - Expressions.subtract( - updatedCountParameter, countParameter), - Expressions.subtract( - countParameter, updatedCountParameter)), + Expressions.subtract(updatedCountParameter, countParameter), long.class)))); - final PhysType physType = - PhysTypeImpl.of( - implementor.getTypeFactory(), - getRowType(), - pref == Prefer.ARRAY - ? JavaRowFormat.ARRAY : JavaRowFormat.SCALAR); + return implementor.result(physType, builder.toBlock()); } + /** + * Builds the replacement row for an UPDATE source row. + * + * @param row source row produced by the child expression + * @param tableFieldCount number of fields in the original table row + * @param updateColumnIndices 0-based indices of the columns being updated + * @return the replacement row + */ + public static Object[] applyUpdate(Object[] row, int tableFieldCount, int[] updateColumnIndices) { + // Source row layout: [originalField_0, ..., originalField_N-1, newValue_0, ..., newValue_M-1] + // where N = tableFieldCount and M = updateColumnIndices.length. + // Copy the first N fields and overwrite the positions named in the SET clause. + final Object[] newRow = new Object[tableFieldCount]; + System.arraycopy(row, 0, newRow, 0, tableFieldCount); + for (int i = 0; i < updateColumnIndices.length; i++) { + newRow[updateColumnIndices[i]] = row[tableFieldCount + i]; + } + return newRow; + } + + /** + * Removes one sink row per source row, matching by field values in table order. Accepts a + * lambda that extracts the key from a sink row. + */ + public static void applyDeleteRowsByKey(Enumerable sourceKeys, + Collection sinkRows, Function1 sinkKeySelector) { + + // Build a map of source keys to the number of sink rows that must be removed for each. + final Map, Integer> pendingByKey = new HashMap<>(); + try (Enumerator e = sourceKeys.enumerator()) { + while (e.moveNext()) { + final List key = keyOf(e.current()); + pendingByKey.put(key, pendingByKey.getOrDefault(key, 0) + 1); + } + } + + // Iterate over sink rows and remove matching rows based on key. + for (java.util.Iterator it = sinkRows.iterator(); it.hasNext();) { + final List key = keyOf(sinkKeySelector.apply(it.next())); + final Integer pending = pendingByKey.get(key); + if (pending == null || pending == 0) { + continue; + } + + it.remove(); + + if (pending == 1) { + pendingByKey.remove(key); + } else { + pendingByKey.put(key, pending - 1); + } + } + } + + /** + * Returns an equatable key for a row. + * + * @param rowValues row values + * @return key for row + */ + private static List keyOf(Object[] rowValues) { + return Arrays.asList(Arrays.copyOf(rowValues, rowValues.length)); + } + } diff --git a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java index 81fc9d6f0414..c529c076e5f5 100644 --- a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java +++ b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java @@ -181,6 +181,8 @@ public enum BuiltInMethod { Integer.class, int.class, int.class, BigDecimal.class, RoundingMode.class), INTO(ExtendedEnumerable.class, "into", Collection.class), REMOVE_ALL(ExtendedEnumerable.class, "removeAll", Collection.class), + UPDATE(ExtendedEnumerable.class, "update", List.class, Function1.class, + Function1.class, Function1.class), SCHEMA_GET_SUB_SCHEMA(Schema.class, "getSubSchema", String.class), SCHEMA_GET_TABLE(Schema.class, "getTable", String.class), SCHEMA_PLUS_ADD_TABLE(SchemaPlus.class, "add", String.class, Table.class), diff --git a/core/src/test/java/org/apache/calcite/adapter/enumerable/EnumerableTableModifyTest.java b/core/src/test/java/org/apache/calcite/adapter/enumerable/EnumerableTableModifyTest.java new file mode 100644 index 000000000000..603d22fc7946 --- /dev/null +++ b/core/src/test/java/org/apache/calcite/adapter/enumerable/EnumerableTableModifyTest.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.adapter.enumerable; + +import org.apache.calcite.linq4j.Linq4j; + +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; + +/** Tests for {@link EnumerableTableModify} row-consumption semantics. */ +class EnumerableTableModifyTest { + + @Test void testApplyUpdateOneToOneUpdatesOnlyFirstNMatchingRows() { + final List sink = new ArrayList<>(); + sink.add(new Object[] {1, 10}); + sink.add(new Object[] {1, 10}); + sink.add(new Object[] {1, 10}); + sink.add(new Object[] {2, 20}); + + // Source row layout: [original_i, original_j, new_j]. + final List source = + Arrays.asList(new Object[] {1, 10, 100}, new Object[] {1, 10, 200}); + + final long count = + EnumerableTableModify.applyUpdateOneToOne(Linq4j.asEnumerable(source), sink, 2, + new int[] {1}); + + assertThat(count, is(2L)); + assertThat(toValueRows(sink), + is( + Arrays.asList( + Arrays.asList(1, 100), + Arrays.asList(1, 200), + Arrays.asList(1, 10), + Arrays.asList(2, 20)))); + } + + @Test void testApplyDeleteDoesNotSkipRowsWhenSourceBackedBySink() { + final List sink = new ArrayList<>(Arrays.asList(1, 1, 1)); + final List source = + Arrays.asList(new Object[] {1}, new Object[] {1}, new Object[] {1}); + + EnumerableTableModify.applyDeleteRowsByKey( + Linq4j.asEnumerable(source), sink, row -> new Object[] {row}); + + assertThat(sink, is(Collections.emptyList())); + } + + private static List> toValueRows(List rows) { + final List> valueRows = new ArrayList<>(); + for (Object[] row : rows) { + valueRows.add(Arrays.asList(row)); + } + return valueRows; + } +} diff --git a/core/src/test/java/org/apache/calcite/test/JdbcFrontLinqBackTest.java b/core/src/test/java/org/apache/calcite/test/JdbcFrontLinqBackTest.java index 95283f14d795..97ac8449af63 100644 --- a/core/src/test/java/org/apache/calcite/test/JdbcFrontLinqBackTest.java +++ b/core/src/test/java/org/apache/calcite/test/JdbcFrontLinqBackTest.java @@ -278,14 +278,14 @@ public class JdbcFrontLinqBackTest { final List employees = new ArrayList<>(); CalciteAssert.AssertThat with = mutable(employees); with.query("select * from \"foo\".\"bar\"") - .returnsUnordered( - "empid=0; deptno=0; name=first; salary=0.0; commission=null"); + .returnsUnordered("empid=0; deptno=0; name=first; salary=0.0; commission=null"); with.query("insert into \"foo\".\"bar\" select * from \"hr\".\"emps\"") .updates(4); with.query("select count(*) as c from \"foo\".\"bar\"") .returnsUnordered("C=5"); - final String deleteSql = "delete from \"foo\".\"bar\" " - + "where \"deptno\" = 10"; + with.query("select count(*) as c from \"foo\".\"bar\" where \"deptno\" = 10") + .returnsUnordered("C=3"); + final String deleteSql = "delete from \"foo\".\"bar\" where \"deptno\" = 10"; with.query(deleteSql) .updates(3); final String sql = "select \"name\", count(*) as c\n" diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/DefaultEnumerable.java b/linq4j/src/main/java/org/apache/calcite/linq4j/DefaultEnumerable.java index d859519b4178..54dcc20f41e7 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/DefaultEnumerable.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/DefaultEnumerable.java @@ -377,6 +377,15 @@ protected OrderedQueryable asOrderedQueryable() { return EnumerableDefaults.remove(getThis(), sink); } + @Override public long update( + List sink, + Function1 sinkKeySelector, + Function1 sourceKeySelector, + Function1 transform) { + return EnumerableDefaults.update(getThis(), sink, sinkKeySelector, + sourceKeySelector, transform); + } + @Override public Enumerable hashJoin( Enumerable inner, Function1 outerKeySelector, Function1 innerKeySelector, diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java b/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java index 988450df3643..ae26a602132d 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java @@ -61,6 +61,7 @@ import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; +import java.util.ListIterator; import java.util.Map; import java.util.NoSuchElementException; import java.util.Objects; @@ -4399,6 +4400,40 @@ public static > C remove( return sink; } + /** + * Default implementation of + * {@link ExtendedEnumerable#update(List, Function1, Function1, Function1)}. + * + *

      Builds a map from source-row keys to replacement rows in a single pass + * over the source, then performs a single pass over the sink, replacing + * matched rows in place. + */ + public static long update( + Enumerable source, + List sink, + Function1 sinkKeySelector, + Function1 sourceKeySelector, + Function1 sourceTransform) { + final Map updateMap = new HashMap<>(); + try (Enumerator e = source.enumerator()) { + while (e.moveNext()) { + final T row = e.current(); + updateMap.put(sourceKeySelector.apply(row), sourceTransform.apply(row)); + } + } + long updateCount = 0; + final ListIterator it = sink.listIterator(); + while (it.hasNext()) { + final T current = it.next(); + final T newRow = updateMap.get(sinkKeySelector.apply(current)); + if (newRow != null) { + it.set(newRow); + updateCount++; + } + } + return updateCount; + } + /** * Hash table with null-safe key set. * diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedEnumerable.java b/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedEnumerable.java index 160f2afa0b1e..ff28c3fe822c 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedEnumerable.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedEnumerable.java @@ -547,6 +547,32 @@ Enumerable intersect(Enumerable enumerable1, */ > C removeAll(C sink); + /** + * Updates rows of {@code sink} based on the contents of this sequence. + * + *

      For each element {@code x} of this sequence, {@code sourceKeySelector} + * computes a key, and {@code sourceTransform} computes a replacement row. + * Then for each element {@code y} of {@code sink}, {@code sinkKeySelector} + * computes a key; if it matches a key produced from this sequence, {@code y} + * is replaced (in place) with the corresponding replacement row. + * + *

      The sink is a {@link List} so that elements can be replaced + * in place while preserving order. + * + * @param sink List to be updated in place + * @param sinkKeySelector Function that extracts a key from a sink row + * @param sourceKeySelector Function that extracts a key from a source row + * @param transform Function that produces the replacement row from a + * source row + * @param Key type + * @return Number of rows replaced + */ + long update( + List sink, + Function1 sinkKeySelector, + Function1 sourceKeySelector, + Function1 transform); + /** * Correlates the elements of two sequences based on * matching keys. The default equality comparer is used to compare diff --git a/server/src/test/java/org/apache/calcite/test/ServerTest.java b/server/src/test/java/org/apache/calcite/test/ServerTest.java index 40e1430c8799..355d39de7d63 100644 --- a/server/src/test/java/org/apache/calcite/test/ServerTest.java +++ b/server/src/test/java/org/apache/calcite/test/ServerTest.java @@ -108,6 +108,159 @@ static Connection connect() throws SQLException { executor.execute((SqlTruncateTable) o, context); } + @Test void testUpdate() throws Exception { + try (Connection c = connect(); + Statement s = c.createStatement()) { + s.execute("create table t (i int not null, j int not null)"); + s.executeUpdate("insert into t values (1, 10)"); + s.executeUpdate("insert into t values (2, 20)"); + s.executeUpdate("insert into t values (3, 30)"); + + // Update one row + int count = s.executeUpdate("update t set j = 99 where i = 2"); + assertThat(count, is(1)); + + try (ResultSet r = s.executeQuery("select i, j from t order by i")) { + assertThat(r.next(), is(true)); + assertThat(r.getInt(1), is(1)); + assertThat(r.getInt(2), is(10)); + assertThat(r.next(), is(true)); + assertThat(r.getInt(1), is(2)); + assertThat(r.getInt(2), is(99)); + assertThat(r.next(), is(true)); + assertThat(r.getInt(1), is(3)); + assertThat(r.getInt(2), is(30)); + assertThat(r.next(), is(false)); + } + + // Update multiple rows + count = s.executeUpdate("update t set j = 0 where i > 1"); + assertThat(count, is(2)); + + try (ResultSet r = s.executeQuery("select sum(j) from t")) { + assertThat(r.next(), is(true)); + assertThat(r.getInt(1), is(10)); + assertThat(r.next(), is(false)); + } + + // Update zero rows (no predicate match) + count = s.executeUpdate("update t set j = 100 where i = 99"); + assertThat(count, is(0)); + } + } + + @Test void testUpdateDuplicateRows() throws Exception { + try (Connection c = connect(); + Statement s = c.createStatement()) { + s.execute("create table t (i int not null, j int not null)"); + s.executeUpdate("insert into t values (1, 10)"); + s.executeUpdate("insert into t values (1, 10)"); + s.executeUpdate("insert into t values (1, 10)"); + s.executeUpdate("insert into t values (2, 20)"); + + final int count = s.executeUpdate("update t set j = 99 where i = 1 and j = 10"); + assertThat(count, is(3)); + + try (ResultSet r = + s.executeQuery("select " + + "sum(case when i = 1 and j = 99 then 1 else 0 end), " + + "sum(case when i = 1 and j = 10 then 1 else 0 end), " + + "sum(case when i = 2 and j = 20 then 1 else 0 end) " + + "from t")) { + assertThat(r.next(), is(true)); + assertThat(r.getInt(1), is(3)); + assertThat(r.getInt(2), is(0)); + assertThat(r.getInt(3), is(1)); + assertThat(r.next(), is(false)); + } + } + } + + /** Tests that INSERT ... SELECT returns the correct row count when + * 0, 1, or multiple rows are produced by the source query. + * Exercises {@link org.apache.calcite.server.MutableArrayTable} via the + * enumerable INSERT path. */ + @Test void testInsertSelectRowCount() throws Exception { + try (Connection c = connect(); + Statement s = c.createStatement()) { + s.execute("create table src (i int not null, j int not null)"); + s.executeUpdate("insert into src values (1, 10)"); + s.executeUpdate("insert into src values (2, 20)"); + s.execute("create table dst (i int not null, j int not null)"); + + // Insert 0 rows (source query returns nothing) + int count = s.executeUpdate("insert into dst select * from src where 1 = 0"); + assertThat(count, is(0)); + + // Insert 1 row + count = s.executeUpdate("insert into dst select * from src where i = 1"); + assertThat(count, is(1)); + + // Insert multiple rows + count = s.executeUpdate("insert into dst select * from src"); + assertThat(count, is(2)); + } + } + + /** Tests that DELETE returns the correct row count when + * 0, 1, or multiple rows match the predicate. + * Exercises {@link org.apache.calcite.server.MutableArrayTable} via the + * enumerable DELETE path. */ + @Test void testDelete() throws Exception { + try (Connection c = connect(); + Statement s = c.createStatement()) { + s.execute("create table t (i int not null, j int not null)"); + s.executeUpdate("insert into t values (1, 10)"); + s.executeUpdate("insert into t values (2, 20)"); + s.executeUpdate("insert into t values (3, 30)"); + + // Delete 0 rows (no predicate match) + int count = s.executeUpdate("delete from t where i = 99"); + assertThat(count, is(0)); + + // Verify all 3 rows are still present + try (ResultSet r = s.executeQuery("select count(*) from t")) { + assertThat(r.next(), is(true)); + assertThat(r.getInt(1), is(3)); + } + + // Delete 1 row + count = s.executeUpdate("delete from t where i = 2"); + assertThat(count, is(1)); + + // Delete multiple rows (both remaining rows: i=1 and i=3) + count = s.executeUpdate("delete from t where i > 0"); + assertThat(count, is(2)); + + // Verify table is empty + try (ResultSet r = s.executeQuery("select count(*) from t")) { + assertThat(r.next(), is(true)); + assertThat(r.getInt(1), is(0)); + } + } + } + + @Test void testDeleteDuplicateRows() throws Exception { + try (Connection c = connect(); + Statement s = c.createStatement()) { + s.execute("create table t (i int not null, j int not null)"); + s.executeUpdate("insert into t values (1, 10)"); + s.executeUpdate("insert into t values (1, 10)"); + s.executeUpdate("insert into t values (1, 10)"); + s.executeUpdate("insert into t values (2, 20)"); + + final int count = s.executeUpdate("delete from t where i = 1 and j = 10"); + assertThat(count, is(3)); + + try (ResultSet r = s.executeQuery("select i, j from t")) { + assertThat(r.next(), is(true)); + assertThat(r.getInt(1), is(2)); + assertThat(r.getInt(2), is(20)); + assertThat(r.next(), is(false)); + } + } + } + @Test void testStatement() throws Exception { try (Connection c = connect(); Statement s = c.createStatement(); From 7df801d858f14abf5fa0f215ec23585aa0c812e7 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Wed, 10 Jun 2026 16:44:19 -0700 Subject: [PATCH 321/562] [CALCITE-7364] Support the syntax ROW(T.* EXCLUDE cols) for creating nested ROW values Signed-off-by: Mihai Budiu --- babel/src/main/codegen/config.fmpp | 1 - core/src/main/codegen/default_config.fmpp | 1 - core/src/main/codegen/templates/Parser.jj | 86 ++++++++++++------- .../calcite/runtime/CalciteResource.java | 6 ++ .../apache/calcite/sql/SqlStarExclude.java | 5 +- .../sql/validate/SqlValidatorImpl.java | 75 +++++++++------- .../runtime/CalciteResource.properties | 2 + .../calcite/sql/parser/CoreSqlParserTest.java | 46 ++++++++++ .../apache/calcite/test/SqlValidatorTest.java | 39 +++++++++ core/src/test/resources/sql/struct.iq | 51 +++++++++++ site/_docs/reference.md | 10 ++- 11 files changed, 256 insertions(+), 66 deletions(-) diff --git a/babel/src/main/codegen/config.fmpp b/babel/src/main/codegen/config.fmpp index 001bdf2e1034..30c2ce7d6558 100644 --- a/babel/src/main/codegen/config.fmpp +++ b/babel/src/main/codegen/config.fmpp @@ -617,7 +617,6 @@ data: { includePosixOperators: true includeParsingStringLiteralAsArrayLiteral: true includeIntervalWithoutQualifier: true - includeStarExclude: true includeSelectBy: true } } diff --git a/core/src/main/codegen/default_config.fmpp b/core/src/main/codegen/default_config.fmpp index 56d17b82798b..a2547273cb10 100644 --- a/core/src/main/codegen/default_config.fmpp +++ b/core/src/main/codegen/default_config.fmpp @@ -460,5 +460,4 @@ parser: { includeAdditionalDeclarations: false includeParsingStringLiteralAsArrayLiteral: false includeIntervalWithoutQualifier: false - includeStarExclude: false } diff --git a/core/src/main/codegen/templates/Parser.jj b/core/src/main/codegen/templates/Parser.jj index 0aacdb9ff747..d4b589d6c1a7 100644 --- a/core/src/main/codegen/templates/Parser.jj +++ b/core/src/main/codegen/templates/Parser.jj @@ -2026,7 +2026,6 @@ void AddSelectItem(List list) : ) } -<#if (parser.includeStarExclude!default.parser.includeStarExclude)> /** * Parses one unaliased expression in a select list. */ @@ -2035,6 +2034,8 @@ SqlNode SelectExpression() : SqlNode e; SqlNodeList excludeList; SqlNodeList replaceList; + SqlIdentifier sqlIdentifier; + SqlParserPos pos; } { ( @@ -2045,47 +2046,42 @@ SqlNode SelectExpression() : e = Expression(ExprContext.ACCEPT_SUB_QUERY) ) ( -<#if (parser.includeStarExclude!default.parser.includeStarExclude)> excludeList = StarExcludeList() { if (!(e instanceof SqlIdentifier)) { throw SqlUtil.newContextException(excludeList.getParserPosition(), RESOURCE.selectExcludeRequiresStar()); } - final SqlIdentifier sqlIdentifier = (SqlIdentifier) e; + sqlIdentifier = (SqlIdentifier) e; if (!sqlIdentifier.isStar()) { throw SqlUtil.newContextException(excludeList.getParserPosition(), RESOURCE.selectExcludeRequiresStar()); } - final SqlParserPos pos = SqlParserPos.sum( + pos = SqlParserPos.sum( ImmutableList.of(sqlIdentifier.getParserPosition(), excludeList.getParserPosition())); return new SqlStarExclude(pos, sqlIdentifier, excludeList); } | - -<#if (parser.includeStarExclude!default.parser.includeStarExclude)> replaceList = StarReplaceList() { if (!(e instanceof SqlIdentifier)) { throw SqlUtil.newContextException(replaceList.getParserPosition(), RESOURCE.selectReplaceRequiresStar()); } - final SqlIdentifier sqlIdentifier = (SqlIdentifier) e; + sqlIdentifier = (SqlIdentifier) e; if (!sqlIdentifier.isStar()) { throw SqlUtil.newContextException(replaceList.getParserPosition(), RESOURCE.selectReplaceRequiresStar()); } - final SqlParserPos pos = SqlParserPos.sum( + pos = SqlParserPos.sum( ImmutableList.of(sqlIdentifier.getParserPosition(), replaceList.getParserPosition())); return new SqlStarReplace(pos, sqlIdentifier, replaceList); } | - { return e; } ) } -<#if (parser.includeStarExclude!default.parser.includeStarExclude)> SqlNodeList StarExcludeList() : { final Span s; @@ -2106,9 +2102,7 @@ SqlNodeList StarExcludeList() : return new SqlNodeList(list, s.end(this)); } } - -<#if (parser.includeStarExclude!default.parser.includeStarExclude)> SqlNodeList StarReplaceList() : { final Span s; @@ -2134,25 +2128,6 @@ SqlNodeList StarReplaceList() : return new SqlNodeList(list, s.end(this)); } } - -<#else> -/** - * Parses one unaliased expression in a select list. - */ -SqlNode SelectExpression() : -{ - SqlNode e; -} -{ - { - return SqlIdentifier.star(getPos()); - } -| - e = Expression(ExprContext.ACCEPT_SUB_QUERY) { - return e; - } -} - SqlLiteral Natural() : { @@ -4548,12 +4523,42 @@ SqlCall PercentileFunctionCall() : } +/** + * Parses an EXCLUDE or EXCEPT clause following a star inside a ROW constructor, + * e.g. {@code ROW(* EXCLUDE(col1, col2))} or {@code ROW(t.* EXCEPT(t.col))}. + * + * @param starIdentifier the star (e.g. "*" or "t.*") that precedes the EXCLUDE clause + */ +SqlNode RowStarExclude(SqlIdentifier starIdentifier) : +{ + SqlIdentifier id; // current column identifier being parsed + final List list = new ArrayList(); + Span s; +} +{ + ( | ) + { s = span(); } + id = CompoundIdentifier() { list.add(id); } + ( + id = CompoundIdentifier() { list.add(id); } + )* + { + final SqlNodeList excludeList = new SqlNodeList(list, s.end(this)); + return new SqlStarExclude( + SqlParserPos.sum(ImmutableList.of( + starIdentifier.getParserPosition(), + excludeList.getParserPosition())), + starIdentifier, excludeList); + } +} + /** * Parses an atomic row expression. */ SqlNode AtomicRowExpression() : { final SqlNode e; + SqlNode rowStar; } { ( @@ -4584,10 +4589,27 @@ SqlNode AtomicRowExpression() : | e = ContextVariable() | + // Parses "t.*" or "t.* EXCLUDE (col, ...)" inside a ROW constructor e = CompoundIdentifier() + ( + LOOKAHEAD({ allowRowValueStar() + && (getToken(1).kind == EXCLUDE || getToken(1).kind == EXCEPT) }) + rowStar = RowStarExclude((SqlIdentifier) e) { return rowStar; } + | + {} + ) | + // Parses "*" or "* EXCLUDE (col, ...)" inside a ROW constructor LOOKAHEAD({ allowRowValueStar() }) - { return SqlIdentifier.star(getPos()); } + { + final SqlIdentifier starId = SqlIdentifier.star(getPos()); + } + ( + LOOKAHEAD(( | )) + rowStar = RowStarExclude(starId) { return rowStar; } + | + { return starId; } + ) | e = NewSpecification() | diff --git a/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java b/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java index a9cb02d065cb..ea60315ae6ae 100644 --- a/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java +++ b/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java @@ -816,9 +816,15 @@ ExInst illegalArgumentForTableFunctionCall(String a0, @BaseMessage("SELECT * EXCLUDE/EXCEPT list contains unknown column(s): {0}") ExInst selectStarExcludeListContainsUnknownColumns(String columns); + @BaseMessage("ROW(* EXCLUDE/EXCEPT list) contains unknown column(s): {0}") + ExInst rowStarExcludeListContainsUnknownColumns(String columns); + @BaseMessage("SELECT * EXCLUDE/EXCEPT list cannot exclude all columns") ExInst selectStarExcludeCannotExcludeAllColumns(); + @BaseMessage("ROW(* EXCLUDE/EXCEPT list) cannot exclude all columns") + ExInst rowStarExcludeCannotExcludeAllColumns(); + @BaseMessage("SELECT * REPLACE list contains unknown column(s): {0}") ExInst selectStarReplaceListContainsUnknownColumns(String columns); diff --git a/core/src/main/java/org/apache/calcite/sql/SqlStarExclude.java b/core/src/main/java/org/apache/calcite/sql/SqlStarExclude.java index 884c4a846360..fc1d7f298889 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlStarExclude.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlStarExclude.java @@ -27,11 +27,12 @@ import static java.util.Objects.requireNonNull; /** - * Represents {@code SELECT * EXCLUDE(...)}. + * Represents the arguments of {@code SELECT * EXCLUDE(...) or SELECT ROW(* EXCLUDE(...))}, + * without the SELECT itself. */ public class SqlStarExclude extends SqlCall { public static final SqlOperator OPERATOR = - new SqlSpecialOperator("SELECT_STAR_EXCLUDE", SqlKind.OTHER) { + new SqlSpecialOperator("STAR_EXCLUDE", SqlKind.OTHER) { @SuppressWarnings("argument.type.incompatible") @Override public SqlCall createCall( @Nullable SqlLiteral functionQualifier, diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java index f7388bdeb346..e2762d79264c 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java @@ -481,7 +481,7 @@ private boolean expandSelectItem(final SqlNode selectItem, SqlSelect select, } else { final SelectScope scope = (SelectScope) getWhereScope(select); if (expandStar(selectItems, aliases, fields, includeSystemVars, scope, - selectItem)) { + selectItem, false)) { return true; } @@ -670,7 +670,7 @@ private void validateNoQualifiedCommonColumns(SqlNodeList nodeList, private boolean expandStar(List selectItems, Set aliases, PairList fields, boolean includeSystemVars, - SelectScope scope, SqlNode node) { + SelectScope scope, SqlNode node, boolean inRowContext) { final SqlIdentifier identifier; final SqlNodeList excludeList; final SqlNodeList replaceList; @@ -831,9 +831,9 @@ private boolean expandStar(List selectItems, Set aliases, int offset = Math.min(calculatePermuteOffset(selectItems), originalSize); new Permute(from, offset).permute(selectItems, fields); } - throwIfUnknownExcludeColumns(excludeIdentifiers, excludeMatched); + throwIfUnknownExcludeColumns(excludeIdentifiers, excludeMatched, inRowContext); throwIfExcludeEliminatesAllColumns(excludeIdentifiers, fieldsBeforeStar, - fields, identifier); + fields, identifier, inRowContext); throwIfUnknownReplaceColumns(replaceMap, replaceMatched); return true; @@ -909,9 +909,9 @@ private boolean expandStar(List selectItems, Set aliases, } else { throw newValidationError(prefixId, RESOURCE.starRequiresRecordType()); } - throwIfUnknownExcludeColumns(excludeIdentifiers, excludeMatched); + throwIfUnknownExcludeColumns(excludeIdentifiers, excludeMatched, inRowContext); throwIfExcludeEliminatesAllColumns(excludeIdentifiers, fieldsBeforeStar, - fields, identifier); + fields, identifier, inRowContext); throwIfUnknownReplaceColumns(replaceMap, replaceMatched); return true; } @@ -987,7 +987,7 @@ && matchesExcludeIdentifier(columnId, excludeIdentifiers.get(i), nameMatcher)) { } private void throwIfUnknownExcludeColumns(List excludeIdentifiers, - boolean[] excludeMatched) { + boolean[] excludeMatched, boolean inRowContext) { if (excludeIdentifiers.isEmpty()) { return; } @@ -1002,20 +1002,24 @@ private void throwIfUnknownExcludeColumns(List excludeIdentifiers } } if (firstUnknownIndex >= 0) { + final String columns = String.join(", ", unknownExcludeNames); throw newValidationError( excludeIdentifiers.get(firstUnknownIndex), - RESOURCE.selectStarExcludeListContainsUnknownColumns( - String.join(", ", unknownExcludeNames))); + inRowContext + ? RESOURCE.rowStarExcludeListContainsUnknownColumns(columns) + : RESOURCE.selectStarExcludeListContainsUnknownColumns(columns)); } } private void throwIfExcludeEliminatesAllColumns(List excludeIdentifiers, int fieldsBeforeStar, PairList fields, - SqlIdentifier identifier) { + SqlIdentifier identifier, boolean inRowContext) { if (!excludeIdentifiers.isEmpty() && fields.size() == fieldsBeforeStar) { throw newValidationError(identifier, - RESOURCE.selectStarExcludeCannotExcludeAllColumns()); + inRowContext + ? RESOURCE.rowStarExcludeCannotExcludeAllColumns() + : RESOURCE.selectStarExcludeCannotExcludeAllColumns()); } } @@ -1113,7 +1117,8 @@ private boolean addOrExpandField(List selectItems, Set aliases, fields, includeSystemVars, scope, - starExp); + starExp, + false); return true; default: addToSelectList( @@ -7633,6 +7638,11 @@ public static boolean isAmbiguousException(Exception ex) { default: break; } + // SqlStarExclude is expanded by expandStarInRow at the ROW level; + // its exclude identifiers must not be resolved as regular column references. + if (call instanceof SqlStarExclude) { + return call; + } // Only visits arguments which are expressions. We don't want to // qualify non-expressions such as 'x' in 'empno * 5 AS x'. CallCopyingArgHandler argHandler = @@ -7663,8 +7673,9 @@ private SqlNode expandStarInRow(SqlNode node) { if (!(scope instanceof SelectScope)) { // Check if any operand is a star identifier before throwing error for (SqlNode operand : call.getOperandList()) { - if (operand instanceof SqlIdentifier - && ((SqlIdentifier) operand).isStar()) { + if (operand instanceof SqlStarExclude + || (operand instanceof SqlIdentifier + && ((SqlIdentifier) operand).isStar())) { throw validator.newValidationError(node, RESOURCE.rowStarNotAllowed()); } @@ -7675,22 +7686,28 @@ private SqlNode expandStarInRow(SqlNode node) { final List expandedOperands = new ArrayList<>(); boolean expanded = false; for (SqlNode operand : call.getOperandList()) { - if (operand instanceof SqlIdentifier) { - final SqlIdentifier identifier = (SqlIdentifier) operand; - if (identifier.isStar()) { - final boolean expandedStar = - validator.expandStar(expandedOperands, - validator.catalogReader.nameMatcher().createSet(), - PairList.of(), - false, - selectScope, - identifier); - if (!expandedStar) { - throw new AssertionError("Row star expansion failed for " + identifier); - } - expanded = true; - continue; + final SqlIdentifier starId; + if (operand instanceof SqlStarExclude) { + starId = ((SqlStarExclude) operand).getStarIdentifier(); + } else if (operand instanceof SqlIdentifier && ((SqlIdentifier) operand).isStar()) { + starId = (SqlIdentifier) operand; + } else { + starId = null; + } + if (starId != null) { + final boolean expandedStar = + validator.expandStar(expandedOperands, + validator.catalogReader.nameMatcher().createSet(), + PairList.of(), + false, + selectScope, + operand, + true); + if (!expandedStar) { + throw new AssertionError("Row star expansion failed for " + starId); } + expanded = true; + continue; } expandedOperands.add(operand); } diff --git a/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties b/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties index 056aeb7b0715..8906e451214e 100644 --- a/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties +++ b/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties @@ -269,9 +269,11 @@ SelectStarRequiresFrom=SELECT * requires a FROM clause SelectExcludeRequiresStar=EXCLUDE/EXCEPT clause must follow a STAR expression SelectReplaceRequiresStar=REPLACE clause must follow a STAR expression SelectStarExcludeListContainsUnknownColumns=SELECT * EXCLUDE/EXCEPT list contains unknown column(s): {0} +RowStarExcludeListContainsUnknownColumns=ROW(* EXCLUDE/EXCEPT list) contains unknown column(s): {0} SelectStarReplaceListContainsUnknownColumns=SELECT * REPLACE list contains unknown column(s): {0} SelectStarReplaceListContainsDuplicateColumns=SELECT * REPLACE list contains duplicate column(s): {0} SelectStarExcludeCannotExcludeAllColumns=SELECT * EXCLUDE/EXCEPT list cannot exclude all columns +RowStarExcludeCannotExcludeAllColumns=ROW(* EXCLUDE/EXCEPT list) cannot exclude all columns GroupFunctionMustAppearInGroupByClause=Group function ''{0}'' can only appear in GROUP BY clause AuxiliaryWithoutMatchingGroupCall=Call to auxiliary group function ''{0}'' must have matching call to group function ''{1}'' in GROUP BY clause PivotAggMalformed=Measure expression in PIVOT must use aggregate function diff --git a/core/src/test/java/org/apache/calcite/sql/parser/CoreSqlParserTest.java b/core/src/test/java/org/apache/calcite/sql/parser/CoreSqlParserTest.java index c8ad180009ff..1c90c9484dea 100644 --- a/core/src/test/java/org/apache/calcite/sql/parser/CoreSqlParserTest.java +++ b/core/src/test/java/org/apache/calcite/sql/parser/CoreSqlParserTest.java @@ -16,6 +16,7 @@ */ package org.apache.calcite.sql.parser; +import org.apache.calcite.avatica.util.Quoting; import org.apache.calcite.test.DiffTestCase; import com.google.common.collect.ImmutableList; @@ -70,4 +71,49 @@ public class CoreSqlParserTest extends SqlParserTest { private boolean isNotSubclass() { return this.getClass().equals(CoreSqlParserTest.class); } + + /** Test case for + * [CALCITE-7364] + * Support the syntax ROW(T.* EXCLUDE cols) for creating nested ROW values. */ + @Test void testRowStarExclude() { + // Use backticks to ensure that sql(q).same() in general + final SqlParserFixture f = fixture().withConfig(c -> c.withQuoting(Quoting.BACK_TICK)); + final String empExcludeEmpno = "SELECT (ROW(`EMP`.* EXCLUDE (`EMP`.`EMPNO`)))\n" + + "FROM `EMP`"; + + // Simple star with one excluded column + final String starExcludeEmpno = "SELECT (ROW(* EXCLUDE (`EMPNO`)))\n" + + "FROM `EMP`"; + sql("select row(* exclude(empno)) from emp").ok(starExcludeEmpno); + f.sql(starExcludeEmpno).same(); + + // Table-qualified star with excluded column + sql("select row(emp.* exclude(emp.empno)) from emp").ok(empExcludeEmpno); + f.sql(empExcludeEmpno).same(); + + // EXCEPT is normalized to EXCLUDE on unparse + sql("select row(emp.* except(emp.empno)) from emp").ok(empExcludeEmpno); + + // Multiple excluded columns + final String starExcludeEmpnoMgr = "SELECT (ROW(* EXCLUDE (`EMPNO`, `MGR`)))\n" + + "FROM `EMP`"; + sql("select row(* exclude(empno, mgr)) from emp").ok(starExcludeEmpnoMgr); + f.sql(starExcludeEmpnoMgr).same(); + + // Mixed: table-qualified star with exclude, plus plain star + final String empExcludeEmpnoDeptStar = + "SELECT (ROW(`EMP`.* EXCLUDE (`EMP`.`EMPNO`), `DEPT`.*))\n" + + "FROM `EMP`\n" + + "INNER JOIN `DEPT` ON (`EMP`.`DEPTNO` = `DEPT`.`DEPTNO`)"; + sql("select row(emp.* exclude(emp.empno), dept.*)" + + " from emp join dept on emp.deptno = dept.deptno") + .ok(empExcludeEmpnoDeptStar); + f.sql(empExcludeEmpnoDeptStar).same(); + + // Nested ROW with EXCLUDE + final String nestedStarExcludeEmpno = "SELECT (ROW((ROW(* EXCLUDE (`EMPNO`)))))\n" + + "FROM `EMP`"; + sql("select row(row(* exclude(empno))) from emp").ok(nestedStarExcludeEmpno); + f.sql(nestedStarExcludeEmpno).same(); + } } diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index c60c2917edb0..0f40dfbc6dfd 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -2156,6 +2156,45 @@ void testLikeAndSimilarFails() { sql("select row(*) from emp").ok(); sql("select row(emp.*) from emp").ok(); sql("select row(emp.*, dept.*) from emp join dept on emp.deptno = dept.deptno").ok(); + // Nested ROW with star + sql("select row(row(*)) from emp").ok(); + sql("select row(row(emp.*)) from emp").ok(); + } + + /** Test case for + * [CALCITE-7364] + * Support the syntax ROW(T.* EXCLUDE cols) for creating nested ROW values. */ + @Test void testRowWildcardExclude() { + sql("select row(* exclude(empno)) from emp").ok(); + sql("select row(* exclude(empno, deptno)) from emp").ok(); + sql("select row(emp.* exclude(emp.empno)) from emp").ok(); + // EXCEPT is a synonym for EXCLUDE + sql("select row(emp.* except(emp.empno)) from emp").ok(); + sql("select row(emp.* exclude(emp.empno), dept.*)" + + " from emp join dept on emp.deptno = dept.deptno").ok(); + // Nested ROW with EXCLUDE + sql("select row(row(* exclude(empno))) from emp").ok(); + sql("select row(row(emp.* exclude(emp.empno))) from emp").ok(); + // Multiple nested ROWs, one with EXCLUDE + sql("select row(row(* exclude(empno)), row(dept.*)) " + + "from emp join dept on emp.deptno = dept.deptno").ok(); + // Unknown column in exclude list + sql("select row(* exclude(^foo^)) from emp") + .fails("ROW\\(\\* EXCLUDE/EXCEPT list\\) contains unknown column\\(s\\): FOO"); + // Unknown column in nested ROW exclude list + sql("select row(row(* exclude(^foo^))) from emp") + .fails("ROW\\(\\* EXCLUDE/EXCEPT list\\) contains unknown column\\(s\\): FOO"); + // Unknown column in table-qualified exclude list + sql("select row(emp.* exclude(^emp.foo^)) from emp") + .fails("ROW\\(\\* EXCLUDE/EXCEPT list\\) contains unknown column\\(s\\): EMP\\.FOO"); + // Excluding all columns from a ROW expression is not allowed + sql("select row(^*^ exclude(empno, ename, job, mgr, hiredate, sal, comm," + + " deptno, slacker)) from emp") + .fails("ROW\\(\\* EXCLUDE/EXCEPT list\\) cannot exclude all columns"); + // Excluding all columns via qualified name is not allowed + sql("select row(^emp.*^ exclude(emp.empno, emp.ename, emp.job, emp.mgr," + + " emp.hiredate, emp.sal, emp.comm, emp.deptno, emp.slacker)) from emp") + .fails("ROW\\(\\* EXCLUDE/EXCEPT list\\) cannot exclude all columns"); } @Test void testRowWithValidDot() { diff --git a/core/src/test/resources/sql/struct.iq b/core/src/test/resources/sql/struct.iq index 61c892157302..1d894d254e18 100644 --- a/core/src/test/resources/sql/struct.iq +++ b/core/src/test/resources/sql/struct.iq @@ -187,4 +187,55 @@ select row(d.*, row(d.*)) from dept d limit 1; !ok +# [CALCITE-7364] Support the syntax ROW(T.* EXCLUDE cols) for creating nested ROW values +select row(* exclude(empno)) from emp order by empno limit 1; ++----------------------------------------------------+ +| EXPR$0 | ++----------------------------------------------------+ +| {SMITH, CLERK, 7902, 1980-12-17, 800.00, null, 20} | ++----------------------------------------------------+ +(1 row) + +!ok + +select row(emp.* exclude(emp.empno)) from emp order by empno limit 1; ++----------------------------------------------------+ +| EXPR$0 | ++----------------------------------------------------+ +| {SMITH, CLERK, 7902, 1980-12-17, 800.00, null, 20} | ++----------------------------------------------------+ +(1 row) + +!ok + +select row(* exclude(empno, mgr)) from emp order by empno limit 1; ++----------------------------------------------+ +| EXPR$0 | ++----------------------------------------------+ +| {SMITH, CLERK, 1980-12-17, 800.00, null, 20} | ++----------------------------------------------+ +(1 row) + +!ok + +select row(emp.* exclude(emp.empno), dept.*) from emp join dept on emp.deptno = dept.deptno order by emp.empno limit 1; ++--------------------------------------------------------------------------+ +| EXPR$0 | ++--------------------------------------------------------------------------+ +| {SMITH, CLERK, 7902, 1980-12-17, 800.00, null, 20, 20, RESEARCH, DALLAS} | ++--------------------------------------------------------------------------+ +(1 row) + +!ok + +select row(emp.* exclude(emp.empno), dept.* exclude(dept.deptno)) from emp join dept on emp.deptno = dept.deptno order by emp.empno limit 1; ++----------------------------------------------------------------------+ +| EXPR$0 | ++----------------------------------------------------------------------+ +| {SMITH, CLERK, 7902, 1980-12-17, 800.00, null, 20, RESEARCH, DALLAS} | ++----------------------------------------------------------------------+ +(1 row) + +!ok + # End struct.iq diff --git a/site/_docs/reference.md b/site/_docs/reference.md index e50acf936f1c..026fa59c4741 100644 --- a/site/_docs/reference.md +++ b/site/_docs/reference.md @@ -248,9 +248,16 @@ starWithReplace: * | * REPLACE '(' expression AS column [, expression AS column ]* ')' +rowStarItem: + * + | tableAlias . * + | * { EXCLUDE | EXCEPT } '(' column [, column ]* ')' + | tableAlias . * { EXCLUDE | EXCEPT } '(' column [, column ]* ')' + Note: -* `SELECT * EXCLUDE (...)` and `SELECT * REPLACE (...)` are recognized only when the Babel parser is enabled. `EXCLUDE` (or the alias `EXCEPT`) removes the specified columns from the star expansion; `REPLACE` substitutes the given expressions for the matching columns while keeping the original column order. For `REPLACE`, the column alias must either be a simple identifier or, for a table-qualified star such as `t.*`, a qualified identifier whose prefix matches the star's table alias. +* `EXCLUDE` (or the alias `EXCEPT`) removes the specified columns from the star expansion; `REPLACE` substitutes the given expressions for the matching columns while keeping the original column order. For `REPLACE`, the column alias must either be a simple identifier or, for a table-qualified star such as `t.*`, a qualified identifier whose prefix matches the star's table alias. +* `ROW(rowStarItem [, rowStarItem ]*)` creates a nested ROW from all columns (or all columns except the excluded ones) of one or more tables. `EXCEPT` is an alias for `EXCLUDE` in this context. projectItem: expression [ [ AS ] columnAlias ] @@ -1793,6 +1800,7 @@ Implicit type coercion of following cases are ignored: | Operator syntax | Description |:--------------- |:----------- | ROW (value [, value ]*) | Creates a row from a list of values. +| ROW (rowStarItem [, rowStarItem ]*) | Creates a row from all columns, or all columns except those excluded, of one or more tables. | (value [, value ]* ) | Creates a row from a list of values. | row '[' index ']' | Returns the element at a particular location in a row (1-based index). | row '[' name ']' | Returns the element of a row with a particular name. From a4f1ae1a7d1c78efb5c575e33a9493bb4213446c Mon Sep 17 00:00:00 2001 From: iwanttobepowerful <745778074@qq.com> Date: Fri, 12 Jun 2026 14:21:01 +0800 Subject: [PATCH 322/562] [CALCITE-7587] RelDecorrelator fails on correlated scalar subquery with ROW_NUMBER window function due to RexOver nullability mismatch --- .../calcite/sql2rel/RelDecorrelator.java | 112 ++++++++++++++++++ core/src/test/resources/sql/sub-query.iq | 33 ++++++ 2 files changed, 145 insertions(+) diff --git a/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java b/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java index f9e7049af21a..a5a817579372 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java @@ -2695,6 +2695,118 @@ private RexNode createCaseExpression( return literal; } + /** + * Decorrelates a window expression ({@link RexOver}) that may reference + * correlation variables in its {@code PARTITION BY} / {@code ORDER BY} + * keys (or the aggregate arguments). + * + *

      For each correlation field reachable from {@code over}, we: + *

        + *
      1. rewrite it to an input reference of the (already decorrelated) + * left-hand side of the surrounding correlate, and
        2. + *
      2. append that reference to the window's {@code partitionKeys} + * (if not already present), so that the window is evaluated + * independently per outer row group, matching the original + * per-correlated-row semantics.
        3. + *
      + * + *

      If the scalar sub-query has been pulled above a LEFT correlate, the + * result is wrapped in a {@code CASE} on the null-indicator so that it + * stays {@code NULL} when the right side did not match. + * + *

      Concrete example. For the SQL: + * 

      {@code
+     * SELECT e.ename,
+     *        (SELECT ROW_NUMBER() OVER (PARTITION BY e.deptno
+     *                                   ORDER BY e.empno, d.deptno)
+     *           FROM dept d WHERE e.deptno = d.deptno) AS rn
+     * FROM   emp e
+     * ORDER BY e.empno
+     * }
      + * + *

      BEFORE this method (window expression as seen on entry): + * 

      {@code
+     * ROW_NUMBER() OVER (
+     *   PARTITION BY $cor2.DEPTNO
+     *   ORDER BY    $cor2.EMPNO, $0)
+     *   partitionKeys = [$cor2.DEPTNO]
+     *   orderKeys     = [$cor2.EMPNO, $0]
+     * }
      + * + *

      AFTER this method (with {@code projectPulledAboveLeftCorrelator=true} + * and a null-indicator at column {@code $3}; correlation fields are + * rewritten to input refs from the outer side and {@code $cor2.EMPNO} is + * additionally appended to the partition keys): + * 

      {@code
+     * CASE(IS NULL($3), null:BIGINT,
+     *      CAST(
+     *        ROW_NUMBER() OVER (
+     *          PARTITION BY CASE(IS NULL($3), null:TINYINT,  CAST($2):TINYINT),
+     *                       CASE(IS NULL($3), null:SMALLINT, CAST($0):SMALLINT)
+     *          ORDER BY    CASE(IS NULL($3), null:SMALLINT, CAST($0):SMALLINT),
+     *                       $3)
+     *      ):BIGINT)
+     *   newOver.partitionKeys =
+     *     [CASE(IS NULL($3), null:TINYINT,  CAST($2):TINYINT),
+     *      CASE(IS NULL($3), null:SMALLINT, CAST($0):SMALLINT)]
+     *   newOver.orderKeys     =
+     *     [CASE(IS NULL($3), null:SMALLINT, CAST($0):SMALLINT), $3]
+     * }
      + * + *

      Note that {@code $cor2.EMPNO} only appeared in the original + * {@code ORDER BY}; without appending its decorrelated form to + * {@code partitionKeys} the rewritten window would silently widen its + * computation scope across outer rows and produce wrong results. + */ + @Override public RexNode visitOver(RexOver over) { + // Collect correlation fields that are referenced directly by the window + // expression. They need to be added to the window partition keys so that + // decorrelation does not widen the window computation scope. + final List correlationFields = new ArrayList<>(); + over.accept(new RexVisitorImpl(true) { + @Override public Void visitFieldAccess(RexFieldAccess fieldAccess) { + if (cm.mapFieldAccessToCorRef.containsKey(fieldAccess) + && !correlationFields.contains(fieldAccess)) { + correlationFields.add(fieldAccess); + } + return super.visitFieldAccess(fieldAccess); + } + }); + + RexOver newOver = (RexOver) super.visitOver(over); + if (!correlationFields.isEmpty()) { + final List partitionKeys = new ArrayList<>(newOver.getWindow().partitionKeys); + boolean update = false; + for (RexFieldAccess fieldAccess : correlationFields) { + // Rewrite the correlation field to its decorrelated input reference, + // then use it as an additional partition key for the window. + RexNode partitionKey = visitFieldAccess(fieldAccess); + if (!partitionKeys.contains(partitionKey)) { + partitionKeys.add(partitionKey); + update = true; + } + } + if (update) { + newOver = + (RexOver) rexBuilder.makeOver(newOver.getParserPosition(), + newOver.getType(), + newOver.getAggOperator(), newOver.getOperands(), partitionKeys, + newOver.getWindow().orderKeys, + newOver.getWindow().getLowerBound(), + newOver.getWindow().getUpperBound(), + newOver.getWindow().getExclude(), + newOver.getWindow().isRows(), true, false, newOver.isDistinct(), + newOver.ignoreNulls()); + } + } + if (projectPulledAboveLeftCorrelator && (nullIndicator != null)) { + // Once a scalar sub-query is pulled above a left correlate, the result + // must remain nullable when there is no matching row on the right side. + return createCaseExpression(nullIndicator, null, newOver); + } + return newOver; + } + @Override public RexNode visitCall(final RexCall call) { RexNode newCall; diff --git a/core/src/test/resources/sql/sub-query.iq b/core/src/test/resources/sql/sub-query.iq index db69258df248..4d4398cfc4f0 100644 --- a/core/src/test/resources/sql/sub-query.iq +++ b/core/src/test/resources/sql/sub-query.iq @@ -8168,6 +8168,39 @@ SELECT deptno FROM dept WHERE 1000.00 > !ok +# [CALCITE-7587] RelDecorrelator fails on correlated scalar subquery with ROW_NUMBER window function +# due to RexOver nullability mismatch +# Correlated scalar sub-query in the SELECT list that contains a window function. +# Decorrelation must add the correlation key to the window partition and preserve nullability +# of the OVER expression after pulling it above the left correlate. +SELECT e.ename, + (SELECT ROW_NUMBER() OVER (PARTITION BY e.deptno ORDER BY e.empno, d.deptno) + FROM dept d + WHERE e.deptno = d.deptno) AS rn +FROM emp e +ORDER BY e.empno; ++--------+----+ +| ENAME | RN | ++--------+----+ +| SMITH | 1 | +| ALLEN | 1 | +| WARD | 1 | +| JONES | 1 | +| MARTIN | 1 | +| BLAKE | 1 | +| CLARK | 1 | +| SCOTT | 1 | +| KING | 1 | +| TURNER | 1 | +| ADAMS | 1 | +| JAMES | 1 | +| FORD | 1 | +| MILLER | 1 | ++--------+----+ +(14 rows) + +!ok + # [CALCITE-7584] RelDecorrelator produces incorrect results for correlated LATERAL sub-queries with window functions # Correlated LATERAL sub-query with a window expression. # The equality predicate between the inner and outer query must remain applied From 0a4f7209053ce971810d883918fedcca5faebde9 Mon Sep 17 00:00:00 2001 From: xuzifu666 <1206332514@qq.com> Date: Thu, 11 Jun 2026 20:29:53 +0800 Subject: [PATCH 323/562] [CALCITE-5929] Improve LogicalWindow print plan to add the constant value --- .../org/apache/calcite/rel/core/Window.java | 88 +++++++++++++- .../rel/logical/LogicalWindowTest.java | 110 ++++++++++++++++++ .../org/apache/calcite/test/JdbcTest.java | 2 +- .../apache/calcite/test/RelOptRulesTest.xml | 14 +-- 4 files changed, 205 insertions(+), 9 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rel/core/Window.java b/core/src/main/java/org/apache/calcite/rel/core/Window.java index b444c8468f42..705e349b8465 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/Window.java +++ b/core/src/main/java/org/apache/calcite/rel/core/Window.java @@ -35,6 +35,7 @@ import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexChecker; import org.apache.calcite.rex.RexFieldCollation; +import org.apache.calcite.rex.RexInputRef; import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexLocalRef; import org.apache.calcite.rex.RexNode; @@ -162,8 +163,10 @@ public Window(RelOptCluster cluster, RelTraitSet traitSet, RelNode input, @Override public RelWriter explainTerms(RelWriter pw) { super.explainTerms(pw); + final int inputFieldCount = getInput().getRowType().getFieldCount(); for (Ord window : Ord.zip(groups)) { - pw.item("window#" + window.i, window.e.toString()); + pw.item("window#" + window.i, + window.e.computeDisplayString(constants, inputFieldCount)); } if (this.constants != null && this.constants.size() > 0) { pw.item("constants", constants); @@ -345,6 +348,89 @@ private String computeString(@UnderInitialization Group this) { return buf.toString(); } + /** Returns a display string with constant offsets in window bounds expanded + * to their values. Unlike {@link #toString()}, this is for display + * only and does not affect {@link #equals} or {@link #hashCode}. + * Constants can be literals or expressions (e.g., 5+5). */ + public String computeDisplayString(List constants, int inputFieldCount) { + final StringBuilder buf = new StringBuilder("window("); + final int i = buf.length(); + if (!keys.isEmpty()) { + buf.append("partition "); + buf.append(keys); + } + if (!orderKeys.getFieldCollations().isEmpty()) { + if (buf.length() > i) { + buf.append(' '); + } + buf.append("order by "); + buf.append(orderKeys); + } + if (orderKeys.getFieldCollations().isEmpty() + && lowerBound.isUnboundedPreceding() + && upperBound.isUnboundedFollowing()) { + // skip + } else if (!orderKeys.getFieldCollations().isEmpty() + && lowerBound.isUnboundedPreceding() + && upperBound.isCurrentRow() + && !isRows) { + // skip + } else { + if (buf.length() > i) { + buf.append(' '); + } + buf.append(isRows ? "rows " : "range "); + buf.append("between "); + buf.append(expandBound(lowerBound, constants, inputFieldCount)); + buf.append(" and "); + buf.append(expandBound(upperBound, constants, inputFieldCount)); + if (exclude != RexWindowExclusion.EXCLUDE_NO_OTHER) { + buf.append(" ").append(exclude); + } + } + if (!aggCalls.isEmpty()) { + if (buf.length() > i) { + buf.append(' '); + } + buf.append("aggs "); + buf.append(aggCalls); + } + buf.append(")"); + return buf.toString(); + } + + /** Expands a window bound by replacing RexInputRef constants with their values. + * + *

      If the bound offset is a RexInputRef pointing to a constant: + * - For RexLiteral constants, extracts the actual value (e.g., 10) + * - For other expressions, uses toString() to show the expression digest + * + *

      Examples: + * - RexInputRef(1) pointing to RexLiteral(10) → "10 PRECEDING" + * - RexInputRef(1) pointing to RexCall(+, 5, 5) → digest representation + */ + private static String expandBound(RexWindowBound bound, + List constants, int inputFieldCount) { + if (bound.isUnbounded() || bound.isCurrentRow()) { + return bound.toString(); + } + final RexNode offset = bound.getOffset(); + if (offset instanceof RexInputRef) { + final int index = ((RexInputRef) offset).getIndex(); + if (index >= inputFieldCount && index - inputFieldCount < constants.size()) { + final RexNode constant = constants.get(index - inputFieldCount); + // Constants can be literals or constant expressions (e.g., 5+5 = RexCall). + // For literals, use getValue2() to get the actual value. + // For expressions, use toString() which shows the expression digest. + final String value = (constant instanceof RexLiteral) + ? String.valueOf(((RexLiteral) constant).getValue2()) + : constant.toString(); + return value + " " + (bound.isPreceding() ? "PRECEDING" : "FOLLOWING"); + } + } + return bound.toString(); + } + @Override public boolean equals(@Nullable Object obj) { return this == obj || obj instanceof Group diff --git a/core/src/test/java/org/apache/calcite/rel/logical/LogicalWindowTest.java b/core/src/test/java/org/apache/calcite/rel/logical/LogicalWindowTest.java index 79182be959fe..726fd1cbefdd 100644 --- a/core/src/test/java/org/apache/calcite/rel/logical/LogicalWindowTest.java +++ b/core/src/test/java/org/apache/calcite/rel/logical/LogicalWindowTest.java @@ -20,6 +20,7 @@ import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.AbstractRelNode; +import org.apache.calcite.rel.RelCollations; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Window; import org.apache.calcite.rel.type.RelDataType; @@ -27,20 +28,30 @@ import org.apache.calcite.rel.type.RelDataTypeSystem; import org.apache.calcite.rel.type.RelDataTypeSystemImpl; import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexInputRef; import org.apache.calcite.rex.RexLiteral; +import org.apache.calcite.rex.RexWindowBound; +import org.apache.calcite.rex.RexWindowBounds; +import org.apache.calcite.rex.RexWindowExclusion; +import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.type.BasicSqlType; import org.apache.calcite.sql.type.SqlTypeFactoryImpl; import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.test.MockRelOptPlanner; +import org.apache.calcite.util.ImmutableBitSet; import org.junit.jupiter.api.Test; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import static org.apache.calcite.rel.core.Window.Group; import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.hasSize; import static org.junit.jupiter.api.Assertions.assertNotSame; import static org.junit.jupiter.api.Assertions.assertSame; @@ -84,4 +95,103 @@ public class LogicalWindowTest { assertThat(updated.getConstants(), hasSize(1)); assertSame(newConstants.get(0), updated.getConstants().get(0)); } + + /** Test case of + * [CALCITE-5929] + * Improve LogicalWindow print plan to add the constant value. */ + @Test void testComputeDisplayStringWithLiteralConstant() { + // Test that computeDisplayString() correctly expands literal constants + // in window bounds (e.g., "10 PRECEDING" instead of "$1 PRECEDING") + final MockRelOptPlanner planner = new MockRelOptPlanner(Contexts.empty()); + final SqlTypeFactoryImpl typeFactory = + new SqlTypeFactoryImpl(org.apache.calcite.rel.type.RelDataTypeSystem.DEFAULT); + final RexBuilder rexBuilder = new RexBuilder(typeFactory); + final RelOptCluster cluster = RelOptCluster.create(planner, rexBuilder); + final RelTraitSet traitSet = RelTraitSet.createEmpty(); + final RelNode relNode = new AbstractRelNode(cluster, traitSet) { + }; + + // Create a literal constant: 10 + final RexLiteral literalTen = + rexBuilder.makeExactLiteral(java.math.BigDecimal.TEN, + typeFactory.createSqlType(SqlTypeName.BIGINT)); + final List constants = Collections.singletonList(literalTen); + + // Create window bounds: 10 PRECEDING to CURRENT ROW + // The offset is RexInputRef(1) which maps to constants[0] = 10 + final int inputFieldCount = 1; + final RexInputRef offsetRef = new RexInputRef(inputFieldCount, literalTen.getType()); + final RexWindowBound lowerBound = RexWindowBounds.preceding(offsetRef); + + // Create a window group with this bound + final List aggCalls = new ArrayList<>(); + final Group group = + new Group(ImmutableBitSet.of(), + true, // isRows + lowerBound, + RexWindowBounds.CURRENT_ROW, + RexWindowExclusion.EXCLUDE_NO_OTHER, + RelCollations.EMPTY, + aggCalls); + + // Call computeDisplayString and verify it expands "10 PRECEDING" + final String displayString = group.computeDisplayString(constants, inputFieldCount); + assertThat(displayString, containsString("10 PRECEDING")); + assertThat(displayString, containsString("CURRENT ROW")); + } + + @Test void testComputeDisplayStringWithConstantExpression() { + // Test that computeDisplayString() correctly handles constant expressions + // (not just literals) in window bounds. For example, when a window bound + // contains RexCall representing an expression like 5+5. + final MockRelOptPlanner planner = new MockRelOptPlanner(Contexts.empty()); + final SqlTypeFactoryImpl typeFactory = + new SqlTypeFactoryImpl(org.apache.calcite.rel.type.RelDataTypeSystem.DEFAULT); + final RexBuilder rexBuilder = new RexBuilder(typeFactory); + final RelOptCluster cluster = RelOptCluster.create(planner, rexBuilder); + final RelTraitSet traitSet = RelTraitSet.createEmpty(); + final RelNode relNode = new AbstractRelNode(cluster, traitSet) { + }; + + // Create a constant expression: 5 + 5 + final RexLiteral five = + rexBuilder.makeExactLiteral(java.math.BigDecimal.valueOf(5), + typeFactory.createSqlType(SqlTypeName.BIGINT)); + final SqlOperator plusOp = SqlStdOperatorTable.PLUS; + final RexCall addExpr = + (RexCall) rexBuilder.makeCall(plusOp, five, five); + + // Test that expandBound() correctly handles both literals and expressions. + // Although the API accepts List, at runtime constants can include + // expressions like RexCall(+, 5, 5). We use an unchecked cast to simulate this. + @SuppressWarnings("unchecked") + final List constants = + (List) (List) Collections.singletonList(addExpr); + + // Create window bounds with RexInputRef pointing to this expression + final int inputFieldCount = 1; + final RexInputRef offsetRef = new RexInputRef(inputFieldCount, addExpr.getType()); + final RexWindowBound lowerBound = RexWindowBounds.preceding(offsetRef); + + // Create a window group + final List aggCalls = new ArrayList<>(); + final Group group = + new Group(ImmutableBitSet.of(), + true, // isRows + lowerBound, + RexWindowBounds.CURRENT_ROW, + RexWindowExclusion.EXCLUDE_NO_OTHER, + RelCollations.EMPTY, + aggCalls); + + // Call computeDisplayString and verify it correctly renders the expression. + // Since the constant is RexCall(+, 5, 5), expandBound() should call toString() + // on it (the non-literal branch), which returns the digest representation. + final String displayString = group.computeDisplayString(constants, inputFieldCount); + + // Verify the expression 5+5 is shown as "+(5:BIGINT, 5:BIGINT) PRECEDING", + // not as unexpanded "$1 PRECEDING" + assertThat(displayString, containsString("+(5:BIGINT, 5:BIGINT) PRECEDING")); + assertThat(displayString, containsString("CURRENT ROW")); + } } diff --git a/core/src/test/java/org/apache/calcite/test/JdbcTest.java b/core/src/test/java/org/apache/calcite/test/JdbcTest.java index f90c5525cad2..9732b10c8f1b 100644 --- a/core/src/test/java/org/apache/calcite/test/JdbcTest.java +++ b/core/src/test/java/org/apache/calcite/test/JdbcTest.java @@ -4411,7 +4411,7 @@ void testOrderByOnSortedTable2(String format) { "[deptno INTEGER NOT NULL, empid INTEGER NOT NULL, S REAL, FIVE INTEGER NOT NULL, M REAL, C BIGINT NOT NULL]") .explainContains("" + "EnumerableCalc(expr#0..7=[{inputs}], expr#8=[0:BIGINT], expr#9=[>($t4, $t8)], expr#10=[null:JavaType(class java.lang.Float)], expr#11=[CASE($t9, $t5, $t10)], expr#12=[5], deptno=[$t1], empid=[$t0], S=[$t11], FIVE=[$t12], M=[$t6], C=[$t7])\n" - + " EnumerableWindow(window#0=[window(partition {1} order by [0] rows between $4 PRECEDING and CURRENT ROW aggs [COUNT($3), $SUM0($3), MIN($2), COUNT()])], constants=[[1]])\n" + + " EnumerableWindow(window#0=[window(partition {1} order by [0] rows between 1 PRECEDING and CURRENT ROW aggs [COUNT($3), $SUM0($3), MIN($2), COUNT()])], constants=[[1]])\n" + " EnumerableCalc(expr#0..4=[{inputs}], expr#5=[+($t3, $t0)], proj#0..1=[{exprs}], salary=[$t3], $3=[$t5])\n" + " EnumerableTableScan(table=[[hr, emps]])\n") .returnsUnordered( diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index c2fb1065553e..29e539a0f7b5 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -5522,7 +5522,7 @@ ROWS BETWEEN 5 + 5 PRECEDING AND 1 PRECEDING) AS w_count from emp @@ -7476,7 +7476,7 @@ LogicalProject(EMPNO=[$0], DEPTNO=[$1], W_COUNT=[$2]) @@ -10914,17 +10914,17 @@ FROM t1]]> @@ -11978,7 +11978,7 @@ LogicalProject(EXPR$0=[CAST(/(CASE(>(COUNT($5) OVER (ORDER BY $0 ROWS 3 PRECEDIN ($2, 0), $3, null:INTEGER), $2)):INTEGER]) - LogicalWindow(window#0=[window(order by [0] rows between $2 PRECEDING and CURRENT ROW aggs [COUNT($1), $SUM0($1)])], constants=[[3]]) + LogicalWindow(window#0=[window(order by [0] rows between 3 PRECEDING and CURRENT ROW aggs [COUNT($1), $SUM0($1)])], constants=[[3]]) LogicalProject(EMPNO=[$0], SAL=[$5]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) ]]> From 84136b4278c93bf38535ef24b94ebd6fed47da50 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Thu, 11 Jun 2026 21:14:49 -0700 Subject: [PATCH 324/562] [CALCITE-7556] NameMultimap.remove(key, value) leaves an empty key bucket behind Signed-off-by: Mihai Budiu --- .../java/org/apache/calcite/util/NameMultimap.java | 9 +++++---- .../test/java/org/apache/calcite/util/UtilTest.java | 13 +++++++++++++ 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/util/NameMultimap.java b/core/src/main/java/org/apache/calcite/util/NameMultimap.java index 8d1505746eb6..d37f5366c98e 100644 --- a/core/src/main/java/org/apache/calcite/util/NameMultimap.java +++ b/core/src/main/java/org/apache/calcite/util/NameMultimap.java @@ -16,8 +16,6 @@ */ package org.apache.calcite.util; -import org.apache.calcite.linq4j.function.Experimental; - import org.checkerframework.checker.nullness.qual.Nullable; import java.util.ArrayList; @@ -71,13 +69,16 @@ public void put(String name, V v) { /** Removes all entries that have the given case-sensitive key. * * @return Whether a value was removed */ - @Experimental public boolean remove(String key, V value) { final List list = map().get(key); if (list == null) { return false; } - return list.remove(value); + boolean result = list.remove(value); + if (list.isEmpty()) { + map().remove(key); + } + return result; } /** Returns a map containing all the entries in this multimap that match the diff --git a/core/src/test/java/org/apache/calcite/util/UtilTest.java b/core/src/test/java/org/apache/calcite/util/UtilTest.java index 2d7ab0f606b5..7a6df9a5a07f 100644 --- a/core/src/test/java/org/apache/calcite/util/UtilTest.java +++ b/core/src/test/java/org/apache/calcite/util/UtilTest.java @@ -3292,4 +3292,17 @@ static String describe(Matcher m) { m.describeTo(d); return d.toString(); } + + /** Test case for [CALCITE-7556] + * NameMultimap.remove(key, value) leaves an empty key bucket behind. */ + @Test void testNameMultimapRemoveLastValueRemovesKey() { + final NameMultimap map = new NameMultimap<>(); + map.put("baz", 1); + + assertTrue(map.remove("baz", 1)); + assertThat(map.range("baz", true), hasSize(0)); + assertThat(map.containsKey("baz", true), is(false)); + assertThat(map.containsKey("BAZ", false), is(false)); + assertThat(map.map(), aMapWithSize(0)); + } } From 29430a393e15a4cf32d44708517ff4926649e0f7 Mon Sep 17 00:00:00 2001 From: zzwqqq Date: Sun, 14 Jun 2026 07:46:06 +0800 Subject: [PATCH 325/562] [CALCITE-7585] SqlMerge unparse EXISTS predicates in ON clause without parentheses --- .../org/apache/calcite/sql/SqlDelete.java | 3 +- .../java/org/apache/calcite/sql/SqlMerge.java | 5 ++- .../apache/calcite/sql/SqlSelectOperator.java | 3 +- .../org/apache/calcite/sql/SqlUpdate.java | 3 +- .../java/org/apache/calcite/sql/SqlUtil.java | 34 ++++++++----------- .../rel/rel2sql/RelToSqlConverterTest.java | 17 ++++++++++ 6 files changed, 40 insertions(+), 25 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql/SqlDelete.java b/core/src/main/java/org/apache/calcite/sql/SqlDelete.java index c9da35c48bf5..7b3cf41c5cfe 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlDelete.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlDelete.java @@ -152,7 +152,8 @@ public SqlNode getTargetTable() { } SqlNode condition = this.condition; if (condition != null) { - SqlUtil.unparseWhereClause(writer, condition, opLeft, opRight); + writer.sep("WHERE"); + SqlUtil.unparseConditionClause(writer, condition, opLeft, opRight); } writer.endList(frame); } diff --git a/core/src/main/java/org/apache/calcite/sql/SqlMerge.java b/core/src/main/java/org/apache/calcite/sql/SqlMerge.java index ddadf32959e1..df450974a56b 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlMerge.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlMerge.java @@ -197,9 +197,8 @@ public void setSourceSelect(SqlSelect sourceSelect) { writer.keyword("USING"); source.unparse(writer, opLeft, opRight); - writer.newlineAndIndent(); - writer.keyword("ON"); - condition.unparse(writer, opLeft, opRight); + writer.sep("ON"); + SqlUtil.unparseConditionClause(writer, condition, opLeft, opRight); SqlUpdate updateCall = this.updateCall; if (updateCall != null) { diff --git a/core/src/main/java/org/apache/calcite/sql/SqlSelectOperator.java b/core/src/main/java/org/apache/calcite/sql/SqlSelectOperator.java index 7e37ac732f2c..8605eef7b31d 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlSelectOperator.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlSelectOperator.java @@ -173,7 +173,8 @@ public SqlSelect createCall( SqlNode where = select.where; if (where != null) { - SqlUtil.unparseWhereClause(writer, where, 0, 0); + writer.sep("WHERE"); + SqlUtil.unparseConditionClause(writer, where, 0, 0); } if (select.groupBy != null) { SqlNodeList groupBy = diff --git a/core/src/main/java/org/apache/calcite/sql/SqlUpdate.java b/core/src/main/java/org/apache/calcite/sql/SqlUpdate.java index 22d669efcf0b..0f743540d18e 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlUpdate.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlUpdate.java @@ -202,7 +202,8 @@ public void setSourceSelect(SqlSelect sourceSelect) { writer.endList(setFrame); SqlNode condition = this.condition; if (condition != null) { - SqlUtil.unparseWhereClause(writer, condition, opLeft, opRight); + writer.sep("WHERE"); + SqlUtil.unparseConditionClause(writer, condition, opLeft, opRight); } writer.endList(frame); } diff --git a/core/src/main/java/org/apache/calcite/sql/SqlUtil.java b/core/src/main/java/org/apache/calcite/sql/SqlUtil.java index 1bafd7cfff85..b5c91a1c7ad0 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlUtil.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlUtil.java @@ -455,34 +455,30 @@ public static void unparseBinarySyntax( } /** - * Unparses a WHERE clause. + * Unparses a condition using a + * {@link SqlWriter.FrameTypeEnum#WHERE_LIST} frame, which provides + * predicate-list formatting. * - *

      Unparsing the condition in a {@link SqlWriter.FrameTypeEnum#WHERE_LIST} - * frame lets sub-queries in predicates recognize that they need - * parentheses. - * - * @param writer Writer - * @param where WHERE condition + * @param writer Writer + * @param condition Condition * @param leftPrec Left precedence * @param rightPrec Right precedence */ - public static void unparseWhereClause(SqlWriter writer, SqlNode where, - int leftPrec, int rightPrec) { - writer.sep("WHERE"); - + public static void unparseConditionClause(SqlWriter writer, + SqlNode condition, int leftPrec, int rightPrec) { if (!writer.isAlwaysUseParentheses()) { - SqlNode node = where; + SqlNode node = condition; // Decide whether to split on ORs or ANDs. - SqlBinaryOperator whereSep = SqlStdOperatorTable.AND; + SqlBinaryOperator conditionSep = SqlStdOperatorTable.AND; if ((node instanceof SqlCall) && node.getKind() == SqlKind.OR) { - whereSep = SqlStdOperatorTable.OR; + conditionSep = SqlStdOperatorTable.OR; } - // Unroll whereClause. + // Unroll condition. final List list = new ArrayList<>(0); - while (node.getKind() == whereSep.kind) { + while (node.getKind() == conditionSep.kind) { assert node instanceof SqlCall; final SqlCall call1 = (SqlCall) node; list.add(0, call1.operand(1)); @@ -490,10 +486,10 @@ public static void unparseWhereClause(SqlWriter writer, SqlNode where, } list.add(0, node); - writer.list(SqlWriter.FrameTypeEnum.WHERE_LIST, whereSep, - new SqlNodeList(list, where.getParserPosition())); + writer.list(SqlWriter.FrameTypeEnum.WHERE_LIST, conditionSep, + new SqlNodeList(list, condition.getParserPosition())); } else { - where.unparse(writer, leftPrec, rightPrec); + condition.unparse(writer, leftPrec, rightPrec); } } diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java index 97d782fcaeef..4e2825bb3b62 100644 --- a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java +++ b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java @@ -10046,6 +10046,23 @@ private void checkLiteral2(String expression, String expected) { sql(sql7) .schema(CalciteAssert.SchemaSpec.JDBC_SCOTT) .ok(expected7); + + // [CALCITE-7585] SqlMerge unparse EXISTS predicates in ON clause + // without parentheses. + final String sql8 = "merge into \"DEPT\" as \"t\"\n" + + "using \"DEPT\" as \"s\"\n" + + "on exists (select 1 from \"EMP\" as \"e\" where \"e\".\"DEPTNO\" = \"s\".\"DEPTNO\")\n" + + "when matched then\n" + + "update set \"DNAME\" = \"s\".\"DNAME\""; + final String expected8 = "MERGE INTO \"SCOTT\".\"DEPT\" AS \"DEPT0\"\n" + + "USING \"SCOTT\".\"DEPT\"\n" + + "ON EXISTS (SELECT *\n" + + "FROM \"SCOTT\".\"EMP\"\n" + + "WHERE \"DEPTNO\" = \"DEPT\".\"DEPTNO\")\n" + + "WHEN MATCHED THEN UPDATE SET \"DNAME\" = \"DEPT\".\"DNAME\""; + sql(sql8) + .schema(CalciteAssert.SchemaSpec.JDBC_SCOTT) + .ok(expected8); } /** Test case for From d3f094fa429ff433d09ca27025d407d04ba65840 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Fri, 12 Jun 2026 20:04:46 -0700 Subject: [PATCH 326/562] [CALCITE-7603] Support ROW constructors that name fields Signed-off-by: Mihai Budiu --- core/src/main/codegen/templates/Parser.jj | 123 +++++++++++++++--- .../calcite/sql/fun/SqlRowOperator.java | 89 +++++++++++-- .../calcite/sql/parser/CoreSqlParserTest.java | 46 ------- .../apache/calcite/test/SqlValidatorTest.java | 22 ++++ core/src/test/resources/sql/struct.iq | 46 +++++++ site/_docs/reference.md | 22 ++-- .../calcite/sql/parser/SqlParserTest.java | 69 ++++++++++ 7 files changed, 332 insertions(+), 85 deletions(-) diff --git a/core/src/main/codegen/templates/Parser.jj b/core/src/main/codegen/templates/Parser.jj index d4b589d6c1a7..b340415b958c 100644 --- a/core/src/main/codegen/templates/Parser.jj +++ b/core/src/main/codegen/templates/Parser.jj @@ -109,6 +109,7 @@ import org.apache.calcite.sql.SqlWithItem; import org.apache.calcite.sql.fun.SqlCase; import org.apache.calcite.sql.fun.SqlInternalOperators; import org.apache.calcite.sql.fun.SqlLibraryOperators; +import org.apache.calcite.sql.fun.SqlRowOperator; import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.fun.SqlTrimFunction; import org.apache.calcite.sql.parser.Span; @@ -2703,12 +2704,14 @@ void AddRowConstructor(List list) : /** * Parses a row constructor in the context of a VALUES expression. + * Supports optional field-name aliases: {@code ROW(expr AS fieldName, ...)}. */ SqlNode RowConstructor() : { - final SqlNodeList valueList; + SqlNodeList valueList; final SqlNode value; final Span s; + final List nameList = new ArrayList(); } { // hints are necessary here due to common LPAREN prefixes @@ -2720,15 +2723,24 @@ SqlNode RowConstructor() : { s = span(); } valueList = ParenthesizedQueryOrCommaListWithDefault(ExprContext.ACCEPT_NONCURSOR) - { s.add(this); } + + { + s.add(this); + return buildRowCall(s.end(valueList), valueList, null); + } | + // Standard forms: ROW(e1 [AS n1], e2, ...) or (e1 [AS n1], e2, ...) LOOKAHEAD(3) ( { s = span(); } | { s = Span.of(); } ) - valueList = ParenthesizedQueryOrCommaListWithDefault(ExprContext.ACCEPT_NONCURSOR) + { nameList.clear(); } + valueList = RowArgListWithParens(ExprContext.ACCEPT_NONCURSOR, nameList) + { + return buildRowCall(s.end(valueList), valueList, nameList); + } | value = Expression(ExprContext.ACCEPT_NONCURSOR) { @@ -2741,15 +2753,9 @@ SqlNode RowConstructor() : s = Span.of(value); valueList = new SqlNodeList(ImmutableList.of(value), value.getParserPosition()); + return buildRowCall(s.end(valueList), valueList, null); } ) - { - // REVIEW jvs 8-Feb-2004: Should we discriminate between scalar - // sub-queries inside of ROW and row sub-queries? The standard does, - // but the distinction seems to be purely syntactic. - return SqlStdOperatorTable.ROW.createCall(s.end(valueList), - (List) valueList); - } } /** Parses a WHERE clause for SELECT, DELETE, and UPDATE. */ @@ -4173,6 +4179,27 @@ SqlKind comp() : } } +/** + * Builds a ROW call from parsed expressions and optional field-name aliases. + * Uses the singleton ROW operator when all names are absent, and a per-instance + * SqlRowOperator (carrying the names) when any AS alias was written. + */ +JAVACODE SqlNode buildRowCall(SqlParserPos pos, SqlNodeList exprList, List nameList) { + if (nameList != null) { + for (int i = 0; i < nameList.size(); i++) { + if (nameList.get(i) != null) { + final List fieldNames = new ArrayList(); + for (int j = 0; j < nameList.size(); j++) { + SqlNode n = nameList.get(j); + fieldNames.add(n instanceof SqlLiteral ? ((SqlLiteral) n).getValueAs(String.class) : null); + } + return new SqlRowOperator("ROW", fieldNames).createCall(pos, (List) exprList.getList()); + } + } + } + return SqlStdOperatorTable.ROW.createCall(pos, (List) exprList.getList()); +} + /** * Parses a unary row expression, or a parenthesized expression of any * kind. @@ -4184,6 +4211,9 @@ SqlNode Expression3(ExprContext exprContext) : final SqlNodeList list1; final Span s; final Span rowSpan; + // Populated by RowArgListWithParens: one entry per expression, either a + // SqlLiteral string for an AS-aliased field name, or null if unnamed. + final List rowFieldNames = new ArrayList(); } { LOOKAHEAD(2) @@ -4200,7 +4230,7 @@ SqlNode Expression3(ExprContext exprContext) : s = span(); pushRowValueStar(); } - list = ParenthesizedQueryOrCommaList(exprContext) { + list = RowArgListWithParens(exprContext, rowFieldNames) { try { if (exprContext != ExprContext.ACCEPT_ALL && exprContext != ExprContext.ACCEPT_CURSOR @@ -4209,7 +4239,7 @@ SqlNode Expression3(ExprContext exprContext) : throw SqlUtil.newContextException(s.end(list), RESOURCE.illegalRowExpression()); } - return SqlStdOperatorTable.ROW.createCall(list); + return buildRowCall(list.getParserPosition(), list, rowFieldNames); } finally { popRowValueStar(); } @@ -4219,12 +4249,11 @@ SqlNode Expression3(ExprContext exprContext) : { rowSpan = span(); pushRowValueStar(); } | { rowSpan = null; } ) - list1 = ParenthesizedQueryOrCommaList(exprContext) { + list1 = RowArgListWithParens(exprContext, rowFieldNames) { try { if (rowSpan != null) { // interpret as row constructor - return SqlStdOperatorTable.ROW.createCall(rowSpan.end(list1), - (List) list1); + return buildRowCall(rowSpan.end(list1), list1, rowFieldNames); } } finally { if (rowSpan != null) { @@ -4280,10 +4309,68 @@ SqlNode Expression3(ExprContext exprContext) : return list1.get(0).clone(list1.getParserPosition()); } else { // interpret as row constructor - return SqlStdOperatorTable.ROW.createCall(span().end(list1), - (List) list1); + return buildRowCall(span().end(list1), list1, rowFieldNames); + } + } +} + +/** + * Parses a parenthesized comma list for ROW constructors. + * Supports optional field-name aliases: {@code (expr AS fieldName, ...)}. + * Populates {@code nameList} with field names (or null for unnamed fields). + * Returns a SqlNodeList of the value expressions. + */ +SqlNodeList RowArgListWithParens(ExprContext exprContext, List nameList) : +{ + SqlNode e; + SqlIdentifier alias; + final List exprList = new ArrayList(); + ExprContext firstExprContext = exprContext; + final Span s; +} +{ + + { + s = span(); + switch (exprContext) { + case ACCEPT_SUB_QUERY: + firstExprContext = ExprContext.ACCEPT_NONCURSOR; + break; + case ACCEPT_CURSOR: + firstExprContext = ExprContext.ACCEPT_ALL; + break; } } + ( + e = OrderedQueryOrExpr(firstExprContext) + ( + alias = SimpleIdentifier() + { exprList.add(e); nameList.add(SqlLiteral.createCharString(alias.getSimple(), alias.getParserPosition())); } + | + { exprList.add(e); nameList.add(null); } + ) + | + e = Default() { exprList.add(e); nameList.add(null); } + ) + ( + + { + checkNonQueryExpression(exprContext); + } + ( + e = Expression(exprContext) + ( + alias = SimpleIdentifier() + { exprList.add(e); nameList.add(SqlLiteral.createCharString(alias.getSimple(), alias.getParserPosition())); } + | + { exprList.add(e); nameList.add(null); } + ) + | + e = Default() { exprList.add(e); nameList.add(null); } + ) + )* + + { return new SqlNodeList(exprList, s.end(this)); } } /** @@ -5407,7 +5494,7 @@ SqlNode PeriodConstructor() : AddExpression(args, ExprContext.ACCEPT_SUB_QUERY) { - return SqlStdOperatorTable.ROW.createCall(s.end(this), args); + return buildRowCall(s.end(this), new SqlNodeList(args, s.end(this)), null); } } diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlRowOperator.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlRowOperator.java index b20d38176525..f73a3bceb7a3 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlRowOperator.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlRowOperator.java @@ -20,29 +20,63 @@ import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.sql.SqlCall; import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlOperatorBinding; import org.apache.calcite.sql.SqlSpecialOperator; import org.apache.calcite.sql.SqlUtil; import org.apache.calcite.sql.SqlWriter; import org.apache.calcite.sql.type.InferTypes; import org.apache.calcite.sql.type.OperandTypes; +import org.apache.calcite.sql.validate.SqlValidator; +import org.apache.calcite.sql.validate.SqlValidatorScope; +import org.apache.calcite.util.ImmutableNullableList; + +import org.checkerframework.checker.nullness.qual.Nullable; + +import java.util.List; /** - * SqlRowOperator represents the special ROW constructor. + * SqlRowOperator represents the special ROW constructor, {@code ROW(v1, v2, ...)}. * - *

      TODO: describe usage for row-value construction and row-type construction - * (SQL supports both). + *

      Fields may be given explicit names using AS aliases: + * {@code ROW(v1 AS f1, v2 AS f2, ...)}. When aliases are present, a + * per-instance operator (rather than the singleton {@link + * org.apache.calcite.sql.fun.SqlStdOperatorTable#ROW}) is used to carry the + * field names through type inference. After type inference the resulting + * {@link org.apache.calcite.rel.type.RelDataType} carries the names, so + * downstream code does not need to inspect the operator. + * + *

      When no aliases are given, field names are auto-generated + * ({@code EXPR$0}, {@code EXPR$1}, …). */ public class SqlRowOperator extends SqlSpecialOperator { - //~ Constructors ----------------------------------------------------------- + /** + * Optional explicit field names. When null, field names are auto-generated + * ({@code EXPR$0}, {@code EXPR$1}, …). Individual entries may be null to + * mix named and unnamed fields. + */ + private final @Nullable List<@Nullable String> fieldNames; + + /** Constructor for the singleton (no field-name aliases). */ public SqlRowOperator(String name) { + this(name, null); + } + + /** Constructor for a named ROW operator with explicit field-name aliases. + * Field names may be null, in which case they are auto-generated. */ + public SqlRowOperator(String name, @Nullable List<@Nullable String> fieldNames) { super(name, SqlKind.ROW, MDX_PRECEDENCE, false, null, InferTypes.RETURN_TYPE, OperandTypes.VARIADIC); + if (fieldNames == null) { + this.fieldNames = null; + } else { + this.fieldNames = ImmutableNullableList.copyOf(fieldNames); + } } //~ Methods ---------------------------------------------------------------- @@ -50,13 +84,16 @@ public SqlRowOperator(String name) { @Override public RelDataType inferReturnType( final SqlOperatorBinding opBinding) { // The type of a ROW(e1,e2) expression is a record with the types - // {e1type,e2type}. According to the standard, field names are - // implementation-defined. + // ROW(e1type,e2type). Field names come from AS aliases when present; + // otherwise they are implementation-defined. final RelDataTypeFactory typeFactory = opBinding.getTypeFactory(); final RelDataTypeFactory.Builder builder = typeFactory.builder(); - for (int index = 0; index < opBinding.getOperandCount(); index++) { - builder.add(SqlUtil.deriveAliasFromOrdinal(index), - opBinding.getOperandType(index)); + for (int i = 0; i < opBinding.getOperandCount(); i++) { + final String fieldName = + fieldNames != null && fieldNames.get(i) != null + ? fieldNames.get(i) + : SqlUtil.deriveAliasFromOrdinal(i); + builder.add(fieldName, opBinding.getOperandType(i)); } final RelDataType recordType = builder.build(); @@ -68,12 +105,44 @@ public SqlRowOperator(String name) { return typeFactory.createTypeWithNullability(recordType, nullable); } + @Override public RelDataType deriveType( + SqlValidator validator, + SqlValidatorScope scope, + SqlCall call) { + if (fieldNames == null) { + return super.deriveType(validator, scope, call); + } + // For named ROW: validate operand types without replacing this operator + // via lookupRoutine (which would substitute the singleton and lose field names). + for (SqlNode operand : call.getOperandList()) { + validator.deriveType(scope, operand); + } + return validateOperands(validator, scope, call); + } + @Override public void unparse( SqlWriter writer, SqlCall call, int leftPrec, int rightPrec) { - SqlUtil.unparseFunctionSyntax(this, writer, call, false); + if (fieldNames == null) { + SqlUtil.unparseFunctionSyntax(this, writer, call, false); + return; + } + writer.print("ROW"); + writer.setNeedWhitespace(false); + final SqlWriter.Frame frame = + writer.startList(SqlWriter.FrameTypeEnum.FUN_CALL, "(", ")"); + for (int i = 0; i < call.operandCount(); i++) { + writer.sep(","); + call.operand(i).unparse(writer, 0, 0); + final String name = fieldNames.get(i); + if (name != null) { + writer.keyword("AS"); + writer.identifier(name, true); + } + } + writer.endList(frame); } // override SqlOperator diff --git a/core/src/test/java/org/apache/calcite/sql/parser/CoreSqlParserTest.java b/core/src/test/java/org/apache/calcite/sql/parser/CoreSqlParserTest.java index 1c90c9484dea..c8ad180009ff 100644 --- a/core/src/test/java/org/apache/calcite/sql/parser/CoreSqlParserTest.java +++ b/core/src/test/java/org/apache/calcite/sql/parser/CoreSqlParserTest.java @@ -16,7 +16,6 @@ */ package org.apache.calcite.sql.parser; -import org.apache.calcite.avatica.util.Quoting; import org.apache.calcite.test.DiffTestCase; import com.google.common.collect.ImmutableList; @@ -71,49 +70,4 @@ public class CoreSqlParserTest extends SqlParserTest { private boolean isNotSubclass() { return this.getClass().equals(CoreSqlParserTest.class); } - - /** Test case for - * [CALCITE-7364] - * Support the syntax ROW(T.* EXCLUDE cols) for creating nested ROW values. */ - @Test void testRowStarExclude() { - // Use backticks to ensure that sql(q).same() in general - final SqlParserFixture f = fixture().withConfig(c -> c.withQuoting(Quoting.BACK_TICK)); - final String empExcludeEmpno = "SELECT (ROW(`EMP`.* EXCLUDE (`EMP`.`EMPNO`)))\n" - + "FROM `EMP`"; - - // Simple star with one excluded column - final String starExcludeEmpno = "SELECT (ROW(* EXCLUDE (`EMPNO`)))\n" - + "FROM `EMP`"; - sql("select row(* exclude(empno)) from emp").ok(starExcludeEmpno); - f.sql(starExcludeEmpno).same(); - - // Table-qualified star with excluded column - sql("select row(emp.* exclude(emp.empno)) from emp").ok(empExcludeEmpno); - f.sql(empExcludeEmpno).same(); - - // EXCEPT is normalized to EXCLUDE on unparse - sql("select row(emp.* except(emp.empno)) from emp").ok(empExcludeEmpno); - - // Multiple excluded columns - final String starExcludeEmpnoMgr = "SELECT (ROW(* EXCLUDE (`EMPNO`, `MGR`)))\n" - + "FROM `EMP`"; - sql("select row(* exclude(empno, mgr)) from emp").ok(starExcludeEmpnoMgr); - f.sql(starExcludeEmpnoMgr).same(); - - // Mixed: table-qualified star with exclude, plus plain star - final String empExcludeEmpnoDeptStar = - "SELECT (ROW(`EMP`.* EXCLUDE (`EMP`.`EMPNO`), `DEPT`.*))\n" - + "FROM `EMP`\n" - + "INNER JOIN `DEPT` ON (`EMP`.`DEPTNO` = `DEPT`.`DEPTNO`)"; - sql("select row(emp.* exclude(emp.empno), dept.*)" - + " from emp join dept on emp.deptno = dept.deptno") - .ok(empExcludeEmpnoDeptStar); - f.sql(empExcludeEmpnoDeptStar).same(); - - // Nested ROW with EXCLUDE - final String nestedStarExcludeEmpno = "SELECT (ROW((ROW(* EXCLUDE (`EMPNO`)))))\n" - + "FROM `EMP`"; - sql("select row(row(* exclude(empno))) from emp").ok(nestedStarExcludeEmpno); - f.sql(nestedStarExcludeEmpno).same(); - } } diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index 0f40dfbc6dfd..f473cf60db37 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -2197,6 +2197,28 @@ void testLikeAndSimilarFails() { .fails("ROW\\(\\* EXCLUDE/EXCEPT list\\) cannot exclude all columns"); } + /** Test case for + * [CALCITE-7603] + * Support ROW constructors that name fields. */ + @Test void testRowWithFieldNames() { + // All fields named: the returned type should use the specified names + sql("select row(1 as a, 'hello' as b) from emp") + .columnType("RecordType(INTEGER NOT NULL A, CHAR(5) NOT NULL B) NOT NULL"); + // Mixed: named and unnamed fields + sql("select row(empno as eno, ename) from emp") + .columnType("RecordType(INTEGER NOT NULL ENO, VARCHAR(20) NOT NULL EXPR$1) NOT NULL"); + // No names: existing auto-generated behavior unchanged + sql("select row(empno, ename) from emp") + .columnType("RecordType(INTEGER NOT NULL EXPR$0, VARCHAR(20) NOT NULL EXPR$1) NOT NULL"); + // Access a named field by name + sql("select row(empno as eno, ename as en).eno from emp") + .columnType("INTEGER NOT NULL"); + // Nested ROW with named fields + sql("select row(row(1 as x, 2 as y) as inner_row) from emp") + .columnType("RecordType(RecordType(INTEGER NOT NULL X, INTEGER NOT NULL Y)" + + " NOT NULL INNER_ROW) NOT NULL"); + } + @Test void testRowWithValidDot() { sql("select ((1,2),(3,4,5)).\"EXPR$1\".\"EXPR$2\"\n from dept") .columnType("INTEGER NOT NULL"); diff --git a/core/src/test/resources/sql/struct.iq b/core/src/test/resources/sql/struct.iq index 1d894d254e18..198ec492ae1e 100644 --- a/core/src/test/resources/sql/struct.iq +++ b/core/src/test/resources/sql/struct.iq @@ -238,4 +238,50 @@ select row(emp.* exclude(emp.empno), dept.* exclude(dept.deptno)) from emp join !ok +# [CALCITE-7603] Support ROW constructors that name fields +select row(1 as a, 'hello' as b); ++------------+ +| EXPR$0 | ++------------+ +| {1, hello} | ++------------+ +(1 row) + +!ok + +# Named field access: .field selects a field from a named-field ROW +select row(1 as a, 'hello' as b).a; ++--------+ +| EXPR$0 | ++--------+ +| 1 | ++--------+ +(1 row) + +!ok + +# Named field access on a column expression +select row(empno as eno, ename as en).eno from emp order by empno limit 3; ++--------+ +| EXPR$0 | ++--------+ +| 7369 | +| 7499 | +| 7521 | ++--------+ +(3 rows) + +!ok + +# Nested named-field ROW, and access to inner field +select row(row(1 as x, 2 as y) as inner_row, 'hello' as name).inner_row.x; ++--------+ +| EXPR$0 | ++--------+ +| 1 | ++--------+ +(1 row) + +!ok + # End struct.iq diff --git a/site/_docs/reference.md b/site/_docs/reference.md index 026fa59c4741..e546757d0bce 100644 --- a/site/_docs/reference.md +++ b/site/_docs/reference.md @@ -1797,17 +1797,17 @@ Implicit type coercion of following cases are ignored: ### Value constructors -| Operator syntax | Description -|:--------------- |:----------- -| ROW (value [, value ]*) | Creates a row from a list of values. -| ROW (rowStarItem [, rowStarItem ]*) | Creates a row from all columns, or all columns except those excluded, of one or more tables. -| (value [, value ]* ) | Creates a row from a list of values. -| row '[' index ']' | Returns the element at a particular location in a row (1-based index). -| row '[' name ']' | Returns the element of a row with a particular name. -| map '[' key ']' | Returns the element of a map with a particular key. -| array '[' index ']' | Returns the element at a particular location in an array (1-based index). -| ARRAY '[' value [, value ]* ']' | Creates an array from a list of values. -| MAP '[' key, value [, key, value ]* ']' | Creates a map from a list of key-value pairs. +| Operator syntax | Description +|:-------------------------------------------|:----------- +| ROW (value [AS name] [, value [AS name]]*) | Creates a row from a list of values. +| ROW (rowStarItem [, rowStarItem ]*) | Creates a row from all columns, or all columns except those excluded, of one or more tables. +| (value [, value ]* ) | Creates a row from a list of values. +| row '[' index ']' | Returns the element at a particular location in a row (1-based index). +| row '[' name ']' | Returns the element of a row with a particular name. +| map '[' key ']' | Returns the element of a map with a particular key. +| array '[' index ']' | Returns the element at a particular location in an array (1-based index). +| ARRAY '[' value [, value ]* ']' | Creates an array from a list of values. +| MAP '[' key, value [, key, value ]* ']' | Creates a map from a list of key-value pairs. ### Value constructors by query diff --git a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java index 4ab6f776ba4a..a9e3c6b95943 100644 --- a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java +++ b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java @@ -10242,4 +10242,73 @@ public void checkExpFails(String sql, String expected) { .fails(expected.replace("$op", op)); } } + + /** Test case for + * [CALCITE-7603] + * Support ROW constructors that name fields. */ + @Test void testRowWithFieldNames() { + // All fields named + sql("select row(1 as a, 'hello' as b) from emp") + .ok("SELECT (ROW(1 AS `A`, 'hello' AS `B`))\nFROM `EMP`"); + // Mixed: some fields named, some not + sql("select row(1 as a, 2) from emp") + .ok("SELECT (ROW(1 AS `A`, 2))\nFROM `EMP`"); + // No field names (existing behavior unchanged) + sql("select row(1, 2) from emp") + .ok("SELECT (ROW(1, 2))\nFROM `EMP`"); + // Expression with AS + sql("select row(empno + 1 as eno, ename as en) from emp") + .ok("SELECT (ROW((`EMPNO` + 1) AS `ENO`, `ENAME` AS `EN`))\nFROM `EMP`"); + // Round-trip: the canonical form can be re-parsed + final SqlParserFixture f = fixture().withConfig(c -> c.withQuoting(Quoting.BACK_TICK)); + f.sql("SELECT (ROW(1 AS `A`, 2))\nFROM `EMP`").same(); + // AS is not optional in ROW constructors + sql("select row(1 ^a^, 'hello' b) from emp") + .fails("(?s)Encountered \"a\" at line 1, column 14.*"); + } + + /** Test case for + * [CALCITE-7364] + * Support the syntax ROW(T.* EXCLUDE cols) for creating nested ROW values. */ + @Test void testRowStarExclude() { + // Use backticks to ensure that sql(q).same() in general + final SqlParserFixture f = fixture().withConfig(c -> c.withQuoting(Quoting.BACK_TICK)); + final String empExcludeEmpno = "SELECT (ROW(`EMP`.* EXCLUDE (`EMP`.`EMPNO`)))\n" + + "FROM `EMP`"; + + // Simple star with one excluded column + final String starExcludeEmpno = "SELECT (ROW(* EXCLUDE (`EMPNO`)))\n" + + "FROM `EMP`"; + sql("select row(* exclude(empno)) from emp").ok(starExcludeEmpno); + f.sql(starExcludeEmpno).same(); + + // Table-qualified star with excluded column + sql("select row(emp.* exclude(emp.empno)) from emp").ok(empExcludeEmpno); + f.sql(empExcludeEmpno).same(); + + // EXCEPT is normalized to EXCLUDE on unparse + sql("select row(emp.* except(emp.empno)) from emp").ok(empExcludeEmpno); + + // Multiple excluded columns + final String starExcludeEmpnoMgr = "SELECT (ROW(* EXCLUDE (`EMPNO`, `MGR`)))\n" + + "FROM `EMP`"; + sql("select row(* exclude(empno, mgr)) from emp").ok(starExcludeEmpnoMgr); + f.sql(starExcludeEmpnoMgr).same(); + + // Mixed: table-qualified star with exclude, plus plain star + final String empExcludeEmpnoDeptStar = + "SELECT (ROW(`EMP`.* EXCLUDE (`EMP`.`EMPNO`), `DEPT`.*))\n" + + "FROM `EMP`\n" + + "INNER JOIN `DEPT` ON (`EMP`.`DEPTNO` = `DEPT`.`DEPTNO`)"; + sql("select row(emp.* exclude(emp.empno), dept.*)" + + " from emp join dept on emp.deptno = dept.deptno") + .ok(empExcludeEmpnoDeptStar); + f.sql(empExcludeEmpnoDeptStar).same(); + + // Nested ROW with EXCLUDE + final String nestedStarExcludeEmpno = "SELECT (ROW((ROW(* EXCLUDE (`EMPNO`)))))\n" + + "FROM `EMP`"; + sql("select row(row(* exclude(empno))) from emp").ok(nestedStarExcludeEmpno); + f.sql(nestedStarExcludeEmpno).same(); + } } From 11141c743a2f4e3566ac95e99ca0f62f4b83cfd6 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Mon, 15 Jun 2026 21:53:22 -0700 Subject: [PATCH 327/562] [CALCITE-7606] ROW field names should be used to infer column names Signed-off-by: Mihai Budiu --- .../sql/validate/SqlValidatorUtil.java | 4 +++ .../apache/calcite/test/SqlValidatorTest.java | 2 +- .../calcite/test/SqlToRelConverterTest.xml | 12 +++---- core/src/test/resources/sql/struct.iq | 35 ++++++++++--------- 4 files changed, 29 insertions(+), 24 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorUtil.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorUtil.java index f05c58a8c7f3..1c5bfcf9a6c0 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorUtil.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorUtil.java @@ -366,6 +366,10 @@ public static String alias(SqlNode node, int ordinal) { // E.g. "foo.bar" --> "bar" return Util.last(((SqlIdentifier) node).names); + case DOT: + // E.g. "row(a as x).x" --> "x" + return alias_(((SqlCall) node).operand(1), ordinal); + default: if (ordinal < 0) { return null; diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index f473cf60db37..0fd03a456b12 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -14266,7 +14266,7 @@ private static String missingFilters(String... args) { @Test void testAccessingNestedFieldsOfNullableRecord() { sql("select ROW_COLUMN_ARRAY[0].NOT_NULL_FIELD from NULLABLEROWS.NR_T1") .withExtendedCatalog() - .type("RecordType(BIGINT EXPR$0) NOT NULL"); + .type("RecordType(BIGINT NOT_NULL_FIELD) NOT NULL"); sql("select ROW_COLUMN_ARRAY[0]['NOT_NULL_FIELD'] from NULLABLEROWS.NR_T1") .withExtendedCatalog() .type("RecordType(BIGINT EXPR$0) NOT NULL"); diff --git a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml index 0bf249f3afc4..192a0140daed 100644 --- a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml @@ -553,7 +553,7 @@ LogicalProject(EXPR$0=[ROW(ITEM(ITEM(ITEM(ITEM($3, 0), 'detail'), 'skills'), 0). @@ -2030,7 +2030,7 @@ LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$ @@ -2041,7 +2041,7 @@ LogicalProject(EXPR$0=[$1.CITY]) @@ -2052,7 +2052,7 @@ LogicalProject(EXPR$0=[ROW(ROW(1, 2), ROW(3, 4, 5)).EXPR$1.EXPR$2]) @@ -4112,7 +4112,7 @@ natural join customer.contact_peek t2]]> Date: Mon, 15 Jun 2026 15:05:40 -0700 Subject: [PATCH 328/562] [CALCITE-7602] ROW(*) loses column names Signed-off-by: Mihai Budiu --- .../calcite/sql/fun/SqlRowOperator.java | 8 +- .../sql/validate/SqlValidatorImpl.java | 39 ++++++++- .../apache/calcite/test/SqlValidatorTest.java | 86 +++++++++++++++++++ core/src/test/resources/sql/struct.iq | 52 +++++++++++ 4 files changed, 180 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlRowOperator.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlRowOperator.java index f73a3bceb7a3..eaa77f03d55e 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlRowOperator.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlRowOperator.java @@ -63,9 +63,15 @@ public SqlRowOperator(String name) { this(name, null); } + /** Returns the explicit field-name aliases, or null if none were specified + * (in which case names are auto-generated as {@code EXPR$0}, etc.). */ + public @Nullable List<@Nullable String> getFieldNames() { + return fieldNames; + } + /** Constructor for a named ROW operator with explicit field-name aliases. * Field names may be null, in which case they are auto-generated. */ - public SqlRowOperator(String name, @Nullable List<@Nullable String> fieldNames) { + public SqlRowOperator(String name, @Nullable List fieldNames) { super(name, SqlKind.ROW, MDX_PRECEDENCE, false, diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java index e2762d79264c..12b1b2733570 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java @@ -95,6 +95,7 @@ import org.apache.calcite.sql.TableCharacteristic; import org.apache.calcite.sql.fun.SqlCase; import org.apache.calcite.sql.fun.SqlInternalOperators; +import org.apache.calcite.sql.fun.SqlRowOperator; import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.sql.type.AssignableOperandTypeChecker; @@ -7684,7 +7685,16 @@ private SqlNode expandStarInRow(SqlNode node) { } final SelectScope selectScope = (SelectScope) scope; final List expandedOperands = new ArrayList<>(); + final List<@Nullable String> expandedNames = new ArrayList<>(); boolean expanded = false; + + // Retrieve field names stored in the operator (from ROW(v AS name, ...) syntax). + final @Nullable List<@Nullable String> origFieldNames = + call.getOperator() instanceof SqlRowOperator + ? ((SqlRowOperator) call.getOperator()).getFieldNames() + : null; + + int origIdx = 0; for (SqlNode operand : call.getOperandList()) { final SqlIdentifier starId; if (operand instanceof SqlStarExclude) { @@ -7695,6 +7705,7 @@ private SqlNode expandStarInRow(SqlNode node) { starId = null; } if (starId != null) { + final int sizeBefore = expandedOperands.size(); final boolean expandedStar = validator.expandStar(expandedOperands, validator.catalogReader.nameMatcher().createSet(), @@ -7706,16 +7717,36 @@ private SqlNode expandStarInRow(SqlNode node) { if (!expandedStar) { throw new AssertionError("Row star expansion failed for " + starId); } + // Each newly added operand is a SqlIdentifier; its last name component + // is the original column name. + for (int i = sizeBefore; i < expandedOperands.size(); i++) { + expandedNames.add(SqlValidatorUtil.alias(expandedOperands.get(i))); + } expanded = true; - continue; + } else { + expandedOperands.add(operand); + // Prefer the name from the original named-ROW operator; fall back to + // whatever alias can be derived from the operand expression itself. + final @Nullable String name = + origFieldNames != null && origIdx < origFieldNames.size() + && origFieldNames.get(origIdx) != null + ? origFieldNames.get(origIdx) + : SqlValidatorUtil.alias(operand); + expandedNames.add(name); } - expandedOperands.add(operand); + origIdx++; } if (!expanded) { return node; } - return SqlStdOperatorTable.ROW.createCall( - call.getParserPosition(), expandedOperands); + // Assign unique names to all fields. The first occurrence of a name keeps + // it as-is; subsequent duplicates get a compiler-generated suffix based on + // the original column name (e.g. a second DEPTNO becomes DEPTNO0). + final boolean caseSensitive = validator.catalogReader.nameMatcher().isCaseSensitive(); + final List uniqueNames = + SqlValidatorUtil.uniquify(expandedNames, SqlValidatorUtil.EXPR_SUGGESTER, caseSensitive); + return new SqlRowOperator("ROW", uniqueNames) + .createCall(call.getParserPosition(), expandedOperands); } protected SqlNode expandDynamicStar(SqlIdentifier id, SqlIdentifier fqId) { diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index 0fd03a456b12..0e9c533dbd4f 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -2197,6 +2197,76 @@ void testLikeAndSimilarFails() { .fails("ROW\\(\\* EXCLUDE/EXCEPT list\\) cannot exclude all columns"); } + /** Test case for + * [CALCITE-7602] + * ROW(*) loses column names. */ + @Test void testRowWildcardColumnNames() { + // ROW(*) should produce a struct with the original column names + sql("select row(*) from emp") + .columnType(EMP_RECORD_TYPE); + // ROW(T.*) should also preserve column names + sql("select row(emp.*) from emp") + .columnType(EMP_RECORD_TYPE); + // ROW(col, T.*) where the star repeats a column already named: the duplicate + // gets a compiler-assigned suffix (EMPNO0) rather than losing all names + sql("select row(empno, emp.*) from emp") + .columnType("RecordType(INTEGER NOT NULL EMPNO," + + " INTEGER NOT NULL EMPNO0," + + " VARCHAR(20) NOT NULL ENAME," + + " VARCHAR(10) NOT NULL JOB," + + " INTEGER MGR," + + " TIMESTAMP(0) NOT NULL HIREDATE," + + " INTEGER NOT NULL SAL," + + " INTEGER NOT NULL COMM," + + " INTEGER NOT NULL DEPTNO," + + " BOOLEAN NOT NULL SLACKER) NOT NULL"); + // ROW(T1.*, T2.*) with a shared column name (DEPTNO): the second DEPTNO + // gets the suffix DEPTNO0 while all other names are preserved + sql("select row(emp.*, dept.*) from emp join dept on emp.deptno = dept.deptno") + .columnType("RecordType(INTEGER NOT NULL EMPNO," + + " VARCHAR(20) NOT NULL ENAME," + + " VARCHAR(10) NOT NULL JOB," + + " INTEGER MGR," + + " TIMESTAMP(0) NOT NULL HIREDATE," + + " INTEGER NOT NULL SAL," + + " INTEGER NOT NULL COMM," + + " INTEGER NOT NULL DEPTNO," + + " BOOLEAN NOT NULL SLACKER," + + " INTEGER NOT NULL DEPTNO0," + + " VARCHAR(10) NOT NULL NAME) NOT NULL"); + // ROW(* EXCLUDE(col)) should preserve names of the remaining columns + sql("select row(* exclude(empno)) from emp") + .columnType("RecordType(VARCHAR(20) NOT NULL ENAME," + + " VARCHAR(10) NOT NULL JOB," + + " INTEGER MGR," + + " TIMESTAMP(0) NOT NULL HIREDATE," + + " INTEGER NOT NULL SAL," + + " INTEGER NOT NULL COMM," + + " INTEGER NOT NULL DEPTNO," + + " BOOLEAN NOT NULL SLACKER) NOT NULL"); + // ROW(T.* EXCLUDE(T.col)) should preserve names of the remaining columns + sql("select row(emp.* exclude(emp.empno)) from emp") + .columnType("RecordType(VARCHAR(20) NOT NULL ENAME," + + " VARCHAR(10) NOT NULL JOB," + + " INTEGER MGR," + + " TIMESTAMP(0) NOT NULL HIREDATE," + + " INTEGER NOT NULL SAL," + + " INTEGER NOT NULL COMM," + + " INTEGER NOT NULL DEPTNO," + + " BOOLEAN NOT NULL SLACKER) NOT NULL"); + // Named field combined with star expansion: explicit name takes precedence + sql("select row(empno as eno, emp.* exclude(emp.empno)) from emp") + .columnType("RecordType(INTEGER NOT NULL ENO," + + " VARCHAR(20) NOT NULL ENAME," + + " VARCHAR(10) NOT NULL JOB," + + " INTEGER MGR," + + " TIMESTAMP(0) NOT NULL HIREDATE," + + " INTEGER NOT NULL SAL," + + " INTEGER NOT NULL COMM," + + " INTEGER NOT NULL DEPTNO," + + " BOOLEAN NOT NULL SLACKER) NOT NULL"); + } + /** Test case for * [CALCITE-7603] * Support ROW constructors that name fields. */ @@ -2226,6 +2296,22 @@ void testLikeAndSimilarFails() { .columnType("INTEGER NOT NULL"); sql("select t.a.\"EXPR$1\" from (select row(1,2) as a from (values (1))) as t") .columnType("INTEGER NOT NULL"); + // After star expansion the ROW carries real column names, so field access + // by original name works without quoting EXPR$N ordinals. + sql("select row(*).empno from emp") + .columnType("INTEGER NOT NULL"); + sql("select row(emp.*).ename from emp") + .columnType("VARCHAR(20) NOT NULL"); + sql("select row(* exclude(empno)).ename from emp") + .columnType("VARCHAR(20) NOT NULL"); + // When two stars produce a duplicate name the second gets a suffix; + // both the original and the suffixed name are accessible. + sql("select row(emp.*, dept.*).deptno from emp" + + " join dept on emp.deptno = dept.deptno") + .columnType("INTEGER NOT NULL"); + sql("select row(emp.*, dept.*).deptno0 from emp" + + " join dept on emp.deptno = dept.deptno") + .columnType("INTEGER NOT NULL"); } @Test void testRowWithInvalidDotOperation() { diff --git a/core/src/test/resources/sql/struct.iq b/core/src/test/resources/sql/struct.iq index 9f2f3939376e..bf25d4cadb28 100644 --- a/core/src/test/resources/sql/struct.iq +++ b/core/src/test/resources/sql/struct.iq @@ -285,4 +285,56 @@ select row(row(1 as x, 2 as y) as inner_row, 'hello' as name).inner_row.x; !ok +# [CALCITE-7602] ROW(*), ROW(T.*) and ROW(T.* EXCLUDE(cols)) preserve original column names + +# Field access by original column name on ROW(*) +select row(*).empno from emp order by empno limit 3; ++-------+ +| EMPNO | ++-------+ +| 7369 | +| 7499 | +| 7521 | ++-------+ +(3 rows) + +!ok + +# Field access by original column name on ROW(T.*) +select row(emp.*).ename from emp order by empno limit 3; ++-------+ +| ENAME | ++-------+ +| SMITH | +| ALLEN | +| WARD | ++-------+ +(3 rows) + +!ok + +# Field access on ROW(* EXCLUDE(col)) using a preserved name +select row(* exclude(empno)).ename from emp order by empno limit 3; ++-------+ +| ENAME | ++-------+ +| SMITH | +| ALLEN | +| WARD | ++-------+ +(3 rows) + +!ok + +# ROW(T1.*, T2.*): shared column DEPTNO is renamed to DEPTNO0 for the second table +select row(emp.*, dept.*).deptno0 from emp join dept on emp.deptno = dept.deptno order by emp.empno limit 1; ++---------+ +| DEPTNO0 | ++---------+ +| 20 | ++---------+ +(1 row) + +!ok + # End struct.iq From 267ab1515252741bcda472c08d207c2fb57db22f Mon Sep 17 00:00:00 2001 From: Tisya Bhatia Date: Mon, 8 Jun 2026 15:34:35 -0500 Subject: [PATCH 329/562] [CALCITE-7594] Support GROUP BY ALL When GROUP BY ALL appears with no grouping items, group by every expression in the SELECT clause that does not contain an aggregate or window function and is not a measure. Parser forks the ALL branch on LOOKAHEAD(2); a trailing grouping list keeps the existing CALCITE-5089 ALL/DISTINCT set-quantifier behaviour; nothing trailing emits a GROUP_BY_ALL marker that the validator expands before base group validation. Constants are kept as grouping keys (grouping only by constants over empty input returns 0 rows, matching standard SQL and major dialects); SELECT * is rejected. Includes parser, validator, and execution tests plus reference docs. --- core/src/main/codegen/templates/Parser.jj | 41 +++++++++++++------ .../calcite/runtime/CalciteResource.java | 3 ++ .../java/org/apache/calcite/sql/SqlKind.java | 3 ++ .../apache/calcite/sql/SqlSelectOperator.java | 4 ++ .../calcite/sql/fun/SqlInternalOperators.java | 5 +++ .../sql/validate/SqlValidatorImpl.java | 27 ++++++++++++ .../runtime/CalciteResource.properties | 1 + .../org/apache/calcite/test/JdbcTest.java | 11 +++++ .../apache/calcite/test/SqlValidatorTest.java | 30 ++++++++++++++ site/_docs/reference.md | 9 +++- .../calcite/sql/parser/SqlParserTest.java | 16 ++++++++ 11 files changed, 136 insertions(+), 14 deletions(-) diff --git a/core/src/main/codegen/templates/Parser.jj b/core/src/main/codegen/templates/Parser.jj index b340415b958c..4bbf49840c1f 100644 --- a/core/src/main/codegen/templates/Parser.jj +++ b/core/src/main/codegen/templates/Parser.jj @@ -2773,25 +2773,42 @@ SqlNode Where() : SqlNodeList GroupBy() : { final List list; - final boolean distinct; final Span s; + SqlParserPos pos; } { { s = span(); } ( - { distinct = true; } - | { distinct = false; } - | { distinct = false; } + + list = GroupingElementList() { + pos = s.end(this); + return new SqlNodeList( + ImmutableList.of( + SqlInternalOperators.GROUP_BY_DISTINCT.createCall(pos, list)), pos + ); + } + | + + ( + LOOKAHEAD(2) + list = GroupingElementList() { + return new SqlNodeList(list, s.end(this)); + } + | + { + pos = s.end(this); + return new SqlNodeList( + ImmutableList.of( + SqlInternalOperators.GROUP_BY_ALL.createCall(pos)), pos + ); + } + ) + | + list = GroupingElementList() { + return new SqlNodeList(list, s.end(this)); + } ) - list = GroupingElementList() { - final SqlParserPos pos = s.end(this); - final List list2 = distinct - ? ImmutableList.of( - SqlInternalOperators.GROUP_BY_DISTINCT.createCall(pos, list)) - : list; - return new SqlNodeList(list2, pos); - } } List GroupingElementList() : diff --git a/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java b/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java index ea60315ae6ae..13745bf0dfdb 100644 --- a/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java +++ b/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java @@ -383,6 +383,9 @@ ExInst naturalOrUsingColumnNotCompatible(String a0, @BaseMessage("Windowed aggregate expression is illegal in {0} clause") ExInst windowedAggregateIllegalInClause(String a0); + @BaseMessage("GROUP BY ALL requires an explicit SELECT list; ''*'' is not supported") + ExInst groupByAllRequiresExplicitSelectList(); + @BaseMessage("Aggregate expressions cannot be nested") ExInst nestedAggIllegal(); diff --git a/core/src/main/java/org/apache/calcite/sql/SqlKind.java b/core/src/main/java/org/apache/calcite/sql/SqlKind.java index 40111fe0afcb..680c833c1721 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlKind.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlKind.java @@ -183,6 +183,9 @@ public enum SqlKind { /** The DISTINCT keyword of the GROUP BY clause. */ GROUP_BY_DISTINCT, + /** The ALL keyword of the GROUP BY clause. */ + GROUP_BY_ALL, + /** * ORDER BY clause. * diff --git a/core/src/main/java/org/apache/calcite/sql/SqlSelectOperator.java b/core/src/main/java/org/apache/calcite/sql/SqlSelectOperator.java index 8605eef7b31d..fea1c2235fe9 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlSelectOperator.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlSelectOperator.java @@ -186,6 +186,10 @@ public SqlSelect createCall( writer.sep("GROUP BY DISTINCT"); List operandList = ((SqlCall) groupBy.get(0)).getOperandList(); groupBy = new SqlNodeList(operandList, groupBy.getParserPosition()); + } else if (groupBy.size() == 1 && groupBy.get(0) != null + && groupBy.get(0).getKind() == SqlKind.GROUP_BY_ALL) { + writer.sep("GROUP BY ALL"); + groupBy = new SqlNodeList(groupBy.getParserPosition()); } else { writer.sep("GROUP BY"); } diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlInternalOperators.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlInternalOperators.java index dd1098fe4fce..8757e1e17753 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlInternalOperators.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlInternalOperators.java @@ -203,6 +203,11 @@ private SqlInternalOperators() { public static final SqlInternalOperator GROUP_BY_DISTINCT = new SqlRollupOperator("GROUP BY DISTINCT", SqlKind.GROUP_BY_DISTINCT); + /** {@code GROUP BY ALL}, a placeholder expanded during validation into + * a standard {@code GROUP BY}. */ + public static final SqlInternalOperator GROUP_BY_ALL = + new SqlRollupOperator("GROUP BY ALL", SqlKind.GROUP_BY_ALL); + /** Fetch operator is ONLY used for its precedence during unparsing. */ public static final SqlOperator FETCH = SqlBasicOperator.create("FETCH") diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java index 12b1b2733570..7989b7e7d830 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java @@ -5274,6 +5274,7 @@ private static SqlNode measureToValue(SqlNode e) { * called even if no GROUP BY clause is present. */ protected void validateGroupClause(SqlSelect select) { + rewriteGroupByAll(select); SqlNodeList groupList = select.getGroup(); if (groupList == null) { return; @@ -5345,6 +5346,32 @@ protected void validateGroupClause(SqlSelect select) { } } + /** If GROUP BY clause is the {@code GROUP BY ALL} placeholder, replaces it + * with every non-aggregated expression from the SELECT clause. */ + private void rewriteGroupByAll(SqlSelect select) { + final SqlNodeList groupList = select.getGroup(); + if (groupList == null + || groupList.size() != 1 + || groupList.get(0).getKind() != SqlKind.GROUP_BY_ALL) { + return; + } + final List keys = new ArrayList<>(); + for (SqlNode selectItem : select.getSelectList()) { + if (SqlValidatorUtil.isMeasure(selectItem)) { + continue; + } + final SqlNode expr = SqlUtil.stripAs(selectItem); + if (expr instanceof SqlIdentifier && ((SqlIdentifier) expr).isStar()) { + throw newValidationError(expr, + RESOURCE.groupByAllRequiresExplicitSelectList()); + } + if (aggOrOverFinder.findAgg(expr) == null) { + keys.add(expr); + } + } + select.setGroupBy(new SqlNodeList(keys, groupList.getParserPosition())); + } + private void validateGroupItem(SqlValidatorScope groupScope, @Nullable AggregatingSelectScope aggregatingScope, SqlNode groupItem) { diff --git a/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties b/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties index 8906e451214e..325b69d87235 100644 --- a/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties +++ b/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties @@ -131,6 +131,7 @@ GroupingInWrongClause={0} operator may only occur in SELECT, HAVING or ORDER BY NotSelectDistinctExpr=Expression ''{0}'' is not in the select clause AggregateIllegalInClause=Aggregate expression is illegal in {0} clause WindowedAggregateIllegalInClause=Windowed aggregate expression is illegal in {0} clause +GroupByAllRequiresExplicitSelectList=GROUP BY ALL requires an explicit SELECT list; ''*'' is not supported NestedAggIllegal=Aggregate expressions cannot be nested MeasureIllegal=Measure expressions can only occur within AGGREGATE function MeasureMustBeInAggregateQuery=Measure expressions can only occur within a GROUP BY query diff --git a/core/src/test/java/org/apache/calcite/test/JdbcTest.java b/core/src/test/java/org/apache/calcite/test/JdbcTest.java index 9732b10c8f1b..ee1afe05cbbc 100644 --- a/core/src/test/java/org/apache/calcite/test/JdbcTest.java +++ b/core/src/test/java/org/apache/calcite/test/JdbcTest.java @@ -1297,6 +1297,17 @@ private void checkResultSetMetaData(Connection connection, String sql) + "c0=1998\n"); } + /** Test case for [CALCITE-7594] GROUP BY ALL: grouping only by a constant + * over empty input returns 0 rows. */ + @Test void testGroupByAllOverEmptyInput() { + CalciteAssert.hr() + .query("select 'x', count(*)\n" + + "from \"hr\".\"emps\"\n" + + "where false\n" + + "group by all") + .returnsCount(0); + } + /** Test case for * [CALCITE-2894] * NullPointerException thrown by RelMdPercentageOriginalRows when explaining diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index 0e9c533dbd4f..283ea802aa24 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -7615,6 +7615,36 @@ public boolean isBangEqualAllowed() { .withConformance(lenient).ok(); } + @Test void testGroupByAll() { + // expands to every non-aggregate SELECT item - so listing a column besides + // an aggregate validates WITHOUT a "not being grouped" error + sql("select deptno, count(*) from emp group by all").ok(); + + // only aggregates -> global aggregation (one group), still valid + sql("select count(*) from emp group by all").ok(); + + // SELECT * cannot be expanded at group-validation time -> clear error + sql("select ^*^ from emp group by all") + .fails("(?s).*GROUP BY ALL requires an explicit SELECT list.*"); + + // contains-an-aggregate + sql("select deptno, substring(job, 1), count(*) + 1 as c, 'x' as x\n" + + "from emp group by all") + .ok(); + + // GROUP BY ALL collects the non-aggregate exprs (sal, x + 1) as + // group keys and "x" still resolves, so the two features coexist. + sql("select sal as x, x + 1 as y, count(*) from emp group by all") + .withValidatorIdentifierExpansion(true) + .withConformance(SqlConformanceEnum.BABEL) + .ok(); + + // GROUP BY ALL is unaffected by isGroupByAlias + sql("select deptno as d, count(*) from emp group by all") + .withConformance(SqlConformanceEnum.LENIENT) + .ok(); + } + /** Test case for * [CALCITE-5507] * HAVING alias failed when aggregate function in condition. */ diff --git a/site/_docs/reference.md b/site/_docs/reference.md index e546757d0bce..454d13456a7e 100644 --- a/site/_docs/reference.md +++ b/site/_docs/reference.md @@ -210,7 +210,7 @@ select: [ BY expression [, expression ]* ] FROM tableExpression [ WHERE booleanExpression ] - [ GROUP BY [ ALL | DISTINCT ] { groupItem [, groupItem ]* } ] + [ GROUP BY { ALL | [ALL | DISTINCT ] groupItem [, groupiTEM ]* } ] [ HAVING booleanExpression ] [ WINDOW windowName AS windowSpec [, windowName AS windowSpec ]* ] [ QUALIFY booleanExpression ] @@ -429,7 +429,12 @@ may refer to tables in the FROM clause of an enclosing query. GROUP BY DISTINCT removes duplicate grouping sets (for example, "GROUP BY DISTINCT GROUPING SETS ((a), (a, b), (a))" is equivalent to "GROUP BY GROUPING SETS ((a), (a, b))"); -GROUP BY ALL is equivalent to GROUP BY. +GROUP BY ALL followed by grouping items is equivalent to GROUP BY +(ALL is the default set quantifier). +GROUP BY ALL on its own groups by every expression in the SELECT clause +that is not an aggregate function; for example, +"SELECT deptno, SUM(sal) FROM emp GROUP BY ALL" is equivalent to +"SELECT deptno, SUM(sal) FROM emp GROUP BY deptno". *selectWithoutFrom* is equivalent to VALUES, but is not standard SQL and is only allowed in certain diff --git a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java index a9e3c6b95943..5c24c0a806db 100644 --- a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java +++ b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java @@ -2637,6 +2637,22 @@ void checkPeriodPredicate(Checker checker) { sql(sql2).ok(expected2); } + @Test void testGroupByAll() { + final String sql = "select x, sum(y) from t\n" + + "group by all"; + final String expected = "SELECT `X`, SUM(`Y`)\n" + + "FROM `T`\n" + + "GROUP BY ALL"; + sql(sql).ok(expected); + + final String sql1 = "select deptno from emp\n" + + "group by all deptno, gender"; + final String expected1 = "SELECT `DEPTNO`\n" + + "FROM `EMP`\n" + + "GROUP BY `DEPTNO`, `GENDER`"; + sql(sql1).ok(expected1); + } + @Test void testGroupByCube2() { final String sql = "select deptno from emp\n" + "group by cube ((a, b), (c, d)) order by a"; From 72f9c44d24930a50bf609b751378e5ea5cea39ee Mon Sep 17 00:00:00 2001 From: xuzifu666 <1206332514@qq.com> Date: Tue, 9 Jun 2026 18:19:50 +0800 Subject: [PATCH 330/562] [CALCITE-7595] Support FILTER clause with window functions --- .../adapter/enumerable/EnumerableWindow.java | 10 +- .../adapter/enumerable/RexImpTable.java | 20 +++ .../apache/calcite/sql/SqlOverOperator.java | 19 ++- .../apache/calcite/test/SqlValidatorTest.java | 24 +++- core/src/test/resources/sql/winagg.iq | 114 ++++++++++++++++++ site/_docs/reference.md | 11 ++ 6 files changed, 192 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java index 6197420b05ea..78ecce8821d7 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java @@ -451,6 +451,7 @@ private static void sampleOfTheGeneratedWindowedAggregate() { hasRows, frameRowCount, partitionRowCount, jDecl, inputPhysTypeFinal); + final RelDataType inputRowType = inputPhysType.getRowType(); final Function> rexArguments = agg -> { List argList = agg.call.getArgList(); List inputTypes = @@ -464,7 +465,7 @@ private static void sampleOfTheGeneratedWindowedAggregate() { return args; }; - implementAdd(aggs, builder7, resultContextBuilder, rexArguments, jDecl); + implementAdd(aggs, builder7, resultContextBuilder, rexArguments, jDecl, inputRowType); BlockStatement forBlock = builder7.toBlock(); // Don't run the aggregate function if current row is excluded @@ -866,7 +867,8 @@ private static void implementAdd(List aggs, final BlockBuilder builder7, final Function frame, final Function> rexArguments, - final DeclarationStatement jDecl) { + final DeclarationStatement jDecl, + final RelDataType inputRowType) { for (final AggImpState agg : aggs) { final WinAggAddContext addContext = new WinAggAddContextImpl(builder7, requireNonNull(agg.state, "agg.state"), frame) { @@ -879,7 +881,9 @@ private static void implementAdd(List aggs, } @Override public @Nullable RexNode rexFilterArgument() { - return null; // REVIEW + return agg.call.filterArg < 0 + ? null + : RexInputRef.of(agg.call.filterArg, inputRowType); } }; agg.implementor.implementAdd(requireNonNull(agg.context, "agg.context"), addContext); diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java index 549bddbf724d..c78336e172c5 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java @@ -417,6 +417,7 @@ import static org.apache.calcite.sql.fun.SqlStdOperatorTable.EVERY; import static org.apache.calcite.sql.fun.SqlStdOperatorTable.EXP; import static org.apache.calcite.sql.fun.SqlStdOperatorTable.EXTRACT; +import static org.apache.calcite.sql.fun.SqlStdOperatorTable.FILTER; import static org.apache.calcite.sql.fun.SqlStdOperatorTable.FIRST_VALUE; import static org.apache.calcite.sql.fun.SqlStdOperatorTable.FLOOR; import static org.apache.calcite.sql.fun.SqlStdOperatorTable.FUSION; @@ -1250,6 +1251,11 @@ void populate2() { NotJsonImplementor.of( new MethodImplementor(BuiltInMethod.IS_JSON_SCALAR.method, NullPolicy.NONE, false))); + // Generates conditional expressions for aggregate FILTER clause: + // e.g. SUM(salary) FILTER (WHERE dept='Sales') → condition ? sum : NULL + // FilterImplementor is used for all FILTER operations, but the restriction only affects + // OVER clause, because normal aggregates never pass through SqlOverOperator. + define(FILTER, new FilterImplementor()); } /** Third step of population. */ @@ -5111,4 +5117,18 @@ private static class ReplaceImplementor extends AbstractRexCallImplementor { operand0, operand1, operand2, Expressions.constant(isCaseSensitive)); } } + + /** Implementor for the FILTER operator. */ + private static class FilterImplementor extends AbstractRexCallImplementor { + FilterImplementor() { + super("filter", NullPolicy.NONE, false); + } + + @Override Expression implementSafe(RexToLixTranslator translator, RexCall call, + List argValueList) { + final Expression value = argValueList.get(0); + final Expression condition = argValueList.get(1); + return Expressions.condition(condition, value, NULL_EXPR); + } + } } diff --git a/core/src/main/java/org/apache/calcite/sql/SqlOverOperator.java b/core/src/main/java/org/apache/calcite/sql/SqlOverOperator.java index cc42a9ad0c41..b1ed13808dac 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlOverOperator.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlOverOperator.java @@ -64,9 +64,14 @@ public SqlOverOperator() { assert call.getOperator() == this; assert call.operandCount() == 2; SqlCall aggCall = call.operand(0); + boolean hasFilter = false; switch (aggCall.getKind()) { case RESPECT_NULLS: case IGNORE_NULLS: + case FILTER: + if (aggCall.getKind() == SqlKind.FILTER) { + hasFilter = true; + } validator.validateCall(aggCall, scope); aggCall = aggCall.operand(0); break; @@ -76,6 +81,11 @@ public SqlOverOperator() { if (!aggCall.getOperator().isAggregator()) { throw validator.newValidationError(aggCall, RESOURCE.overNonAggregate()); } + // COUNT(DISTINCT) is not allowed in window functions with FILTER + if (hasFilter && aggCall.getKind() == SqlKind.COUNT + && aggCall.getFunctionQuantifier() != null) { + throw validator.newValidationError(aggCall, RESOURCE.overNonAggregate()); + } final SqlNode window = call.operand(1); validator.validateWindow(window, scope, aggCall); } @@ -102,7 +112,14 @@ public SqlOverOperator() { SqlNode window = call.operand(1); SqlWindow w = validator.resolveWindow(window, scope); - final SqlCall aggCall = (SqlCall) agg; + SqlCall aggCall = (SqlCall) agg; + // Unwrap FILTER, RESPECT_NULLS, or IGNORE_NULLS to get the actual aggregate call + while (aggCall != null + && (aggCall.getKind() == SqlKind.FILTER + || aggCall.getKind() == SqlKind.RESPECT_NULLS + || aggCall.getKind() == SqlKind.IGNORE_NULLS)) { + aggCall = aggCall.operand(0); + } SqlCallBinding opBinding = new SqlCallBinding(validator, scope, aggCall) { @Override public boolean hasEmptyGroup() { diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index 283ea802aa24..9bdd3177ff29 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -3675,13 +3675,33 @@ void testWinPartClause() { * Validator rejects FILTER in OVER windows. */ @Test void testOverFilter() { winSql("SELECT deptno,\n" - + " ^COUNT(DISTINCT deptno) FILTER (WHERE deptno > 10)^\n" + + " ^COUNT(DISTINCT deptno)^ FILTER (WHERE deptno > 10)\n" + "OVER win AS agg\n" + "FROM emp\n" - + "WINDOW win AS (PARTITION BY empno)") + + "WINDOW win AS (PARTITION BY empno)") .fails("OVER must be applied to aggregate function"); } + /** Test case for [CALCITE-7595] + * Support FILTER clause with window functions. */ + @Test void testFilterWithOver() { + winSql("SELECT SUM(sal) FILTER (WHERE sal > 100) OVER (PARTITION BY deptno) FROM emp") + .ok(); + } + + @Test void testFilterWithOverAndDistinct() { + winSql("SELECT SUM(DISTINCT sal) FILTER (WHERE sal > 100) OVER (ORDER BY deptno) FROM emp") + .ok(); + } + + @Test void testMultipleFiltersWithOver() { + winSql("SELECT " + + "COUNT(*) FILTER (WHERE empno > 100) OVER (PARTITION BY deptno), " + + "SUM(sal) FILTER (WHERE sal > 0) OVER (PARTITION BY deptno) " + + "FROM emp") + .ok(); + } + @Test void testOverInOrderBy() { winSql("select sum(deptno) over ^(order by sum(deptno)\n" + "over(order by deptno))^ from emp") diff --git a/core/src/test/resources/sql/winagg.iq b/core/src/test/resources/sql/winagg.iq index 6a32b3b3f7b0..d8348da422a2 100644 --- a/core/src/test/resources/sql/winagg.iq +++ b/core/src/test/resources/sql/winagg.iq @@ -1173,4 +1173,118 @@ order by 1; (14 rows) !ok + +# [CALCITE-6442] Support FILTER clause with window functions +# The following 4 tests are related to this issue. +# Results were validated on Postgres. + +# Test 1: FILTER with OVER on COUNT +select empno, deptno, + count(*) filter (where sal > 1500) over (partition by deptno) as filtered_count +from emp +order by empno; ++-------+--------+----------------+ +| EMPNO | DEPTNO | FILTERED_COUNT | ++-------+--------+----------------+ +| 7369 | 20 | 0 | +| 7566 | 20 | 5 | +| 7788 | 20 | 5 | +| 7876 | 20 | 0 | +| 7902 | 20 | 5 | +| 7782 | 10 | 3 | +| 7839 | 10 | 3 | +| 7934 | 10 | 0 | +| 7499 | 30 | 6 | +| 7521 | 30 | 0 | +| 7654 | 30 | 0 | +| 7698 | 30 | 6 | +| 7844 | 30 | 0 | +| 7900 | 30 | 0 | ++-------+--------+----------------+ +(14 rows) + +!ok + +# Test 2: FILTER with OVER on SUM +select empno, deptno, + sum(sal) filter (where comm is not null) over (partition by deptno) as filtered_sum +from emp +order by empno; ++-------+--------+--------------+ +| EMPNO | DEPTNO | FILTERED_SUM | ++-------+--------+--------------+ +| 7369 | 20 | | +| 7566 | 20 | | +| 7788 | 20 | | +| 7876 | 20 | | +| 7902 | 20 | | +| 7782 | 10 | | +| 7839 | 10 | | +| 7934 | 10 | | +| 7499 | 30 | 9400.00 | +| 7521 | 30 | 9400.00 | +| 7654 | 30 | 9400.00 | +| 7698 | 30 | | +| 7844 | 30 | 9400.00 | +| 7900 | 30 | | ++-------+--------+--------------+ +(14 rows) + +!ok + +# Test 3: Multiple FILTER with OVER on different aggregates +select empno, deptno, + count(*) filter (where sal > 1500) over (partition by deptno) as high_sal_count, + sum(sal) filter (where sal <= 1500) over (partition by deptno) as low_sal_sum +from emp +order by empno; ++-------+--------+----------------+-------------+ +| EMPNO | DEPTNO | HIGH_SAL_COUNT | LOW_SAL_SUM | ++-------+--------+----------------+-------------+ +| 7369 | 20 | 0 | 10875.00 | +| 7566 | 20 | 5 | | +| 7788 | 20 | 5 | | +| 7876 | 20 | 0 | 10875.00 | +| 7902 | 20 | 5 | | +| 7782 | 10 | 3 | | +| 7839 | 10 | 3 | | +| 7934 | 10 | 0 | 8750.00 | +| 7499 | 30 | 6 | | +| 7521 | 30 | 0 | 9400.00 | +| 7654 | 30 | 0 | 9400.00 | +| 7698 | 30 | 6 | | +| 7844 | 30 | 0 | 9400.00 | +| 7900 | 30 | 0 | 9400.00 | ++-------+--------+----------------+-------------+ +(14 rows) + +!ok + +# Test 4: FILTER with OVER and ORDER BY (running window) +select empno, deptno, sal, + sum(sal) filter (where sal > 1000) over (partition by deptno order by empno rows between unbounded preceding and current row) as running_sum +from emp +order by empno; ++-------+--------+---------+-------------+ +| EMPNO | DEPTNO | SAL | RUNNING_SUM | ++-------+--------+---------+-------------+ +| 7369 | 20 | 800.00 | | +| 7566 | 20 | 2975.00 | 3775.00 | +| 7788 | 20 | 3000.00 | 6775.00 | +| 7876 | 20 | 1100.00 | 7875.00 | +| 7902 | 20 | 3000.00 | 10875.00 | +| 7782 | 10 | 2450.00 | 2450.00 | +| 7839 | 10 | 5000.00 | 7450.00 | +| 7934 | 10 | 1300.00 | 8750.00 | +| 7499 | 30 | 1600.00 | 1600.00 | +| 7521 | 30 | 1250.00 | 2850.00 | +| 7654 | 30 | 1250.00 | 4100.00 | +| 7698 | 30 | 2850.00 | 6950.00 | +| 7844 | 30 | 1500.00 | 8450.00 | +| 7900 | 30 | 950.00 | | ++-------+--------+---------+-------------+ +(14 rows) + +!ok + # End winagg.iq diff --git a/site/_docs/reference.md b/site/_docs/reference.md index 454d13456a7e..80337bbfb125 100644 --- a/site/_docs/reference.md +++ b/site/_docs/reference.md @@ -2121,9 +2121,11 @@ Syntax: windowedAggregateCall: agg '(' [ ALL | DISTINCT ] value [, value ]* ')' [ RESPECT NULLS | IGNORE NULLS ] + [ FILTER '(' WHERE condition ')' ] [ WITHIN GROUP '(' ORDER BY orderItem [, orderItem ]* ')' ] OVER window | agg '(' '*' ')' + [ FILTER '(' WHERE condition ')' ] OVER window {% endhighlight %} @@ -2139,6 +2141,15 @@ The *exclude* clause can be one of: `DISTINCT`, `FILTER` and `WITHIN GROUP` are as described for aggregate functions. +#### FILTER clause in window functions + +When `FILTER` is used with window functions, it is applied in the following order: + +1. Define window rows by `PARTITION BY` and `ORDER BY` +2. Apply `ROWS`/`RANGE` bounds to determine the window frame +3. Apply the `FILTER` condition to rows within that frame +4. Calculate the aggregate function on the filtered rows + | Operator syntax | Description |:----------------------------------------- |:----------- | COUNT(value [, value ]*) OVER window | Returns the number of rows in *window* for which *value* is not null (wholly not null if *value* is composite) From 30f6b0949b26ac7c0e48f4efb70acacbb8d645fc Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Tue, 16 Jun 2026 15:34:57 -0700 Subject: [PATCH 331/562] [CALCITE-2659] Inefficient plan In natural left/right join Signed-off-by: Mihai Budiu --- .../sql/validate/SqlValidatorImpl.java | 26 ++++++++++++++----- .../calcite/test/SqlToRelConverterTest.xml | 21 +++++++-------- 2 files changed, 29 insertions(+), 18 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java index 7989b7e7d830..1e2b29880864 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java @@ -7843,17 +7843,29 @@ private SqlNode expandExprFromJoin(SqlJoin join, SqlIdentifier identifier, Selec assert qualifiedNode.size() == 2; - final SqlCall coalesceCall = - SqlStdOperatorTable.COALESCE.createCall(SqlParserPos.ZERO, qualifiedNode.get(0), - qualifiedNode.get(1)); + // COALESCE is only needed for FULL JOIN; for RIGHT JOIN use the right + // column (always non-null), and for INNER/LEFT JOIN use the left column. + final JoinType joinType = join.getJoinType(); + final SqlNode colRef; + if (joinType.generatesNullsOnLeft() && joinType.generatesNullsOnRight()) { + colRef = + SqlStdOperatorTable.COALESCE.createCall( + qualifiedNode.get(0).getParserPosition() + .plus(qualifiedNode.get(1).getParserPosition()), + qualifiedNode.get(0), qualifiedNode.get(1)); + } else if (joinType.generatesNullsOnLeft()) { + colRef = qualifiedNode.get(1); + } else { + colRef = qualifiedNode.get(0); + } - // If there is an alias for the column, no need to wrap the coalesce with an AS operator + // If there is an alias for the column, no need to wrap with an AS operator boolean haveAlias = fieldAliases.contains(name); if (haveAlias) { - return coalesceCall; + return colRef; } else { - return SqlStdOperatorTable.AS.createCall(SqlParserPos.ZERO, coalesceCall, - new SqlIdentifier(identifier.getSimple(), SqlParserPos.ZERO)); + return SqlStdOperatorTable.AS.createCall(SqlParserPos.ZERO, colRef, + new SqlIdentifier(identifier.getSimple(), identifier.getParserPosition())); } } } diff --git a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml index 192a0140daed..0bf02c9a73b4 100644 --- a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml @@ -4042,12 +4042,11 @@ group by deptno]]> @@ -4063,7 +4062,7 @@ group by grouping sets ((deptno), (deptno, job))]]> Date: Sat, 6 Jun 2026 09:50:36 +0800 Subject: [PATCH 332/562] Add test case for CALCITE-709 --- .../calcite/sql2rel/RelDecorrelatorTest.java | 68 +++++++++++++++++++ core/src/test/resources/sql/sub-query.iq | 12 ++++ 2 files changed, 80 insertions(+) diff --git a/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java b/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java index 245b248157b9..240f9a36ae02 100644 --- a/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java +++ b/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java @@ -1429,6 +1429,74 @@ public static Frameworks.ConfigBuilder config() { assertThat(after, hasTree(planAfter)); } + /** + * Test case for + * [CALCITE-709] + * LIMIT inside scalar sub-query. + */ + @Test void test709() { + final FrameworkConfig frameworkConfig = config().build(); + final RelBuilder builder = RelBuilder.create(frameworkConfig); + final RelOptCluster cluster = builder.getCluster(); + final Planner planner = Frameworks.getPlanner(frameworkConfig); + final String sql = "" + + "SELECT E1.DEPTNO\n" + + "FROM EMP E1\n" + + "WHERE E1.SAL > (SELECT B1.COMM FROM BONUS B1 WHERE E1.ENAME = B1.ENAME LIMIT 2)"; + final RelNode originalRel; + try { + final SqlNode parse = planner.parse(sql); + final SqlNode validate = planner.validate(parse); + originalRel = planner.rel(validate).rel; + } catch (Exception e) { + throw TestUtil.rethrow(e); + } + + final HepProgram hepProgram = HepProgram.builder() + .addRuleCollection( + ImmutableList.of( + // SubQuery program rules + CoreRules.FILTER_SUB_QUERY_TO_CORRELATE, + CoreRules.PROJECT_SUB_QUERY_TO_CORRELATE, + CoreRules.JOIN_SUB_QUERY_TO_CORRELATE)) + .build(); + final Program program = + Programs.of(hepProgram, true, + requireNonNull(cluster.getMetadataProvider())); + final RelNode before = + program.run(cluster.getPlanner(), originalRel, cluster.traitSet(), + Collections.emptyList(), Collections.emptyList()); + final String planBefore = "" + + "LogicalProject(DEPTNO=[$7])\n" + + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7])\n" + + " LogicalFilter(condition=[>($5, $8)])\n" + + " LogicalCorrelate(correlation=[$cor0], joinType=[left], requiredColumns=[{1}])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalAggregate(group=[{}], agg#0=[SINGLE_VALUE($0)])\n" + + " LogicalSort(fetch=[2])\n" + + " LogicalProject(COMM=[$3])\n" + + " LogicalFilter(condition=[=($cor0.ENAME, $0)])\n" + + " LogicalTableScan(table=[[scott, BONUS]])\n"; + assertThat(before, hasTree(planBefore)); + + // Decorrelate without any rules, just "purely" decorrelation algorithm on RelDecorrelator + final RelNode after = + RelDecorrelator.decorrelateQuery(before, builder, RuleSets.ofList(Collections.emptyList()), + RuleSets.ofList(Collections.emptyList())); + // Verify plan + final String planAfter = "" + + "LogicalProject(DEPTNO=[$7])\n" + + " LogicalJoin(condition=[AND(=($1, $8), >($5, $9))], joinType=[inner])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalAggregate(group=[{0}], agg#0=[SINGLE_VALUE($1)])\n" + + " LogicalProject(ENAME=[$1], COMM=[$0])\n" + + " LogicalFilter(condition=[<=($2, 2)])\n" + + " LogicalProject(COMM=[$3], ENAME=[$0], rn=[ROW_NUMBER() OVER (PARTITION BY $0)])\n" + + " LogicalFilter(condition=[IS NOT NULL($0)])\n" + + " LogicalTableScan(table=[[scott, BONUS]])\n"; + assertThat(after, hasTree(planAfter)); + } + /** Test case for [CALCITE-7257] * Subqueries cannot be decorrelated if join condition contains RexFieldAccess. */ @Test void testJoinConditionContainsRexFieldAccess() { diff --git a/core/src/test/resources/sql/sub-query.iq b/core/src/test/resources/sql/sub-query.iq index 4d4398cfc4f0..4b699d14b85f 100644 --- a/core/src/test/resources/sql/sub-query.iq +++ b/core/src/test/resources/sql/sub-query.iq @@ -10017,5 +10017,17 @@ INNER JOIN emp e +--------+------------+----------+-------+--------+----------+------+------------+---------+---------+---------+ (8 rows) +!ok + +# [CALCITE-709] LIMIT inside scalar sub-query +SELECT E1.DEPTNO +FROM EMP E1 +WHERE E1.SAL > (SELECT B1.COMM FROM BONUS B1 WHERE E1.ENAME = B1.ENAME LIMIT 2); ++--------+ +| DEPTNO | ++--------+ ++--------+ +(0 rows) + !ok # End sub-query.iq From 8d2d54eb37b540326d17083e15415386cbea6dfa Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Wed, 17 Jun 2026 11:30:28 -0700 Subject: [PATCH 333/562] [CALCITE-7583] UNNEST with multiple array arguments returns wrong result Signed-off-by: Mihai Budiu --- .../enumerable/EnumerableUncollect.java | 4 +- .../apache/calcite/rel/core/Uncollect.java | 26 +- .../apache/calcite/runtime/SqlFunctions.java | 159 ++++++-- .../apache/calcite/sql/SqlUnnestOperator.java | 21 +- .../apache/calcite/util/BuiltInMethod.java | 2 +- .../org/apache/calcite/test/JdbcTest.java | 186 --------- .../apache/calcite/test/SqlFunctionsTest.java | 107 +++++ .../apache/calcite/test/SqlValidatorTest.java | 32 +- core/src/test/resources/sql/unnest.iq | 364 ++++++++++++++++++ 9 files changed, 673 insertions(+), 228 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableUncollect.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableUncollect.java index 187a1cca7554..de167cd08e4f 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableUncollect.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableUncollect.java @@ -89,7 +89,7 @@ public static EnumerableUncollect create(RelTraitSet traitSet, RelNode input, JavaRowFormat.LIST); // final Enumerable> child = <>; - // return child.selectMany(FLAT_PRODUCT); + // return child.selectMany(FLAT_ZIP); final Expression child_ = builder.append( "child", result.block); @@ -125,7 +125,7 @@ public static EnumerableUncollect create(RelTraitSet traitSet, RelNode input, final Expression lambda = lambdaForStructWithSingleItem != null ? lambdaForStructWithSingleItem - : Expressions.call(BuiltInMethod.FLAT_PRODUCT.method, + : Expressions.call(BuiltInMethod.FLAT_ZIP.method, Expressions.constant(Ints.toArray(fieldCounts)), Expressions.constant(withOrdinality), Expressions.constant( diff --git a/core/src/main/java/org/apache/calcite/rel/core/Uncollect.java b/core/src/main/java/org/apache/calcite/rel/core/Uncollect.java index 09b4d5822f07..2d4c3620a74c 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/Uncollect.java +++ b/core/src/main/java/org/apache/calcite/rel/core/Uncollect.java @@ -169,21 +169,34 @@ public static RelDataType deriveUncollectRowType(RelNode rel, .build(); } + // With multiple collections, zip semantics pads shorter collections with + // NULL, so all output columns from a multi-collection UNNEST are nullable. + final boolean padNullable = fields.size() > 1; + for (int i = 0; i < fields.size(); i++) { RelDataTypeField field = fields.get(i); if (field.getType() instanceof MapSqlType) { // This code is similar to SqlUnnestOperator::inferReturnType. MapSqlType mapType = (MapSqlType) field.getType(); - builder.add(SqlUnnestOperator.MAP_KEY_COLUMN_NAME, mapType.getKeyType()); - builder.add(SqlUnnestOperator.MAP_VALUE_COLUMN_NAME, mapType.getValueType()); + RelDataType keyType = padNullable + ? typeFactory.enforceTypeWithNullability(mapType.getKeyType(), true) + : mapType.getKeyType(); + RelDataType valueType = padNullable + ? typeFactory.enforceTypeWithNullability(mapType.getValueType(), true) + : mapType.getValueType(); + builder.add(SqlUnnestOperator.MAP_KEY_COLUMN_NAME, keyType); + builder.add(SqlUnnestOperator.MAP_VALUE_COLUMN_NAME, valueType); } else { RelDataType componentType = field.getType().getComponentType(); if (null == componentType) { throw RESOURCE.unnestArgument().ex(); } - boolean isNullable = componentType.isNullable(); + boolean isNullable = componentType.isNullable() || padNullable; if (requireAlias) { - builder.add(itemAliases.get(i), componentType); + RelDataType colType = padNullable + ? typeFactory.enforceTypeWithNullability(componentType, true) + : componentType; + builder.add(itemAliases.get(i), colType); } else if (componentType.isStruct()) { for (RelDataTypeField fieldInfo : componentType.getFieldList()) { RelDataType fieldType = fieldInfo.getType(); @@ -194,7 +207,10 @@ public static RelDataType deriveUncollectRowType(RelNode rel, } } else { // Element type is not a record, use the field name of the element directly - builder.add(field.getName(), componentType); + RelDataType colType = padNullable + ? typeFactory.enforceTypeWithNullability(componentType, true) + : componentType; + builder.add(field.getName(), colType); } } } diff --git a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java index 90ed84c7618f..789fb9d6c794 100644 --- a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java +++ b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java @@ -7428,68 +7428,87 @@ public static Function1, Enumerable> flatList() { return inputList -> Linq4j.asEnumerable(inputList).select(v -> structAccess(v, 0, null)); } - public static Function1>> flatProduct( + /** + * Returns a function that, given a row containing one or more collection + * fields, produces an {@link Enumerable} of combined element rows using + * zip (positional pairing) semantics. + * + *

      This is the standard semantics for SQL {@code UNNEST(a, b, ...)}: the + * i-th output row pairs element {@code a[i]} with element {@code b[i]}. + * Shorter collections are padded with {@code NULL}. + */ + public static Function1>> flatZip( final int[] fieldCounts, final boolean withOrdinality, final FlatProductInputType[] inputTypes) { if (fieldCounts.length == 1) { if (!withOrdinality && inputTypes[0] == FlatProductInputType.SCALAR) { + // Simple unnest without ordinality //noinspection unchecked return (Function1) LIST_AS_ENUMERABLE; } else { - return row -> p2(new Object[] { row }, fieldCounts, withOrdinality, - inputTypes); + // unnest with ordinality for a single scalar column + return row -> z2(new Object[] { row }, fieldCounts, withOrdinality, inputTypes); } } - return lists -> p2((Object[]) lists, fieldCounts, withOrdinality, - inputTypes); + return lists -> z2((Object[]) lists, fieldCounts, withOrdinality, inputTypes); } - private static Enumerable> p2( + /** + * Helper for {@link #flatZip}: unpacks each collection in {@code lists} + * into an enumerator and combines them using zip (positional) semantics, + * padding shorter collections with {@code NULL}. + * + * @param lists one element per collection (scalar list, struct list, or map) + * @param fieldCounts output column count for each collection (-1 for a collection of scalars) + * @param withOrdinality whether to append a 1-based ordinality column + * @param inputTypes type of elements in each collection (SCALAR, LIST, or MAP) + */ + @SuppressWarnings("rawtypes") + private static Enumerable> z2( Object[] lists, int[] fieldCounts, boolean withOrdinality, FlatProductInputType[] inputTypes) { final List>> enumerators = new ArrayList<>(); + final int[] widths = new int[lists.length]; int totalFieldCount = 0; for (int i = 0; i < lists.length; i++) { - int fieldCount = fieldCounts[i]; - FlatProductInputType inputType = inputTypes[i]; - Object inputObject = lists[i]; + final int fieldCount = fieldCounts[i]; + final FlatProductInputType inputType = inputTypes[i]; + final Object inputObject = lists[i]; switch (inputType) { case SCALAR: @SuppressWarnings("unchecked") List list = (List) inputObject; - enumerators.add( - Linq4j.transform( - Linq4j.enumerator(list), FlatLists::of)); + enumerators.add(Linq4j.transform(Linq4j.enumerator(list), FlatLists::of)); + widths[i] = 1; break; case LIST: @SuppressWarnings("unchecked") List> listList = (List>) inputObject; enumerators.add(Linq4j.enumerator(listList)); + widths[i] = fieldCount; break; case MAP: @SuppressWarnings("unchecked") Map map = (Map) inputObject; Enumerator> enumerator = Linq4j.enumerator(map.entrySet()); - - Enumerator> transformed = - Linq4j.transform(enumerator, - e -> FlatLists.of(e.getKey(), e.getValue())); - enumerators.add(transformed); + enumerators.add(Linq4j.transform(enumerator, e -> FlatLists.of(e.getKey(), e.getValue()))); + widths[i] = 2; break; default: - break; - } - if (fieldCount < 0) { - ++totalFieldCount; - } else { - totalFieldCount += fieldCount; + throw new IllegalArgumentException("Unknown input type: " + inputType); } + totalFieldCount += (fieldCount < 0) ? 1 : fieldCount; } if (withOrdinality) { ++totalFieldCount; } - return product(enumerators, totalFieldCount, withOrdinality); + final int fieldCount = totalFieldCount; + return new AbstractEnumerable>() { + @Override public Enumerator> enumerator() { + return new ZipPaddedEnumerator(enumerators, widths, fieldCount, withOrdinality); + } + }; } public static Object[] array(Object... args) { @@ -7549,6 +7568,96 @@ public static Enumerable> pro }; } + /** + * Enumerates over the positional zip of the given collection enumerators, + * padding shorter collections with {@code NULL}. + */ + @SuppressWarnings({"rawtypes", "unchecked"}) + private static class ZipPaddedEnumerator + implements Enumerator> { + /** One enumerator per input collection; each yields one element row per step. + * For a scalar collection the inner list has exactly one element. */ + private final List>> enumerators; + /** Output column count contributed by each collection (parallel to {@link #enumerators}). */ + private final int[] widths; + /** {@code end_of_collection[i]} is {@code true} once collection {@code i} is exhausted. */ + private final boolean[] endOfCollection; + /** Preallocated output buffer where each result row is constructed. */ + final @Nullable Object[] flatElements; + /** Reused {@link List} view over {@link #flatElements}, passed to {@link FlatLists#of} + * to produce the final result for each output row. */ + final List<@Nullable Object> list; + private final boolean withOrdinality; + /** 1-based counter incremented on each successful {@link #moveNext()}. */ + private int currentOrdinality; + + ZipPaddedEnumerator(List>> enumerators, + int[] widths, int fieldCount, boolean withOrdinality) { + this.enumerators = enumerators; + this.widths = widths; + this.withOrdinality = withOrdinality; + this.endOfCollection = new boolean[enumerators.size()]; + flatElements = new Object[fieldCount]; + list = Arrays.asList(flatElements); + } + + @Override public boolean moveNext() { + boolean allCompleted = true; + for (int i = 0; i < enumerators.size(); i++) { + if (!endOfCollection[i]) { + endOfCollection[i] = !enumerators.get(i).moveNext(); + } + allCompleted &= endOfCollection[i]; + } + if (!allCompleted && withOrdinality) { + currentOrdinality++; + } + return !allCompleted; + } + + @Override public FlatLists.ComparableList current() { + int column = 0; + for (int i = 0; i < enumerators.size(); i++) { + int width = widths[i]; + if (!endOfCollection[i]) { + final Object elemRow = enumerators.get(i).current(); + if (elemRow instanceof Object[]) { + final Object[] arr = (Object[]) elemRow; + for (int p = 0; p < width; p++) { + flatElements[column + p] = p < arr.length ? arr[p] : null; + } + } else { + final List lst = (List) elemRow; + for (int p = 0; p < width; p++) { + flatElements[column + p] = p < lst.size() ? lst.get(p) : null; + } + } + } else { + Arrays.fill(flatElements, column, column + width, null); + } + column += width; + } + if (withOrdinality) { + flatElements[column] = currentOrdinality; + } + return (FlatLists.ComparableList) FlatLists.of(list); + } + + @Override public void reset() { + for (Enumerator> e : enumerators) { + e.reset(); + } + Arrays.fill(endOfCollection, false); + currentOrdinality = 0; + } + + @Override public void close() { + for (Enumerator> e : enumerators) { + e.close(); + } + } + } + /** * Implements the {@code .} (field access) operator on an object * whose type is not known until runtime. @@ -7644,7 +7753,7 @@ public enum JsonScope { JSON_KEYS, JSON_KEYS_AND_VALUES, JSON_VALUES } - /** Type of argument passed into {@link #flatProduct}. */ + /** Type of argument passed into {@link #flatZip}. */ public enum FlatProductInputType { SCALAR, LIST, MAP } diff --git a/core/src/main/java/org/apache/calcite/sql/SqlUnnestOperator.java b/core/src/main/java/org/apache/calcite/sql/SqlUnnestOperator.java index bd6023936c17..af1753f8e8b5 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlUnnestOperator.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlUnnestOperator.java @@ -83,13 +83,22 @@ public SqlUnnestOperator(boolean withOrdinality) { assert type instanceof ArraySqlType || type instanceof MultisetSqlType || type instanceof MapSqlType; // If a type is nullable, all field accesses inside the type are also nullable + // With multiple collections, zip semantics pad shorter collections with + // NULL, so all output columns from a multi-collection UNNEST are nullable. + final boolean padNullable = opBinding.getOperandCount() > 1; if (type instanceof MapSqlType) { MapSqlType mapType = (MapSqlType) type; - builder.add(MAP_KEY_COLUMN_NAME, mapType.getKeyType()); - builder.add(MAP_VALUE_COLUMN_NAME, mapType.getValueType()); + RelDataType keyType = padNullable + ? typeFactory.enforceTypeWithNullability(mapType.getKeyType(), true) + : mapType.getKeyType(); + RelDataType valueType = padNullable + ? typeFactory.enforceTypeWithNullability(mapType.getValueType(), true) + : mapType.getValueType(); + builder.add(MAP_KEY_COLUMN_NAME, keyType); + builder.add(MAP_VALUE_COLUMN_NAME, valueType); } else { RelDataType componentType = requireNonNull(type.getComponentType(), "componentType"); - boolean isNullable = componentType.isNullable(); + boolean isNullable = componentType.isNullable() || padNullable; if (!allowAliasUnnestItems(opBinding) && componentType.isStruct()) { for (RelDataTypeField field : componentType.getFieldList()) { RelDataType fieldType = field.getType(); @@ -99,8 +108,10 @@ public SqlUnnestOperator(boolean withOrdinality) { builder.add(field.getName(), fieldType); } } else { - builder.add(SqlUtil.deriveAliasFromOrdinal(operand), - componentType); + RelDataType colType = padNullable + ? typeFactory.enforceTypeWithNullability(componentType, true) + : componentType; + builder.add(SqlUtil.deriveAliasFromOrdinal(operand), colType); } } } diff --git a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java index c529c076e5f5..295d7d662e62 100644 --- a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java +++ b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java @@ -327,7 +327,7 @@ public enum BuiltInMethod { @SuppressWarnings("deprecation") PAIR_LIST_COPY_OF(PairList.Helper.class, "copyOf", Object.class, Object.class, Object[].class), - FLAT_PRODUCT(SqlFunctions.class, "flatProduct", int[].class, boolean.class, + FLAT_ZIP(SqlFunctions.class, "flatZip", int[].class, boolean.class, FlatProductInputType[].class), FLAT_LIST(SqlFunctions.class, "flatList"), LIST_N(FlatLists.class, "copyOf", Comparable[].class), diff --git a/core/src/test/java/org/apache/calcite/test/JdbcTest.java b/core/src/test/java/org/apache/calcite/test/JdbcTest.java index ee1afe05cbbc..a3c004c441ad 100644 --- a/core/src/test/java/org/apache/calcite/test/JdbcTest.java +++ b/core/src/test/java/org/apache/calcite/test/JdbcTest.java @@ -2490,76 +2490,6 @@ void checkMultisetQueryWithSingleColumn() { .returnsUnordered("A=[10, 20, 10, 10]"); } - @Test void testUnnestArray() { - CalciteAssert.that() - .query("select*from unnest(array[1,2])") - .returnsUnordered("EXPR$0=1", - "EXPR$0=2"); - } - - @Test void testUnnestArrayWithOrdinality() { - CalciteAssert.that() - .query("select*from unnest(array[10,20]) with ordinality as t(i, o)") - .returnsUnordered("I=10; O=1", - "I=20; O=2"); - } - - @Test void testUnnestRecordType() { - // unnest(RecordType(Array)) - CalciteAssert.that() - .query("select * from unnest\n" - + "(select t.x from (values array[10, 20], array[30, 40]) as t(x))\n" - + " with ordinality as t(a, o)") - .returnsUnordered("A=10; O=1", "A=20; O=2", - "A=30; O=1", "A=40; O=2"); - - // unnest(RecordType(Multiset)) - CalciteAssert.that() - .query("select * from unnest\n" - + "(select t.x from (values multiset[10, 20], array[30, 40]) as t(x))\n" - + " with ordinality as t(a, o)") - .returnsUnordered("A=10; O=1", "A=20; O=2", - "A=30; O=1", "A=40; O=2"); - - // unnest(RecordType(Map)) - CalciteAssert.that() - .query("select * from unnest\n" - + "(select t.x from (values map['a', 20], map['b', 30], map['c', 40]) as t(x))\n" - + " with ordinality as t(a, b, o)") - .returnsUnordered("A=a; B=20; O=1", - "A=b; B=30; O=1", - "A=c; B=40; O=1"); - } - - @Test void testUnnestMultiset() { - CalciteAssert.that() - .with(CalciteAssert.Config.REGULAR) - .query("select*from unnest(multiset[1,2]) as t(c)") - .returnsUnordered("C=1", "C=2"); - } - - @Test void testUnnestMultiset2() { - CalciteAssert.that() - .with(CalciteAssert.Config.REGULAR) - .query("select*from unnest(\n" - + " select \"employees\" from \"hr\".\"depts\"\n" - + " where \"deptno\" = 10)") - .returnsUnordered( - "empid=100; deptno=10; name=Bill; salary=10000.0; commission=1000", - "empid=150; deptno=10; name=Sebastian; salary=7000.0; commission=null"); - } - - /** Test case for - * [CALCITE-2391] - * Aggregate query with UNNEST or LATERAL fails with - * ClassCastException. */ - @Test void testAggUnnestColumn() { - final String sql = "select count(d.\"name\") as c\n" - + "from \"hr\".\"depts\" as d,\n" - + " UNNEST(d.\"employees\") as e"; - CalciteAssert.hr().query(sql).returnsUnordered("C=3"); - } - @Test void testArrayElement() { CalciteAssert.that() .with(CalciteAssert.Config.REGULAR) @@ -2599,122 +2529,6 @@ void checkMultisetQueryWithSingleColumn() { "name=Theodore; deptno=10; M=120"); } - /** Per SQL std, UNNEST is implicitly LATERAL. */ - @Test void testUnnestArrayColumn() { - CalciteAssert.hr() - .query("select d.\"name\", e.*\n" - + "from \"hr\".\"depts\" as d,\n" - + " UNNEST(d.\"employees\") as e") - .returnsUnordered( - "name=HR; empid=200; deptno=20; name0=Eric; salary=8000.0; commission=500", - "name=Sales; empid=100; deptno=10; name0=Bill; salary=10000.0; commission=1000", - "name=Sales; empid=150; deptno=10; name0=Sebastian; salary=7000.0; commission=null"); - } - - @Test void testUnnestArrayScalarArray() { - CalciteAssert.hr() - .query("select d.\"name\", e.*\n" - + "from \"hr\".\"depts\" as d,\n" - + " UNNEST(d.\"employees\", array[1, 2]) as e") - .returnsUnordered( - "name=HR; empid=200; deptno=20; name0=Eric; salary=8000.0; commission=500; EXPR$1=1", - "name=HR; empid=200; deptno=20; name0=Eric; salary=8000.0; commission=500; EXPR$1=2", - "name=Sales; empid=100; deptno=10; name0=Bill; salary=10000.0; commission=1000; EXPR$1=1", - "name=Sales; empid=100; deptno=10; name0=Bill; salary=10000.0; commission=1000; EXPR$1=2", - "name=Sales; empid=150; deptno=10; name0=Sebastian; salary=7000.0; commission=null; EXPR$1=1", - "name=Sales; empid=150; deptno=10; name0=Sebastian; salary=7000.0; commission=null; EXPR$1=2"); - } - - @Test void testUnnestArrayScalarArrayAliased() { - CalciteAssert.hr() - .query("select d.\"name\", e.*\n" - + "from \"hr\".\"depts\" as d,\n" - + " UNNEST(d.\"employees\", array[1, 2]) as e (ei, d, n, s, c, i)\n" - + "where ei + i > 151") - .returnsUnordered( - "name=HR; EI=200; D=20; N=Eric; S=8000.0; C=500; I=1", - "name=HR; EI=200; D=20; N=Eric; S=8000.0; C=500; I=2", - "name=Sales; EI=150; D=10; N=Sebastian; S=7000.0; C=null; I=2"); - } - - @Test void testUnnestArrayScalarArrayWithOrdinal() { - CalciteAssert.hr() - .query("select d.\"name\", e.*\n" - + "from \"hr\".\"depts\" as d,\n" - + " UNNEST(d.\"employees\", array[1, 2]) with ordinality as e (ei, d, n, s, c, i, o)\n" - + "where ei + i > 151") - .returnsUnordered( - "name=HR; EI=200; D=20; N=Eric; S=8000.0; C=500; I=1; O=1", - "name=HR; EI=200; D=20; N=Eric; S=8000.0; C=500; I=2; O=2", - "name=Sales; EI=150; D=10; N=Sebastian; S=7000.0; C=null; I=2; O=4"); - } - - /** Test case for - * [CALCITE-3498] - * Unnest operation's ordinality should be deterministic. */ - @Test void testUnnestArrayWithDeterministicOrdinality() { - CalciteAssert.that() - .query("select v, o\n" - + "from unnest(array[100, 200]) with ordinality as t1(v, o)\n" - + "where v > 1") - .returns("V=100; O=1\n" - + "V=200; O=2\n"); - - CalciteAssert.that() - .query("with\n" - + " x as (select * from unnest(array[100, 200]) with ordinality as t1(v, o)), " - + " y as (select * from unnest(array[1000, 2000]) with ordinality as t2(v, o))\n" - + "select x.o as o1, x.v as v1, y.o as o2, y.v as v2 " - + "from x join y on x.o=y.o") - .returnsUnordered( - "O1=1; V1=100; O2=1; V2=1000", - "O1=2; V1=200; O2=2; V2=2000"); - } - - /** Test case for - * [CALCITE-1250] - * UNNEST applied to MAP data type. */ - @Test void testUnnestItemsInMap() throws SQLException { - Connection connection = DriverManager.getConnection("jdbc:calcite:"); - final String sql = "select * from unnest(MAP['a', 1, 'b', 2]) as um(k, v)"; - ResultSet resultSet = connection.createStatement().executeQuery(sql); - final String expected = "K=a; V=1\n" - + "K=b; V=2\n"; - assertThat(CalciteAssert.toString(resultSet), is(expected)); - connection.close(); - } - - @Test void testUnnestItemsInMapWithOrdinality() throws SQLException { - Connection connection = DriverManager.getConnection("jdbc:calcite:"); - final String sql = "select *\n" - + "from unnest(MAP['a', 1, 'b', 2]) with ordinality as um(k, v, i)"; - ResultSet resultSet = connection.createStatement().executeQuery(sql); - final String expected = "K=a; V=1; I=1\n" - + "K=b; V=2; I=2\n"; - assertThat(CalciteAssert.toString(resultSet), is(expected)); - connection.close(); - } - - @Test void testUnnestItemsInMapWithNoAliasAndAdditionalArgument() - throws SQLException { - Connection connection = DriverManager.getConnection("jdbc:calcite:"); - final String sql = - "select * from unnest(MAP['a', 1, 'b', 2], array[5, 6, 7])"; - ResultSet resultSet = connection.createStatement().executeQuery(sql); - - List map = FlatLists.of("KEY=a; VALUE=1", "KEY=b; VALUE=2"); - List array = FlatLists.of(" EXPR$1=5", " EXPR$1=6", " EXPR$1=7"); - - final StringBuilder b = new StringBuilder(); - for (List row : Linq4j.product(FlatLists.of(map, array))) { - b.append(row.get(0)).append(";").append(row.get(1)).append("\n"); - } - final String expected = b.toString(); - - assertThat(CalciteAssert.toString(resultSet), is(expected)); - connection.close(); - } - private CalciteAssert.AssertQuery withFoodMartQuery(int id) throws IOException { final FoodMartQuerySet set = FoodMartQuerySet.instance(); diff --git a/core/src/test/java/org/apache/calcite/test/SqlFunctionsTest.java b/core/src/test/java/org/apache/calcite/test/SqlFunctionsTest.java index 96c111bd39d0..959e2fabc258 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlFunctionsTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlFunctionsTest.java @@ -17,7 +17,10 @@ package org.apache.calcite.test; import org.apache.calcite.avatica.util.ByteString; import org.apache.calcite.avatica.util.DateTimeUtils; +import org.apache.calcite.linq4j.Enumerable; +import org.apache.calcite.linq4j.function.Function1; import org.apache.calcite.runtime.CalciteException; +import org.apache.calcite.runtime.FlatLists; import org.apache.calcite.runtime.SqlFunctions; import org.apache.calcite.runtime.Utilities; @@ -41,6 +44,8 @@ import static org.apache.calcite.avatica.util.DateTimeUtils.dateStringToUnixDate; import static org.apache.calcite.avatica.util.DateTimeUtils.timeStringToUnixDate; import static org.apache.calcite.avatica.util.DateTimeUtils.timestampStringToUnixDate; +import static org.apache.calcite.runtime.SqlFunctions.FlatProductInputType.LIST; +import static org.apache.calcite.runtime.SqlFunctions.FlatProductInputType.SCALAR; import static org.apache.calcite.runtime.SqlFunctions.arraysOverlap; import static org.apache.calcite.runtime.SqlFunctions.charLength; import static org.apache.calcite.runtime.SqlFunctions.concat; @@ -2112,4 +2117,106 @@ private long sqlTimestamp(String str) { () -> parse.parseTimestamp("%Y-%m-%d %H:%M:%S", "2024-01-01 00:00:00", "Asia/Sanghai")); } + + // Tests for ZipPaddedEnumerator, accessed via the public SqlFunctions.flatZip API. + + /** Invokes {@link SqlFunctions#flatZip} over scalar collections and collects output rows. */ + @SuppressWarnings({"rawtypes", "unchecked"}) + private static List> zipScalars( + boolean withOrdinality, List... inputs) { + final int n = inputs.length; + final int[] fieldCounts = new int[n]; + Arrays.fill(fieldCounts, 1); + final SqlFunctions.FlatProductInputType[] types = + new SqlFunctions.FlatProductInputType[n]; + Arrays.fill(types, SCALAR); + final Function1>> fn = + SqlFunctions.flatZip(fieldCounts, withOrdinality, types); + final Object arg = n == 1 ? inputs[0] : inputs; + final List> rows = new ArrayList<>(); + for (FlatLists.ComparableList row : fn.apply(arg)) { + rows.add(new ArrayList<>(row)); + } + return rows; + } + + @Test void testZipPaddedSingleCollectionWithOrdinality() { + // Single scalar collection with ordinality uses ZipPaddedEnumerator, not + // the LIST_AS_ENUMERABLE shortcut. + List> rows = zipScalars(true, Arrays.asList(10, 20, 30)); + assertThat(rows, hasSize(3)); + assertThat(rows.get(0), is(list(10, 1))); + assertThat(rows.get(1), is(list(20, 2))); + assertThat(rows.get(2), is(list(30, 3))); + } + + @Test void testZipPaddedEqualLength() { + List> rows = + zipScalars(false, Arrays.asList(10, 20), Arrays.asList(4, 5)); + assertThat(rows, hasSize(2)); + assertThat(rows.get(0), is(list(10, 4))); + assertThat(rows.get(1), is(list(20, 5))); + } + + @Test void testZipPaddedFirstLonger() { + List> rows = + zipScalars(false, Arrays.asList(10, 20, 30), Arrays.asList(4, 5)); + assertThat(rows, hasSize(3)); + assertThat(rows.get(0), is(list(10, 4))); + assertThat(rows.get(1), is(list(20, 5))); + assertThat(rows.get(2), is(Arrays.asList(30, null))); + } + + @Test void testZipPaddedSecondLonger() { + List> rows = + zipScalars(false, Arrays.asList(10), Arrays.asList(4, 5, 6)); + assertThat(rows, hasSize(3)); + assertThat(rows.get(0), is(list(10, 4))); + assertThat(rows.get(1), is(Arrays.asList(null, 5))); + assertThat(rows.get(2), is(Arrays.asList(null, 6))); + } + + @Test void testZipPaddedBothEmpty() { + List> rows = + zipScalars(false, Collections.emptyList(), Collections.emptyList()); + assertThat(rows, hasSize(0)); + } + + @Test void testZipPaddedOneEmpty() { + List> rows = + zipScalars(false, Arrays.asList(4, 5), Collections.emptyList()); + assertThat(rows, hasSize(2)); + assertThat(rows.get(0), is(Arrays.asList(4, null))); + assertThat(rows.get(1), is(Arrays.asList(5, null))); + } + + @Test void testZipPaddedWithOrdinality() { + List> rows = + zipScalars(true, Arrays.asList(10, 20, 30), Arrays.asList(4, 5)); + assertThat(rows, hasSize(3)); + assertThat(rows.get(0), is(list(10, 4, 1))); + assertThat(rows.get(1), is(list(20, 5, 2))); + assertThat(rows.get(2), is(Arrays.asList(30, null, 3))); + } + + @Test void testZipPaddedStructElements() { + // Two LIST (struct) collections of width 2: [(1,2),(3,4)] and [(10,20)] + // → [1,2,10,20], [3,4,null,null] + @SuppressWarnings({"rawtypes", "unchecked"}) + final Function1>> fn = + SqlFunctions.flatZip(new int[]{2, 2}, false, + new SqlFunctions.FlatProductInputType[]{LIST, LIST}); + final List> col1 = + Arrays.asList(FlatLists.of(1, 2), FlatLists.of(3, 4)); + final List> col2 = + Collections.singletonList(FlatLists.of(10, 20)); + final List> rows = new ArrayList<>(); + for (FlatLists.ComparableList row + : fn.apply(new Object[]{col1, col2})) { + rows.add(new ArrayList<>(row)); + } + assertThat(rows, hasSize(2)); + assertThat(rows.get(0), is(list(1, 2, 10, 20))); + assertThat(rows.get(1), is(Arrays.asList(3, 4, null, null))); + } } diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index 9bdd3177ff29..4b4d2302de52 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -2117,10 +2117,8 @@ void testLikeAndSimilarFails() { .type("RecordType(BOOLEAN NOT NULL EXPR$0) NOT NULL"); sql("^values ('1'),(2)^") .fails("Values passed to VALUES operator must have compatible types"); - if (TODO) { - sql("values (1),(2.0),(3)") - .columnType("ROWTYPE(DOUBLE)"); - } + sql("values (1),(2.0),(3)") + .columnType("DECIMAL(11, 1) NOT NULL"); } @Test void testRow() { @@ -14517,4 +14515,30 @@ private static SqlIdentifier rewriteIdentifier(SqlIdentifier sqlIdentifier) { + ">'\\. " + "Supported form\\(s\\): ' = '"); } + + /** Test case for + * [CALCITE-7583] + * UNNEST with multiple array arguments returns wrong result. + * + *

      When UNNEST is given multiple collection arguments, zip (positional) + * semantics pad shorter collections with NULL, so all output columns must be + * nullable even when the element types of the input arrays are not nullable. */ + @Test void testUnnestMultiArgNullability() { + // Single-arg UNNEST: element type is NOT NULL, result column is NOT NULL. + sql("select * from unnest(array[1, 2])") + .type("RecordType(INTEGER NOT NULL EXPR$0) NOT NULL"); + // Multi-arg UNNEST: result columns are nullable (padded with NULL when lengths differ). + sql("select * from unnest(array[1, 2], array[3, 4])") + .type("RecordType(INTEGER EXPR$0, INTEGER EXPR$1) NOT NULL"); + // Ordinality column itself is always NOT NULL. + sql("select * from unnest(array[1, 2], array[3, 4]) with ordinality") + .type("RecordType(INTEGER EXPR$0, INTEGER EXPR$1," + + " INTEGER NOT NULL ORDINALITY) NOT NULL"); + // Different element types: both result columns are nullable. + sql("select * from unnest(array['a', 'b'], array[1, 2])") + .type("RecordType(CHAR(1) EXPR$0, INTEGER EXPR$1) NOT NULL"); + // Struct array + scalar array: all result columns (including struct fields) are nullable. + sql("select * from unnest(array[(1, 'a'), (2, 'b')], array[10, 20]) as t(p, q, r)") + .type("RecordType(INTEGER P, CHAR(1) Q, INTEGER R) NOT NULL"); + } } diff --git a/core/src/test/resources/sql/unnest.iq b/core/src/test/resources/sql/unnest.iq index 88488141ad3b..8defd3b01ce6 100644 --- a/core/src/test/resources/sql/unnest.iq +++ b/core/src/test/resources/sql/unnest.iq @@ -243,6 +243,370 @@ FROM UNNEST(array [0, 2, 4, 4, 5]) as x; !ok +# Tests for [CALCITE-7583] UNNEST with multiple array arguments returns wrong result +# 6 tests validated on Postgres + +# Multi-collection UNNEST uses zip (positional) semantics, not cartesian +# product. Elements are paired by position; shorter collections are padded +# with NULL. + +# Equal-length: zip and product give the same result. +select * +from unnest(array[10, 20, 30], array[1, 2, 3]) as t(a, b); ++----+---+ +| A | B | ++----+---+ +| 10 | 1 | +| 20 | 2 | +| 30 | 3 | ++----+---+ +(3 rows) + +!ok + +# First collection longer: second is padded with NULL for the extra rows. +select * +from unnest(array[10, 20, 30], array[1, 2]) as t(a, b); ++----+---+ +| A | B | ++----+---+ +| 10 | 1 | +| 20 | 2 | +| 30 | | ++----+---+ +(3 rows) + +!ok + +# Second collection longer: first is padded with NULL for the extra rows. +select * +from unnest(array[10, 20], array[1, 2, 3]) as t(a, b); ++----+---+ +| A | B | ++----+---+ +| 10 | 1 | +| 20 | 2 | +| | 3 | ++----+---+ +(3 rows) + +!ok + +# Ordinality counts all output rows, including NULL-padded ones. +select * +from unnest(array[10, 20, 30], array[1, 2]) with ordinality as t(a, b, o); ++----+---+---+ +| A | B | O | ++----+---+---+ +| 10 | 1 | 1 | +| 20 | 2 | 2 | +| 30 | | 3 | ++----+---+---+ +(3 rows) + +!ok + +# MAP + scalar array: MAP (2 entries) exhausts before the scalar (3 elements). +# Postgres does not support MAP values, so this is not validated on Postgres +select * +from unnest(map['x', 10, 'y', 20], array[1, 2, 3]) as t(k, v, n); ++---+----+---+ +| K | V | N | ++---+----+---+ +| x | 10 | 1 | +| y | 20 | 2 | +| | | 3 | ++---+----+---+ +(3 rows) + +!ok + +# Struct array + scalar array: struct array has 3 elements, scalar has 2. +# The third struct row is produced normally; scalar column is NULL. +# Postgres requires a slightly different syntax, but produces the same result. +select * +from unnest(array[(1, 'a'), (2, 'b'), (3, 'c')], array[10, 20]) as t(x, y, z); ++---+---+----+ +| X | Y | Z | ++---+---+----+ +| 1 | a | 10 | +| 2 | b | 20 | +| 3 | c | | ++---+---+----+ +(3 rows) + +!ok + +# [CALCITE-7583] UNNEST with multiple array arguments returns wrong result +# Three arrays of different lengths with ordinality: output has max(3,2,4)=4 rows. +select * +from unnest(array[10, 20, 30], array[1, 2], array['a', 'b', 'c', 'd']) +with ordinality as t(x, y, z, o); ++----+---+---+---+ +| X | Y | Z | O | ++----+---+---+---+ +| 10 | 1 | a | 1 | +| 20 | 2 | b | 2 | +| 30 | | c | 3 | +| | | d | 4 | ++----+---+---+---+ +(4 rows) + +!ok + +# Unnest a plain scalar array; default column name is EXPR$0. +select * from unnest(array[1, 2]); ++--------+ +| EXPR$0 | ++--------+ +| 1 | +| 2 | ++--------+ +(2 rows) + +!ok + +# Unnest with ordinality: explicit aliases i and o. +select * from unnest(array[10, 20]) with ordinality as t(i, o); ++----+---+ +| I | O | ++----+---+ +| 10 | 1 | +| 20 | 2 | ++----+---+ +(2 rows) + +!ok + +# Unnest a subquery that returns a RecordType(Array). +# Ordinality resets to 1 for each input row. +select * from unnest +(select t.x from (values array[10, 20], array[30, 40]) as t(x)) + with ordinality as t(a, o); ++----+---+ +| A | O | ++----+---+ +| 10 | 1 | +| 20 | 2 | +| 30 | 1 | +| 40 | 2 | ++----+---+ +(4 rows) + +!ok + +# Unnest a subquery that returns a RecordType(Multiset). +# Ordinality resets to 1 for each input row. +select * from unnest +(select t.x from (values multiset[10, 20], array[30, 40]) as t(x)) + with ordinality as t(a, o); ++----+---+ +| A | O | ++----+---+ +| 10 | 1 | +| 20 | 2 | +| 30 | 1 | +| 40 | 2 | ++----+---+ +(4 rows) + +!ok + +# Unnest a subquery that returns a RecordType(Map). +# Each map is one row; ordinality is always 1 because each map has one entry. +select * from unnest +(select t.x from (values map['a', 20], map['b', 30], map['c', 40]) as t(x)) + with ordinality as t(a, b, o); ++---+----+---+ +| A | B | O | ++---+----+---+ +| a | 20 | 1 | +| b | 30 | 1 | +| c | 40 | 1 | ++---+----+---+ +(3 rows) + +!ok + +# Unnest a multiset literal with an explicit column alias. +select * from unnest(multiset[1, 2]) as t(c); ++---+ +| C | ++---+ +| 1 | +| 2 | ++---+ +(2 rows) + +!ok + +# [CALCITE-3498] Unnest operation's ordinality should be deterministic. +# A filter applied after WITH ORDINALITY must not disturb the ordinality values. +select v, o +from unnest(array[100, 200]) with ordinality as t1(v, o) +where v > 1; ++-----+---+ +| V | O | ++-----+---+ +| 100 | 1 | +| 200 | 2 | ++-----+---+ +(2 rows) + +!ok + +# [CALCITE-3498] Unnest operation's ordinality should be deterministic. +with + x as (select * from unnest(array[100, 200]) with ordinality as t1(v, o)), + y as (select * from unnest(array[1000, 2000]) with ordinality as t2(v, o)) +select x.o as o1, x.v as v1, y.o as o2, y.v as v2 +from x join y on x.o = y.o; ++----+-----+----+------+ +| O1 | V1 | O2 | V2 | ++----+-----+----+------+ +| 1 | 100 | 1 | 1000 | +| 2 | 200 | 2 | 2000 | ++----+-----+----+------+ +(2 rows) + +!ok + +# [CALCITE-1250] UNNEST applied to MAP data type. +select * from unnest(MAP['a', 1, 'b', 2]) as um(k, v); ++---+---+ +| K | V | ++---+---+ +| a | 1 | +| b | 2 | ++---+---+ +(2 rows) + +!ok + +select * +from unnest(MAP['a', 1, 'b', 2]) with ordinality as um(k, v, i); ++---+---+---+ +| K | V | I | ++---+---+---+ +| a | 1 | 1 | +| b | 2 | 2 | ++---+---+---+ +(2 rows) + +!ok + +# [CALCITE-7583] UNNEST with multiple array arguments returns wrong result +# Validated on Postgres +# MAP has 2 entries; array has 3 elements; the third row pads MAP columns with NULL. +select * from unnest(MAP['a', 1, 'b', 2], array[5, 6, 7]); ++-----+-------+--------+ +| KEY | VALUE | EXPR$1 | ++-----+-------+--------+ +| a | 1 | 5 | +| b | 2 | 6 | +| | | 7 | ++-----+-------+--------+ +(3 rows) + +!ok + +!use hr + +# Unnest a subquery result that returns an array-of-struct column (hr.depts.employees). +select * from unnest( + select "employees" from "hr"."depts" + where "deptno" = 10); ++-------+--------+-----------+---------+------------+ +| empid | deptno | name | salary | commission | ++-------+--------+-----------+---------+------------+ +| 100 | 10 | Bill | 10000.0 | 1000 | +| 150 | 10 | Sebastian | 7000.0 | | ++-------+--------+-----------+---------+------------+ +(2 rows) + +!ok + +# [CALCITE-2391] Aggregate query with UNNEST or LATERAL fails with ClassCastException. +# Total employee count across all depts: Sales(2) + Marketing(0) + HR(1) = 3. +select count(d."name") as c +from "hr"."depts" as d, +UNNEST(d."employees") as e; ++---+ +| C | ++---+ +| 3 | ++---+ +(1 row) + +!ok + +# Per SQL standard, UNNEST is implicitly LATERAL. +select d."name", e.* +from "hr"."depts" as d, +UNNEST(d."employees") as e; ++-------+-------+--------+-----------+---------+------------+ +| name | empid | deptno | name0 | salary | commission | ++-------+-------+--------+-----------+---------+------------+ +| HR | 200 | 20 | Eric | 8000.0 | 500 | +| Sales | 100 | 10 | Bill | 10000.0 | 1000 | +| Sales | 150 | 10 | Sebastian | 7000.0 | | ++-------+-------+--------+-----------+---------+------------+ +(3 rows) + +!ok + +# [CALCITE-7583] UNNEST with multiple array arguments returns wrong result +# Validated on Postgres +select d."name", e.* +from "hr"."depts" as d, + UNNEST(d."employees", array[1, 2]) as e; ++-----------+-------+--------+-----------+---------+------------+--------+ +| name | empid | deptno | name0 | salary | commission | EXPR$1 | ++-----------+-------+--------+-----------+---------+------------+--------+ +| HR | 200 | 20 | Eric | 8000.0 | 500 | 1 | +| HR | | | | | | 2 | +| Marketing | | | | | | 1 | +| Marketing | | | | | | 2 | +| Sales | 100 | 10 | Bill | 10000.0 | 1000 | 1 | +| Sales | 150 | 10 | Sebastian | 7000.0 | | 2 | ++-----------+-------+--------+-----------+---------+------------+--------+ +(6 rows) + +!ok + +# [CALCITE-7583] UNNEST with multiple array arguments returns wrong result +# Validated on Postgres +select d."name", e.* +from "hr"."depts" as d, + UNNEST(d."employees", array[1, 2]) as e (ei, d, n, s, c, i) +where ei + i > 151; ++-------+-----+----+-----------+--------+-----+---+ +| name | EI | D | N | S | C | I | ++-------+-----+----+-----------+--------+-----+---+ +| HR | 200 | 20 | Eric | 8000.0 | 500 | 1 | +| Sales | 150 | 10 | Sebastian | 7000.0 | | 2 | ++-------+-----+----+-----------+--------+-----+---+ +(2 rows) + +!ok + +# [CALCITE-7583] UNNEST with multiple array arguments returns wrong result +# Validated on Postgres +# Same query with WITH ORDINALITY added. +select d."name", e.* +from "hr"."depts" as d, + UNNEST(d."employees", array[1, 2]) with ordinality as e (ei, d, n, s, c, i, o) +where ei + i > 151; ++-------+-----+----+-----------+--------+-----+---+---+ +| name | EI | D | N | S | C | I | O | ++-------+-----+----+-----------+--------+-----+---+---+ +| HR | 200 | 20 | Eric | 8000.0 | 500 | 1 | 1 | +| Sales | 150 | 10 | Sebastian | 7000.0 | | 2 | 2 | ++-------+-----+----+-----------+--------+-----+---+---+ +(2 rows) + +!ok + !use bookstore # [CALCITE-4773] RelDecorrelator's RemoveSingleAggregateRule can produce result with wrong row type From aadfc7ca1c45a297eee5c3829d0550a5e7dbb33d Mon Sep 17 00:00:00 2001 From: "cc.cai" <2356672992@qq.com> Date: Fri, 19 Jun 2026 15:32:38 +0800 Subject: [PATCH 334/562] Revert "[CALCITE-7539] Upgrade Arrow adapter dependencies to 16.0.0" This reverts commit a33d6a9ddeb67877b3d51983eaf55bfccabe463d. --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 14d29dfa356d..fd823182640c 100644 --- a/gradle.properties +++ b/gradle.properties @@ -81,7 +81,7 @@ jandex.version=3.5.3 # elasticsearch does not like asm:6.2.1+ aggdesigner-algorithm.version=6.1 apiguardian-api.version=1.1.2 -arrow.version=16.0.0 +arrow.version=15.0.0 asm.version=9.9.1 byte-buddy.version=1.18.8 cassandra-all.version=4.1.6 From 87530b934f6b8c8c1023bec5077fb9348eb6d752 Mon Sep 17 00:00:00 2001 From: "cc.cai" <2356672992@qq.com> Date: Fri, 19 Jun 2026 15:32:39 +0800 Subject: [PATCH 335/562] Revert "[CALCITE-7580] Remove Gandiva dependency from Arrow adapter" This reverts commit bf2b81b27d76ef3571ba676dbc7bce4ccb4a0413. --- arrow/build.gradle.kts | 1 + .../arrow/AbstractArrowEnumerator.java | 30 +-- .../adapter/arrow/ArrowDirectEnumerator.java | 31 ++- .../adapter/arrow/ArrowEnumerable.java | 25 +- .../adapter/arrow/ArrowFilterEnumerator.java | 238 +++++------------- .../adapter/arrow/ArrowProjectEnumerator.java | 80 ++++++ .../calcite/adapter/arrow/ArrowRules.java | 2 +- .../calcite/adapter/arrow/ArrowTable.java | 143 ++++++++++- .../adapter/arrow/ArrowTranslator.java | 69 ++--- .../calcite/adapter/arrow/ConditionToken.java | 51 +--- .../adapter/arrow/ArrowAdapterTest.java | 50 ---- .../calcite/adapter/arrow/ArrowDataTest.java | 29 --- .../calcite/adapter/arrow/ArrowExtension.java | 23 +- bom/build.gradle.kts | 1 + .../org/apache/calcite/rex/RexSimplify.java | 26 +- .../calcite/sql/type/SqlTypeFactoryImpl.java | 2 +- .../calcite/sql2rel/SqlToRelConverter.java | 34 ++- .../apache/calcite/rex/RexProgramTest.java | 20 ++ .../calcite/sql/type/SqlTypeFactoryTest.java | 25 ++ .../calcite/test/SqlToRelConverterTest.java | 18 ++ .../calcite/test/SqlToRelConverterTest.xml | 24 ++ gradle.properties | 1 + site/_docs/history.md | 5 - .../apache/calcite/test/SqlToRelFixture.java | 5 + 24 files changed, 525 insertions(+), 408 deletions(-) create mode 100644 arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowProjectEnumerator.java diff --git a/arrow/build.gradle.kts b/arrow/build.gradle.kts index c75a8b5f6751..598aa8a87972 100644 --- a/arrow/build.gradle.kts +++ b/arrow/build.gradle.kts @@ -23,6 +23,7 @@ dependencies { implementation("com.google.guava:guava") implementation("org.apache.arrow:arrow-memory-netty") implementation("org.apache.arrow:arrow-vector") + implementation("org.apache.arrow.gandiva:arrow-gandiva") annotationProcessor("org.immutables:value") compileOnly("org.immutables:value-annotations") diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/AbstractArrowEnumerator.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/AbstractArrowEnumerator.java index 486e3f60bd8d..e188757b0d2c 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/AbstractArrowEnumerator.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/AbstractArrowEnumerator.java @@ -24,7 +24,9 @@ import org.apache.arrow.vector.TimeStampVector; import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.VectorUnloader; import org.apache.arrow.vector.ipc.ArrowFileReader; +import org.apache.arrow.vector.ipc.message.ArrowRecordBatch; import org.apache.arrow.vector.types.TimeUnit; import org.apache.arrow.vector.types.pojo.ArrowType; @@ -49,6 +51,8 @@ abstract class AbstractArrowEnumerator implements Enumerator { this.currRowIndex = -1; } + abstract void evaluateOperator(ArrowRecordBatch arrowRecordBatch); + protected void loadNextArrowBatch() { try { final VectorSchemaRoot vsr = arrowFileReader.getVectorSchemaRoot(); @@ -56,32 +60,14 @@ protected void loadNextArrowBatch() { this.valueVectors.add(vsr.getVector(i)); } this.rowCount = vsr.getRowCount(); + VectorUnloader vectorUnloader = new VectorUnloader(vsr); + ArrowRecordBatch arrowRecordBatch = vectorUnloader.getRecordBatch(); + evaluateOperator(arrowRecordBatch); } catch (IOException e) { throw Util.toUnchecked(e); } } - /** Loads the next non-empty Arrow batch. */ - protected boolean loadNextNonEmptyArrowBatch() { - while (true) { - final boolean hasNextBatch; - try { - hasNextBatch = arrowFileReader.loadNextBatch(); - } catch (IOException e) { - throw Util.toUnchecked(e); - } - if (!hasNextBatch) { - return false; - } - currRowIndex = -1; - valueVectors.clear(); - loadNextArrowBatch(); - if (rowCount > 0) { - return true; - } - } - } - @Override public Object current() { if (fields.size() == 1) { return getValue(this.valueVectors.get(0), currRowIndex); @@ -99,7 +85,7 @@ protected boolean loadNextNonEmptyArrowBatch() { *

      For {@link TimeStampVector}, converts the raw value to * milliseconds since epoch, which is the representation used by * Calcite's Enumerable runtime for TIMESTAMP types. */ - protected static Object getValue(ValueVector vector, int index) { + private static Object getValue(ValueVector vector, int index) { if (vector instanceof TimeStampVector) { if (vector.isNull(index)) { return null; diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowDirectEnumerator.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowDirectEnumerator.java index 787ffd88d933..0cdec7baeb80 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowDirectEnumerator.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowDirectEnumerator.java @@ -17,11 +17,19 @@ package org.apache.calcite.adapter.arrow; import org.apache.calcite.util.ImmutableIntList; +import org.apache.calcite.util.Util; import org.apache.arrow.vector.ipc.ArrowFileReader; +import org.apache.arrow.vector.ipc.message.ArrowRecordBatch; + +import java.io.IOException; /** * Enumerator that reads projected Arrow value-vectors directly. + * + *

      This path is used for identity projections that Gandiva cannot project + * through the existing {@code Projector} path, such as Arrow binary vectors. + * It is not a replacement for Gandiva expression evaluation. */ class ArrowDirectEnumerator extends AbstractArrowEnumerator { private final Runnable onClose; @@ -32,14 +40,27 @@ class ArrowDirectEnumerator extends AbstractArrowEnumerator { this.onClose = onClose; } + @Override protected void evaluateOperator(ArrowRecordBatch arrowRecordBatch) { + } + @Override public boolean moveNext() { - while (currRowIndex >= rowCount - 1) { - if (!loadNextNonEmptyArrowBatch()) { - return false; + if (currRowIndex >= rowCount - 1) { + final boolean hasNextBatch; + try { + hasNextBatch = arrowFileReader.loadNextBatch(); + } catch (IOException e) { + throw Util.toUnchecked(e); + } + if (hasNextBatch) { + currRowIndex = 0; + this.valueVectors.clear(); + loadNextArrowBatch(); } + return hasNextBatch; + } else { + currRowIndex++; + return true; } - currRowIndex++; - return true; } @Override public void close() { diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowEnumerable.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowEnumerable.java index b9c0c4171e65..84ed5997aab2 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowEnumerable.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowEnumerable.java @@ -21,10 +21,11 @@ import org.apache.calcite.util.ImmutableIntList; import org.apache.calcite.util.Util; +import org.apache.arrow.gandiva.evaluator.Filter; +import org.apache.arrow.gandiva.evaluator.Projector; import org.apache.arrow.vector.ipc.ArrowFileReader; -import org.apache.arrow.vector.types.pojo.Schema; -import java.util.List; +import org.checkerframework.checker.nullness.qual.Nullable; /** * Enumerable that reads from Arrow value-vectors. @@ -32,24 +33,28 @@ class ArrowEnumerable extends AbstractEnumerable { private final ArrowFileReader arrowFileReader; private final ImmutableIntList fields; - private final List>> conditions; - private final Schema schema; + private final @Nullable Projector projector; + private final @Nullable Filter filter; private final Runnable onClose; ArrowEnumerable(ArrowFileReader arrowFileReader, ImmutableIntList fields, - List>> conditions, Schema schema, Runnable onClose) { + @Nullable Projector projector, @Nullable Filter filter, + Runnable onClose) { this.arrowFileReader = arrowFileReader; - this.conditions = conditions; - this.schema = schema; + this.projector = projector; + this.filter = filter; this.fields = fields; this.onClose = onClose; } @Override public Enumerator enumerator() { try { - if (!conditions.isEmpty()) { - return new ArrowFilterEnumerator(arrowFileReader, fields, - conditions, schema, onClose); + if (projector != null) { + return new ArrowProjectEnumerator(arrowFileReader, fields, projector, + onClose); + } else if (filter != null) { + return new ArrowFilterEnumerator(arrowFileReader, fields, filter, + onClose); } // No projector and no filter means the query is an identity projection // that should read selected value-vectors directly. diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowFilterEnumerator.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowFilterEnumerator.java index 2f154cc6ef7e..5eddec224909 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowFilterEnumerator.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowFilterEnumerator.java @@ -19,215 +19,91 @@ import org.apache.calcite.util.ImmutableIntList; import org.apache.calcite.util.Util; -import org.apache.arrow.vector.ValueVector; -import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.gandiva.evaluator.Filter; +import org.apache.arrow.gandiva.evaluator.SelectionVector; +import org.apache.arrow.gandiva.evaluator.SelectionVectorInt16; +import org.apache.arrow.gandiva.exceptions.GandivaException; +import org.apache.arrow.memory.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.ipc.ArrowFileReader; -import org.apache.arrow.vector.types.pojo.Field; -import org.apache.arrow.vector.types.pojo.Schema; +import org.apache.arrow.vector.ipc.message.ArrowRecordBatch; + +import org.checkerframework.checker.nullness.qual.Nullable; import java.io.IOException; -import java.math.BigDecimal; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.regex.Pattern; import static java.util.Objects.requireNonNull; /** - * Enumerator that evaluates Arrow filter tokens in Java. + * Enumerator that reads from a filtered collection of Arrow value-vectors. */ class ArrowFilterEnumerator extends AbstractArrowEnumerator { - private final List> conditions; - private final Schema schema; + private final BufferAllocator allocator; + private final Filter filter; + private @Nullable ArrowBuf buf; + private @Nullable SelectionVector selectionVector; + private int selectionVectorIndex; + private final Runnable onClose; - private final List filterVectors; - private final Map likePatterns; - ArrowFilterEnumerator(ArrowFileReader arrowFileReader, - ImmutableIntList fields, List>> conditions, - Schema schema, Runnable onClose) { + ArrowFilterEnumerator(ArrowFileReader arrowFileReader, ImmutableIntList fields, + Filter filter, Runnable onClose) { super(arrowFileReader, fields); - this.conditions = toConditionTokens(conditions); - this.schema = schema; + this.allocator = new RootAllocator(Long.MAX_VALUE); + this.filter = filter; this.onClose = onClose; - this.filterVectors = new ArrayList<>(schema.getFields().size()); - this.likePatterns = new HashMap<>(); } - @Override protected void loadNextArrowBatch() { - super.loadNextArrowBatch(); - final VectorSchemaRoot root; + @Override void evaluateOperator(ArrowRecordBatch arrowRecordBatch) { try { - root = arrowFileReader.getVectorSchemaRoot(); - } catch (IOException e) { + this.buf = this.allocator.buffer((long) rowCount * 2); + this.selectionVector = new SelectionVectorInt16(buf); + filter.evaluate(arrowRecordBatch, selectionVector); + } catch (GandivaException e) { throw Util.toUnchecked(e); } - filterVectors.clear(); - for (int i = 0; i < schema.getFields().size(); i++) { - filterVectors.add(root.getVector(i)); - } } @Override public boolean moveNext() { - while (true) { - if (currRowIndex >= rowCount - 1) { - if (!loadNextNonEmptyArrowBatch()) { - return false; + if (selectionVector == null + || selectionVectorIndex >= selectionVector.getRecordCount()) { + boolean hasNextBatch; + while (true) { + try { + hasNextBatch = arrowFileReader.loadNextBatch(); + } catch (IOException e) { + throw Util.toUnchecked(e); } - } - currRowIndex++; - if (matches(currRowIndex)) { - return true; - } - } - } - - private boolean matches(int rowIndex) { - for (List orGroup : conditions) { - boolean any = false; - for (ConditionToken token : orGroup) { - if (matches(token, rowIndex)) { - any = true; - break; + if (hasNextBatch) { + selectionVectorIndex = 0; + this.valueVectors.clear(); + loadNextArrowBatch(); + requireNonNull(selectionVector, "selectionVector"); + if (selectionVectorIndex >= selectionVector.getRecordCount()) { + // the "filtered" batch is empty, but there may be more batches to fetch + continue; + } + currRowIndex = selectionVector.getIndex(selectionVectorIndex++); } + return hasNextBatch; } - if (!any) { - return false; - } - } - return true; - } - - private boolean matches(ConditionToken token, int rowIndex) { - final Object value = getValue(fieldVector(token.fieldName), rowIndex); - switch (token.operator) { - case IS_NULL: - return value == null; - case IS_NOT_NULL: - return value != null; - case IS_TRUE: - return Boolean.TRUE.equals(value); - case IS_FALSE: - return Boolean.FALSE.equals(value); - case IS_NOT_TRUE: - return !Boolean.TRUE.equals(value); - case IS_NOT_FALSE: - return !Boolean.FALSE.equals(value); - case EQUAL: - return value != null && compare(value, literal(token)) == 0; - case NOT_EQUAL: - return value != null && compare(value, literal(token)) != 0; - case LESS_THAN: - return value != null && compare(value, literal(token)) < 0; - case LESS_THAN_OR_EQUAL: - return value != null && compare(value, literal(token)) <= 0; - case GREATER_THAN: - return value != null && compare(value, literal(token)) > 0; - case GREATER_THAN_OR_EQUAL: - return value != null && compare(value, literal(token)) >= 0; - case LIKE: - return value != null - && like(value.toString(), requireNonNull(token.value, "value")); - default: - throw new AssertionError("Unhandled Arrow filter operator: " + token.operator); - } - } - - private ValueVector fieldVector(String fieldName) { - final Field field = schema.findField(fieldName); - final int index = schema.getFields().indexOf(field); - if (index < 0) { - throw new IllegalArgumentException("Unknown Arrow field: " + fieldName); - } - return filterVectors.get(index); - } - - private static Object literal(ConditionToken token) { - final String type = requireNonNull(token.valueType, "valueType"); - final String value = requireNonNull(token.value, "value"); - if (type.startsWith("decimal")) { - return new BigDecimal(value); - } else if (type.equals("integer")) { - return Integer.valueOf(value); - } else if (type.equals("long")) { - return Long.valueOf(value); - } else if (type.equals("float")) { - return Float.valueOf(value); - } else if (type.equals("double")) { - return Double.valueOf(value); - } else if (type.equals("string")) { - return unquote(value); - } - throw new UnsupportedOperationException("Unsupported literal type: " + type); - } - - private static int compare(Object left, Object right) { - if (left instanceof BigDecimal || right instanceof BigDecimal) { - return toBigDecimal(left).compareTo(toBigDecimal(right)); - } - if (left instanceof Number && right instanceof Number) { - return Double.compare(((Number) left).doubleValue(), - ((Number) right).doubleValue()); - } - return left.toString().compareTo(right.toString()); - } - - private static BigDecimal toBigDecimal(Object value) { - if (value instanceof BigDecimal) { - return (BigDecimal) value; - } - return new BigDecimal(value.toString()); - } - - private boolean like(String value, String pattern) { - final String unquotedPattern = unquote(pattern); - final Pattern compiledPattern = - likePatterns.computeIfAbsent(unquotedPattern, p -> { - return Pattern.compile(toRegex(p), Pattern.DOTALL); - }); - return compiledPattern.matcher(value).matches(); - } - - private static String toRegex(String pattern) { - final StringBuilder builder = new StringBuilder(); - for (int i = 0; i < pattern.length(); i++) { - final char c = pattern.charAt(i); - if (c == '%') { - builder.append(".*"); - } else if (c == '_') { - builder.append('.'); - } else { - builder.append(Pattern.quote(String.valueOf(c))); - } + } else { + currRowIndex = selectionVector.getIndex(selectionVectorIndex++); + return true; } - return builder.toString(); } - private static String unquote(String value) { - if (value.length() >= 2 && value.charAt(0) == '\'' - && value.charAt(value.length() - 1) == '\'') { - return value.substring(1, value.length() - 1).replace("''", "'"); - } - return value; - } - - private static List> toConditionTokens( - List>> conditions) { - final List> result = - new ArrayList<>(conditions.size()); - for (List> orGroup : conditions) { - final List tokens = new ArrayList<>(orGroup.size()); - for (List token : orGroup) { - tokens.add(ConditionToken.fromTokenList(token)); + @Override public void close() { + try { + if (buf != null) { + buf.close(); } - result.add(tokens); + filter.close(); + } catch (GandivaException e) { + throw Util.toUnchecked(e); + } finally { + onClose.run(); } - return result; - } - - @Override public void close() { - onClose.run(); } } diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowProjectEnumerator.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowProjectEnumerator.java new file mode 100644 index 000000000000..0895f36cf15f --- /dev/null +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowProjectEnumerator.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.adapter.arrow; + +import org.apache.calcite.util.ImmutableIntList; +import org.apache.calcite.util.Util; + +import org.apache.arrow.gandiva.evaluator.Projector; +import org.apache.arrow.gandiva.exceptions.GandivaException; +import org.apache.arrow.vector.ipc.ArrowFileReader; +import org.apache.arrow.vector.ipc.message.ArrowRecordBatch; + +import java.io.IOException; + +/** + * Enumerator that reads from a projected collection of Arrow value-vectors. + */ +class ArrowProjectEnumerator extends AbstractArrowEnumerator { + private final Projector projector; + private final Runnable onClose; + + ArrowProjectEnumerator(ArrowFileReader arrowFileReader, ImmutableIntList fields, + Projector projector, Runnable onClose) { + super(arrowFileReader, fields); + this.projector = projector; + this.onClose = onClose; + } + + @Override protected void evaluateOperator(ArrowRecordBatch arrowRecordBatch) { + try { + projector.evaluate(arrowRecordBatch, valueVectors); + } catch (GandivaException e) { + throw Util.toUnchecked(e); + } + } + + @Override public boolean moveNext() { + if (currRowIndex >= rowCount - 1) { + final boolean hasNextBatch; + try { + hasNextBatch = arrowFileReader.loadNextBatch(); + } catch (IOException e) { + throw Util.toUnchecked(e); + } + if (hasNextBatch) { + currRowIndex = 0; + this.valueVectors.clear(); + loadNextArrowBatch(); + } + return hasNextBatch; + } else { + currRowIndex++; + return true; + } + } + + @Override public void close() { + try { + projector.close(); + } catch (GandivaException e) { + throw Util.toUnchecked(e); + } finally { + onClose.run(); + } + } +} diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowRules.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowRules.java index 3da1527f1014..6e268d646928 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowRules.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowRules.java @@ -100,7 +100,7 @@ RelNode convert(Filter filter) { final RelTraitSet traitSet = filter.getTraitSet().replace(ArrowRel.CONVENTION); // Expand SEARCH (e.g. IN, BETWEEN) before pushing to Arrow, - // since the Arrow adapter does not support SEARCH natively. + // since Gandiva does not support SEARCH natively. final RexNode condition = RexUtil.expandSearch(filter.getCluster().getRexBuilder(), null, filter.getCondition()); return new ArrowFilter(filter.getCluster(), traitSet, diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTable.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTable.java index 2585f5d156ea..74438efe2a33 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTable.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTable.java @@ -38,9 +38,17 @@ import org.apache.calcite.util.ImmutableIntList; import org.apache.calcite.util.Util; +import org.apache.arrow.gandiva.evaluator.Filter; +import org.apache.arrow.gandiva.evaluator.Projector; +import org.apache.arrow.gandiva.exceptions.GandivaException; +import org.apache.arrow.gandiva.expression.Condition; +import org.apache.arrow.gandiva.expression.ExpressionTree; +import org.apache.arrow.gandiva.expression.TreeBuilder; +import org.apache.arrow.gandiva.expression.TreeNode; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.ipc.ArrowFileReader; import org.apache.arrow.vector.ipc.SeekableReadChannel; +import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.Schema; @@ -50,15 +58,20 @@ import java.io.FileInputStream; import java.io.IOException; import java.lang.reflect.Type; +import java.util.ArrayList; import java.util.List; +import static java.lang.Double.parseDouble; +import static java.lang.Float.parseFloat; +import static java.lang.Integer.parseInt; +import static java.lang.Long.parseLong; import static java.util.Objects.requireNonNull; /** * Table backed by an Apache Arrow file. * - *

      Reads data from an Arrow IPC file on disk. Projections and filters read - * directly from Arrow value-vectors. + *

      Reads data from an Arrow IPC file on disk and supports projection + * and filter push-down via the Gandiva expression compiler. * *

      Implements {@link TranslatableTable} so that it can be converted into * an {@link ArrowTableScan} for query planning, and {@link QueryableTable} @@ -103,6 +116,43 @@ public class ArrowTable extends AbstractTable public Enumerable query(DataContext root, ImmutableIntList fields, List>> conditions) { requireNonNull(fields, "fields"); + final Projector projector; + final Filter filter; + + if (conditions.isEmpty()) { + filter = null; + projector = makeProjector(fields); + } else { + projector = null; + + final List conjuncts = new ArrayList<>(conditions.size()); + for (List> orGroup : conditions) { + final List disjuncts = new ArrayList<>(orGroup.size()); + for (List conditionParts : orGroup) { + disjuncts.add( + convertConditionToGandiva( + ConditionToken.fromTokenList(conditionParts))); + } + if (disjuncts.size() == 1) { + conjuncts.add(disjuncts.get(0)); + } else { + conjuncts.add(TreeBuilder.makeOr(disjuncts)); + } + } + final Condition filterCondition; + if (conjuncts.size() == 1) { + filterCondition = TreeBuilder.makeCondition(conjuncts.get(0)); + } else { + filterCondition = + TreeBuilder.makeCondition(TreeBuilder.makeAnd(conjuncts)); + } + + try { + filter = Filter.make(schema, filterCondition); + } catch (GandivaException e) { + throw Util.toUnchecked(e); + } + } FileInputStream fis = null; try { @@ -113,7 +163,7 @@ public Enumerable query(DataContext root, ImmutableIntList fields, final FileInputStream fisRef = fis; final Runnable onClose = () -> closeSilently(fisRef); fis = null; // ownership transferred to onClose - return new ArrowEnumerable(reader, fields, conditions, schema, onClose); + return new ArrowEnumerable(reader, fields, projector, filter, onClose); } catch (IOException e) { throw Util.toUnchecked(e); } finally { @@ -152,6 +202,70 @@ private static RelDataType deduceRowType(Schema schema, return builder.build(); } + private @Nullable Projector makeProjector(ImmutableIntList fields) { + if (requiresDirectVectorProjection(fields)) { + // Returning null selects ArrowEnumerable's direct vector-read path. + // Use that path because Gandiva does not support identity projection + // expressions over Arrow List and binary vectors. + return null; + } + + final List expressionTrees = new ArrayList<>(); + for (int fieldOrdinal : fields) { + Field field = schema.getFields().get(fieldOrdinal); + TreeNode node = TreeBuilder.makeField(field); + expressionTrees.add(TreeBuilder.makeExpression(node, field)); + } + try { + return Projector.make(schema, expressionTrees); + } catch (GandivaException e) { + throw Util.toUnchecked(e); + } + } + + /** Returns whether selected fields should be projected by reading Arrow + * value-vectors directly rather than by creating a Gandiva projector. + * + *

      CALCITE-7541 extends this direct projection path for Arrow binary vector + * families because Gandiva cannot project them through the existing identity + * projection path. Queries with filters still use Gandiva filters; this direct + * path only applies to no-filter projections. + */ + private boolean requiresDirectVectorProjection(ImmutableIntList fields) { + for (int fieldOrdinal : fields) { + switch (schema.getFields().get(fieldOrdinal).getType().getTypeID()) { + case List: + case Binary: + case LargeBinary: + case FixedSizeBinary: + return true; + default: + break; + } + } + return false; + } + + /** Converts a single {@link ConditionToken} into a Gandiva {@link TreeNode}. */ + private TreeNode convertConditionToGandiva(ConditionToken token) { + final List treeNodes = new ArrayList<>(2); + treeNodes.add( + TreeBuilder.makeField(schema.getFields() + .get( + schema.getFields().indexOf( + schema.findField(token.fieldName))))); + + if (token.isBinary()) { + treeNodes.add( + makeLiteralNode( + requireNonNull(token.value, "value"), + requireNonNull(token.valueType, "valueType"))); + } + + return TreeBuilder.makeFunction( + token.operator, treeNodes, new ArrowType.Bool()); + } + /** Closes an {@link AutoCloseable} without throwing. */ private static void closeSilently(AutoCloseable closeable) { try { @@ -161,6 +275,29 @@ private static void closeSilently(AutoCloseable closeable) { } } + private static TreeNode makeLiteralNode(String literal, String type) { + if (type.startsWith("decimal")) { + String[] typeParts = + type.substring(type.indexOf('(') + 1, type.indexOf(')')).split(","); + int precision = parseInt(typeParts[0]); + int scale = parseInt(typeParts[1]); + return TreeBuilder.makeDecimalLiteral(literal, precision, scale); + } else if (type.equals("integer")) { + return TreeBuilder.makeLiteral(parseInt(literal)); + } else if (type.equals("long")) { + return TreeBuilder.makeLiteral(parseLong(literal)); + } else if (type.equals("float")) { + return TreeBuilder.makeLiteral(parseFloat(literal)); + } else if (type.equals("double")) { + return TreeBuilder.makeLiteral(parseDouble(literal)); + } else if (type.equals("string")) { + return TreeBuilder.makeStringLiteral(literal.substring(1, literal.length() - 1)); + } else { + throw new IllegalArgumentException("Invalid literal " + literal + + ", type " + type); + } + } + /** * Implementation of {@link Queryable} based on a {@link ArrowTable}. * diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTranslator.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTranslator.java index 2b4229598153..0ec680405270 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTranslator.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTranslator.java @@ -35,26 +35,13 @@ import java.util.ArrayList; import java.util.List; -import static org.apache.calcite.adapter.arrow.ConditionToken.Operator.EQUAL; -import static org.apache.calcite.adapter.arrow.ConditionToken.Operator.GREATER_THAN; -import static org.apache.calcite.adapter.arrow.ConditionToken.Operator.GREATER_THAN_OR_EQUAL; -import static org.apache.calcite.adapter.arrow.ConditionToken.Operator.IS_FALSE; -import static org.apache.calcite.adapter.arrow.ConditionToken.Operator.IS_NOT_FALSE; -import static org.apache.calcite.adapter.arrow.ConditionToken.Operator.IS_NOT_NULL; -import static org.apache.calcite.adapter.arrow.ConditionToken.Operator.IS_NOT_TRUE; -import static org.apache.calcite.adapter.arrow.ConditionToken.Operator.IS_NULL; -import static org.apache.calcite.adapter.arrow.ConditionToken.Operator.IS_TRUE; -import static org.apache.calcite.adapter.arrow.ConditionToken.Operator.LESS_THAN; -import static org.apache.calcite.adapter.arrow.ConditionToken.Operator.LESS_THAN_OR_EQUAL; -import static org.apache.calcite.adapter.arrow.ConditionToken.Operator.LIKE; -import static org.apache.calcite.adapter.arrow.ConditionToken.Operator.NOT_EQUAL; import static org.apache.calcite.util.DateTimeStringUtils.ISO_DATETIME_FRACTIONAL_SECOND_FORMAT; import static org.apache.calcite.util.DateTimeStringUtils.getDateFormatter; import static java.util.Objects.requireNonNull; /** - * Translates a {@link RexNode} expression to Arrow predicate tokens. + * Translates a {@link RexNode} expression to Gandiva predicate tokens. */ class ArrowTranslator { final RexBuilder rexBuilder; @@ -78,7 +65,7 @@ public static ArrowTranslator create(RexBuilder rexBuilder, * *

      If exceeded, {@link RexUtil#toCnf(RexBuilder, int, RexNode)} returns * the original expression unchanged, which may cause the subsequent - * translation to Arrow predicates to fail with an + * translation to Gandiva predicates to fail with an * {@link UnsupportedOperationException}. When invoked by the Arrow adapter * module, the exception is caught and the plan falls back to * an Enumerable convention. */ @@ -133,32 +120,32 @@ private static Object literalValue(RexLiteral literal) { private ConditionToken translateMatch2(RexNode node) { switch (node.getKind()) { case EQUALS: - return translateBinary(EQUAL, EQUAL, (RexCall) node); + return translateBinary("equal", "=", (RexCall) node); case NOT_EQUALS: - return translateBinary(NOT_EQUAL, NOT_EQUAL, (RexCall) node); + return translateBinary("not_equal", "<>", (RexCall) node); case LESS_THAN: - return translateBinary(LESS_THAN, GREATER_THAN, (RexCall) node); + return translateBinary("less_than", ">", (RexCall) node); case LESS_THAN_OR_EQUAL: - return translateBinary(LESS_THAN_OR_EQUAL, GREATER_THAN_OR_EQUAL, (RexCall) node); + return translateBinary("less_than_or_equal_to", ">=", (RexCall) node); case GREATER_THAN: - return translateBinary(GREATER_THAN, LESS_THAN, (RexCall) node); + return translateBinary("greater_than", "<", (RexCall) node); case GREATER_THAN_OR_EQUAL: - return translateBinary(GREATER_THAN_OR_EQUAL, LESS_THAN_OR_EQUAL, (RexCall) node); + return translateBinary("greater_than_or_equal_to", "<=", (RexCall) node); case IS_NULL: - return translateUnary(IS_NULL, (RexCall) node); + return translateUnary("isnull", (RexCall) node); case IS_NOT_NULL: - return translateUnary(IS_NOT_NULL, (RexCall) node); + return translateUnary("isnotnull", (RexCall) node); case IS_NOT_TRUE: - return translateUnary(IS_NOT_TRUE, (RexCall) node); + return translateUnary("isnottrue", (RexCall) node); case IS_NOT_FALSE: - return translateUnary(IS_NOT_FALSE, (RexCall) node); + return translateUnary("isnotfalse", (RexCall) node); case INPUT_REF: final RexInputRef inputRef = (RexInputRef) node; - return ConditionToken.unary(fieldNames.get(inputRef.getIndex()), IS_TRUE); + return ConditionToken.unary(fieldNames.get(inputRef.getIndex()), "istrue"); case NOT: - return translateUnary(IS_FALSE, (RexCall) node); + return translateUnary("isfalse", (RexCall) node); case LIKE: - return translateBinaryNoReverse(LIKE, (RexCall) node); + return translateBinary("like", null, (RexCall) node); default: throw new UnsupportedOperationException("Unsupported operator " + node); } @@ -168,8 +155,7 @@ private ConditionToken translateMatch2(RexNode node) { * Translates a call to a binary operator, reversing arguments if * necessary. */ - private ConditionToken translateBinary(ConditionToken.Operator op, - ConditionToken.Operator rop, RexCall call) { + private ConditionToken translateBinary(String op, String rop, RexCall call) { final RexNode left = call.operands.get(0); final RexNode right = call.operands.get(1); @Nullable ConditionToken expression = translateBinary2(op, left, right); @@ -183,21 +169,9 @@ private ConditionToken translateBinary(ConditionToken.Operator op, throw new UnsupportedOperationException("Unsupported binary operator " + call); } - /** Translates a call to a binary operator without reversing arguments. */ - private ConditionToken translateBinaryNoReverse(ConditionToken.Operator op, - RexCall call) { - final RexNode left = call.operands.get(0); - final RexNode right = call.operands.get(1); - @Nullable ConditionToken expression = translateBinary2(op, left, right); - if (expression != null) { - return expression; - } - throw new UnsupportedOperationException("Unsupported binary operator " + call); - } - /** Translates a call to a binary operator. Returns null on failure. */ - private @Nullable ConditionToken translateBinary2( - ConditionToken.Operator op, RexNode left, RexNode right) { + private @Nullable ConditionToken translateBinary2(String op, RexNode left, + RexNode right) { if (right.getKind() != SqlKind.LITERAL) { return null; } @@ -217,7 +191,7 @@ private ConditionToken translateBinaryNoReverse(ConditionToken.Operator op, /** Combines a field name, operator, and literal to produce a binary * condition token. */ - private ConditionToken translateOp2(ConditionToken.Operator op, String name, + private ConditionToken translateOp2(String op, String name, RexLiteral right) { Object value = literalValue(right); String valueString = value.toString(); @@ -235,7 +209,7 @@ private ConditionToken translateOp2(ConditionToken.Operator op, String name, } /** Translates a call to a unary operator. */ - private ConditionToken translateUnary(ConditionToken.Operator op, RexCall call) { + private ConditionToken translateUnary(String op, RexCall call) { final RexNode opNode = call.operands.get(0); @Nullable ConditionToken expression = translateUnary2(op, opNode); @@ -247,8 +221,7 @@ private ConditionToken translateUnary(ConditionToken.Operator op, RexCall call) } /** Translates a call to a unary operator. Returns null on failure. */ - private @Nullable ConditionToken translateUnary2(ConditionToken.Operator op, - RexNode opNode) { + private @Nullable ConditionToken translateUnary2(String op, RexNode opNode) { if (opNode.getKind() == SqlKind.INPUT_REF) { final RexInputRef inputRef = (RexInputRef) opNode; final String name = fieldNames.get(inputRef.getIndex()); diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ConditionToken.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ConditionToken.java index c5b690840add..44d3facea77f 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ConditionToken.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ConditionToken.java @@ -25,7 +25,7 @@ import static java.util.Objects.requireNonNull; /** - * A structured representation of a single Arrow predicate condition. + * A structured representation of a single Gandiva predicate condition. * *

      A condition is either unary (e.g. {@code IS NULL}) or binary * (e.g. {@code =}, {@code <}). Unary conditions have a field name @@ -36,11 +36,11 @@ */ class ConditionToken { final String fieldName; - final Operator operator; + final String operator; final @Nullable String value; final @Nullable String valueType; - private ConditionToken(String fieldName, Operator operator, + private ConditionToken(String fieldName, String operator, @Nullable String value, @Nullable String valueType) { this.fieldName = requireNonNull(fieldName, "fieldName"); this.operator = requireNonNull(operator, "operator"); @@ -50,7 +50,7 @@ private ConditionToken(String fieldName, Operator operator, /** Creates a binary condition token * (e.g. {@code intField equal 12 integer}). */ - static ConditionToken binary(String fieldName, Operator operator, + static ConditionToken binary(String fieldName, String operator, String value, String valueType) { return new ConditionToken(fieldName, operator, requireNonNull(value, "value"), @@ -59,7 +59,7 @@ static ConditionToken binary(String fieldName, Operator operator, /** Creates a unary condition token * (e.g. {@code intField isnull}). */ - static ConditionToken unary(String fieldName, Operator operator) { + static ConditionToken unary(String fieldName, String operator) { return new ConditionToken(fieldName, operator, null, null); } @@ -76,55 +76,22 @@ boolean isBinary() { * binary conditions. */ List toTokenList() { if (isBinary()) { - return ImmutableList.of(fieldName, operator.token, + return ImmutableList.of(fieldName, operator, requireNonNull(value, "value"), requireNonNull(valueType, "valueType")); } - return ImmutableList.of(fieldName, operator.token); + return ImmutableList.of(fieldName, operator); } /** Creates a {@code ConditionToken} from a serialized string list. */ static ConditionToken fromTokenList(List tokens) { final int size = tokens.size(); if (size == 4) { - return binary(tokens.get(0), Operator.of(tokens.get(1)), + return binary(tokens.get(0), tokens.get(1), tokens.get(2), tokens.get(3)); } else if (size == 2) { - return unary(tokens.get(0), Operator.of(tokens.get(1))); + return unary(tokens.get(0), tokens.get(1)); } throw new IllegalArgumentException("Invalid condition tokens: " + tokens); } - - /** Operators supported by the Arrow adapter filter representation. */ - enum Operator { - IS_NULL("isnull"), - IS_NOT_NULL("isnotnull"), - IS_TRUE("istrue"), - IS_FALSE("isfalse"), - IS_NOT_TRUE("isnottrue"), - IS_NOT_FALSE("isnotfalse"), - EQUAL("equal"), - NOT_EQUAL("not_equal"), - LESS_THAN("less_than"), - LESS_THAN_OR_EQUAL("less_than_or_equal_to"), - GREATER_THAN("greater_than"), - GREATER_THAN_OR_EQUAL("greater_than_or_equal_to"), - LIKE("like"); - - final String token; - - Operator(String token) { - this.token = token; - } - - static Operator of(String token) { - for (Operator operator : values()) { - if (operator.token.equals(token)) { - return operator; - } - } - throw new UnsupportedOperationException( - "Unsupported Arrow filter operator: " + token); - } - } } diff --git a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java index 8c9fda7f081f..275bdd0be76c 100644 --- a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java +++ b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java @@ -82,11 +82,6 @@ static void initializeArrowState(@TempDir Path sharedTempDir) arrowDataGenerator.writeArrowData(dataLocationFile); arrowDataGenerator.writeScottEmpData(arrowFilesDirectory); - File emptyBatchDataLocationFile = - arrowFilesDirectory.resolve("arrowemptybatch.arrow").toFile(); - ArrowDataTest emptyBatchDataGenerator = new ArrowDataTest(); - emptyBatchDataGenerator.writeArrowDataWithEmptyBatch(emptyBatchDataLocationFile); - File datatypeLocationFile = arrowFilesDirectory.resolve("arrowdatatype.arrow").toFile(); ArrowDataTest arrowtypeDataGenerator = new ArrowDataTest(); arrowtypeDataGenerator.writeArrowDataType(datatypeLocationFile); @@ -265,51 +260,6 @@ static void initializeArrowState(@TempDir Path sharedTempDir) .explainContains(plan); } - /** Test case for - * [CALCITE-7580] - * Remove Gandiva dependency from Arrow adapter. */ - @Test void testArrowProjectSkipsEmptyBatch() { - String sql = "select \"intField\", \"stringField\" from arrowemptybatch\n"; - String result = "intField=0; stringField=0\n" - + "intField=1; stringField=1\n" - + "intField=2; stringField=2\n"; - - CalciteAssert.that() - .with(arrow) - .query(sql) - .returns(result); - } - - /** Test case for - * [CALCITE-7580] - * Remove Gandiva dependency from Arrow adapter. */ - @Test void testArrowFilterSkipsEmptyBatch() { - String sql = "select \"intField\", \"stringField\"\n" - + "from arrowemptybatch\n" - + "where \"intField\" > 0"; - String result = "intField=1; stringField=1\n" - + "intField=2; stringField=2\n"; - - CalciteAssert.that() - .with(arrow) - .query(sql) - .returns(result); - } - - /** Test case for - * [CALCITE-7580] - * Remove Gandiva dependency from Arrow adapter. */ - @Test void testArrowFilterSkipsEmptyBatchWithNoMatches() { - String sql = "select \"intField\", \"stringField\"\n" - + "from arrowemptybatch\n" - + "where \"intField\" < 0"; - - CalciteAssert.that() - .with(arrow) - .query(sql) - .returns(""); - } - @Test void testArrowProjectFieldsWithIntegerFilter() { String sql = "select \"intField\", \"stringField\"\n" + "from arrowdata\n" diff --git a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowDataTest.java b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowDataTest.java index 4b9af441aabf..a53cc1231ad3 100644 --- a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowDataTest.java +++ b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowDataTest.java @@ -301,35 +301,6 @@ public void writeArrowBinaryData(File file) throws IOException { fileOutputStream.close(); } - public void writeArrowDataWithEmptyBatch(File file) throws IOException { - Schema arrowSchema = makeArrowSchema(); - try (RootAllocator allocator = new RootAllocator(Integer.MAX_VALUE); - VectorSchemaRoot vectorSchemaRoot = - VectorSchemaRoot.create(arrowSchema, allocator); - FileOutputStream fileOutputStream = new FileOutputStream(file); - ArrowFileWriter arrowFileWriter = - new ArrowFileWriter(vectorSchemaRoot, null, - fileOutputStream.getChannel())) { - arrowFileWriter.start(); - - vectorSchemaRoot.setRowCount(0); - for (Field field : vectorSchemaRoot.getSchema().getFields()) { - vectorSchemaRoot.getVector(field.getName()).setValueCount(0); - } - arrowFileWriter.writeBatch(); - - int rowCount = 3; - vectorSchemaRoot.setRowCount(rowCount); - intField(vectorSchemaRoot.getVector("intField"), rowCount); - varCharField(vectorSchemaRoot.getVector("stringField"), rowCount); - floatField(vectorSchemaRoot.getVector("floatField"), rowCount); - longField(vectorSchemaRoot.getVector("longField"), rowCount); - arrowFileWriter.writeBatch(); - - arrowFileWriter.end(); - } - } - public void writeArrowDataType(File file) throws IOException { FileOutputStream fileOutputStream = new FileOutputStream(file); Schema arrowSchema = makeArrowDateTypeSchema(); diff --git a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowExtension.java b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowExtension.java index 4600dab1001c..ab1f5c2a88c1 100644 --- a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowExtension.java +++ b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowExtension.java @@ -18,12 +18,22 @@ import org.apache.calcite.config.CalciteSystemProperty; +import org.apache.arrow.gandiva.evaluator.Projector; +import org.apache.arrow.gandiva.exceptions.GandivaException; +import org.apache.arrow.gandiva.expression.ExpressionTree; +import org.apache.arrow.vector.types.pojo.Schema; + import org.junit.jupiter.api.extension.ConditionEvaluationResult; import org.junit.jupiter.api.extension.ExecutionCondition; import org.junit.jupiter.api.extension.ExtensionContext; +import java.util.ArrayList; +import java.util.List; + /** * JUnit5 extension to handle Arrow tests. + * + *

      Tests will be skipped if the Gandiva library cannot be loaded on the given platform. */ class ArrowExtension implements ExecutionCondition { @@ -31,7 +41,8 @@ class ArrowExtension implements ExecutionCondition { * Whether to run this test. * *

      Enabled by default, unless explicitly disabled from command line - * ({@code -Dcalcite.test.arrow=false}). + * ({@code -Dcalcite.test.arrow=false}) or if Gandiva library, used to implement arrow + * filtering/projection, cannot be loaded. * * @return {@code true} if the test is enabled and can run in the current environment, * {@code false} otherwise @@ -40,6 +51,16 @@ class ArrowExtension implements ExecutionCondition { final ExtensionContext context) { boolean enabled = CalciteSystemProperty.TEST_ARROW.value(); + try { + Schema emptySchema = new Schema(new ArrayList<>(), null); + List expressions = new ArrayList<>(); + Projector.make(emptySchema, expressions); + } catch (GandivaException e) { + // this exception comes from using an empty expression, + // but the JNI library was loaded properly + } catch (UnsatisfiedLinkError e) { + enabled = false; + } if (enabled) { return ConditionEvaluationResult.enabled("Arrow tests enabled"); diff --git a/bom/build.gradle.kts b/bom/build.gradle.kts index 64cb532afdc3..f00ed7d8e556 100644 --- a/bom/build.gradle.kts +++ b/bom/build.gradle.kts @@ -101,6 +101,7 @@ dependencies { apiv("org.apache.arrow:arrow-memory-netty", "arrow") apiv("org.apache.arrow:arrow-vector", "arrow") apiv("org.apache.arrow:arrow-jdbc", "arrow") + apiv("org.apache.arrow.gandiva:arrow-gandiva", "arrow-gandiva") apiv("org.apache.calcite.avatica:avatica-core", "calcite.avatica") apiv("org.apache.calcite.avatica:avatica-server", "calcite.avatica") apiv("org.apache.cassandra:cassandra-all") diff --git a/core/src/main/java/org/apache/calcite/rex/RexSimplify.java b/core/src/main/java/org/apache/calcite/rex/RexSimplify.java index 4d7702295bff..0ffb60454e33 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexSimplify.java +++ b/core/src/main/java/org/apache/calcite/rex/RexSimplify.java @@ -529,11 +529,14 @@ private RexNode simplifyLike(RexCall e, RexUnknownAs unknownAs) { } if (e.operands.size() == 3 && e.operands.get(2) instanceof RexLiteral) { final RexLiteral escapeLiteral = (RexLiteral) e.operands.get(2); - Character escape = requireNonNull(escapeLiteral.getValueAs(Character.class)); - e = (RexCall) rexBuilder - .makeCall(e.getParserPosition(), e.getOperator(), e.operands.get(0), - rexBuilder.makeLiteral(simplifyLikeString(likeStr, escape, '%'), - e.operands.get(1).getType(), true, true), escapeLiteral); + final String escapeStr = requireNonNull(escapeLiteral.getValueAs(String.class)); + if (escapeStr.length() == 1) { + char escape = escapeStr.charAt(0); + e = (RexCall) rexBuilder + .makeCall(e.getParserPosition(), e.getOperator(), e.operands.get(0), + rexBuilder.makeLiteral(simplifyLikeString(likeStr, escape, '%'), + e.operands.get(1).getType(), true, true), escapeLiteral); + } } } return simplifyGenericNode(e); @@ -543,7 +546,7 @@ private RexNode simplifyLike(RexCall e, RexUnknownAs unknownAs) { // string with even escapes 'AA\\\\%%__%%AA' simplify to 'AA\\__%AA' // string with odd escapes 'AA\\\\\\%%__%%AA' simplify to 'AA\\\\\\%__%AA' private String simplifyMixedWildcards(String str, char escape) { - Pattern pattern = Pattern.compile("[_%]+"); + Pattern pattern = getWildCardPattern(escape); Matcher matcher = pattern.matcher(str); StringBuilder builder = new StringBuilder(); int from = 0; @@ -567,6 +570,17 @@ && consecutiveSameCharCountBefore(str, start - 1, escape) % 2 == 1) { return builder.toString(); } + private static Pattern getWildCardPattern(char escape) { + switch (escape) { + case '%': + return Pattern.compile("_+"); + case '_': + return Pattern.compile("%+"); + default: + return Pattern.compile("[_%]+"); + } + } + // Tool method: count the number of consecutive identical characters before index private int consecutiveSameCharCountBefore(String str, int index, char escape) { int count = 0; diff --git a/core/src/main/java/org/apache/calcite/sql/type/SqlTypeFactoryImpl.java b/core/src/main/java/org/apache/calcite/sql/type/SqlTypeFactoryImpl.java index 115b66fa215f..d0ccab7dfd2f 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/SqlTypeFactoryImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/type/SqlTypeFactoryImpl.java @@ -561,7 +561,7 @@ private static void assertBasic(SqlTypeName typeName) { } } - if (type.getSqlTypeName() == resultType.getSqlTypeName() + if (type.getSqlTypeName().getFamily() == resultType.getSqlTypeName().getFamily() && type.getSqlTypeName().allowsPrec() && type.getPrecision() != resultType.getPrecision()) { final int precision = diff --git a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java index 4d14e16937ef..882d42387098 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java @@ -1870,7 +1870,7 @@ public RelNode convertToSingleValueSubq( if (leftKeys.size() == 1) { SqlCall sqlCall = comparisonOp.createCall(rightVals.getParserPosition(), leftKeys.get(0), rightVals); - rexComparison = bb.convertExpression(sqlCall); + rexComparison = ensureComparisonTypes(bb.convertExpression(sqlCall)); } else { assert rightVals instanceof SqlCall; final SqlBasicCall call = (SqlBasicCall) rightVals; @@ -1880,9 +1880,10 @@ public RelNode convertToSingleValueSubq( RexUtil.composeConjunction(rexBuilder, transform( Pair.zip(leftKeys, call.getOperandList()), - pair -> bb.convertExpression( - comparisonOp.createCall(rightVals.getParserPosition(), - pair.left, pair.right)))); + pair -> ensureComparisonTypes( + bb.convertExpression( + comparisonOp.createCall(rightVals.getParserPosition(), + pair.left, pair.right))))); } comparisons.add(rexComparison); } @@ -1901,6 +1902,31 @@ public RelNode convertToSingleValueSubq( } } + /** + * Ensures that a comparison expression has matching operand types. If the + * operands have different type names, casts the right operand to match the + * left operand's type. This handles the case where type coercion is disabled + * and the IN-to-OR expansion produces comparisons with mismatched types + * (e.g., DATE = CHAR). + */ + private RexNode ensureComparisonTypes(RexNode node) { + if (validator != null && validator.config().typeCoercionEnabled()) { + return node; + } + if (node instanceof RexCall) { + final RexCall call = (RexCall) node; + if (call.operands.size() == 2) { + final RexNode left = call.operands.get(0); + final RexNode right = call.operands.get(1); + if (left.getType().getSqlTypeName() != right.getType().getSqlTypeName()) { + final RexNode castRight = rexBuilder.ensureType(left.getType(), right, true); + return rexBuilder.makeCall(call.getOperator(), left, castRight); + } + } + } + return node; + } + /** * Converts a {@link SqlNodeList} (for example an IN-list or VALUES list) * into a relational expression and produces a Rex-level sub-query that diff --git a/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java b/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java index 2bdc7a1d56c4..ac7b5aebe6c3 100644 --- a/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java +++ b/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java @@ -4358,6 +4358,10 @@ private void checkSarg(String message, Sarg sarg, * Multiple consecutive '%' in the string matched by LIKE should simplify to a single '%', * [CALCITE-7153] * Mixed wildcards of _ and % need to be simplified in LIKE operator. + * [CALCITE-7578] + * LIKE with empty ESCAPE might fail with StringIndexOutOfBoundsException. + * [CALCITE-7588] + * LIKE with ESCAPE symbols containing wildcards fails. * */ @Test void testSimplifyLike() { final RexNode ref = input(tVarchar(true, 10), 0); @@ -4421,6 +4425,14 @@ private void checkSarg(String message, Sarg sarg, "LIKE($0, '###%%#%#%A#%%#%A%###%%', '#')"); checkSimplifyUnchanged(like(ref, literal("A"), literal("#"))); checkSimplifyUnchanged(like(ref, literal("%A"), literal("#"))); + checkSimplifyUnchanged(like(ref, literal("TE%_ST"), literal("%"))); + checkSimplifyUnchanged(like(ref, literal("TE%%ST"), literal("%"))); + checkSimplifyUnchanged(like(ref, literal("a%_b%%c"), literal("%"))); + checkSimplifyUnchanged(like(ref, literal("%_%%A%_"), literal("%"))); + // escape char equal to the '_' wildcard. '%E__S%' ESCAPE '_' is a literal '_'. + checkSimplifyUnchanged(like(ref, literal("%E__S%"), literal("_"))); + checkSimplifyUnchanged(like(ref, literal("TE_%ST"), literal("_"))); + checkSimplifyUnchanged(like(ref, literal("a_%b__c"), literal("_"))); // As above, but ref is NOT NULL final RexNode refMandatory = vVarcharNotNull(0); @@ -4451,6 +4463,14 @@ private void checkSarg(String message, Sarg sarg, // NOT(SIMILAR TO) is not optimized checkSimplifyUnchanged( not(rexBuilder.makeCall(SqlStdOperatorTable.SIMILAR_TO, ref, literal("%")))); + + try { + // Empty ESCAPE + checkSimplifyUnchanged(like(ref, literal("a"), literal(""))); + } catch (RuntimeException e) { + assertThat(e.getMessage(), + containsString("Invalid escape character ''")); + } } @Test void testSimplifyNullCheckInFilter() { diff --git a/core/src/test/java/org/apache/calcite/sql/type/SqlTypeFactoryTest.java b/core/src/test/java/org/apache/calcite/sql/type/SqlTypeFactoryTest.java index 8f5e5e4018db..f0ff190d28c7 100644 --- a/core/src/test/java/org/apache/calcite/sql/type/SqlTypeFactoryTest.java +++ b/core/src/test/java/org/apache/calcite/sql/type/SqlTypeFactoryTest.java @@ -33,6 +33,7 @@ import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.notNullValue; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.hasToString; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -172,6 +173,30 @@ class SqlTypeFactoryTest { assertThat(leastRestrictive.getPrecision(), is(3)); } + /** + * Test case for + * + * LeastRetrictiveSqlType for TIMESTAMP, TIMESTAMP_LTZ might ignore precision. */ + @Test void testLeastRestrictiveForTimestampAndTimestampLtz() { + SqlTypeFixture f = new SqlTypeFixture(); + RelDataType ltz0 = + f.typeFactory.createSqlType(SqlTypeName.TIMESTAMP_WITH_LOCAL_TIME_ZONE, 0); + RelDataType leastRestrictive = + f.typeFactory.leastRestrictive(Lists.newArrayList(ltz0, f.sqlTimestampPrec3)); + assertThat(leastRestrictive, is(notNullValue())); + assertThat(leastRestrictive.getPrecision(), is(3)); + } + + @Test void testLeastRestrictiveForTimestampLtzAndTimestamp() { + SqlTypeFixture f = new SqlTypeFixture(); + RelDataType ltz0 = + f.typeFactory.createSqlType(SqlTypeName.TIMESTAMP_WITH_LOCAL_TIME_ZONE, 0); + RelDataType leastRestrictive = + f.typeFactory.leastRestrictive(Lists.newArrayList(f.sqlTimestampPrec3, ltz0)); + assertThat(leastRestrictive, is(notNullValue())); + assertThat(leastRestrictive.getPrecision(), is(3)); + } + @Test void testLeastRestrictiveForTimestampAndDate() { SqlTypeFixture f = new SqlTypeFixture(); RelDataType leastRestrictive = diff --git a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java index 694572683e99..26000620af68 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java @@ -2213,6 +2213,24 @@ void checkCorrelatedMapSubQuery(boolean expand) { sql(sql).withExpand(false).ok(); } + /** Test case for + * [CALCITE-7562] + * SqlToRel misses CAST in case IN expression without type coercion. */ + @Test void testInDateColumnWithoutTypeCoercion() { + final String sql = + "select * from emp_b where birthdate in ('2000-06-30', '2000-09-27')"; + sql(sql).withTypeCoercion(false).ok(); + } + + /** Test case for + * [CALCITE-7562] + * SqlToRel misses CAST in case IN expression without type coercion. */ + @Test void testInDateColumnWithTypeCoercion() { + final String sql = + "select * from emp_b where birthdate in ('2000-06-30', '2000-09-27')"; + sql(sql).ok(); + } + @Test void testInValueListLong() { // Go over the default threshold of 20 to force a sub-query. final String sql = "select empno from emp where deptno in" diff --git a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml index 0bf02c9a73b4..71fe294d2f0c 100644 --- a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml @@ -3305,6 +3305,30 @@ LogicalFilter(condition=[=($cor0.DEPTNO, $7)]) LogicalAggregate(group=[{0}], S=[SUM($1)], agg#1=[COUNT()]) LogicalProject(DEPTNO=[$7], SAL=[$5]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +]]> + + + + + + + + + + + + + + + + diff --git a/gradle.properties b/gradle.properties index fd823182640c..eb0bd778ae14 100644 --- a/gradle.properties +++ b/gradle.properties @@ -81,6 +81,7 @@ jandex.version=3.5.3 # elasticsearch does not like asm:6.2.1+ aggdesigner-algorithm.version=6.1 apiguardian-api.version=1.1.2 +arrow-gandiva.version=15.0.0 arrow.version=15.0.0 asm.version=9.9.1 byte-buddy.version=1.18.8 diff --git a/site/_docs/history.md b/site/_docs/history.md index c72d6f78d938..15e2561d398a 100644 --- a/site/_docs/history.md +++ b/site/_docs/history.md @@ -49,11 +49,6 @@ other software versions as specified in gradle.properties. #### Breaking Changes {: #breaking-1-43-0} -* [CALCITE-7580] - Remove Gandiva dependency from Arrow adapter. Arrow adapter projection and - filter evaluation now run in Java, and the `arrow-gandiva` dependency is no - longer included in the Arrow module or BOM. - #### New features {: #new-features-1-43-0} diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlToRelFixture.java b/testkit/src/main/java/org/apache/calcite/test/SqlToRelFixture.java index a6685cf952f5..784cb1eba1dd 100644 --- a/testkit/src/main/java/org/apache/calcite/test/SqlToRelFixture.java +++ b/testkit/src/main/java/org/apache/calcite/test/SqlToRelFixture.java @@ -178,6 +178,11 @@ public SqlToRelFixture withConformance(SqlConformance conformance) { .withValidatorConfig(c -> c.withConformance(conformance))); } + public SqlToRelFixture withTypeCoercion(boolean enabled) { + return withFactory(f -> + f.withValidatorConfig(c -> c.withTypeCoercionEnabled(enabled))); + } + public SqlToRelFixture withDiffRepos(DiffRepository diffRepos) { return new SqlToRelFixture(sql, decorrelate, tester, factory, trim, expression, diffRepos); From d5a19142a6b9dc11c1b449ddd3c1b93beabc7907 Mon Sep 17 00:00:00 2001 From: "cc.cai" <2356672992@qq.com> Date: Fri, 19 Jun 2026 17:23:35 +0800 Subject: [PATCH 336/562] [CALCITE-7580] Remove Gandiva dependency from Arrow adapter --- arrow/build.gradle.kts | 1 - .../arrow/AbstractArrowEnumerator.java | 30 ++- .../adapter/arrow/ArrowDirectEnumerator.java | 31 +-- .../adapter/arrow/ArrowEnumerable.java | 25 +- .../adapter/arrow/ArrowFilterEnumerator.java | 238 +++++++++++++----- .../adapter/arrow/ArrowProjectEnumerator.java | 80 ------ .../calcite/adapter/arrow/ArrowRules.java | 2 +- .../calcite/adapter/arrow/ArrowTable.java | 143 +---------- .../adapter/arrow/ArrowTranslator.java | 69 +++-- .../calcite/adapter/arrow/ConditionToken.java | 51 +++- .../adapter/arrow/ArrowAdapterTest.java | 50 ++++ .../calcite/adapter/arrow/ArrowDataTest.java | 29 +++ .../calcite/adapter/arrow/ArrowExtension.java | 23 +- bom/build.gradle.kts | 1 - gradle.properties | 1 - site/_docs/history.md | 5 + 16 files changed, 397 insertions(+), 382 deletions(-) delete mode 100644 arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowProjectEnumerator.java diff --git a/arrow/build.gradle.kts b/arrow/build.gradle.kts index 598aa8a87972..c75a8b5f6751 100644 --- a/arrow/build.gradle.kts +++ b/arrow/build.gradle.kts @@ -23,7 +23,6 @@ dependencies { implementation("com.google.guava:guava") implementation("org.apache.arrow:arrow-memory-netty") implementation("org.apache.arrow:arrow-vector") - implementation("org.apache.arrow.gandiva:arrow-gandiva") annotationProcessor("org.immutables:value") compileOnly("org.immutables:value-annotations") diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/AbstractArrowEnumerator.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/AbstractArrowEnumerator.java index e188757b0d2c..486e3f60bd8d 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/AbstractArrowEnumerator.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/AbstractArrowEnumerator.java @@ -24,9 +24,7 @@ import org.apache.arrow.vector.TimeStampVector; import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.VectorSchemaRoot; -import org.apache.arrow.vector.VectorUnloader; import org.apache.arrow.vector.ipc.ArrowFileReader; -import org.apache.arrow.vector.ipc.message.ArrowRecordBatch; import org.apache.arrow.vector.types.TimeUnit; import org.apache.arrow.vector.types.pojo.ArrowType; @@ -51,8 +49,6 @@ abstract class AbstractArrowEnumerator implements Enumerator { this.currRowIndex = -1; } - abstract void evaluateOperator(ArrowRecordBatch arrowRecordBatch); - protected void loadNextArrowBatch() { try { final VectorSchemaRoot vsr = arrowFileReader.getVectorSchemaRoot(); @@ -60,14 +56,32 @@ protected void loadNextArrowBatch() { this.valueVectors.add(vsr.getVector(i)); } this.rowCount = vsr.getRowCount(); - VectorUnloader vectorUnloader = new VectorUnloader(vsr); - ArrowRecordBatch arrowRecordBatch = vectorUnloader.getRecordBatch(); - evaluateOperator(arrowRecordBatch); } catch (IOException e) { throw Util.toUnchecked(e); } } + /** Loads the next non-empty Arrow batch. */ + protected boolean loadNextNonEmptyArrowBatch() { + while (true) { + final boolean hasNextBatch; + try { + hasNextBatch = arrowFileReader.loadNextBatch(); + } catch (IOException e) { + throw Util.toUnchecked(e); + } + if (!hasNextBatch) { + return false; + } + currRowIndex = -1; + valueVectors.clear(); + loadNextArrowBatch(); + if (rowCount > 0) { + return true; + } + } + } + @Override public Object current() { if (fields.size() == 1) { return getValue(this.valueVectors.get(0), currRowIndex); @@ -85,7 +99,7 @@ protected void loadNextArrowBatch() { *

      For {@link TimeStampVector}, converts the raw value to * milliseconds since epoch, which is the representation used by * Calcite's Enumerable runtime for TIMESTAMP types. */ - private static Object getValue(ValueVector vector, int index) { + protected static Object getValue(ValueVector vector, int index) { if (vector instanceof TimeStampVector) { if (vector.isNull(index)) { return null; diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowDirectEnumerator.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowDirectEnumerator.java index 0cdec7baeb80..787ffd88d933 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowDirectEnumerator.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowDirectEnumerator.java @@ -17,19 +17,11 @@ package org.apache.calcite.adapter.arrow; import org.apache.calcite.util.ImmutableIntList; -import org.apache.calcite.util.Util; import org.apache.arrow.vector.ipc.ArrowFileReader; -import org.apache.arrow.vector.ipc.message.ArrowRecordBatch; - -import java.io.IOException; /** * Enumerator that reads projected Arrow value-vectors directly. - * - *

      This path is used for identity projections that Gandiva cannot project - * through the existing {@code Projector} path, such as Arrow binary vectors. - * It is not a replacement for Gandiva expression evaluation. */ class ArrowDirectEnumerator extends AbstractArrowEnumerator { private final Runnable onClose; @@ -40,27 +32,14 @@ class ArrowDirectEnumerator extends AbstractArrowEnumerator { this.onClose = onClose; } - @Override protected void evaluateOperator(ArrowRecordBatch arrowRecordBatch) { - } - @Override public boolean moveNext() { - if (currRowIndex >= rowCount - 1) { - final boolean hasNextBatch; - try { - hasNextBatch = arrowFileReader.loadNextBatch(); - } catch (IOException e) { - throw Util.toUnchecked(e); - } - if (hasNextBatch) { - currRowIndex = 0; - this.valueVectors.clear(); - loadNextArrowBatch(); + while (currRowIndex >= rowCount - 1) { + if (!loadNextNonEmptyArrowBatch()) { + return false; } - return hasNextBatch; - } else { - currRowIndex++; - return true; } + currRowIndex++; + return true; } @Override public void close() { diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowEnumerable.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowEnumerable.java index 84ed5997aab2..b9c0c4171e65 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowEnumerable.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowEnumerable.java @@ -21,11 +21,10 @@ import org.apache.calcite.util.ImmutableIntList; import org.apache.calcite.util.Util; -import org.apache.arrow.gandiva.evaluator.Filter; -import org.apache.arrow.gandiva.evaluator.Projector; import org.apache.arrow.vector.ipc.ArrowFileReader; +import org.apache.arrow.vector.types.pojo.Schema; -import org.checkerframework.checker.nullness.qual.Nullable; +import java.util.List; /** * Enumerable that reads from Arrow value-vectors. @@ -33,28 +32,24 @@ class ArrowEnumerable extends AbstractEnumerable { private final ArrowFileReader arrowFileReader; private final ImmutableIntList fields; - private final @Nullable Projector projector; - private final @Nullable Filter filter; + private final List>> conditions; + private final Schema schema; private final Runnable onClose; ArrowEnumerable(ArrowFileReader arrowFileReader, ImmutableIntList fields, - @Nullable Projector projector, @Nullable Filter filter, - Runnable onClose) { + List>> conditions, Schema schema, Runnable onClose) { this.arrowFileReader = arrowFileReader; - this.projector = projector; - this.filter = filter; + this.conditions = conditions; + this.schema = schema; this.fields = fields; this.onClose = onClose; } @Override public Enumerator enumerator() { try { - if (projector != null) { - return new ArrowProjectEnumerator(arrowFileReader, fields, projector, - onClose); - } else if (filter != null) { - return new ArrowFilterEnumerator(arrowFileReader, fields, filter, - onClose); + if (!conditions.isEmpty()) { + return new ArrowFilterEnumerator(arrowFileReader, fields, + conditions, schema, onClose); } // No projector and no filter means the query is an identity projection // that should read selected value-vectors directly. diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowFilterEnumerator.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowFilterEnumerator.java index 5eddec224909..2f154cc6ef7e 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowFilterEnumerator.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowFilterEnumerator.java @@ -19,91 +19,215 @@ import org.apache.calcite.util.ImmutableIntList; import org.apache.calcite.util.Util; -import org.apache.arrow.gandiva.evaluator.Filter; -import org.apache.arrow.gandiva.evaluator.SelectionVector; -import org.apache.arrow.gandiva.evaluator.SelectionVectorInt16; -import org.apache.arrow.gandiva.exceptions.GandivaException; -import org.apache.arrow.memory.ArrowBuf; -import org.apache.arrow.memory.BufferAllocator; -import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.ValueVector; +import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.arrow.vector.ipc.ArrowFileReader; -import org.apache.arrow.vector.ipc.message.ArrowRecordBatch; - -import org.checkerframework.checker.nullness.qual.Nullable; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.Schema; import java.io.IOException; +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; import static java.util.Objects.requireNonNull; /** - * Enumerator that reads from a filtered collection of Arrow value-vectors. + * Enumerator that evaluates Arrow filter tokens in Java. */ class ArrowFilterEnumerator extends AbstractArrowEnumerator { - private final BufferAllocator allocator; - private final Filter filter; - private @Nullable ArrowBuf buf; - private @Nullable SelectionVector selectionVector; - private int selectionVectorIndex; - + private final List> conditions; + private final Schema schema; private final Runnable onClose; + private final List filterVectors; + private final Map likePatterns; - ArrowFilterEnumerator(ArrowFileReader arrowFileReader, ImmutableIntList fields, - Filter filter, Runnable onClose) { + ArrowFilterEnumerator(ArrowFileReader arrowFileReader, + ImmutableIntList fields, List>> conditions, + Schema schema, Runnable onClose) { super(arrowFileReader, fields); - this.allocator = new RootAllocator(Long.MAX_VALUE); - this.filter = filter; + this.conditions = toConditionTokens(conditions); + this.schema = schema; this.onClose = onClose; + this.filterVectors = new ArrayList<>(schema.getFields().size()); + this.likePatterns = new HashMap<>(); } - @Override void evaluateOperator(ArrowRecordBatch arrowRecordBatch) { + @Override protected void loadNextArrowBatch() { + super.loadNextArrowBatch(); + final VectorSchemaRoot root; try { - this.buf = this.allocator.buffer((long) rowCount * 2); - this.selectionVector = new SelectionVectorInt16(buf); - filter.evaluate(arrowRecordBatch, selectionVector); - } catch (GandivaException e) { + root = arrowFileReader.getVectorSchemaRoot(); + } catch (IOException e) { throw Util.toUnchecked(e); } + filterVectors.clear(); + for (int i = 0; i < schema.getFields().size(); i++) { + filterVectors.add(root.getVector(i)); + } } @Override public boolean moveNext() { - if (selectionVector == null - || selectionVectorIndex >= selectionVector.getRecordCount()) { - boolean hasNextBatch; - while (true) { - try { - hasNextBatch = arrowFileReader.loadNextBatch(); - } catch (IOException e) { - throw Util.toUnchecked(e); + while (true) { + if (currRowIndex >= rowCount - 1) { + if (!loadNextNonEmptyArrowBatch()) { + return false; } - if (hasNextBatch) { - selectionVectorIndex = 0; - this.valueVectors.clear(); - loadNextArrowBatch(); - requireNonNull(selectionVector, "selectionVector"); - if (selectionVectorIndex >= selectionVector.getRecordCount()) { - // the "filtered" batch is empty, but there may be more batches to fetch - continue; - } - currRowIndex = selectionVector.getIndex(selectionVectorIndex++); + } + currRowIndex++; + if (matches(currRowIndex)) { + return true; + } + } + } + + private boolean matches(int rowIndex) { + for (List orGroup : conditions) { + boolean any = false; + for (ConditionToken token : orGroup) { + if (matches(token, rowIndex)) { + any = true; + break; } - return hasNextBatch; } - } else { - currRowIndex = selectionVector.getIndex(selectionVectorIndex++); - return true; + if (!any) { + return false; + } } + return true; } - @Override public void close() { - try { - if (buf != null) { - buf.close(); + private boolean matches(ConditionToken token, int rowIndex) { + final Object value = getValue(fieldVector(token.fieldName), rowIndex); + switch (token.operator) { + case IS_NULL: + return value == null; + case IS_NOT_NULL: + return value != null; + case IS_TRUE: + return Boolean.TRUE.equals(value); + case IS_FALSE: + return Boolean.FALSE.equals(value); + case IS_NOT_TRUE: + return !Boolean.TRUE.equals(value); + case IS_NOT_FALSE: + return !Boolean.FALSE.equals(value); + case EQUAL: + return value != null && compare(value, literal(token)) == 0; + case NOT_EQUAL: + return value != null && compare(value, literal(token)) != 0; + case LESS_THAN: + return value != null && compare(value, literal(token)) < 0; + case LESS_THAN_OR_EQUAL: + return value != null && compare(value, literal(token)) <= 0; + case GREATER_THAN: + return value != null && compare(value, literal(token)) > 0; + case GREATER_THAN_OR_EQUAL: + return value != null && compare(value, literal(token)) >= 0; + case LIKE: + return value != null + && like(value.toString(), requireNonNull(token.value, "value")); + default: + throw new AssertionError("Unhandled Arrow filter operator: " + token.operator); + } + } + + private ValueVector fieldVector(String fieldName) { + final Field field = schema.findField(fieldName); + final int index = schema.getFields().indexOf(field); + if (index < 0) { + throw new IllegalArgumentException("Unknown Arrow field: " + fieldName); + } + return filterVectors.get(index); + } + + private static Object literal(ConditionToken token) { + final String type = requireNonNull(token.valueType, "valueType"); + final String value = requireNonNull(token.value, "value"); + if (type.startsWith("decimal")) { + return new BigDecimal(value); + } else if (type.equals("integer")) { + return Integer.valueOf(value); + } else if (type.equals("long")) { + return Long.valueOf(value); + } else if (type.equals("float")) { + return Float.valueOf(value); + } else if (type.equals("double")) { + return Double.valueOf(value); + } else if (type.equals("string")) { + return unquote(value); + } + throw new UnsupportedOperationException("Unsupported literal type: " + type); + } + + private static int compare(Object left, Object right) { + if (left instanceof BigDecimal || right instanceof BigDecimal) { + return toBigDecimal(left).compareTo(toBigDecimal(right)); + } + if (left instanceof Number && right instanceof Number) { + return Double.compare(((Number) left).doubleValue(), + ((Number) right).doubleValue()); + } + return left.toString().compareTo(right.toString()); + } + + private static BigDecimal toBigDecimal(Object value) { + if (value instanceof BigDecimal) { + return (BigDecimal) value; + } + return new BigDecimal(value.toString()); + } + + private boolean like(String value, String pattern) { + final String unquotedPattern = unquote(pattern); + final Pattern compiledPattern = + likePatterns.computeIfAbsent(unquotedPattern, p -> { + return Pattern.compile(toRegex(p), Pattern.DOTALL); + }); + return compiledPattern.matcher(value).matches(); + } + + private static String toRegex(String pattern) { + final StringBuilder builder = new StringBuilder(); + for (int i = 0; i < pattern.length(); i++) { + final char c = pattern.charAt(i); + if (c == '%') { + builder.append(".*"); + } else if (c == '_') { + builder.append('.'); + } else { + builder.append(Pattern.quote(String.valueOf(c))); } - filter.close(); - } catch (GandivaException e) { - throw Util.toUnchecked(e); - } finally { - onClose.run(); } + return builder.toString(); + } + + private static String unquote(String value) { + if (value.length() >= 2 && value.charAt(0) == '\'' + && value.charAt(value.length() - 1) == '\'') { + return value.substring(1, value.length() - 1).replace("''", "'"); + } + return value; + } + + private static List> toConditionTokens( + List>> conditions) { + final List> result = + new ArrayList<>(conditions.size()); + for (List> orGroup : conditions) { + final List tokens = new ArrayList<>(orGroup.size()); + for (List token : orGroup) { + tokens.add(ConditionToken.fromTokenList(token)); + } + result.add(tokens); + } + return result; + } + + @Override public void close() { + onClose.run(); } } diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowProjectEnumerator.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowProjectEnumerator.java deleted file mode 100644 index 0895f36cf15f..000000000000 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowProjectEnumerator.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.calcite.adapter.arrow; - -import org.apache.calcite.util.ImmutableIntList; -import org.apache.calcite.util.Util; - -import org.apache.arrow.gandiva.evaluator.Projector; -import org.apache.arrow.gandiva.exceptions.GandivaException; -import org.apache.arrow.vector.ipc.ArrowFileReader; -import org.apache.arrow.vector.ipc.message.ArrowRecordBatch; - -import java.io.IOException; - -/** - * Enumerator that reads from a projected collection of Arrow value-vectors. - */ -class ArrowProjectEnumerator extends AbstractArrowEnumerator { - private final Projector projector; - private final Runnable onClose; - - ArrowProjectEnumerator(ArrowFileReader arrowFileReader, ImmutableIntList fields, - Projector projector, Runnable onClose) { - super(arrowFileReader, fields); - this.projector = projector; - this.onClose = onClose; - } - - @Override protected void evaluateOperator(ArrowRecordBatch arrowRecordBatch) { - try { - projector.evaluate(arrowRecordBatch, valueVectors); - } catch (GandivaException e) { - throw Util.toUnchecked(e); - } - } - - @Override public boolean moveNext() { - if (currRowIndex >= rowCount - 1) { - final boolean hasNextBatch; - try { - hasNextBatch = arrowFileReader.loadNextBatch(); - } catch (IOException e) { - throw Util.toUnchecked(e); - } - if (hasNextBatch) { - currRowIndex = 0; - this.valueVectors.clear(); - loadNextArrowBatch(); - } - return hasNextBatch; - } else { - currRowIndex++; - return true; - } - } - - @Override public void close() { - try { - projector.close(); - } catch (GandivaException e) { - throw Util.toUnchecked(e); - } finally { - onClose.run(); - } - } -} diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowRules.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowRules.java index 6e268d646928..3da1527f1014 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowRules.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowRules.java @@ -100,7 +100,7 @@ RelNode convert(Filter filter) { final RelTraitSet traitSet = filter.getTraitSet().replace(ArrowRel.CONVENTION); // Expand SEARCH (e.g. IN, BETWEEN) before pushing to Arrow, - // since Gandiva does not support SEARCH natively. + // since the Arrow adapter does not support SEARCH natively. final RexNode condition = RexUtil.expandSearch(filter.getCluster().getRexBuilder(), null, filter.getCondition()); return new ArrowFilter(filter.getCluster(), traitSet, diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTable.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTable.java index 74438efe2a33..2585f5d156ea 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTable.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTable.java @@ -38,17 +38,9 @@ import org.apache.calcite.util.ImmutableIntList; import org.apache.calcite.util.Util; -import org.apache.arrow.gandiva.evaluator.Filter; -import org.apache.arrow.gandiva.evaluator.Projector; -import org.apache.arrow.gandiva.exceptions.GandivaException; -import org.apache.arrow.gandiva.expression.Condition; -import org.apache.arrow.gandiva.expression.ExpressionTree; -import org.apache.arrow.gandiva.expression.TreeBuilder; -import org.apache.arrow.gandiva.expression.TreeNode; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.ipc.ArrowFileReader; import org.apache.arrow.vector.ipc.SeekableReadChannel; -import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.Schema; @@ -58,20 +50,15 @@ import java.io.FileInputStream; import java.io.IOException; import java.lang.reflect.Type; -import java.util.ArrayList; import java.util.List; -import static java.lang.Double.parseDouble; -import static java.lang.Float.parseFloat; -import static java.lang.Integer.parseInt; -import static java.lang.Long.parseLong; import static java.util.Objects.requireNonNull; /** * Table backed by an Apache Arrow file. * - *

      Reads data from an Arrow IPC file on disk and supports projection - * and filter push-down via the Gandiva expression compiler. + *

      Reads data from an Arrow IPC file on disk. Projections and filters read + * directly from Arrow value-vectors. * *

      Implements {@link TranslatableTable} so that it can be converted into * an {@link ArrowTableScan} for query planning, and {@link QueryableTable} @@ -116,43 +103,6 @@ public class ArrowTable extends AbstractTable public Enumerable query(DataContext root, ImmutableIntList fields, List>> conditions) { requireNonNull(fields, "fields"); - final Projector projector; - final Filter filter; - - if (conditions.isEmpty()) { - filter = null; - projector = makeProjector(fields); - } else { - projector = null; - - final List conjuncts = new ArrayList<>(conditions.size()); - for (List> orGroup : conditions) { - final List disjuncts = new ArrayList<>(orGroup.size()); - for (List conditionParts : orGroup) { - disjuncts.add( - convertConditionToGandiva( - ConditionToken.fromTokenList(conditionParts))); - } - if (disjuncts.size() == 1) { - conjuncts.add(disjuncts.get(0)); - } else { - conjuncts.add(TreeBuilder.makeOr(disjuncts)); - } - } - final Condition filterCondition; - if (conjuncts.size() == 1) { - filterCondition = TreeBuilder.makeCondition(conjuncts.get(0)); - } else { - filterCondition = - TreeBuilder.makeCondition(TreeBuilder.makeAnd(conjuncts)); - } - - try { - filter = Filter.make(schema, filterCondition); - } catch (GandivaException e) { - throw Util.toUnchecked(e); - } - } FileInputStream fis = null; try { @@ -163,7 +113,7 @@ public Enumerable query(DataContext root, ImmutableIntList fields, final FileInputStream fisRef = fis; final Runnable onClose = () -> closeSilently(fisRef); fis = null; // ownership transferred to onClose - return new ArrowEnumerable(reader, fields, projector, filter, onClose); + return new ArrowEnumerable(reader, fields, conditions, schema, onClose); } catch (IOException e) { throw Util.toUnchecked(e); } finally { @@ -202,70 +152,6 @@ private static RelDataType deduceRowType(Schema schema, return builder.build(); } - private @Nullable Projector makeProjector(ImmutableIntList fields) { - if (requiresDirectVectorProjection(fields)) { - // Returning null selects ArrowEnumerable's direct vector-read path. - // Use that path because Gandiva does not support identity projection - // expressions over Arrow List and binary vectors. - return null; - } - - final List expressionTrees = new ArrayList<>(); - for (int fieldOrdinal : fields) { - Field field = schema.getFields().get(fieldOrdinal); - TreeNode node = TreeBuilder.makeField(field); - expressionTrees.add(TreeBuilder.makeExpression(node, field)); - } - try { - return Projector.make(schema, expressionTrees); - } catch (GandivaException e) { - throw Util.toUnchecked(e); - } - } - - /** Returns whether selected fields should be projected by reading Arrow - * value-vectors directly rather than by creating a Gandiva projector. - * - *

      CALCITE-7541 extends this direct projection path for Arrow binary vector - * families because Gandiva cannot project them through the existing identity - * projection path. Queries with filters still use Gandiva filters; this direct - * path only applies to no-filter projections. - */ - private boolean requiresDirectVectorProjection(ImmutableIntList fields) { - for (int fieldOrdinal : fields) { - switch (schema.getFields().get(fieldOrdinal).getType().getTypeID()) { - case List: - case Binary: - case LargeBinary: - case FixedSizeBinary: - return true; - default: - break; - } - } - return false; - } - - /** Converts a single {@link ConditionToken} into a Gandiva {@link TreeNode}. */ - private TreeNode convertConditionToGandiva(ConditionToken token) { - final List treeNodes = new ArrayList<>(2); - treeNodes.add( - TreeBuilder.makeField(schema.getFields() - .get( - schema.getFields().indexOf( - schema.findField(token.fieldName))))); - - if (token.isBinary()) { - treeNodes.add( - makeLiteralNode( - requireNonNull(token.value, "value"), - requireNonNull(token.valueType, "valueType"))); - } - - return TreeBuilder.makeFunction( - token.operator, treeNodes, new ArrowType.Bool()); - } - /** Closes an {@link AutoCloseable} without throwing. */ private static void closeSilently(AutoCloseable closeable) { try { @@ -275,29 +161,6 @@ private static void closeSilently(AutoCloseable closeable) { } } - private static TreeNode makeLiteralNode(String literal, String type) { - if (type.startsWith("decimal")) { - String[] typeParts = - type.substring(type.indexOf('(') + 1, type.indexOf(')')).split(","); - int precision = parseInt(typeParts[0]); - int scale = parseInt(typeParts[1]); - return TreeBuilder.makeDecimalLiteral(literal, precision, scale); - } else if (type.equals("integer")) { - return TreeBuilder.makeLiteral(parseInt(literal)); - } else if (type.equals("long")) { - return TreeBuilder.makeLiteral(parseLong(literal)); - } else if (type.equals("float")) { - return TreeBuilder.makeLiteral(parseFloat(literal)); - } else if (type.equals("double")) { - return TreeBuilder.makeLiteral(parseDouble(literal)); - } else if (type.equals("string")) { - return TreeBuilder.makeStringLiteral(literal.substring(1, literal.length() - 1)); - } else { - throw new IllegalArgumentException("Invalid literal " + literal - + ", type " + type); - } - } - /** * Implementation of {@link Queryable} based on a {@link ArrowTable}. * diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTranslator.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTranslator.java index 0ec680405270..2b4229598153 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTranslator.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTranslator.java @@ -35,13 +35,26 @@ import java.util.ArrayList; import java.util.List; +import static org.apache.calcite.adapter.arrow.ConditionToken.Operator.EQUAL; +import static org.apache.calcite.adapter.arrow.ConditionToken.Operator.GREATER_THAN; +import static org.apache.calcite.adapter.arrow.ConditionToken.Operator.GREATER_THAN_OR_EQUAL; +import static org.apache.calcite.adapter.arrow.ConditionToken.Operator.IS_FALSE; +import static org.apache.calcite.adapter.arrow.ConditionToken.Operator.IS_NOT_FALSE; +import static org.apache.calcite.adapter.arrow.ConditionToken.Operator.IS_NOT_NULL; +import static org.apache.calcite.adapter.arrow.ConditionToken.Operator.IS_NOT_TRUE; +import static org.apache.calcite.adapter.arrow.ConditionToken.Operator.IS_NULL; +import static org.apache.calcite.adapter.arrow.ConditionToken.Operator.IS_TRUE; +import static org.apache.calcite.adapter.arrow.ConditionToken.Operator.LESS_THAN; +import static org.apache.calcite.adapter.arrow.ConditionToken.Operator.LESS_THAN_OR_EQUAL; +import static org.apache.calcite.adapter.arrow.ConditionToken.Operator.LIKE; +import static org.apache.calcite.adapter.arrow.ConditionToken.Operator.NOT_EQUAL; import static org.apache.calcite.util.DateTimeStringUtils.ISO_DATETIME_FRACTIONAL_SECOND_FORMAT; import static org.apache.calcite.util.DateTimeStringUtils.getDateFormatter; import static java.util.Objects.requireNonNull; /** - * Translates a {@link RexNode} expression to Gandiva predicate tokens. + * Translates a {@link RexNode} expression to Arrow predicate tokens. */ class ArrowTranslator { final RexBuilder rexBuilder; @@ -65,7 +78,7 @@ public static ArrowTranslator create(RexBuilder rexBuilder, * *

      If exceeded, {@link RexUtil#toCnf(RexBuilder, int, RexNode)} returns * the original expression unchanged, which may cause the subsequent - * translation to Gandiva predicates to fail with an + * translation to Arrow predicates to fail with an * {@link UnsupportedOperationException}. When invoked by the Arrow adapter * module, the exception is caught and the plan falls back to * an Enumerable convention. */ @@ -120,32 +133,32 @@ private static Object literalValue(RexLiteral literal) { private ConditionToken translateMatch2(RexNode node) { switch (node.getKind()) { case EQUALS: - return translateBinary("equal", "=", (RexCall) node); + return translateBinary(EQUAL, EQUAL, (RexCall) node); case NOT_EQUALS: - return translateBinary("not_equal", "<>", (RexCall) node); + return translateBinary(NOT_EQUAL, NOT_EQUAL, (RexCall) node); case LESS_THAN: - return translateBinary("less_than", ">", (RexCall) node); + return translateBinary(LESS_THAN, GREATER_THAN, (RexCall) node); case LESS_THAN_OR_EQUAL: - return translateBinary("less_than_or_equal_to", ">=", (RexCall) node); + return translateBinary(LESS_THAN_OR_EQUAL, GREATER_THAN_OR_EQUAL, (RexCall) node); case GREATER_THAN: - return translateBinary("greater_than", "<", (RexCall) node); + return translateBinary(GREATER_THAN, LESS_THAN, (RexCall) node); case GREATER_THAN_OR_EQUAL: - return translateBinary("greater_than_or_equal_to", "<=", (RexCall) node); + return translateBinary(GREATER_THAN_OR_EQUAL, LESS_THAN_OR_EQUAL, (RexCall) node); case IS_NULL: - return translateUnary("isnull", (RexCall) node); + return translateUnary(IS_NULL, (RexCall) node); case IS_NOT_NULL: - return translateUnary("isnotnull", (RexCall) node); + return translateUnary(IS_NOT_NULL, (RexCall) node); case IS_NOT_TRUE: - return translateUnary("isnottrue", (RexCall) node); + return translateUnary(IS_NOT_TRUE, (RexCall) node); case IS_NOT_FALSE: - return translateUnary("isnotfalse", (RexCall) node); + return translateUnary(IS_NOT_FALSE, (RexCall) node); case INPUT_REF: final RexInputRef inputRef = (RexInputRef) node; - return ConditionToken.unary(fieldNames.get(inputRef.getIndex()), "istrue"); + return ConditionToken.unary(fieldNames.get(inputRef.getIndex()), IS_TRUE); case NOT: - return translateUnary("isfalse", (RexCall) node); + return translateUnary(IS_FALSE, (RexCall) node); case LIKE: - return translateBinary("like", null, (RexCall) node); + return translateBinaryNoReverse(LIKE, (RexCall) node); default: throw new UnsupportedOperationException("Unsupported operator " + node); } @@ -155,7 +168,8 @@ private ConditionToken translateMatch2(RexNode node) { * Translates a call to a binary operator, reversing arguments if * necessary. */ - private ConditionToken translateBinary(String op, String rop, RexCall call) { + private ConditionToken translateBinary(ConditionToken.Operator op, + ConditionToken.Operator rop, RexCall call) { final RexNode left = call.operands.get(0); final RexNode right = call.operands.get(1); @Nullable ConditionToken expression = translateBinary2(op, left, right); @@ -169,9 +183,21 @@ private ConditionToken translateBinary(String op, String rop, RexCall call) { throw new UnsupportedOperationException("Unsupported binary operator " + call); } + /** Translates a call to a binary operator without reversing arguments. */ + private ConditionToken translateBinaryNoReverse(ConditionToken.Operator op, + RexCall call) { + final RexNode left = call.operands.get(0); + final RexNode right = call.operands.get(1); + @Nullable ConditionToken expression = translateBinary2(op, left, right); + if (expression != null) { + return expression; + } + throw new UnsupportedOperationException("Unsupported binary operator " + call); + } + /** Translates a call to a binary operator. Returns null on failure. */ - private @Nullable ConditionToken translateBinary2(String op, RexNode left, - RexNode right) { + private @Nullable ConditionToken translateBinary2( + ConditionToken.Operator op, RexNode left, RexNode right) { if (right.getKind() != SqlKind.LITERAL) { return null; } @@ -191,7 +217,7 @@ private ConditionToken translateBinary(String op, String rop, RexCall call) { /** Combines a field name, operator, and literal to produce a binary * condition token. */ - private ConditionToken translateOp2(String op, String name, + private ConditionToken translateOp2(ConditionToken.Operator op, String name, RexLiteral right) { Object value = literalValue(right); String valueString = value.toString(); @@ -209,7 +235,7 @@ private ConditionToken translateOp2(String op, String name, } /** Translates a call to a unary operator. */ - private ConditionToken translateUnary(String op, RexCall call) { + private ConditionToken translateUnary(ConditionToken.Operator op, RexCall call) { final RexNode opNode = call.operands.get(0); @Nullable ConditionToken expression = translateUnary2(op, opNode); @@ -221,7 +247,8 @@ private ConditionToken translateUnary(String op, RexCall call) { } /** Translates a call to a unary operator. Returns null on failure. */ - private @Nullable ConditionToken translateUnary2(String op, RexNode opNode) { + private @Nullable ConditionToken translateUnary2(ConditionToken.Operator op, + RexNode opNode) { if (opNode.getKind() == SqlKind.INPUT_REF) { final RexInputRef inputRef = (RexInputRef) opNode; final String name = fieldNames.get(inputRef.getIndex()); diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ConditionToken.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ConditionToken.java index 44d3facea77f..c5b690840add 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ConditionToken.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ConditionToken.java @@ -25,7 +25,7 @@ import static java.util.Objects.requireNonNull; /** - * A structured representation of a single Gandiva predicate condition. + * A structured representation of a single Arrow predicate condition. * *

      A condition is either unary (e.g. {@code IS NULL}) or binary * (e.g. {@code =}, {@code <}). Unary conditions have a field name @@ -36,11 +36,11 @@ */ class ConditionToken { final String fieldName; - final String operator; + final Operator operator; final @Nullable String value; final @Nullable String valueType; - private ConditionToken(String fieldName, String operator, + private ConditionToken(String fieldName, Operator operator, @Nullable String value, @Nullable String valueType) { this.fieldName = requireNonNull(fieldName, "fieldName"); this.operator = requireNonNull(operator, "operator"); @@ -50,7 +50,7 @@ private ConditionToken(String fieldName, String operator, /** Creates a binary condition token * (e.g. {@code intField equal 12 integer}). */ - static ConditionToken binary(String fieldName, String operator, + static ConditionToken binary(String fieldName, Operator operator, String value, String valueType) { return new ConditionToken(fieldName, operator, requireNonNull(value, "value"), @@ -59,7 +59,7 @@ static ConditionToken binary(String fieldName, String operator, /** Creates a unary condition token * (e.g. {@code intField isnull}). */ - static ConditionToken unary(String fieldName, String operator) { + static ConditionToken unary(String fieldName, Operator operator) { return new ConditionToken(fieldName, operator, null, null); } @@ -76,22 +76,55 @@ boolean isBinary() { * binary conditions. */ List toTokenList() { if (isBinary()) { - return ImmutableList.of(fieldName, operator, + return ImmutableList.of(fieldName, operator.token, requireNonNull(value, "value"), requireNonNull(valueType, "valueType")); } - return ImmutableList.of(fieldName, operator); + return ImmutableList.of(fieldName, operator.token); } /** Creates a {@code ConditionToken} from a serialized string list. */ static ConditionToken fromTokenList(List tokens) { final int size = tokens.size(); if (size == 4) { - return binary(tokens.get(0), tokens.get(1), + return binary(tokens.get(0), Operator.of(tokens.get(1)), tokens.get(2), tokens.get(3)); } else if (size == 2) { - return unary(tokens.get(0), tokens.get(1)); + return unary(tokens.get(0), Operator.of(tokens.get(1))); } throw new IllegalArgumentException("Invalid condition tokens: " + tokens); } + + /** Operators supported by the Arrow adapter filter representation. */ + enum Operator { + IS_NULL("isnull"), + IS_NOT_NULL("isnotnull"), + IS_TRUE("istrue"), + IS_FALSE("isfalse"), + IS_NOT_TRUE("isnottrue"), + IS_NOT_FALSE("isnotfalse"), + EQUAL("equal"), + NOT_EQUAL("not_equal"), + LESS_THAN("less_than"), + LESS_THAN_OR_EQUAL("less_than_or_equal_to"), + GREATER_THAN("greater_than"), + GREATER_THAN_OR_EQUAL("greater_than_or_equal_to"), + LIKE("like"); + + final String token; + + Operator(String token) { + this.token = token; + } + + static Operator of(String token) { + for (Operator operator : values()) { + if (operator.token.equals(token)) { + return operator; + } + } + throw new UnsupportedOperationException( + "Unsupported Arrow filter operator: " + token); + } + } } diff --git a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java index 275bdd0be76c..8c9fda7f081f 100644 --- a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java +++ b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java @@ -82,6 +82,11 @@ static void initializeArrowState(@TempDir Path sharedTempDir) arrowDataGenerator.writeArrowData(dataLocationFile); arrowDataGenerator.writeScottEmpData(arrowFilesDirectory); + File emptyBatchDataLocationFile = + arrowFilesDirectory.resolve("arrowemptybatch.arrow").toFile(); + ArrowDataTest emptyBatchDataGenerator = new ArrowDataTest(); + emptyBatchDataGenerator.writeArrowDataWithEmptyBatch(emptyBatchDataLocationFile); + File datatypeLocationFile = arrowFilesDirectory.resolve("arrowdatatype.arrow").toFile(); ArrowDataTest arrowtypeDataGenerator = new ArrowDataTest(); arrowtypeDataGenerator.writeArrowDataType(datatypeLocationFile); @@ -260,6 +265,51 @@ static void initializeArrowState(@TempDir Path sharedTempDir) .explainContains(plan); } + /** Test case for + * [CALCITE-7580] + * Remove Gandiva dependency from Arrow adapter. */ + @Test void testArrowProjectSkipsEmptyBatch() { + String sql = "select \"intField\", \"stringField\" from arrowemptybatch\n"; + String result = "intField=0; stringField=0\n" + + "intField=1; stringField=1\n" + + "intField=2; stringField=2\n"; + + CalciteAssert.that() + .with(arrow) + .query(sql) + .returns(result); + } + + /** Test case for + * [CALCITE-7580] + * Remove Gandiva dependency from Arrow adapter. */ + @Test void testArrowFilterSkipsEmptyBatch() { + String sql = "select \"intField\", \"stringField\"\n" + + "from arrowemptybatch\n" + + "where \"intField\" > 0"; + String result = "intField=1; stringField=1\n" + + "intField=2; stringField=2\n"; + + CalciteAssert.that() + .with(arrow) + .query(sql) + .returns(result); + } + + /** Test case for + * [CALCITE-7580] + * Remove Gandiva dependency from Arrow adapter. */ + @Test void testArrowFilterSkipsEmptyBatchWithNoMatches() { + String sql = "select \"intField\", \"stringField\"\n" + + "from arrowemptybatch\n" + + "where \"intField\" < 0"; + + CalciteAssert.that() + .with(arrow) + .query(sql) + .returns(""); + } + @Test void testArrowProjectFieldsWithIntegerFilter() { String sql = "select \"intField\", \"stringField\"\n" + "from arrowdata\n" diff --git a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowDataTest.java b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowDataTest.java index a53cc1231ad3..4b9af441aabf 100644 --- a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowDataTest.java +++ b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowDataTest.java @@ -301,6 +301,35 @@ public void writeArrowBinaryData(File file) throws IOException { fileOutputStream.close(); } + public void writeArrowDataWithEmptyBatch(File file) throws IOException { + Schema arrowSchema = makeArrowSchema(); + try (RootAllocator allocator = new RootAllocator(Integer.MAX_VALUE); + VectorSchemaRoot vectorSchemaRoot = + VectorSchemaRoot.create(arrowSchema, allocator); + FileOutputStream fileOutputStream = new FileOutputStream(file); + ArrowFileWriter arrowFileWriter = + new ArrowFileWriter(vectorSchemaRoot, null, + fileOutputStream.getChannel())) { + arrowFileWriter.start(); + + vectorSchemaRoot.setRowCount(0); + for (Field field : vectorSchemaRoot.getSchema().getFields()) { + vectorSchemaRoot.getVector(field.getName()).setValueCount(0); + } + arrowFileWriter.writeBatch(); + + int rowCount = 3; + vectorSchemaRoot.setRowCount(rowCount); + intField(vectorSchemaRoot.getVector("intField"), rowCount); + varCharField(vectorSchemaRoot.getVector("stringField"), rowCount); + floatField(vectorSchemaRoot.getVector("floatField"), rowCount); + longField(vectorSchemaRoot.getVector("longField"), rowCount); + arrowFileWriter.writeBatch(); + + arrowFileWriter.end(); + } + } + public void writeArrowDataType(File file) throws IOException { FileOutputStream fileOutputStream = new FileOutputStream(file); Schema arrowSchema = makeArrowDateTypeSchema(); diff --git a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowExtension.java b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowExtension.java index ab1f5c2a88c1..4600dab1001c 100644 --- a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowExtension.java +++ b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowExtension.java @@ -18,22 +18,12 @@ import org.apache.calcite.config.CalciteSystemProperty; -import org.apache.arrow.gandiva.evaluator.Projector; -import org.apache.arrow.gandiva.exceptions.GandivaException; -import org.apache.arrow.gandiva.expression.ExpressionTree; -import org.apache.arrow.vector.types.pojo.Schema; - import org.junit.jupiter.api.extension.ConditionEvaluationResult; import org.junit.jupiter.api.extension.ExecutionCondition; import org.junit.jupiter.api.extension.ExtensionContext; -import java.util.ArrayList; -import java.util.List; - /** * JUnit5 extension to handle Arrow tests. - * - *

      Tests will be skipped if the Gandiva library cannot be loaded on the given platform. */ class ArrowExtension implements ExecutionCondition { @@ -41,8 +31,7 @@ class ArrowExtension implements ExecutionCondition { * Whether to run this test. * *

      Enabled by default, unless explicitly disabled from command line - * ({@code -Dcalcite.test.arrow=false}) or if Gandiva library, used to implement arrow - * filtering/projection, cannot be loaded. + * ({@code -Dcalcite.test.arrow=false}). * * @return {@code true} if the test is enabled and can run in the current environment, * {@code false} otherwise @@ -51,16 +40,6 @@ class ArrowExtension implements ExecutionCondition { final ExtensionContext context) { boolean enabled = CalciteSystemProperty.TEST_ARROW.value(); - try { - Schema emptySchema = new Schema(new ArrayList<>(), null); - List expressions = new ArrayList<>(); - Projector.make(emptySchema, expressions); - } catch (GandivaException e) { - // this exception comes from using an empty expression, - // but the JNI library was loaded properly - } catch (UnsatisfiedLinkError e) { - enabled = false; - } if (enabled) { return ConditionEvaluationResult.enabled("Arrow tests enabled"); diff --git a/bom/build.gradle.kts b/bom/build.gradle.kts index f00ed7d8e556..64cb532afdc3 100644 --- a/bom/build.gradle.kts +++ b/bom/build.gradle.kts @@ -101,7 +101,6 @@ dependencies { apiv("org.apache.arrow:arrow-memory-netty", "arrow") apiv("org.apache.arrow:arrow-vector", "arrow") apiv("org.apache.arrow:arrow-jdbc", "arrow") - apiv("org.apache.arrow.gandiva:arrow-gandiva", "arrow-gandiva") apiv("org.apache.calcite.avatica:avatica-core", "calcite.avatica") apiv("org.apache.calcite.avatica:avatica-server", "calcite.avatica") apiv("org.apache.cassandra:cassandra-all") diff --git a/gradle.properties b/gradle.properties index eb0bd778ae14..fd823182640c 100644 --- a/gradle.properties +++ b/gradle.properties @@ -81,7 +81,6 @@ jandex.version=3.5.3 # elasticsearch does not like asm:6.2.1+ aggdesigner-algorithm.version=6.1 apiguardian-api.version=1.1.2 -arrow-gandiva.version=15.0.0 arrow.version=15.0.0 asm.version=9.9.1 byte-buddy.version=1.18.8 diff --git a/site/_docs/history.md b/site/_docs/history.md index 15e2561d398a..c72d6f78d938 100644 --- a/site/_docs/history.md +++ b/site/_docs/history.md @@ -49,6 +49,11 @@ other software versions as specified in gradle.properties. #### Breaking Changes {: #breaking-1-43-0} +* [CALCITE-7580] + Remove Gandiva dependency from Arrow adapter. Arrow adapter projection and + filter evaluation now run in Java, and the `arrow-gandiva` dependency is no + longer included in the Arrow module or BOM. + #### New features {: #new-features-1-43-0} From c59ba228f965b0b5bfd0cb4b8036db346fb7d2ae Mon Sep 17 00:00:00 2001 From: Sergey Nuyanzin Date: Fri, 5 Jun 2026 14:51:13 +0200 Subject: [PATCH 337/562] [CALCITE-7589] `JOIN ... USING` might fail with disabled type coercion --- .../apache/calcite/sql2rel/SqlToRelConverter.java | 7 ++++--- .../apache/calcite/test/SqlToRelConverterTest.java | 7 +++++++ .../apache/calcite/test/SqlToRelConverterTest.xml | 13 +++++++++++++ 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java index 882d42387098..53d54dd6d2a1 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java @@ -3642,9 +3642,10 @@ private RexNode convertUsing(SqlValidatorNamespace leftNamespace, offset += rowType.getFieldList().size(); } - RelDataType resultType = - validator().getTypeCoercion().commonTypeForBinaryComparison( - comparedTypes.get(0), comparedTypes.get(1)); + RelDataType resultType = validator().config().typeCoercionEnabled() + ? validator().getTypeCoercion().commonTypeForBinaryComparison( + comparedTypes.get(0), comparedTypes.get(1)) + : null; if (resultType == null) { // Leave call unchanged (as it happens in TypeCoercionImpl#binaryComparisonCoercion) list.add(rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, operands)); diff --git a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java index 26000620af68..218f018fb683 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java @@ -5397,6 +5397,13 @@ void checkUserDefinedOrderByOver(NullCollation nullCollation) { .ok(); } + /** Test case for [CALCITE-7589] + * JOIN ... USING might fail with disabled type coercion. */ + @Test void testNaturalJoinCastNoCoercion2() { + final String sql = "select * from emp join dept using(deptno)"; + sql(sql).withTypeCoercion(false).ok(); + } + /** Tests LEFT JOIN LATERAL with multiple columns from outer. */ @Test void testLeftJoinLateral4() { final String sql = "select * from (values (4,5)) as t(c,d)\n" diff --git a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml index 71fe294d2f0c..6900b06ff8fb 100644 --- a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml @@ -5693,6 +5693,19 @@ LogicalProject(X=[CAST($0):DECIMAL(2, 1) NOT NULL]) LogicalJoin(condition=[=($0, $1)], joinType=[inner]) LogicalValues(tuples=[[{ 'x' }]]) LogicalValues(tuples=[[{ 0.0 }]]) +]]> + + + + + + + + From 746bd45ec536ed655f2c69efa313da775611e778 Mon Sep 17 00:00:00 2001 From: "cc.cai" <2356672992@qq.com> Date: Sun, 21 Jun 2026 11:56:49 +0800 Subject: [PATCH 338/562] [CALCITE-7539] Upgrade Arrow adapter dependencies to 16.0.0 --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index fd823182640c..14d29dfa356d 100644 --- a/gradle.properties +++ b/gradle.properties @@ -81,7 +81,7 @@ jandex.version=3.5.3 # elasticsearch does not like asm:6.2.1+ aggdesigner-algorithm.version=6.1 apiguardian-api.version=1.1.2 -arrow.version=15.0.0 +arrow.version=16.0.0 asm.version=9.9.1 byte-buddy.version=1.18.8 cassandra-all.version=4.1.6 From 49783ada164a457afbeb1296ab9deb6ed4c9cd4c Mon Sep 17 00:00:00 2001 From: xuzifu666 <1206332514@qq.com> Date: Mon, 11 May 2026 16:54:46 +0800 Subject: [PATCH 339/562] [CALCITE-5406] Support the SELECT DISTINCT ON statement for PostgreSQL dialect --- babel/src/main/codegen/config.fmpp | 1 + .../org/apache/calcite/test/BabelTest.java | 36 +++++ babel/src/test/resources/sql/select.iq | 86 ++++++++++++ core/src/main/codegen/config.fmpp | 5 +- core/src/main/codegen/default_config.fmpp | 2 + core/src/main/codegen/templates/Parser.jj | 31 ++++- .../calcite/runtime/CalciteResource.java | 9 ++ .../org/apache/calcite/sql/SqlSelect.java | 54 ++++++-- .../apache/calcite/sql/SqlSelectOperator.java | 13 +- .../sql/validate/SqlAbstractConformance.java | 4 + .../calcite/sql/validate/SqlConformance.java | 10 ++ .../sql/validate/SqlConformanceEnum.java | 10 ++ .../validate/SqlDelegatingConformance.java | 4 + .../sql/validate/SqlValidatorImpl.java | 53 +++++++- .../calcite/sql2rel/SqlToRelConverter.java | 98 +++++++++++++- .../runtime/CalciteResource.properties | 3 + core/src/test/codegen/config.fmpp | 1 + .../calcite/test/SqlToRelConverterTest.java | 31 +++++ .../apache/calcite/test/SqlValidatorTest.java | 125 +++++++++++++++++- .../calcite/test/SqlToRelConverterTest.xml | 56 ++++++++ server/src/main/codegen/config.fmpp | 1 + site/_docs/reference.md | 21 ++- .../calcite/sql/parser/SqlParserTest.java | 98 ++++++++++++++ 23 files changed, 733 insertions(+), 19 deletions(-) diff --git a/babel/src/main/codegen/config.fmpp b/babel/src/main/codegen/config.fmpp index 30c2ce7d6558..a4e0e9d1ce5f 100644 --- a/babel/src/main/codegen/config.fmpp +++ b/babel/src/main/codegen/config.fmpp @@ -618,6 +618,7 @@ data: { includeParsingStringLiteralAsArrayLiteral: true includeIntervalWithoutQualifier: true includeSelectBy: true + includeDistinctOn: true } } diff --git a/babel/src/test/java/org/apache/calcite/test/BabelTest.java b/babel/src/test/java/org/apache/calcite/test/BabelTest.java index 1deda157fa28..6103d61ed4ce 100644 --- a/babel/src/test/java/org/apache/calcite/test/BabelTest.java +++ b/babel/src/test/java/org/apache/calcite/test/BabelTest.java @@ -594,6 +594,41 @@ private void checkSqlResult(String funLibrary, String query, String result) { .returns(result); } + /** Test case for + * [CALCITE-5406] + * Support the SELECT DISTINCT ON statement. */ + @Test void testDistinctOn() { + final SqlValidatorFixture v = Fixtures.forValidator() + .withParserConfig(c -> c.withParserFactory(SqlBabelParserImpl.FACTORY)) + .withConformance(SqlConformanceEnum.BABEL); + + // Basic DISTINCT ON + v.withSql("select distinct on (deptno) empno, ename from emp order by deptno, empno") + .ok(); + + // DISTINCT ON with multiple columns + v.withSql("select distinct on (deptno, job) empno, ename from emp order by deptno, job, empno") + .ok(); + + // DISTINCT ON with expression + v.withSql("select distinct on (deptno) empno, sal * 12 as annual_sal " + + "from emp order by deptno, sal desc").ok(); + + // DISTINCT ON without ORDER BY should fail + v.withSql("^select distinct on (deptno) empno from emp^") + .fails("SELECT DISTINCT ON requires an ORDER BY clause"); + + // DISTINCT ON with ORDER BY mismatch should fail + v.withSql("select distinct on (deptno) empno from emp order by ^empno^") + .fails("SELECT DISTINCT ON expressions must match ORDER BY expressions"); + + // DISTINCT and DISTINCT ON are mutually exclusive + v.withSql("SELECT DISTINCT ^DISTINCT^ ON (deptno) empno, ename\n" + + "FROM emp\n" + + "ORDER BY deptno, empno") + .fails("(?s)Incorrect syntax near the keyword 'DISTINCT' at line 1, column 17.*"); + } + /** Test case for * [CALCITE-7337] * Add age function (enabled in PostgreSQL library). */ @@ -637,4 +672,5 @@ private void checkSqlResult(String funLibrary, String query, String result) { .query("SELECT AGE(timestamp '2023-12-25') FROM (VALUES (1)) t") .runs(); } + } diff --git a/babel/src/test/resources/sql/select.iq b/babel/src/test/resources/sql/select.iq index f94905c16e9e..60af913f1401 100755 --- a/babel/src/test/resources/sql/select.iq +++ b/babel/src/test/resources/sql/select.iq @@ -332,4 +332,90 @@ from emp e join dept d on e.deptno = d.deptno; SELECT * REPLACE list contains unknown column(s): DEPTNO !error +# [CALCITE-5406] Support the SELECT DISTINCT ON statement for PostgreSQL dialect +# Note: All results in this section have been verified against PostgreSQL and are identical + +# Test basic DISTINCT ON +SELECT DISTINCT ON (deptno) empno, ename, deptno +FROM emp +ORDER BY deptno, empno; ++-------+-------+--------+ +| EMPNO | ENAME | DEPTNO | ++-------+-------+--------+ +| 7782 | CLARK | 10 | +| 7369 | SMITH | 20 | +| 7499 | ALLEN | 30 | ++-------+-------+--------+ +(3 rows) + +!ok + +# Test DISTINCT ON with descending order +SELECT DISTINCT ON (deptno) empno, ename, sal, deptno +FROM emp +ORDER BY deptno, sal DESC, empno; ++-------+-------+---------+--------+ +| EMPNO | ENAME | SAL | DEPTNO | ++-------+-------+---------+--------+ +| 7839 | KING | 5000.00 | 10 | +| 7788 | SCOTT | 3000.00 | 20 | +| 7698 | BLAKE | 2850.00 | 30 | ++-------+-------+---------+--------+ +(3 rows) + +!ok + +# Test DISTINCT ON with multiple columns +SELECT DISTINCT ON (deptno, job) empno, ename, deptno, job +FROM emp +ORDER BY deptno, job, empno; ++-------+--------+--------+-----------+ +| EMPNO | ENAME | DEPTNO | JOB | ++-------+--------+--------+-----------+ +| 7934 | MILLER | 10 | CLERK | +| 7782 | CLARK | 10 | MANAGER | +| 7839 | KING | 10 | PRESIDENT | +| 7788 | SCOTT | 20 | ANALYST | +| 7369 | SMITH | 20 | CLERK | +| 7566 | JONES | 20 | MANAGER | +| 7900 | JAMES | 30 | CLERK | +| 7698 | BLAKE | 30 | MANAGER | +| 7499 | ALLEN | 30 | SALESMAN | ++-------+--------+--------+-----------+ +(9 rows) + +!ok + +# Test DISTINCT ON with window function +SELECT DISTINCT ON (deptno) empno, ename, ROW_NUMBER() OVER (ORDER BY sal) as rn +FROM emp +ORDER BY deptno, empno; ++-------+-------+----+ +| EMPNO | ENAME | RN | ++-------+-------+----+ +| 7782 | CLARK | 9 | +| 7369 | SMITH | 1 | +| 7499 | ALLEN | 8 | ++-------+-------+----+ +(3 rows) + +!ok + +# Test DISTINCT and DISTINCT ON are mutually exclusive +SELECT DISTINCT DISTINCT ON (deptno) empno, ename, deptno +FROM emp +ORDER BY deptno, empno; +Incorrect syntax near the keyword 'DISTINCT' +!error + +# Test DISTINCT ON unparse +SELECT DISTINCT ON (deptno) empno, ename, deptno +FROM emp +ORDER BY deptno, empno; + +SELECT DISTINCT ON ("DEPTNO") "EMP"."EMPNO", "EMP"."ENAME", "EMP"."DEPTNO" +FROM "scott"."EMP" AS "EMP" +ORDER BY "DEPTNO", "EMPNO" +!explain-validated-on all + # End select.iq diff --git a/core/src/main/codegen/config.fmpp b/core/src/main/codegen/config.fmpp index 73d981bf3934..d8286c4ce1d0 100644 --- a/core/src/main/codegen/config.fmpp +++ b/core/src/main/codegen/config.fmpp @@ -40,8 +40,9 @@ data: { # FMPP will use the declaration from default_config.fmpp. parser: { # Generated parser implementation package and class name. - package: "org.apache.calcite.sql.parser.impl", - class: "SqlParserImpl", + package: "org.apache.calcite.sql.parser.impl" + class: "SqlParserImpl" + includeDistinctOn: true # List of files in @includes directory that have parser method # implementations for parsing custom SQL statements, literals or types diff --git a/core/src/main/codegen/default_config.fmpp b/core/src/main/codegen/default_config.fmpp index a2547273cb10..9cb054233719 100644 --- a/core/src/main/codegen/default_config.fmpp +++ b/core/src/main/codegen/default_config.fmpp @@ -460,4 +460,6 @@ parser: { includeAdditionalDeclarations: false includeParsingStringLiteralAsArrayLiteral: false includeIntervalWithoutQualifier: false + includeStarExclude: false + includeDistinctOn: false } diff --git a/core/src/main/codegen/templates/Parser.jj b/core/src/main/codegen/templates/Parser.jj index 4bbf49840c1f..3b2209538e21 100644 --- a/core/src/main/codegen/templates/Parser.jj +++ b/core/src/main/codegen/templates/Parser.jj @@ -1023,6 +1023,24 @@ List FunctionParameterList(ExprContext exprContext) : } } +void AllOrDistinctOrDistinctOn(List keywords, List distinctOnList) : +{ + final Span s; + final SqlNodeList distinctOn; +} +{ + { keywords.add(SqlSelectKeyword.ALL.symbol(getPos())); } +| + { s = span(); } + ( + + distinctOn = ExpressionCommaList(s, ExprContext.ACCEPT_SUB_QUERY) + + { distinctOnList.addAll(distinctOn.getList()); } + )? + { keywords.add(SqlSelectKeyword.DISTINCT.symbol(s.end(this))); } +} + SqlLiteral AllOrDistinct() : { } @@ -1389,6 +1407,7 @@ SqlSelect SqlSelect() : final SqlNode qualify; final SqlNodeList by; final List hints = new ArrayList(); + final List distinctOnList = new ArrayList(); final Span s; } { @@ -1400,9 +1419,15 @@ SqlSelect SqlSelect() : keywords.add(SqlSelectKeyword.STREAM.symbol(getPos())); } )? +<#if parser.includeDistinctOn!default.parser.includeDistinctOn> + ( + AllOrDistinctOrDistinctOn(keywords, distinctOnList) + )? +<#else> ( keyword = AllOrDistinct() { keywords.add(keyword); } )? + { keywordList = new SqlNodeList(keywords, s.addAll(keywords).pos()); } @@ -1431,10 +1456,14 @@ SqlSelect SqlSelect() : } ) { + final SqlNodeList distinctOn = distinctOnList.isEmpty() + ? null + : new SqlNodeList(distinctOnList, Span.of(distinctOnList).pos()); final SqlSelect select = new SqlSelect(s.end(this), keywordList, new SqlNodeList(selectList, Span.of(selectList).pos()), fromClause, where, groupBy, having, windowDecls, qualify, - null, null, null, new SqlNodeList(hints, getPos())); + null, null, null, new SqlNodeList(hints, getPos()), + distinctOn); SqlByRewriter.rewrite(select, by); return select; } diff --git a/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java b/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java index 13745bf0dfdb..171a8da68aa1 100644 --- a/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java +++ b/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java @@ -508,6 +508,15 @@ ExInst intervalFieldExceedsPrecision(Number a0, @BaseMessage("QUALIFY expression ''{0}'' must contain a window function") ExInst qualifyExpressionMustContainWindowFunction(String a0); + @BaseMessage("SELECT DISTINCT ON is not supported under the current SQL conformance level") + ExInst distinctOnNotAllowed(); + + @BaseMessage("SELECT DISTINCT ON requires an ORDER BY clause") + ExInst distinctOnRequiresOrderBy(); + + @BaseMessage("SELECT DISTINCT ON expressions must match ORDER BY expressions") + ExInst distinctOnOrderByMismatch(); + @BaseMessage("ROW/RANGE not allowed with RANK, DENSE_RANK, ROW_NUMBER, PERCENTILE_CONT/DISC or LAG/LEAD functions") ExInst rankWithFrame(); diff --git a/core/src/main/java/org/apache/calcite/sql/SqlSelect.java b/core/src/main/java/org/apache/calcite/sql/SqlSelect.java index 0faa1d024f57..ad542c31a005 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlSelect.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlSelect.java @@ -43,6 +43,7 @@ public class SqlSelect extends SqlCall { public static final int WHERE_OPERAND = 3; public static final int HAVING_OPERAND = 5; public static final int QUALIFY_OPERAND = 7; + public static final int DISTINCT_ON_OPERAND = 12; SqlNodeList keywordList; SqlNodeList selectList; @@ -56,6 +57,7 @@ public class SqlSelect extends SqlCall { @Nullable SqlNode offset; @Nullable SqlNode fetch; @Nullable SqlNodeList hints; + @Nullable SqlNodeList distinctOn; boolean hasByClause; //~ Constructors ----------------------------------------------------------- @@ -72,7 +74,8 @@ public SqlSelect(SqlParserPos pos, @Nullable SqlNodeList orderBy, @Nullable SqlNode offset, @Nullable SqlNode fetch, - @Nullable SqlNodeList hints) { + @Nullable SqlNodeList hints, + @Nullable SqlNodeList distinctOn) { super(pos); this.keywordList = requireNonNull(keywordList != null ? keywordList : new SqlNodeList(pos)); @@ -88,10 +91,29 @@ public SqlSelect(SqlParserPos pos, this.offset = offset; this.fetch = fetch; this.hints = hints; + this.distinctOn = distinctOn; this.hasByClause = false; } - /** deprecated, without {@code qualify}. */ + /** Constructor without {@code distinctOn}; distinctOn defaults to null. */ + public SqlSelect(SqlParserPos pos, + @Nullable SqlNodeList keywordList, + SqlNodeList selectList, + @Nullable SqlNode from, + @Nullable SqlNode where, + @Nullable SqlNodeList groupBy, + @Nullable SqlNode having, + @Nullable SqlNodeList windowDecls, + @Nullable SqlNode qualify, + @Nullable SqlNodeList orderBy, + @Nullable SqlNode offset, + @Nullable SqlNode fetch, + @Nullable SqlNodeList hints) { + this(pos, keywordList, selectList, from, where, groupBy, having, + windowDecls, qualify, orderBy, offset, fetch, hints, null); + } + + /** deprecated, without {@code qualify} and {@code distinctOn}. */ @Deprecated // to be removed before 2.0 public SqlSelect(SqlParserPos pos, @Nullable SqlNodeList keywordList, @@ -106,7 +128,7 @@ public SqlSelect(SqlParserPos pos, @Nullable SqlNode fetch, @Nullable SqlNodeList hints) { this(pos, keywordList, selectList, from, where, groupBy, having, - windowDecls, null, orderBy, offset, fetch, hints); + windowDecls, null, orderBy, offset, fetch, hints, null); } //~ Methods ---------------------------------------------------------------- @@ -122,7 +144,8 @@ public SqlSelect(SqlParserPos pos, @SuppressWarnings("nullness") @Override public List getOperandList() { return ImmutableNullableList.of(keywordList, selectList, from, where, - groupBy, having, windowDecls, qualify, orderBy, offset, fetch, hints); + groupBy, having, windowDecls, qualify, orderBy, offset, fetch, hints, + distinctOn); } @Override public void setOperand(int i, @Nullable SqlNode operand) { @@ -160,12 +183,23 @@ public SqlSelect(SqlParserPos pos, case 10: fetch = operand; break; + case 11: + hints = (SqlNodeList) operand; + break; + case 12: + distinctOn = (SqlNodeList) operand; + break; default: throw new AssertionError(i); } } public final boolean isDistinct() { + // DISTINCT ON is mutually exclusive with DISTINCT, so when DISTINCT ON is present, + // we return false to indicate that standard DISTINCT processing is not needed + if (isDistinctOn()) { + return false; + } return getModifierNode(SqlSelectKeyword.DISTINCT) != null; } @@ -273,10 +307,6 @@ public void setFetch(@Nullable SqlNode fetch) { this.fetch = fetch; } - public void setHints(@Nullable SqlNodeList hints) { - this.hints = hints; - } - @Pure public @Nullable SqlNodeList getHints() { return this.hints; @@ -327,4 +357,12 @@ public boolean hasWhere() { public boolean isKeywordPresent(SqlSelectKeyword targetKeyWord) { return getModifierNode(targetKeyWord) != null; } + + public boolean isDistinctOn() { + return distinctOn != null && !distinctOn.isEmpty(); + } + + public @Nullable SqlNodeList getDistinctOn() { + return distinctOn; + } } diff --git a/core/src/main/java/org/apache/calcite/sql/SqlSelectOperator.java b/core/src/main/java/org/apache/calcite/sql/SqlSelectOperator.java index fea1c2235fe9..1c1a8edfe8c3 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlSelectOperator.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlSelectOperator.java @@ -78,7 +78,8 @@ private SqlSelectOperator() { (SqlNodeList) operands[8], operands[9], operands[10], - (SqlNodeList) operands[11]); + (SqlNodeList) operands[11], + operands.length > 12 ? (SqlNodeList) operands[12] : null); } /** @@ -114,7 +115,8 @@ public SqlSelect createCall( orderBy, offset, fetch, - hints); + hints, + null); } @Override public void acceptCall( @@ -150,6 +152,13 @@ public SqlSelect createCall( final SqlNode keyword = select.keywordList.get(i); keyword.unparse(writer, 0, 0); } + if (select.isDistinctOn()) { + writer.keyword("ON"); + final SqlWriter.Frame frame = + writer.startList("(", ")"); + castNonNull(select.distinctOn).unparse(writer, 0, 0); + writer.endList(frame); + } writer.topN(select.fetch, select.offset); final SqlNodeList selectClause = select.selectList; writer.list(SqlWriter.FrameTypeEnum.SELECT_LIST, SqlWriter.COMMA, diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlAbstractConformance.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlAbstractConformance.java index b19d8c071d10..82f06a48d168 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlAbstractConformance.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlAbstractConformance.java @@ -172,4 +172,8 @@ public abstract class SqlAbstractConformance implements SqlConformance { @Override public boolean supportsUnsignedTypes() { return SqlConformanceEnum.DEFAULT.supportsUnsignedTypes(); } + + @Override public boolean isDistinctOnAllowed() { + return SqlConformanceEnum.DEFAULT.isDistinctOnAllowed(); + } } diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlConformance.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlConformance.java index fcd3b46641f6..32b4a03b90d9 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlConformance.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlConformance.java @@ -681,4 +681,14 @@ default boolean isColonFieldAccessAllowed() { * True when the unsigned versions of integer types are supported. */ boolean supportsUnsignedTypes(); + + /** + * Whether {@code SELECT DISTINCT ON} is allowed. + * + *

      Among the built-in conformance levels, true in + * {@link SqlConformanceEnum#BABEL}, + * {@link SqlConformanceEnum#LENIENT}; + * false otherwise. + */ + boolean isDistinctOnAllowed(); } diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlConformanceEnum.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlConformanceEnum.java index f0f258171b3d..4475c4ce8096 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlConformanceEnum.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlConformanceEnum.java @@ -532,4 +532,14 @@ public enum SqlConformanceEnum implements SqlConformance { return false; } } + + @Override public boolean isDistinctOnAllowed() { + switch (this) { + case BABEL: + case LENIENT: + return true; + default: + return false; + } + } } diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlDelegatingConformance.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlDelegatingConformance.java index 25f8d7e03747..0d415d8aec24 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlDelegatingConformance.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlDelegatingConformance.java @@ -177,4 +177,8 @@ protected SqlDelegatingConformance(SqlConformance delegate) { @Override public boolean supportsUnsignedTypes() { return delegate.supportsUnsignedTypes(); } + + @Override public boolean isDistinctOnAllowed() { + return delegate.isDistinctOnAllowed(); + } } diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java index 1e2b29880864..a41be8c67b6c 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java @@ -3267,8 +3267,10 @@ private void registerQuery( if (orderList != null) { // If the query is 'SELECT DISTINCT', restrict the columns // available to the ORDER BY clause. + // DISTINCT ON is an exception: ORDER BY may reference columns + // not in the SELECT list. final SqlValidatorScope selectScope3 = - select.isDistinct() + (select.isDistinct() && !select.isDistinctOn()) ? new AggregatingSelectScope(selectScope, select, true) : selectScope2; OrderByScope orderScope = @@ -4463,6 +4465,7 @@ protected void validateSelect( // dialects you can refer to columns of the select list, e.g. // "SELECT empno AS x FROM emp ORDER BY x" validateOrderList(select); + validateDistinctOnClause(select); if (shouldCheckForRollUp(from)) { checkRollUpInSelectList(select); @@ -4997,6 +5000,54 @@ protected void validateQualifyClause(SqlSelect select) { } } + protected void validateDistinctOnClause(SqlSelect select) { + SqlNodeList distinctOn = select.getDistinctOn(); + if (distinctOn == null || distinctOn.isEmpty()) { + return; + } + + if (!config.conformance().isDistinctOnAllowed()) { + throw newValidationError(select, RESOURCE.distinctOnNotAllowed()); + } + + SqlNodeList orderList = select.getOrderList(); + if (orderList == null || orderList.isEmpty()) { + throw newValidationError(select, RESOURCE.distinctOnRequiresOrderBy()); + } + + if (orderList.size() < distinctOn.size()) { + throw newValidationError(orderList.get(orderList.size() - 1), + RESOURCE.distinctOnOrderByMismatch()); + } + + final SqlValidatorScope orderScope = getOrderScope(select); + for (int i = 0; i < distinctOn.size(); i++) { + SqlNode distinctOnExpr = expand(distinctOn.get(i), orderScope); + SqlNode orderItem = orderList.get(i); + SqlNode orderExpr = stripOrderByModifiers(orderItem); + orderExpr = expand(orderExpr, orderScope); + if (!SqlNode.equalDeep(distinctOnExpr, orderExpr, Litmus.IGNORE)) { + throw newValidationError(orderItem, RESOURCE.distinctOnOrderByMismatch()); + } + } + } + + /** Strips ASC, DESC, NULLS FIRST, NULLS LAST from an ORDER BY item. */ + private static SqlNode stripOrderByModifiers(SqlNode orderExpr) { + while (orderExpr instanceof SqlCall) { + SqlCall call = (SqlCall) orderExpr; + SqlKind kind = call.getKind(); + if (kind == SqlKind.DESCENDING + || kind == SqlKind.NULLS_FIRST + || kind == SqlKind.NULLS_LAST) { + orderExpr = call.operand(0); + } else { + break; + } + } + return orderExpr; + } + protected void validateMustFilterRequirements(SqlSelect select, SelectNamespace ns) { ns.filterRequirement = FilterRequirement.EMPTY; if (select.getFrom() != null) { diff --git a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java index 53d54dd6d2a1..1551d1630da4 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java @@ -720,6 +720,9 @@ private static RelCollation requiredCollation(RelNode r) { if (r instanceof Project) { return requiredCollation(((Project) r).getInput()); } + if (r instanceof Filter) { + return requiredCollation(((Filter) r).getInput()); + } if (r instanceof Delta) { return requiredCollation(((Delta) r).getInput()); } @@ -816,9 +819,13 @@ protected void convertSelectImpl( distinctify(bb, true); } - convertOrder( - select, bb, collation, orderExprList, select.getOffset(), - select.getFetch()); + if (select.isDistinctOn()) { + convertOrder(select, bb, collation, orderExprList, null, null); + } else { + convertOrder( + select, bb, collation, orderExprList, select.getOffset(), + select.getFetch()); + } if (select.hasHints()) { final List hints = SqlUtil.getRelHint(hintStrategies, select.getHints()); @@ -839,6 +846,20 @@ protected void convertSelectImpl( } else { bb.setRoot(bb.root(), true); } + + if (select.isDistinctOn()) { + convertDistinctOn(bb, select, collationList); + final @Nullable RexNode offsetExpr = select.getOffset() == null + ? null : convertExpression(select.getOffset()); + final @Nullable RexNode fetchExpr = select.getFetch() == null + ? null : convertExpression(select.getFetch()); + if (offsetExpr != null || fetchExpr != null) { + bb.setRoot( + LogicalSort.create(bb.root(), RelCollations.EMPTY, + offsetExpr, fetchExpr), + false); + } + } } /** @@ -1021,6 +1042,77 @@ private void distinctify( rel.getRowType().getFieldNames(), ImmutableSet.of()), false); } + /** + * Converts a SELECT DISTINCT ON clause into a relational expression + * using ROW_NUMBER() window function. + */ + private void convertDistinctOn(Blackboard bb, SqlSelect select, + List collationList) { + if (bb.root == null) { + throw new IllegalArgumentException("rel must not be null"); + } + final SqlNodeList distinctOn = requireNonNull(select.getDistinctOn(), "distinctOn"); + + relBuilder.push(bb.root()); + final RelDataType inputRowType = bb.root().getRowType(); + + // Build PARTITION BY expressions from DISTINCT ON. + // DISTINCT ON expressions are a prefix of ORDER BY, + // so we can use the field indices from collationList. + final List partitionKeys = new ArrayList<>(); + for (int i = 0; i < distinctOn.size(); i++) { + RelFieldCollation fieldCollation = collationList.get(i); + partitionKeys.add( + rexBuilder.makeInputRef(inputRowType, fieldCollation.getFieldIndex())); + } + + // Build ORDER BY expressions for the window function + final List orderKeys = new ArrayList<>(); + for (RelFieldCollation fieldCollation : collationList) { + RexNode ref = rexBuilder.makeInputRef(inputRowType, fieldCollation.getFieldIndex()); + final Set kinds = new HashSet<>(); + if (fieldCollation.getDirection() == RelFieldCollation.Direction.DESCENDING) { + kinds.add(SqlKind.DESCENDING); + } + switch (fieldCollation.nullDirection) { + case FIRST: + kinds.add(SqlKind.NULLS_FIRST); + break; + case LAST: + kinds.add(SqlKind.NULLS_LAST); + break; + default: + break; + } + orderKeys.add(new RexFieldCollation(ref, kinds)); + } + + final RelDataType bigintType = + typeFactory.createSqlType(SqlTypeName.BIGINT); + final RexNode rowNumber = + rexBuilder.makeOver(bigintType, SqlStdOperatorTable.ROW_NUMBER, ImmutableList.of(), + partitionKeys, ImmutableList.copyOf(orderKeys), + RexWindowBounds.UNBOUNDED_PRECEDING, RexWindowBounds.CURRENT_ROW, + RexWindowExclusion.EXCLUDE_NO_OTHER, true, true, false, false, false); + + // Add ROW_NUMBER as the last column + final List fields = new ArrayList<>(relBuilder.fields()); + fields.add(rowNumber); + relBuilder.project(fields); + + // Filter rn = 1 + relBuilder.filter( + relBuilder.equals( + Util.last(relBuilder.fields()), + relBuilder.literal(BigDecimal.ONE))); + + // Remove the ROW_NUMBER column + relBuilder.project( + Util.skipLast(relBuilder.fields())); + + bb.setRoot(relBuilder.build(), false); + } + /** * Converts a query's ORDER BY clause, if any. * diff --git a/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties b/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties index 325b69d87235..4096537b7fb9 100644 --- a/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties +++ b/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties @@ -171,6 +171,9 @@ FollowingBeforePrecedingError=Upper frame boundary cannot be PRECEDING when lowe WindowNameMustBeSimple=Window name must be a simple identifier DuplicateWindowName=Duplicate window names not allowed EmptyWindowSpec=Empty window specification not allowed +DistinctOnNotAllowed=SELECT DISTINCT ON is not supported under the current SQL conformance level +DistinctOnRequiresOrderBy=SELECT DISTINCT ON requires an ORDER BY clause +DistinctOnOrderByMismatch=SELECT DISTINCT ON expressions must match ORDER BY expressions DupWindowSpec=Duplicate window specification not allowed in the same window clause QualifyExpressionMustContainWindowFunction=QUALIFY expression ''{0}'' must contain a window function RankWithFrame=ROW/RANGE not allowed with RANK, DENSE_RANK, ROW_NUMBER, PERCENTILE_CONT/DISC or LAG/LEAD functions diff --git a/core/src/test/codegen/config.fmpp b/core/src/test/codegen/config.fmpp index 99ecb93da962..3f985001913a 100644 --- a/core/src/test/codegen/config.fmpp +++ b/core/src/test/codegen/config.fmpp @@ -69,6 +69,7 @@ data: { implementationFiles: [ "parserImpls.ftl" ] + includeDistinctOn: true } } diff --git a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java index 218f018fb683..c9f409f8c367 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java @@ -6239,4 +6239,35 @@ void checkUserDefinedOrderByOver(NullCollation nullCollation) { assertThat(plan, not(containsString("FLOOR(FLOOR"))); assertThat(plan, containsString("FLOOR($4, FLAG(WEEK))")); } + + /** Test case of + * [CALCITE-5406] + * Support the SELECT DISTINCT ON statement for PostgreSQL dialect. */ + @Test void testDistinctOnSimple() { + final String sql = "SELECT DISTINCT ON (deptno) empno, ename\n" + + "FROM emp\n" + + "ORDER BY deptno, empno"; + sql(sql).withConformance(SqlConformanceEnum.LENIENT).ok(); + } + + /** Test case of + * [CALCITE-5406] + * Support the SELECT DISTINCT ON statement for PostgreSQL dialect. */ + @Test void testDistinctOnMultiple() { + final String sql = "SELECT DISTINCT ON (deptno, job) empno, ename\n" + + "FROM emp\n" + + "ORDER BY deptno, job, hiredate DESC"; + sql(sql).withConformance(SqlConformanceEnum.LENIENT).ok(); + } + + /** Test case of + * [CALCITE-5406] + * Support the SELECT DISTINCT ON statement for PostgreSQL dialect. */ + @Test void testDistinctOnWithLimit() { + final String sql = "SELECT DISTINCT ON (deptno) empno, ename\n" + + "FROM emp\n" + + "ORDER BY deptno, empno\n" + + "LIMIT 5"; + sql(sql).withConformance(SqlConformanceEnum.LENIENT).ok(); + } } diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index 4b4d2302de52..c4e8993839d9 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -6817,7 +6817,130 @@ void testReturnsCorrectRowTypeOnCombinedJoin() { f.withSql(qualifyOnMultipleWindowFunctions).ok(); } - /** Negative tests for the {@code QUALIFY} clause. */ + /** Test case of + * [CALCITE-5406] + * Support the SELECT DISTINCT ON statement for PostgreSQL dialect. */ + @Test void testDistinctOnNotAllowed() { + // DISTINCT ON is not allowed under default SQL conformance + sql("^SELECT DISTINCT ON (deptno) empno FROM emp^ ORDER BY deptno") + .fails("SELECT DISTINCT ON is not supported under the current SQL conformance level"); + } + + /** Test case of + * [CALCITE-5406] + * Support the SELECT DISTINCT ON statement for PostgreSQL dialect. */ + @Test void testDistinctOnPositive() { + final SqlValidatorFixture f = + fixture().withConformance(SqlConformanceEnum.LENIENT); + + f.withSql("SELECT DISTINCT ON (deptno) empno, ename FROM emp ORDER BY deptno, empno") + .ok(); + + f.withSql("SELECT DISTINCT ON (deptno, job) empno FROM emp ORDER BY deptno, job, hiredate") + .ok(); + + f.withSql("SELECT DISTINCT ON (deptno) empno FROM emp ORDER BY deptno DESC") + .ok(); + + f.withSql("SELECT DISTINCT ON (deptno) empno FROM emp ORDER BY deptno NULLS FIRST") + .ok(); + } + + /** Test case of + * [CALCITE-5406] + * Support the SELECT DISTINCT ON statement for PostgreSQL dialect. */ + @Test void testDistinctOnNegative() { + final SqlValidatorFixture f = + fixture().withConformance(SqlConformanceEnum.LENIENT); + + // DISTINCT ON requires ORDER BY + f.withSql("^SELECT DISTINCT ON (deptno) empno FROM emp^") + .fails("SELECT DISTINCT ON requires an ORDER BY clause"); + + // ORDER BY must contain all DISTINCT ON expressions as prefix + f.withSql("SELECT DISTINCT ON (deptno, job) empno FROM emp ORDER BY ^deptno^") + .fails("SELECT DISTINCT ON expressions must match ORDER BY expressions"); + + // ORDER BY prefix must match exactly + f.withSql("SELECT DISTINCT ON (deptno, job) empno FROM emp ORDER BY ^job^, deptno") + .fails("SELECT DISTINCT ON expressions must match ORDER BY expressions"); + + // DISTINCT ON with extra ORDER BY is ok + f.withSql("SELECT DISTINCT ON (deptno) empno FROM emp ORDER BY deptno, job") + .ok(); + + // Duplicate expressions in DISTINCT ON are allowed but must be matched in ORDER BY + f.withSql("SELECT DISTINCT ON (deptno, deptno) deptno, empno FROM emp ORDER BY deptno, deptno") + .ok(); + + // DISTINCT ON with expression + f.withSql("SELECT DISTINCT ON (empno % 2) empno, ename FROM emp ORDER BY empno % 2") + .ok(); + + // Empty DISTINCT ON is not allowed (parse error) + f.withSql("SELECT DISTINCT ON ((^)^) deptno FROM emp") + .fails("(?s)Encountered \"\\)\" at .*"); + + // DISTINCT ON can reference alias (like ORDER BY) + f.withSql("SELECT DISTINCT ON (x) empno AS x, deptno FROM emp ORDER BY x") + .ok(); + + // DISTINCT ON with alias-column name clash: alias in SELECT takes precedence + f.withSql("SELECT DISTINCT ON (deptno) empno AS deptno, deptno AS d FROM emp ORDER BY deptno") + .ok(); + + // DISTINCT ON with qualified column reference + f.withSql("SELECT DISTINCT ON (e.deptno) e.deptno AS x, e.empno " + + "FROM emp AS e ORDER BY e.deptno") + .ok(); + + // Integer literal in both DISTINCT ON and ORDER BY matches at validator + // (ordinal resolution happens later in SqlToRelConverter) + f.withSql("SELECT DISTINCT ON (2) empno, ename FROM emp ORDER BY 2") + .ok(); + + // Expressions in DISTINCT ON (not ordinals) + f.withSql("SELECT DISTINCT ON (empno % 2, CHAR_LENGTH(ename)) empno, ename " + + "FROM emp ORDER BY empno % 2, CHAR_LENGTH(ename)") + .ok(); + + // DISTINCT ON referencing non-projected column requires ORDER BY + f.withSql("^SELECT DISTINCT ON (deptno) empno, ename FROM emp^") + .fails("SELECT DISTINCT ON requires an ORDER BY clause"); + + // DISTINCT ON with aggregate query + f.withSql("SELECT DISTINCT ON (deptno) deptno, SUM(sal) " + + "FROM emp GROUP BY deptno ORDER BY deptno") + .ok(); + + // DISTINCT ON with aggregate expression + f.withSql("SELECT DISTINCT ON (SUM(sal)) deptno, SUM(sal) " + + "FROM emp GROUP BY deptno ORDER BY SUM(sal)") + .ok(); + + // DISTINCT ON with aggregate query and alias + f.withSql("SELECT DISTINCT ON (sum_sal) deptno, SUM(sal) AS sum_sal " + + "FROM emp GROUP BY deptno ORDER BY sum_sal") + .ok(); + + // DISTINCT ON with USING join (requires qualified reference due to Calcite limitation) + f.withSql("SELECT DISTINCT ON (emp.deptno) * " + + "FROM emp JOIN dept USING (deptno) ORDER BY emp.deptno") + .ok(); + + // DISTINCT ON with NATURAL join (requires qualified reference due to Calcite limitation) + f.withSql("SELECT DISTINCT ON (emp.deptno) * FROM emp NATURAL JOIN dept ORDER BY emp.deptno") + .ok(); + + // DISTINCT ON with unqualified column reference - should work if column is unambiguous + f.withSql("SELECT DISTINCT ON (deptno) empno, deptno FROM emp ORDER BY deptno, empno") + .ok(); + + // DISTINCT ON with unqualified column in ORDER BY - should match + f.withSql("SELECT DISTINCT ON (deptno) empno, deptno FROM emp ORDER BY deptno") + .ok(); + } + @Test void testQualifyNegative() { final SqlValidatorFixture f = fixture().withConformance(SqlConformanceEnum.LENIENT); diff --git a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml index 6900b06ff8fb..1d05fd1b2f18 100644 --- a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml @@ -1991,6 +1991,62 @@ LogicalTableModify(table=[[CATALOG, SALES, EMP]], operation=[DELETE], flattened= LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7], SLACKER=[$8]) LogicalFilter(condition=[=($7, 10)]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +]]> + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/server/src/main/codegen/config.fmpp b/server/src/main/codegen/config.fmpp index e5c3eaf4d27a..ba64f0b71a51 100644 --- a/server/src/main/codegen/config.fmpp +++ b/server/src/main/codegen/config.fmpp @@ -99,6 +99,7 @@ data: { implementationFiles: [ "parserImpls.ftl" ] + includeDistinctOn: true } } diff --git a/site/_docs/reference.md b/site/_docs/reference.md index 80337bbfb125..b6ecb6b52bfa 100644 --- a/site/_docs/reference.md +++ b/site/_docs/reference.md @@ -205,7 +205,7 @@ orderItem: expression [ ASC | DESC ] [ NULLS FIRST | NULLS LAST ] select: - SELECT [ hintComment ] [ STREAM ] [ ALL | DISTINCT ] + SELECT [ hintComment ] [ STREAM ] [ ALL | DISTINCT [ ON '(' expression [, expression ]* ')' ] ] { starWithExclude | projectItem [, projectItem ]* } [ BY expression [, expression ]* ] FROM tableExpression @@ -214,6 +214,8 @@ select: [ HAVING booleanExpression ] [ WINDOW windowName AS windowSpec [, windowName AS windowSpec ]* ] [ QUALIFY booleanExpression ] + [ ORDER BY orderItem [, orderItem ]* ] + [ LIMIT expression [ OFFSET expression ] ] The optional, non-standard `BY` clause groups and orders the query by the specified expressions, and automatically adds them to the SELECT list @@ -221,6 +223,13 @@ for naming and positional reference. But `SELECT ... BY` cannot be combined with an explicit `GROUP BY` or `ORDER BY` clause in the same query. `SELECT ... BY` is recognized only when the Babel parser is enabled. It sets the generated parser configuration flag `includeSelectBy` to `true`. +The optional `DISTINCT ON` clause is a PostgreSQL extension that allows you to +eliminate duplicate rows based on specified expressions, keeping the first row +in each group as determined by the `ORDER BY` clause. This is recognized only +when the Babel parser is enabled. It sets the generated parser configuration flag +`includeDistinctOn` to `true`. When using `DISTINCT ON`, the expressions in the +`DISTINCT ON` clause must match the beginning of the `ORDER BY` clause. + For example: {% highlight sql %} @@ -236,6 +245,16 @@ GROUP BY deptno ORDER BY deptno {% endhighlight %} +Example of `DISTINCT ON`: + +{% highlight sql %} +SELECT DISTINCT ON (deptno) empno, ename, deptno +FROM emp +ORDER BY deptno, empno +{% endhighlight %} + +This query keeps only the first employee (ordered by empno) in each department. + selectWithoutFrom: SELECT [ ALL | DISTINCT ] { * | projectItem [, projectItem ]* } diff --git a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java index 5c24c0a806db..0adc5be12aba 100644 --- a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java +++ b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java @@ -6230,6 +6230,104 @@ private static Matcher isCharLiteral(String s) { sql(sql).fails("(?s).*Encountered \"QUALIFY\" at .*"); } + /** Test case of + * [CALCITE-5406] + * Support the SELECT DISTINCT ON statement for PostgreSQL dialect. */ + @Test void testDistinctOn() { + final String sql = "SELECT DISTINCT ON (deptno) empno, ename\n" + + "FROM emp\n" + + "ORDER BY deptno, empno"; + + final String expected = "SELECT DISTINCT ON (`DEPTNO`) `EMPNO`, `ENAME`\n" + + "FROM `EMP`\n" + + "ORDER BY `DEPTNO`, `EMPNO`"; + sql(sql).ok(expected); + } + + /** Test case of + * [CALCITE-5406] + * Support the SELECT DISTINCT ON statement for PostgreSQL dialect. */ + @Test void testDistinctOnMultiple() { + final String sql = "SELECT DISTINCT ON (deptno, job) empno, ename\n" + + "FROM emp\n" + + "ORDER BY deptno, job, hiredate DESC"; + + final String expected = "SELECT DISTINCT ON (`DEPTNO`, `JOB`) `EMPNO`, `ENAME`\n" + + "FROM `EMP`\n" + + "ORDER BY `DEPTNO`, `JOB`, `HIREDATE` DESC"; + sql(sql).ok(expected); + } + + /** Test case of + * [CALCITE-5406] + * Support the SELECT DISTINCT ON statement for PostgreSQL dialect. */ + @Test void testDistinctOnWithEverything() { + final String sql = "SELECT DISTINCT ON (deptno) empno, ename\n" + + "FROM emp\n" + + "WHERE deptno > 3\n" + + "GROUP BY deptno, empno, ename\n" + + "HAVING COUNT(*) > 1\n" + + "ORDER BY deptno, empno\n" + + "LIMIT 5\n"; + + final String expected = "SELECT DISTINCT ON (`DEPTNO`) `EMPNO`, `ENAME`\n" + + "FROM `EMP`\n" + + "WHERE (`DEPTNO` > 3)\n" + + "GROUP BY `DEPTNO`, `EMPNO`, `ENAME`\n" + + "HAVING (COUNT(*) > 1)\n" + + "ORDER BY `DEPTNO`, `EMPNO`\n" + + "FETCH NEXT 5 ROWS ONLY"; + sql(sql).ok(expected); + } + + /** Test case of + * [CALCITE-5406] + * Support the SELECT DISTINCT ON statement for PostgreSQL dialect. */ + @Test void testDistinctOnWithOffset() { + final String sql = "SELECT DISTINCT ON (deptno) empno, ename\n" + + "FROM emp\n" + + "ORDER BY deptno, empno\n" + + "OFFSET 10 ROWS\n" + + "FETCH NEXT 5 ROWS ONLY"; + + final String expected = "SELECT DISTINCT ON (`DEPTNO`) `EMPNO`, `ENAME`\n" + + "FROM `EMP`\n" + + "ORDER BY `DEPTNO`, `EMPNO`\n" + + "OFFSET 10 ROWS\n" + + "FETCH NEXT 5 ROWS ONLY"; + sql(sql).ok(expected); + } + + /** Test case of + * [CALCITE-5406] + * Support the SELECT DISTINCT ON statement for PostgreSQL dialect. */ + @Test void testDistinctOnWithExpression() { + final String sql = "SELECT DISTINCT ON (deptno + 1) empno, ename\n" + + "FROM emp\n" + + "ORDER BY deptno + 1, empno"; + + final String expected = "SELECT DISTINCT ON ((`DEPTNO` + 1)) `EMPNO`, `ENAME`\n" + + "FROM `EMP`\n" + + "ORDER BY (`DEPTNO` + 1), `EMPNO`"; + sql(sql).ok(expected); + } + + /** Test case of + * [CALCITE-5406] + * Support the SELECT DISTINCT ON statement for PostgreSQL dialect. */ + @Test void testDistinctOnWithJoin() { + final String sql = "SELECT DISTINCT ON (e.deptno) e.empno, d.dname\n" + + "FROM emp AS e JOIN dept AS d ON e.deptno = d.deptno\n" + + "ORDER BY e.deptno, e.empno"; + + final String expected = "SELECT DISTINCT ON (`E`.`DEPTNO`) " + + "`E`.`EMPNO`, `D`.`DNAME`\n" + + "FROM `EMP` AS `E`\n" + + "INNER JOIN `DEPT` AS `D` ON (`E`.`DEPTNO` = `D`.`DEPTNO`)\n" + + "ORDER BY `E`.`DEPTNO`, `E`.`EMPNO`"; + sql(sql).ok(expected); + } + @Test void testNullTreatment() { sql("select lead(x) respect nulls over (w) from t") .ok("SELECT (LEAD(`X`) RESPECT NULLS OVER (`W`))\n" From f36a9062cf1596b7e6a0ef03e2267ca358cd2d0b Mon Sep 17 00:00:00 2001 From: Sergey Nuyanzin Date: Sat, 20 Jun 2026 22:59:04 +0200 Subject: [PATCH 340/562] [CALCITE-7565] `TRIM` without `FROM` fails at parsing time --- core/src/main/codegen/templates/Parser.jj | 13 ++++++++++--- .../apache/calcite/sql/parser/SqlParserTest.java | 8 ++++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/core/src/main/codegen/templates/Parser.jj b/core/src/main/codegen/templates/Parser.jj index 3b2209538e21..077481c262a4 100644 --- a/core/src/main/codegen/templates/Parser.jj +++ b/core/src/main/codegen/templates/Parser.jj @@ -6953,9 +6953,16 @@ SqlNode BuiltinFunctionCall() : flag = SqlTrimFunction.Flag.LEADING.symbol(getPos()); } ) - [ trimChars = Expression(ExprContext.ACCEPT_SUB_QUERY) ] - { fromPos = getPos(); } - e = Expression(ExprContext.ACCEPT_SUB_QUERY) + ( + { fromPos = getPos() ;} + e = Expression(ExprContext.ACCEPT_SUB_QUERY) + | + e = Expression(ExprContext.ACCEPT_SUB_QUERY) + [ + { trimChars = e; fromPos = getPos(); } + e = Expression(ExprContext.ACCEPT_SUB_QUERY) + ] + ) ) | ( diff --git a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java index 0adc5be12aba..0fa8c43fd9d3 100644 --- a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java +++ b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java @@ -5945,6 +5945,14 @@ private static Matcher isCharLiteral(String s) { // Parser accepts a call to TRIM() with no arguments expr("trim(^)^") .fails("(?s).*Encountered \"\\)\" at line 1, column 6\\..*"); + // Test cases for [CALCITE-7565] https://issues.apache.org/jira/browse/CALCITE-7565 + // TRIM without FROM fails at parsing time + expr("trim(both ' a ')") + .ok("TRIM(BOTH ' ' FROM ' a ')"); + expr("trim(leading ' a ')") + .ok("TRIM(LEADING ' ' FROM ' a ')"); + expr("trim(trailing ' a ')") + .ok("TRIM(TRAILING ' ' FROM ' a ')"); } @Test void testConvertAndTranslate() { From 15570034da93656e0ad81c3762f179966d7dfcf4 Mon Sep 17 00:00:00 2001 From: Tisya Bhatia Date: Mon, 22 Jun 2026 10:03:01 -0500 Subject: [PATCH 341/562] [CALCITE-7597] Support ORDER BY ALL --- core/src/main/codegen/templates/Parser.jj | 31 ++++++++-- .../calcite/runtime/CalciteResource.java | 3 + .../java/org/apache/calcite/sql/SqlKind.java | 4 ++ .../calcite/sql/fun/SqlInternalOperators.java | 12 ++++ .../sql/validate/SqlValidatorImpl.java | 57 +++++++++++++++++++ .../runtime/CalciteResource.properties | 1 + .../calcite/sql/test/SqlAdvisorTest.java | 12 +++- .../apache/calcite/test/SqlValidatorTest.java | 24 ++++++++ core/src/test/resources/sql/sort.iq | 14 +++++ site/_docs/reference.md | 8 ++- .../calcite/sql/parser/SqlParserTest.java | 30 ++++++++++ 11 files changed, 188 insertions(+), 8 deletions(-) diff --git a/core/src/main/codegen/templates/Parser.jj b/core/src/main/codegen/templates/Parser.jj index 077481c262a4..5519dd566d2f 100644 --- a/core/src/main/codegen/templates/Parser.jj +++ b/core/src/main/codegen/templates/Parser.jj @@ -3093,6 +3093,7 @@ SqlNodeList OrderBy(boolean accept) : { final List list = new ArrayList(); final Span s; + SqlNode all; } { { @@ -3104,10 +3105,32 @@ SqlNodeList OrderBy(boolean accept) : throw SqlUtil.newContextException(s.pos(), RESOURCE.illegalOrderBy()); } } - OrderItemList(list) - { - return new SqlNodeList(list, s.addAll(list).pos()); - } + + ( + { all = SqlInternalOperators.ORDER_BY_ALL.createCall(getPos()); } + ( + + | { all = SqlStdOperatorTable.DESC.createCall(getPos(), all); } + )? + ( + LOOKAHEAD(2) + { + all = SqlStdOperatorTable.NULLS_FIRST.createCall(getPos(), all); + } + | + { + all = SqlStdOperatorTable.NULLS_LAST.createCall(getPos(), all); + } + )? + { + list.add(all); + return new SqlNodeList(list, s.addAll(list).pos()); + } + | + OrderItemList(list) { + return new SqlNodeList(list, s.addAll(list).pos()); + } + ) } <#if parser.includeSelectBy!false> diff --git a/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java b/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java index 171a8da68aa1..ac2ed8501980 100644 --- a/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java +++ b/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java @@ -777,6 +777,9 @@ ExInst illegalArgumentForTableFunctionCall(String a0, @BaseMessage("Streaming ORDER BY must start with monotonic expression") ExInst streamMustOrderByMonotonic(); + @BaseMessage("ORDER BY ALL requires an explicit SELECT list; ''*'' is not supported") + ExInst orderByAllRequiresExplicitSelectList(); + @BaseMessage("Set operator cannot combine streaming and non-streaming inputs") ExInst streamSetOpInconsistentInputs(); diff --git a/core/src/main/java/org/apache/calcite/sql/SqlKind.java b/core/src/main/java/org/apache/calcite/sql/SqlKind.java index 680c833c1721..a70c71df7614 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlKind.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlKind.java @@ -195,6 +195,9 @@ public enum SqlKind { */ ORDER_BY, + /** The ALL keyword of the ORDER BY clause. */ + ORDER_BY_ALL, + /** WITH clause. */ WITH, @@ -1482,6 +1485,7 @@ public enum SqlKind { TIMESTAMP_ADD, TIMESTAMP_DIFF, TIMESTAMP_SUB, EXTRACT, INTERVAL, LITERAL_CHAIN, JDBC_FN, PRECEDING, FOLLOWING, ORDER_BY, + ORDER_BY_ALL, NULLS_FIRST, NULLS_LAST, COLLECTION_TABLE, TABLESAMPLE, VALUES, WITH, WITH_ITEM, ITEM, SKIP_TO_FIRST, SKIP_TO_LAST, JSON_VALUE_EXPRESSION, UNNEST), diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlInternalOperators.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlInternalOperators.java index 8757e1e17753..d1429f501b04 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlInternalOperators.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlInternalOperators.java @@ -208,6 +208,18 @@ private SqlInternalOperators() { public static final SqlInternalOperator GROUP_BY_ALL = new SqlRollupOperator("GROUP BY ALL", SqlKind.GROUP_BY_ALL); + /** {@code ORDER BY ALL}, a placeholder expanded during validation into + * a standard {@code ORDER BY}. */ + public static final SqlInternalOperator ORDER_BY_ALL = + // High precedence so the placeholder is never wrapped in parentheses when it + // appears alone or inside DESC / NULLS FIRST | LAST in an always-parentheses writer + new SqlInternalOperator("ORDER BY ALL", SqlKind.ORDER_BY_ALL, 100) { + @Override public void unparse(SqlWriter writer, SqlCall call, + int leftPrec, int rightPrec) { + writer.keyword("ALL"); + } + }; + /** Fetch operator is ONLY used for its precedence during unparsing. */ public static final SqlOperator FETCH = SqlBasicOperator.create("FETCH") diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java index a41be8c67b6c..37f6712e90a7 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java @@ -5209,6 +5209,7 @@ protected void validateOrderList(SqlSelect select) { // ORDER BY is validated in a scope where aliases in the SELECT clause // are visible. For example, "SELECT empno AS x FROM emp ORDER BY x" // is valid. + rewriteOrderByAll(select); SqlNodeList orderList = select.getOrderList(); if (orderList == null) { return; @@ -5236,6 +5237,62 @@ protected void validateOrderList(SqlSelect select) { } } + protected void rewriteOrderByAll(SqlSelect select) { + final SqlNodeList orderList = select.getOrderList(); + if (orderList == null || orderList.size() != 1) { + return; + } + + SqlNode node = orderList.get(0); + boolean desc = false; + SqlKind nulls = null; + + while (node instanceof SqlCall) { + final SqlKind kind = node.getKind(); + if (kind == SqlKind.NULLS_FIRST || kind == SqlKind.NULLS_LAST) { + nulls = kind; + node = ((SqlCall) node).operand(0); + } else if (kind == SqlKind.DESCENDING) { + desc = true; + node = ((SqlCall) node).operand(0); + } else { + break; + } + } + + if (node.getKind() != SqlKind.ORDER_BY_ALL) { + return; + } + final SqlParserPos pos = orderList.getParserPosition(); + final List keys = new ArrayList<>(); + + for (SqlNode selectItem : select.getSelectList()) { + final SqlNode expr = SqlUtil.stripAs(selectItem); + if (expr instanceof SqlIdentifier && ((SqlIdentifier) expr).isStar()) { + throw newValidationError(expr, + RESOURCE.orderByAllRequiresExplicitSelectList()); + } + keys.add(applyOrderByAllDirection(expr, desc, nulls, pos)); + } + select.setOrderBy(new SqlNodeList(keys, pos)); + } + + /** Wraps a single ORDER BY ALL key with the optional descending direction + * and null-ordering that apply to every expanded key. */ + private static SqlNode applyOrderByAllDirection(SqlNode key, boolean desc, + @Nullable SqlKind nulls, SqlParserPos pos) { + SqlNode result = key; + if (desc) { + result = SqlStdOperatorTable.DESC.createCall(pos, result); + } + if (nulls == SqlKind.NULLS_FIRST) { + result = SqlStdOperatorTable.NULLS_FIRST.createCall(pos, result); + } else if (nulls == SqlKind.NULLS_LAST) { + result = SqlStdOperatorTable.NULLS_LAST.createCall(pos, result); + } + return result; + } + /** * Validates an item in the GROUP BY clause of a SELECT statement. * diff --git a/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties b/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties index 4096537b7fb9..0799d8d6ccf5 100644 --- a/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties +++ b/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties @@ -256,6 +256,7 @@ CannotConvertToStream=Cannot convert table ''{0}'' to stream CannotConvertToRelation=Cannot convert stream ''{0}'' to relation StreamMustGroupByMonotonic=Streaming aggregation requires at least one monotonic expression in GROUP BY clause StreamMustOrderByMonotonic=Streaming ORDER BY must start with monotonic expression +OrderByAllRequiresExplicitSelectList=ORDER BY ALL requires an explicit SELECT list; ''*'' is not supported StreamSetOpInconsistentInputs=Set operator cannot combine streaming and non-streaming inputs CannotStreamValues=Cannot stream VALUES CyclicDefinition=Cannot resolve ''{0}''; it references view ''{1}'', whose definition is cyclic diff --git a/core/src/test/java/org/apache/calcite/sql/test/SqlAdvisorTest.java b/core/src/test/java/org/apache/calcite/sql/test/SqlAdvisorTest.java index 94cbc6fb9ed4..fad2778a384e 100644 --- a/core/src/test/java/org/apache/calcite/sql/test/SqlAdvisorTest.java +++ b/core/src/test/java/org/apache/calcite/sql/test/SqlAdvisorTest.java @@ -71,6 +71,10 @@ class SqlAdvisorTest extends SqlValidatorTestCase { Collections.singletonList( "KEYWORD(*)"); + private static final List ORDER_BY_ALL_KEYWORD = + Collections.singletonList( + "KEYWORD(ALL)"); + protected static final List FROM_KEYWORDS = Arrays.asList( "KEYWORD(()", @@ -744,10 +748,12 @@ protected List getJoinKeywords() { String sql; sql = "select emp.empno from sales.emp where empno=1 order by ^dummy"; - f.withSql(sql).assertHint(EXPR_KEYWORDS, EMP_COLUMNS, EMP_TABLE); + f.withSql(sql).assertHint(EXPR_KEYWORDS, ORDER_BY_ALL_KEYWORD, EMP_COLUMNS, + EMP_TABLE); sql = "select emp.empno from sales.emp where empno=1 order by ^"; - f.withSql(sql).assertComplete(EXPR_KEYWORDS, EMP_COLUMNS, EMP_TABLE); + f.withSql(sql).assertComplete(EXPR_KEYWORDS, ORDER_BY_ALL_KEYWORD, + EMP_COLUMNS, EMP_TABLE); sql = "select emp.empno\n" @@ -755,7 +761,7 @@ protected List getJoinKeywords() { + " mpno,name,ob,gr,iredate,al,omm,eptno,lacker)\n" + "where e.mpno=1 order by ^"; f.withSql(sql) - .assertComplete(EXPR_KEYWORDS, + .assertComplete(EXPR_KEYWORDS, ORDER_BY_ALL_KEYWORD, Arrays.asList("COLUMN(MPNO)", "COLUMN(NAME)", "COLUMN(OB)", diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index c4e8993839d9..f85c6c75bf30 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -7324,6 +7324,30 @@ public boolean isBangEqualAllowed() { .ok(); } + /** Test case for + * [CALCITE-7597] + * Support ORDER BY ALL. */ + @Test void testOrderByAll() { + // expands to all SELECT items, validates fine + sql("select deptno, sal from emp order by all").ok(); + // direction applies to every expanded key + sql("select deptno, sal from emp order by all desc").ok(); + // SELECT * can't be expanded here + sql("select ^*^ from emp order by all") + .fails("(?s).*ORDER BY ALL requires an explicit SELECT list.*"); + // Aliases that shadow other column names must not confuse expansion + sql("select empno as deptno, deptno as empno from emp order by all").ok(); + // verify "x" still resolves and the two features coexist + sql("select sal as x, x + 1 as y from emp order by all") + .withValidatorIdentifierExpansion(true) + .withConformance(SqlConformanceEnum.BABEL) + .ok(); + sql("select sal as x, x + 1 as y from emp order by all desc") + .withValidatorIdentifierExpansion(true) + .withConformance(SqlConformanceEnum.BABEL) + .ok(); + } + @Test void testOrder() { final SqlConformance conformance = fixture().conformance(); sql("select empno as x from emp order by empno").ok(); diff --git a/core/src/test/resources/sql/sort.iq b/core/src/test/resources/sql/sort.iq index b7df52bff2c6..f9149a54822a 100644 --- a/core/src/test/resources/sql/sort.iq +++ b/core/src/test/resources/sql/sort.iq @@ -549,4 +549,18 @@ order by au."books"[1]."title"; !ok +# [CALCITE-7597] ORDER BY ALL sorts by every SELECT expression, in list order +select x, y from (values (2, 'b'), (1, 'a'), (1, 'c')) as t(x, y) +order by all; ++---+---+ +| X | Y | ++---+---+ +| 1 | a | +| 1 | c | +| 2 | b | ++---+---+ +(3 rows) + +!ok + # End sort.iq diff --git a/site/_docs/reference.md b/site/_docs/reference.md index b6ecb6b52bfa..6250a6d63943 100644 --- a/site/_docs/reference.md +++ b/site/_docs/reference.md @@ -191,7 +191,7 @@ query: | query MINUS [ ALL | DISTINCT ] query | query INTERSECT [ ALL | DISTINCT ] query } - [ ORDER BY orderItem [, orderItem ]* ] + [ ORDER BY { ALL [ ASC | DESC ] [ NULLS FIRST | NULLS LAST ] | orderItem [, orderItem]* } ] [ LIMIT [ start, ] { count | ALL } ] [ OFFSET start { ROW | ROWS } ] [ FETCH { FIRST | NEXT } [ count ] { ROW | ROWS } ONLY ] @@ -421,6 +421,12 @@ in those same conformance levels, any *column* in *insert* may be replaced by In *orderItem*, if *expression* is a positive integer *n*, it denotes the nth item in the SELECT clause. +`ORDER BY ALL` sorts by every expression in the SELECT clause, +in the order that they appear in the list; for example: +"SELECT x, y FROM t ORDER BY ALL" is equivalent to +"SELECT x, y FROM t ORDER BY x, y" +An optional trailing ASC / DESC and NULLS FIRST / NULLS LAST applies to all keys. + In *query*, *count* and *start* may each be either an unsigned integer literal or a dynamic parameter whose value is an integer. diff --git a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java index 0fa8c43fd9d3..591278ebfa5b 100644 --- a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java +++ b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java @@ -3947,6 +3947,36 @@ void checkPeriodPredicate(Checker checker) { + "ORDER BY `EMPNO`, `GENDER` DESC, `DEPTNO`, `EMPNO`, `NAME` DESC"); } + @Test void testOrderByAll() { + final String sql = "select x, y from t\n" + + "order by all"; + final String expected = "SELECT `X`, `Y`\n" + + "FROM `T`\n" + + "ORDER BY ALL"; + sql(sql).ok(expected); + + final String sql1 = "select x, y from t\n" + + "order by all desc"; + final String expected1 = "SELECT `X`, `Y`\n" + + "FROM `T`\n" + + "ORDER BY ALL DESC"; + sql(sql1).ok(expected1); + + final String sql2 = "select x, y from t\n" + + "order by all desc nulls last"; + final String expected2 = "SELECT `X`, `Y`\n" + + "FROM `T`\n" + + "ORDER BY ALL DESC NULLS LAST"; + sql(sql2).ok(expected2); + + final String sql3 = "select x, y from t\n" + + "order by all nulls first"; + final String expected3 = "SELECT `X`, `Y`\n" + + "FROM `T`\n" + + "ORDER BY ALL NULLS FIRST"; + sql(sql3).ok(expected3); + } + @Test void testOrderNullsFirst() { final String sql = "select * from emp\n" + "order by gender desc nulls last,\n" From d9157876411f1fd8b3cb3d166536ab4a7e8bb345 Mon Sep 17 00:00:00 2001 From: Takaaki Nakama Date: Wed, 24 Jun 2026 09:58:21 +0900 Subject: [PATCH 342/562] [CALCITE-7614] UNNEST of an unqualified struct-rooted array path fails validation: "Column 's.s' not found" When the operand of UNNEST is an unqualified identifier whose leading component is a PEEK_FIELDS struct column (e.g. UNNEST(s.arr) where s is a PEEK_FIELDS struct containing array field arr), validation failed with "Column 's.s' not found in table 't'", while the table-qualified form UNNEST(r.s.arr) worked. While DelegatingScope.fullyQualify resolves the qualifying table for the unqualified operand, that resolution re-enters validation of the UNNEST namespace, which rewrites the very same operand identifier's names in place to the fully-qualified form. fullyQualify then re-qualified the already-qualified identifier, duplicating the struct-column segment and producing the doubled "s.s". Fix fullyQualify to qualify a copy of the identifier taken before the re-entrant resolution, so the PEEK_FIELDS branch always works from the original names. Co-Authored-By: Claude Opus 4.8 Co-authored-by: Cursor --- .../calcite/sql/validate/DelegatingScope.java | 7 ++++++- .../apache/calcite/sql/test/SqlAdvisorTest.java | 1 + .../org/apache/calcite/test/SqlValidatorTest.java | 15 +++++++++++++++ .../org/apache/calcite/test/catalog/Fixture.java | 7 +++++++ .../test/catalog/MockCatalogReaderSimple.java | 12 ++++++++++++ 5 files changed, 41 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/calcite/sql/validate/DelegatingScope.java b/core/src/main/java/org/apache/calcite/sql/validate/DelegatingScope.java index 5153d3160e50..16a0e3a53b4d 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/DelegatingScope.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/DelegatingScope.java @@ -329,6 +329,9 @@ protected void addColumnNames( final ResolvedImpl resolved = new ResolvedImpl(); int size = identifier.names.size(); int i = size - 1; + // Snapshot: resolution below may rewrite identifier's names in place [CALCITE-7614] + final SqlIdentifier originalIdentifier = + (SqlIdentifier) identifier.clone(identifier.getParserPosition()); for (; i > 0; i--) { final SqlIdentifier prefix = identifier.getComponent(0, i); resolved.clear(); @@ -388,7 +391,9 @@ protected void addColumnNames( fromNs = resolve.namespace; fromPath = resolve.path; fromRowType = resolve.rowType(); - identifier = identifier + // Qualify the original (pre-resolution) names; see the comment + // on originalIdentifier above ([CALCITE-7614]). + identifier = originalIdentifier .setName(0, columnName) .add(0, tableName2, SqlParserPos.ZERO); ++i; diff --git a/core/src/test/java/org/apache/calcite/sql/test/SqlAdvisorTest.java b/core/src/test/java/org/apache/calcite/sql/test/SqlAdvisorTest.java index fad2778a384e..01488fcc4bad 100644 --- a/core/src/test/java/org/apache/calcite/sql/test/SqlAdvisorTest.java +++ b/core/src/test/java/org/apache/calcite/sql/test/SqlAdvisorTest.java @@ -100,6 +100,7 @@ class SqlAdvisorTest extends SqlValidatorTestCase { "TABLE(CATALOG.SALES.DOUBLE_PK)", "TABLE(CATALOG.SALES.DEPT_NESTED)", "TABLE(CATALOG.SALES.DEPT_NESTED_EXPANDED)", + "TABLE(CATALOG.SALES.DEPT_NESTED_PEEK)", "TABLE(CATALOG.SALES.BONUS)", "TABLE(CATALOG.SALES.ORDERS)", "TABLE(CATALOG.SALES.SALGRADE)", diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index f85c6c75bf30..0cf1966f99a8 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -9486,6 +9486,21 @@ void testGroupExpressionEquivalenceParams() { sql(sql3).fails("Table 'D' not found"); } + /** Test case for + * [CALCITE-7614] + * UNNEST of an array field in a PEEK_FIELDS struct column fails when the table + * is not qualified. */ + @Test void testUnnestPeekFieldsArrayColumn() { + // Table-qualified form already works. + sql("select * from dept_nested_peek as r CROSS JOIN UNNEST(r.s.arr) as x").ok(); + // Unqualified form must work too (used to fail with + // "Column 'S.S' not found in table 'DEPT_NESTED_PEEK'"). + sql("select * from dept_nested_peek CROSS JOIN UNNEST(s.arr) as x").ok(); + // Comma-join variants, for parity with testUnnestArrayColumn. + sql("select * from dept_nested_peek as r, UNNEST(r.s.arr) as x").ok(); + sql("select * from dept_nested_peek, UNNEST(s.arr) as x").ok(); + } + @Test void testUnnestWithOrdinality() { sql("select*from unnest(array[1, 2]) with ordinality") .type("RecordType(INTEGER NOT NULL EXPR$0, INTEGER NOT NULL ORDINALITY) NOT NULL"); diff --git a/testkit/src/main/java/org/apache/calcite/test/catalog/Fixture.java b/testkit/src/main/java/org/apache/calcite/test/catalog/Fixture.java index be6d20179fb8..d067e6bb4854 100644 --- a/testkit/src/main/java/org/apache/calcite/test/catalog/Fixture.java +++ b/testkit/src/main/java/org/apache/calcite/test/catalog/Fixture.java @@ -136,6 +136,13 @@ final class Fixture extends AbstractFixture { final RelDataType varchar5ArrayType = array(varchar5Type); final RelDataType intArrayArrayType = array(intArrayType); final RelDataType varchar5ArrayArrayType = array(varchar5ArrayType); + // A "peek" struct that contains an array field, e.g. Row(ARR varchar(5) array) + // with StructKind.PEEK_FIELDS so that "ARR" can be referenced without the + // struct-column prefix. + final RelDataType peekArrayType = typeFactory.builder() + .add("ARR", varchar5ArrayType) + .kind(StructKind.PEEK_FIELDS) + .build(); final RelDataType intMultisetType = typeFactory.createMultisetType(intType, -1); final RelDataType varchar5MultisetType = typeFactory.createMultisetType(varchar5Type, -1); final RelDataType intMultisetArrayType = array(intMultisetType); diff --git a/testkit/src/main/java/org/apache/calcite/test/catalog/MockCatalogReaderSimple.java b/testkit/src/main/java/org/apache/calcite/test/catalog/MockCatalogReaderSimple.java index 8723266ea0e1..1f7be1141091 100644 --- a/testkit/src/main/java/org/apache/calcite/test/catalog/MockCatalogReaderSimple.java +++ b/testkit/src/main/java/org/apache/calcite/test/catalog/MockCatalogReaderSimple.java @@ -177,6 +177,14 @@ private void registerTableDeptNestedExpanded(MockSchema salesSchema, Fixture fix registerTable(deptNestedExpandedTable); } + private void registerTableDeptNestedPeek(MockSchema salesSchema, Fixture fixture) { + MockTable deptNestedPeekTable = + MockTable.create(this, salesSchema, "DEPT_NESTED_PEEK", false, 4); + deptNestedPeekTable.addColumn("DEPTNO", fixture.intType, true); + deptNestedPeekTable.addColumn("S", fixture.peekArrayType); + registerTable(deptNestedPeekTable); + } + private void registerTableBonus(MockSchema salesSchema, Fixture fixture) { MockTable bonusTable = MockTable.create(this, salesSchema, "BONUS", false, 0); @@ -512,6 +520,10 @@ private void registerTableDoublePK(MockSchema salesSchema, Fixture fixture) { // Register "DEPT_NESTED_EXPANDED" table. registerTableDeptNestedExpanded(salesSchema, fixture); + // Register "DEPT_NESTED_PEEK" table, which has a PEEK_FIELDS struct column + // "S" containing an array field "ARR". + registerTableDeptNestedPeek(salesSchema, fixture); + // Register "BONUS" table. registerTableBonus(salesSchema, fixture); From c6d0a6b62afbb022d802e0ff5991e2abff860fd3 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Tue, 23 Jun 2026 11:34:41 -0700 Subject: [PATCH 343/562] [CALCITE-7619] RexSimplify incorrectly simplifies IS_FALSE(x) when x is nullable Signed-off-by: Mihai Budiu --- .../org/apache/calcite/rex/RexSimplify.java | 8 ++++++ .../org/apache/calcite/tools/RelBuilder.java | 6 ++++- .../apache/calcite/rex/RexProgramTest.java | 27 +++++++++++++++++++ .../apache/calcite/test/RelOptRulesTest.java | 24 +++++++++++++++++ .../apache/calcite/test/RelOptRulesTest.xml | 19 +++++++++++++ .../calcite/test/SqlToRelConverterTest.xml | 4 +-- core/src/test/resources/sql/sub-query.iq | 2 +- 7 files changed, 86 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rex/RexSimplify.java b/core/src/main/java/org/apache/calcite/rex/RexSimplify.java index 0ffb60454e33..ae4b3501c955 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexSimplify.java +++ b/core/src/main/java/org/apache/calcite/rex/RexSimplify.java @@ -200,6 +200,14 @@ public RexNode simplifyPreservingType(RexNode e, RexUnknownAs unknownAs, && SqlTypeUtil.equalSansNullability(rexBuilder.typeFactory, e2.getType(), e.getType())) { return e2; } + // If simplification widens nullability (NOT NULL → nullable) without changing + // the base type, using a CAST to NOT NULL is wrong: e.g. + // x IS FALSE is not the same as CAST(NOT(x) AS BOOLEAN NOT NULL). + // Return the original expression to preserve both semantics and type. + if (!e.getType().isNullable() && e2.getType().isNullable() + && SqlTypeUtil.equalSansNullability(rexBuilder.typeFactory, e2.getType(), e.getType())) { + return e; + } final RexNode e3 = rexBuilder.makeCast(e.getType(), e2, matchNullability, false); if (e3.equals(e)) { return e; diff --git a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java index cf9ccecdefef..54d662a3a13c 100644 --- a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java +++ b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java @@ -1938,9 +1938,13 @@ public RelBuilder filter(Iterable variablesSet, if (config.simplify()) { conjunctionPredicates = simplifier.simplifyFilterPredicates(predicates); } else { + // The config says "do not simplify", but without the following optimizations + // filter construction may fail because the predicates do not respect + // invariants checked by the filter constructor in Filter.isValid(). List simplified = new ArrayList<>(); for (RexNode predicate : predicates) { - RexNode simple = RexSimplify.simplifyComparisonWithNull(predicate, getRexBuilder()); + RexNode simple = simplifier.removeNullabilityCast(predicate); + simple = RexSimplify.simplifyComparisonWithNull(simple, getRexBuilder()); simplified.add(simple); } conjunctionPredicates = diff --git a/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java b/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java index ac7b5aebe6c3..91af635e5203 100644 --- a/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java +++ b/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java @@ -3841,6 +3841,33 @@ private static String getString(Map map) { assertThat(result2.getOperands().get(0), is(booleanInput)); } + /** Test cases for [CALCITE-7619] + * RexSimplify incorrectly simplifies IS_FALSE(x) when x is nullable. */ + @Test void testSimplifyPreservingTypeIsNotNullCast() { + // IS_FALSE(nullable_bool) has type BOOLEAN NOT NULL. + final RexNode isFalseExpr = isFalse(vBool()); + assertThat("IS_FALSE has NOT NULL type", isFalseExpr.getType().isNullable(), is(false)); + final RexNode s0 = simplify.simplifyPreservingType(isFalseExpr, RexUnknownAs.FALSE, true); + // nullable_bool IS FALSE != CAST(NOT(nullable_bool) AS BOOL NOT NULL) + assertThat(s0.isA(SqlKind.CAST), is(false)); + + // simplify(IS_FALSE(nullable_bool), FALSE) = NOT(nullable_bool), which is nullable. + final RexNode s1 = simplify.simplify(isFalseExpr, RexUnknownAs.FALSE); + assertThat(s1.isA(SqlKind.NOT), is(true)); + assertThat(s1.getType().isNullable(), is(true)); + + // IS_TRUE(nullable_bool) has type BOOLEAN NOT NULL. + final RexNode isTrueExpr = isTrue(vBool()); + assertThat(isTrueExpr.getType().isNullable(), is(false)); + final RexNode s2 = simplify.simplifyPreservingType(isTrueExpr, RexUnknownAs.FALSE, true); + // nullable_bool IS TRUE != CAST(nullable_bool AS BOOL NOT NULL) + assertThat(s2.isA(SqlKind.CAST), is(false)); + + // simplify(IS_TRUE(nullable_bool), FALSE) = nullable_bool, which is nullable. + final RexNode s3 = simplify.simplify(isTrueExpr, RexUnknownAs.FALSE); + assertThat(s3.getType().isNullable(), is(true)); + } + @Test void testSimplifyNot() { // "NOT(NOT(x))" => "x" checkSimplify(not(not(vBool())), "?0.bool0"); diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index 715d9f4f7b8e..b10a5c065c67 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -87,6 +87,7 @@ import org.apache.calcite.rel.rules.LoptOptimizeJoinRule; import org.apache.calcite.rel.rules.MeasureRules; import org.apache.calcite.rel.rules.MultiJoin; +import org.apache.calcite.rel.rules.MultiJoinOptimizeBushyRule; import org.apache.calcite.rel.rules.ProjectCorrelateTransposeRule; import org.apache.calcite.rel.rules.ProjectFilterTransposeRule; import org.apache.calcite.rel.rules.ProjectJoinTransposeRule; @@ -2328,6 +2329,29 @@ private void checkSemiOrAntiJoinProjectTranspose(JoinRelType type) { .check(); } + /** Test case for [CALCITE-7619] + * RexSimplify incorrectly simplifies IS_FALSE(x) when x is nullable. */ + @Test void testExpressionSimplification3() { + final String sql = "WITH tmp(bool_col) AS (\n" + + " VALUES (TRUE),\n" + + " (FALSE),\n" + + " (NULL)\n" + + ")\n" + + "SELECT *\n" + + "FROM tmp\n" + + "WHERE bool_col IS FALSE"; + // Simplify actually hides the bug we are trying to solve, so we disable it. + // Without this fix the reduce rule will fail with an assertion failure + // becase it tries to create a filter with a cast that strips nullability. + RelBuilder.Config config = RelBuilder.Config.DEFAULT.withSimplify(false); + RelOptRule reduce = + ReduceExpressionsRule.FilterReduceExpressionsRule.FilterReduceExpressionsRuleConfig.DEFAULT + .withRelBuilderFactory(RelBuilder.proto(config)).toRule(); + sql(sql) + .withRule(reduce) + .checkUnchanged(); + } + @Test void testReduceAverage() { final String sql = "select name, max(name), avg(deptno), min(name)\n" + "from sales.dept group by name"; diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index 29e539a0f7b5..2a47eefd0d7a 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -5565,6 +5565,25 @@ LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$ LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7], SLACKER=[$8]) LogicalFilter(condition=[SEARCH($1, Sarg[(-∞..'':VARCHAR(20)), ('':VARCHAR(20)..'3':VARCHAR(20)), ('3':VARCHAR(20)..+∞)]:VARCHAR(20))]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +]]> + + + + + + + + diff --git a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml index 1d05fd1b2f18..b5361cbbd35d 100644 --- a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml @@ -98,10 +98,10 @@ from empnullables]]> diff --git a/core/src/test/resources/sql/sub-query.iq b/core/src/test/resources/sql/sub-query.iq index 4b699d14b85f..2f190ed040dd 100644 --- a/core/src/test/resources/sql/sub-query.iq +++ b/core/src/test/resources/sql/sub-query.iq @@ -2425,7 +2425,7 @@ select sal from "scott".emp e !ok !if (use_old_decorr) { -EnumerableCalc(expr#0..5=[{inputs}], expr#6=[RAND()], expr#7=[CAST($t6):INTEGER NOT NULL], expr#8=[2], expr#9=[MOD($t7, $t8)], expr#10=[3], expr#11=[=($t9, $t10)], expr#12=[OR($t11, $t3)], SAL=[$t1], $condition=[$t12]) +EnumerableCalc(expr#0..5=[{inputs}], expr#6=[RAND()], expr#7=[CAST($t6):INTEGER NOT NULL], expr#8=[2], expr#9=[MOD($t7, $t8)], expr#10=[3], expr#11=[=($t9, $t10)], expr#12=[IS NOT FALSE($t3)], expr#13=[IS NOT NULL($t3)], expr#14=[AND($t12, $t13)], expr#15=[OR($t11, $t14)], SAL=[$t1], $condition=[$t15]) EnumerableMergeJoin(condition=[=($2, $4)], joinType=[left]) EnumerableSort(sort0=[$2], dir0=[ASC]) EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], SAL=[$t5], DEPTNO=[$t7]) From 5b4b18bb76e3634bd4dec6b8bfee8854f85e8fe4 Mon Sep 17 00:00:00 2001 From: Stamatis Zampetakis Date: Mon, 15 Jun 2026 11:15:27 +0200 Subject: [PATCH 344/562] [CALCITE-7604] Add rule to pull up GROUP BY above JOIN --- .../rel/rules/AggregateJoinTransposeRule.java | 2 +- .../apache/calcite/rel/rules/CoreRules.java | 5 + .../rel/rules/JoinAggregateTransposeRule.java | 210 ++++++++++++++++++ .../test/JoinAggregateTransposeRuleTest.java | 168 ++++++++++++++ .../test/JoinAggregateTransposeRuleTest.xml | 202 +++++++++++++++++ .../test/resources/sql/join-agg-transpose.iq | 78 +++++++ 6 files changed, 664 insertions(+), 1 deletion(-) create mode 100644 core/src/main/java/org/apache/calcite/rel/rules/JoinAggregateTransposeRule.java create mode 100644 core/src/test/java/org/apache/calcite/test/JoinAggregateTransposeRuleTest.java create mode 100644 core/src/test/resources/org/apache/calcite/test/JoinAggregateTransposeRuleTest.xml create mode 100644 core/src/test/resources/sql/join-agg-transpose.iq diff --git a/core/src/main/java/org/apache/calcite/rel/rules/AggregateJoinTransposeRule.java b/core/src/main/java/org/apache/calcite/rel/rules/AggregateJoinTransposeRule.java index 934b7633a9f7..c56f61ba1388 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/AggregateJoinTransposeRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/AggregateJoinTransposeRule.java @@ -129,7 +129,7 @@ public AggregateJoinTransposeRule(Class aggregateClass, allowFunctions); } - private static boolean isAggregateSupported(Aggregate aggregate, + static boolean isAggregateSupported(Aggregate aggregate, boolean allowFunctions) { if (!allowFunctions && !aggregate.getAggCallList().isEmpty()) { return false; diff --git a/core/src/main/java/org/apache/calcite/rel/rules/CoreRules.java b/core/src/main/java/org/apache/calcite/rel/rules/CoreRules.java index 873d04016f6f..bc21c41dd366 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/CoreRules.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/CoreRules.java @@ -654,6 +654,11 @@ private CoreRules() {} public static final JoinExtractFilterRule JOIN_EXTRACT_FILTER = JoinExtractFilterRule.Config.DEFAULT.toRule(); + /** Rule that pulls an {@link Aggregate} from the left input of a + * {@link Join} to above the join (group-by pull up). */ + public static final JoinAggregateTransposeRule JOIN_AGGREGATE_TRANSPOSE = + JoinAggregateTransposeRule.Config.DEFAULT.toRule(); + /** Rule that matches a {@link LogicalJoin} whose inputs are * {@link LogicalProject}s, and pulls the project expressions up. */ public static final JoinProjectTransposeRule JOIN_PROJECT_BOTH_TRANSPOSE = diff --git a/core/src/main/java/org/apache/calcite/rel/rules/JoinAggregateTransposeRule.java b/core/src/main/java/org/apache/calcite/rel/rules/JoinAggregateTransposeRule.java new file mode 100644 index 000000000000..99cf46141d7c --- /dev/null +++ b/core/src/main/java/org/apache/calcite/rel/rules/JoinAggregateTransposeRule.java @@ -0,0 +1,210 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.rel.rules; + +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelRule; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.Aggregate; +import org.apache.calcite.rel.core.Join; +import org.apache.calcite.rel.core.JoinInfo; +import org.apache.calcite.rel.core.JoinRelType; +import org.apache.calcite.rel.metadata.RelMetadataQuery; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexUtil; +import org.apache.calcite.tools.RelBuilder; +import org.apache.calcite.util.ImmutableBitSet; +import org.apache.calcite.util.mapping.MappingType; +import org.apache.calcite.util.mapping.Mappings; + +import org.immutables.value.Value; + +import java.util.ArrayList; +import java.util.List; + +import static org.apache.calcite.rel.rules.AggregateJoinTransposeRule.isAggregateSupported; + +/** + * Planner rule that pulls an + * {@link org.apache.calcite.rel.core.Aggregate} + * from below a {@link org.apache.calcite.rel.core.Join} to above it. + * + *

      Before + * 

      
+ * SELECT s.sales
+ * FROM (SELECT ss_sold_date_sk, SUM(ss_sales_price) AS sales
+ *       FROM store_sales
+ *       GROUP BY ss_sold_date_sk) s
+ * JOIN date_dim d
+ *   ON s.ss_sold_date_sk = d.d_date_sk
+ * WHERE d.d_year = 2000
+ *
      + * + *

      After + * 

      
+ * SELECT SUM(ss_sales_price) AS sales
+ * FROM store_sales s
+ * JOIN date_dim d
+ *   ON s.ss_sold_date_sk = d.d_date_sk
+ * WHERE d.d_year = 2000
+ * GROUP BY s.ss_sold_date_sk
+ *
      + * + *

      This rule implements the simplest form of group-by pull up transformation + * described in the following papers: + * + *

        + *
      • Weipeng P. Yan, and Per-Ake Larson. "Interchanging the order of grouping and join". Technical + * Report CS 95-09, Dept. of Computer Science, University of Waterloo, Canada, 1995.
        • + *
      • Weipeng P. Yan, and Per-Ake Larson. "Eager Aggregation and Lazy Aggregation." Proceedings + * of the 21th International Conference on Very Large Data Bases. 1995.
        • + *
      + * + *

      The papers contain additional variants ("lazy" aggregation) not currently + * implemented. + * + * @see CoreRules#JOIN_AGGREGATE_TRANSPOSE + */ +@Value.Enclosing +public class JoinAggregateTransposeRule + extends RelRule + implements TransformationRule { + + protected JoinAggregateTransposeRule(Config config) { + super(config); + } + + @Override public final boolean matches(RelOptRuleCall call) { + final Join join = call.rel(0); + final Aggregate left = call.rel(1); + final RelNode right = call.rel(2); + final JoinInfo info = join.analyzeCondition(); + final RelMetadataQuery mq = call.getMetadataQuery(); + + // Only handle INNER equijoins with simple aggregates for now. + // Join keys on the agg side must reference only group-by columns + // (ensures row elimination removes whole groups, not partial) + ImmutableBitSet groupOutput = ImmutableBitSet.range(left.getGroupCount()); + return join.getJoinType() == JoinRelType.INNER + && info.isEqui() + // We could potentially relax the check for the supported functions + // in this rule. I opted to keep things more constrained for now + // in case we decide to extend this rule for lazy aggregation. + && isAggregateSupported(left, true) + && groupOutput.contains(info.leftSet()) + // The right side must be unique on its join keys (no row duplication) + && Boolean.TRUE.equals(mq.areColumnsUnique(right, info.rightSet())); + } + + @Override public void onMatch(RelOptRuleCall call) { + final Join join = call.rel(0); + final Aggregate left = call.rel(1); + final RelNode aggInput = left.getInput(); + final RelNode right = join.getRight(); + + // Build the transformation + final int rawFieldCount = aggInput.getRowType().getFieldCount(); + final int leftFields = left.getRowType().getFieldCount(); + final int rightFields = right.getRowType().getFieldCount(); + final List groupList = left.getGroupSet().toList(); + + // Remap join condition: replace references to left output columns + // with references to raw aggInput columns in the new join layout. + // Old join: [agg output (leftFields) | other (rightFields)] + // New join: [aggInput (rawFieldCount) | other (rightFields)] + final int oldJoinWidth = join.getRowType().getFieldCount(); + final int newJoinWidth = rawFieldCount + rightFields; + + final Mappings.TargetMapping condMapping = + Mappings.create(MappingType.FUNCTION, oldJoinWidth, newJoinWidth); + // Agg output positions 0..groupCount-1 -> raw aggInput column positions + for (int i = 0; i < groupList.size(); i++) { + condMapping.set(i, groupList.get(i)); + } + // Other-side columns shift: from leftFields+j to rawFieldCount+j + for (int j = 0; j < rightFields; j++) { + condMapping.set(leftFields + j, rawFieldCount + j); + } + final RexNode newCondition = RexUtil.apply(condMapping, join.getCondition()); + + // Build new join + final RelBuilder relBuilder = call.builder(); + relBuilder.push(aggInput).push(right); + relBuilder.join(JoinRelType.INNER, newCondition); + + // Build new left above the join. + // New group-by set: original group columns (at their raw positions in + // aggInput) plus all other-side columns (to preserve them). + final ImmutableBitSet.Builder newGroupSetBuilder = ImmutableBitSet.builder(); + for (int col : groupList) { + newGroupSetBuilder.set(col); + } + for (int j = 0; j < rightFields; j++) { + newGroupSetBuilder.set(rawFieldCount + j); + } + final ImmutableBitSet newGroupSet = newGroupSetBuilder.build(); + + relBuilder.aggregate(relBuilder.groupKey(newGroupSet), left.getAggCallList()); + + // Add project to restore original join output column order. + // Original output: [group(left_cols), agg_calls, right_cols] + // New output: [group(left_cols, right_cols), agg_calls] + + // Create a mapping between the input (source) and the output (target) + // columns of the new aggregate. For example: + // + // Aggregate: Aggregate(group=[{7, 9, 10}]) + // Mapping: { 7 -> 0, 9 -> 1, 10 -> 2 } + final Mappings.TargetMapping newGroupMap = Mappings.target(newGroupSet.toList(), newJoinWidth); + + final List projects = new ArrayList<>(); + // Group-by columns of original left + for (int col : groupList) { + int pos = newGroupMap.getTarget(col); + projects.add(relBuilder.field(pos)); + } + // Aggregate call results + int aggCallBase = newGroupSet.cardinality(); + for (int k = 0; k < left.getAggCallList().size(); k++) { + projects.add(relBuilder.field(aggCallBase + k)); + } + // Right-side columns + for (int j = 0; j < rightFields; j++) { + int pos = newGroupMap.getTarget(rawFieldCount + j); + projects.add(relBuilder.field(pos)); + } + + relBuilder.project(projects, join.getRowType().getFieldNames()); + + call.transformTo(relBuilder.build()); + } + + /** Rule configuration. */ + @Value.Immutable + public interface Config extends RelRule.Config { + Config DEFAULT = ImmutableJoinAggregateTransposeRule.Config.of() + .withOperandSupplier(b0 -> + b0.operand(Join.class).inputs( + b1 -> b1.operand(Aggregate.class).anyInputs(), + b2 -> b2.operand(RelNode.class).anyInputs())) + .withDescription("JoinAggregateTransposeRule"); + + @Override default JoinAggregateTransposeRule toRule() { + return new JoinAggregateTransposeRule(this); + } + } +} diff --git a/core/src/test/java/org/apache/calcite/test/JoinAggregateTransposeRuleTest.java b/core/src/test/java/org/apache/calcite/test/JoinAggregateTransposeRuleTest.java new file mode 100644 index 000000000000..97f784522a61 --- /dev/null +++ b/core/src/test/java/org/apache/calcite/test/JoinAggregateTransposeRuleTest.java @@ -0,0 +1,168 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.test; + +import org.apache.calcite.config.CalciteConnectionConfig; +import org.apache.calcite.config.CalciteConnectionProperty; +import org.apache.calcite.jdbc.CalciteSchema; +import org.apache.calcite.plan.hep.HepProgram; +import org.apache.calcite.prepare.CalciteCatalogReader; +import org.apache.calcite.rel.rules.CoreRules; +import org.apache.calcite.schema.SchemaPlus; +import org.apache.calcite.tools.Frameworks; + +import com.google.common.collect.ImmutableList; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Test; + +/** + * Unit tests for {@link org.apache.calcite.rel.rules.JoinAggregateTransposeRule}. + * + *

      Relevant tickets: + *

      + */ +class JoinAggregateTransposeRuleTest { + + private static RelOptFixture fixture() { + // Use SCOTT schema to keep unit and end-to-end (Quidem) tests aligned. + SchemaPlus rootSchema = Frameworks.createRootSchema(true); + CalciteAssert.addSchema(rootSchema, CalciteAssert.SchemaSpec.SCOTT); + CalciteConnectionConfig config = + CalciteConnectionConfig.DEFAULT.set(CalciteConnectionProperty.CASE_SENSITIVE, "false"); + return RelOptFixture.DEFAULT + .withCatalogReaderFactory( + (typeFactory, caseSensitive) -> + new CalciteCatalogReader( + CalciteSchema.from(rootSchema), + ImmutableList.of("SCOTT"), + typeFactory, + config)) + .withDiffRepos(DiffRepository.lookup(JoinAggregateTransposeRuleTest.class)); + } + + private static RelOptFixture sql(String sql) { + return fixture().sql(sql); + } + + /** + * Tests that the rule can pull the group by from the left side of the join + * in the trivial case where there are no aggregate functions. + */ + @Test void testPullGroupByWithoutAggregateFunctions() { + final String sql = "select g.deptno\n" + + "from (select deptno from emp group by deptno) g\n" + + "join dept d on g.deptno = d.deptno"; + sql(sql).withRule(CoreRules.JOIN_AGGREGATE_TRANSPOSE).check(); + } + + /** + * Tests that the rule can pull the group by from the left side of the join + * when that is a simple aggregation. + */ + @Test void testPullGroupByFromLeftWithSimpleAggregation() { + final String sql = "select g.deptno, g.total_sal, d.dname\n" + + "from (select deptno, sum(sal) as total_sal\n" + + " from emp group by deptno) g\n" + + "join dept d on g.deptno = d.deptno"; + sql(sql).withRule(CoreRules.JOIN_AGGREGATE_TRANSPOSE).check(); + } + + /** + * Tests that the rule can pull the group by from the left side of the join + * when that is a simple aggregation with multiple aggregate functions. + */ + @Test void testPullGroupByFromLeftWithSimpleAggregationMultipleFunctions() { + final String sql = "select g.deptno, g.total_sal, g.low_sal, g.high_sal, d.dname\n" + + "from (select deptno, sum(sal) as total_sal, min(sal) as low_sal, max(sal) as high_sal\n" + + " from emp group by deptno) g\n" + + "join dept d on g.deptno = d.deptno"; + sql(sql).withRule(CoreRules.JOIN_AGGREGATE_TRANSPOSE).check(); + } + + /** + * Tests that the rule can pull the group by from the right side of the join + * by exploiting the join commutativity. Demonstrates that there is no + * need for implementing separate rule/logic for pulling the group by from the + * right side of the join. + */ + @Test void testPullGroupByFromRightWithSimpleAggregation() { + final String sql = "select g.deptno, g.total_sal, d.dname\n" + + "from dept d\n" + + "join (select deptno, sum(sal) as total_sal\n" + + " from emp group by deptno) g\n" + + " on g.deptno = d.deptno"; + HepProgram program = HepProgram.builder() + // Without a limit here the commute rule would keep flipping the + // join indefinitely causing stack overflow. + .addMatchLimit(1) + .addRuleInstance(CoreRules.JOIN_COMMUTE) + .addMatchLimit(HepProgram.MATCH_UNTIL_FIXPOINT) + .addRuleInstance(CoreRules.JOIN_AGGREGATE_TRANSPOSE) + .build(); + sql(sql) + .withProgram(program) + .check(); + } + + /** + * Tests that the rule can pull the group by from the left side of the join + * even when there is a filtering on the right side. + */ + @Test void testPullGroupByFromLeftWithFilterOnRight() { + final String sql = "select g.deptno, g.total_sal, d.dname\n" + + "from (select deptno, sum(sal) as total_sal\n" + + " from emp group by deptno) g\n" + + "join dept d on g.deptno = d.deptno\n" + + "where d.dname = 'RESEARCH'"; + sql(sql) + .withPreRule(CoreRules.FILTER_INTO_JOIN) + .withRule(CoreRules.JOIN_AGGREGATE_TRANSPOSE) + .check(); + } + + /** + * Tests that the rules not apply when the join is not an equijoin. + */ + @Test void testNoPullGroupByAboveNonEquiJoin() { + final String sql = "select g.deptno\n" + + "from (select deptno from emp group by deptno) g\n" + + "join dept d on g.deptno > d.deptno"; + sql(sql).withRule(CoreRules.JOIN_AGGREGATE_TRANSPOSE).checkUnchanged(); + } + + /** + * Tests that the rule does not apply when the right join keys are not unique. + * The job column is not unique thus the rule bails out. + */ + @Test void testNoPullGroupByWhenRightJoinKeysNotUnique() { + final String sql = "select g.job, g.cnt\n" + + "from (select job, count(*) as cnt\n" + + " from emp group by job) g\n" + + "join emp e on g.job = e.job"; + sql(sql).withRule(CoreRules.JOIN_AGGREGATE_TRANSPOSE).checkUnchanged(); + } + + + @AfterAll static void checkActualAndReferenceFiles() { + fixture().diffRepos.checkActualAndReferenceFiles(); + } +} diff --git a/core/src/test/resources/org/apache/calcite/test/JoinAggregateTransposeRuleTest.xml b/core/src/test/resources/org/apache/calcite/test/JoinAggregateTransposeRuleTest.xml new file mode 100644 index 000000000000..8c7c7c59640e --- /dev/null +++ b/core/src/test/resources/org/apache/calcite/test/JoinAggregateTransposeRuleTest.xml @@ -0,0 +1,202 @@ + + + + + + d.deptno]]> + + + ($0, $1)], joinType=[inner]) + LogicalAggregate(group=[{0}]) + LogicalProject(DEPTNO=[$7]) + LogicalTableScan(table=[[scott, EMP]]) + LogicalTableScan(table=[[scott, DEPT]]) +]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/core/src/test/resources/sql/join-agg-transpose.iq b/core/src/test/resources/sql/join-agg-transpose.iq new file mode 100644 index 000000000000..3cccb32da2a9 --- /dev/null +++ b/core/src/test/resources/sql/join-agg-transpose.iq @@ -0,0 +1,78 @@ +# join-agg-transpose.iq - [CALCITE-7604] Add rule to pull up GROUP BY above join +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +!use post +!set outputformat mysql +!use scott + +!set hep-rules " ++FILTER_INTO_JOIN, ++JOIN_AGGREGATE_TRANSPOSE" + +# Tests the rule can pull the group by from the left side of the join +# in the trivial case where there are no aggregate functions + +select g.deptno +from (select deptno from emp group by deptno) g +join dept d on g.deptno = d.deptno; ++--------+ +| DEPTNO | ++--------+ +| 10 | +| 20 | +| 30 | ++--------+ +(3 rows) + +!ok + +EnumerableCalc(expr#0..1=[{inputs}], DEPTNO=[$t0]) + EnumerableAggregate(group=[{0, 1}]) + EnumerableHashJoin(condition=[=($0, $1)], joinType=[inner]) + EnumerableCalc(expr#0..7=[{inputs}], DEPTNO=[$t7]) + EnumerableTableScan(table=[[scott, EMP]]) + EnumerableCalc(expr#0..2=[{inputs}], DEPTNO=[$t0]) + EnumerableTableScan(table=[[scott, DEPT]]) +!plan + +# Tests the rule can pull the group by from the left side of the join +# even when there is a filtering on the right side. + +select g.deptno, g.total_sal, d.dname +from (select deptno, sum(sal) as total_sal + from emp group by deptno) g +join dept d on g.deptno = d.deptno +where d.dname = 'RESEARCH'; ++--------+-----------+----------+ +| DEPTNO | TOTAL_SAL | DNAME | ++--------+-----------+----------+ +| 20 | 10875.00 | RESEARCH | ++--------+-----------+----------+ +(1 row) + +!ok + +EnumerableCalc(expr#0..3=[{inputs}], DEPTNO=[$t0], TOTAL_SAL=[$t3], DNAME=[$t2]) + EnumerableAggregate(group=[{0, 2, 3}], TOTAL_SAL=[SUM($1)]) + EnumerableHashJoin(condition=[=($0, $2)], joinType=[inner]) + EnumerableCalc(expr#0..7=[{inputs}], DEPTNO=[$t7], SAL=[$t5]) + EnumerableTableScan(table=[[scott, EMP]]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=['RESEARCH':VARCHAR(14)], expr#4=[=($t1, $t3)], proj#0..1=[{exprs}], $condition=[$t4]) + EnumerableTableScan(table=[[scott, DEPT]]) +!plan + +# End join-agg-transpose.iq From c15f1cab205972662f7b7f984fe6e9c85d12244a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 23 Jun 2026 14:39:55 +0000 Subject: [PATCH 345/562] Bump nokogiri from 1.19.3 to 1.19.4 in /site Bumps [nokogiri](https://github.com/sparklemotion/nokogiri) from 1.19.3 to 1.19.4. - [Release notes](https://github.com/sparklemotion/nokogiri/releases) - [Changelog](https://github.com/sparklemotion/nokogiri/blob/main/CHANGELOG.md) - [Commits](https://github.com/sparklemotion/nokogiri/compare/v1.19.3...v1.19.4) --- updated-dependencies: - dependency-name: nokogiri dependency-version: 1.19.4 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- site/Gemfile | 2 +- site/Gemfile.lock | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/site/Gemfile b/site/Gemfile index 815bd370ea29..6d8fbf6a516e 100644 --- a/site/Gemfile +++ b/site/Gemfile @@ -16,7 +16,7 @@ source 'https://rubygems.org' gem 'jekyll', '~>4' gem "webrick", "~> 1.9.1" -gem "nokogiri", "~> 1.19.3" +gem "nokogiri", "~> 1.19.4" gem "csv", "~> 3.3.2" gem "base64", "~> 0.2.0" diff --git a/site/Gemfile.lock b/site/Gemfile.lock index 5d750044420e..794c90e93970 100644 --- a/site/Gemfile.lock +++ b/site/Gemfile.lock @@ -74,21 +74,21 @@ GEM rb-fsevent (~> 0.10, >= 0.10.3) rb-inotify (~> 0.9, >= 0.9.10) mercenary (0.4.0) - nokogiri (1.19.3-aarch64-linux-gnu) + nokogiri (1.19.4-aarch64-linux-gnu) racc (~> 1.4) - nokogiri (1.19.3-aarch64-linux-musl) + nokogiri (1.19.4-aarch64-linux-musl) racc (~> 1.4) - nokogiri (1.19.3-arm-linux-gnu) + nokogiri (1.19.4-arm-linux-gnu) racc (~> 1.4) - nokogiri (1.19.3-arm-linux-musl) + nokogiri (1.19.4-arm-linux-musl) racc (~> 1.4) - nokogiri (1.19.3-arm64-darwin) + nokogiri (1.19.4-arm64-darwin) racc (~> 1.4) - nokogiri (1.19.3-x86_64-darwin) + nokogiri (1.19.4-x86_64-darwin) racc (~> 1.4) - nokogiri (1.19.3-x86_64-linux-gnu) + nokogiri (1.19.4-x86_64-linux-gnu) racc (~> 1.4) - nokogiri (1.19.3-x86_64-linux-musl) + nokogiri (1.19.4-x86_64-linux-musl) racc (~> 1.4) pathutil (0.16.2) forwardable-extended (~> 2.6) @@ -140,7 +140,7 @@ DEPENDENCIES csv (~> 3.3.2) jekyll (~> 4) jekyll-redirect-from - nokogiri (~> 1.19.3) + nokogiri (~> 1.19.4) webrick (~> 1.9.1) BUNDLED WITH From 069dde7d5ac13fd8529e765516499fe966ccd830 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 24 Jun 2026 00:08:38 +0000 Subject: [PATCH 346/562] Bump concurrent-ruby from 1.3.5 to 1.3.7 in /site Bumps [concurrent-ruby](https://github.com/ruby-concurrency/concurrent-ruby) from 1.3.5 to 1.3.7. - [Release notes](https://github.com/ruby-concurrency/concurrent-ruby/releases) - [Changelog](https://github.com/ruby-concurrency/concurrent-ruby/blob/master/CHANGELOG.md) - [Commits](https://github.com/ruby-concurrency/concurrent-ruby/compare/v1.3.5...v1.3.7) --- updated-dependencies: - dependency-name: concurrent-ruby dependency-version: 1.3.7 dependency-type: indirect ... Signed-off-by: dependabot[bot] --- site/Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site/Gemfile.lock b/site/Gemfile.lock index 794c90e93970..7fc66d79a4b4 100644 --- a/site/Gemfile.lock +++ b/site/Gemfile.lock @@ -6,7 +6,7 @@ GEM base64 (0.2.0) bigdecimal (3.1.9) colorator (1.1.0) - concurrent-ruby (1.3.5) + concurrent-ruby (1.3.7) csv (3.3.2) em-websocket (0.5.3) eventmachine (>= 0.12.9) From 4a76f418d9a13ac1888e24ae486c6c4fa55b5646 Mon Sep 17 00:00:00 2001 From: Chris Dennis Date: Wed, 10 Jun 2026 15:00:41 -0400 Subject: [PATCH 347/562] [CALCITE-7599] Enhance parser to allow numeric literals as values in k/v hints as suggested by documentation Signed-off-by: Chris Dennis --- core/src/main/codegen/templates/Parser.jj | 2 ++ .../main/java/org/apache/calcite/sql/SqlHint.java | 7 ++++--- .../apache/calcite/test/SqlHintsConverterTest.java | 8 +++++++- .../apache/calcite/test/SqlHintsConverterTest.xml | 13 ++++++++++++- site/_docs/reference.md | 4 +++- .../apache/calcite/sql/parser/SqlParserTest.java | 7 ++++++- 6 files changed, 34 insertions(+), 7 deletions(-) diff --git a/core/src/main/codegen/templates/Parser.jj b/core/src/main/codegen/templates/Parser.jj index 5519dd566d2f..c5d392e2fb0c 100644 --- a/core/src/main/codegen/templates/Parser.jj +++ b/core/src/main/codegen/templates/Parser.jj @@ -1296,6 +1296,8 @@ void AddKeyValueOption(List list) : ) ( + value = NumericLiteral() + | value = StringLiteral() | value = SimpleIdentifier() diff --git a/core/src/main/java/org/apache/calcite/sql/SqlHint.java b/core/src/main/java/org/apache/calcite/sql/SqlHint.java index 38f54636ebe8..8c61828daec9 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlHint.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlHint.java @@ -197,18 +197,19 @@ public enum HintOptionFormat implements Symbolizable { * The hint options are list of key-value pairs. * For each pair, * the key is a simple identifier or string literal, - * the value is a string literal. + * the value is a string or numeric literal. */ KV_LIST } //~ Tools ------------------------------------------------------------------ - private static String getOptionAsString(SqlNode node) { + private String getOptionAsString(SqlNode node) { assert node instanceof SqlIdentifier || SqlUtil.isLiteral(node); if (node instanceof SqlIdentifier) { return ((SqlIdentifier) node).getSimple(); } - return ((SqlLiteral) node).getValueAs(String.class); + return requireNonNull(((SqlLiteral) node).toValue(), + () -> "null hint literal in " + options); } } diff --git a/core/src/test/java/org/apache/calcite/test/SqlHintsConverterTest.java b/core/src/test/java/org/apache/calcite/test/SqlHintsConverterTest.java index 9183134cd4e4..346e0c57e6b9 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlHintsConverterTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlHintsConverterTest.java @@ -172,7 +172,7 @@ public final Fixture sql(String sql) { /** Test case for [CALCITE-7498] * The parser rejects the example hints from the documentation. */ @Test void testDocumentationExample() { - final String sql = "SELECT /*+ hint1, hint2(a='1', b='2') */ *\n" + final String sql = "SELECT /*+ hint1, hint2(a=1, b=2) */ *\n" + "FROM emp /*+ hint3(5, 'x') */\n" + "JOIN dept /*+ hint4(c=id), hint5 */\n" + "ON emp.deptno = dept.deptno"; @@ -201,6 +201,12 @@ public final Fixture sql(String sql) { sql(sql).ok(); } + @Test void testQueryHintWithKeyValueNumericLiteralOptions() { + final String sql = "select /*+ hint2(a=1, b=-1, c=1.1, d=-1.1, e=1e3, f=-1e-4) */ *\n" + + "from emp"; + sql(sql).ok(); + } + @Test void testNestedQueryHint() { final String sql = "select /*+ resource(parallelism='3'), repartition(10) */ empno\n" + "from (select /*+ resource(mem='20Mb')*/ empno, ename from emp)"; diff --git a/core/src/test/resources/org/apache/calcite/test/SqlHintsConverterTest.xml b/core/src/test/resources/org/apache/calcite/test/SqlHintsConverterTest.xml index e2cd95622d57..d06db165d0e2 100644 --- a/core/src/test/resources/org/apache/calcite/test/SqlHintsConverterTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/SqlHintsConverterTest.xml @@ -60,7 +60,7 @@ Correlate:[[USE_HASH_JOIN inheritPath:[0] options:[ORDERS, PRODUCTS_TEMPORAL]]] - @@ -350,6 +350,17 @@ LogicalJoin:[[NO_HASH_JOIN inheritPath:[0, 0]]] TableScan:[[PROPERTIES inheritPath:[0, 0, 0] options:{K1=v1, K2=v2}], [INDEX inheritPath:[0, 0, 0] options:[ENAME]]] TableScan:[[PROPERTIES inheritPath:[0, 0, 1] options:{K1=v1, K2=v2}], [INDEX inheritPath:[0, 0, 1] options:[ENAME]]] TableScan:[[PROPERTIES inheritPath:[0, 1, 0] options:{K1=v1, K2=v2}], [INDEX inheritPath:[0, 1, 0] options:[ENAME]]] +]]> + + + + + + + + diff --git a/site/_docs/reference.md b/site/_docs/reference.md index 6250a6d63943..064d6fb95e40 100644 --- a/site/_docs/reference.md +++ b/site/_docs/reference.md @@ -3647,11 +3647,13 @@ optionKey: optionVal: simpleIdentifier + | numericLiteral | stringLiteral hintOption: simpleIdentifier - | stringLiteral + | numericLiteral + | stringLiteral {% endhighlight %} It is experimental in Calcite, and yet not fully implemented, what we have implemented are: diff --git a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java index 591278ebfa5b..bb7566c9904c 100644 --- a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java +++ b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java @@ -9845,7 +9845,12 @@ private static Consumer> checkWarnings( final String sql1 = "select " + "/*+ properties(^k1^=123, k2='v2'), no_hash_join() */ " + "empno, ename, deptno from emps"; - sql(sql1).fails("(?s).*Encountered \"k1 = 123\" at .*"); + // Allow numeric literal k/v values. + final String expected1 = "SELECT\n" + + "/*+ `PROPERTIES`(`K1` = 123, `K2` = 'v2'), `NO_HASH_JOIN` */\n" + + "`EMPNO`, `ENAME`, `DEPTNO`\n" + + "FROM `EMPS`"; + sql(sql1).ok(expected1); final String sql2 = "select " + "/*+ properties(k1, k2^=^'v2'), no_hash_join */ " + "empno, ename, deptno from emps"; From 65b32186fd58dac2b155bd087c776bc6ea235478 Mon Sep 17 00:00:00 2001 From: xuzifu666 <1206332514@qq.com> Date: Tue, 23 Jun 2026 18:32:49 +0800 Subject: [PATCH 348/562] [CALCITE-7615] MAP_CONCAT does not accept NULL as an argument --- .../calcite/runtime/CalciteResource.java | 2 +- .../calcite/sql/fun/SqlLibraryOperators.java | 50 ++++++++++++++----- .../runtime/CalciteResource.properties | 2 +- .../apache/calcite/test/SqlOperatorTest.java | 31 +++++++----- 4 files changed, 58 insertions(+), 27 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java b/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java index ac2ed8501980..ea8e1772c678 100644 --- a/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java +++ b/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java @@ -741,7 +741,7 @@ ExInst illegalArgumentForTableFunctionCall(String a0, @BaseMessage("Map requires an even number of arguments") ExInst mapRequiresEvenArgCount(); - @BaseMessage("Function ''{0}'' should all be of type map, but it is ''{1}''") + @BaseMessage("Arguments of function ''{0}'' should all be of type MAP, but ''{1}'' was found") ExInst typesShouldAllBeMap(String funcName, String type); @BaseMessage("Incompatible types") diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java index 2479a92ba247..539d6b327fd0 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java @@ -1763,23 +1763,47 @@ private static RelDataType deriveTypeArraysZip(SqlOperatorBinding opBinding) { OperandTypes.ARRAY.or(OperandTypes.ARRAY_BOOLEAN_LITERAL)); private static RelDataType deriveTypeMapConcat(SqlOperatorBinding opBinding) { + final RelDataTypeFactory typeFactory = opBinding.getTypeFactory(); if (opBinding.getOperandCount() == 0) { - final RelDataTypeFactory typeFactory = opBinding.getTypeFactory(); - final RelDataType type = typeFactory.createSqlType(SqlTypeName.VARCHAR); - requireNonNull(type, "type"); + final RelDataType type = typeFactory.createSqlType(SqlTypeName.ANY); return SqlTypeUtil.createMapType(typeFactory, type, type, true); - } else { - final List operandTypes = opBinding.collectOperandTypes(); - for (RelDataType operandType : operandTypes) { - if (!SqlTypeUtil.isMap(operandType)) { - throw opBinding.newError( - RESOURCE.typesShouldAllBeMap( - opBinding.getOperator().getName(), - operandType.getFullTypeString())); - } + } + final List operandTypes = opBinding.collectOperandTypes(); + final List mapTypes = new ArrayList<>(); + boolean hasNull = false; + for (RelDataType operandType : operandTypes) { + if (operandType.getSqlTypeName() == SqlTypeName.NULL) { + hasNull = true; + } else if (SqlTypeUtil.isMap(operandType)) { + mapTypes.add(operandType); + } else { + throw opBinding.newError( + RESOURCE.typesShouldAllBeMap( + opBinding.getOperator().getName(), + operandType.getFullTypeString())); + } + } + if (mapTypes.isEmpty() || hasNull) { + // All arguments are NULL literals, or at least one null argument; + // the result is NULL. + return typeFactory.createSqlType(SqlTypeName.NULL); + } + // If there are MAP placeholders (e.g. from map_concat() with no + // arguments) alongside more specific MAP types, ignore the placeholders and + // infer the type from the specific maps. + final List concreteMapTypes = new ArrayList<>(); + for (RelDataType mapType : mapTypes) { + final RelDataType keyType = requireNonNull(mapType.getKeyType()); + final RelDataType valueType = requireNonNull(mapType.getValueType()); + if (keyType.getSqlTypeName() == SqlTypeName.ANY + && valueType.getSqlTypeName() == SqlTypeName.ANY) { + continue; } - return requireNonNull(opBinding.getTypeFactory().leastRestrictive(operandTypes)); + concreteMapTypes.add(mapType); } + final List typesToUse = + concreteMapTypes.isEmpty() ? mapTypes : concreteMapTypes; + return requireNonNull(typeFactory.leastRestrictive(typesToUse)); } /** The "MAP_CONCAT(map [, map]*)" function. */ diff --git a/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties b/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties index 0799d8d6ccf5..49552b41985d 100644 --- a/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties +++ b/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties @@ -244,7 +244,7 @@ DuplicateNameInColumnList=Duplicate name ''{0}'' in column list RequireAtLeastOneArg=Require at least 1 argument MapRequiresTwoOrMoreArgs=Map requires at least 2 arguments MapRequiresEvenArgCount=Map requires an even number of arguments -TypesShouldAllBeMap=Function ''{0}'' should all be of type map, but it is ''{1}'' +TypesShouldAllBeMap=Arguments of function ''{0}'' should all be of type MAP, but ''{1}'' was found IncompatibleTypes=Incompatible types ColumnCountMismatch=Number of columns must match number of query columns DuplicateColumnAndNoColumnList=Column has duplicate column name ''{0}'' and no column list specified diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java index 080a704a8cc8..2119f79df20e 100644 --- a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java +++ b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java @@ -9230,7 +9230,9 @@ void checkArrayReverseFunc(SqlOperatorFixture f0, SqlFunction function, "(INTEGER NOT NULL, INTEGER) MAP NOT NULL"); // test zero arg, but it should return empty map. f.checkScalar("map_concat()", "{}", - "(VARCHAR NOT NULL, VARCHAR NOT NULL) MAP"); + "(ANY NOT NULL, ANY NOT NULL) MAP"); + f.checkScalar("map_concat(map_concat(), map[1, 2])", "{1=2}", + "(INTEGER NOT NULL, INTEGER NOT NULL) MAP NOT NULL"); // after calcite supports cast(null as map), it should add these tests. if (TODO) { @@ -9244,16 +9246,18 @@ void checkArrayReverseFunc(SqlOperatorFixture f0, SqlFunction function, // test only has one operand, but it is not map type. f.checkFails("^map_concat(1)^", - "Function 'MAP_CONCAT' should all be of type map, but it is 'INTEGER NOT NULL'", false); - f.checkFails("^map_concat(null)^", - "Function 'MAP_CONCAT' should all be of type map, but it is 'NULL'", false); + "Arguments of function 'MAP_CONCAT' should all be of type MAP, " + + "but 'INTEGER NOT NULL' was found", false); + // test operand is the NULL literal. + f.checkNull("map_concat(null)"); // test operands in same type family, but it is not map type. f.checkFails("^map_concat(array[1], array[1])^", - "Function 'MAP_CONCAT' should all be of type map, " - + "but it is 'INTEGER NOT NULL ARRAY NOT NULL'", false); - f.checkFails("^map_concat(map['foo', 1], null)^", - "Function 'MAP_CONCAT' should all be of type map, " - + "but it is 'NULL'", false); + "Arguments of function 'MAP_CONCAT' should all be of type MAP, " + + "but 'INTEGER NOT NULL ARRAY NOT NULL' was found", false); + // test map operand with NULL literal. + f.checkNull("map_concat(map['foo', 1], null)"); + f.checkType("map_concat(map['foo', 1], null)", "NULL"); + f.checkNull("map_concat(null, map['foo', 1])"); // test operands not in same type family. f.checkFails("^map_concat(map[1, null], array[1])^", "Parameters must be of the same type", false); @@ -9272,6 +9276,9 @@ void checkArrayReverseFunc(SqlOperatorFixture f0, SqlFunction function, "(INTEGER NOT NULL, INTEGER) MAP NOT NULL"); f1.checkScalar("map_concat(map('foo', 1), map())", "{foo=1}", "(CHAR(3) NOT NULL, INTEGER NOT NULL) MAP NOT NULL"); + // test zero arg, but it should return empty map. + f1.checkScalar("map_concat()", "{}", + "(ANY NOT NULL, ANY NOT NULL) MAP"); // test operand is null map f1.checkNull("map_concat(map('foo', 1), cast(null as map))"); @@ -9281,9 +9288,9 @@ void checkArrayReverseFunc(SqlOperatorFixture f0, SqlFunction function, f1.checkType("map_concat(cast(null as map), map['foo', 1])", "(VARCHAR NOT NULL, INTEGER) MAP"); - f1.checkFails("^map_concat(map('foo', 1), null)^", - "Function 'MAP_CONCAT' should all be of type map, " - + "but it is 'NULL'", false); + f1.checkNull("map_concat(map('foo', 1), null)"); + f1.checkType("map_concat(map('foo', 1), null)", "NULL"); + f1.checkNull("map_concat(null, map('foo', 1))"); // test operands not in same type family. f1.checkFails("^map_concat(map(1, null), array[1])^", "Parameters must be of the same type", false); From 382ef42be36c28eb55911b6cd9d34189f0d65f47 Mon Sep 17 00:00:00 2001 From: leishp <2233047175@qq.com> Date: Tue, 23 Jun 2026 20:12:21 +0800 Subject: [PATCH 349/562] [CALCITE-7616] ProjectToLogicalProjectAndWindowRule should only match logical Project nodes --- .../rel/rules/ProjectToWindowRule.java | 3 +- .../apache/calcite/test/JdbcAdapterTest.java | 102 ++++++++++++++++++ 2 files changed, 104 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/calcite/rel/rules/ProjectToWindowRule.java b/core/src/main/java/org/apache/calcite/rel/rules/ProjectToWindowRule.java index 931169135fd4..536f5834e468 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/ProjectToWindowRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/ProjectToWindowRule.java @@ -25,6 +25,7 @@ import org.apache.calcite.rel.core.Project; import org.apache.calcite.rel.hint.RelHint; import org.apache.calcite.rel.logical.LogicalCalc; +import org.apache.calcite.rel.logical.LogicalProject; import org.apache.calcite.rel.logical.LogicalWindow; import org.apache.calcite.rex.RexBiVisitorImpl; import org.apache.calcite.rex.RexCall; @@ -193,7 +194,7 @@ public interface ProjectToLogicalProjectAndWindowRuleConfig ProjectToLogicalProjectAndWindowRuleConfig DEFAULT = ImmutableProjectToLogicalProjectAndWindowRuleConfig.of() .withOperandSupplier(b -> - b.operand(Project.class) + b.operand(LogicalProject.class) .predicate(Project::containsOver) .anyInputs()) .withDescription("ProjectToWindowRule:project"); diff --git a/core/src/test/java/org/apache/calcite/test/JdbcAdapterTest.java b/core/src/test/java/org/apache/calcite/test/JdbcAdapterTest.java index df12fbe8a8d1..7bed49bb1037 100644 --- a/core/src/test/java/org/apache/calcite/test/JdbcAdapterTest.java +++ b/core/src/test/java/org/apache/calcite/test/JdbcAdapterTest.java @@ -18,11 +18,17 @@ import org.apache.calcite.adapter.enumerable.EnumerableRules; import org.apache.calcite.adapter.java.ReflectiveSchema; +import org.apache.calcite.adapter.jdbc.JdbcSchema; import org.apache.calcite.config.CalciteConnectionProperty; import org.apache.calcite.config.Lex; import org.apache.calcite.plan.RelOptPlanner; import org.apache.calcite.prepare.Prepare; import org.apache.calcite.runtime.Hook; +import org.apache.calcite.schema.Schema; +import org.apache.calcite.schema.SchemaFactory; +import org.apache.calcite.schema.SchemaPlus; +import org.apache.calcite.sql.SqlDialect; +import org.apache.calcite.sql.dialect.MysqlSqlDialect; import org.apache.calcite.test.CalciteAssert.AssertThat; import org.apache.calcite.test.CalciteAssert.DatabaseInstance; import org.apache.calcite.test.schemata.foodmart.FoodmartSchema; @@ -40,6 +46,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; +import java.util.Map; import java.util.Properties; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; @@ -47,7 +54,9 @@ import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.not; import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.stringContainsInOrder; import static org.junit.jupiter.api.Assertions.assertFalse; /** @@ -1696,6 +1705,99 @@ private LockWrapper exclusiveCleanDb(Connection c) throws SQLException { calciteConnection.close(); } + /** Test case for + * [CALCITE-7616] + * ProjectToLogicalProjectAndWindowRule should not match non-logical Project + * nodes with JDBC convention. + * + *

      When a JDBC schema's dialect supports window functions (e.g. MySQL), + * a query with window functions (RANK, ROW_NUMBER, etc.) should not throw + * AssertionError because {@code ProjectToLogicalProjectAndWindowRule} + * fires on {@code JdbcProject} and creates {@code LogicalWindow} with + * JDBC convention. + * + *

      Uses an in-memory HSQLDB database with a custom schema factory + * ({@link WindowSupportingJdbcSchemaFactory}) that wraps the HSQLDB + * connection with MySQL dialect. HSQLDB's own dialect reports + * {@code supportsWindowFunctions() = false}, so MySQL dialect (which + * reports {@code true}) is needed to trigger + * {@code JdbcProjectRule} to convert projects containing OVER expressions + * into {@code JdbcProject} nodes. */ + @Test void testWindowFunctionJdbcConvention() throws Exception { + final String jdbcUrl = "jdbc:hsqldb:mem:jdbcwindowconventiontest"; + try (Connection conn = DriverManager.getConnection(jdbcUrl, "SA", "")) { + try (Statement stmt = conn.createStatement()) { + stmt.execute("CREATE TABLE emp (" + + "empno INT, " + + "sal DECIMAL(10,2)" + + ")"); + stmt.execute("INSERT INTO emp VALUES " + + "(1, 100.00), " + + "(2, 200.00), " + + "(3, 300.00)"); + } + } + + final String model = "{\n" + + " version: '1.0',\n" + + " defaultSchema: 'TEST',\n" + + " schemas: [{\n" + + " type: 'custom',\n" + + " name: 'TEST',\n" + + " factory: '" + WindowSupportingJdbcSchemaFactory.class.getName() + "',\n" + + " operand: {\n" + + " jdbcUrl: '" + jdbcUrl + "',\n" + + " jdbcDriver: 'org.hsqldb.jdbcDriver',\n" + + " jdbcUser: 'SA',\n" + + " jdbcPassword: ''\n" + + " }\n" + + " }]\n" + + "}"; + + final String sql = "SELECT empno, RANK() OVER (ORDER BY sal DESC) AS rnk\n" + + "FROM emp"; + + try { + CalciteAssert.model(model) + .query(sql) + .runs(); + } catch (Exception e) { + // After fix, the convention AssertionError should NOT occur. + // The query may fail because HSQLDB does not support RANK(), but + // that is a runtime SQL error, not a planner convention error. + assertThat(e.getMessage(), + not(stringContainsInOrder("calling-convention"))); + } + } + + /** Custom JDBC schema factory wrapping HSQLDB with MySQL dialect. + * + *

      HSQLDB's dialect reports {@code supportsWindowFunctions() = false}, + * so MySQL dialect is used to trigger the JdbcProjectRule path that + * exercises CALCITE-7616. */ + public static class WindowSupportingJdbcSchemaFactory + implements SchemaFactory { + + @Override public Schema create(SchemaPlus parentSchema, String name, + Map operand) { + final String jdbcUrl = (String) operand.get("jdbcUrl"); + final String jdbcDriver = (String) operand.get("jdbcDriver"); + final String jdbcUser = (String) operand.get("jdbcUser"); + final String jdbcPassword = (String) operand.get("jdbcPassword"); + + final javax.sql.DataSource dataSource = + JdbcSchema.dataSource(jdbcUrl, jdbcDriver, jdbcUser, jdbcPassword); + + final SqlDialect dialect = + new MysqlSqlDialect( + SqlDialect.EMPTY_CONTEXT + .withDatabaseProduct(SqlDialect.DatabaseProduct.MYSQL)); + + return JdbcSchema.create(parentSchema, name, dataSource, + databaseMetaData -> dialect, null, null); + } + } + /** Acquires a lock, and releases it when closed. */ static class LockWrapper implements AutoCloseable { private final Lock lock; From 07d2f2aab2c24fa07cb9cc43355715b413e6750d Mon Sep 17 00:00:00 2001 From: xuzifu666 <1206332514@qq.com> Date: Tue, 23 Jun 2026 11:42:37 +0800 Subject: [PATCH 350/562] [CALCITE-7620] Result of FILTER clause in window functions is incorrect --- .../calcite/rel/metadata/RelMdCollation.java | 11 +- .../rel/rules/ProjectToWindowRule.java | 3 + .../calcite/sql2rel/SqlToRelConverter.java | 62 +++++++++ core/src/test/resources/sql/sub-query.iq | 22 ++-- core/src/test/resources/sql/winagg.iq | 124 +++++++++++------- 5 files changed, 163 insertions(+), 59 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdCollation.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdCollation.java index 6ffe9c1f62df..e6ae979b1ab3 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdCollation.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdCollation.java @@ -66,6 +66,7 @@ import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -372,13 +373,13 @@ public static List sort(RelCollation collation) { /** Helper method to determine a * {@link org.apache.calcite.rel.core.Window}'s collation. * - *

      A Window projects the fields of its input first, followed by the output - * from each of its windows. Assuming (quite reasonably) that the - * implementation does not re-order its input rows, then any collations of its - * input are preserved. */ + *

      A Window operator groups rows by PARTITION BY keys and sorts each + * partition by ORDER BY keys. The output order is therefore not defined by + * a simple collation in the general case, so we conservatively report no + * collations. */ public static @Nullable List window(RelMetadataQuery mq, RelNode input, ImmutableList groups) { - return mq.collations(input); + return Collections.emptyList(); } /** Helper method to determine a diff --git a/core/src/main/java/org/apache/calcite/rel/rules/ProjectToWindowRule.java b/core/src/main/java/org/apache/calcite/rel/rules/ProjectToWindowRule.java index 536f5834e468..14a333aace48 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/ProjectToWindowRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/ProjectToWindowRule.java @@ -20,6 +20,7 @@ import org.apache.calcite.plan.RelOptRuleCall; import org.apache.calcite.plan.RelRule; import org.apache.calcite.plan.RelTraitSet; +import org.apache.calcite.rel.RelCollationTraitDef; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Calc; import org.apache.calcite.rel.core.Project; @@ -53,6 +54,7 @@ import java.util.ArrayDeque; import java.util.ArrayList; +import java.util.Collections; import java.util.Deque; import java.util.HashSet; import java.util.List; @@ -262,6 +264,7 @@ static class WindowedAggRelSplitter extends CalcRelSplitter { RelBuilder relBuilder, RelNode input, RexProgram program, List hints) { checkArgument(program.getCondition() == null, "WindowedAggregateRel cannot accept a condition"); + traitSet = traitSet.replaceIfs(RelCollationTraitDef.INSTANCE, Collections::emptyList); return LogicalWindow.create(cluster, traitSet, relBuilder, input, program, hints); } diff --git a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java index 1551d1630da4..507c14428829 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java @@ -2504,6 +2504,11 @@ private RexNode convertOver(Blackboard bb, SqlNode node) { SqlCall call = (SqlCall) node; bb.getValidator().deriveType(bb.scope, call); SqlCall aggCall = call.operand(0); + @Nullable SqlNode filter = null; + if (aggCall.getKind() == SqlKind.FILTER) { + filter = aggCall.operand(1); + aggCall = aggCall.operand(0); + } boolean ignoreNulls = false; switch (aggCall.getKind()) { case IGNORE_NULLS: @@ -2515,6 +2520,22 @@ private RexNode convertOver(Blackboard bb, SqlNode node) { default: break; } + if (filter != null) { + final SqlOperator op = aggCall.getOperator(); + if (op instanceof SqlAggFunction + && !((SqlAggFunction) op).requiresOver()) { + // FILTER on a windowed aggregate can be implemented by wrapping the + // aggregate arguments in CASE expressions, because true aggregates + // ignore NULL inputs. This does not work for window value functions + // (FIRST_VALUE, LAST_VALUE, NTH_VALUE, LEAD, LAG, etc.) which do not + // ignore NULL inputs. + aggCall = applyFilterToAggArgs(aggCall, filter); + bb.getValidator().deriveType(bb.scope, aggCall); + } else { + throw new UnsupportedOperationException( + "FILTER clause is not supported for window function " + op.getName()); + } + } SqlNode windowOrRef = call.operand(1); final SqlWindow window = @@ -2609,6 +2630,47 @@ private RexNode convertOver(Blackboard bb, SqlNode node) { } } + /** + * Applies a FILTER clause to the arguments of an aggregate call by wrapping + * each argument in a CASE expression. For example, + * {@code SUM(sal) FILTER (WHERE comm IS NOT NULL)} becomes + * {@code SUM(CASE WHEN comm IS NOT NULL THEN sal END)}. + * + *

      This transformation preserves the semantics of the FILTER clause for + * windowed aggregates: rows that do not satisfy the filter contribute NULL + * and are ignored by the aggregate function. + */ + private static SqlCall applyFilterToAggArgs(SqlCall aggCall, SqlNode filter) { + final SqlOperator op = aggCall.getOperator(); + final List operands = aggCall.getOperandList(); + final SqlParserPos pos = aggCall.getParserPosition(); + final SqlLiteral quantifier = aggCall.getFunctionQuantifier(); + final List newOperands = new ArrayList<>(operands.size()); + if (op == SqlStdOperatorTable.COUNT + && operands.size() == 1 + && operands.get(0) instanceof SqlIdentifier + && ((SqlIdentifier) operands.get(0)).isStar()) { + // COUNT(*) FILTER (WHERE x) => COUNT(CASE WHEN x THEN 0 END) + newOperands.add( + new SqlCase(pos, null, SqlNodeList.of(filter), + SqlNodeList.of(SqlLiteral.createExactNumeric("0", pos)), + SqlLiteral.createNull(pos))); + } else { + for (SqlNode operand : operands) { + if (operand instanceof SqlIdentifier + && ((SqlIdentifier) operand).isStar()) { + newOperands.add(operand); + } else { + newOperands.add( + new SqlCase(pos, null, SqlNodeList.of(filter), + SqlNodeList.of(operand), + SqlLiteral.createNull(pos))); + } + } + } + return op.createCall(quantifier, pos, newOperands); + } + protected void convertFrom( Blackboard bb, @Nullable SqlNode from) { diff --git a/core/src/test/resources/sql/sub-query.iq b/core/src/test/resources/sql/sub-query.iq index 2f190ed040dd..84f9aa5183f5 100644 --- a/core/src/test/resources/sql/sub-query.iq +++ b/core/src/test/resources/sql/sub-query.iq @@ -2430,11 +2430,12 @@ EnumerableCalc(expr#0..5=[{inputs}], expr#6=[RAND()], expr#7=[CAST($t6):INTEGER EnumerableSort(sort0=[$2], dir0=[ASC]) EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], SAL=[$t5], DEPTNO=[$t7]) EnumerableTableScan(table=[[scott, EMP]]) - EnumerableSort(sort0=[$1], dir0=[ASC]) - EnumerableCalc(expr#0..1=[{inputs}], expr#2=[false], expr#3=[1], expr#4=[<=($t1, $t3)], cs=[$t2], DEPTNO=[$t0], rn=[$t1], $condition=[$t4]) - EnumerableWindow(window#0=[window(partition {0} rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])], constants=[[false]]) - EnumerableCalc(expr#0..2=[{inputs}], DEPTNO=[$t0]) - EnumerableTableScan(table=[[scott, DEPT]]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1], expr#4=[<=($t2, $t3)], proj#0..2=[{exprs}], $condition=[$t4]) + EnumerableSort(sort0=[$1], dir0=[ASC]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[false], cs=[$t2], DEPTNO=[$t0], rn=[$t1]) + EnumerableWindow(window#0=[window(partition {0} rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])], constants=[[false]]) + EnumerableCalc(expr#0..2=[{inputs}], DEPTNO=[$t0]) + EnumerableTableScan(table=[[scott, DEPT]]) !plan !} @@ -2540,11 +2541,12 @@ EnumerableCalc(expr#0..5=[{inputs}], expr#6=[NOT($t3)], expr#7=[IS NOT NULL($t3) EnumerableSort(sort0=[$2], dir0=[ASC]) EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], SAL=[$t5], DEPTNO=[$t7]) EnumerableTableScan(table=[[scott, EMP]]) - EnumerableSort(sort0=[$1], dir0=[ASC]) - EnumerableCalc(expr#0..1=[{inputs}], expr#2=[false], expr#3=[1], expr#4=[<=($t1, $t3)], cs=[$t2], DEPTNO=[$t0], rn=[$t1], $condition=[$t4]) - EnumerableWindow(window#0=[window(partition {0} rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])], constants=[[false]]) - EnumerableCalc(expr#0..2=[{inputs}], DEPTNO=[$t0]) - EnumerableTableScan(table=[[scott, DEPT]]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1], expr#4=[<=($t2, $t3)], proj#0..2=[{exprs}], $condition=[$t4]) + EnumerableSort(sort0=[$1], dir0=[ASC]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[false], cs=[$t2], DEPTNO=[$t0], rn=[$t1]) + EnumerableWindow(window#0=[window(partition {0} rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])], constants=[[false]]) + EnumerableCalc(expr#0..2=[{inputs}], DEPTNO=[$t0]) + EnumerableTableScan(table=[[scott, DEPT]]) !plan !} diff --git a/core/src/test/resources/sql/winagg.iq b/core/src/test/resources/sql/winagg.iq index d8348da422a2..1f07d4351d8d 100644 --- a/core/src/test/resources/sql/winagg.iq +++ b/core/src/test/resources/sql/winagg.iq @@ -1186,20 +1186,20 @@ order by empno; +-------+--------+----------------+ | EMPNO | DEPTNO | FILTERED_COUNT | +-------+--------+----------------+ -| 7369 | 20 | 0 | -| 7566 | 20 | 5 | -| 7788 | 20 | 5 | -| 7876 | 20 | 0 | -| 7902 | 20 | 5 | -| 7782 | 10 | 3 | -| 7839 | 10 | 3 | -| 7934 | 10 | 0 | -| 7499 | 30 | 6 | -| 7521 | 30 | 0 | -| 7654 | 30 | 0 | -| 7698 | 30 | 6 | -| 7844 | 30 | 0 | -| 7900 | 30 | 0 | +| 7369 | 20 | 3 | +| 7499 | 30 | 2 | +| 7521 | 30 | 2 | +| 7566 | 20 | 3 | +| 7654 | 30 | 2 | +| 7698 | 30 | 2 | +| 7782 | 10 | 2 | +| 7788 | 20 | 3 | +| 7839 | 10 | 2 | +| 7844 | 30 | 2 | +| 7876 | 20 | 3 | +| 7900 | 30 | 2 | +| 7902 | 20 | 3 | +| 7934 | 10 | 2 | +-------+--------+----------------+ (14 rows) @@ -1214,19 +1214,19 @@ order by empno; | EMPNO | DEPTNO | FILTERED_SUM | +-------+--------+--------------+ | 7369 | 20 | | +| 7499 | 30 | 5600.00 | +| 7521 | 30 | 5600.00 | | 7566 | 20 | | +| 7654 | 30 | 5600.00 | +| 7698 | 30 | 5600.00 | +| 7782 | 10 | | | 7788 | 20 | | +| 7839 | 10 | | +| 7844 | 30 | 5600.00 | | 7876 | 20 | | +| 7900 | 30 | 5600.00 | | 7902 | 20 | | -| 7782 | 10 | | -| 7839 | 10 | | | 7934 | 10 | | -| 7499 | 30 | 9400.00 | -| 7521 | 30 | 9400.00 | -| 7654 | 30 | 9400.00 | -| 7698 | 30 | | -| 7844 | 30 | 9400.00 | -| 7900 | 30 | | +-------+--------+--------------+ (14 rows) @@ -1241,20 +1241,20 @@ order by empno; +-------+--------+----------------+-------------+ | EMPNO | DEPTNO | HIGH_SAL_COUNT | LOW_SAL_SUM | +-------+--------+----------------+-------------+ -| 7369 | 20 | 0 | 10875.00 | -| 7566 | 20 | 5 | | -| 7788 | 20 | 5 | | -| 7876 | 20 | 0 | 10875.00 | -| 7902 | 20 | 5 | | -| 7782 | 10 | 3 | | -| 7839 | 10 | 3 | | -| 7934 | 10 | 0 | 8750.00 | -| 7499 | 30 | 6 | | -| 7521 | 30 | 0 | 9400.00 | -| 7654 | 30 | 0 | 9400.00 | -| 7698 | 30 | 6 | | -| 7844 | 30 | 0 | 9400.00 | -| 7900 | 30 | 0 | 9400.00 | +| 7369 | 20 | 3 | 1900.00 | +| 7499 | 30 | 2 | 4950.00 | +| 7521 | 30 | 2 | 4950.00 | +| 7566 | 20 | 3 | 1900.00 | +| 7654 | 30 | 2 | 4950.00 | +| 7698 | 30 | 2 | 4950.00 | +| 7782 | 10 | 2 | 1300.00 | +| 7788 | 20 | 3 | 1900.00 | +| 7839 | 10 | 2 | 1300.00 | +| 7844 | 30 | 2 | 4950.00 | +| 7876 | 20 | 3 | 1900.00 | +| 7900 | 30 | 2 | 4950.00 | +| 7902 | 20 | 3 | 1900.00 | +| 7934 | 10 | 2 | 1300.00 | +-------+--------+----------------+-------------+ (14 rows) @@ -1269,22 +1269,58 @@ order by empno; | EMPNO | DEPTNO | SAL | RUNNING_SUM | +-------+--------+---------+-------------+ | 7369 | 20 | 800.00 | | -| 7566 | 20 | 2975.00 | 3775.00 | -| 7788 | 20 | 3000.00 | 6775.00 | -| 7876 | 20 | 1100.00 | 7875.00 | -| 7902 | 20 | 3000.00 | 10875.00 | -| 7782 | 10 | 2450.00 | 2450.00 | -| 7839 | 10 | 5000.00 | 7450.00 | -| 7934 | 10 | 1300.00 | 8750.00 | | 7499 | 30 | 1600.00 | 1600.00 | | 7521 | 30 | 1250.00 | 2850.00 | +| 7566 | 20 | 2975.00 | 2975.00 | | 7654 | 30 | 1250.00 | 4100.00 | | 7698 | 30 | 2850.00 | 6950.00 | +| 7782 | 10 | 2450.00 | 2450.00 | +| 7788 | 20 | 3000.00 | 5975.00 | +| 7839 | 10 | 5000.00 | 7450.00 | | 7844 | 30 | 1500.00 | 8450.00 | -| 7900 | 30 | 950.00 | | +| 7876 | 20 | 1100.00 | 7075.00 | +| 7900 | 30 | 950.00 | 8450.00 | +| 7902 | 20 | 3000.00 | 10075.00 | +| 7934 | 10 | 1300.00 | 8750.00 | +-------+--------+---------+-------------+ (14 rows) !ok +# Test 5: FILTER with OVER and running window without PARTITION BY +select ename, job, hiredate, + avg(sal) over (order by hiredate, ename rows 3 preceding) as avg_sal, + avg(sal) filter (where job = 'MANAGER') over (order by hiredate, ename rows 3 preceding) + as avg_mgr_sal +from emp +order by hiredate, ename; ++--------+-----------+------------+---------+-------------+ +| ENAME | JOB | HIREDATE | AVG_SAL | AVG_MGR_SAL | ++--------+-----------+------------+---------+-------------+ +| SMITH | CLERK | 1980-12-17 | 800.00 | | +| BLAKE | MANAGER | 1981-01-05 | 1825.00 | 2850.00 | +| JONES | MANAGER | 1981-02-04 | 2208.33 | 2912.50 | +| ALLEN | SALESMAN | 1981-02-20 | 2056.25 | 2912.50 | +| WARD | SALESMAN | 1981-02-22 | 2168.75 | 2912.50 | +| CLARK | MANAGER | 1981-06-09 | 2068.75 | 2712.50 | +| TURNER | SALESMAN | 1981-09-08 | 1700.00 | 2450.00 | +| MARTIN | SALESMAN | 1981-09-28 | 1612.50 | 2450.00 | +| KING | PRESIDENT | 1981-11-17 | 2550.00 | 2450.00 | +| FORD | ANALYST | 1981-12-03 | 2687.50 | | +| JAMES | CLERK | 1981-12-03 | 2550.00 | | +| MILLER | CLERK | 1982-01-23 | 2562.50 | | +| SCOTT | ANALYST | 1987-04-19 | 2062.50 | | +| ADAMS | CLERK | 1987-05-23 | 1587.50 | | ++--------+-----------+------------+---------+-------------+ +(14 rows) + +!ok + +# Test 6: FILTER on window value functions is not supported +select first_value(sal) filter (where job = 'MANAGER') over (order by hiredate) +from emp; +java.sql.SQLException: Error while executing SQL "select first_value(sal) filter (where job = 'MANAGER') over (order by hiredate) +from emp": FILTER clause is not supported for window function FIRST_VALUE +!error + # End winagg.iq From 2f7643b013be12c29c32882b89db5ffffee0a258 Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Thu, 25 Jun 2026 11:48:25 +0800 Subject: [PATCH 351/562] [CALCITE-7623] SemiJoinProjectTransposeRule should support ANTI joins --- .../rules/SemiJoinProjectTransposeRule.java | 80 +++++++++++-------- .../apache/calcite/test/RelOptRulesTest.java | 20 +++++ .../apache/calcite/test/RelOptRulesTest.xml | 18 +++++ 3 files changed, 84 insertions(+), 34 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rel/rules/SemiJoinProjectTransposeRule.java b/core/src/main/java/org/apache/calcite/rel/rules/SemiJoinProjectTransposeRule.java index 42deabf5e3d6..ea4e38633fdc 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/SemiJoinProjectTransposeRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/SemiJoinProjectTransposeRule.java @@ -46,14 +46,19 @@ /** * Planner rule that pushes - * a {@link Join#isSemiJoin semi-join} down in a tree past + * a semi-join or anti-join down in a tree past * a {@link org.apache.calcite.rel.core.Project}. * *

      The intention is to trigger other rules that will convert - * {@code SemiJoin}s. + * {@code SemiJoin}s or {@code AntiJoin}s. * *

      SemiJoin(LogicalProject(X), Y) → LogicalProject(SemiJoin(X, Y)) * + *

      AntiJoin(LogicalProject(X), Y) → LogicalProject(AntiJoin(X, Y)) + * + *

      This rule only transposes a project on the left input. Semi-joins and + * anti-joins only project fields from their left input. + * * @see org.apache.calcite.rel.rules.SemiJoinFilterTransposeRule */ @Value.Enclosing @@ -69,56 +74,56 @@ protected SemiJoinProjectTransposeRule(Config config) { //~ Methods ---------------------------------------------------------------- @Override public void onMatch(RelOptRuleCall call) { - final Join semiJoin = call.rel(0); + final Join join = call.rel(0); final Project project = call.rel(1); - // Convert the LHS semi-join keys to reference the child projection - // expression; all projection expressions must be RexInputRefs, - // otherwise, we wouldn't have created this semi-join. + // Convert the LHS semi-join or anti-join keys to reference the child + // projection expression; all projection expressions must be RexInputRefs, + // otherwise, we wouldn't have created this semi-join or anti-join. - // convert the semijoin condition to reflect the LHS with the project + // convert the join condition to reflect the LHS with the project // pulled up - RexNode newCondition = adjustCondition(project, semiJoin); + RexNode newCondition = adjustCondition(project, join); - LogicalJoin newSemiJoin = + LogicalJoin newJoin = LogicalJoin.create(project.getInput(), - semiJoin.getRight(), + join.getRight(), // No need to copy the hints, the framework would try to do that. ImmutableList.of(), newCondition, - ImmutableSet.of(), JoinRelType.SEMI); + ImmutableSet.of(), join.getJoinType()); // Create the new projection. Note that the projection expressions // are the same as the original because they only reference the LHS - // of the semijoin and the semijoin only projects out the LHS + // of the semi-join or anti-join, which only projects out the LHS. final RelBuilder relBuilder = call.builder(); - relBuilder.push(newSemiJoin); + relBuilder.push(newJoin); relBuilder.project(project.getProjects(), project.getRowType().getFieldNames()); call.transformTo(relBuilder.build()); } /** - * Pulls the project above the semijoin and returns the resulting semijoin - * condition. As a result, the semijoin condition should be modified such - * that references to the LHS of a semijoin should now reference the + * Pulls the project above the semi-join or anti-join and returns the resulting + * join condition. As a result, the join condition should be modified such + * that references to the LHS of the join should now reference the * children of the project that's on the LHS. * - * @param project LogicalProject on the LHS of the semijoin - * @param semiJoin the semijoin - * @return the modified semijoin condition + * @param project LogicalProject on the LHS of the join + * @param join the semi-join or anti-join + * @return the modified join condition */ - private static RexNode adjustCondition(Project project, Join semiJoin) { + private static RexNode adjustCondition(Project project, Join join) { // create two RexPrograms -- the bottom one representing a - // concatenation of the project and the RHS of the semijoin and the - // top one representing the semijoin condition + // concatenation of the project and the RHS of the join and the + // top one representing the join condition final RexBuilder rexBuilder = project.getCluster().getRexBuilder(); final RelDataTypeFactory typeFactory = rexBuilder.getTypeFactory(); - final RelNode rightChild = semiJoin.getRight(); + final RelNode rightChild = join.getRight(); // for the bottom RexProgram, the input is a concatenation of the - // child of the project and the RHS of the semijoin + // child of the project and the RHS of the join RelDataType bottomInputRowType = SqlValidatorUtil.deriveJoinRowType( project.getInput().getRowType(), @@ -126,12 +131,12 @@ private static RexNode adjustCondition(Project project, Join semiJoin) { JoinRelType.INNER, typeFactory, null, - semiJoin.getSystemFieldList()); + join.getSystemFieldList()); RexProgramBuilder bottomProgramBuilder = new RexProgramBuilder(bottomInputRowType, rexBuilder); // add the project expressions, then add input references for the RHS - // of the semijoin + // of the join for (Pair pair : project.getNamedProjects()) { bottomProgramBuilder.addProject(pair.left, pair.right); } @@ -148,8 +153,8 @@ private static RexNode adjustCondition(Project project, Join semiJoin) { } RexProgram bottomProgram = bottomProgramBuilder.getProgram(); - // input rowtype into the top program is the concatenation of the - // project and the RHS of the semijoin + // input row type into the top program is the concatenation of the + // project and the RHS of the join RelDataType topInputRowType = SqlValidatorUtil.deriveJoinRowType( project.getRowType(), @@ -157,18 +162,18 @@ private static RexNode adjustCondition(Project project, Join semiJoin) { JoinRelType.INNER, typeFactory, null, - semiJoin.getSystemFieldList()); + join.getSystemFieldList()); RexProgramBuilder topProgramBuilder = new RexProgramBuilder( topInputRowType, rexBuilder); topProgramBuilder.addIdentity(); - topProgramBuilder.addCondition(semiJoin.getCondition()); + topProgramBuilder.addCondition(join.getCondition()); RexProgram topProgram = topProgramBuilder.getProgram(); // merge the programs and expand out the local references to form - // the new semijoin condition; it now references a concatenation of - // the project's child and the RHS of the semijoin + // the new join condition; it now references a concatenation of + // the project's child and the RHS of the join RexProgram mergedProgram = RexProgramBuilder.mergePrograms( topProgram, @@ -180,6 +185,11 @@ private static RexNode adjustCondition(Project project, Join semiJoin) { () -> "mergedProgram.getCondition() for " + mergedProgram)); } + private static boolean isSemiOrAntiJoin(Join join) { + return join.getJoinType() == JoinRelType.SEMI + || join.getJoinType() == JoinRelType.ANTI; + } + /** Rule configuration. */ @Value.Immutable public interface Config extends RelRule.Config { @@ -194,8 +204,10 @@ public interface Config extends RelRule.Config { default Config withOperandFor(Class joinClass, Class projectClass) { return withOperandSupplier(b -> - b.operand(joinClass).predicate(Join::isSemiJoin).inputs(b2 -> - b2.operand(projectClass).anyInputs())) + b.operand(joinClass) + .predicate(SemiJoinProjectTransposeRule::isSemiOrAntiJoin) + .inputs(b2 -> + b2.operand(projectClass).anyInputs())) .as(Config.class); } } diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index b10a5c065c67..88080fad4702 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -1498,6 +1498,26 @@ private RelOptFixture basePushFilterPastAggWithGroupingSets() { checkSemiOrAntiJoinProjectTranspose(JoinRelType.ANTI); } + /** Test case for + * [CALCITE-7623] + * SemiJoinProjectTransposeRule should support ANTI joins. */ + @Test void testSemiJoinProjectTransposeSupportsAntiJoin() { + final Function relFn = b -> { + RelNode left = b.scan("DEPT") + .project(b.field("DNAME"), b.field("DEPTNO")) + .build(); + RelNode right = b.scan("EMP").build(); + + return b.push(left) + .push(right) + .join(JoinRelType.ANTI, + b.equals(b.field(2, 0, "DEPTNO"), + b.field(2, 1, "DEPTNO"))) + .build(); + }; + relFn(relFn).withRule(CoreRules.SEMI_JOIN_PROJECT_TRANSPOSE).check(); + } + private void checkSemiOrAntiJoinProjectTranspose(JoinRelType type) { final Function relFn = b -> { RelNode left = b.scan("DEPT").build(); diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index 2a47eefd0d7a..e8d6bcd22952 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -17989,6 +17989,24 @@ LogicalProject(DNAME=[$1]) LogicalAggregate(group=[{0}]) LogicalProject($f0=[*(2, $0)]) LogicalTableScan(table=[[scott, DEPT]]) +]]> + + + + + + + + From 5febdedbbf24ae9f7af17073c4184fce77f49ab3 Mon Sep 17 00:00:00 2001 From: Stamatis Zampetakis Date: Fri, 26 Jun 2026 09:25:37 +0200 Subject: [PATCH 352/562] [CALCITE-7625] Unify sqlline scripts from different modules and update tutorial --- .github/workflows/main.yml | 8 +++--- build.gradle.kts | 2 +- example/csv/build.gradle.kts | 24 ----------------- example/csv/sqlline | 51 ------------------------------------ example/csv/sqlline.bat | 37 -------------------------- site/_docs/tutorial.md | 9 +++---- 6 files changed, 9 insertions(+), 122 deletions(-) delete mode 100755 example/csv/sqlline delete mode 100644 example/csv/sqlline.bat diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a7ac82d07666..165f2752ab25 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -76,7 +76,7 @@ jobs: call sqlline.bat -e '!quit' echo. echo Sqlline example/csv - call example/csv/sqlline.bat --verbose -u jdbc:calcite:model=example/csv/src/test/resources/model.json -n admin -p admin -f example/csv/src/test/resources/smoke_test.sql + call sqlline.bat --verbose -u jdbc:calcite:model=example/csv/src/test/resources/model.json -n admin -p admin -f example/csv/src/test/resources/smoke_test.sql echo. echo sqlsh call sqlsh.bat -o headers "select count(*) commits, author from (select substring(author, 1, position(' <' in author)-1) author from git_commits) group by author order by count(*) desc, author limit 20" @@ -106,7 +106,7 @@ jobs: call sqlline.bat -e '!quit' echo. echo Sqlline example/csv - call example/csv/sqlline.bat --verbose -u jdbc:calcite:model=example/csv/src/test/resources/model.json -n admin -p admin -f example/csv/src/test/resources/smoke_test.sql + call sqlline.bat --verbose -u jdbc:calcite:model=example/csv/src/test/resources/model.json -n admin -p admin -f example/csv/src/test/resources/smoke_test.sql echo. echo sqlsh call sqlsh.bat -o headers "select count(*) commits, author from (select substring(author, 1, position(' <' in author)-1) author from git_commits) group by author order by count(*) desc, author limit 20" @@ -136,7 +136,7 @@ jobs: call sqlline.bat -e '!quit' echo. echo Sqlline example/csv - call example/csv/sqlline.bat --verbose -u jdbc:calcite:model=example/csv/src/test/resources/model.json -n admin -p admin -f example/csv/src/test/resources/smoke_test.sql + call sqlline.bat --verbose -u jdbc:calcite:model=example/csv/src/test/resources/model.json -n admin -p admin -f example/csv/src/test/resources/smoke_test.sql echo. echo sqlsh call sqlsh.bat -o headers "select count(*) commits, author from (select substring(author, 1, position(' <' in author)-1) author from git_commits) group by author order by count(*) desc, author limit 20" @@ -328,7 +328,7 @@ jobs: ./sqlline -e '!quit' echo echo Sqlline example/csv - ./example/csv/sqlline --verbose -u jdbc:calcite:model=example/csv/src/test/resources/model.json -n admin -p admin -f example/csv/src/test/resources/smoke_test.sql + ./sqlline --verbose -u jdbc:calcite:model=example/csv/src/test/resources/model.json -n admin -p admin -f example/csv/src/test/resources/smoke_test.sql echo echo sqlsh ./sqlsh -o headers "select count(*) commits, author from (select substring(author, 1, position(' <' in author)-1) author from git_commits) group by author order by count(*) desc, author limit 20" diff --git a/build.gradle.kts b/build.gradle.kts index a66e7107b47e..08cd2bd38989 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -236,7 +236,7 @@ val javadocAggregateIncludingTests by tasks.registering(Javadoc::class) { } val adaptersForSqlline = listOf( - ":arrow", ":babel", ":cassandra", ":druid", ":elasticsearch", + ":arrow", ":babel", ":cassandra", ":druid", ":elasticsearch", "example:csv", ":file", ":geode", ":innodb", ":kafka", ":mongodb", ":pig", ":piglet", ":plus", ":redis", ":server", ":spark", ":splunk") diff --git a/example/csv/build.gradle.kts b/example/csv/build.gradle.kts index edf2d8eed15d..928f794ef058 100644 --- a/example/csv/build.gradle.kts +++ b/example/csv/build.gradle.kts @@ -21,11 +21,6 @@ plugins { id("com.github.vlsi.ide") } -val sqllineClasspath by configurations.creating { - isCanBeConsumed = false - extendsFrom(configurations.testRuntimeClasspath.get()) -} - dependencies { api(project(":core")) api(project(":file")) @@ -40,9 +35,6 @@ dependencies { testImplementation("sqlline:sqlline") testImplementation(project(":testkit")) - sqllineClasspath(project) - sqllineClasspath(files(sourceSets.test.map { it.output })) - annotationProcessor("org.immutables:value") compileOnly("org.immutables:value-annotations") compileOnly("com.google.code.findbugs:jsr305") @@ -82,19 +74,3 @@ ide { generatedSource(annotationProcessorMain) } - -val buildSqllineClasspath by tasks.registering(Jar::class) { - inputs.files(sqllineClasspath).withNormalizer(ClasspathNormalizer::class.java) - archiveFileName.set("sqllineClasspath.jar") - manifest { - attributes( - "Main-Class" to "sqlline.SqlLine", - "Class-Path" to provider { - // Class-Path is a list of URLs - sqllineClasspath.joinToString(" ") { - it.toURI().toURL().toString() - } - } - ) - } -} diff --git a/example/csv/sqlline b/example/csv/sqlline deleted file mode 100755 index 4248fa8373ab..000000000000 --- a/example/csv/sqlline +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/bash -# sqlline - Script to launch SQL shell -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to you under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Example: -# $ ./sqlline -# sqlline> !connect jdbc:calcite:model=src/test/resources/model.json admin admin -# sqlline> !tables - -# Deduce whether we are running cygwin -case $(uname -s) in -(CYGWIN*) cygwin=true;; -(*) cygwin=;; -esac - -# readlink in macOS resolves only links, and it returns empty results if the path points to a file -root=$0 -if [[ -L "$root" ]]; then - root=$(readlink "$root") -fi -root=$(cd "$(dirname "$root")"; pwd) - -CP=$root/build/libs/sqllineClasspath.jar - -if [ "x$CACHE_SQLLINE_CLASSPATH" != "xY" ] || [ ! -f "$CP" ]; then - $root/../../gradlew --console plain -q :example:csv:buildSqllineClasspath -fi - -VM_OPTS= -if [ "$cygwin" ]; then - # Work around https://github.com/jline/jline2/issues/62 - VM_OPTS=-Djline.terminal=jline.UnixTerminal -fi - -export JAVA_OPTS=-Djavax.xml.parsers.DocumentBuilderFactory=com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl - -exec java -Xmx1g $VM_OPTS $JAVA_OPTS -jar "$root/build/libs/sqllineClasspath.jar" "$@" diff --git a/example/csv/sqlline.bat b/example/csv/sqlline.bat deleted file mode 100644 index b29b6f33666d..000000000000 --- a/example/csv/sqlline.bat +++ /dev/null @@ -1,37 +0,0 @@ -@echo off -:: -:: Licensed to the Apache Software Foundation (ASF) under one or more -:: contributor license agreements. See the NOTICE file distributed with -:: this work for additional information regarding copyright ownership. -:: The ASF licenses this file to you under the Apache License, Version 2.0 -:: (the "License"); you may not use this file except in compliance with -:: the License. You may obtain a copy of the License at -:: -:: http://www.apache.org/licenses/LICENSE-2.0 -:: -:: Unless required by applicable law or agreed to in writing, software -:: distributed under the License is distributed on an "AS IS" BASIS, -:: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -:: See the License for the specific language governing permissions and -:: limitations under the License. -:: - -:: sqlline.bat - Windows script to launch SQL shell -:: Example: -:: > sqlline.bat -:: sqlline> !connect jdbc:calcite:model=src\test\resources\model.json admin admin -:: sqlline> !tables - -:: The script updates the classpath on each execution, -:: You might add CACHE_SQLLINE_CLASSPATH environment variable to cache it -:: To build classpath jar manually use gradlew buildSqllineClasspath -set DIRNAME=%~dp0 -if "%DIRNAME%" == "" set DIRNAME=. -set CP=%DIRNAME%\build\libs\sqllineClasspath.jar - -if not defined CACHE_SQLLINE_CLASSPATH ( - if exist "%CP%" del "%CP%" -) -if not exist "%CP%" (call "%DIRNAME%\..\..\gradlew" --console plain -q :example:csv:buildSqllineClasspath) - -java -Xmx1g -jar "%CP%" %* diff --git a/site/_docs/tutorial.md b/site/_docs/tutorial.md index bf50ab6645c2..9ca5c02199d9 100644 --- a/site/_docs/tutorial.md +++ b/site/_docs/tutorial.md @@ -57,7 +57,6 @@ You need Java (version 8, 9 or 10) and Git. {% highlight bash %} $ git clone https://github.com/apache/calcite.git -$ cd calcite/example/csv $ ./sqlline {% endhighlight %} @@ -69,7 +68,7 @@ that is included in this project. {% highlight bash %} $ ./sqlline -sqlline> !connect jdbc:calcite:model=src/test/resources/model.json admin admin +sqlline> !connect jdbc:calcite:model=example/csv/src/test/resources/model.json admin admin {% endhighlight %} (If you are running Windows, the command is `sqlline.bat`.) @@ -395,7 +394,7 @@ There is an example in model-with-custom-table.json: We can query the table in the usual way: {% highlight sql %} -sqlline> !connect jdbc:calcite:model=src/test/resources/model-with-custom-table.json admin admin +sqlline> !connect jdbc:calcite:model=example/csv/src/test/resources/model-with-custom-table.json admin admin sqlline> SELECT empno, name FROM custom_table.emps; +--------+--------+ | EMPNO | NAME | @@ -481,7 +480,7 @@ a subset of columns from a CSV file. Let's run the same query against two very similar schemas: {% highlight sql %} -sqlline> !connect jdbc:calcite:model=src/test/resources/model.json admin admin +sqlline> !connect jdbc:calcite:model=example/csv/src/test/resources/model.json admin admin sqlline> explain plan for select name from emps; +-----------------------------------------------------+ | PLAN | @@ -489,7 +488,7 @@ sqlline> explain plan for select name from emps; | EnumerableCalc(expr#0..9=[{inputs}], NAME=[$t1]) | | EnumerableTableScan(table=[[SALES, EMPS]]) | +-----------------------------------------------------+ -sqlline> !connect jdbc:calcite:model=src/test/resources/smart.json admin admin +sqlline> !connect jdbc:calcite:model=example/csv/src/test/resources/smart.json admin admin sqlline> explain plan for select name from emps; +-----------------------------------------------------+ | PLAN | From e945d482c49eb7285bf6d342093a0cd00185cc03 Mon Sep 17 00:00:00 2001 From: Yash Limbad Date: Wed, 24 Jun 2026 12:06:32 +0530 Subject: [PATCH 353/562] [CALCITE-7622] Don't fire JoinProjectTransposeRule for ANTI/SEMI/LEFT_MARK JOIN --- .../rel/rules/JoinProjectTransposeRule.java | 6 +++ .../apache/calcite/test/RelOptRulesTest.java | 37 +++++++++++++++++++ .../apache/calcite/test/RelOptRulesTest.xml | 33 +++++++++++++++++ 3 files changed, 76 insertions(+) diff --git a/core/src/main/java/org/apache/calcite/rel/rules/JoinProjectTransposeRule.java b/core/src/main/java/org/apache/calcite/rel/rules/JoinProjectTransposeRule.java index 6902a5ef8054..e82f57e96b0f 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/JoinProjectTransposeRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/JoinProjectTransposeRule.java @@ -112,6 +112,12 @@ public JoinProjectTransposeRule(RelOptRuleOperand operand, //~ Methods ---------------------------------------------------------------- + @Override public boolean matches(RelOptRuleCall call) { + Join join = call.rel(0); + // SEMI/ANTI/LEFT_MARK join cannot be swapped. + return join.getJoinType().projectsRight(); + } + @Override public void onMatch(RelOptRuleCall call) { final Join join = call.rel(0); final JoinRelType joinType = join.getJoinType(); diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index 88080fad4702..72d94a5a0858 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -1538,6 +1538,43 @@ private void checkSemiOrAntiJoinProjectTranspose(JoinRelType type) { relFn(relFn).withRule(CoreRules.PROJECT_JOIN_TRANSPOSE).check(); } + /** Test case for + * [CALCITE-7622] + * Don't fire JoinProjectTransposeRule for ANTI/SEMI/LEFT_MARK JOIN. */ + @Test void testJoinProjectTransposeDoesNotMatchSemiJoin() { + checkJoinProjectTransposeDoesNotMatch(JoinRelType.SEMI); + } + + /** Test case for + * [CALCITE-7622] + * Don't fire JoinProjectTransposeRule for ANTI/SEMI/LEFT_MARK JOIN. */ + @Test void testJoinProjectTransposeDoesNotMatchAntiJoin() { + checkJoinProjectTransposeDoesNotMatch(JoinRelType.ANTI); + } + + /** Test case for + * [CALCITE-7622] + * Don't fire JoinProjectTransposeRule for ANTI/SEMI/LEFT_MARK JOIN. */ + @Test void testJoinProjectTransposeDoesNotMatchLeftMarkJoin() { + checkJoinProjectTransposeDoesNotMatch(JoinRelType.LEFT_MARK); + } + + /** A SEMI, ANTI or LEFT_MARK join does not project its right input, so + * {@link JoinProjectTransposeRule} must not pull projects above it. */ + private void checkJoinProjectTransposeDoesNotMatch(JoinRelType type) { + final Function relFn = b -> b + .scan("EMP") + .project(b.field("DEPTNO")) + .scan("DEPT") + .project(b.field("DEPTNO")) + .join(type, + b.equals( + b.field(2, 0, 0), + b.field(2, 1, 0))) + .build(); + relFn(relFn).withRule(CoreRules.JOIN_PROJECT_BOTH_TRANSPOSE).checkUnchanged(); + } + @Test void testJoinProjectTranspose1() { final String sql = "select a.name\n" + "from dept a\n" diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index e8d6bcd22952..4eae45cc8aa3 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -8463,6 +8463,39 @@ LogicalProject(DEPTNO=[$0], NAME=[$1], NAME0=[$2], EXPR$1=[$3]) LogicalJoin(condition=[=($1, $3)], joinType=[left]) LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) +]]> + + + + + + + + + + + + + + + From 4a1a480e793804f0685f8166fb46c4d5b355099e Mon Sep 17 00:00:00 2001 From: Julian Hyde Date: Sun, 28 Jun 2026 13:13:38 -0700 Subject: [PATCH 354/562] [CALCITE-7628] In the interpreter, MINUS and INTERSECT with 3 or more inputs return wrong result In the interpreter, a query where MINUS or INTERSECT has 3 or more inputs previously returned the wrong result, because SetOpNode evaluated only the first two inputs. It now evaluates all inputs. We add tests in a new file `interpreter.iq`, that runs SQL queries using the interpreter. Add optimized implementations of MINUS DISTINCT and INTERSECT DISTINCT using mutable counts of the number of occurrences of each key; avoid materializing every input of a MINUS or INTERSECT if the output becomes empty. Remove dead code in `RelFieldTrimmer` (killed by CALCITE-3399). Close apache/calcite#5055 --- .../apache/calcite/interpreter/SetOpNode.java | 236 +++++++++++++++--- .../calcite/sql2rel/RelFieldTrimmer.java | 15 +- .../apache/calcite/test/InterpreterTest.java | 2 +- .../org/apache/calcite/test/JdbcTest.java | 9 +- core/src/test/resources/sql/interpreter.iq | 171 +++++++++++++ 5 files changed, 374 insertions(+), 59 deletions(-) create mode 100644 core/src/test/resources/sql/interpreter.iq diff --git a/core/src/main/java/org/apache/calcite/interpreter/SetOpNode.java b/core/src/main/java/org/apache/calcite/interpreter/SetOpNode.java index 2d5750efd799..9e803cb809b9 100644 --- a/core/src/main/java/org/apache/calcite/interpreter/SetOpNode.java +++ b/core/src/main/java/org/apache/calcite/interpreter/SetOpNode.java @@ -18,10 +18,15 @@ import org.apache.calcite.rel.core.SetOp; -import com.google.common.collect.HashMultiset; - -import java.util.Collection; +import java.util.HashMap; import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static com.google.common.base.Preconditions.checkArgument; + +import static org.apache.calcite.util.Util.transformIndexed; /** * Interpreter node that implements a @@ -31,63 +36,216 @@ * {@link org.apache.calcite.rel.core.Intersect}. */ public class SetOpNode implements Node { - private final Source leftSource; - private final Source rightSource; + private final List sources; private final Sink sink; private final SetOp setOp; public SetOpNode(Compiler compiler, SetOp setOp) { - leftSource = compiler.source(setOp, 0); - rightSource = compiler.source(setOp, 1); + final int arity = setOp.getInputs().size(); + checkArgument(arity >= 2, "invalid set op arity %s", arity); + sources = transformIndexed(setOp.getInputs(), (r, i) -> compiler.source(setOp, i)); + assert sources.size() == arity; sink = compiler.sink(setOp); this.setOp = setOp; } @Override public void close() { - leftSource.close(); - rightSource.close(); + for (Source source : sources) { + source.close(); + } } @Override public void run() throws InterruptedException { - final Collection leftRows; - final Collection rightRows; - if (setOp.all) { - leftRows = HashMultiset.create(); - rightRows = HashMultiset.create(); - } else { - leftRows = new HashSet<>(); - rightRows = new HashSet<>(); - } - Row row; - while ((row = leftSource.receive()) != null) { - leftRows.add(row); - } - while ((row = rightSource.receive()) != null) { - rightRows.add(row); - } switch (setOp.kind) { - case INTERSECT: - for (Row leftRow : leftRows) { - if (rightRows.remove(leftRow)) { - sink.send(leftRow); - } + case UNION: + if (setOp.all) { + unionAll(); + } else { + unionDistinct(); } break; - case EXCEPT: - for (Row leftRow : leftRows) { - if (!rightRows.remove(leftRow)) { - sink.send(leftRow); - } + case INTERSECT: + if (setOp.all) { + intersectAll(); + } else { + intersectDistinct(); } break; - case UNION: - leftRows.addAll(rightRows); - for (Row r : leftRows) { - sink.send(r); + case EXCEPT: + if (setOp.all) { + minusAll(); + } else { + minusDistinct(); } break; default: break; } } + + /** Evaluates UNION ALL. Does not need to buffer: sends each row to the + * output as it arrives. */ + private void unionAll() throws InterruptedException { + for (Source source : sources) { + Row row; + while ((row = source.receive()) != null) { + sink.send(row); + } + } + } + + /** Evaluates UNION DISTINCT. Does not need to buffer: sends each row to the + * output as it arrives, eliminating duplicates on the fly. */ + private void unionDistinct() throws InterruptedException { + final Set seen = new HashSet<>(); + for (Source source : sources) { + Row row; + while ((row = source.receive()) != null) { + if (seen.add(row)) { + sink.send(row); + } + } + } + } + + /** Evaluates INTERSECT ALL by counting each value's occurrences in every + * input and emitting it the minimum number of times. 'min' holds the + * smallest count seen across the inputs processed so far; 'current' counts + * the occurrences in the input being processed. */ + private void intersectAll() throws InterruptedException { + final Map counts = new HashMap<>(); + Row row; + final Source first = sources.get(0); + while ((row = first.receive()) != null) { + counts.computeIfAbsent(row, k -> new CountPair()).min++; + } + final int last = sources.size() - 1; + for (int i = 1; i < last && !counts.isEmpty(); i++) { + final Source source = sources.get(i); + while ((row = source.receive()) != null) { + final CountPair pair = counts.get(row); + if (pair != null) { + pair.current++; + } + } + // Reduce the running minimum, dropping values absent from this input. + counts.values().removeIf(pair -> { + if (pair.current == 0) { + return true; + } + pair.min = Math.min(pair.min, pair.current); + pair.current = 0; + return false; + }); + } + // Last input: count occurrences and emit each value that occurs in it + // min(min, current) times. + if (counts.isEmpty()) { + return; + } + final Source source = sources.get(last); + while ((row = source.receive()) != null) { + final CountPair pair = counts.get(row); + if (pair != null) { + pair.current++; + } + } + for (Map.Entry entry : counts.entrySet()) { + final CountPair pair = entry.getValue(); + if (pair.current > 0) { + for (int n = Math.min(pair.min, pair.current); n > 0; n--) { + sink.send(entry.getKey()); + } + } + } + } + + /** Evaluates INTERSECT DISTINCT by retaining, for each successive input, the + * rows it has in common with the result so far. Inputs after the first are + * streamed, so only the result set (which only shrinks) is held in memory. */ + private void intersectDistinct() throws InterruptedException { + Set result = read(sources.get(0)); + for (int i = 1; i < sources.size() && !result.isEmpty(); i++) { + final Source source = sources.get(i); + final Set next = new HashSet<>(); + Row row; + while ((row = source.receive()) != null) { + if (result.contains(row)) { + next.add(row); + } + } + result = next; + } + for (Row r : result) { + sink.send(r); + } + } + + /** Evaluates EXCEPT ALL by counting occurrences of each value in a map. A row + * from the first input increments its value's count; a row from a later input + * decrements it, and a value whose count reaches zero is removed. After all + * inputs have been read, emit each surviving value as many times as its + * remaining count. */ + private void minusAll() throws InterruptedException { + final Map counts = new HashMap<>(); + Row row; + final Source first = sources.get(0); + while ((row = first.receive()) != null) { + counts.computeIfAbsent(row, k -> new Count()).i++; + } + for (int i = 1; i < sources.size() && !counts.isEmpty(); i++) { + final Source source = sources.get(i); + while ((row = source.receive()) != null) { + final Count count = counts.get(row); + if (count != null && --count.i == 0) { + counts.remove(row); + } + } + } + for (Map.Entry entry : counts.entrySet()) { + for (int n = entry.getValue().i; n > 0; n--) { + sink.send(entry.getKey()); + } + } + } + + /** Evaluates EXCEPT DISTINCT by removing from the running result every row + * that occurs in a later input. Later inputs are streamed, so only the result + * set is held in memory. */ + private void minusDistinct() throws InterruptedException { + final Set result = read(sources.get(0)); + for (int i = 1; i < sources.size() && !result.isEmpty(); i++) { + final Source source = sources.get(i); + Row row; + while ((row = source.receive()) != null) { + result.remove(row); + } + } + for (Row r : result) { + sink.send(r); + } + } + + /** Reads a single input into a set, eliminating duplicates. */ + private static Set read(Source source) { + final Set rows = new HashSet<>(); + Row row; + while ((row = source.receive()) != null) { + rows.add(row); + } + return rows; + } + + /** Mutable count of the occurrences of a row, used by {@link #minusAll()}. */ + private static class Count { + int i; + } + + /** Mutable pair of counts used by {@link #intersectAll()}: the minimum + * multiplicity of a row across the inputs seen so far, and its count in the + * input currently being processed. */ + private static class CountPair { + int min; + int current; + } } diff --git a/core/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java b/core/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java index 48cb1f052139..d47adb17b5e0 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java @@ -1076,20 +1076,7 @@ public TrimResult trimFields( return result(setOp, mapping); } - switch (setOp.kind) { - case UNION: - relBuilder.union(setOp.all, setOp.getInputs().size()); - break; - case INTERSECT: - relBuilder.intersect(setOp.all, setOp.getInputs().size()); - break; - case EXCEPT: - assert setOp.getInputs().size() == 2; - relBuilder.minus(setOp.all); - break; - default: - throw new AssertionError("unknown setOp " + setOp); - } + relBuilder.union(true, setOp.getInputs().size()); return result(relBuilder.build(), mapping, setOp); } diff --git a/core/src/test/java/org/apache/calcite/test/InterpreterTest.java b/core/src/test/java/org/apache/calcite/test/InterpreterTest.java index 1409b02d4a9f..7470e2163649 100644 --- a/core/src/test/java/org/apache/calcite/test/InterpreterTest.java +++ b/core/src/test/java/org/apache/calcite/test/InterpreterTest.java @@ -471,7 +471,7 @@ private static void assertRows(Interpreter interpreter, + "(select x, y from (values (1, 'a'), (2, 'b'), (2, 'b'), (3, 'c')) as t(x, y))\n" + "except all\n" + "(select x, y from (values (1, 'a'), (2, 'c'), (4, 'x')) as t2(x, y))"; - sql(sql).returnsRows("[2, b]", "[2, b]", "[3, c]"); + sql(sql).returnsRowsUnordered("[2, b]", "[2, b]", "[3, c]"); } @Test void testDuplicateRowInterpretMinusAll() { diff --git a/core/src/test/java/org/apache/calcite/test/JdbcTest.java b/core/src/test/java/org/apache/calcite/test/JdbcTest.java index a3c004c441ad..dbeb4f7a4675 100644 --- a/core/src/test/java/org/apache/calcite/test/JdbcTest.java +++ b/core/src/test/java/org/apache/calcite/test/JdbcTest.java @@ -8249,11 +8249,10 @@ void checkCalciteSchemaGetSubSchemaMap(boolean cache) { + " BindableTableScan(table=[[hr, emps]])\n" + " BindableProject(empid=[$0], deptno=[$1])\n" + " BindableTableScan(table=[[hr, emps]])") - .returns("" - + "empid=150; deptno=10\n" - + "empid=100; deptno=10\n" - + "empid=200; deptno=20\n" - + "empid=110; deptno=10\n"); + .returnsUnordered("empid=150; deptno=10", + "empid=100; deptno=10", + "empid=200; deptno=20", + "empid=110; deptno=10"); } } diff --git a/core/src/test/resources/sql/interpreter.iq b/core/src/test/resources/sql/interpreter.iq new file mode 100644 index 000000000000..1b33d63c21a8 --- /dev/null +++ b/core/src/test/resources/sql/interpreter.iq @@ -0,0 +1,171 @@ +# interpreter.iq - Queries executed by the interpreter +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Setting 'bindable' makes the planner target BindableConvention. Operators +# that have no native bindable implementation - including Minus, Intersect and +# Union - are executed by the interpreter (org.apache.calcite.interpreter); see +# Bindables and SetOpNode. +!use blank +!set outputformat mysql +!set bindable true + +# A project and filter, to confirm queries run via the interpreter. +select i, i * 10 as j +from (values 1, 2, 3, 4) as t (i) +where i <> 2 +order by i; ++---+----+ +| I | J | ++---+----+ +| 1 | 10 | +| 3 | 30 | +| 4 | 40 | ++---+----+ +(3 rows) + +!ok + +# The plan uses bindable/interpreter operators, not enumerable ones. +BindableProject(I=[$0], J=[*($0, 10)]) + BindableFilter(condition=[<>($0, 2)]) + BindableValues(tuples=[[{ 1 }, { 2 }, { 3 }, { 4 }]]) +!plan + +# Aggregate. +select count(*) as c, sum(i) as s, min(i) as mn, max(i) as mx +from (values 1, 2, 3, 4) as t (i); ++---+----+----+----+ +| C | S | MN | MX | ++---+----+----+----+ +| 4 | 10 | 1 | 4 | ++---+----+----+----+ +(1 row) + +!ok + +# Join. +select t.i, u.y +from (values (1, 'a'), (2, 'b'), (3, 'c')) as t (i, x) +join (values (1, 'p'), (3, 'q')) as u (j, y) on t.i = u.j +order by t.i; ++---+---+ +| I | Y | ++---+---+ +| 1 | p | +| 3 | q | ++---+---+ +(2 rows) + +!ok + +# Multi-input set operations. +# +# Chained set operations are merged into a single n-input SetOp, so the +# interpreter's SetOpNode must evaluate every input, not just the first two. +# [CALCITE-7628] In the interpreter, MINUS with 3 or more inputs returns wrong +# result + +# Except, three inputs +select i from ( + values 1, 2, 3 except values 3, 4, 5 except values 4, 5, 1 +) as t (i) +order by i; ++---+ +| I | ++---+ +| 2 | ++---+ +(1 row) + +!ok + +# A single three-input BindableMinus confirms the interpreter path. +BindableSort(sort0=[$0], dir0=[ASC]) + BindableMinus(all=[false]) + BindableValues(tuples=[[{ 1 }, { 2 }, { 3 }]]) + BindableValues(tuples=[[{ 3 }, { 4 }, { 5 }]]) + BindableValues(tuples=[[{ 4 }, { 5 }, { 1 }]]) +!plan + +# Except all, three inputs +select i from ( + values 1, 1, 1, 2, 2, 3 except all values 1 except all values 1, 2 +) as t (i) +order by i; ++---+ +| I | ++---+ +| 1 | +| 2 | +| 3 | ++---+ +(3 rows) + +!ok + +# Intersect, three inputs +select i from ( + values 1, 2, 3, 4 intersect values 2, 3, 4, 5 intersect values 3, 4, 5, 6 +) as t (i) +order by i; ++---+ +| I | ++---+ +| 3 | +| 4 | ++---+ +(2 rows) + +!ok + +# Intersect all, three inputs +select i from ( + values 1, 1, 2, 2, 3 intersect all values 1, 2, 2, 3, 3 + intersect all values 1, 2, 3, 3 +) as t (i) +order by i; ++---+ +| I | ++---+ +| 1 | +| 2 | +| 3 | ++---+ +(3 rows) + +!ok + +# Union, three inputs +select i from ( + values 1, 2 union values 2, 3 union values 3, 4 +) as t (i) +order by i; ++---+ +| I | ++---+ +| 1 | +| 2 | +| 3 | +| 4 | ++---+ +(4 rows) + +!ok + +!set bindable false + +# End interpreter.iq From 57209cbc7d13d862592bd184ef5dd3cdad7220f9 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Mon, 29 Jun 2026 13:56:07 -0700 Subject: [PATCH 355/562] [CALCITE-7630] BETWEEN unparses incorrectly when left side contains a BETWEEN expression Signed-off-by: Mihai Budiu --- .../org/apache/calcite/sql/fun/SqlBetweenOperator.java | 4 ++++ .../org/apache/calcite/sql/test/SqlPrettyWriterTest.java | 9 +++++++++ .../org/apache/calcite/sql/test/SqlPrettyWriterTest.xml | 5 +++++ 3 files changed, 18 insertions(+) diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlBetweenOperator.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlBetweenOperator.java index 2d77fd9df7b1..7d11e510a223 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlBetweenOperator.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlBetweenOperator.java @@ -269,6 +269,10 @@ private static class AndFinder extends SqlBasicVisitor { if (operator == SqlStdOperatorTable.AND) { throw Util.FoundOne.NULL; } + // A BETWEEN expression contains an implicit AND keyword + if (call.getKind() == SqlKind.BETWEEN) { + throw Util.FoundOne.NULL; + } return super.visit(call); } diff --git a/core/src/test/java/org/apache/calcite/sql/test/SqlPrettyWriterTest.java b/core/src/test/java/org/apache/calcite/sql/test/SqlPrettyWriterTest.java index 03c65fce99a9..4bef9b541d46 100644 --- a/core/src/test/java/org/apache/calcite/sql/test/SqlPrettyWriterTest.java +++ b/core/src/test/java/org/apache/calcite/sql/test/SqlPrettyWriterTest.java @@ -314,6 +314,15 @@ private SqlPrettyWriterFixture tableDotStar() { // space } + /** Test case for [CALCITE-7630] + * BETWEEN unparses incorrectly when left side contains a BETWEEN expression. */ + @Test void testBetweenAnd3() { + expr("a not between (b between c and d) and e") + .expectingFormatted( + "`A` NOT BETWEEN ASYMMETRIC (`B` BETWEEN ASYMMETRIC `C` AND `D`) AND `E`") + .check(); + } + @Test void testCast() { expr("cast(x + y as decimal(5, 10))") .expectingFormatted("CAST(`X` + `Y` AS DECIMAL(5, 10))") diff --git a/core/src/test/resources/org/apache/calcite/sql/test/SqlPrettyWriterTest.xml b/core/src/test/resources/org/apache/calcite/sql/test/SqlPrettyWriterTest.xml index bf4d3a76b878..32f337f0f4c1 100644 --- a/core/src/test/resources/org/apache/calcite/sql/test/SqlPrettyWriterTest.xml +++ b/core/src/test/resources/org/apache/calcite/sql/test/SqlPrettyWriterTest.xml @@ -21,6 +21,11 @@ + + + + + Date: Mon, 29 Jun 2026 13:56:33 -0700 Subject: [PATCH 356/562] Unparse should not use sep unless inside a list Signed-off-by: Mihai Budiu --- core/src/main/java/org/apache/calcite/sql/SqlStarReplace.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/calcite/sql/SqlStarReplace.java b/core/src/main/java/org/apache/calcite/sql/SqlStarReplace.java index b5149a640379..28b5f5ab4243 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlStarReplace.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlStarReplace.java @@ -76,7 +76,7 @@ public SqlNodeList getReplaceList() { @Override public void unparse(SqlWriter writer, int leftPrec, int rightPrec) { starIdentifier.unparse(writer, leftPrec, rightPrec); - writer.sep("REPLACE"); + writer.keyword("REPLACE"); final SqlWriter.Frame frame = writer.startList("(", ")"); replaceList.unparse(writer, 0, 0); writer.endList(frame); From c1cd5b278173b29a58a9d4b4cc918d0c61a17cd7 Mon Sep 17 00:00:00 2001 From: Takaaki Nakama Date: Thu, 25 Jun 2026 15:39:59 +0900 Subject: [PATCH 357/562] [CALCITE-7546] NullPointerException in SqlToRelConverter for UNNEST(array) AS alias under conformance with allowAliasUnnestItems=true Under a SqlConformance where allowAliasUnnestItems() is true (SqlConformanceEnum.PRESTO and user conformances overriding the flag), the AS branch in convertFrom passes fieldNames=null to convertUnnest when the AS clause omits a column list. The PRESTO-only branch then called requireNonNull(fieldNames, "fieldNames") and threw NPE. Fall back to default item aliases derived from SqlUtil#deriveAliasFromOrdinal, matching the names that SqlUnnestOperator#inferReturnType uses during validation. This keeps the relational row type aligned with the validator's underlying namespace so both struct and scalar element types resolve correctly. Add tests for struct array, scalar array, and the exact array literal reproduction from the issue ("SELECT t FROM UNNEST(ARRAY[1, 2, 3]) AS t"). --- .../calcite/sql2rel/SqlToRelConverter.java | 13 ++++- .../calcite/test/SqlToRelConverterTest.java | 33 +++++++++++++ .../calcite/test/SqlToRelConverterTest.xml | 48 +++++++++++++++++++ 3 files changed, 93 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java index 507c14428829..ed6d52491a42 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java @@ -2857,10 +2857,21 @@ private void convertUnnest(Blackboard bb, SqlCall call, @Nullable List f RelNode uncollect; try { if (validator().config().conformance().allowAliasUnnestItems()) { + // Without an AS column list, mirror SqlUnnestOperator#inferReturnType + // so Uncollect's row type stays aligned with the validator. + List itemAliases; + if (fieldNames != null) { + itemAliases = fieldNames; + } else { + itemAliases = new ArrayList<>(nodes.size()); + for (int i = 0; i < nodes.size(); i++) { + itemAliases.add(SqlUtil.deriveAliasFromOrdinal(i)); + } + } uncollect = relBuilder .push(child) .project(exprs) - .uncollect(requireNonNull(fieldNames, "fieldNames"), operator.withOrdinality) + .uncollect(itemAliases, operator.withOrdinality) .build(); } else { // REVIEW danny 2020-04-26: should we unify the normal field aliases and diff --git a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java index c9f409f8c367..1d5833c5160b 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java @@ -1967,6 +1967,39 @@ public static void checkActualAndReferenceFiles() { sql(sql).withConformance(SqlConformanceEnum.BIG_QUERY).ok(); } + /** + * Test case for + * [CALCITE-7546] + * NullPointerException in SqlToRelConverter for UNNEST(array) AS alias under + * conformance with allowAliasUnnestItems=true. + */ + @Test void testAliasUnnestArrayPlanWithoutColumnList() { + final String sql = "select d.deptno, e.empno\n" + + "from dept_nested_expanded as d,\n" + + " UNNEST(d.employees) as e"; + sql(sql).withConformance(SqlConformanceEnum.PRESTO).ok(); + } + + @Test void testAliasUnnestScalarArrayPlanWithoutColumnList() { + final String sql = "select d.deptno, a\n" + + "from dept_nested_expanded as d,\n" + + " UNNEST(d.admins) as a"; + sql(sql).withConformance(SqlConformanceEnum.PRESTO).ok(); + } + + /** + * Test case for + * [CALCITE-7546] + * NullPointerException in SqlToRelConverter for UNNEST(array) AS alias under + * conformance with allowAliasUnnestItems=true, using the exact array + * literal reproduction from the issue. + */ + @Test void testAliasUnnestArrayLiteralPlanWithoutColumnList() { + final String sql = "select t\n" + + "from UNNEST(ARRAY[1, 2, 3]) as t"; + sql(sql).withConformance(SqlConformanceEnum.PRESTO).ok(); + } + @Test void testArrayOfRecord() { sql("select employees[1].detail.skills[2+3].desc from dept_nested").ok(); } diff --git a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml index b5361cbbd35d..90b3c09a2432 100644 --- a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml @@ -325,6 +325,20 @@ LogicalProject(A=[$0], B=[$1], C=[$2], DEPTNO=[$3], NAME=[$4]) LogicalProject(A=[$2], B=[$1], C=[$0]) LogicalValues(tuples=[[{ 1, 2, 3 }]]) LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) +]]> + + + + + + + + @@ -396,6 +410,40 @@ from dept_nested_expanded as d, UNNEST(d.employees) as t(employee)]]> + + + + + + + + + + + + + + + + Date: Fri, 26 Jun 2026 13:51:19 +0200 Subject: [PATCH 358/562] [CALCITE-7626] Pass user-specified JAVA_OPTS to sqlline/sqlsh --- sqlline | 2 +- sqlline.bat | 4 +++- sqlsh | 2 +- sqlsh.bat | 9 +++++---- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/sqlline b/sqlline index 2da73e43c166..edb4be54861a 100755 --- a/sqlline +++ b/sqlline @@ -49,6 +49,6 @@ if [ "$cygwin" ]; then VM_OPTS=-Djline.terminal=jline.UnixTerminal fi -export JAVA_OPTS=-Djavax.xml.parsers.DocumentBuilderFactory=com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl +JAVA_OPTS="-Djavax.xml.parsers.DocumentBuilderFactory=com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl ${JAVA_OPTS}" exec java -Xmx1g $VM_OPTS $JAVA_OPTS -jar "$root/build/libs/sqllineClasspath.jar" "$@" diff --git a/sqlline.bat b/sqlline.bat index d98fbd62a4f2..d2fe22b0d3f6 100644 --- a/sqlline.bat +++ b/sqlline.bat @@ -33,4 +33,6 @@ if not defined CACHE_SQLLINE_CLASSPATH ( ) if not exist "%CP%" (call "%DIRNAME%\gradlew" --console plain -q :buildSqllineClasspath) -java -Xmx1g -jar "%CP%" %* +set JAVA_OPTS=-Djavax.xml.parsers.DocumentBuilderFactory=com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl %JAVA_OPTS% + +java -Xmx1g %JAVA_OPTS% -jar "%CP%" %* diff --git a/sqlsh b/sqlsh index 1904d9dd3bed..437060a249f6 100755 --- a/sqlsh +++ b/sqlsh @@ -43,6 +43,6 @@ if [ "x$CACHE_SQLLINE_CLASSPATH" != "xY" ] || [ ! -f "$CP" ]; then fi VM_OPTS= -export JAVA_OPTS=-Djavax.xml.parsers.DocumentBuilderFactory=com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl +JAVA_OPTS="-Djavax.xml.parsers.DocumentBuilderFactory=com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl ${JAVA_OPTS}" exec java $VM_OPTS -cp "${CP}" $JAVA_OPTS org.apache.calcite.adapter.os.SqlShell "$@" diff --git a/sqlsh.bat b/sqlsh.bat index 45d7187b0b4e..2ad45c54392b 100644 --- a/sqlsh.bat +++ b/sqlsh.bat @@ -16,10 +16,9 @@ :: limitations under the License. :: -:: sqlline.bat - Windows script to launch SQL shell +:: sqlsh.bat - Windows script to launch SQL shell :: Example: -:: > sqlline.bat -:: sqlline> !connect jdbc:calcite: admin admin +:: > sqlsh.bat select * from du order by 1 limit 3 :: The script updates the classpath on each execution, :: You might add CACHE_SQLLINE_CLASSPATH environment variable to cache it @@ -33,4 +32,6 @@ if not defined CACHE_SQLLINE_CLASSPATH ( ) if not exist "%CP%" (call "%DIRNAME%\gradlew" --console plain -q :buildSqllineClasspath) -java -Xmx1g -cp "%CP%" org.apache.calcite.adapter.os.SqlShell %* +set JAVA_OPTS=-Djavax.xml.parsers.DocumentBuilderFactory=com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl %JAVA_OPTS% + +java -Xmx1g -cp "%CP%" %JAVA_OPTS% org.apache.calcite.adapter.os.SqlShell %* From 3616b3c94d7e7467c63b2c7349f3c3a0ec211b26 Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Wed, 1 Jul 2026 13:56:52 +0800 Subject: [PATCH 359/562] Site: Update Sergey Nuyanzin to PMC Signed-off-by: xiedeyantu Co-authored-by: Sergey Nuyanzin --- site/_data/contributors.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/site/_data/contributors.yml b/site/_data/contributors.yml index a0a737eac889..c64c356bf590 100644 --- a/site/_data/contributors.yml +++ b/site/_data/contributors.yml @@ -315,8 +315,8 @@ - name: Sergey Nuyanzin apacheId: snuyanzin githubId: snuyanzin - org: EPAM - role: Committer + org: Confluent + role: PMC - name: Shuyi Chen apacheId: shuyichen githubId: suez1224 From b5ba85b71a3587ceeaa6a9fa32dd93fd59239520 Mon Sep 17 00:00:00 2001 From: liuzhengri <1289206629@qq.com> Date: Wed, 1 Jul 2026 01:46:18 +0800 Subject: [PATCH 360/562] [CALCITE-7632] Replace java.util.Stack with java.util.ArrayDeque in Pattern and TopDownRuleDriver Replace java.util.Stack (which extends Vector and provides unnecessary synchronized access) with java.util.ArrayDeque in: - runtime/Pattern.java: PatternBuilder.stack field - plan/volcano/TopDownRuleDriver.java: tasks field Also remove @SuppressWarnings("JdkObsolete") annotations and TODO comments that requested this change. Stack is officially discouraged by the Java documentation, which recommends using Deque implementations like ArrayDeque instead. ArrayDeque is faster for single-threaded use since it avoids the synchronization overhead inherited from Vector. --- .../apache/calcite/plan/volcano/TopDownRuleDriver.java | 10 +++++----- .../main/java/org/apache/calcite/runtime/Pattern.java | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/plan/volcano/TopDownRuleDriver.java b/core/src/main/java/org/apache/calcite/plan/volcano/TopDownRuleDriver.java index 8e25a8e45b5d..6940b0b1ae50 100644 --- a/core/src/main/java/org/apache/calcite/plan/volcano/TopDownRuleDriver.java +++ b/core/src/main/java/org/apache/calcite/plan/volcano/TopDownRuleDriver.java @@ -28,12 +28,13 @@ import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; +import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Collection; +import java.util.Deque; import java.util.HashSet; import java.util.List; import java.util.Set; -import java.util.Stack; import java.util.function.Predicate; import static java.util.Objects.requireNonNull; @@ -47,7 +48,6 @@ * A Task is a piece of work to be executed, it may apply some rules * or schedule other tasks. */ -@SuppressWarnings("JdkObsolete") class TopDownRuleDriver implements RuleDriver { private static final Logger LOGGER = CalciteTrace.getPlannerTaskTracer(); @@ -62,7 +62,7 @@ class TopDownRuleDriver implements RuleDriver { /** * All tasks waiting for execution. */ - private final Stack tasks = new Stack<>(); // TODO: replace with Deque + private final Deque tasks = new ArrayDeque<>(); /** * A task that is currently applying and may generate new RelNode. @@ -350,7 +350,7 @@ private class OptimizeGroup implements Task { for (RelNode rel : physicals) { Task task = getOptimizeInputTask(rel, group); if (task != null) { - tasks.add(task); + tasks.push(task); } } } @@ -602,7 +602,7 @@ private class ApplyRule implements GeneratorTask { requireNonNull(group.getTraitSet().getConvention(), () -> "convention for " + group)))); if (match != null) { - tasks.add(new ApplyRule(match, group, false)); + tasks.push(new ApplyRule(match, group, false)); } return null; } diff --git a/core/src/main/java/org/apache/calcite/runtime/Pattern.java b/core/src/main/java/org/apache/calcite/runtime/Pattern.java index 6fb331d80caa..dfeaac36345c 100644 --- a/core/src/main/java/org/apache/calcite/runtime/Pattern.java +++ b/core/src/main/java/org/apache/calcite/runtime/Pattern.java @@ -18,7 +18,8 @@ import com.google.common.collect.ImmutableList; -import java.util.Stack; +import java.util.ArrayDeque; +import java.util.Deque; import java.util.stream.Collectors; import static com.google.common.base.Preconditions.checkArgument; @@ -68,9 +69,8 @@ enum Op { } /** Builds a pattern expression. */ - @SuppressWarnings("JdkObsolete") class PatternBuilder { - final Stack stack = new Stack<>(); // TODO: replace with Deque + final Deque stack = new ArrayDeque<>(); private PatternBuilder() {} From dee6545bae9db6d8531912586cee0c5937faac2f Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Wed, 1 Jul 2026 20:20:54 +0800 Subject: [PATCH 361/562] Re-enable [CALCITE-685] Correlated scalar sub-query in SELECT clause throws Signed-off-by: xiedeyantu --- .../org/apache/calcite/test/JdbcTest.java | 33 ++++++++++++++----- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/core/src/test/java/org/apache/calcite/test/JdbcTest.java b/core/src/test/java/org/apache/calcite/test/JdbcTest.java index dbeb4f7a4675..8ebc5b87c966 100644 --- a/core/src/test/java/org/apache/calcite/test/JdbcTest.java +++ b/core/src/test/java/org/apache/calcite/test/JdbcTest.java @@ -5500,7 +5500,6 @@ private CalciteAssert.AssertQuery predicate(String foo) { /** Test case for * [CALCITE-685] * Correlated scalar sub-query in SELECT clause throws. */ - @Disabled("[CALCITE-685]") @Test void testCorrelatedScalarSubQuery() { final String sql = "select e.department_id, sum(e.employee_id),\n" + " ( select sum(e2.employee_id)\n" @@ -5508,19 +5507,35 @@ private CalciteAssert.AssertQuery predicate(String foo) { + " where e.department_id = e2.department_id\n" + " )\n" + "from employee e\n" - + "group by e.department_id\n"; - final String explain = "EnumerableNestedLoopJoin(condition=[true], joinType=[left])\n" - + " EnumerableAggregate(group=[{7}], EXPR$1=[$SUM0($0)])\n" - + " EnumerableTableScan(table=[[foodmart2, employee]])\n" - + " EnumerableAggregate(group=[{}], EXPR$0=[SUM($0)])\n" - + " EnumerableCalc(expr#0..16=[{inputs}], expr#17=[$cor0], expr#18=[$t17.department_id], expr#19=[=($t18, $t7)], employee_id=[$t0], department_id=[$t7], $condition=[$t19])\n" - + " EnumerableTableScan(table=[[foodmart2, employee]])\n"; + + "group by e.department_id\n" + + "order by e.department_id"; + final String explain = "EnumerableCalc(expr#0..3=[{inputs}], proj#0..1=[{exprs}], " + + "EXPR$0=[$t3])\n" + + " EnumerableMergeJoin(condition=[=($0, $2)], joinType=[left])\n" + + " EnumerableSort(sort0=[$0], dir0=[ASC])\n" + + " EnumerableAggregate(group=[{7}], EXPR$1=[$SUM0($0)])\n" + + " EnumerableTableScan(table=[[foodmart2, employee]])\n" + + " EnumerableSort(sort0=[$0], dir0=[ASC])\n" + + " EnumerableAggregate(group=[{7}], EXPR$0=[$SUM0($0)])\n" + + " EnumerableTableScan(table=[[foodmart2, employee]])\n"; CalciteAssert.that() .with(CalciteAssert.Config.FOODMART_CLONE) .with(Lex.JAVA) .query(sql) .explainContains(explain) - .returnsCount(0); + .returnsOrdered( + "department_id=1; EXPR$1=75; EXPR$2=75", + "department_id=2; EXPR$1=160; EXPR$2=160", + "department_id=3; EXPR$1=126; EXPR$2=126", + "department_id=4; EXPR$1=87; EXPR$2=87", + "department_id=5; EXPR$1=398; EXPR$2=398", + "department_id=11; EXPR$1=44166; EXPR$2=44166", + "department_id=14; EXPR$1=8859; EXPR$2=8859", + "department_id=15; EXPR$1=133341; EXPR$2=133341", + "department_id=16; EXPR$1=160636; EXPR$2=160636", + "department_id=17; EXPR$1=137346; EXPR$2=137346", + "department_id=18; EXPR$1=165879; EXPR$2=165879", + "department_id=19; EXPR$1=17670; EXPR$2=17670"); } @Test void testLeftJoin() { From bd226072c3396cb0769c8b3a0cc8c0591fa606e5 Mon Sep 17 00:00:00 2001 From: xuzifu666 <1206332514@qq.com> Date: Mon, 29 Jun 2026 20:45:12 +0800 Subject: [PATCH 362/562] [CALCITE-5261] UNION(ALL) inside of the CURSOR throws an exception while validating the query --- .../calcite/sql/fun/SqlCursorConstructor.java | 8 +-- .../calcite/sql/validate/SqlValidator.java | 8 +-- .../sql/validate/SqlValidatorImpl.java | 50 +++++++++++++------ .../calcite/test/SqlToRelConverterTest.java | 14 ++++++ .../apache/calcite/test/SqlValidatorTest.java | 36 +++++++++++++ .../calcite/test/SqlToRelConverterTest.xml | 32 ++++++++++++ 6 files changed, 125 insertions(+), 23 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlCursorConstructor.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlCursorConstructor.java index 7d23c4123eab..cb6fb9ae1490 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlCursorConstructor.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlCursorConstructor.java @@ -19,7 +19,7 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.sql.SqlCall; import org.apache.calcite.sql.SqlKind; -import org.apache.calcite.sql.SqlSelect; +import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlSpecialOperator; import org.apache.calcite.sql.SqlWriter; import org.apache.calcite.sql.type.OperandTypes; @@ -50,9 +50,9 @@ public SqlCursorConstructor() { SqlValidator validator, SqlValidatorScope scope, SqlCall call) { - SqlSelect subSelect = call.operand(0); - validator.declareCursor(subSelect, scope); - subSelect.validateExpr(validator, scope); + final SqlNode query = call.operand(0); + validator.declareCursor(query, scope); + query.validateExpr(validator, scope); return super.deriveType(validator, scope, call); } diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidator.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidator.java index 3846ed5daf67..813247376d39 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidator.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidator.java @@ -635,12 +635,12 @@ SqlNodeList expandStar(SqlNodeList selectList, SqlSelect query, SqlValidatorScope getEmptyScope(); /** - * Declares a SELECT expression as a cursor. + * Declares a query expression as a cursor. * - * @param select select expression associated with the cursor - * @param scope scope of the parent query associated with the cursor + * @param query query expression associated with the cursor + * @param scope scope of the parent query associated with the cursor */ - void declareCursor(SqlSelect select, SqlValidatorScope scope); + void declareCursor(SqlNode query, SqlValidatorScope scope); /** * Pushes a new instance of a function call on to a function call stack. diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java index 37f6712e90a7..26faab77bc68 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java @@ -415,27 +415,47 @@ public SqlConformance getConformance() { return new SqlNodeList(list, SqlParserPos.ZERO); } - @Override public void declareCursor(SqlSelect select, + @Override public void declareCursor(SqlNode query, SqlValidatorScope parentScope) { - cursorSet.add(select); + cursorSet.add(query); - // add the cursor to a map that maps the cursor to its select based on + // add the cursor to a map that maps the cursor to its query based on // the position of the cursor relative to other cursors in that call FunctionParamInfo funcParamInfo = requireNonNull(functionCallStack.peek(), "functionCall"); - Map cursorMap = funcParamInfo.cursorPosToSelectMap; + Map cursorMap = funcParamInfo.cursorPosToQueryMap; final int cursorCount = cursorMap.size(); - cursorMap.put(cursorCount, select); + cursorMap.put(cursorCount, query); - // create a namespace associated with the result of the select + // create a namespace associated with the result of the query // that is the argument to the cursor constructor; register it // with a scope corresponding to the cursor - SelectScope cursorScope = - new SelectScope(parentScope, getEmptyScope(), select); - clauseScopes.put(IdPair.of(select, Clause.CURSOR), cursorScope); - final SelectNamespace selectNs = createSelectNamespace(select, select); - final String alias = SqlValidatorUtil.alias(select, nextGeneratedId++); - registerNamespace(cursorScope, alias, selectNs, false); + final SqlValidatorNamespace ns; + final SqlValidatorScope cursorScope; + if (query instanceof SqlSelect) { + SqlSelect select = (SqlSelect) query; + cursorScope = new SelectScope(parentScope, getEmptyScope(), select); + clauseScopes.put(IdPair.of(select, Clause.CURSOR), cursorScope); + ns = createSelectNamespace(select, select); + } else { + final SqlCall call = (SqlCall) query; + cursorScope = new ListScope(parentScope) { + @Override public SqlNode getNode() { + return call; + } + }; + if (query.isA(SqlKind.SET_QUERY)) { + ns = createSetopNamespace(call, call); + } else if (query.getKind() == SqlKind.VALUES) { + ns = new TableConstructorNamespace(this, call, cursorScope, call); + } else if (query.getKind() == SqlKind.WITH) { + ns = new WithNamespace(this, (SqlWith) call, call); + } else { + throw Util.unexpected(query.getKind()); + } + } + final String alias = SqlValidatorUtil.alias(query, nextGeneratedId++); + registerNamespace(cursorScope, alias, ns, false); } @Override public void pushFunctionCall() { @@ -8469,10 +8489,10 @@ public IdInfo(SqlValidatorScope scope, SqlIdentifier id) { protected static class FunctionParamInfo { /** * Maps a cursor (based on its position relative to other cursor - * parameters within a function call) to the SELECT associated with the + * parameters within a function call) to the query associated with the * cursor. */ - public final Map cursorPosToSelectMap; + public final Map cursorPosToQueryMap; /** * Maps a column list parameter to the parent cursor parameter it @@ -8481,7 +8501,7 @@ protected static class FunctionParamInfo { public final Map columnListParamToParentCursorMap; public FunctionParamInfo() { - cursorPosToSelectMap = new HashMap<>(); + cursorPosToQueryMap = new HashMap<>(); columnListParamToParentCursorMap = new HashMap<>(); } } diff --git a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java index 1d5833c5160b..1016acc41078 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java @@ -1886,6 +1886,20 @@ public static void checkActualAndReferenceFiles() { sql(sql).withDecorrelate(false).ok(); } + /** Test case for CURSOR containing UNION ALL. */ + @Test void testCollectionTableWithCursorParamUnion() { + final String sql = "select * from table(dedup(" + + "cursor(select ename from emp union all select ename from emp), 'NAME'))"; + sql(sql).withDecorrelate(false).ok(); + } + + /** Test case for CURSOR containing UNION (distinct). */ + @Test void testCollectionTableWithCursorParamUnionDistinct() { + final String sql = "select * from table(dedup(" + + "cursor(select ename from emp union select ename from emp), 'NAME'))"; + sql(sql).withDecorrelate(false).ok(); + } + @Test void testUnnest() { final String sql = "select*from unnest(multiset[1,2])"; sql(sql).ok(); diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index 0cf1966f99a8..0fb4fa4055b6 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -34,6 +34,7 @@ import org.apache.calcite.sql.SqlFunctionCategory; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlLiteral; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlNodeList; import org.apache.calcite.sql.SqlOperator; @@ -47,6 +48,7 @@ import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.parser.SqlParseException; import org.apache.calcite.sql.parser.SqlParser; +import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.sql.type.ArraySqlType; import org.apache.calcite.sql.type.OperandTypes; import org.apache.calcite.sql.type.ReturnTypes; @@ -63,6 +65,7 @@ import org.apache.calcite.sql.validate.SqlValidator; import org.apache.calcite.sql.validate.SqlValidatorCatalogReader; import org.apache.calcite.sql.validate.SqlValidatorImpl; +import org.apache.calcite.sql.validate.SqlValidatorScope; import org.apache.calcite.sql.validate.SqlValidatorUtil; import org.apache.calcite.test.catalog.CountingFactory; import org.apache.calcite.test.catalog.MockCatalogReaderSimple; @@ -101,12 +104,14 @@ import static org.apache.calcite.test.Matchers.isCharset; +import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.nullValue; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.hasToString; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; import static org.junit.jupiter.api.Assumptions.assumeTrue; @@ -10089,11 +10094,42 @@ void testGroupExpressionEquivalenceParams() { + "`CATALOG`.`SALES`.`EMP` AS `EMP`"); } + /** Test case for + * [CALCITE-5261] + * UNION(ALL) inside of the CURSOR throws an exception while validating the query. + * */ @Test void testCollectionTableWithCursorParam() { sql("select * from table(dedup(cursor(select * from emp),'ename'))") .type("RecordType(VARCHAR(1024) NOT NULL NAME) NOT NULL"); sql("select * from table(dedup(cursor(select * from ^bloop^),'ename'))") .fails("Object 'BLOOP' not found"); + sql("select * from table(dedup(cursor(select ename from emp union all " + + "select ename from emp), 'ename'))") + .type("RecordType(VARCHAR(1024) NOT NULL NAME) NOT NULL"); + sql("select * from table(dedup(cursor(select ename from emp union " + + "select ename from emp), 'ename'))") + .type("RecordType(VARCHAR(1024) NOT NULL NAME) NOT NULL"); + sql("select * from table(dedup(cursor(values ('a'), ('b')), 'COLUMN0'))") + .type("RecordType(VARCHAR(1024) NOT NULL NAME) NOT NULL"); + sql("select * from table(dedup(cursor(with cte as (select ename from emp) " + + "select * from cte), 'ENAME'))") + .type("RecordType(VARCHAR(1024) NOT NULL NAME) NOT NULL"); + } + + @Test void testDeclareCursorUnexpectedKind() { + final SqlValidator validator = fixture().factory.createValidator(); + validator.pushFunctionCall(); + final SqlCall query = + new SqlBasicCall(SqlStdOperatorTable.EXPLICIT_TABLE, + new SqlNodeList( + ImmutableList.of(SqlLiteral.createNull(SqlParserPos.ZERO)), + SqlParserPos.ZERO), + SqlParserPos.ZERO); + final SqlValidatorScope scope = validator.getEmptyScope(); + final AssertionError error = + assertThrows(AssertionError.class, () -> + ((SqlValidatorImpl) validator).declareCursor(query, scope)); + assertThat(error.getMessage(), containsString("EXPLICIT_TABLE")); } @Test void testTemporalTable() { diff --git a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml index 90b3c09a2432..bd6a2246a143 100644 --- a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml @@ -898,6 +898,38 @@ LogicalProject(NAME=[$0]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) LogicalProject(NAME=[$1]) LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) +]]> + + + + + + + + + + + + + + + + From 9eedeedbb0a84c96f8f364aa7f9591dccf6d42ce Mon Sep 17 00:00:00 2001 From: Darpan Date: Tue, 30 Jun 2026 10:05:40 +0530 Subject: [PATCH 363/562] [CALCITE-7631] Introduce a composable RexImplementorTable SPI for operator code generation Enumerable code generation resolved operator implementors only through the RexImpTable singleton, whose sole extension hook is ImplementableFunction for schema user-defined functions. Operators registered through a SqlOperatorTable (custom or dialect operators) therefore had no code-generation or constant-reduction path. Extract a RexImplementorTable interface (the get() lookups for scalar, aggregate, match and table-function operators) that RexImpTable implements, and add RexImplementorTables.chain() so an extension can layer its own implementors ahead of the built-ins -- the code-generation counterpart of SqlOperatorTable. Thread an injectable RexImplementorTable (defaulting to the built-ins) through RexToLixTranslator (scalar code generation) and RexExecutorImpl (constant folding); deprecate the table-less translateProjects/translateCondition overloads and migrate all callers. The match and table-function lookups now return null on a miss so a chained table can fall through. --- .../calcite/adapter/enumerable/EnumUtils.java | 2 +- .../adapter/enumerable/EnumerableCalc.java | 6 +- .../adapter/enumerable/EnumerableMatch.java | 11 +- .../enumerable/EnumerableRelImplementor.java | 7 + .../adapter/enumerable/RexImpTable.java | 33 +-- .../enumerable/RexImplementorTable.java | 56 +++++ .../enumerable/RexImplementorTables.java | 126 ++++++++++ .../enumerable/RexToLixTranslator.java | 81 +++++- .../interpreter/JaninoRexCompiler.java | 4 +- .../apache/calcite/rex/RexExecutorImpl.java | 25 +- .../enumerable/RexImplementorTableTest.java | 230 ++++++++++++++++++ .../calcite/adapter/spark/SparkRules.java | 5 +- 12 files changed, 543 insertions(+), 43 deletions(-) create mode 100644 core/src/main/java/org/apache/calcite/adapter/enumerable/RexImplementorTable.java create mode 100644 core/src/main/java/org/apache/calcite/adapter/enumerable/RexImplementorTables.java create mode 100644 core/src/test/java/org/apache/calcite/adapter/enumerable/RexImplementorTableTest.java diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java index 018ba8bb7551..2ed244dfa885 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java @@ -987,7 +987,7 @@ static Expression generatePredicate( right_, rightPhysType)), implementor.allCorrelateVariables, implementor.getConformance(), - nullable))); + nullable, implementor.getRexImplementorTable()))); Class clazz = nullable ? NullablePredicate2.class : Predicate2.class; return Expressions.lambda(clazz, builder.toBlock(), left_, right_); } diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableCalc.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableCalc.java index e1b1cdcb291e..03cd0e419a0d 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableCalc.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableCalc.java @@ -166,7 +166,8 @@ public static EnumerableCalc create(final RelNode input, typeFactory, builder2, new RexToLixTranslator.InputGetterImpl(input, result.physType), - implementor.allCorrelateVariables, implementor.getConformance()); + implementor.allCorrelateVariables, implementor.getConformance(), + false, implementor.getRexImplementorTable()); builder2.add( Expressions.ifThen( condition, @@ -198,7 +199,8 @@ public static EnumerableCalc create(final RelNode input, physType, DataContext.ROOT, new RexToLixTranslator.InputGetterImpl(input, result.physType), - implementor.allCorrelateVariables); + implementor.allCorrelateVariables, + implementor.getRexImplementorTable()); builder3.add( Expressions.return_( null, physType.record(expressions))); diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMatch.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMatch.java index 2fdd2b41ec9b..77d86cf5d729 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMatch.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMatch.java @@ -247,7 +247,9 @@ private static Expression implementMeasure(RexToLixTranslator translator, case PREV: case CLASSIFIER: matchFunction = (SqlMatchFunction) ((RexCall) value).getOperator(); - matchImplementor = RexImpTable.INSTANCE.get(matchFunction); + matchImplementor = + requireNonNull(RexImpTable.INSTANCE.get(matchFunction), + () -> "no implementor for match function " + matchFunction); // Work with the implementor return matchImplementor.implement(translator, (RexCall) value, @@ -266,7 +268,9 @@ private static Expression implementMeasure(RexToLixTranslator translator, case CLASSIFIER: final RexCall call = (RexCall) operands.get(0); matchFunction = (SqlMatchFunction) call.getOperator(); - matchImplementor = RexImpTable.INSTANCE.get(matchFunction); + matchImplementor = + requireNonNull(RexImpTable.INSTANCE.get(matchFunction), + () -> "no implementor for match function " + matchFunction); // Work with the implementor requireNonNull((PassedRowsInputGetter) translator.inputGetter, "inputGetter") .setIndex(null); @@ -316,7 +320,8 @@ private Expression implementMatcher(EnumerableRelImplementor implementor, builder2, inputGetter1, implementor.allCorrelateVariables, - implementor.getConformance()); + implementor.getConformance(), + false, implementor.getRexImplementorTable()); builder2.add(Expressions.return_(null, condition)); final Expression predicate_ = diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableRelImplementor.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableRelImplementor.java index 53d0296f7a6b..8d27d6f74c56 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableRelImplementor.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableRelImplementor.java @@ -488,6 +488,13 @@ public EnumerableRel.Result result(PhysType physType, BlockStatement block) { SqlConformanceEnum.DEFAULT); } + /** Returns the table of code-generation implementors to use, defaulting to + * the built-in {@link RexImpTable#instance()}. */ + public RexImplementorTable getRexImplementorTable() { + return (RexImplementorTable) map.getOrDefault("_rexImplementorTable", + RexImpTable.INSTANCE); + } + /** Visitor that finds types in an {@link Expression} tree. */ @VisibleForTesting static class TypeFinder extends VisitorImpl { diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java index c78336e172c5..acbd66cb1cc7 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java @@ -557,7 +557,7 @@ * *

      Immutable. */ -public class RexImpTable { +public class RexImpTable implements RexImplementorTable { /** The singleton instance. */ public static final RexImpTable INSTANCE; @@ -569,6 +569,11 @@ public class RexImpTable { INSTANCE = new RexImpTable(builder); } + /** Returns the table of built-in implementors. */ + public static RexImplementorTable instance() { + return INSTANCE; + } + public static final ConstantExpression NULL_EXPR = Expressions.constant(null); public static final ConstantExpression FALSE_EXPR = @@ -1457,7 +1462,9 @@ private static RexCallImplementor createRexCallImplementor( }; } - private static RexCallImplementor wrapAsRexCallImplementor( + /** Wraps a {@link CallImplementor} (for example, one built with + * {@link #createImplementor}) as a {@link RexCallImplementor}. */ + public static RexCallImplementor wrapAsRexCallImplementor( final CallImplementor implementor) { return new AbstractRexCallImplementor("udf", NullPolicy.NONE, false) { @Override Expression implementSafe(RexToLixTranslator translator, @@ -1467,7 +1474,7 @@ private static RexCallImplementor wrapAsRexCallImplementor( }; } - public @Nullable RexCallImplementor get(final SqlOperator operator) { + @Override public @Nullable RexCallImplementor get(final SqlOperator operator) { if (operator instanceof SqlUserDefinedFunction) { org.apache.calcite.schema.Function udf = ((SqlUserDefinedFunction) operator).getFunction(); @@ -1502,7 +1509,7 @@ private static RexCallImplementor wrapAsRexCallImplementor( return null; } - public @Nullable AggImplementor get(final SqlAggFunction aggregation, + @Override public @Nullable AggImplementor get(final SqlAggFunction aggregation, boolean forWindowAggregate) { if (aggregation instanceof SqlUserDefinedAggFunction) { final SqlUserDefinedAggFunction udaf = @@ -1531,24 +1538,18 @@ private static RexCallImplementor wrapAsRexCallImplementor( return aggSupplier.get(); } - public MatchImplementor get(final SqlMatchFunction function) { + @Override public @Nullable MatchImplementor get( + final SqlMatchFunction function) { final Supplier supplier = matchMap.get(function); - if (supplier != null) { - return supplier.get(); - } else { - throw new IllegalStateException("Supplier should not be null"); - } + return supplier != null ? supplier.get() : null; } - public TableFunctionCallImplementor get(final SqlWindowTableFunction operator) { + @Override public @Nullable TableFunctionCallImplementor get( + final SqlWindowTableFunction operator) { final Supplier supplier = tvfImplementorMap.get(operator); - if (supplier != null) { - return supplier.get(); - } else { - throw new IllegalStateException("Supplier should not be null"); - } + return supplier != null ? supplier.get() : null; } static Expression optimize(Expression expression) { diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImplementorTable.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImplementorTable.java new file mode 100644 index 000000000000..9c135bf83a09 --- /dev/null +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImplementorTable.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.adapter.enumerable; + +import org.apache.calcite.adapter.enumerable.RexImpTable.RexCallImplementor; +import org.apache.calcite.sql.SqlAggFunction; +import org.apache.calcite.sql.SqlMatchFunction; +import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.SqlWindowTableFunction; + +import org.checkerframework.checker.nullness.qual.Nullable; + +/** + * Provides the implementor that generates code for calls to an operator. + * + *

      Enumerable code generation translates each operator call into + * {@linkplain org.apache.calcite.linq4j.tree.Expression linq4j code} using an + * implementor. This table looks up that implementor for a scalar operator, an + * aggregate function, a {@code MATCH_RECOGNIZE} function, or a windowed table + * function. + * + *

      A lookup returns {@code null} if this table has no implementor for the + * given operator. + */ +public interface RexImplementorTable { + /** Returns the implementor of a scalar operator, or null if this table has + * none. */ + @Nullable RexCallImplementor get(SqlOperator operator); + + /** Returns the implementor of an aggregate function (in window context when + * {@code forWindowAggregate} is true), or null if this table has none. */ + @Nullable AggImplementor get(SqlAggFunction aggregation, + boolean forWindowAggregate); + + /** Returns the implementor of a {@code MATCH_RECOGNIZE} function, or null if + * this table has none. */ + @Nullable MatchImplementor get(SqlMatchFunction function); + + /** Returns the implementor of a windowed table function, or null if this + * table has none. */ + @Nullable TableFunctionCallImplementor get(SqlWindowTableFunction operator); +} diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImplementorTables.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImplementorTables.java new file mode 100644 index 000000000000..22b3340ee88e --- /dev/null +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImplementorTables.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.adapter.enumerable; + +import org.apache.calcite.adapter.enumerable.RexImpTable.RexCallImplementor; +import org.apache.calcite.sql.SqlAggFunction; +import org.apache.calcite.sql.SqlMatchFunction; +import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.SqlWindowTableFunction; + +import com.google.common.collect.ImmutableList; + +import org.checkerframework.checker.nullness.qual.Nullable; + +import java.util.ArrayList; +import java.util.List; + +/** + * Utilities for {@link RexImplementorTable}. + */ +public abstract class RexImplementorTables { + private RexImplementorTables() { + } + + /** Creates a table that consults each of the given tables in turn, returning + * the first non-null implementor. + * + *

      Earlier tables take precedence: when more than one table has an + * implementor for the same operator, the one earliest in the list is + * returned. Listing {@link RexImpTable#instance()} last makes the built-in + * implementors the fallback. */ + public static RexImplementorTable chain(RexImplementorTable... tables) { + return chain(ImmutableList.copyOf(tables)); + } + + /** Creates a table that consults each of the given tables in turn. + * + * @see #chain(RexImplementorTable...) */ + public static RexImplementorTable chain( + Iterable tables) { + final List list = new ArrayList<>(); + for (RexImplementorTable table : tables) { + addFlattened(list, table); + } + if (list.size() == 1) { + return list.get(0); + } + return new Chain(ImmutableList.copyOf(list)); + } + + private static void addFlattened(List list, + RexImplementorTable table) { + if (table instanceof Chain) { + list.addAll(((Chain) table).tables); + } else { + list.add(table); + } + } + + /** Implementor table that consults a list of tables in order, returning the + * first non-null implementor. */ + private static class Chain implements RexImplementorTable { + final ImmutableList tables; + + Chain(ImmutableList tables) { + this.tables = tables; + } + + @Override public @Nullable RexCallImplementor get(SqlOperator operator) { + for (RexImplementorTable table : tables) { + final RexCallImplementor implementor = table.get(operator); + if (implementor != null) { + return implementor; + } + } + return null; + } + + @Override public @Nullable AggImplementor get(SqlAggFunction aggregation, + boolean forWindowAggregate) { + for (RexImplementorTable table : tables) { + final AggImplementor implementor = + table.get(aggregation, forWindowAggregate); + if (implementor != null) { + return implementor; + } + } + return null; + } + + @Override public @Nullable MatchImplementor get(SqlMatchFunction function) { + for (RexImplementorTable table : tables) { + final MatchImplementor implementor = table.get(function); + if (implementor != null) { + return implementor; + } + } + return null; + } + + @Override public @Nullable TableFunctionCallImplementor get( + SqlWindowTableFunction operator) { + for (RexImplementorTable table : tables) { + final TableFunctionCallImplementor implementor = table.get(operator); + if (implementor != null) { + return implementor; + } + } + return null; + } + } +} diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java index 7d2cb6b7320c..15924844ca9d 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java @@ -127,6 +127,8 @@ public class RexToLixTranslator implements RexVisitor private final @Nullable BlockBuilder staticList; private final @Nullable Function1 correlates; + private final RexImplementorTable implementorTable; + /** * Map from RexLiteral's variable name to its literal, which is often a * ({@link org.apache.calcite.linq4j.tree.ConstantExpression})) @@ -163,7 +165,8 @@ private RexToLixTranslator(@Nullable RexProgram program, @Nullable BlockBuilder staticList, RexBuilder builder, SqlConformance conformance, - @Nullable Function1 correlates) { + @Nullable Function1 correlates, + RexImplementorTable implementorTable) { this.program = program; // may be null this.typeFactory = requireNonNull(typeFactory, "typeFactory"); this.conformance = requireNonNull(conformance, "conformance"); @@ -173,6 +176,23 @@ private RexToLixTranslator(@Nullable RexProgram program, this.staticList = staticList; this.builder = requireNonNull(builder, "builder"); this.correlates = correlates; // may be null + this.implementorTable = requireNonNull(implementorTable, "implementorTable"); + } + + /** + * Translates a {@link RexProgram} to a sequence of expressions and + * declarations, using the built-in implementor table. + * + * @deprecated Use {@link #translateProjects(RexProgram, JavaTypeFactory, SqlConformance, BlockBuilder, BlockBuilder, PhysType, Expression, InputGetter, Function1, RexImplementorTable)}. + */ + @Deprecated // to be removed before 2.0 + public static List translateProjects(RexProgram program, + JavaTypeFactory typeFactory, SqlConformance conformance, + BlockBuilder list, @Nullable BlockBuilder staticList, + @Nullable PhysType outputPhysType, Expression root, + InputGetter inputGetter, @Nullable Function1 correlates) { + return translateProjects(program, typeFactory, conformance, list, staticList, + outputPhysType, root, inputGetter, correlates, RexImpTable.INSTANCE); } /** @@ -189,13 +209,15 @@ private RexToLixTranslator(@Nullable RexProgram program, * @param inputGetter Generates expressions for inputs * @param correlates Provider of references to the values of correlated * variables + * @param implementorTable Table of implementors for operator code generation * @return Sequence of expressions, optional condition */ public static List translateProjects(RexProgram program, JavaTypeFactory typeFactory, SqlConformance conformance, BlockBuilder list, @Nullable BlockBuilder staticList, @Nullable PhysType outputPhysType, Expression root, - InputGetter inputGetter, @Nullable Function1 correlates) { + InputGetter inputGetter, @Nullable Function1 correlates, + RexImplementorTable implementorTable) { List storageTypes = null; if (outputPhysType != null) { final RelDataType rowType = outputPhysType.getRowType(); @@ -205,18 +227,25 @@ public static List translateProjects(RexProgram program, } } return new RexToLixTranslator(program, typeFactory, root, inputGetter, - list, staticList, new RexBuilder(typeFactory), conformance, null) + list, staticList, new RexBuilder(typeFactory), conformance, null, + implementorTable) .setCorrelates(correlates) .translateList(program.getProjectList(), storageTypes); } + /** + * Translates a {@link RexProgram} to a sequence of expressions and + * declarations, using the built-in implementor table. + * + * @deprecated Use {@link #translateProjects(RexProgram, JavaTypeFactory, SqlConformance, BlockBuilder, BlockBuilder, PhysType, Expression, InputGetter, Function1, RexImplementorTable)}. + */ @Deprecated // to be removed before 2.0 public static List translateProjects(RexProgram program, JavaTypeFactory typeFactory, SqlConformance conformance, BlockBuilder list, @Nullable PhysType outputPhysType, Expression root, InputGetter inputGetter, @Nullable Function1 correlates) { return translateProjects(program, typeFactory, conformance, list, null, - outputPhysType, root, inputGetter, correlates); + outputPhysType, root, inputGetter, correlates, RexImpTable.INSTANCE); } public static Expression translateTableFunction(JavaTypeFactory typeFactory, @@ -225,7 +254,8 @@ public static Expression translateTableFunction(JavaTypeFactory typeFactory, PhysType inputPhysType, PhysType outputPhysType) { final RexToLixTranslator translator = new RexToLixTranslator(null, typeFactory, root, null, list, - null, new RexBuilder(typeFactory), conformance, null); + null, new RexBuilder(typeFactory), conformance, null, + RexImpTable.INSTANCE); return translator .translateTableFunction(rexCall, inputEnumerable, inputPhysType, outputPhysType); @@ -237,7 +267,8 @@ public static RexToLixTranslator forAggregation(JavaTypeFactory typeFactory, SqlConformance conformance) { final ParameterExpression root = DataContext.ROOT; return new RexToLixTranslator(null, typeFactory, root, inputGetter, list, - null, new RexBuilder(typeFactory), conformance, null); + null, new RexBuilder(typeFactory), conformance, null, + RexImpTable.INSTANCE); } Expression translate(RexNode expr) { @@ -1218,7 +1249,7 @@ private Expression translateTableFunction(RexCall rexCall, Expression inputEnume PhysType inputPhysType, PhysType outputPhysType) { assert rexCall.getOperator() instanceof SqlWindowTableFunction; TableFunctionCallImplementor implementor = - RexImpTable.INSTANCE.get((SqlWindowTableFunction) rexCall.getOperator()); + implementorTable.get((SqlWindowTableFunction) rexCall.getOperator()); if (implementor == null) { throw Util.needToImplement("implementor of " + rexCall.getOperator().getName()); } @@ -1226,16 +1257,41 @@ private Expression translateTableFunction(RexCall rexCall, Expression inputEnume this, inputEnumerable, rexCall, inputPhysType, outputPhysType); } + /** + * Translates the condition of a {@link RexProgram} to a Java expression, + * using the built-in implementor table. + * + * @deprecated Use {@link #translateCondition(RexProgram, JavaTypeFactory, BlockBuilder, InputGetter, Function1, SqlConformance, boolean, RexImplementorTable)}. + */ + @Deprecated // to be removed before 2.0 public static Expression translateCondition(RexProgram program, JavaTypeFactory typeFactory, BlockBuilder list, InputGetter inputGetter, Function1 correlates, SqlConformance conformance) { return translateCondition(program, typeFactory, list, inputGetter, - correlates, conformance, false); + correlates, conformance, false, RexImpTable.INSTANCE); } + /** + * Translates the condition of a {@link RexProgram} to a Java expression, + * using the built-in implementor table. + * + * @deprecated Use {@link #translateCondition(RexProgram, JavaTypeFactory, BlockBuilder, InputGetter, Function1, SqlConformance, boolean, RexImplementorTable)}. + */ + @Deprecated // to be removed before 2.0 public static Expression translateCondition(RexProgram program, JavaTypeFactory typeFactory, BlockBuilder list, InputGetter inputGetter, Function1 correlates, SqlConformance conformance, boolean nullable) { + return translateCondition(program, typeFactory, list, inputGetter, correlates, + conformance, nullable, RexImpTable.INSTANCE); + } + + /** + * Translates the condition of a {@link RexProgram} to a Java expression. + */ + public static Expression translateCondition(RexProgram program, + JavaTypeFactory typeFactory, BlockBuilder list, InputGetter inputGetter, + Function1 correlates, SqlConformance conformance, + boolean nullable, RexImplementorTable implementorTable) { RexLocalRef condition = program.getCondition(); if (condition == null) { return RexImpTable.TRUE_EXPR; @@ -1243,7 +1299,8 @@ public static Expression translateCondition(RexProgram program, final ParameterExpression root = DataContext.ROOT; RexToLixTranslator translator = new RexToLixTranslator(program, typeFactory, root, inputGetter, list, - null, new RexBuilder(typeFactory), conformance, null); + null, new RexBuilder(typeFactory), conformance, null, + implementorTable); translator = translator.setCorrelates(correlates); return translator.translate( condition, @@ -1264,7 +1321,7 @@ public RexToLixTranslator setBlock(BlockBuilder list) { return this; } return new RexToLixTranslator(program, typeFactory, root, inputGetter, list, - staticList, builder, conformance, correlates); + staticList, builder, conformance, correlates, implementorTable); } public RexToLixTranslator setCorrelates( @@ -1273,7 +1330,7 @@ public RexToLixTranslator setCorrelates( return this; } return new RexToLixTranslator(program, typeFactory, root, inputGetter, list, - staticList, builder, conformance, correlates); + staticList, builder, conformance, correlates, implementorTable); } public Expression getRoot() { @@ -1494,7 +1551,7 @@ private ConstantExpression getTypedNullLiteral(RexLiteral literal) { return RexUtil.expandSearch(builder, program, call).accept(this); } final RexImpTable.RexCallImplementor implementor = - RexImpTable.INSTANCE.get(operator); + implementorTable.get(operator); if (implementor == null) { throw new RuntimeException("cannot translate call " + call); } diff --git a/core/src/main/java/org/apache/calcite/interpreter/JaninoRexCompiler.java b/core/src/main/java/org/apache/calcite/interpreter/JaninoRexCompiler.java index bca4f85ef501..db553ca9fb03 100644 --- a/core/src/main/java/org/apache/calcite/interpreter/JaninoRexCompiler.java +++ b/core/src/main/java/org/apache/calcite/interpreter/JaninoRexCompiler.java @@ -19,6 +19,7 @@ import org.apache.calcite.DataContext; import org.apache.calcite.adapter.enumerable.JavaRowFormat; import org.apache.calcite.adapter.enumerable.PhysTypeImpl; +import org.apache.calcite.adapter.enumerable.RexImpTable; import org.apache.calcite.adapter.enumerable.RexToLixTranslator; import org.apache.calcite.config.CalciteSystemProperty; import org.apache.calcite.jdbc.JavaTypeFactoryImpl; @@ -103,7 +104,8 @@ public JaninoRexCompiler(RexBuilder rexBuilder) { SqlConformanceEnum.DEFAULT; // TODO: get this from implementor final List expressionList = RexToLixTranslator.translateProjects(program, javaTypeFactory, - conformance, list, staticList, null, root, inputGetter, correlates); + conformance, list, staticList, null, root, inputGetter, correlates, + RexImpTable.INSTANCE); Ord.forEach(expressionList, (expression, i) -> list.add( Expressions.statement( diff --git a/core/src/main/java/org/apache/calcite/rex/RexExecutorImpl.java b/core/src/main/java/org/apache/calcite/rex/RexExecutorImpl.java index acfbbdc56d4e..7f06abf58c2f 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexExecutorImpl.java +++ b/core/src/main/java/org/apache/calcite/rex/RexExecutorImpl.java @@ -18,6 +18,8 @@ import org.apache.calcite.DataContext; import org.apache.calcite.adapter.enumerable.EnumUtils; +import org.apache.calcite.adapter.enumerable.RexImpTable; +import org.apache.calcite.adapter.enumerable.RexImplementorTable; import org.apache.calcite.adapter.enumerable.RexToLixTranslator; import org.apache.calcite.adapter.enumerable.RexToLixTranslator.InputGetter; import org.apache.calcite.adapter.java.JavaTypeFactory; @@ -57,20 +59,29 @@ public class RexExecutorImpl implements RexExecutor { private final DataContext dataContext; + private final RexImplementorTable implementorTable; public RexExecutorImpl(DataContext dataContext) { + this(dataContext, RexImpTable.INSTANCE); + } + + public RexExecutorImpl(DataContext dataContext, + RexImplementorTable implementorTable) { this.dataContext = dataContext; + this.implementorTable = implementorTable; } private static String compile(RexBuilder rexBuilder, List constExps, - RexToLixTranslator.InputGetter getter) { + RexToLixTranslator.InputGetter getter, + RexImplementorTable implementorTable) { final RelDataTypeFactory typeFactory = rexBuilder.getTypeFactory(); final RelDataType emptyRowType = typeFactory.builder().build(); - return compile(rexBuilder, constExps, getter, emptyRowType); + return compile(rexBuilder, constExps, getter, emptyRowType, implementorTable); } private static String compile(RexBuilder rexBuilder, List constExps, - RexToLixTranslator.InputGetter getter, RelDataType rowType) { + RexToLixTranslator.InputGetter getter, RelDataType rowType, + RexImplementorTable implementorTable) { final RexProgramBuilder programBuilder = new RexProgramBuilder(rowType, rexBuilder); for (RexNode node : constExps) { @@ -93,7 +104,8 @@ private static String compile(RexBuilder rexBuilder, List constExps, final RexProgram program = programBuilder.getProgram(); final List expressions = RexToLixTranslator.translateProjects(program, javaTypeFactory, - conformance, blockBuilder, null, null, root_, getter, null); + conformance, blockBuilder, null, null, root_, getter, null, + implementorTable); blockBuilder.add( Expressions.return_(null, Expressions.newArrayInit(Object[].class, expressions))); @@ -121,7 +133,8 @@ public static RexExecutable getExecutable(RexBuilder rexBuilder, List e final JavaTypeFactoryImpl typeFactory = new JavaTypeFactoryImpl(rexBuilder.getTypeFactory().getTypeSystem()); final InputGetter getter = new DataContextInputGetter(rowType, typeFactory); - final String code = compile(rexBuilder, exps, getter, rowType); + final String code = + compile(rexBuilder, exps, getter, rowType, RexImpTable.INSTANCE); return new RexExecutable(code, "generated Rex code"); } @@ -155,7 +168,7 @@ public static RexExecutable getExecutable(RexBuilder rexBuilder, List e try { String code = compile(rexBuilder, exps, (list, index, storageType) -> { throw new UnsupportedOperationException(); - }); + }, implementorTable); final RexExecutable executable = new RexExecutable(code, exps); executable.setDataContext(dataContext); diff --git a/core/src/test/java/org/apache/calcite/adapter/enumerable/RexImplementorTableTest.java b/core/src/test/java/org/apache/calcite/adapter/enumerable/RexImplementorTableTest.java new file mode 100644 index 000000000000..6fb3039a61d0 --- /dev/null +++ b/core/src/test/java/org/apache/calcite/adapter/enumerable/RexImplementorTableTest.java @@ -0,0 +1,230 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.adapter.enumerable; + +import org.apache.calcite.DataContext; +import org.apache.calcite.adapter.enumerable.RexImpTable.RexCallImplementor; +import org.apache.calcite.adapter.java.JavaTypeFactory; +import org.apache.calcite.jdbc.JavaTypeFactoryImpl; +import org.apache.calcite.linq4j.tree.BlockBuilder; +import org.apache.calcite.linq4j.tree.Expression; +import org.apache.calcite.linq4j.tree.Expressions; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeSystem; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexProgram; +import org.apache.calcite.rex.RexProgramBuilder; +import org.apache.calcite.sql.SqlAggFunction; +import org.apache.calcite.sql.SqlFunction; +import org.apache.calcite.sql.SqlFunctionCategory; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlMatchFunction; +import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.SqlWindowTableFunction; +import org.apache.calcite.sql.fun.SqlBasicAggFunction; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.type.OperandTypes; +import org.apache.calcite.sql.type.ReturnTypes; +import org.apache.calcite.sql.validate.SqlConformanceEnum; + +import org.checkerframework.checker.nullness.qual.Nullable; +import org.junit.jupiter.api.Test; + +import java.lang.reflect.Type; +import java.util.List; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.hamcrest.CoreMatchers.nullValue; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.sameInstance; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Tests for the {@link RexImplementorTable} SPI, its + * {@link RexImplementorTables#chain composition}, and its use by + * {@link RexToLixTranslator}. + */ +class RexImplementorTableTest { + /** A scalar operator that has no built-in implementor. */ + private static final SqlOperator MY_FN = + new SqlFunction("MY_CUSTOM_FN", SqlKind.OTHER_FUNCTION, + ReturnTypes.BOOLEAN, null, OperandTypes.ANY, + SqlFunctionCategory.USER_DEFINED_FUNCTION); + + /** A sentinel implementor; only its identity matters to these tests. */ + private static final RexCallImplementor SENTINEL = + (translator, call, arguments) -> { + throw new UnsupportedOperationException("sentinel"); + }; + + /** A sentinel aggregate implementor; only its identity matters. */ + private static final AggImplementor AGG_SENTINEL = new AggImplementor() { + @Override public List getStateType(AggContext info) { + throw new UnsupportedOperationException(); + } + + @Override public void implementReset(AggContext info, AggResetContext reset) { + throw new UnsupportedOperationException(); + } + + @Override public void implementAdd(AggContext info, AggAddContext add) { + throw new UnsupportedOperationException(); + } + + @Override public Expression implementResult(AggContext info, + AggResultContext result) { + throw new UnsupportedOperationException(); + } + }; + + /** An aggregate function with no built-in implementor. */ + private static final SqlAggFunction MY_AGG = + SqlBasicAggFunction.create("MY_CUSTOM_AGG", SqlKind.OTHER_FUNCTION, + ReturnTypes.BIGINT, OperandTypes.ANY); + + /** Implementor table that knows a single scalar operator. */ + private static final class SingleScalarTable implements RexImplementorTable { + private final SqlOperator operator; + private final RexCallImplementor implementor; + + SingleScalarTable(SqlOperator operator, RexCallImplementor implementor) { + this.operator = operator; + this.implementor = implementor; + } + + @Override public @Nullable RexCallImplementor get(SqlOperator op) { + return op == operator ? implementor : null; + } + + @Override public @Nullable AggImplementor get(SqlAggFunction aggregation, + boolean forWindowAggregate) { + return null; + } + + @Override public @Nullable MatchImplementor get(SqlMatchFunction function) { + return null; + } + + @Override public @Nullable TableFunctionCallImplementor get( + SqlWindowTableFunction operator) { + return null; + } + } + + /** The built-in table has no implementor for an unregistered operator. */ + @Test void builtinHasNoImplementorForUnknownOperator() { + assertThat(RexImpTable.instance().get(MY_FN), is(nullValue())); + } + + /** A chained extension table supplies the implementor for its own operator, + * while the built-in table still resolves standard operators. */ + @Test void chainResolvesExtensionThenFallsBackToBuiltin() { + final RexImplementorTable chain = + RexImplementorTables.chain(new SingleScalarTable(MY_FN, SENTINEL), + RexImpTable.instance()); + assertThat(chain.get(MY_FN), is(sameInstance(SENTINEL))); + assertThat(chain.get(SqlStdOperatorTable.UPPER), is(notNullValue())); + } + + /** A table earlier in the chain overrides a built-in implementor. */ + @Test void earlierTableOverridesBuiltin() { + final RexImplementorTable chain = + RexImplementorTables.chain( + new SingleScalarTable(SqlStdOperatorTable.UPPER, SENTINEL), + RexImpTable.instance()); + assertThat(chain.get(SqlStdOperatorTable.UPPER), is(sameInstance(SENTINEL))); + } + + /** A single-element chain returns that table itself, with no wrapper. */ + @Test void singleElementChainIsIdentity() { + final RexImplementorTable table = new SingleScalarTable(MY_FN, SENTINEL); + assertThat(RexImplementorTables.chain(table), is(sameInstance(table))); + } + + /** An injected table drives project code generation for an operator that the + * built-in table cannot translate on its own. */ + @Test void injectedTableDrivesProjectCodeGen() { + final JavaTypeFactory typeFactory = + new JavaTypeFactoryImpl(RelDataTypeSystem.DEFAULT); + final RexBuilder rexBuilder = new RexBuilder(typeFactory); + final RelDataType emptyRowType = typeFactory.builder().build(); + final RexNode call = + rexBuilder.makeCall(MY_FN, rexBuilder.makeLiteral(true)); + final RexProgramBuilder programBuilder = + new RexProgramBuilder(emptyRowType, rexBuilder); + programBuilder.addProject(call, "c0"); + final RexProgram program = programBuilder.getProgram(); + final RexToLixTranslator.InputGetter inputGetter = + (list, index, storageType) -> { + throw new UnsupportedOperationException(); + }; + + // The built-in table alone cannot translate MY_FN. + assertThrows(RuntimeException.class, () -> + RexToLixTranslator.translateProjects(program, typeFactory, + SqlConformanceEnum.DEFAULT, new BlockBuilder(), null, null, + DataContext.ROOT, inputGetter, null, RexImpTable.instance())); + + // A chained table that supplies MY_FN's implementor makes code-gen succeed. + final RexCallImplementor implementor = + RexImpTable.wrapAsRexCallImplementor( + RexImpTable.createImplementor( + (translator, c, operands) -> Expressions.constant(true), + NullPolicy.NONE, false)); + final RexImplementorTable table = + RexImplementorTables.chain(new SingleScalarTable(MY_FN, implementor), + RexImpTable.instance()); + final List expressions = + RexToLixTranslator.translateProjects(program, typeFactory, + SqlConformanceEnum.DEFAULT, new BlockBuilder(), null, null, + DataContext.ROOT, inputGetter, null, table); + assertThat(expressions, is(notNullValue())); + assertThat(expressions, hasSize(1)); + } + + /** A chained extension table supplies an aggregate implementor that the + * built-in table does not have, while still resolving built-in aggregates. */ + @Test void chainResolvesCustomAggregateImplementor() { + final RexImplementorTable custom = new RexImplementorTable() { + @Override public @Nullable RexCallImplementor get(SqlOperator operator) { + return null; + } + + @Override public @Nullable AggImplementor get(SqlAggFunction aggregation, + boolean forWindowAggregate) { + return aggregation == MY_AGG ? AGG_SENTINEL : null; + } + + @Override public @Nullable MatchImplementor get(SqlMatchFunction function) { + return null; + } + + @Override public @Nullable TableFunctionCallImplementor get( + SqlWindowTableFunction operator) { + return null; + } + }; + final RexImplementorTable chain = + RexImplementorTables.chain(custom, RexImpTable.instance()); + assertThat(RexImpTable.instance().get(MY_AGG, false), is(nullValue())); + assertThat(chain.get(MY_AGG, false), is(sameInstance(AGG_SENTINEL))); + assertThat(chain.get(SqlStdOperatorTable.COUNT, false), is(notNullValue())); + } +} diff --git a/spark/src/main/java/org/apache/calcite/adapter/spark/SparkRules.java b/spark/src/main/java/org/apache/calcite/adapter/spark/SparkRules.java index 2ebda2879c5b..eb38a6dc17f2 100644 --- a/spark/src/main/java/org/apache/calcite/adapter/spark/SparkRules.java +++ b/spark/src/main/java/org/apache/calcite/adapter/spark/SparkRules.java @@ -376,7 +376,7 @@ public int getFlags() { typeFactory, builder2, new RexToLixTranslator.InputGetterImpl(e_, result.physType), - null, implementor.getConformance()); + null, implementor.getConformance(), false, RexImpTable.INSTANCE); builder2.add( Expressions.ifThen( Expressions.not(condition), @@ -396,7 +396,8 @@ public int getFlags() { null, DataContext.ROOT, new RexToLixTranslator.InputGetterImpl(e_, result.physType), - null); + null, + RexImpTable.INSTANCE); builder2.add( Expressions.return_(null, Expressions.convert_( From 8b18b12890629cbea1c31903035d0be339120d07 Mon Sep 17 00:00:00 2001 From: xuzifu666 <1206332514@qq.com> Date: Wed, 1 Jul 2026 14:12:49 +0800 Subject: [PATCH 364/562] [CALCITE-6344] RelToSqlConverter invalid quotation for arrays and item operator(ansi dialect) --- .../calcite/rel/rel2sql/SqlImplementor.java | 16 +++++++++++++--- .../rel/rel2sql/RelToSqlConverterTest.java | 18 +++++++++++++++--- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java b/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java index 7d579d184cc8..403d7f6e2f8d 100644 --- a/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java +++ b/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java @@ -713,9 +713,19 @@ public SqlNode toSql(@Nullable RexProgram program, RexNode rex) { break; case ROW: case ITEM: - final SqlNode expr = toSql(program, referencedExpr); - sqlIdentifier = new SqlIdentifier(expr.toString(), POS); - break; + // The referenced expression (e.g. an array/map ITEM access) must be + // unparsed as its own SqlNode so that the target dialect quotes each + // part correctly. Combine it with the accessed field names using the + // DOT operator instead of collapsing everything into a single + // identifier name (which would get quoted as one token). + SqlNode dotNode = toSql(program, referencedExpr); + RexFieldAccess dotAccess; + while ((dotAccess = accesses.pollLast()) != null) { + dotNode = + SqlStdOperatorTable.DOT.createCall(POS, dotNode, + new SqlIdentifier(dotAccess.getField().getName(), POS)); + } + return dotNode; default: sqlIdentifier = (SqlIdentifier) toSql(program, referencedExpr); } diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java index 4e2825bb3b62..1d3dda5af67e 100644 --- a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java +++ b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java @@ -2998,14 +2998,26 @@ private SqlDialect nonOrdinalDialect() { * SqlItemOperator fails in RelToSqlConverter. */ @Test void testSqlItemOperator() { sql("SELECT foo[0].\"EXPR$1\" FROM (SELECT ARRAY[ROW('a', 'b')] AS foo)") - .ok("SELECT \"ARRAY[ROW('a', 'b')][0]\".\"EXPR$1\"\n" + .ok("SELECT ARRAY[ROW('a', 'b')][0].\"EXPR$1\"\n" + "FROM (VALUES (0)) AS \"t\" (\"ZERO\")"); sql("SELECT foo['k'].\"EXPR$1\" FROM (SELECT MAP['k', ROW('a', 'b')] AS foo)") - .ok("SELECT \"MAP['k', ROW('a', 'b')]['k']\".\"EXPR$1\"\n" + .ok("SELECT MAP['k', ROW('a', 'b')]['k'].\"EXPR$1\"\n" + "FROM (VALUES (0)) AS \"t\" (\"ZERO\")"); sql("select\"books\"[0].\"title\" from \"authors\"") .schema(CalciteAssert.SchemaSpec.BOOKSTORE) - .ok("SELECT \"`books`[0]\".\"title\"\n" + .ok("SELECT \"books\"[0].\"title\"\n" + + "FROM \"bookstore\".\"authors\""); + } + + /** Test case for + * [CALCITE-6344] + * RelToSqlConverter invalid quotation for arrays and item operator + * (ansi dialect). */ + @Test void testSqlItemOperator2() { + sql("SELECT \"books\"[0].\"title\" from \"bookstore\".\"authors\"") + .schema(CalciteAssert.SchemaSpec.BOOKSTORE) + .withPostgresql() + .ok("SELECT \"books\"[0].\"title\"\n" + "FROM \"bookstore\".\"authors\""); } From 81f801224fa3d54659e725991e1efe65a1cea830 Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Wed, 1 Jul 2026 17:23:33 +0800 Subject: [PATCH 365/562] Site: Update Ran Tao githubId Signed-off-by: xiedeyantu --- site/_data/contributors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site/_data/contributors.yml b/site/_data/contributors.yml index c64c356bf590..7596f97b1a79 100644 --- a/site/_data/contributors.yml +++ b/site/_data/contributors.yml @@ -292,7 +292,7 @@ role: Committer - name: Ran Tao apacheId: taoran - githubId: chucheng92 + githubId: taoran92 pronouns: he/him org: ByteDance role: Committer From e44d82b3b3068cd904acd8cb146731151cd1e0ba Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Wed, 1 Jul 2026 10:21:56 +0800 Subject: [PATCH 366/562] [CALCITE-7242] Implement a rule to eliminate LITERAL_AGG so that other databases can handle it Co-authored-by: Weihua Zhang <745778074@qq.com> --- .../rules/AggregateRemoveLiteralAggRule.java | 156 ++++++++++++++++++ .../apache/calcite/rel/rules/CoreRules.java | 5 + .../apache/calcite/test/InterpreterTest.java | 36 ++++ .../apache/calcite/test/RelOptRulesTest.java | 86 ++++++++++ .../apache/calcite/test/RelOptRulesTest.xml | 134 +++++++++++++++ 5 files changed, 417 insertions(+) create mode 100644 core/src/main/java/org/apache/calcite/rel/rules/AggregateRemoveLiteralAggRule.java diff --git a/core/src/main/java/org/apache/calcite/rel/rules/AggregateRemoveLiteralAggRule.java b/core/src/main/java/org/apache/calcite/rel/rules/AggregateRemoveLiteralAggRule.java new file mode 100644 index 000000000000..6aedbb612cb3 --- /dev/null +++ b/core/src/main/java/org/apache/calcite/rel/rules/AggregateRemoveLiteralAggRule.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.rel.rules; + +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelRule; +import org.apache.calcite.rel.RelCollations; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.Aggregate; +import org.apache.calcite.rel.core.AggregateCall; +import org.apache.calcite.rel.logical.LogicalAggregate; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.tools.RelBuilder; +import org.apache.calcite.util.ImmutableBitSet; + +import org.immutables.value.Value; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * Planner rule that removes {@code LITERAL_AGG} aggregate calls from an + * {@link Aggregate}. + * + *

      {@code LITERAL_AGG} is an internal aggregate used by Calcite to mark + * whether a group exists, and many external databases cannot implement it. This + * rule keeps the grouping operation, removes the literal aggregate calls, and + * adds a {@code Project} that restores the original row type with the literal + * values. + * + *

      For example, + * + * 

      {@code
+ * LogicalAggregate(group=[{0}], i=[LITERAL_AGG(true)])
+ *   LogicalTableScan(table=[[EMP]])
+ * }
      + * + *

      becomes + * + * 

      {@code
+ * LogicalProject(DEPTNO=[$0], i=[true])
+ *   LogicalAggregate(group=[{0}])
+ *     LogicalTableScan(table=[[EMP]])
+ * }
      + */ +@Value.Enclosing +public class AggregateRemoveLiteralAggRule + extends RelRule + implements TransformationRule { + + /** Creates an AggregateRemoveLiteralAggRule. */ + protected AggregateRemoveLiteralAggRule(Config config) { + super(config); + } + + @Override public void onMatch(RelOptRuleCall call) { + final RelBuilder relBuilder = call.builder(); + final Aggregate aggregate = call.rel(0); + final List aggCalls = aggregate.getAggCallList(); + if (aggCalls.isEmpty()) { + return; + } + + final boolean[] literalAggs = new boolean[aggCalls.size()]; + int literalAggCount = 0; + for (int i = 0; i < aggCalls.size(); i++) { + if (aggCalls.get(i).getAggregation().getKind() == SqlKind.LITERAL_AGG) { + literalAggs[i] = true; + literalAggCount++; + } + } + if (literalAggCount == 0) { + return; + } + + final List newAggCalls = + new ArrayList<>(aggCalls.size() - literalAggCount); + final int[] oldAggIndexToNewAggIndex = new int[aggCalls.size()]; + int newAggPos = 0; + for (int i = 0; i < aggCalls.size(); i++) { + if (!literalAggs[i]) { + newAggCalls.add(aggCalls.get(i)); + oldAggIndexToNewAggIndex[i] = newAggPos++; + } + } + if (newAggCalls.isEmpty() && aggregate.getGroupCount() == 0) { + newAggCalls.add( + AggregateCall.create(SqlStdOperatorTable.COUNT, false, false, false, + Collections.emptyList(), Collections.emptyList(), -1, + null, RelCollations.EMPTY, + aggregate.getGroupSets().contains(ImmutableBitSet.of()), + aggregate.getInput(), null, null)); + } + + final RelNode newAggregate = + aggregate.copy(aggregate.getTraitSet(), aggregate.getInput(), + aggregate.getGroupSet(), aggregate.getGroupSets(), newAggCalls); + relBuilder.push(newAggregate); + + final int groupCount = aggregate.getGroupCount(); + final int outputCount = aggregate.getRowType().getFieldCount(); + final List projects = new ArrayList<>(outputCount); + for (int outPos = 0; outPos < outputCount; outPos++) { + if (outPos < groupCount) { + projects.add(relBuilder.field(outPos)); + } else { + final int aggIndex = outPos - groupCount; + final AggregateCall aggCall = aggCalls.get(aggIndex); + if (literalAggs[aggIndex]) { + projects.add(aggCall.rexList.get(0)); + } else { + projects.add(relBuilder.field(groupCount + oldAggIndexToNewAggIndex[aggIndex])); + } + } + } + + relBuilder.project(projects, aggregate.getRowType().getFieldNames()); + call.transformTo(relBuilder.build()); + } + + /** Rule configuration. */ + @Value.Immutable + public interface Config extends RelRule.Config { + Config DEFAULT = ImmutableAggregateRemoveLiteralAggRule.Config.of() + .withOperandSupplier(b0 -> + b0.operand(LogicalAggregate.class).anyInputs()); + + @Override default AggregateRemoveLiteralAggRule toRule() { + return new AggregateRemoveLiteralAggRule(this); + } + + /** Defines an operand tree for the given aggregate class. */ + default Config withOperandFor(Class aggregateClass) { + return withOperandSupplier(b0 -> + b0.operand(aggregateClass).anyInputs()) + .as(Config.class); + } + } +} diff --git a/core/src/main/java/org/apache/calcite/rel/rules/CoreRules.java b/core/src/main/java/org/apache/calcite/rel/rules/CoreRules.java index bc21c41dd366..104e34bfaebe 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/CoreRules.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/CoreRules.java @@ -990,6 +990,11 @@ private CoreRules() {} public static final AggregateGroupingSetsToUnionRule AGGREGATE_GROUPING_SETS_TO_UNION = AggregateGroupingSetsToUnionRule.Config.DEFAULT.toRule(); + /** Rule that removes {@code LITERAL_AGG} aggregate calls by replacing them + * with literal expressions in a {@link Project}. */ + public static final AggregateRemoveLiteralAggRule AGGREGATE_REMOVE_LITERAL_AGG = + AggregateRemoveLiteralAggRule.Config.DEFAULT.toRule(); + /** Rule that converts a {@link Correlate} after an {@link Uncollect} into a simple * Uncollect, if possible. */ public static final RelOptRule UNNEST_DECORRELATE = diff --git a/core/src/test/java/org/apache/calcite/test/InterpreterTest.java b/core/src/test/java/org/apache/calcite/test/InterpreterTest.java index 7470e2163649..c138056e727a 100644 --- a/core/src/test/java/org/apache/calcite/test/InterpreterTest.java +++ b/core/src/test/java/org/apache/calcite/test/InterpreterTest.java @@ -371,6 +371,42 @@ private static void assertRows(Interpreter interpreter, "[1943, 1, true, -3]"); } + /** Tests a GROUP BY query after replacing + * {@link org.apache.calcite.sql.fun.SqlInternalOperators#LITERAL_AGG}. */ + @Test void testAggregateRemoveLiteralAggOnEmptyInput() { + rootSchema().add("beatles", new ScannableTableTest.BeatlesTable()); + final Function relFn = + b -> applyAggregateRemoveLiteralAggRule(b.scan("beatles") + .empty() + .aggregate(b.groupKey("k"), + b.literalAgg(true).as("t")) + .build()); + fixture().withRel(relFn) + .returnsRows(); + } + + /** Tests a GROUP BY () query after replacing + * {@link org.apache.calcite.sql.fun.SqlInternalOperators#LITERAL_AGG}. */ + @Test void testAggregateRemoveLiteralAggWithOnlyLiteralAgg() { + rootSchema().add("beatles", new ScannableTableTest.BeatlesTable()); + final Function relFn = + b -> applyAggregateRemoveLiteralAggRule(b.scan("beatles") + .aggregate(b.groupKey(), + b.literalAgg(true).as("t")) + .build()); + fixture().withRel(relFn) + .returnsRows("[true]"); + } + + private static RelNode applyAggregateRemoveLiteralAggRule(RelNode rel) { + final HepProgram program = HepProgram.builder() + .addRuleInstance(CoreRules.AGGREGATE_REMOVE_LITERAL_AGG) + .build(); + final HepPlanner hep = new HepPlanner(program); + hep.setRoot(rel); + return hep.findBestExp(); + } + /** Tests executing a plan on a single-column * {@link org.apache.calcite.schema.ScannableTable} using an interpreter. */ @Test void testInterpretSimpleScannableTable() { diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index 72d94a5a0858..0dee9a6b74c4 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -12003,6 +12003,92 @@ private void checkLoptOptimizeJoinRule(LoptOptimizeJoinRule rule) { .check(); } + /** Test case of + * [CALCITE-7242] + * Implement a rule to eliminate LITERAL_AGG so that other databases can handle it. */ + @Test void testAggregateRemoveLiteralAggRuleWithAnySubQuery() { + final String sql = "select deptno, name = ANY (\n" + + " select mgr from emp)\n" + + "from dept"; + sql(sql) + .withSubQueryRules() + .withLateDecorrelate(true) + .withAfter((fixture, rel) -> applyAggregateRemoveLiteralAggRule(rel)) + .check(); + } + + /** Test case of + * [CALCITE-7242] + * Implement a rule to eliminate LITERAL_AGG so that other databases can handle it. */ + @Test void testAggregateRemoveLiteralAggRuleWithInSubQuery() { + final String sql = "select empno\n" + + "from sales.emp\n" + + "where deptno in (select deptno from sales.emp where empno < 20)\n" + + "or emp.sal < 100"; + sql(sql) + .withSubQueryRules() + .withLateDecorrelate(true) + .withAfter((fixture, rel) -> applyAggregateRemoveLiteralAggRule(rel)) + .check(); + } + + /** Test case of + * [CALCITE-7242] + * Implement a rule to eliminate LITERAL_AGG so that other databases can handle it. */ + @Test void testAggregateRemoveLiteralAggRuleWithRegularAggCall() { + final Function relFn = b -> b + .scan("EMP") + .aggregate(b.groupKey("DEPTNO"), + b.count().as("c"), + b.literalAgg(true).as("i")) + .build(); + relFn(relFn) + .withRule(CoreRules.AGGREGATE_REMOVE_LITERAL_AGG) + .check(); + } + + /** Test case of + * [CALCITE-7242] + * Implement a rule to eliminate LITERAL_AGG so that other databases can handle it. */ + @Test void testAggregateRemoveLiteralAggRuleWithEmptyInput() { + final Function relFn = b -> { + final RelBuilder builder = + RelBuilderTest.createBuilder(c -> c.withAggregateUnique(true)); + return builder + .scan("EMP") + .empty() + .aggregate(builder.groupKey("DEPTNO"), + builder.literalAgg(true).as("i")) + .build(); + }; + relFn(relFn) + .withRule(CoreRules.AGGREGATE_REMOVE_LITERAL_AGG) + .check(); + } + + /** Test case of + * [CALCITE-7242] + * Implement a rule to eliminate LITERAL_AGG so that other databases can handle it. */ + @Test void testAggregateRemoveLiteralAggRuleWithOnlyLiteralAgg() { + final Function relFn = b -> b + .scan("EMP") + .aggregate(b.groupKey(), + b.literalAgg(true).as("i")) + .build(); + relFn(relFn) + .withRule(CoreRules.AGGREGATE_REMOVE_LITERAL_AGG) + .check(); + } + + private static RelNode applyAggregateRemoveLiteralAggRule(RelNode rel) { + final HepProgram program = HepProgram.builder() + .addRuleInstance(CoreRules.AGGREGATE_REMOVE_LITERAL_AGG) + .build(); + final HepPlanner hep = new HepPlanner(program); + hep.setRoot(rel); + return hep.findBestExp(); + } + /** Test case of * [CALCITE-7178] * FETCH and OFFSET in EnumerableMergeUnionRule do not support BIGINT. */ diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index 4eae45cc8aa3..4cb1da4092b9 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -1130,6 +1130,140 @@ LogicalProject(MGR=[$0], SUM_SAL=[$2]) LogicalAggregate(group=[{0, 1}], SUM_SAL=[SUM($2)]) LogicalProject(MGR=[$3], DEPTNO=[$7], SAL=[$5]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +]]> + + + + + + + + + + + ($2, 0)), AND(<($3, $2), null, <>($2, 0), IS NULL($5)))]) + LogicalJoin(condition=[=(CAST($1):INTEGER NOT NULL, $4)], joinType=[left]) + LogicalJoin(condition=[true], joinType=[inner]) + LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) + LogicalAggregate(group=[{}], c=[COUNT()], ck=[COUNT($0)]) + LogicalProject(MGR=[$3]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) + LogicalAggregate(group=[{0}], i=[LITERAL_AGG(true)]) + LogicalProject(MGR=[$3]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +]]> + + + ($2, 0)), AND(<($3, $2), null, <>($2, 0), IS NULL($5)))]) + LogicalJoin(condition=[=(CAST($1):INTEGER NOT NULL, $4)], joinType=[left]) + LogicalJoin(condition=[true], joinType=[inner]) + LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) + LogicalAggregate(group=[{}], c=[COUNT()], ck=[COUNT($0)]) + LogicalProject(MGR=[$3]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) + LogicalProject(MGR=[$0], i=[true]) + LogicalAggregate(group=[{0}]) + LogicalProject(MGR=[$3]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 04a4e8e5c4de4adda9e0e9c9ff855aaece8c45e1 Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Fri, 3 Jul 2026 12:51:45 +0800 Subject: [PATCH 367/562] [CALCITE-7634] JoinExpandOrToUnionRule incorrectly expands OR branches with non-equi predicates referencing both join inputs Signed-off-by: xiedeyantu --- .../rel/rules/JoinExpandOrToUnionRule.java | 24 +++++++----- .../apache/calcite/test/RelOptRulesTest.java | 13 +++++++ .../apache/calcite/test/RelOptRulesTest.xml | 38 ++++++++++++++++--- core/src/test/resources/sql/hep.iq | 38 +++++++++++++++++++ 4 files changed, 99 insertions(+), 14 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rel/rules/JoinExpandOrToUnionRule.java b/core/src/main/java/org/apache/calcite/rel/rules/JoinExpandOrToUnionRule.java index 6e3f137e56dd..67e061d5f7cb 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/JoinExpandOrToUnionRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/JoinExpandOrToUnionRule.java @@ -29,6 +29,7 @@ import org.apache.calcite.rex.RexUtil; import org.apache.calcite.rex.RexVisitorImpl; import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.tools.RelBuilder; import org.immutables.value.Value; @@ -155,10 +156,6 @@ private static boolean isValidCond(RexNode node, int leftFieldCount) { // equality (when the above conditions are met), that are // single-side (refer to only on of the collections joined), // or which are constant, they will all trigger the expansion. - if (!doesNotReferToBothInputs(cond, leftFieldCount)) { - return false; - } - if (RexUtil.SubQueryFinder.find(cond) != null || RexUtil.containsCorrelation(cond)) { // The "call" does not support sub-queries or correlation yet @@ -170,8 +167,15 @@ private static boolean isValidCond(RexNode node, int leftFieldCount) { // Checks if the "call" is valid for use as a join key. if (isEquiJoinCond(call, leftFieldCount)) { hasJoinKeyCond = true; + continue; } } + + // Non-equality predicates may be pushed into the expanded branch only + // if they do not correlate the two join inputs. + if (!doesNotReferToBothInputs(cond, leftFieldCount)) { + return false; + } } return hasJoinKeyCond; } @@ -204,9 +208,9 @@ private static boolean doesNotReferToBothInputs(RexNode rex, int leftFieldCount) /** * Counts the number of InputRefs in a RexNode expression. */ private static class RexInputRefCounter extends RexVisitorImpl { - private int leftFieldCount; - public int leftInputRefCount = 0; - public int rightInputRefCount = 0; + private final int leftFieldCount; + private int leftInputRefCount = 0; + private int rightInputRefCount = 0; RexInputRefCounter(int leftFieldCount) { super(true); @@ -215,7 +219,7 @@ private static class RexInputRefCounter extends RexVisitorImpl { @Override public Void visitInputRef(RexInputRef inputRef) { if (inputRef.getIndex() < leftFieldCount) { - leftFieldCount++; + leftInputRefCount++; } else { rightInputRefCount++; } @@ -367,7 +371,9 @@ private List expandInnerJoinToRelNodes(Join join, List orConds for (int i = 0; i < orConds.size(); i++) { RexNode orCond = orConds.get(i); for (int j = 0; j < i; j++) { - orCond = relBuilder.and(orCond, relBuilder.not(orConds.get(j))); + orCond = + relBuilder.and(orCond, + relBuilder.call(SqlStdOperatorTable.IS_NOT_TRUE, orConds.get(j))); } relBuilder.push(join.getLeft()) diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index 0dee9a6b74c4..4fb62fb3460c 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -10008,6 +10008,19 @@ private RelOptFixture spatial(String sql) { .check(); } + /** Test case for + * [CALCITE-7634] + * JoinExpandOrToUnionRule incorrectly expands OR branches with non-equi + * predicates referencing both join inputs. */ + @Test void testJoinConditionOrExpansionRuleWithCrossInputPredicate() { + String sql = "select * from EMP as p1\n" + + "inner join EMP as p2 on (p1.empno = p2.empno and p1.sal < p2.sal)\n" + + "or (p1.mgr = p2.mgr and p1.comm < p2.comm)\n" + + "or p1.deptno = p2.deptno"; + sql(sql).withRule(CoreRules.JOIN_EXPAND_OR_TO_UNION_RULE) + .check(); + } + /** Test case for * [CALCITE-6930] * Implementing JoinConditionOrExpansionRule. */ diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index 4cb1da4092b9..7178366a670f 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -7849,10 +7849,10 @@ LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$ LogicalJoin(condition=[AND(=($0, $9), =($2, 'Job1'), SEARCH($1, Sarg['a':VARCHAR(20), 'bb':VARCHAR(20), 'cc':VARCHAR(20)]:VARCHAR(20)), SEARCH($5, Sarg[(120..3000)]), =($3, $6))], joinType=[inner]) LogicalTableScan(table=[[CATALOG, SALES, EMPNULLABLES]]) LogicalTableScan(table=[[CATALOG, SALES, EMPNULLABLES]]) - LogicalJoin(condition=[AND(=($7, $16), =($11, 'Job2'), SEARCH($10, Sarg['a':VARCHAR(20), 'bb':VARCHAR(20), 'cc':VARCHAR(20)]:VARCHAR(20)), SEARCH($14, Sarg[(110..3000)]), <(CAST(+($3, 10)):DOUBLE, LN(15)), OR(<>($0, $9), <>($2, 'Job1'), SEARCH($1, Sarg[(-∞..'a':VARCHAR(20)), ('a':VARCHAR(20)..'bb':VARCHAR(20)), ('bb':VARCHAR(20)..'cc':VARCHAR(20)), ('cc':VARCHAR(20)..+∞)]:VARCHAR(20)), SEARCH($5, Sarg[(-∞..120], [3000..+∞)]), <>($3, $6)))], joinType=[inner]) + LogicalJoin(condition=[AND(=($7, $16), =($11, 'Job2'), SEARCH($10, Sarg['a':VARCHAR(20), 'bb':VARCHAR(20), 'cc':VARCHAR(20)]:VARCHAR(20)), SEARCH($14, Sarg[(110..3000)]), <(CAST(+($3, 10)):DOUBLE, LN(15)), IS NOT TRUE(AND(=($0, $9), =($2, 'Job1'), SEARCH($1, Sarg['a':VARCHAR(20), 'bb':VARCHAR(20), 'cc':VARCHAR(20)]:VARCHAR(20)), SEARCH($5, Sarg[(120..3000)]), =($3, $6))))], joinType=[inner]) LogicalTableScan(table=[[CATALOG, SALES, EMPNULLABLES]]) LogicalTableScan(table=[[CATALOG, SALES, EMPNULLABLES]]) - LogicalJoin(condition=[AND(OR(AND(=($1, 'Jensen'), >($15, 10)), SEARCH(CAST($3):DECIMAL(11, 1), Sarg[[10.0:DECIMAL(11, 1)..20.0:DECIMAL(11, 1)]]:DECIMAL(11, 1))), OR(<>($0, $9), <>($2, 'Job1'), SEARCH($1, Sarg[(-∞..'a':VARCHAR(20)), ('a':VARCHAR(20)..'bb':VARCHAR(20)), ('bb':VARCHAR(20)..'cc':VARCHAR(20)), ('cc':VARCHAR(20)..+∞)]:VARCHAR(20)), SEARCH($5, Sarg[(-∞..120], [3000..+∞)]), <>($3, $6)), OR(<>($7, $16), <>($11, 'Job2'), SEARCH($10, Sarg[(-∞..'a':VARCHAR(20)), ('a':VARCHAR(20)..'bb':VARCHAR(20)), ('bb':VARCHAR(20)..'cc':VARCHAR(20)), ('cc':VARCHAR(20)..+∞)]:VARCHAR(20)), SEARCH($14, Sarg[(-∞..110], [3000..+∞)]), >=(CAST(+($3, 10)):DOUBLE, LN(15))))], joinType=[inner]) + LogicalJoin(condition=[AND(OR(AND(=($1, 'Jensen'), >($15, 10)), SEARCH(CAST($3):DECIMAL(11, 1), Sarg[[10.0:DECIMAL(11, 1)..20.0:DECIMAL(11, 1)]]:DECIMAL(11, 1))), IS NOT TRUE(AND(=($0, $9), =($2, 'Job1'), SEARCH($1, Sarg['a':VARCHAR(20), 'bb':VARCHAR(20), 'cc':VARCHAR(20)]:VARCHAR(20)), SEARCH($5, Sarg[(120..3000)]), =($3, $6))), IS NOT TRUE(AND(=($7, $16), =($11, 'Job2'), SEARCH($10, Sarg['a':VARCHAR(20), 'bb':VARCHAR(20), 'cc':VARCHAR(20)]:VARCHAR(20)), SEARCH($14, Sarg[(110..3000)]), <(CAST(+($3, 10)):DOUBLE, LN(15)))))], joinType=[inner]) LogicalTableScan(table=[[CATALOG, SALES, EMPNULLABLES]]) LogicalTableScan(table=[[CATALOG, SALES, EMPNULLABLES]]) ]]> @@ -7890,7 +7890,7 @@ LogicalUnion(all=[true]) LogicalJoin(condition=[=($0, $10)], joinType=[inner]) LogicalTableScan(table=[[scott, DEPT]]) LogicalTableScan(table=[[scott, EMP]]) - LogicalJoin(condition=[AND(=($1, $5), <>($0, $10))], joinType=[inner]) + LogicalJoin(condition=[AND(=($1, $5), IS NOT TRUE(=($0, $10)))], joinType=[inner]) LogicalTableScan(table=[[scott, DEPT]]) LogicalTableScan(table=[[scott, EMP]]) LogicalProject($f0=[null:TINYINT], $f1=[null:VARCHAR(14)], $f2=[null:VARCHAR(13)], EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7]) @@ -7963,10 +7963,10 @@ LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$ LogicalJoin(condition=[<($3, $12)], joinType=[inner]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) - LogicalJoin(condition=[AND(=($0, $9), >=($3, $12))], joinType=[inner]) + LogicalJoin(condition=[AND(=($0, $9), IS NOT TRUE(<($3, $12)))], joinType=[inner]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) - LogicalJoin(condition=[AND(OR(<($5, 0), <(LN($5), 10.0E0)), >=($3, $12), <>($0, $9))], joinType=[inner]) + LogicalJoin(condition=[AND(OR(<($5, 0), <(LN($5), 10.0E0)), IS NOT TRUE(<($3, $12)), <>($0, $9))], joinType=[inner]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) ]]> @@ -8002,6 +8002,34 @@ LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$ LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +]]> + + + + + + + + + + + diff --git a/core/src/test/resources/sql/hep.iq b/core/src/test/resources/sql/hep.iq index 6e1146c90d48..7fbcbc7f2975 100644 --- a/core/src/test/resources/sql/hep.iq +++ b/core/src/test/resources/sql/hep.iq @@ -238,6 +238,44 @@ EnumerableHashJoin(condition=[AND(=($0, $6), OR(AND(>($1, 11), <=($7, 32)), AND( !} !set hep-rules original +# [CALCITE-7634] JoinExpandOrToUnionRule incorrectly expands OR branches with non-equi predicates referencing both join inputs +!set hep-rules " ++CoreRules.JOIN_EXPAND_OR_TO_UNION_RULE" + +with emp_nulls (empno, mgr, sal, comm, deptno) as (values + (1, 10, 100, cast(null as integer), 20), + (2, 10, 200, 5, 20), + (3, 30, 300, cast(null as integer), 30)) +select p1.empno as e1, p2.empno as e2 +from emp_nulls as p1 +inner join emp_nulls as p2 on (p1.empno = p2.empno and p1.sal < p2.sal) +or (p1.mgr = p2.mgr and p1.comm < p2.comm) +or p1.deptno = p2.deptno +order by e1, e2; ++----+----+ +| E1 | E2 | ++----+----+ +| 1 | 1 | +| 1 | 2 | +| 2 | 1 | +| 2 | 2 | +| 3 | 3 | ++----+----+ +(5 rows) + +!ok +EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) + EnumerableCalc(expr#0..9=[{inputs}], E1=[$t0], E2=[$t5]) + EnumerableUnion(all=[true]) + EnumerableNestedLoopJoin(condition=[OR(AND(=($0, $5), <($2, $7)), AND(=($1, $6), <($3, $8)))], joinType=[inner]) + EnumerableValues(tuples=[[{ 1, 10, 100, null, 20 }, { 2, 10, 200, 5, 20 }, { 3, 30, 300, null, 30 }]]) + EnumerableValues(tuples=[[{ 1, 10, 100, null, 20 }, { 2, 10, 200, 5, 20 }, { 3, 30, 300, null, 30 }]]) + EnumerableMergeJoin(condition=[AND(=($4, $9), IS NOT TRUE(OR(AND(=($0, $5), <($2, $7)), AND(=($1, $6), <($3, $8)))))], joinType=[inner]) + EnumerableValues(tuples=[[{ 1, 10, 100, null, 20 }, { 2, 10, 200, 5, 20 }, { 3, 30, 300, null, 30 }]]) + EnumerableValues(tuples=[[{ 1, 10, 100, null, 20 }, { 2, 10, 200, 5, 20 }, { 3, 30, 300, null, 30 }]]) +!plan +!set hep-rules original + # [CALCITE-5740] Support for AggToSemiJoinRule !set hep-rules " +CoreRules.AGGREGATE_PROJECT_MERGE, From 439fc95423950af13d08605f24c1afb588ee83d9 Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Sun, 5 Jul 2026 08:59:52 +0800 Subject: [PATCH 368/562] [CALCITE-7638] SetOpToFilterRule MINUS drops rows when right-side filters evaluate to UNKNOWN Signed-off-by: xiedeyantu --- .../calcite/rel/rules/SetOpToFilterRule.java | 18 +++++++----- .../apache/calcite/test/RelOptRulesTest.java | 13 +++++++++ .../apache/calcite/test/RelOptRulesTest.xml | 27 ++++++++++++++++++ core/src/test/resources/sql/hep.iq | 28 +++++++++++++++++++ 4 files changed, 79 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rel/rules/SetOpToFilterRule.java b/core/src/main/java/org/apache/calcite/rel/rules/SetOpToFilterRule.java index e00e3e50f685..848cc6c797fa 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/SetOpToFilterRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/SetOpToFilterRule.java @@ -26,6 +26,7 @@ import org.apache.calcite.rel.core.Union; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexUtil; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.tools.RelBuilder; import org.apache.calcite.util.Pair; @@ -96,7 +97,7 @@ * is rewritten to * * SELECT DISTINCT mgr, comm FROM emp - * WHERE mgr = 12 AND NOT(comm = 5) + * WHERE mgr = 12 AND (comm = 5) IS NOT TRUE * */ @Value.Enclosing @@ -214,17 +215,19 @@ private static RelBuilder buildSetOp(RelBuilder builder, int count, RelNode setO /** * Creates a combined condition where the first condition - * is kept as-is and all subsequent conditions are negated, + * is kept as-is and all subsequent conditions are IS NOT TRUE, * then joined with AND operators. * *

      For example, given conditions [cond1, cond2, cond3], - * this constructs (cond1 AND NOT(cond2) AND NOT(cond3)). + * this constructs (cond1 AND cond2 IS NOT TRUE AND cond3 IS NOT TRUE). + * The right-side filter matches only TRUE rows, so FALSE and UNKNOWN + * both represent rows that are not present in that MINUS input. */ - private static RexNode andFirstNotRest(RelBuilder builder, List conds) { + private static RexNode andFirstIsNotTrueRest(RelBuilder builder, List conds) { List allConds = new ArrayList<>(); allConds.add(conds.get(0)); for (int i = 1; i < conds.size(); i++) { - allConds.add(builder.not(conds.get(i))); + allConds.add(builder.call(SqlStdOperatorTable.IS_NOT_TRUE, conds.get(i))); } return builder.and(allConds); } @@ -233,7 +236,8 @@ private static RexNode andFirstNotRest(RelBuilder builder, List conds) * Combines conditions according to set operation: * UNION: OR combination * INTERSECT: AND combination - * MINUS: Special handling where first source uses AND-NOT combination. + * MINUS: Special handling where first source keeps rows that match the + * first input and do not match any subsequent input. */ private static RexNode combineConditions(RelBuilder builder, List conds, SetOp setOp, boolean isFirstSource) { @@ -243,7 +247,7 @@ private static RexNode combineConditions(RelBuilder builder, List conds return builder.and(conds); } else if (setOp instanceof Minus) { return isFirstSource - ? andFirstNotRest(builder, conds) + ? andFirstIsNotTrueRest(builder, conds) : builder.or(conds); } // unreachable diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index 4fb62fb3460c..d56bde9808fc 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -11438,6 +11438,19 @@ private void checkLoptOptimizeJoinRule(LoptOptimizeJoinRule rule) { .check(); } + /** Test case of + * [CALCITE-7638] + * SetOpToFilterRule MINUS drops rows when right-side filters evaluate to UNKNOWN. */ + @Test void testMinusToFilterRuleWithNullableFilter() { + final String sql = "SELECT mgr, comm FROM empnullables WHERE mgr = 12\n" + + "EXCEPT\n" + + "SELECT mgr, comm FROM empnullables WHERE comm = 5\n"; + sql(sql) + .withPreRule(CoreRules.PROJECT_FILTER_TRANSPOSE) + .withRule(CoreRules.MINUS_FILTER_TO_FILTER) + .check(); + } + /** Test case of * [CALCITE-6973] * Add rule for convert Minus to Filter. */ diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index 7178366a670f..cdf6956ae4c6 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -10935,6 +10935,33 @@ LogicalMinus(all=[false]) LogicalFilter(condition=[>($0, 8)]) LogicalProject(DEPTNO=[$0]) LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) +]]> + + + + + + + + + + + diff --git a/core/src/test/resources/sql/hep.iq b/core/src/test/resources/sql/hep.iq index 7fbcbc7f2975..18cdaf27222e 100644 --- a/core/src/test/resources/sql/hep.iq +++ b/core/src/test/resources/sql/hep.iq @@ -77,6 +77,34 @@ EnumerableAggregate(group=[{0, 1}]) !plan !set hep-rules original +# [CALCITE-7638] SetOpToFilterRule MINUS drops rows when right-side filters evaluate to UNKNOWN. +!set hep-rules " ++CoreRules.PROJECT_FILTER_TRANSPOSE, ++CoreRules.MINUS_FILTER_TO_FILTER" + +SELECT mgr, comm FROM ( + SELECT mgr, comm FROM emp WHERE mgr = 7698 + EXCEPT + SELECT mgr, comm FROM emp WHERE comm = 500 +) ORDER BY comm NULLS LAST; ++------+---------+ +| MGR | COMM | ++------+---------+ +| 7698 | 0.00 | +| 7698 | 300.00 | +| 7698 | 1400.00 | +| 7698 | | ++------+---------+ +(4 rows) + +!ok +EnumerableSort(sort0=[$1], dir0=[ASC]) + EnumerableAggregate(group=[{0, 1}]) + EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t3):INTEGER], expr#9=[7698], expr#10=[=($t8, $t9)], expr#11=[CAST($t6):DECIMAL(12, 2)], expr#12=[500.00:DECIMAL(12, 2)], expr#13=[=($t11, $t12)], expr#14=[IS NOT TRUE($t13)], expr#15=[AND($t10, $t14)], MGR=[$t3], COMM=[$t6], $condition=[$t15]) + EnumerableTableScan(table=[[scott, EMP]]) +!plan +!set hep-rules original + # Testing with the planner-rules shows that due to cost-based selection issues, # the planner fails to choose a plan that includes the Aggregate operator. !set planner-rules " From 6f58163f00dcaf72fdac16d9cbf47bd43f16ac7b Mon Sep 17 00:00:00 2001 From: Alessandro Solimando Date: Fri, 3 Jul 2026 11:26:50 +0200 Subject: [PATCH 369/562] [CALCITE-7636] Materialized view union rewriting drops rows when the view filter is not null-rejecting --- .../MaterializedViewAggregateRule.java | 2 +- .../materialize/MaterializedViewJoinRule.java | 3 +- .../calcite/test/MaterializationTest.java | 23 ++++++++++++++ .../test/MaterializedViewRelOptRulesTest.java | 30 +++++++++++++++++++ 4 files changed, 56 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rel/rules/materialize/MaterializedViewAggregateRule.java b/core/src/main/java/org/apache/calcite/rel/rules/materialize/MaterializedViewAggregateRule.java index e6cf9e1058d0..404d0200bab1 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/materialize/MaterializedViewAggregateRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/materialize/MaterializedViewAggregateRule.java @@ -291,7 +291,7 @@ public abstract class MaterializedViewAggregateRule[CALCITE-7636] + * Materialized view union rewriting drops rows when the view filter is not + * null-rejecting. + * + *

      The view filters on {@code commission > 400}, which is UNKNOWN for the + * null-commission employee. The union-rewritten result must keep that row and + * therefore match the result computed without materializations. */ + @Test void testUnionRewritingNullableAggregatePredicate() { + try (TryThreadLocal.Memo ignored = Prepare.THREAD_TRIM.push(true)) { + MaterializationService.setThreadLocal(); + CalciteAssert.that() + .withMaterializations(HR_FKUK_MODEL, "m0", + "select \"deptno\", sum(\"salary\") as s from \"emps\" " + + "where \"deptno\" > 5 and \"commission\" > 400 group by \"deptno\"") + .query("select \"deptno\", sum(\"salary\") as s from \"emps\" " + + "where \"deptno\" > 5 group by \"deptno\"") + .enableMaterializations(true) + .explainContains("EnumerableUnion(all=[true])") + .sameResultWithMaterializationsDisabled(); + } + } + @Test void testViewMaterialization() { try (TryThreadLocal.Memo ignored = Prepare.THREAD_TRIM.push(true)) { MaterializationService.setThreadLocal(); diff --git a/core/src/test/java/org/apache/calcite/test/MaterializedViewRelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/MaterializedViewRelOptRulesTest.java index a7a088081e58..febf47cbbe8d 100644 --- a/core/src/test/java/org/apache/calcite/test/MaterializedViewRelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/MaterializedViewRelOptRulesTest.java @@ -364,6 +364,36 @@ protected final MaterializedViewFixture sql(String materialize, .ok(); } + /** Test case for + * [CALCITE-7636] + * Materialized view union rewriting drops rows when the view filter is not + * null-rejecting. + * + *

      The view is filtered on a predicate over the nullable {@code commission} + * column, so the predicate is UNKNOWN for rows where {@code commission} is + * null. The query-branch of the union must keep those rows (using + * {@code IS NOT TRUE}), otherwise they are dropped from both branches and + * whole groups disappear from the result. */ + @Test void testAggregateMaterializationUnionRewritingNullablePredicate() { + sql("select \"deptno\", sum(\"salary\") as s\n" + + "from \"emps\" where \"deptno\" > 5 and \"commission\" > 1000\n" + + "group by \"deptno\"", + "select \"deptno\", sum(\"salary\") as s\n" + + "from \"emps\" where \"deptno\" > 5\n" + + "group by \"deptno\"") + .checkingThatResultContains("" + + "EnumerableAggregate(group=[{0}], S=[$SUM0($1)])\n" + + " EnumerableUnion(all=[true])\n" + + " EnumerableAggregate(group=[{1}], S=[$SUM0($3)])\n" + + " EnumerableCalc(expr#0..4=[{inputs}], expr#5=[CAST($t1):INTEGER NOT NULL], " + + "expr#6=[5], expr#7=[>($t5, $t6)], expr#8=[1000], expr#9=[CAST($t4):INTEGER], " + + "expr#10=[<($t8, $t9)], expr#11=[IS NOT TRUE($t10)], expr#12=[AND($t7, $t11)], " + + "proj#0..4=[{exprs}], $condition=[$t12])\n" + + " EnumerableTableScan(table=[[hr, emps]])\n" + + " EnumerableTableScan(table=[[hr, MV0]])") + .ok(); + } + @Test void testJoinAggregateMaterializationNoAggregateFuncs1() { sql("select \"empid\", \"depts\".\"deptno\" from \"emps\"\n" + "join \"depts\" using (\"deptno\") where \"depts\".\"deptno\" > 10\n" From 52184f71845a186f442a9d9285b25d3eed1c009b Mon Sep 17 00:00:00 2001 From: Anas Khan <83116240+anxkhn@users.noreply.github.com> Date: Fri, 3 Jul 2026 07:18:57 +0530 Subject: [PATCH 370/562] [CALCITE-7206] Avoid duplicate compare(Object, Object) bridge method in the Enumerable merge join comparator PhysTypeImpl.generateComparator always appends a bridge compare(Object, Object) method when EnumerableRules.BRIDGE_METHODS is enabled. When the boxed row Java class is already Object (for example a merge join key of type ANY, represented as a bare Object), the primary compare method already has the signature compare(Object, Object), so the bridge collapses to the same signature and the generated Comparator declares two identical methods, which fails to compile with "Error while compiling generated Java code". Skip the bridge method when javaRowClass is Object, since the primary method already satisfies the Comparator contract. The normal Object[] row case is unchanged. --- .../adapter/enumerable/PhysTypeImpl.java | 5 ++- .../adapter/enumerable/PhysTypeTest.java | 33 +++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/PhysTypeImpl.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/PhysTypeImpl.java index a44d4f69df70..ce7d016a6ca0 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/PhysTypeImpl.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/PhysTypeImpl.java @@ -499,7 +499,10 @@ private Expression generateComparator(RelCollation collation, ImmutableList.of(parameterV0, parameterV1), body.toBlock())); - if (EnumerableRules.BRIDGE_METHODS) { + // When javaRowClass is Object the primary compare method already has the + // signature compare(Object, Object), so a bridge method would be a + // duplicate and the generated Comparator would fail to compile. + if (EnumerableRules.BRIDGE_METHODS && javaRowClass != Object.class) { final ParameterExpression parameterO0 = Expressions.parameter(Object.class, "o0"); final ParameterExpression parameterO1 = diff --git a/core/src/test/java/org/apache/calcite/adapter/enumerable/PhysTypeTest.java b/core/src/test/java/org/apache/calcite/adapter/enumerable/PhysTypeTest.java index 494dc21a2a11..ed61ad90fc8d 100644 --- a/core/src/test/java/org/apache/calcite/adapter/enumerable/PhysTypeTest.java +++ b/core/src/test/java/org/apache/calcite/adapter/enumerable/PhysTypeTest.java @@ -21,6 +21,8 @@ import org.apache.calcite.linq4j.Enumerable; import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.linq4j.tree.Expressions; +import org.apache.calcite.rel.RelCollation; +import org.apache.calcite.rel.RelCollations; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.sql.type.SqlTypeName; @@ -29,7 +31,9 @@ import org.junit.jupiter.api.Test; import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.not; import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; /** * Test for {@link org.apache.calcite.adapter.enumerable.PhysTypeImpl}. @@ -98,4 +102,33 @@ public final class PhysTypeTest { + ")"; assertThat(expected, is(Expressions.toString(e))); } + + /** Test case for + * [CALCITE-7206] + * Duplicate 'compare' method in the Enumerable merge join comparator when the + * row Java class is Object. + * + *

      When the row is a single column whose Java class is {@link Object} (for + * example a merge join key of type {@code ANY}), the primary + * {@code compare(Object, Object)} method and the generated bridge method have + * the same signature, so the emitted {@link java.util.Comparator} must not + * declare the bridge method twice or it fails to compile. */ + @Test void testMergeJoinComparatorWithObjectRowHasNoDuplicateBridge() { + final RelDataType rowType = + TYPE_FACTORY.createStructType( + ImmutableList.of( + TYPE_FACTORY.createSqlType(SqlTypeName.ANY)), + ImmutableList.of("anyField")); + final PhysType rowPhysType = + PhysTypeImpl.of(TYPE_FACTORY, rowType, JavaRowFormat.SCALAR, false); + final RelCollation collation = RelCollations.of(0); + final Expression comparator = + rowPhysType.generateMergeJoinComparator(collation); + final String generated = Expressions.toString(comparator); + // The primary compare method is still generated ... + assertThat(generated, containsString("public int compare(Object v0, Object v1)")); + // ... but the bridge compare(Object o0, Object o1) would collide with it and + // must therefore be omitted. + assertThat(generated, not(containsString("Object o0, Object o1"))); + } } From c2f08cffcc79659e7cceb8b3c13b4cec61dba3e7 Mon Sep 17 00:00:00 2001 From: Ruben Quesada Lopez Date: Thu, 2 Jul 2026 17:42:48 +0100 Subject: [PATCH 371/562] [CALCITE-7635] Simplification result of conjunction of comparisons depends on terms order --- .../org/apache/calcite/rex/RexSimplify.java | 15 ++++++++-- .../apache/calcite/rex/RexProgramTest.java | 30 ++++++++++++++++++- 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rex/RexSimplify.java b/core/src/main/java/org/apache/calcite/rex/RexSimplify.java index ae4b3501c955..58979c0f5c60 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexSimplify.java +++ b/core/src/main/java/org/apache/calcite/rex/RexSimplify.java @@ -800,7 +800,7 @@ private > RexNode simplifyComparison(RexCall e, } else { e2 = rexBuilder.makeCall(e.getParserPosition(), e.op, operands); } - return simplifyUsingPredicates(e2, clazz); + return simplifyUsingPredicates(e2, clazz, unknownAs); } @@ -1972,7 +1972,9 @@ private > RexNode simplifyAnd2ForUnknownAsFalse( // or weaken terms that are partially implied. // E.g. given predicate "x >= 5" and term "x between 3 and 10" // we weaken to term to "x between 5 and 10". - final RexNode term2 = simplifyUsingPredicates(term, clazz); + // Note: we use RexUnknownAs.FALSE because the current method + // simplifies AND expressions "For Unknown As False". + final RexNode term2 = simplifyUsingPredicates(term, clazz, FALSE); if (term2 != term) { terms.set(i, term = term2); } @@ -2097,7 +2099,7 @@ private > RexNode simplifyAnd2ForUnknownAsFalse( } private > RexNode simplifyUsingPredicates(RexNode e, - Class clazz) { + Class clazz, RexUnknownAs unknownAs) { if (predicates.pulledUpPredicates.isEmpty()) { return e; } @@ -2126,6 +2128,13 @@ private > RexNode simplifyUsingPredicates(RexNode e, } else if (rangeSet2.equals(RangeSets.rangeSetAll())) { // Range is always satisfied given these predicates; but nullability might // be problematic + if (unknownAs != UNKNOWN) { + // If unknownAs FALSE: row is already excluded for null input, so the IS_NOT_NULL + // guard is redundant, just return TRUE. + // If unknownAs TRUE: null rows pass regardless, and non-null rows also pass + // (range satisfied), the overall result is always TRUE. + return rexBuilder.makeLiteral(true); + } return simplify( rexBuilder.makeCall(RexUtil.getPos(e), SqlStdOperatorTable.IS_NOT_NULL, comparison.ref), RexUnknownAs.UNKNOWN); diff --git a/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java b/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java index 91af635e5203..e15af17d81fc 100644 --- a/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java +++ b/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java @@ -48,6 +48,7 @@ import org.apache.calcite.util.TimestampWithTimeZoneString; import org.apache.calcite.util.Util; +import com.google.common.collect.Collections2; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableRangeSet; @@ -2284,6 +2285,33 @@ private void checkExponentialCnf(int n) { checkSimplify(rexNode, "false"); } + /** Test case for + * [CALCITE-7635] + * Simplification result of conjunction of comparisons depends on terms order. */ + @Test void testSimplifyAndComparison() { + List args = + ImmutableList.of(lt(vInt(), literal(10)), + gt(vInt(), literal(0)), + lt(vInt(), literal(20))); + for (List params : Collections2.permutations(args)) { + checkSimplifyFilter( + and(params), + "SEARCH(?0.int0, Sarg[(0..10)])"); + } + } + + @Test void testSimplifyComparisonWithPredicates() { + RelOptPredicateList relOptPredicateList = + RelOptPredicateList.of(rexBuilder, + ImmutableList.of(lt(vInt(), literal(10)), gt(vInt(), literal(0)))); + checkSimplifyWithPredicates(lt(vInt(), literal(20)), relOptPredicateList, + RexUnknownAs.UNKNOWN, "IS NOT NULL(?0.int0)"); + checkSimplifyWithPredicates(lt(vInt(), literal(20)), relOptPredicateList, + RexUnknownAs.FALSE, "true"); + checkSimplifyWithPredicates(lt(vInt(), literal(20)), relOptPredicateList, + RexUnknownAs.TRUE, "true"); + } + /** Test case for * [CALCITE-7160] * Simplify AND/OR with DISTINCT predicates to SEARCH. */ @@ -4375,7 +4403,7 @@ private void checkSarg(String message, Sarg sarg, checkSimplifyFilter(ne(refNullable, literal(9)), relOptPredicateList, "false"); checkSimplifyFilter(ne(refNullable, literal(5)), relOptPredicateList, - "IS NOT NULL($0)"); + "true"); } /** Tests From 5e3f4a49c9a5101eb20aeb234ad7adf57bad85de Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Sun, 5 Jul 2026 20:01:47 +0800 Subject: [PATCH 372/562] [CALCITE-7645] AggregateUnionTransposeRule drops aggregate FILTER when rebuilding child aggregate calls Signed-off-by: xiedeyantu --- .../rules/AggregateUnionTransposeRule.java | 5 +-- .../apache/calcite/test/RelOptRulesTest.java | 16 ++++++++++ .../apache/calcite/test/RelOptRulesTest.xml | 32 +++++++++++++++++++ core/src/test/resources/sql/hep.iq | 19 +++++++++++ 4 files changed, 70 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rel/rules/AggregateUnionTransposeRule.java b/core/src/main/java/org/apache/calcite/rel/rules/AggregateUnionTransposeRule.java index 923311a4f5b2..614c510a4e18 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/AggregateUnionTransposeRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/AggregateUnionTransposeRule.java @@ -166,8 +166,9 @@ public AggregateUnionTransposeRule(Class aggregateClass, AggregateCall newCall = AggregateCall.create(origCall.getParserPosition(), origCall.getAggregation(), origCall.isDistinct(), origCall.isApproximate(), origCall.ignoreNulls(), - origCall.rexList, origCall.getArgList(), -1, origCall.distinctKeys, - origCall.collation, aggRel.getGroupSet().isEmpty(), input, null, + origCall.rexList, origCall.getArgList(), origCall.filterArg, + origCall.distinctKeys, origCall.collation, + aggRel.getGroupSet().isEmpty(), input, null, origCall.getName()); childAggCalls.set(i, newCall); } diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index d56bde9808fc..18777b1ad7bc 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -8641,6 +8641,22 @@ private void checkSemiJoinRuleOnAntiJoin(RelOptRule rule) { .check(); } + /** Test case for + * [CALCITE-7645] + * AggregateUnionTransposeRule drops aggregate FILTER when rebuilding + * pushed-down aggregate calls. */ + @Test void testAggregateUnionTransposeWithFilterAndNullableInput() { + final String sql = "select min(v) filter (where p)\n" + + "from (\n" + + " select * from (values (10, false), (100, true)) as t(v, p)\n" + + " union all\n" + + " select * from (values (cast(null as integer), true), (200, true)) as t(v, p)\n" + + ")"; + sql(sql) + .withRule(CoreRules.AGGREGATE_UNION_TRANSPOSE) + .check(); + } + /** If all inputs to UNION are already unique, AggregateUnionTransposeRule is * a no-op. */ @Test void testAggregateUnionTransposeWithAllInputsUnique() { diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index cdf6956ae4c6..2eb42b693c72 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -1308,6 +1308,38 @@ LogicalAggregate(group=[{0}], EXPR$1=[SUM($1)]) LogicalAggregate(group=[{0, 1}]) LogicalProject(DEPTNO=[$7], T=[2]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +]]> + + + + + + + + + + + diff --git a/core/src/test/resources/sql/hep.iq b/core/src/test/resources/sql/hep.iq index 18cdaf27222e..812663177a8d 100644 --- a/core/src/test/resources/sql/hep.iq +++ b/core/src/test/resources/sql/hep.iq @@ -103,6 +103,25 @@ EnumerableSort(sort0=[$1], dir0=[ASC]) EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t3):INTEGER], expr#9=[7698], expr#10=[=($t8, $t9)], expr#11=[CAST($t6):DECIMAL(12, 2)], expr#12=[500.00:DECIMAL(12, 2)], expr#13=[=($t11, $t12)], expr#14=[IS NOT TRUE($t13)], expr#15=[AND($t10, $t14)], MGR=[$t3], COMM=[$t6], $condition=[$t15]) EnumerableTableScan(table=[[scott, EMP]]) !plan + +# [CALCITE-7645] AggregateUnionTransposeRule must preserve aggregate FILTER when creating pushed-down aggregates +!set hep-rules " ++AGGREGATE_UNION_TRANSPOSE" + +select min(v) filter (where p) as m +from ( + select * from (values (10, false), (100, true)) as t(v, p) + union all + select * from (values (cast(null as integer), true), (200, true)) as t(v, p) +); ++-----+ +| M | ++-----+ +| 100 | ++-----+ +(1 row) + +!ok !set hep-rules original # Testing with the planner-rules shows that due to cost-based selection issues, From e252b54b5c23b448209f3c8686f0aa252be1c5ab Mon Sep 17 00:00:00 2001 From: Terran Date: Fri, 8 May 2026 16:31:32 +0800 Subject: [PATCH 373/562] [CALCITE-6242] Enhance lambda closure --- .../enumerable/RexToLixTranslator.java | 7 +- .../org/apache/calcite/plan/RelOptUtil.java | 13 ++ .../apache/calcite/rex/RexBiVisitorImpl.java | 12 +- .../apache/calcite/rex/RexProgramBuilder.java | 5 +- .../apache/calcite/rex/RexVisitorImpl.java | 11 +- .../calcite/runtime/CalciteResource.java | 6 + .../sql/validate/SqlAbstractConformance.java | 4 + .../calcite/sql/validate/SqlConformance.java | 22 ++ .../sql/validate/SqlConformanceEnum.java | 11 + .../validate/SqlDelegatingConformance.java | 4 + .../calcite/sql/validate/SqlLambdaScope.java | 42 ++-- .../sql/validate/SqlValidatorImpl.java | 31 ++- .../calcite/sql2rel/SqlToRelConverter.java | 23 +- .../runtime/CalciteResource.properties | 2 + .../calcite/test/SqlToRelConverterTest.java | 13 ++ .../apache/calcite/test/SqlValidatorTest.java | 198 +++++++++++++++++- .../calcite/test/SqlToRelConverterTest.xml | 11 + core/src/test/resources/sql/lambda.iq | 43 ++++ .../calcite/sql/parser/SqlParserTest.java | 5 + 19 files changed, 431 insertions(+), 32 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java index 15924844ca9d..ea4ff65127c3 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java @@ -1861,9 +1861,14 @@ private Result toInnerStorageType(Result result, Type storageType) { new ParameterExpression[rexLambdaRefs.size()]; for (int i = 0; i < rexLambdaRefs.size(); i++) { final RexLambdaRef rexLambdaRef = rexLambdaRefs.get(i); + // Declare lambda parameters as 'final' so that Janino can capture them + // from nested anonymous classes (e.g., nested lambdas like x -> y -> x + y). + // Janino requires captured local variables to be explicitly final, + // unlike javac which supports effectively-final variables. parameterExpressions[i] = Expressions.parameter( - typeFactory.getJavaClass(rexLambdaRef.getType()), rexLambdaRef.getName()); + Modifier.FINAL, typeFactory.getJavaClass(rexLambdaRef.getType()), + rexLambdaRef.getName()); } // Generate code for lambda expression body diff --git a/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java b/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java index 9b24cf2ad201..b615c31d8533 100644 --- a/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java +++ b/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java @@ -70,6 +70,7 @@ import org.apache.calcite.rex.RexExecutorImpl; import org.apache.calcite.rex.RexFieldAccess; import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexLambda; import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexLocalRef; import org.apache.calcite.rex.RexNode; @@ -3363,6 +3364,12 @@ private static RexShuttle pushShuttle(final Project project) { @Override public RexNode visitInputRef(RexInputRef ref) { return project.getProjects().get(ref.getIndex()); } + + @Override public RexNode visitLambda(RexLambda lambda) { + // Lambda body references are at a different scope level. + // Do not remap indices inside lambda body against this project. + return lambda; + } }; } @@ -3386,6 +3393,12 @@ private static RexShuttle pushShuttle(final Calc calc) { @Override public RexNode visitInputRef(RexInputRef ref) { return projects.get(ref.getIndex()); } + + @Override public RexNode visitLambda(RexLambda lambda) { + // Lambda body references are at a different scope level. + // Do not remap indices inside lambda body against this calc. + return lambda; + } }; } diff --git a/core/src/main/java/org/apache/calcite/rex/RexBiVisitorImpl.java b/core/src/main/java/org/apache/calcite/rex/RexBiVisitorImpl.java index 5a11bf771e0e..0c8f220bff29 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexBiVisitorImpl.java +++ b/core/src/main/java/org/apache/calcite/rex/RexBiVisitorImpl.java @@ -119,8 +119,18 @@ protected RexBiVisitorImpl(boolean deep) { return null; } + /** + * Visits a lambda expression. When {@code deep} is true, recurses into + * the lambda body so that analysis visitors (e.g. InputFinder) can discover + * field references inside the lambda. When {@code deep} is false, returns + * null without recursing — this is the shallow traversal mode used by + * visitors that only need top-level information. + */ @Override public R visitLambda(RexLambda lambda, P arg) { - return null; + if (!deep) { + return null; + } + return lambda.getExpression().accept(this, arg); } @Override public R visitNodeAndFieldIndex(RexNodeAndFieldIndex nodeAndFieldIndex, P arg) { diff --git a/core/src/main/java/org/apache/calcite/rex/RexProgramBuilder.java b/core/src/main/java/org/apache/calcite/rex/RexProgramBuilder.java index 2c8cdfbd9a39..ebeb12aa609e 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexProgramBuilder.java +++ b/core/src/main/java/org/apache/calcite/rex/RexProgramBuilder.java @@ -910,8 +910,9 @@ private abstract class RegisterShuttle extends RexShuttle { } @Override public RexNode visitLambda(RexLambda lambda) { - super.visitLambda(lambda); - return registerInternal(lambda); + // Lambda body references are at a different scope level. + // Do not validate or register lambda body indices against this program's input. + return lambda; } } diff --git a/core/src/main/java/org/apache/calcite/rex/RexVisitorImpl.java b/core/src/main/java/org/apache/calcite/rex/RexVisitorImpl.java index 6ebbe92679f6..d6d5931d9768 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexVisitorImpl.java +++ b/core/src/main/java/org/apache/calcite/rex/RexVisitorImpl.java @@ -118,8 +118,17 @@ protected RexVisitorImpl(boolean deep) { return null; } + /** + * Visits a lambda expression. When {@code deep} is true, recurses into + * the lambda body to analyze its sub-expressions (critical for InputFinder + * to detect field references inside lambda bodies during pushDownJoinConditions). + * When {@code deep} is false, returns null without recursing. + */ @Override public R visitLambda(RexLambda lambda) { - return null; + if (!deep) { + return null; + } + return lambda.getExpression().accept(this); } @Override public R visitLambdaRef(RexLambdaRef lambdaRef) { diff --git a/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java b/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java index ea8e1772c678..01ea9cbda0b1 100644 --- a/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java +++ b/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java @@ -244,9 +244,15 @@ ExInst columnNotFoundInTableDidYouMean(String a0, ExInst paramNotFoundInFunctionDidYouMean(String a0, String a1, String a2); + @BaseMessage("Lambda closure is not allowed in this conformance: reference to ''{0}'' from enclosing scope") + ExInst lambdaClosureNotAllowed(String identifier); + @BaseMessage("Param ''{0}'' not found in lambda expression ''{1}''") ExInst paramNotFoundInLambdaExpression(String a0, String a1); + @BaseMessage("Duplicate lambda parameter ''{0}''") + ExInst duplicateLambdaParameter(String paramName); + @BaseMessage("Operand {0} must be a query") ExInst needQueryOp(String a0); diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlAbstractConformance.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlAbstractConformance.java index 82f06a48d168..9ef7559c5329 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlAbstractConformance.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlAbstractConformance.java @@ -149,6 +149,10 @@ public abstract class SqlAbstractConformance implements SqlConformance { return SqlConformanceEnum.DEFAULT.allowQualifyingCommonColumn(); } + @Override public boolean allowLambdaClosure() { + return SqlConformanceEnum.DEFAULT.allowLambdaClosure(); + } + @Override public boolean allowAliasUnnestItems() { return SqlConformanceEnum.DEFAULT.allowAliasUnnestItems(); } diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlConformance.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlConformance.java index 32b4a03b90d9..3ccc860a0d71 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlConformance.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlConformance.java @@ -611,6 +611,28 @@ default boolean isColonFieldAccessAllowed() { */ boolean allowQualifyingCommonColumn(); + /** + * Whether to allow lambda expressions to access variables from enclosing + * scopes (closure semantics). + * + *

      For example, in a higher-order function context like: + * + * 

      +   * SELECT *
+   * FROM t1
+   * JOIN t2 ON EXISTS(t1.arr, x -> x = t2.v)
      + * + *

      The {@code t2.v} from the enclosing scope would be accessible inside + * the lambda body if closures are allowed. + * + *

      Among the built-in conformance levels, false in + * {@link SqlConformanceEnum#STRICT_92}, + * {@link SqlConformanceEnum#STRICT_99}, + * {@link SqlConformanceEnum#STRICT_2003}; + * true otherwise. + */ + boolean allowLambdaClosure(); + /** * Whether {@code VALUE} is allowed as an alternative to {@code VALUES} in * the parser. diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlConformanceEnum.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlConformanceEnum.java index 4475c4ce8096..fd083560e025 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlConformanceEnum.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlConformanceEnum.java @@ -433,6 +433,17 @@ public enum SqlConformanceEnum implements SqlConformance { } } + @Override public boolean allowLambdaClosure() { + switch (this) { + case STRICT_92: + case STRICT_99: + case STRICT_2003: + return false; + default: + return true; + } + } + @Override public boolean allowAliasUnnestItems() { switch (this) { case BIG_QUERY: diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlDelegatingConformance.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlDelegatingConformance.java index 0d415d8aec24..daac4b5c416a 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlDelegatingConformance.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlDelegatingConformance.java @@ -158,6 +158,10 @@ protected SqlDelegatingConformance(SqlConformance delegate) { return delegate.allowQualifyingCommonColumn(); } + @Override public boolean allowLambdaClosure() { + return delegate.allowLambdaClosure(); + } + @Override public boolean isValueAllowed() { return delegate.isValueAllowed(); } diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlLambdaScope.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlLambdaScope.java index 22003912d457..0db4b457115b 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlLambdaScope.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlLambdaScope.java @@ -21,15 +21,12 @@ import org.apache.calcite.sql.SqlLambda; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.type.SqlTypeName; -import org.apache.calcite.util.Litmus; import org.checkerframework.checker.nullness.qual.Nullable; import java.util.HashMap; import java.util.Map; -import static com.google.common.base.Preconditions.checkArgument; - import static org.apache.calcite.util.Static.RESOURCE; /** @@ -54,7 +51,10 @@ public SqlLambdaScope( /** True if the identifier matches one of the parameter names. */ public boolean isParameter(SqlIdentifier id) { - return this.parameterTypes.containsKey(id.toString()); + final SqlNameMatcher nameMatcher = validator.catalogReader.nameMatcher(); + final String name = id.getSimple(); + return parameterTypes.keySet().stream() + .anyMatch(paramName -> nameMatcher.matches(paramName, name)); } @Override public SqlNode getNode() { @@ -62,21 +62,35 @@ public boolean isParameter(SqlIdentifier id) { } @Override public SqlQualified fullyQualify(SqlIdentifier identifier) { - boolean found = lambdaExpr.getParameters() - .stream() - .anyMatch(param -> param.equalsDeep(identifier, Litmus.IGNORE)); - if (found) { - return SqlQualified.create(this, 1, null, identifier); - } else { + if (identifier.isSimple()) { + final SqlNameMatcher nameMatcher = validator.catalogReader.nameMatcher(); + final String name = identifier.getSimple(); + boolean found = lambdaExpr.getParameters() + .stream() + .anyMatch(param -> + nameMatcher.matches(((SqlIdentifier) param).getSimple(), name)); + if (found) { + return SqlQualified.create(this, 1, null, identifier); + } + } + if (!validator.config().conformance().allowLambdaClosure()) { throw validator.newValidationError(identifier, - RESOURCE.paramNotFoundInLambdaExpression(identifier.toString(), lambdaExpr.toString())); + RESOURCE.lambdaClosureNotAllowed(identifier.toString())); } + return parent.fullyQualify(identifier); } @Override public @Nullable RelDataType resolveColumn(String columnName, SqlNode ctx) { - checkArgument(parameterTypes.containsKey(columnName), - "column %s not found", columnName); - return parameterTypes.get(columnName); + final SqlNameMatcher nameMatcher = validator.catalogReader.nameMatcher(); + for (Map.Entry entry : parameterTypes.entrySet()) { + if (nameMatcher.matches(entry.getKey(), columnName)) { + return entry.getValue(); + } + } + // Delegate to parent scope for nested lambda closure resolution. + // In a nested lambda like x -> EXISTS(arr, y -> x + y), the inner lambda + // scope does not contain 'x', but the outer lambda scope does. + return parent.resolveColumn(columnName, ctx); } public Map getParameterTypes() { diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java index 26faab77bc68..2897202926a0 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java @@ -3353,10 +3353,9 @@ private void registerQuery( alias, lambdaNamespace, forceNullable); - operands = call.getOperandList(); - for (int i = 0; i < operands.size(); i++) { - registerOperandSubQueries(parentScope, call, i); - } + // Register sub-queries inside the body under lambdaScope, so that + // nested lambdas can resolve outer lambda parameters. + registerOperandSubQueries(lambdaScope, call, 1); break; case WITH: @@ -6642,6 +6641,30 @@ public void setOriginal(SqlNode expr, SqlNode original) { final LambdaNamespace ns = getNamespaceOrThrow(lambdaExpr).unwrap(LambdaNamespace.class); + // Check for duplicate lambda parameter names + final SqlNameMatcher nameMatcher = catalogReader.nameMatcher(); + final Set seen = nameMatcher.createSet(); + for (SqlNode param : lambdaExpr.getParameters()) { + final String name = ((SqlIdentifier) param).getSimple(); + if (!seen.add(name)) { + throw newValidationError(param, + RESOURCE.duplicateLambdaParameter(name)); + } + // Check against enclosing lambda scopes: x -> ... x -> ... + SqlValidatorScope parentScope = scope.getParent(); + while (parentScope instanceof DelegatingScope) { + if (parentScope instanceof SqlLambdaScope) { + final SqlLambdaScope parentLambda = (SqlLambdaScope) parentScope; + if (parentLambda.getParameterTypes().keySet().stream() + .anyMatch(p -> nameMatcher.matches(p, name))) { + throw newValidationError(param, + RESOURCE.duplicateLambdaParameter(name)); + } + } + parentScope = ((DelegatingScope) parentScope).getParent(); + } + } + deriveType(scope, lambdaExpr.getExpression()); RelDataType type = deriveTypeImpl(scope, lambdaExpr); setValidatedNodeType(lambdaExpr, type); diff --git a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java index ed6d52491a42..240b8bd02b0d 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java @@ -2480,6 +2480,13 @@ private RexNode convertLambda(Blackboard bb, SqlNode node) { final SqlLambdaScope scope = (SqlLambdaScope) validator().getLambdaScope(call); final Map nameToNodeMap = new HashMap<>(); + // For nested lambdas, inherit the parent blackboard's nameToNodeMap so that + // the inner lambda can resolve references to outer lambda parameters. + // e.g., in x -> EXISTS(arr, y -> x + y = 4), the inner lambda's blackboard + // needs access to "X" from the outer lambda's nameToNodeMap. + if (bb.nameToNodeMap != null) { + nameToNodeMap.putAll(bb.nameToNodeMap); + } final List parameters = new ArrayList<>(scope.getParameterTypes().size()); final Map parameterTypes = scope.getParameterTypes(); @@ -5759,11 +5766,16 @@ void setRoot(List inputs) { SqlQualified qualified) { if (nameToNodeMap != null && qualified.prefixLength == 1) { RexNode node = nameToNodeMap.get(qualified.identifier.names.get(0)); - if (node == null) { + if (node != null) { + return Pair.of(node, null); + } + // If the identifier is not found in nameToNodeMap and the current scope + // is a lambda scope, fall through to standard scope resolution to allow + // external references (e.g., t2.v in a JOIN ON lambda expression). + if (!(scope instanceof SqlLambdaScope)) { throw new AssertionError("Unknown identifier '" + qualified.identifier + "' encountered while expanding expression"); } - return Pair.of(node, null); } final SqlNameMatcher nameMatcher = scope.getValidator().getCatalogReader().nameMatcher(); @@ -5782,6 +5794,13 @@ void setRoot(List inputs) { // preserved. final SqlValidatorScope ancestorScope = resolve.scope; boolean isParent = ancestorScope != scope; + // When in a lambda scope, external references to tables that are part + // of the current blackboard's inputs should be resolved locally, not + // as correlation variables. The lambda blackboard inherits inputs from + // its parent blackboard. + if (isParent && scope instanceof SqlLambdaScope && inputs != null) { + isParent = false; + } if ((inputs != null) && !isParent) { final LookupContext rels = new LookupContext(this, inputs, systemFieldList.size()); diff --git a/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties b/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties index 49552b41985d..27e52cbce900 100644 --- a/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties +++ b/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties @@ -398,4 +398,6 @@ SelectByCannotWithGroupBy=SELECT BY cannot be used with GROUP BY SelectByCannotWithOrderBy=SELECT BY cannot be used with ORDER BY DescriptorMustBeIdentifier=The argument of DESCRIPTOR must be an identifier MeasureAliasDuplicate=Duplicate name ''{0}'' in MATCH_RECOGNIZE MEASURE alias list +LambdaClosureNotAllowed=Lambda closure is not allowed in this conformance: reference to ''{0}'' from enclosing scope +DuplicateLambdaParameter=Duplicate lambda parameter ''{0}'' # End CalciteResource.properties diff --git a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java index 1016acc41078..6b3255e653c7 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java @@ -182,6 +182,19 @@ public static void checkActualAndReferenceFiles() { .ok(); } + /** Test case for nested lambda: inner lambda references outer lambda + * parameter. Verifies that 'x' in the inner lambda is resolved from the + * outer lambda scope, not treated as a table column name. */ + @Test void testNestedLambdaExpression() { + final String sql = + "select \"EXISTS\"(array(1,2,3), x -> \"EXISTS\"(array(1,2,3), y -> x + y = 4))"; + fixture() + .withFactory(c -> + c.withOperatorTable(t -> SqlValidatorTest.operatorTableFor(SqlLibrary.SPARK))) + .withSql(sql) + .ok(); + } + @Test void testDotLiteralAfterRow() { final String sql = "select row(1,2).\"EXPR$1\" from emp"; sql(sql).ok(); diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index 0fb4fa4055b6..123dbe01d018 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -8607,7 +8607,10 @@ void testGroupExpressionEquivalenceParams() { /** Test case for * [CALCITE-3679] - * Allow lambda expressions in SQL queries. */ + * Allow lambda expressions in SQL queries. + * [CALCITE-6242] + * Enhance lambda closure parsing. + * */ @Test void testHigherOrderFunction() { final SqlValidatorFixture s = fixture() .withOperatorTable(MockSqlOperatorTable.standard().extend()); @@ -8621,6 +8624,10 @@ void testGroupExpressionEquivalenceParams() { .type("RecordType(INTEGER NOT NULL EXPR$0) NOT NULL"); s.withSql("select HIGHER_ORDER_FUNCTION2(1, () -> 0.1)") .type("RecordType(INTEGER NOT NULL EXPR$0) NOT NULL"); + s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> x + 1 + ^emp.deptno^) from emp") + .ok(); + s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> x + 1 + ^deptno^) from emp") + .ok(); // test for type check s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> ^x + 1^)") @@ -8638,13 +8645,70 @@ void testGroupExpressionEquivalenceParams() { .fails("Cannot apply '(?s).*HIGHER_ORDER_FUNCTION' to arguments of type " + "'HIGHER_ORDER_FUNCTION\\(, ANY>\\)'.*"); - // test for illegal parameters - s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> x + 1 + ^emp.deptno^) from emp") - .fails("Param 'EMP\\.DEPTNO' not found in lambda expression " - + "'\\(`X`, `Y`\\) -> `X` \\+ 1 \\+ `EMP`\\.`DEPTNO`'"); + } + + /** Test case for lambda closure conformance checking. + * Tests that lambda expressions can or cannot access variables from enclosing + * scopes based on the SQL conformance level. + * [CALCITE-6242] + * Enhance lambda closure parsing. + * */ + @Test void testLambdaClosureConformance() { + final SqlValidatorFixture s = fixture() + .withOperatorTable(MockSqlOperatorTable.standard().extend()); + + // Lambda accessing outer scope variable (closure) + // In DEFAULT conformance, closure is allowed + s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> x + 1 + deptno) from emp") + .withConformance(SqlConformanceEnum.DEFAULT) + .ok(); + + // In STRICT_92, closure is NOT allowed s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> x + 1 + ^deptno^) from emp") - .fails("Param 'DEPTNO' not found in lambda expression " - + "'\\(`X`, `Y`\\) -> `X` \\+ 1 \\+ `DEPTNO`'"); + .withConformance(SqlConformanceEnum.STRICT_92) + .fails("Lambda closure is not allowed in this conformance: " + + "reference to 'DEPTNO' from enclosing scope"); + + // In STRICT_99, closure is NOT allowed + s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> x + 1 + ^deptno^) from emp") + .withConformance(SqlConformanceEnum.STRICT_99) + .fails("Lambda closure is not allowed in this conformance: " + + "reference to 'DEPTNO' from enclosing scope"); + + // In STRICT_2003, closure is NOT allowed + s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> x + 1 + ^deptno^) from emp") + .withConformance(SqlConformanceEnum.STRICT_2003) + .fails("Lambda closure is not allowed in this conformance: " + + "reference to 'DEPTNO' from enclosing scope"); + + // In BABEL conformance, closure is allowed + s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> x + 1 + deptno) from emp") + .withConformance(SqlConformanceEnum.BABEL) + .ok(); + + // In LENIENT conformance, closure is allowed + s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> x + 1 + deptno) from emp") + .withConformance(SqlConformanceEnum.LENIENT) + .ok(); + + // Lambda using only its own parameters (no closure) - should always work + s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> x + 1) from emp") + .withConformance(SqlConformanceEnum.STRICT_92) + .ok(); + + s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> y) from emp") + .withConformance(SqlConformanceEnum.STRICT_92) + .ok(); + + // Test with qualified column name in closure + s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> x + 1 + emp.deptno) from emp") + .withConformance(SqlConformanceEnum.DEFAULT) + .ok(); + + s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> x + 1 + ^emp.deptno^) from emp") + .withConformance(SqlConformanceEnum.STRICT_92) + .fails("Lambda closure is not allowed in this conformance: " + + "reference to 'EMP.DEPTNO' from enclosing scope"); } /** Test case for [CALCITE-7193] @@ -8734,6 +8798,126 @@ void testGroupExpressionEquivalenceParams() { .assertBindType(is("RecordType(INTEGER ?0)")); } + /** Test case for + * [CALCITE-6242] + * Enhance lambda closure parsing. + * Tests that nested lambda expressions validate correctly: in the + * expression {@code x -> EXISTS(arr, y -> x + y = 4)}, the inner lambda + * references 'x' from the outer lambda's scope. 'x' should not be treated + * like a table column name. */ + @Test void testNestedLambdaClosure() { + final SqlOperatorTable opTable = operatorTableFor(SqlLibrary.SPARK); + + // Nested lambda: inner lambda references outer lambda parameter + sql("select \"EXISTS\"(array(1,2,3), x -> \"EXISTS\"(array(1,2,3), y -> x + y = 4))") + .withOperatorTable(opTable) + .ok(); + + // Nested lambda with FROM clause: outer lambda parameter 'x' should resolve + // from the outer lambda scope, not as a table column + sql("select \"EXISTS\"(array(1,2,3), x -> \"EXISTS\"(array(1,2,3), y -> x + y > deptno))" + + " from emp") + .withOperatorTable(opTable) + .ok(); + + // In STRICT mode, inner lambda referencing outer lambda param is treated + // as closure and is rejected + sql("select \"EXISTS\"(array(1,2,3), x -> \"EXISTS\"(array(1,2,3), y -> ^x^ + y = 4))" + + " from emp") + .withOperatorTable(opTable) + .withConformance(SqlConformanceEnum.STRICT_2003) + .fails("Lambda closure is not allowed in this conformance: " + + "reference to 'X' from enclosing scope"); + } + + /** Test case for + * [CALCITE-6242] + * Enhance lambda closure parsing. + * Tests that lambda parameter names follow the same case-sensitivity + * rules as other identifiers, including quoting. */ + @Test void testLambdaParameterCaseSensitivity() { + final SqlOperatorTable opTable = operatorTableFor(SqlLibrary.SPARK); + + // Case-insensitive mode with UNCHANGED casing: + // parameter defined as 'x', referenced as 'X' should match + final SqlValidatorFixture insensitive = fixture() + .withCaseSensitive(false) + .withUnquotedCasing(Casing.UNCHANGED) + .withOperatorTable(opTable); + + insensitive.withSql("select \"EXISTS\"(array(1,2,3), x -> x + 1 > 0)").ok(); + insensitive.withSql("select \"EXISTS\"(array(1,2,3), x -> X + 1 > 0)").ok(); + insensitive.withSql("select \"EXISTS\"(array(1,2,3), X -> x + 1 > 0)").ok(); + + // Nested lambda: inner lambda references outer parameter with different case + insensitive.withSql("select \"EXISTS\"(array(1,2,3)," + + " x -> \"EXISTS\"(array(1,2,3), y -> X + y = 4))").ok(); + + // Case-sensitive mode with UNCHANGED casing: + // parameter defined as 'x', referenced as 'X' should NOT match + final SqlValidatorFixture sensitive = fixture() + .withCaseSensitive(true) + .withUnquotedCasing(Casing.UNCHANGED) + .withQuoting(Quoting.DOUBLE_QUOTE) + .withOperatorTable(opTable); + + // Same case: should work + sensitive.withSql("select \"EXISTS\"(array(1,2,3), x -> x + 1 > 0)").ok(); + + // Different case: should fail in case-sensitive mode + sensitive.withSql("select \"EXISTS\"(array(1,2,3), x -> ^X^ + 1 > 0)") + .fails("Column 'X' not found in any table"); + + // Quoted parameter names: quoting preserves case + // In default config (unquotedCasing=TO_UPPER), quoted lowercase stays lowercase + final SqlValidatorFixture defaultFixture = fixture() + .withOperatorTable(opTable); + + // Unquoted parameter 'x' is converted to 'X', unquoted reference 'x' is also 'X' + defaultFixture.withSql("select \"EXISTS\"(array(1,2,3), x -> x + 1 > 0)").ok(); + } + + /** Test case for + * [CALCITE-6242] + * Enhance lambda closure parsing. + * Tests that duplicate lambda parameter names are rejected, both within + * a single lambda and across nested lambdas (shadowing). */ + @Test void testLambdaDuplicateParameterName() { + final SqlOperatorTable opTable = operatorTableFor(SqlLibrary.SPARK); + final SqlValidatorFixture f = fixture().withOperatorTable(opTable); + + // Same parameter name used twice in one lambda + f.withSql("select HIGHER_ORDER_FUNCTION(1, (x, ^x^) -> x + 1)") + .fails("Duplicate lambda parameter 'X'"); + + // Same parameter name in nested lambdas (shadowing) + f.withSql("select \"EXISTS\"(array(1,2,3)," + + " x -> \"EXISTS\"(array(1,2,3), ^x^ -> x + 1 > 0))") + .fails("Duplicate lambda parameter 'X'"); + + // Different parameter names: should work + f.withSql("select \"EXISTS\"(array(1,2,3)," + + " x -> \"EXISTS\"(array(1,2,3), y -> x + y = 4))").ok(); + + // Case-insensitive: x and X are same parameter (shadowing detected) + final SqlValidatorFixture insensitive = fixture() + .withCaseSensitive(false) + .withUnquotedCasing(Casing.UNCHANGED) + .withOperatorTable(opTable); + insensitive.withSql("select \"EXISTS\"(array(1,2,3)," + + " x -> \"EXISTS\"(array(1,2,3), ^X^ -> X + 1 > 0))") + .fails("Duplicate lambda parameter 'X'"); + + // Case-sensitive mode: x and X are different parameters (no shadowing) + final SqlValidatorFixture sensitive = fixture() + .withCaseSensitive(true) + .withUnquotedCasing(Casing.UNCHANGED) + .withQuoting(Quoting.DOUBLE_QUOTE) + .withOperatorTable(opTable); + sensitive.withSql("select \"EXISTS\"(array(1,2,3)," + + " x -> \"EXISTS\"(array(1,2,3), X -> X + 1 > 0))").ok(); + } + @Test void testPercentileFunctionsBigQuery() { final SqlOperatorTable opTable = operatorTableFor(SqlLibrary.BIG_QUERY); final String sql = "select\n" diff --git a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml index bd6a2246a143..f2ab8fec37e7 100644 --- a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml @@ -5963,6 +5963,17 @@ LogicalProject(D2=[$0], D3=[$1]) LogicalFilter(condition=[=($1, $0)]) LogicalProject(D4=[+($0, 4)], D5=[+($0, 5)], D6=[+($0, 6)]) LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) +]]> + + + + + "EXISTS"(array(1,2,3), y -> x + y = 4))]]> + + + EXISTS(ARRAY(1, 2, 3), (Y) -> =(+(X, Y), 4)))]) + LogicalValues(tuples=[[{ 0 }]]) ]]> diff --git a/core/src/test/resources/sql/lambda.iq b/core/src/test/resources/sql/lambda.iq index 207543ec77cb..82808ff19dad 100644 --- a/core/src/test/resources/sql/lambda.iq +++ b/core/src/test/resources/sql/lambda.iq @@ -102,3 +102,46 @@ select "EXISTS"(array[array[1, 2], array[3, 4]], x -> x[1] = 1); (1 row) !ok + +# [CALCITE-6242] Enhance lambda closure parsing +select * + from (select array(1, 2, 3) as arr) as t1 inner join + (select 1 as v) as t2 on "EXISTS"(arr, x -> x = t2.v); ++-----------+---+ +| ARR | V | ++-----------+---+ +| [1, 2, 3] | 1 | ++-----------+---+ +(1 row) + +!ok + +# Nested lambda: x -> (y -> x + y). +# The inner closure 'x' refers to the outer lambda parameter, +# and must not be treated like a table column name. +# For x in (1,2,3) and y in (1,2,3), x + y = 4 holds (e.g. 1+3, 2+2, 3+1). +select "EXISTS"(array(1, 2, 3), x -> "EXISTS"(array(1, 2, 3), y -> x + y = 4)); ++--------+ +| EXPR$0 | ++--------+ +| true | ++--------+ +(1 row) + +!ok + +# Nested lambda with a FROM clause whose column is also named 'x'. +# The inner closure 'x' must resolve to the outer lambda parameter +# (values 1,2,3), NOT to the table column 'x' (value 100). +# If 'x' were resolved as the column, 100 + y = 4 would never hold +# and the result would be false. +select "EXISTS"(array(1, 2, 3), x -> "EXISTS"(array(1, 2, 3), y -> x + y = 4)) as r + from (select 100 as x) as t; ++------+ +| R | ++------+ +| true | ++------+ +(1 row) + +!ok diff --git a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java index bb7566c9904c..e61cc66d5662 100644 --- a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java +++ b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java @@ -9945,6 +9945,11 @@ private static Consumer> checkWarnings( sql("select 1 || (a, b) ^->^ a + b") .fails(errorMessage2); + + // Nested lambda: inner lambda in a function call within the outer lambda body + sql("select higher_order_func(x -> higher_order_func(y -> x + y, 1), 1) from t") + .ok("SELECT `HIGHER_ORDER_FUNC`(`X` -> `HIGHER_ORDER_FUNC`(`Y` -> (`X` + `Y`), 1), 1)\n" + + "FROM `T`"); } /** From 3c657a2ba43a6b5d572562f6f4a24c532d04320d Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Sun, 5 Jul 2026 13:15:39 +0800 Subject: [PATCH 374/562] [CALCITE-7643] AggregateMinMaxToLimitRule drops FILTER condition for filtered MIN/MAX aggregates Signed-off-by: xiedeyantu --- .../rel/rules/AggregateMinMaxToLimitRule.java | 23 ++++++--- .../apache/calcite/test/RelOptRulesTest.java | 13 +++++ .../apache/calcite/test/RelOptRulesTest.xml | 29 +++++++++++ core/src/test/resources/sql/planner.iq | 49 +++++++++++++++++++ 4 files changed, 108 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rel/rules/AggregateMinMaxToLimitRule.java b/core/src/main/java/org/apache/calcite/rel/rules/AggregateMinMaxToLimitRule.java index 51c29961769b..e76b3de3deb4 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/AggregateMinMaxToLimitRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/AggregateMinMaxToLimitRule.java @@ -92,12 +92,23 @@ protected AggregateMinMaxToLimitRule(Config config) { // MIN is ASC, MAX is DESC final boolean isDesc = aggCall.getAggregation().kind == SqlKind.MAX; - RexNode subQuery = builder.scalarQuery(b -> b.push(aggInput) - .project(r) - .filter(b.isNotNull(r)) - .sortLimit(0, 1, - isDesc ? builder.desc(r) : r) - .build()); + RexNode subQuery = builder.scalarQuery(b -> { + b.push(aggInput); + final RexNode inputField = b.field(idx); + final List predicates = new ArrayList<>(); + predicates.add(b.isNotNull(inputField)); + if (aggCall.hasFilter()) { + // filterArg references the aggregate input, so apply it before projection + predicates.add(b.field(aggCall.filterArg)); + } + b.filter(predicates) + // Scalar sub-query must return only the MIN/MAX argument + .project(inputField); + final RexNode sortField = b.field(0); + return b.sortLimit(0, 1, + isDesc ? b.desc(sortField) : sortField) + .build(); + }); newProjects.add(subQuery); } diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index 18777b1ad7bc..d94ae4652659 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -11699,6 +11699,19 @@ private void checkLoptOptimizeJoinRule(LoptOptimizeJoinRule rule) { .check(); } + /** Test case for + * [CALCITE-7643] + * AggregateMinMaxToLimitRule drops FILTER condition for filtered + * MIN/MAX aggregates. */ + @Test void testAggregateMinMaxToLimitRuleWithFilter() { + final String sql = "select min(v) filter (where p), max(v) filter (where p)\n" + + "from (values (10, false), (100, true), (200, true),\n" + + " (300, false), (cast(null as integer), true)) as t(v, p)"; + sql(sql) + .withRule(CoreRules.AGGREGATE_MIN_MAX_TO_LIMIT) + .check(); + } + @Test void testOuterJoinForDphyp() { HepProgram program = new HepProgramBuilder() .addMatchOrder(HepMatchOrder.BOTTOM_UP) diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index 2eb42b693c72..64061b8910f8 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -931,6 +931,35 @@ LogicalSort(sort0=[$0], dir0=[DESC], fetch=[1]) + + + + + + + + + + + Date: Sat, 4 Jul 2026 15:25:48 +0530 Subject: [PATCH 375/562] Change typo in Pig script schema mismatch error message --- .../src/main/java/org/apache/calcite/piglet/PigRelBuilder.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piglet/src/main/java/org/apache/calcite/piglet/PigRelBuilder.java b/piglet/src/main/java/org/apache/calcite/piglet/PigRelBuilder.java index df9f57a30365..87c467209b01 100644 --- a/piglet/src/main/java/org/apache/calcite/piglet/PigRelBuilder.java +++ b/piglet/src/main/java/org/apache/calcite/piglet/PigRelBuilder.java @@ -246,7 +246,7 @@ public RelBuilder scan(RelOptTable userSchema, String... tableNames) { // If both schemas are valid, they must be compatible throw new IllegalArgumentException( "Pig script schema does not match database schema for table " + names + ".\n" - + "\t Scrip schema: " + userSchema.getRowType().getFullTypeString() + "\n" + + "\t Script schema: " + userSchema.getRowType().getFullTypeString() + "\n" + "\t Database schema: " + systemSchema.getRowType().getFullTypeString()); } // We choose to use systemSchema if it is valid From 7bac14eb9d5864135d76f0aff0ba2b52dd7aaf13 Mon Sep 17 00:00:00 2001 From: Anas Khan <83116240+anxkhn@users.noreply.github.com> Date: Tue, 7 Jul 2026 10:50:59 +0530 Subject: [PATCH 376/562] Change typo in Pig aggregate UDF exception message --- .../main/java/org/apache/calcite/piglet/PigRelUdfConverter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piglet/src/main/java/org/apache/calcite/piglet/PigRelUdfConverter.java b/piglet/src/main/java/org/apache/calcite/piglet/PigRelUdfConverter.java index bade6463075f..b378002e0aa0 100644 --- a/piglet/src/main/java/org/apache/calcite/piglet/PigRelUdfConverter.java +++ b/piglet/src/main/java/org/apache/calcite/piglet/PigRelUdfConverter.java @@ -183,7 +183,7 @@ static SqlAggFunction getSqlAggFuncForPigUdf(RexCall call) { ((ScalarFunctionImpl) pigUdf.getFunction()).method.getDeclaringClass(); if (Accumulator.class.isAssignableFrom(udfClass)) { throw new UnsupportedOperationException( - "Cannot find corresponding SqlAgg func for Pig aggegate " + pigUdfClassName); + "Cannot find corresponding SqlAgg func for Pig aggregate " + pigUdfClassName); } } return sqlAggFunction; From ed7ee43900832fb54a49996d9deee17179c3662b Mon Sep 17 00:00:00 2001 From: Anas Khan <83116240+anxkhn@users.noreply.github.com> Date: Tue, 7 Jul 2026 10:50:59 +0530 Subject: [PATCH 377/562] Change typo in TableNamespace exception message --- .../java/org/apache/calcite/sql/validate/TableNamespace.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/calcite/sql/validate/TableNamespace.java b/core/src/main/java/org/apache/calcite/sql/validate/TableNamespace.java index abc3cd1abf51..0b4b38336eca 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/TableNamespace.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/TableNamespace.java @@ -126,7 +126,7 @@ public TableNamespace extend(SqlNodeList extendList) { final SqlValidatorTable validatorTable = requireNonNull( relOptTable.unwrap(SqlValidatorTable.class), - () -> "cant unwrap SqlValidatorTable from " + relOptTable); + () -> "can't unwrap SqlValidatorTable from " + relOptTable); return new TableNamespace(validator, validatorTable, ImmutableList.of()); } return new TableNamespace(validator, table, extendedFields); From 17ed107a0432c1c34c0f4f16796f4c8261ce4177 Mon Sep 17 00:00:00 2001 From: Anas Khan <83116240+anxkhn@users.noreply.github.com> Date: Fri, 3 Jul 2026 10:31:54 +0530 Subject: [PATCH 378/562] [CALCITE-7555] Sarg.compareTo() collapses semantically different search arguments that have different nullAs Sarg.equals and Sarg.hashCode both account for nullAs, but compareTo ordered only by the range set. Two Sargs over the same ranges but with different null semantics (for example RexUnknownAs.UNKNOWN versus RexUnknownAs.FALSE) were therefore unequal as objects yet compared as 0, violating the Comparable contract. A sorted collection keyed on Sarg, such as a TreeSet or TreeMap, could silently drop one of two semantically distinct search arguments. Tie-break on nullAs after the range-set comparison so that compareTo agrees with equals. --- .../java/org/apache/calcite/util/Sarg.java | 10 ++++++- .../apache/calcite/rex/RexProgramTest.java | 26 +++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/calcite/util/Sarg.java b/core/src/main/java/org/apache/calcite/util/Sarg.java index fc1373571747..89e4c6e9a1c8 100644 --- a/core/src/main/java/org/apache/calcite/util/Sarg.java +++ b/core/src/main/java/org/apache/calcite/util/Sarg.java @@ -199,7 +199,15 @@ public StringBuilder printTo(StringBuilder sb, } @Override public int compareTo(Sarg o) { - return RangeSets.compare(rangeSet, o.rangeSet); + int c = RangeSets.compare(rangeSet, o.rangeSet); + if (c != 0) { + return c; + } + // Tie-break on nullAs so that compareTo is consistent with equals and + // hashCode, which both account for nullAs. Two Sargs over the same ranges + // but with different null semantics must not compare as equal, otherwise a + // sorted collection keyed on Sarg would silently drop one of them. + return Integer.compare(nullAs.ordinal(), o.nullAs.ordinal()); } @Override public int hashCode() { diff --git a/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java b/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java index e15af17d81fc..60fd60a83751 100644 --- a/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java +++ b/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java @@ -68,6 +68,7 @@ import java.util.List; import java.util.Map; import java.util.TreeMap; +import java.util.TreeSet; import java.util.function.Supplier; import static org.apache.calcite.test.Matchers.isRangeSet; @@ -4126,6 +4127,31 @@ private void checkSarg(String message, Sarg sarg, assertThat(sarg.isComplementedPoints(), is(true)); } + /** Unit test for + * [CALCITE-7555] + * {@code Sarg.compareTo} collapses semantically different search arguments + * that have different {@code nullAs}. + * + *

      {@link Sarg#equals} and {@link Sarg#hashCode} account for + * {@link Sarg#nullAs}, but {@link Sarg#compareTo} used to order only by the + * range set, so two Sargs over the same ranges but with different null + * semantics were unequal yet compared as {@code 0}. A sorted collection keyed + * on Sarg would then silently drop one of them. */ + @Test void testSargCompareToIsConsistentWithEquals() { + final ImmutableRangeSet singleton = + ImmutableRangeSet.of(Range.singleton(1)); + final Sarg unknown = Sarg.of(RexUnknownAs.UNKNOWN, singleton); + final Sarg falseSarg = Sarg.of(RexUnknownAs.FALSE, singleton); + + assertFalse(unknown.equals(falseSarg)); + assertThat(unknown.compareTo(falseSarg) == 0, is(false)); + + final TreeSet> values = new TreeSet<>(); + values.add(unknown); + values.add(falseSarg); + assertThat(values, hasSize(2)); + } + @Test void testInterpreter() { assertThat(eval(trueLiteral), is(true)); assertThat(eval(nullInt), is(NullSentinel.INSTANCE)); From 6197ddb6debbf85957c5edcf17a96b5be71b4344 Mon Sep 17 00:00:00 2001 From: Ruben Quesada Lopez Date: Tue, 7 Jul 2026 11:47:03 +0100 Subject: [PATCH 379/562] [CALCITE-7641] Materialize view rules with UnionRewritingPullProgram on a HepPlanner throws IllegalArgumentException --- .../MaterializedViewAggregateRule.java | 5 + .../materialize/MaterializedViewJoinRule.java | 9 +- .../test/MaterializedViewRelOptRulesTest.java | 112 ++++++++++++++++++ 3 files changed, 124 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rel/rules/materialize/MaterializedViewAggregateRule.java b/core/src/main/java/org/apache/calcite/rel/rules/materialize/MaterializedViewAggregateRule.java index 404d0200bab1..538723661d60 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/materialize/MaterializedViewAggregateRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/materialize/MaterializedViewAggregateRule.java @@ -23,6 +23,7 @@ import org.apache.calcite.plan.hep.HepPlanner; import org.apache.calcite.plan.hep.HepProgram; import org.apache.calcite.plan.hep.HepProgramBuilder; +import org.apache.calcite.plan.hep.HepRelVertex; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Aggregate; import org.apache.calcite.rel.core.AggregateCall; @@ -259,6 +260,10 @@ public abstract class MaterializedViewAggregateRule[CALCITE-7641] + * Materialize view rules with UnionRewritingPullProgram on a HepPlanner + * throws IllegalArgumentException. */ + @Test void testJoinAggregateMaterializationNoAggregateFuncs9Hep() { + // Tester using a HepPlanner instead of Volcano + final MaterializedViewTester hepTester = + new MaterializedViewTester() { + @Override protected List optimize(RelNode queryRel, + List materializationList) { + // Dummy UnionRewritingPullProgram + final HepProgram unionRewritingPullProgram = new HepProgramBuilder().build(); + // MaterializedViewRule with UnionRewritingPullProgram + final HepProgram mainProgram = new HepProgramBuilder() + .addRuleInstance(MaterializedViewOnlyAggregateRule.Config.DEFAULT + .withUnionRewritingPullProgram(unionRewritingPullProgram).toRule()) + .build(); + final HepPlanner hepPlanner = new HepPlanner(mainProgram); + final Program program = + (planner, rel, requiredOutputTraits, materializations, lattices) -> { + for (RelOptMaterialization materialization : materializations) { + planner.addMaterialization(materialization); + } + planner.setRoot(rel); + return planner.findBestExp(); + }; + return ImmutableList.of( + program.run(hepPlanner, queryRel, queryRel.getCluster().traitSet(), + materializationList, ImmutableList.of())); + } + }; + + String materialize = "select \"depts\".\"deptno\", \"dependents\".\"empid\"\n" + + "from \"depts\"\n" + + "join \"dependents\" on (\"depts\".\"name\" = \"dependents\".\"name\")\n" + + "join \"locations\" on (\"locations\".\"name\" = \"dependents\".\"name\")\n" + + "join \"emps\" on (\"emps\".\"deptno\" = \"depts\".\"deptno\")\n" + + "where \"depts\".\"deptno\" > 11 and \"depts\".\"deptno\" < 19\n" + + "group by \"depts\".\"deptno\", \"dependents\".\"empid\""; + String query = "select \"dependents\".\"empid\"\n" + + "from \"depts\"\n" + + "join \"dependents\" on (\"depts\".\"name\" = \"dependents\".\"name\")\n" + + "join \"locations\" on (\"locations\".\"name\" = \"dependents\".\"name\")\n" + + "join \"emps\" on (\"emps\".\"deptno\" = \"depts\".\"deptno\")\n" + + "where \"depts\".\"deptno\" > 10 and \"depts\".\"deptno\" < 20\n" + + "group by \"dependents\".\"empid\""; + + MaterializedViewFixture.create(query, hepTester) + .withMaterializations(ImmutableList.of(Pair.of(materialize, "MV0"))) + .checkingThatResultContains("EnumerableTableScan(table=[[hr, MV0]])") + .ok(); + } + @Test void testJoinAggregateMaterializationNoAggregateFuncs10() { sql("select \"depts\".\"name\", \"dependents\".\"name\" as \"name2\", " + "\"emps\".\"deptno\", \"depts\".\"deptno\" as \"deptno2\", " @@ -950,6 +1010,58 @@ protected final MaterializedViewFixture sql(String materialize, .ok(); } + /** Test case for + * [CALCITE-7641] + * Materialize view rules with UnionRewritingPullProgram on a HepPlanner + * throws IllegalArgumentException. */ + @Test void testJoinMaterialization10Hep() { + // Tester using a HepPlanner instead of Volcano + final MaterializedViewTester hepTester = + new MaterializedViewTester() { + @Override protected List optimize(RelNode queryRel, + List materializationList) { + // Dummy UnionRewritingPullProgram + final HepProgram unionRewritingPullProgram = new HepProgramBuilder().build(); + final HepProgram mainProgram = new HepProgramBuilder() + .addRuleInstance(MaterializedViewRules.JOIN) + .addRuleInstance(MaterializedViewRules.PROJECT_JOIN) + .addRuleInstance(MaterializedViewRules.PROJECT_FILTER) + // MaterializedViewOnlyFilterRule with UnionRewritingPullProgram + .addRuleInstance(MaterializedViewOnlyFilterRule.Config.DEFAULT + .withUnionRewritingPullProgram(unionRewritingPullProgram).toRule()) + .build(); + final HepPlanner hepPlanner = new HepPlanner(mainProgram); + final Program program = + (planner, rel, requiredOutputTraits, materializations, lattices) -> { + for (RelOptMaterialization materialization : materializations) { + planner.addMaterialization(materialization); + } + planner.setRoot(rel); + return planner.findBestExp(); + }; + return ImmutableList.of( + program.run(hepPlanner, queryRel, queryRel.getCluster().traitSet(), + materializationList, ImmutableList.of())); + } + }; + + String materialize = "select \"depts\".\"deptno\", \"dependents\".\"empid\"\n" + + "from \"depts\"\n" + + "join \"dependents\" on (\"depts\".\"name\" = \"dependents\".\"name\")\n" + + "join \"emps\" on (\"emps\".\"deptno\" = \"depts\".\"deptno\")\n" + + "where \"depts\".\"deptno\" > 30"; + String query = "select \"dependents\".\"empid\"\n" + + "from \"depts\"\n" + + "join \"dependents\" on (\"depts\".\"name\" = \"dependents\".\"name\")\n" + + "join \"emps\" on (\"emps\".\"deptno\" = \"depts\".\"deptno\")\n" + + "where \"depts\".\"deptno\" > 10"; + + MaterializedViewFixture.create(query, hepTester) + .withMaterializations(ImmutableList.of(Pair.of(materialize, "MV0"))) + .checkingThatResultContains("EnumerableTableScan(table=[[hr, MV0]])") + .ok(); + } + @Test void testJoinMaterialization11() { sql("select \"empid\" from \"emps\"\n" + "join \"depts\" using (\"deptno\")", From 476dd8c0e609b30d4243bc59eefc4645ea257bfa Mon Sep 17 00:00:00 2001 From: xuzifu666 <1206332514@qq.com> Date: Sun, 5 Jul 2026 07:19:29 +0800 Subject: [PATCH 380/562] [CALCITE-6036] Support WITHIN GROUP(ORDER BY x) OVER (PARTITION BY y) --- .../test/resources/sql/within-group-over.iq | 93 +++++++++++++++++++ .../calcite/rel/rel2sql/SqlImplementor.java | 36 ++++++- .../apache/calcite/sql/SqlOverOperator.java | 26 +++++- .../sql/validate/SqlAbstractConformance.java | 4 + .../calcite/sql/validate/SqlConformance.java | 18 ++++ .../sql/validate/SqlConformanceEnum.java | 9 ++ .../validate/SqlDelegatingConformance.java | 4 + .../calcite/sql/validate/SqlValidator.java | 23 ++++- .../sql/validate/SqlValidatorImpl.java | 13 ++- .../calcite/sql2rel/SqlToRelConverter.java | 26 +++++- .../org/apache/calcite/tools/RelBuilder.java | 9 ++ .../rel/rel2sql/RelToSqlConverterTest.java | 86 +++++++++++++++++ .../apache/calcite/test/SqlValidatorTest.java | 27 ++++++ site/_docs/reference.md | 22 +++++ 14 files changed, 384 insertions(+), 12 deletions(-) create mode 100644 babel/src/test/resources/sql/within-group-over.iq diff --git a/babel/src/test/resources/sql/within-group-over.iq b/babel/src/test/resources/sql/within-group-over.iq new file mode 100644 index 000000000000..82df38fcd862 --- /dev/null +++ b/babel/src/test/resources/sql/within-group-over.iq @@ -0,0 +1,93 @@ +# within-group-over.iq - WITHIN GROUP (ORDER BY) OVER (PARTITION BY) +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Tests for the non-standard (Oracle) syntax that lets an aggregate function +# with a "WITHIN GROUP (ORDER BY ...)" sort key also carry an "OVER (...)" +# clause, so that it behaves as an analytic (window) function. The WITHIN GROUP +# order key orders the aggregate's input but does not restrict the window frame: +# the aggregate is computed over the whole partition and broadcast to every row +# (matching Oracle). This syntax is enabled by the BABEL conformance +# (SqlConformance.allowWithinGroupOverAggregate). +# +!use scott-babel +!set outputformat mysql + +# The following 3 tests are related to this issue. +# Results were validated on Oracle. + +# LISTAGG as an analytic function, partitioned by deptno. The result is the same +# for every row of the partition (broadcast), which distinguishes the analytic +# form from a per-row accumulating window. +select deptno, + listagg(ename, ',') within group (order by ename) + over (partition by deptno) as names +from emp +where deptno in (10, 20) +order by deptno, ename; ++--------+------------------------------+ +| DEPTNO | NAMES | ++--------+------------------------------+ +| 10 | CLARK,KING,MILLER | +| 10 | CLARK,KING,MILLER | +| 10 | CLARK,KING,MILLER | +| 20 | ADAMS,FORD,JONES,SCOTT,SMITH | +| 20 | ADAMS,FORD,JONES,SCOTT,SMITH | +| 20 | ADAMS,FORD,JONES,SCOTT,SMITH | +| 20 | ADAMS,FORD,JONES,SCOTT,SMITH | +| 20 | ADAMS,FORD,JONES,SCOTT,SMITH | ++--------+------------------------------+ +(8 rows) + +!ok + +# The WITHIN GROUP order key controls the concatenation order (here descending), +# independently of the OVER partition. +select deptno, + listagg(ename, ',') within group (order by ename desc) + over (partition by deptno) as names +from emp +where deptno = 10 +order by ename; ++--------+-------------------+ +| DEPTNO | NAMES | ++--------+-------------------+ +| 10 | MILLER,KING,CLARK | +| 10 | MILLER,KING,CLARK | +| 10 | MILLER,KING,CLARK | ++--------+-------------------+ +(3 rows) + +!ok + +# Contrast: the aggregate (GROUP BY) form collapses each group to a single row. +select deptno, + listagg(ename, ',') within group (order by ename) as names +from emp +where deptno in (10, 20) +group by deptno +order by deptno; ++--------+------------------------------+ +| DEPTNO | NAMES | ++--------+------------------------------+ +| 10 | CLARK,KING,MILLER | +| 20 | ADAMS,FORD,JONES,SCOTT,SMITH | ++--------+------------------------------+ +(2 rows) + +!ok + +# End within-group-over.iq diff --git a/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java b/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java index 403d7f6e2f8d..d8c02a56f3f4 100644 --- a/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java +++ b/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java @@ -105,6 +105,7 @@ import org.apache.calcite.util.DateString; import org.apache.calcite.util.ImmutableBitSet; import org.apache.calcite.util.NlsString; +import org.apache.calcite.util.Optionality; import org.apache.calcite.util.Pair; import org.apache.calcite.util.RangeSets; import org.apache.calcite.util.Sarg; @@ -1119,6 +1120,23 @@ private SqlCall toSql(@Nullable RexProgram program, RexOver rexOver) { for (RexFieldCollation rfc : rexWindow.orderKeys) { addOrderItem(orderNodes, program, rfc); } + + SqlAggFunction sqlAggregateFunction = rexOver.getAggOperator(); + + // Inverse distribution functions such as PERCENTILE_CONT/DISC take their + // sort key from a "WITHIN GROUP (ORDER BY ...)" clause rather than the + // window's ORDER BY, as in + // "PERCENTILE_CONT(x) WITHIN GROUP (ORDER BY y) OVER (PARTITION BY z)". + // Route the window's order keys into a WITHIN GROUP wrapper and leave the + // OVER clause with only the partition. + final SqlNodeList groupOrderList; + if (sqlAggregateFunction.requiresGroupOrder() == Optionality.MANDATORY + && !orderNodes.isEmpty()) { + groupOrderList = new SqlNodeList(orderNodes, POS); + orderNodes = Expressions.list(); + } else { + groupOrderList = null; + } final SqlNodeList orderList = new SqlNodeList(orderNodes, POS); @@ -1132,8 +1150,6 @@ private SqlCall toSql(@Nullable RexProgram program, RexOver rexOver) { // "disallow partial" and set the allowPartial = false. final SqlLiteral allowPartial = null; - SqlAggFunction sqlAggregateFunction = rexOver.getAggOperator(); - SqlNode lowerBound = null; SqlNode upperBound = null; SqlLiteral exclude = toSql(rexWindow.getExclude()); @@ -1149,15 +1165,22 @@ private SqlCall toSql(@Nullable RexProgram program, RexOver rexOver) { final List nodeList = toSql(program, rexOver.getOperands()); return createOverCall(sqlAggregateFunction, nodeList, sqlWindow, - rexOver.isDistinct(), rexOver.ignoreNulls()); + rexOver.isDistinct(), rexOver.ignoreNulls(), groupOrderList); } private static SqlCall createOverCall(SqlAggFunction op, List operands, SqlWindow window, boolean isDistinct, boolean ignoreNulls) { + return createOverCall(op, operands, window, isDistinct, ignoreNulls, null); + } + + private static SqlCall createOverCall(SqlAggFunction op, List operands, + SqlWindow window, boolean isDistinct, boolean ignoreNulls, + @Nullable SqlNodeList groupOrderList) { if (op instanceof SqlSumEmptyIsZeroAggFunction) { // Rewrite "SUM0(x) OVER w" to "COALESCE(SUM(x) OVER w, 0)" final SqlCall node = - createOverCall(SqlStdOperatorTable.SUM, operands, window, isDistinct, ignoreNulls); + createOverCall(SqlStdOperatorTable.SUM, operands, window, isDistinct, ignoreNulls, + groupOrderList); return SqlStdOperatorTable.COALESCE.createCall(POS, node, ZERO); } SqlCall aggFunctionCall; @@ -1171,6 +1194,11 @@ private static SqlCall createOverCall(SqlAggFunction op, List operands, aggFunctionCall = SqlStdOperatorTable.IGNORE_NULLS.createCall(null, POS, aggFunctionCall); } + if (groupOrderList != null && !groupOrderList.isEmpty()) { + aggFunctionCall = + SqlStdOperatorTable.WITHIN_GROUP.createCall(POS, aggFunctionCall, + groupOrderList); + } return SqlStdOperatorTable.OVER.createCall(POS, aggFunctionCall, window); } diff --git a/core/src/main/java/org/apache/calcite/sql/SqlOverOperator.java b/core/src/main/java/org/apache/calcite/sql/SqlOverOperator.java index b1ed13808dac..1ec62f3309a2 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlOverOperator.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlOverOperator.java @@ -78,6 +78,19 @@ public SqlOverOperator() { default: break; } + // Support "agg WITHIN GROUP (ORDER BY ...) OVER (...)" for inverse + // distribution functions such as PERCENTILE_CONT/PERCENTILE_DISC. The + // WITHIN GROUP wrapper carries the sort key, and the underlying operand + // is the actual aggregate. This is non-standard (Oracle) syntax, gated by + // conformance; otherwise the WITHIN GROUP call is not an aggregator and the + // overNonAggregate error below fires. + SqlNodeList groupOrderList = null; + if (aggCall.getKind() == SqlKind.WITHIN_GROUP + && validator.config().conformance().allowWithinGroupOverAggregate()) { + validator.validateCall(aggCall, scope); + groupOrderList = aggCall.operand(1); + aggCall = aggCall.operand(0); + } if (!aggCall.getOperator().isAggregator()) { throw validator.newValidationError(aggCall, RESOURCE.overNonAggregate()); } @@ -87,7 +100,7 @@ public SqlOverOperator() { throw validator.newValidationError(aggCall, RESOURCE.overNonAggregate()); } final SqlNode window = call.operand(1); - validator.validateWindow(window, scope, aggCall); + validator.validateWindow(window, scope, aggCall, groupOrderList); } @Override public RelDataType deriveType( @@ -113,6 +126,17 @@ public SqlOverOperator() { SqlWindow w = validator.resolveWindow(window, scope); SqlCall aggCall = (SqlCall) agg; + // "agg WITHIN GROUP (ORDER BY ...) OVER (...)": the WITHIN GROUP wrapper + // has already derived the correct return type (e.g. the collation column + // type for PERCENTILE_CONT/DISC via SqlWithinGroupOperator.deriveType), so + // reuse it rather than re-inferring from the bare aggregate call, which + // would fail because the sort key is not available to the aggregate alone. + if (aggCall.getKind() == SqlKind.WITHIN_GROUP) { + RelDataType ret = validator.deriveType(scope, aggCall); + validator.setValidatedNodeType(call, ret); + validator.setValidatedNodeType(agg, ret); + return ret; + } // Unwrap FILTER, RESPECT_NULLS, or IGNORE_NULLS to get the actual aggregate call while (aggCall != null && (aggCall.getKind() == SqlKind.FILTER diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlAbstractConformance.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlAbstractConformance.java index 9ef7559c5329..84cf77e3c96a 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlAbstractConformance.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlAbstractConformance.java @@ -33,6 +33,10 @@ public abstract class SqlAbstractConformance implements SqlConformance { return SqlConformanceEnum.DEFAULT.allowCharLiteralAlias(); } + @Override public boolean allowWithinGroupOverAggregate() { + return SqlConformanceEnum.DEFAULT.allowWithinGroupOverAggregate(); + } + @Override public boolean isSupportedDualTable() { return SqlConformanceEnum.DEFAULT.isSupportedDualTable(); } diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlConformance.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlConformance.java index 3ccc860a0d71..db0dd693a99c 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlConformance.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlConformance.java @@ -89,6 +89,24 @@ public interface SqlConformance { */ boolean allowCharLiteralAlias(); + /** + * Whether to allow an inverse distribution function such as + * {@code PERCENTILE_CONT} or {@code PERCENTILE_DISC} to combine a + * {@code WITHIN GROUP (ORDER BY ...)} clause with an {@code OVER} clause, so + * that it may be used as an analytic (window) function. For example, + * + * 

      +   *   PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY x)
+   *     OVER (PARTITION BY y)
      + * + *

      This is non-standard SQL supported by Oracle. + * + *

      Among the built-in conformance levels, true in + * {@link SqlConformanceEnum#BABEL}; + * false otherwise. + */ + boolean allowWithinGroupOverAggregate(); + /** * Whether to allow aliases from the {@code SELECT} clause to be used as * column names in the {@code GROUP BY} clause. diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlConformanceEnum.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlConformanceEnum.java index fd083560e025..4ed6cb93c285 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlConformanceEnum.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlConformanceEnum.java @@ -103,6 +103,15 @@ public enum SqlConformanceEnum implements SqlConformance { } } + @Override public boolean allowWithinGroupOverAggregate() { + switch (this) { + case BABEL: + return true; + default: + return false; + } + } + @Override public boolean isSupportedDualTable() { switch (this) { case MYSQL_5: diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlDelegatingConformance.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlDelegatingConformance.java index daac4b5c416a..e142893891e1 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlDelegatingConformance.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlDelegatingConformance.java @@ -39,6 +39,10 @@ protected SqlDelegatingConformance(SqlConformance delegate) { return delegate.allowCharLiteralAlias(); } + @Override public boolean allowWithinGroupOverAggregate() { + return delegate.allowWithinGroupOverAggregate(); + } + @Override public boolean isSupportedDualTable() { return delegate.isSupportedDualTable(); } diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidator.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidator.java index 813247376d39..94b90c6789bc 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidator.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidator.java @@ -290,10 +290,31 @@ void validateQuery(SqlNode node, SqlValidatorScope scope, * @param call the SqlNode if a function call if the window is attached * to one. */ + default void validateWindow( + SqlNode windowOrId, + SqlValidatorScope scope, + @Nullable SqlCall call) { + validateWindow(windowOrId, scope, call, null); + } + + /** + * Validates a window clause where the windowed function carries an inline + * {@code WITHIN GROUP (ORDER BY ...)} sort key, as in + * {@code PERCENTILE_CONT(x) WITHIN GROUP (ORDER BY y) OVER (PARTITION BY z)}. + * + * @param windowOrId SqlNode that can be either SqlWindow with all the + * components of a window spec or a SqlIdentifier with the + * name of a window spec. + * @param scope Naming scope + * @param call the aggregate function call the window is attached to. + * @param groupOrderList the {@code WITHIN GROUP} order list carried by the + * aggregate, or null if there is none. + */ void validateWindow( SqlNode windowOrId, SqlValidatorScope scope, - @Nullable SqlCall call); + @Nullable SqlCall call, + @Nullable SqlNodeList groupOrderList); /** Returns whether the validator is currently validating within a window * expression. */ diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java index 2897202926a0..d8fd81a827d1 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java @@ -6604,7 +6604,8 @@ public void setOriginal(SqlNode expr, SqlNode original) { @Override public void validateWindow( SqlNode windowOrId, SqlValidatorScope scope, - @Nullable SqlCall call) { + @Nullable SqlCall call, + @Nullable SqlNodeList groupOrderList) { // Enable nested aggregates with window aggregates (OVER operator) inWindow = true; @@ -6627,9 +6628,15 @@ public void setOriginal(SqlNode expr, SqlNode original) { targetWindow.setWindowCall(call); targetWindow.validate(this, scope); targetWindow.setWindowCall(null); - call.validate(this, scope); + if (groupOrderList == null) { + // A bare "PERCENTILE_CONT(x) WITHIN GROUP (ORDER BY y)" call has already + // been validated by SqlWithinGroupOperator, so re-validating the naked + // aggregate here would fail (it needs the WITHIN GROUP sort key to + // derive its type). Only validate when there is no group order list. + call.validate(this, scope); + } - validateAggregateParams(call, null, null, null, scope); + validateAggregateParams(call, null, null, groupOrderList, scope); // Disable nested aggregates post validation inWindow = false; diff --git a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java index 240b8bd02b0d..f8b661c53fb4 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java @@ -2527,6 +2527,15 @@ private RexNode convertOver(Blackboard bb, SqlNode node) { default: break; } + // "agg WITHIN GROUP (ORDER BY ...) OVER (...)": the WITHIN GROUP sort key + // (used by inverse distribution functions such as PERCENTILE_CONT/DISC) is + // carried as the window's ORDER BY. Oracle forbids ORDER BY inside the OVER + // clause for these functions, so the window's own order list is empty here. + @Nullable SqlNodeList groupOrderList = null; + if (aggCall.getKind() == SqlKind.WITHIN_GROUP) { + groupOrderList = aggCall.operand(1); + aggCall = aggCall.operand(0); + } if (filter != null) { final SqlOperator op = aggCall.getOperator(); if (op instanceof SqlAggFunction @@ -2551,9 +2560,20 @@ private RexNode convertOver(Blackboard bb, SqlNode node) { SqlNode sqlLowerBound = window.getLowerBound(); SqlNode sqlUpperBound = window.getUpperBound(); boolean rows = window.isRows(); - SqlNodeList orderList = window.getOrderList(); - - if (!aggCall.getOperator().allowsFraming()) { + // For "agg WITHIN GROUP (ORDER BY ...) OVER (...)", the sort key comes from + // the WITHIN GROUP clause rather than the window's own (empty) ORDER BY. + SqlNodeList orderList = + groupOrderList != null ? groupOrderList : window.getOrderList(); + + if (groupOrderList != null) { + // For "agg WITHIN GROUP (ORDER BY ...) OVER (...)", the sort key orders + // the aggregate's input but does not restrict the window frame: the + // aggregate is computed over the whole partition and broadcast to every + // row (matching Oracle). Force a full-partition frame so that framing + // aggregates such as LISTAGG do not accumulate row by row. + sqlLowerBound = SqlWindow.createUnboundedPreceding(SqlParserPos.ZERO); + sqlUpperBound = SqlWindow.createUnboundedFollowing(SqlParserPos.ZERO); + } else if (!aggCall.getOperator().allowsFraming()) { // If the operator does not allow framing, bracketing is implicitly // everything up to the current row. sqlLowerBound = SqlWindow.createUnboundedPreceding(SqlParserPos.ZERO); diff --git a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java index 54d662a3a13c..36d56a9f0488 100644 --- a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java +++ b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java @@ -5063,6 +5063,15 @@ private OverCall orderBy_(ImmutableList sortKeys) { @Override public boolean hasEmptyGroup() { return !SqlWindow.isAlwaysNonEmpty(lowerBound, upperBound); } + + @Override public RelDataType getCollationType() { + // Inverse distribution functions such as PERCENTILE_CONT/DISC + // used as analytic functions ("... WITHIN GROUP (ORDER BY x) + // OVER (...)") derive their return type from the sort key. + checkArgument(!sortKeys.isEmpty(), + "collation type requested but no sort key present"); + return sortKeys.get(0).left.getType(); + } }; final RelDataType type = op.inferReturnType(bind); final RexNode over = getRexBuilder() diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java index 1d3dda5af67e..95bbe98f6adf 100644 --- a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java +++ b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java @@ -8275,6 +8275,92 @@ private void checkLiteral2(String expression, String expected) { sql(query).ok(expected); } + /** Test case for + * [CALCITE-6036] + * Support WITHIN GROUP (ORDER BY x) OVER (PARTITION BY y). Checks that an + * inverse distribution function used as an analytic function is unparsed with + * the sort key inside a {@code WITHIN GROUP} clause and the partition inside + * the {@code OVER} clause. */ + @Test void testWithinGroupOver() { + final Function relFn = b -> b + .scan("EMP") + .project( + b.aggregateCall(SqlStdOperatorTable.PERCENTILE_CONT, b.literal(0.5)) + .over() + .partitionBy(b.field("DEPTNO")) + .orderBy(b.field("SAL")) + .rowsUnbounded() + .allowPartial(true) + .nullWhenCountZero(false) + .as("c")) + .build(); + final String expected = "SELECT PERCENTILE_CONT(5E-1) " + + "WITHIN GROUP (ORDER BY \"SAL\") " + + "OVER (PARTITION BY \"DEPTNO\") AS \"c\"\n" + + "FROM \"scott\".\"EMP\""; + relFn(relFn).ok(expected); + } + + /** Test case for + * [CALCITE-6036] + * Support WITHIN GROUP (ORDER BY x) OVER (PARTITION BY y). Checks that + * expressions (rather than plain column references) under both the WITHIN + * GROUP order key and the OVER partition key are unparsed correctly. */ + @Test void testWithinGroupOverWithExpressions() { + final Function relFn = b -> b + .scan("EMP") + .project( + b.aggregateCall(SqlStdOperatorTable.PERCENTILE_CONT, b.literal(0.5)) + .over() + .partitionBy( + b.call(SqlStdOperatorTable.PLUS, b.field("DEPTNO"), + b.literal(1))) + .orderBy( + b.call(SqlStdOperatorTable.MULTIPLY, b.field("SAL"), + b.literal(2))) + .rowsUnbounded() + .allowPartial(true) + .nullWhenCountZero(false) + .as("c")) + .build(); + final String expected = "SELECT PERCENTILE_CONT(5E-1) " + + "WITHIN GROUP (ORDER BY \"SAL\" * 2) " + + "OVER (PARTITION BY \"DEPTNO\" + 1) AS \"c\"\n" + + "FROM \"scott\".\"EMP\""; + relFn(relFn).ok(expected); + } + + /** Test case for + * [CALCITE-6036] + * Support WITHIN GROUP (ORDER BY x) OVER (PARTITION BY y). Checks + * unparsing with multiple partition and sort expressions, including a + * descending sort key. */ + @Test void testWithinGroupOverWithMultipleExpressions() { + final Function relFn = b -> b + .scan("EMP") + .project( + b.aggregateCall(SqlStdOperatorTable.PERCENTILE_DISC, b.literal(0.5)) + .over() + .partitionBy( + b.field("DEPTNO"), + b.call(SqlStdOperatorTable.PLUS, b.field("MGR"), + b.literal(1))) + .orderBy( + b.desc( + b.call(SqlStdOperatorTable.MINUS, b.field("SAL"), + b.field("COMM")))) + .rowsUnbounded() + .allowPartial(true) + .nullWhenCountZero(false) + .as("c")) + .build(); + final String expected = "SELECT PERCENTILE_DISC(5E-1) " + + "WITHIN GROUP (ORDER BY \"SAL\" - \"COMM\" DESC) " + + "OVER (PARTITION BY \"DEPTNO\", \"MGR\" + 1) AS \"c\"\n" + + "FROM \"scott\".\"EMP\""; + relFn(relFn).ok(expected); + } + @Test void testJsonValueExpressionOperator() { String query = "select \"product_name\" format json, " + "\"product_name\" format json encoding utf8, " diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index 123dbe01d018..094e0447b17a 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -8549,6 +8549,33 @@ void testGroupExpressionEquivalenceParams() { .type("RecordType(INTEGER NOT NULL C, INTEGER NOT NULL D) NOT NULL"); } + /** Test case for + * [CALCITE-6036] + * Support WITHIN GROUP (ORDER BY x) OVER (PARTITION BY y). Combining a + * WITHIN GROUP clause with an OVER clause is non-standard (Oracle) syntax that + * is only allowed under a conformance that enables it, such as BABEL. */ + @Test void testPercentileWithinGroupOver() { + final String sql = "select\n" + + " percentile_cont(0.25) within group (order by sal)\n" + + " over (partition by deptno) as c\n" + + "from emp"; + // Enabled under BABEL conformance. + sql(sql) + .withConformance(SqlConformanceEnum.BABEL) + .type("RecordType(INTEGER NOT NULL C) NOT NULL"); + } + + @Test void testPercentileWithinGroupOverFailsInDefaultConformance() { + final String sql = "select\n" + + " ^percentile_cont(0.25) within group (order by sal)^\n" + + " over (partition by deptno) as c\n" + + "from emp"; + // Rejected under the default conformance, which does not allow WITHIN GROUP + // to be combined with an OVER clause. + sql(sql) + .fails("OVER must be applied to aggregate function"); + } + /** Tests that {@code PERCENTILE_CONT} only allows numeric fields. */ @Test void testPercentileContMustOrderByNumeric() { final String sql = "select\n" diff --git a/site/_docs/reference.md b/site/_docs/reference.md index 064d6fb95e40..90511aa40664 100644 --- a/site/_docs/reference.md +++ b/site/_docs/reference.md @@ -2166,6 +2166,28 @@ The *exclude* clause can be one of: `DISTINCT`, `FILTER` and `WITHIN GROUP` are as described for aggregate functions. +#### WITHIN GROUP clause in window functions + +Combining a `WITHIN GROUP (ORDER BY ...)` clause with an `OVER` clause lets an +aggregate function whose ordering is supplied by `WITHIN GROUP` be used as a +window function, as in + +{% highlight sql %} +LISTAGG(ename, ',') WITHIN GROUP (ORDER BY ename) OVER (PARTITION BY deptno) +{% endhighlight %} + +The `WITHIN GROUP` order key orders the function's input but does not restrict +the window frame: the function is computed over the whole partition and the +result is broadcast to every row of that partition. Because the sort key is +supplied by `WITHIN GROUP`, the `OVER` clause must not contain its own +`ORDER BY` or frame specification. + +This is non-standard syntax (supported by Oracle) and is only allowed under a +conformance that returns true for +[SqlConformance.allowWithinGroupOverAggregate()]({{ site.apiRoot }}/org/apache/calcite/sql/validate/SqlConformance.html#allowWithinGroupOverAggregate--), +such as `BABEL`; otherwise the validator reports "OVER must be applied to +aggregate function". + #### FILTER clause in window functions When `FILTER` is used with window functions, it is applied in the following order: From 4c9d4b4a5aaf8150258608ec529e889c0207709d Mon Sep 17 00:00:00 2001 From: Kirill Tkalenko Date: Thu, 9 Jul 2026 10:05:50 +0300 Subject: [PATCH 381/562] [CALCITE-7624] Support BigDecimal for FETCH and OFFSET in Enumerable --- .../calcite/adapter/enumerable/EnumUtils.java | 34 +++ .../adapter/enumerable/EnumerableLimit.java | 38 ++- .../enumerable/EnumerableLimitSort.java | 12 +- .../enumerable/EnumerableRelImplementor.java | 3 + .../enumerable/FetchOffsetRoundingPolicy.java | 29 ++ .../calcite/prepare/CalcitePrepareImpl.java | 9 + .../sql/validate/SqlValidatorImpl.java | 4 +- .../apache/calcite/util/BuiltInMethod.java | 9 +- .../adapter/enumerable/CodeGeneratorTest.java | 60 +++++ .../adapter/enumerable/EnumUtilsTest.java | 15 ++ .../org/apache/calcite/test/JdbcTest.java | 254 +++++++++++++++++- .../apache/calcite/test/SqlValidatorTest.java | 18 ++ .../enumerable/EnumerableLimitSortTest.java | 190 +++++++++++++ core/src/test/resources/sql/sort.iq | 9 +- .../calcite/linq4j/DefaultEnumerable.java | 16 ++ .../calcite/linq4j/EnumerableDefaults.java | 204 ++++++++++++-- .../calcite/linq4j/ExtendedEnumerable.java | 27 ++ .../org/apache/calcite/linq4j/Linq4j.java | 22 ++ .../calcite/linq4j/test/Linq4jTest.java | 44 +++ site/_docs/reference.md | 5 +- 20 files changed, 962 insertions(+), 40 deletions(-) create mode 100644 core/src/main/java/org/apache/calcite/adapter/enumerable/FetchOffsetRoundingPolicy.java diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java index 2ed244dfa885..129180e584c5 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java @@ -72,6 +72,7 @@ import java.lang.reflect.Modifier; import java.lang.reflect.Type; import java.math.BigDecimal; +import java.math.BigInteger; import java.math.RoundingMode; import java.sql.Date; import java.sql.Time; @@ -112,6 +113,39 @@ private EnumUtils() {} public static final List LEFT_RIGHT = ImmutableList.of("left", "right"); + /** Converts a FETCH or OFFSET runtime value to {@link BigDecimal}. + * + *

      The value must be numeric and non-negative. */ + public static BigDecimal numberToBigDecimal(Object value, String kind) { + return numberToBigDecimal(value, kind, FetchOffsetRoundingPolicy.NONE); + } + + /** Converts a FETCH or OFFSET runtime value to {@link BigDecimal}. + * + *

      The value must be numeric and non-negative. The result is adjusted by + * the configured rounding policy. */ + public static BigDecimal numberToBigDecimal(Object value, String kind, + FetchOffsetRoundingPolicy roundingPolicy) { + if (!(value instanceof Number)) { + throw new IllegalArgumentException(kind + " must be a number"); + } + final Number number = (Number) value; + final BigDecimal decimal; + if (number instanceof BigDecimal) { + decimal = (BigDecimal) number; + } else if (number instanceof BigInteger) { + decimal = new BigDecimal((BigInteger) number); + } else if (number instanceof Float || number instanceof Double) { + decimal = BigDecimal.valueOf(number.doubleValue()); + } else { + decimal = BigDecimal.valueOf(number.longValue()); + } + if (decimal.signum() < 0) { + throw new IllegalArgumentException(kind + " must not be negative"); + } + return roundingPolicy.round(decimal); + } + /** Declares a method that overrides another method. */ public static MethodDeclaration overridingMethodDecl(Method method, Iterable parameters, diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimit.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimit.java index 3c046de9228d..02fd54bdad86 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimit.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimit.java @@ -101,36 +101,50 @@ public static EnumerableLimit create(final RelNode input, @Nullable RexNode offs result.format); Expression v = builder.append("child", result.block); + Expression roundingPolicyExp = getRoundingPolicy(implementor); if (offset != null) { v = builder.append("offset", - Expressions.call(v, BuiltInMethod.SKIP.method, - getExpression(offset))); + Expressions.call(BuiltInMethod.SKIP_BIG_DECIMAL.method, v, + getExpression(offset, "OFFSET", roundingPolicyExp))); } if (fetch != null) { v = builder.append("fetch", - Expressions.call(v, BuiltInMethod.TAKE.method, - getExpression(fetch))); + Expressions.call(BuiltInMethod.TAKE_BIG_DECIMAL.method, v, + getExpression(fetch, "FETCH", roundingPolicyExp))); } builder.add(Expressions.return_(null, v)); return implementor.result(physType, builder.toBlock()); } - static Expression getExpression(RexNode rexNode) { + static Expression getExpression(RexNode rexNode, String kind, + Expression roundingPolicy) { + final Expression value; if (rexNode instanceof RexDynamicParam) { final RexDynamicParam param = (RexDynamicParam) rexNode; - return Expressions.convert_( + value = Expressions.call(DataContext.ROOT, BuiltInMethod.DATA_CONTEXT_GET.method, - Expressions.constant("?" + param.getIndex())), - Integer.class); + Expressions.constant("?" + param.getIndex())); } else { - // TODO: Enumerable runtime only supports INT types for FETCH and OFFSET, not BIGINT types. - // Currently, using BIGINT types for execution will result in an error message. - // This issue needs to be fixed. For more information, see CALCITE-7156. - return Expressions.constant(RexLiteral.intValue(rexNode)); + value = Expressions.constant(RexLiteral.bigDecimalValue(rexNode)); } + return Expressions.call( + BuiltInMethod.NUMBER_TO_BIG_DECIMAL_LIMIT.method, + value, + Expressions.constant(kind), + roundingPolicy); + } + + static Expression getRoundingPolicy(EnumerableRelImplementor implementor) { + final Object roundingPolicy = + implementor.map.get(EnumerableRelImplementor.FETCH_OFFSET_ROUNDING_POLICY); + if (roundingPolicy == null) { + return Expressions.field(null, FetchOffsetRoundingPolicy.class, "NONE"); + } + return implementor.stash((FetchOffsetRoundingPolicy) roundingPolicy, + FetchOffsetRoundingPolicy.class); } } diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimitSort.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimitSort.java index defc28f69a53..325fe687ba4d 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimitSort.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimitSort.java @@ -30,7 +30,10 @@ import org.checkerframework.checker.nullness.qual.Nullable; +import java.math.BigDecimal; + import static org.apache.calcite.adapter.enumerable.EnumerableLimit.getExpression; +import static org.apache.calcite.adapter.enumerable.EnumerableLimit.getRoundingPolicy; /** * Implementation of {@link org.apache.calcite.rel.core.Sort} in @@ -95,19 +98,20 @@ public static EnumerableLimitSort create( final PhysType inputPhysType = result.physType; final Pair pair = inputPhysType.generateCollationKey(this.collation.getFieldCollations()); + final Expression roundingPolicyExp = getRoundingPolicy(implementor); final Expression fetchVal; if (this.fetch == null) { - fetchVal = Expressions.constant(Integer.MAX_VALUE); + fetchVal = Expressions.constant(BigDecimal.valueOf(Integer.MAX_VALUE)); } else { - fetchVal = getExpression(this.fetch); + fetchVal = getExpression(this.fetch, "FETCH", roundingPolicyExp); } final Expression offsetVal; if (this.offset == null) { - offsetVal = Expressions.constant(0); + offsetVal = Expressions.constant(BigDecimal.ZERO); } else { - offsetVal = getExpression(this.offset); + offsetVal = getExpression(this.offset, "OFFSET", roundingPolicyExp); } builder.add( diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableRelImplementor.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableRelImplementor.java index 8d27d6f74c56..ae380578c42b 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableRelImplementor.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableRelImplementor.java @@ -78,6 +78,9 @@ * operators of {@link EnumerableConvention} calling convention. */ public class EnumerableRelImplementor extends JavaRelImplementor { + public static final String FETCH_OFFSET_ROUNDING_POLICY = + "_fetchOffsetRoundingPolicy"; + public final Map map; private final Map corrVars = new HashMap<>(); diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/FetchOffsetRoundingPolicy.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/FetchOffsetRoundingPolicy.java new file mode 100644 index 000000000000..027abcade63e --- /dev/null +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/FetchOffsetRoundingPolicy.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.adapter.enumerable; + +import java.math.BigDecimal; + +/** Defines how enumerable FETCH and OFFSET values are rounded. */ +public interface FetchOffsetRoundingPolicy { + /** Default policy that preserves the value produced by validation or + * parameter binding. */ + FetchOffsetRoundingPolicy NONE = value -> value; + + /** Rounds a FETCH or OFFSET value. */ + BigDecimal round(BigDecimal value); +} diff --git a/core/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java b/core/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java index 8439fabd5f3c..06e27e361cff 100644 --- a/core/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java +++ b/core/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java @@ -21,7 +21,9 @@ import org.apache.calcite.adapter.enumerable.EnumerableConvention; import org.apache.calcite.adapter.enumerable.EnumerableInterpretable; import org.apache.calcite.adapter.enumerable.EnumerableRel; +import org.apache.calcite.adapter.enumerable.EnumerableRelImplementor; import org.apache.calcite.adapter.enumerable.EnumerableRules; +import org.apache.calcite.adapter.enumerable.FetchOffsetRoundingPolicy; import org.apache.calcite.adapter.enumerable.RexToLixTranslator; import org.apache.calcite.adapter.java.JavaTypeFactory; import org.apache.calcite.avatica.AvaticaParameter; @@ -1196,6 +1198,13 @@ protected SqlValidator createSqlValidator(CatalogReader catalogReader, CatalogReader.THREAD_LOCAL.set(catalogReader); final SqlConformance conformance = context.config().conformance(); internalParameters.put("_conformance", conformance); + final FetchOffsetRoundingPolicy fetchOffsetRoundingPolicy = + planner.getContext().unwrap(FetchOffsetRoundingPolicy.class); + if (fetchOffsetRoundingPolicy != null) { + internalParameters.put( + EnumerableRelImplementor.FETCH_OFFSET_ROUNDING_POLICY, + fetchOffsetRoundingPolicy); + } bindable = EnumerableInterpretable.toBindable(internalParameters, context.spark(), enumerable, diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java index d8fd81a827d1..331c8849a699 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java @@ -1763,11 +1763,11 @@ SqlValidatorNamespace getNamespaceOrThrow(SqlIdentifier id, private void handleOffsetFetch(@Nullable SqlNode offset, @Nullable SqlNode fetch) { if (offset instanceof SqlDynamicParam) { setValidatedNodeType(offset, - typeFactory.createSqlType(SqlTypeName.INTEGER)); + typeFactory.createSqlType(SqlTypeName.DECIMAL)); } if (fetch instanceof SqlDynamicParam) { setValidatedNodeType(fetch, - typeFactory.createSqlType(SqlTypeName.INTEGER)); + typeFactory.createSqlType(SqlTypeName.DECIMAL)); } } diff --git a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java index 295d7d662e62..2b07069b6598 100644 --- a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java +++ b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java @@ -21,6 +21,7 @@ import org.apache.calcite.adapter.enumerable.BasicAggregateLambdaFactory; import org.apache.calcite.adapter.enumerable.BasicLazyAccumulator; import org.apache.calcite.adapter.enumerable.EnumUtils; +import org.apache.calcite.adapter.enumerable.FetchOffsetRoundingPolicy; import org.apache.calcite.adapter.enumerable.LazyAggregateLambdaFactory; import org.apache.calcite.adapter.enumerable.MatchUtils; import org.apache.calcite.adapter.enumerable.SourceSorter; @@ -296,7 +297,7 @@ public enum BuiltInMethod { ORDER_BY(ExtendedEnumerable.class, "orderBy", Function1.class, Comparator.class), ORDER_BY_WITH_FETCH_AND_OFFSET(EnumerableDefaults.class, "orderBy", Enumerable.class, - Function1.class, Comparator.class, int.class, int.class), + Function1.class, Comparator.class, BigDecimal.class, BigDecimal.class), UNION(ExtendedEnumerable.class, "union", Enumerable.class), CONCAT(ExtendedEnumerable.class, "concat", Enumerable.class), REPEAT_UNION(EnumerableDefaults.class, "repeatUnion", Enumerable.class, @@ -308,7 +309,11 @@ public enum BuiltInMethod { INTERSECT(ExtendedEnumerable.class, "intersect", Enumerable.class, boolean.class), EXCEPT(ExtendedEnumerable.class, "except", Enumerable.class, boolean.class), SKIP(ExtendedEnumerable.class, "skip", int.class), + SKIP_BIG_DECIMAL(EnumerableDefaults.class, "skip", Enumerable.class, + BigDecimal.class), TAKE(ExtendedEnumerable.class, "take", int.class), + TAKE_BIG_DECIMAL(EnumerableDefaults.class, "take", Enumerable.class, + BigDecimal.class), SINGLETON_ENUMERABLE(Linq4j.class, "singletonEnumerable", Object.class), EMPTY_ENUMERABLE(Linq4j.class, "emptyEnumerable"), NULLS_COMPARATOR(Functions.class, "nullsComparator", boolean.class, @@ -390,6 +395,8 @@ public enum BuiltInMethod { Calendar.class), TIME_ZONE_GET_OFFSET(TimeZone.class, "getOffset", long.class), LONG_VALUE(Number.class, "longValue"), + NUMBER_TO_BIG_DECIMAL_LIMIT(EnumUtils.class, "numberToBigDecimal", + Object.class, String.class, FetchOffsetRoundingPolicy.class), STRING_TO_UPPER(String.class, "toUpperCase"), COMPARATOR_COMPARE(Comparator.class, "compare", Object.class, Object.class), COLLECTIONS_REVERSE_ORDER(Collections.class, "reverseOrder"), diff --git a/core/src/test/java/org/apache/calcite/adapter/enumerable/CodeGeneratorTest.java b/core/src/test/java/org/apache/calcite/adapter/enumerable/CodeGeneratorTest.java index 1a32d8de0197..d8e605e39176 100644 --- a/core/src/test/java/org/apache/calcite/adapter/enumerable/CodeGeneratorTest.java +++ b/core/src/test/java/org/apache/calcite/adapter/enumerable/CodeGeneratorTest.java @@ -16,7 +16,9 @@ */ package org.apache.calcite.adapter.enumerable; +import org.apache.calcite.DataContexts; import org.apache.calcite.jdbc.JavaTypeFactoryImpl; +import org.apache.calcite.linq4j.Enumerable; import org.apache.calcite.linq4j.tree.ClassDeclaration; import org.apache.calcite.linq4j.tree.Expressions; import org.apache.calcite.plan.ConventionTraitDef; @@ -26,19 +28,32 @@ import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.RelRoot; import org.apache.calcite.rel.rules.CoreRules; +import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rel.type.RelDataTypeSystem; import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexLiteral; +import org.apache.calcite.runtime.Bindable; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.parser.SqlParseException; import org.apache.calcite.sql.parser.SqlParser; import org.apache.calcite.sql.test.SqlTestFactory; +import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.sql.validate.SqlValidator; import org.apache.calcite.sql2rel.SqlToRelConverter; +import com.google.common.collect.ImmutableList; + import org.junit.jupiter.api.Test; +import java.math.BigDecimal; +import java.math.RoundingMode; import java.util.HashMap; +import java.util.List; +import java.util.Map; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.hasSize; import static org.junit.jupiter.api.Assertions.assertFalse; /** @@ -97,4 +112,49 @@ public class CodeGeneratorTest { assertFalse(javaCode.contains("case_when_value3")); assertFalse(javaCode.contains("case_when_value4")); } + + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test public void testFetchOffsetRoundingPolicy() { + final JavaTypeFactoryImpl typeFactory = + new JavaTypeFactoryImpl(RelDataTypeSystem.DEFAULT); + final RexBuilder rexBuilder = new RexBuilder(typeFactory); + final VolcanoPlanner planner = new VolcanoPlanner(); + planner.addRelTraitDef(ConventionTraitDef.INSTANCE); + final RelOptCluster cluster = RelOptCluster.create(planner, rexBuilder); + final RelDataType rowType = + typeFactory.builder().add("i", SqlTypeName.INTEGER).build(); + final RelDataType integerType = rowType.getFieldList().get(0).getType(); + final ImmutableList> tuples = + ImmutableList.of( + ImmutableList.of( + rexBuilder.makeExactLiteral(BigDecimal.ONE, + integerType)), + ImmutableList.of( + rexBuilder.makeExactLiteral(BigDecimal.valueOf(2), + integerType)), + ImmutableList.of( + rexBuilder.makeExactLiteral(BigDecimal.valueOf(3), + integerType))); + final EnumerableValues values = + EnumerableValues.create(cluster, rowType, tuples); + final RelDataType decimalType = + typeFactory.createSqlType(SqlTypeName.DECIMAL, 2, 1); + final EnumerableLimit limit = + EnumerableLimit.create(values, null, + rexBuilder.makeExactLiteral(new BigDecimal("1.5"), decimalType)); + + final Map parameters = new HashMap<>(); + parameters.put(EnumerableRelImplementor.FETCH_OFFSET_ROUNDING_POLICY, + (FetchOffsetRoundingPolicy) value -> + value.setScale(0, RoundingMode.DOWN)); + final Bindable bindable = + EnumerableInterpretable.toBindable(parameters, null, limit, + EnumerableRel.Prefer.ARRAY); + + final Enumerable enumerable = bindable.bind(DataContexts.of(parameters)); + final List result = enumerable.toList(); + assertThat(result, hasSize(1)); + } } diff --git a/core/src/test/java/org/apache/calcite/adapter/enumerable/EnumUtilsTest.java b/core/src/test/java/org/apache/calcite/adapter/enumerable/EnumUtilsTest.java index 267aa39ca162..40b825939df9 100644 --- a/core/src/test/java/org/apache/calcite/adapter/enumerable/EnumUtilsTest.java +++ b/core/src/test/java/org/apache/calcite/adapter/enumerable/EnumUtilsTest.java @@ -29,6 +29,7 @@ import org.junit.jupiter.api.Test; import java.math.BigDecimal; +import java.math.RoundingMode; import java.util.Arrays; import static org.hamcrest.CoreMatchers.is; @@ -171,6 +172,20 @@ public final class EnumUtilsTest { assertThat(Expressions.toString(e2), is("(String) (Object) null")); } + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test void testNumberToBigDecimalPreservesFractionalNumber() { + final FetchOffsetRoundingPolicy ceiling = + value -> value.setScale(0, RoundingMode.CEILING); + assertThat(EnumUtils.numberToBigDecimal(1.5D, "FETCH"), + is(new BigDecimal("1.5"))); + assertThat(EnumUtils.numberToBigDecimal(1.5F, "OFFSET"), + is(new BigDecimal("1.5"))); + assertThat(EnumUtils.numberToBigDecimal(1.5D, "FETCH", ceiling), + is(BigDecimal.valueOf(2))); + } + @Test void testMethodCallExpression() { // test for Object.class method parameter type final ConstantExpression arg0 = Expressions.constant(1, int.class); diff --git a/core/src/test/java/org/apache/calcite/test/JdbcTest.java b/core/src/test/java/org/apache/calcite/test/JdbcTest.java index 8ebc5b87c966..0c5ab19b2ec4 100644 --- a/core/src/test/java/org/apache/calcite/test/JdbcTest.java +++ b/core/src/test/java/org/apache/calcite/test/JdbcTest.java @@ -5808,6 +5808,256 @@ private CalciteAssert.AssertQuery withEmpDept(String sql) { Matchers.returnsUnordered("name=Eric")); } + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test void testPreparedOffsetFetchWithBigDecimal() { + CalciteAssert.hr() + .query("select \"name\"\n" + + "from \"hr\".\"emps\"\n" + + "order by \"empid\" offset ? fetch next ? rows only") + .explainContains("EnumerableLimit(offset=[?0], fetch=[?1])") + .consumesPreparedStatement(p -> { + p.setBigDecimal(1, BigDecimal.valueOf(3)); + p.setBigDecimal(2, BigDecimal.valueOf(4)); + }) + .returnsUnordered("name=Eric"); + } + + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test void testPreparedFetchWithFractionalBigDecimal() { + CalciteAssert.hr() + .query("select \"name\"\n" + + "from \"hr\".\"emps\"\n" + + "order by \"empid\" offset ? fetch next ? rows only") + .explainContains("EnumerableLimit(offset=[?0], fetch=[?1])") + .consumesPreparedStatement(p -> { + p.setBigDecimal(1, BigDecimal.ZERO); + p.setBigDecimal(2, new BigDecimal("1.5")); + }) + .returnsUnordered("name=Bill", "name=Theodore"); + } + + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test void testPreparedOffsetWithFractionalBigDecimal() { + CalciteAssert.hr() + .query("select \"name\"\n" + + "from \"hr\".\"emps\"\n" + + "order by \"empid\" offset ? fetch next ? rows only") + .explainContains("EnumerableLimit(offset=[?0], fetch=[?1])") + .consumesPreparedStatement(p -> { + p.setBigDecimal(1, new BigDecimal("1.5")); + p.setBigDecimal(2, BigDecimal.ONE); + }) + .returnsUnordered("name=Sebastian"); + } + + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test void testFetchLiteralFractionalBigDecimal() { + CalciteAssert.hr() + .query("select \"name\"\n" + + "from \"hr\".\"emps\"\n" + + "order by \"empid\" offset 0 fetch next 1.5 rows only") + .explainContains("EnumerableLimit(offset=[0], fetch=[1.5:DECIMAL(2, 1)])") + .returnsUnordered("name=Bill", "name=Theodore"); + } + + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test void testOffsetLiteralFractionalBigDecimal() { + CalciteAssert.hr() + .query("select \"name\"\n" + + "from \"hr\".\"emps\"\n" + + "order by \"empid\" offset 1.5 fetch next 1 rows only") + .explainContains("EnumerableLimit(offset=[1.5:DECIMAL(2, 1)], fetch=[1])") + .returnsUnordered("name=Sebastian"); + } + + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test void testPreparedOffsetFetchWithIntegerParameters() { + CalciteAssert.hr() + .query("select \"name\"\n" + + "from \"hr\".\"emps\"\n" + + "order by \"empid\" offset ? fetch next ? rows only") + .explainContains("EnumerableLimit(offset=[?0], fetch=[?1])") + .consumesPreparedStatement(p -> { + p.setInt(1, 1); + p.setInt(2, 2); + }) + .returnsUnordered("name=Theodore", "name=Sebastian"); + } + + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test void testPreparedOffsetFetchWithLongParameters() { + CalciteAssert.hr() + .query("select \"name\"\n" + + "from \"hr\".\"emps\"\n" + + "order by \"empid\" offset ? fetch next ? rows only") + .explainContains("EnumerableLimit(offset=[?0], fetch=[?1])") + .consumesPreparedStatement(p -> { + p.setLong(1, 1L); + p.setLong(2, 2L); + }) + .returnsUnordered("name=Theodore", "name=Sebastian"); + } + + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test void testPreparedOffsetWithBigDecimalAboveIntegerMax() { + CalciteAssert.hr() + .query("select \"name\"\n" + + "from \"hr\".\"emps\"\n" + + "order by \"empid\" offset ? fetch next ? rows only") + .explainContains("EnumerableLimit(offset=[?0], fetch=[?1])") + .consumesPreparedStatement(p -> { + p.setBigDecimal(1, + BigDecimal.valueOf(Integer.MAX_VALUE).add(BigDecimal.ONE)); + p.setBigDecimal(2, BigDecimal.ONE); + }) + .returnsUnordered(); + } + + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test void testOffsetLiteralAboveIntegerMax() { + CalciteAssert.hr() + .query("select \"name\"\n" + + "from \"hr\".\"emps\"\n" + + "order by \"empid\" offset 2147483648 fetch next 1 rows only") + .explainContains("EnumerableLimit(offset=[2147483648:BIGINT], fetch=[1])") + .returnsUnordered(); + } + + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test void testPreparedOffsetFetchWithBigDecimalWithoutOrderBy() { + CalciteAssert.hr() + .query("select \"name\"\n" + + "from \"hr\".\"emps\"\n" + + "offset ? fetch next ? rows only") + .explainContains("EnumerableLimit(offset=[?0], fetch=[?1])") + .consumesPreparedStatement(p -> { + p.setBigDecimal(1, BigDecimal.ZERO); + p.setBigDecimal(2, BigDecimal.ZERO); + }) + .returnsUnordered(); + } + + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test void testPreparedNegativeFetchWithBigDecimalWithoutOrderBy() + throws Exception { + CalciteAssert.hr() + .doWithConnection(connection -> { + final String sql = "select \"name\"\n" + + "from \"hr\".\"emps\"\n" + + "offset ? fetch next ? rows only"; + try (PreparedStatement p = connection.prepareStatement(sql)) { + p.setBigDecimal(1, BigDecimal.ZERO); + p.setBigDecimal(2, BigDecimal.valueOf(-1)); + final SQLException e = assertThrows(SQLException.class, p::executeQuery); + assertThat(e.getMessage(), containsString("FETCH must not be negative")); + } catch (SQLException e) { + throw TestUtil.rethrow(e); + } + }); + } + + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test void testPreparedNegativeOffsetWithBigDecimalWithoutOrderBy() + throws Exception { + CalciteAssert.hr() + .doWithConnection(connection -> { + final String sql = "select \"name\"\n" + + "from \"hr\".\"emps\"\n" + + "offset ? fetch next ? rows only"; + try (PreparedStatement p = connection.prepareStatement(sql)) { + p.setBigDecimal(1, BigDecimal.valueOf(-1)); + p.setBigDecimal(2, BigDecimal.valueOf(2)); + final SQLException e = assertThrows(SQLException.class, p::executeQuery); + assertThat(e.getMessage(), containsString("OFFSET must not be negative")); + } catch (SQLException e) { + throw TestUtil.rethrow(e); + } + }); + } + + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test void testPreparedNegativeLimitWithBigDecimalWithoutOrderBy() + throws Exception { + CalciteAssert.hr() + .doWithConnection(connection -> { + final String sql = "select \"name\"\n" + + "from \"hr\".\"emps\"\n" + + "limit ? offset ?"; + try (PreparedStatement p = connection.prepareStatement(sql)) { + p.setBigDecimal(1, BigDecimal.valueOf(-1)); + p.setBigDecimal(2, BigDecimal.ZERO); + final SQLException e = assertThrows(SQLException.class, p::executeQuery); + assertThat(e.getMessage(), containsString("FETCH must not be negative")); + } catch (SQLException e) { + throw TestUtil.rethrow(e); + } + }); + } + + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test void testPreparedFetchWithBigDecimalAboveLongMaxWithoutOrderBy() { + CalciteAssert.hr() + .query("select \"name\"\n" + + "from \"hr\".\"emps\"\n" + + "offset ? fetch next ? rows only") + .explainContains("EnumerableLimit(offset=[?0], fetch=[?1])") + .consumesPreparedStatement(p -> { + p.setBigDecimal(1, BigDecimal.ZERO); + p.setBigDecimal(2, + BigDecimal.valueOf(Long.MAX_VALUE).add(BigDecimal.ONE)); + }) + .returnsUnordered( + "name=Bill", + "name=Eric", + "name=Sebastian", + "name=Theodore"); + } + + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test void testFetchLiteralAboveLongMaxWithoutOrderBy() { + CalciteAssert.hr() + .query("select \"name\"\n" + + "from \"hr\".\"emps\"\n" + + "offset 0 fetch next 9223372036854775808 rows only") + .explainContains("EnumerableLimit(offset=[0], " + + "fetch=[9223372036854775808:DECIMAL(19, 0)])") + .returnsUnordered( + "name=Bill", + "name=Eric", + "name=Sebastian", + "name=Theodore"); + } + private void checkPreparedOffsetFetch(final int offset, final int fetch, final Matcher matcher) throws Exception { CalciteAssert.hr() @@ -5819,8 +6069,8 @@ private void checkPreparedOffsetFetch(final int offset, final int fetch, connection.prepareStatement(sql)) { final ParameterMetaData pmd = p.getParameterMetaData(); assertThat(pmd.getParameterCount(), is(2)); - assertThat(pmd.getParameterType(1), is(Types.INTEGER)); - assertThat(pmd.getParameterType(2), is(Types.INTEGER)); + assertThat(pmd.getParameterType(1), is(Types.DECIMAL)); + assertThat(pmd.getParameterType(2), is(Types.DECIMAL)); p.setInt(1, offset); p.setInt(2, fetch); try (ResultSet r = p.executeQuery()) { diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index 094e0447b17a..efa86f9c0b02 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -10630,6 +10630,24 @@ void testGroupExpressionEquivalenceParams() { .rewritesTo(expected); } + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test void testNegativeFetchOffsetLimit() { + sql("select name from dept limit ^-^1") + .fails("(?s).*Encountered \"-\".*"); + sql("select name from dept offset ^-^1") + .fails("(?s).*Encountered \"-\".*"); + sql("select name from dept fetch next ^-^1 rows only") + .fails("(?s).*Encountered \"-\".*"); + sql("select name from dept order by name limit ^-^1") + .fails("(?s).*Encountered \"-\".*"); + sql("select name from dept order by name offset ^-^1") + .fails("(?s).*Encountered \"-\".*"); + sql("select name from dept order by name fetch next ^-^1 rows only") + .fails("(?s).*Encountered \"-\".*"); + } + @Test void testRewriteWithUnionFetchWithoutOrderBy() { final String sql = "select name from dept union all select name from dept limit 2"; diff --git a/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableLimitSortTest.java b/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableLimitSortTest.java index 5c0d35ca4b5f..90256f8fdd4c 100644 --- a/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableLimitSortTest.java +++ b/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableLimitSortTest.java @@ -24,11 +24,19 @@ import org.apache.calcite.runtime.Hook; import org.apache.calcite.test.CalciteAssert; import org.apache.calcite.test.schemata.hr.HrSchemaBig; +import org.apache.calcite.util.TestUtil; import org.junit.jupiter.api.Test; +import java.math.BigDecimal; +import java.sql.PreparedStatement; +import java.sql.SQLException; import java.util.function.Consumer; +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; + /** Tests for * {@link org.apache.calcite.adapter.enumerable.EnumerableLimitSort}. */ public class EnumerableLimitSortTest { @@ -158,6 +166,165 @@ public class EnumerableLimitSortTest { "commission=250; empid=36"); } + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test void limitWithoutOffsetUsesBigDecimalDefaultOffset() { + tester("select empid from emps order by empid limit 2") + .explainContains("EnumerableLimitSort(sort0=[$0], dir0=[ASC], " + + "fetch=[2])") + .returnsOrdered( + "empid=1", + "empid=2"); + } + + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test void offsetWithoutLimitUsesBigDecimalDefaultFetch() { + tester("select empid from emps where empid <= 4 order by empid offset 2") + .explainContains("EnumerableLimitSort(sort0=[$0], dir0=[ASC], " + + "offset=[2])") + .returnsOrdered( + "empid=3", + "empid=4"); + } + + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test void fractionalOffsetAndFetchUseIndependentRowCounts() { + tester("select empid from emps where empid <= 4 order by empid " + + "offset 1.5 fetch next 1.5 rows only") + .explainContains("EnumerableLimitSort(sort0=[$0], dir0=[ASC], " + + "offset=[1.5:DECIMAL(2, 1)], fetch=[1.5:DECIMAL(2, 1)])") + .returnsOrdered( + "empid=3", + "empid=4"); + } + + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test void dynamicParametersWithNegativeBigDecimalFetch() throws Exception { + final String sql = "select empid from emps where empid <= 2 order by empid " + + "offset ? fetch next ? rows only"; + tester(sql) + .explainContains("EnumerableLimitSort(sort0=[$0], dir0=[ASC], " + + "offset=[?0], fetch=[?1])"); + failsWithNegativeParameter(sql, p -> { + p.setBigDecimal(1, BigDecimal.ZERO); + p.setBigDecimal(2, BigDecimal.valueOf(-1)); + }, "FETCH must not be negative"); + } + + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test void dynamicParametersWithNegativeBigDecimalOffset() throws Exception { + final String sql = "select empid from emps where empid <= 3 order by empid " + + "offset ? fetch next ? rows only"; + tester(sql) + .explainContains("EnumerableLimitSort(sort0=[$0], dir0=[ASC], " + + "offset=[?0], fetch=[?1])"); + failsWithNegativeParameter(sql, p -> { + p.setBigDecimal(1, BigDecimal.valueOf(-1)); + p.setBigDecimal(2, BigDecimal.valueOf(2)); + }, "OFFSET must not be negative"); + } + + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test void dynamicParametersWithNegativeBigDecimalLimit() throws Exception { + final String sql = "select empid from emps where empid <= 2 order by empid " + + "limit ? offset ?"; + tester(sql) + .explainContains("EnumerableLimitSort(sort0=[$0], dir0=[ASC], " + + "offset=[?1], fetch=[?0])"); + failsWithNegativeParameter(sql, p -> { + p.setBigDecimal(1, BigDecimal.valueOf(-1)); + p.setBigDecimal(2, BigDecimal.ZERO); + }, "FETCH must not be negative"); + } + + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test void dynamicParametersWithBigDecimal() { + tester("select commission from emps order by commission nulls last " + + "offset ? fetch next ? rows only") + .explainContains("EnumerableLimitSort(sort0=[$0], dir0=[ASC], " + + "offset=[?0], fetch=[?1])\n" + + " EnumerableCalc(expr#0..4=[{inputs}], commission=[$t4])\n" + + " EnumerableTableScan(table=[[s, emps]])") + .consumesPreparedStatement(p -> { + p.setBigDecimal(1, BigDecimal.valueOf(1)); + p.setBigDecimal(2, BigDecimal.valueOf(2)); + }) + .returnsOrdered( + "commission=250", + "commission=250"); + } + + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test void dynamicParametersWithBigDecimalAboveIntegerMax() { + tester("select commission from emps order by commission nulls last " + + "offset ? fetch next ? rows only") + .explainContains("EnumerableLimitSort(sort0=[$0], dir0=[ASC], " + + "offset=[?0], fetch=[?1])") + .consumesPreparedStatement(p -> { + p.setBigDecimal(1, + BigDecimal.valueOf(Integer.MAX_VALUE).add(BigDecimal.ONE)); + p.setBigDecimal(2, BigDecimal.ONE); + }) + .returnsOrdered(); + } + + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test void offsetLiteralAboveIntegerMax() { + tester("select commission from emps order by commission nulls last " + + "offset 2147483648 fetch next 1 rows only") + .explainContains("EnumerableLimitSort(sort0=[$4], dir0=[ASC], " + + "offset=[2147483648:BIGINT], fetch=[1])") + .returnsOrdered(); + } + + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test void dynamicParametersWithBigDecimalAboveLongMax() { + tester("select empid from emps where empid <= 2 order by empid " + + "offset ? fetch next ? rows only") + .explainContains("EnumerableLimitSort(sort0=[$0], dir0=[ASC], " + + "offset=[?0], fetch=[?1])") + .consumesPreparedStatement(p -> { + p.setBigDecimal(1, BigDecimal.ZERO); + p.setBigDecimal(2, + BigDecimal.valueOf(Long.MAX_VALUE).add(BigDecimal.ONE)); + }) + .returnsOrdered( + "empid=1", + "empid=2"); + } + + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test void fetchLiteralAboveLongMax() { + tester("select empid from emps where empid <= 2 order by empid " + + "offset 0 fetch next 9223372036854775808 rows only") + .explainContains("EnumerableLimitSort(sort0=[$0], dir0=[ASC], " + + "offset=[0], fetch=[9223372036854775808:DECIMAL(19, 0)])") + .returnsOrdered( + "empid=1", + "empid=2"); + } + private CalciteAssert.AssertQuery tester(String sqlQuery) { return CalciteAssert.that() .with(CalciteConnectionProperty.LEX, Lex.JAVA) @@ -169,4 +336,27 @@ private CalciteAssert.AssertQuery tester(String sqlQuery) { planner.addRule(EnumerableRules.ENUMERABLE_LIMIT_SORT_RULE); }); } + + private void failsWithNegativeParameter(String sql, + CalciteAssert.PreparedStatementConsumer consumer, String message) + throws Exception { + CalciteAssert.that() + .with(CalciteConnectionProperty.LEX, Lex.JAVA) + .with(CalciteConnectionProperty.FORCE_DECORRELATE, false) + .withSchema("s", new ReflectiveSchema(new HrSchemaBig())) + .doWithConnection(connection -> { + try (Hook.Closeable ignored = + Hook.PLANNER.addThread((Consumer) planner -> { + planner.removeRule(EnumerableRules.ENUMERABLE_SORT_RULE); + planner.addRule(EnumerableRules.ENUMERABLE_LIMIT_SORT_RULE); + }); + PreparedStatement p = connection.prepareStatement(sql)) { + consumer.accept(p); + final SQLException e = assertThrows(SQLException.class, p::executeQuery); + assertThat(e.getMessage(), containsString(message)); + } catch (SQLException e) { + throw TestUtil.rethrow(e); + } + }); + } } diff --git a/core/src/test/resources/sql/sort.iq b/core/src/test/resources/sql/sort.iq index f9149a54822a..0e8d84cff947 100644 --- a/core/src/test/resources/sql/sort.iq +++ b/core/src/test/resources/sql/sort.iq @@ -419,8 +419,13 @@ order by arr desc; # [CALCITE-7156] OFFSET and FETCH in EnumerableLimit need to support BIGINT select * from "hr"."emps" limit 3000000000 offset 2500000000; -java.lang.ArithmeticException: Integer overflow: 2500000000 is out of range for INT -!error ++-------+--------+------+--------+------------+ +| empid | deptno | name | salary | commission | ++-------+--------+------+--------+------------+ ++-------+--------+------+--------+------------+ +(0 rows) + +!ok # [CALCITE-7367] NULLS FIRST throws ClassCastException when sorting arrays select * from diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/DefaultEnumerable.java b/linq4j/src/main/java/org/apache/calcite/linq4j/DefaultEnumerable.java index 54dcc20f41e7..c02001b346d4 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/DefaultEnumerable.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/DefaultEnumerable.java @@ -693,6 +693,10 @@ protected OrderedQueryable asOrderedQueryable() { return EnumerableDefaults.skip(getThis(), count); } + @Override public Enumerable skip(BigDecimal count) { + return EnumerableDefaults.skip(getThis(), count); + } + @Override public Enumerable skipWhile(Predicate1 predicate) { return EnumerableDefaults.skipWhile(getThis(), predicate); } @@ -701,6 +705,10 @@ protected OrderedQueryable asOrderedQueryable() { return EnumerableDefaults.skipWhile(getThis(), predicate); } + @Override public Enumerable skipWhileBigDecimal(Predicate2 predicate) { + return EnumerableDefaults.skipWhileBigDecimal(getThis(), predicate); + } + @Override public BigDecimal sum(BigDecimalFunction1 selector) { return EnumerableDefaults.sum(getThis(), selector); } @@ -745,6 +753,10 @@ protected OrderedQueryable asOrderedQueryable() { return EnumerableDefaults.take(getThis(), count); } + @Override public Enumerable take(BigDecimal count) { + return EnumerableDefaults.take(getThis(), count); + } + @Override public Enumerable takeWhile(Predicate1 predicate) { return EnumerableDefaults.takeWhile(getThis(), predicate); } @@ -753,6 +765,10 @@ protected OrderedQueryable asOrderedQueryable() { return EnumerableDefaults.takeWhile(getThis(), predicate); } + @Override public Enumerable takeWhileBigDecimal(Predicate2 predicate) { + return EnumerableDefaults.takeWhileBigDecimal(getThis(), predicate); + } + @Override public > OrderedEnumerable thenBy( Function1 keySelector) { return EnumerableDefaults.thenBy(getThisOrdered(), keySelector); diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java b/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java index ae26a602132d..0ff4afddb6dd 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java @@ -48,6 +48,7 @@ import org.checkerframework.framework.qual.HasQualifierParameter; import java.math.BigDecimal; +import java.math.RoundingMode; import java.util.AbstractList; import java.util.AbstractMap; import java.util.AbstractSet; @@ -3244,26 +3245,43 @@ public static Enumerable orderBy( Function1 keySelector, Comparator comparator, int offset, int fetch) { + return orderBy(source, keySelector, comparator, + BigDecimal.valueOf(offset), BigDecimal.valueOf(fetch)); + } + + /** + * A sort implementation optimized for a sort with a fetch size (LIMIT). + * + * @param offset how many rows are skipped from the sorted output. + * Must be greater than or equal to 0. + * @param fetch how many rows are retrieved. Must be greater than or equal to 0. + */ + public static Enumerable orderBy( + Enumerable source, + Function1 keySelector, + Comparator comparator, + BigDecimal offset, BigDecimal fetch) { // As discussed in CALCITE-3920 and CALCITE-4157, this method avoids to sort the complete input, // if only the first N rows are actually needed. A TreeMap implementation has been chosen, // so that it behaves similar to the orderBy method without fetch/offset. // The TreeMap has a better performance if there are few distinct sort keys. return new AbstractEnumerable() { @Override public Enumerator enumerator() { - if (fetch == 0) { + if (fetch.compareTo(BigDecimal.ZERO) <= 0) { return Linq4j.emptyEnumerator(); } TreeMap> map = new TreeMap<>(comparator); - long size = 0; - long needed = fetch + (long) offset; + BigDecimal size = BigDecimal.ZERO; + BigDecimal actualOffset = offset.max(BigDecimal.ZERO); + BigDecimal needed = rowsRequired(actualOffset).add(rowsRequired(fetch)); // read the input into a tree map try (Enumerator os = source.enumerator()) { while (os.moveNext()) { TSource o = os.current(); TKey key = keySelector.apply(o); - if (needed >= 0 && size >= needed) { + if (needed.signum() >= 0 && size.compareTo(needed) >= 0) { // the current row will never appear in the output, so just skip it @KeyFor("map") TKey lastKey = map.lastKey(); if (comparator.compare(key, lastKey) >= 0) { @@ -3277,7 +3295,7 @@ public static Enumerable orderBy( } else { l.remove(l.size() - 1); } - size--; + size = size.subtract(BigDecimal.ONE); } // add the current element to the map map.compute(key, (k, l) -> { @@ -3292,24 +3310,30 @@ public static Enumerable orderBy( l.add(o); return l; }); - size++; + size = size.add(BigDecimal.ONE); } } // skip the first 'offset' rows by deleting them from the map - if (offset > 0) { - // search the key up to (but excluding) which we have to remove entries from the map - int skipped = 0; + if (actualOffset.compareTo(BigDecimal.ZERO) > 0) { + // search the key up to which we have to remove entries from the map + BigDecimal skipped = BigDecimal.ZERO; + BigDecimal rowsToSkip = rowsRequired(actualOffset); TKey until = (TKey) DUMMY; + boolean removeUntilInclusive = false; for (Map.Entry> e : map.entrySet()) { - skipped += e.getValue().size(); + skipped = skipped.add(BigDecimal.valueOf(e.getValue().size())); - if (skipped > offset) { + if (skipped.compareTo(rowsToSkip) >= 0) { // we might need to remove entries from the list List l = e.getValue(); - int toKeep = skipped - offset; - if (toKeep < l.size()) { - l.subList(0, l.size() - toKeep).clear(); + BigDecimal toKeep = skipped.subtract(rowsToSkip); + if (toKeep.compareTo(BigDecimal.valueOf(l.size())) < 0) { + if (toKeep.signum() == 0) { + removeUntilInclusive = true; + } else { + l.subList(0, l.size() - toKeep.intValueExact()).clear(); + } } until = e.getKey(); @@ -3320,7 +3344,7 @@ public static Enumerable orderBy( // the offset is bigger than the number of rows in the map return Linq4j.emptyEnumerator(); } - map.headMap(until, false).clear(); + map.headMap(until, removeUntilInclusive).clear(); } return new LookupImpl<>(map).valuesEnumerable().enumerator(); @@ -3790,6 +3814,10 @@ public static TSource single(Enumerable source, return toRet; } + private static BigDecimal rowsRequired(BigDecimal count) { + return count.max(BigDecimal.ZERO).setScale(0, RoundingMode.CEILING); + } + /** * Bypasses a specified number of elements in a * sequence and then returns the remaining elements. @@ -3802,6 +3830,18 @@ public static Enumerable skip(Enumerable source, }); } + /** + * Bypasses a specified number of elements in a + * sequence and then returns the remaining elements. + */ + public static Enumerable skip(Enumerable source, + final BigDecimal count) { + return skipWhileBigDecimal(source, (v1, v2) -> { + // Count is 1-based + return v2.compareTo(count) < 0; + }); + } + /** * Bypasses elements in a sequence as long as a * specified condition is true and then returns the remaining @@ -3957,6 +3997,19 @@ public static Enumerable take(Enumerable source, }); } + /** + * Returns a specified number of contiguous elements + * from the start of a sequence. + */ + public static Enumerable take(Enumerable source, + final BigDecimal count) { + return takeWhileBigDecimal( + source, (v1, v2) -> { + // Count is 1-based + return v2.compareTo(count) < 0; + }); + } + /** * Returns elements from a sequence as long as a * specified condition is true. @@ -3997,6 +4050,36 @@ public static Enumerable takeWhileLong( }; } + /** + * Returns elements from a sequence as long as a + * specified condition is true. The element's index is used in the + * logic of the predicate function. + */ + public static Enumerable takeWhileBigDecimal( + final Enumerable source, + final Predicate2 predicate) { + return new AbstractEnumerable() { + @Override public Enumerator enumerator() { + return new TakeWhileBigDecimalEnumerator<>(source.enumerator(), predicate); + } + }; + } + + /** + * Bypasses elements in a sequence as long as a + * specified condition is true. The element's index is used in the + * logic of the predicate function. + */ + public static Enumerable skipWhileBigDecimal( + final Enumerable source, + final Predicate2 predicate) { + return new AbstractEnumerable() { + @Override public Enumerator enumerator() { + return new SkipWhileBigDecimalEnumerator<>(source.enumerator(), predicate); + } + }; + } + /** * Performs a subsequent ordering of the elements in a sequence according * to a key. @@ -4577,6 +4660,50 @@ static class TakeWhileLongEnumerator implements Enumerator { } } + /** Enumerable that implements take-while. + * + * @param element type */ + static class TakeWhileBigDecimalEnumerator implements Enumerator { + private final Enumerator enumerator; + private final Predicate2 predicate; + + boolean done = false; + BigDecimal n = BigDecimal.valueOf(-1); + + TakeWhileBigDecimalEnumerator(Enumerator enumerator, + Predicate2 predicate) { + this.enumerator = enumerator; + this.predicate = predicate; + } + + @Override public TSource current() { + return enumerator.current(); + } + + @Override public boolean moveNext() { + if (!done) { + if (enumerator.moveNext()) { + n = n.add(BigDecimal.ONE); + if (predicate.apply(enumerator.current(), n)) { + return true; + } + } + done = true; + } + return false; + } + + @Override public void reset() { + enumerator.reset(); + done = false; + n = BigDecimal.valueOf(-1); + } + + @Override public void close() { + enumerator.close(); + } + } + /** Enumerator that implements skip-while. * * @param element type */ @@ -4623,6 +4750,53 @@ static class SkipWhileEnumerator implements Enumerator { } } + /** Enumerator that implements skip-while. + * + * @param element type */ + static class SkipWhileBigDecimalEnumerator implements Enumerator { + private final Enumerator enumerator; + private final Predicate2 predicate; + + boolean started = false; + BigDecimal n = BigDecimal.valueOf(-1); + + SkipWhileBigDecimalEnumerator(Enumerator enumerator, + Predicate2 predicate) { + this.enumerator = enumerator; + this.predicate = predicate; + } + + @Override public TSource current() { + return enumerator.current(); + } + + @Override public boolean moveNext() { + for (;;) { + if (!enumerator.moveNext()) { + return false; + } + if (started) { + return true; + } + n = n.add(BigDecimal.ONE); + if (!predicate.apply(enumerator.current(), n)) { + started = true; + return true; + } + } + } + + @Override public void reset() { + enumerator.reset(); + started = false; + n = BigDecimal.valueOf(-1); + } + + @Override public void close() { + enumerator.close(); + } + } + /** Enumerator that casts each value. * *

      If the source type {@code F} is not nullable, the target element type diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedEnumerable.java b/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedEnumerable.java index ff28c3fe822c..362372464ac4 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedEnumerable.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedEnumerable.java @@ -1095,6 +1095,12 @@ boolean sequenceEqual(Enumerable enumerable1, */ Enumerable skip(int count); + /** + * Bypasses a specified number of elements in a + * sequence and then returns the remaining elements. + */ + Enumerable skip(BigDecimal count); + /** * Bypasses elements in a sequence as long as a * specified condition is true and then returns the remaining @@ -1110,6 +1116,14 @@ boolean sequenceEqual(Enumerable enumerable1, */ Enumerable skipWhile(Predicate2 predicate); + /** + * Bypasses elements in a sequence as long as a + * specified condition is true and then returns the remaining + * elements. The element's index is used in the logic of the + * predicate function. + */ + Enumerable skipWhileBigDecimal(Predicate2 predicate); + /** * Computes the sum of the sequence of Decimal values * that are obtained by invoking a transform function on each @@ -1186,6 +1200,12 @@ boolean sequenceEqual(Enumerable enumerable1, */ Enumerable take(int count); + /** + * Returns a specified number of contiguous elements + * from the start of a sequence. + */ + Enumerable take(BigDecimal count); + /** * Returns elements from a sequence as long as a * specified condition is true. @@ -1199,6 +1219,13 @@ boolean sequenceEqual(Enumerable enumerable1, */ Enumerable takeWhile(Predicate2 predicate); + /** + * Returns elements from a sequence as long as a + * specified condition is true. The element's index is used in the + * logic of the predicate function. + */ + Enumerable takeWhileBigDecimal(Predicate2 predicate); + /** * Creates a {@code Map} from an * {@code Enumerable} according to a specified key selector diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/Linq4j.java b/linq4j/src/main/java/org/apache/calcite/linq4j/Linq4j.java index 9e2fa64a4c03..df9e39e35bd2 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/Linq4j.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/Linq4j.java @@ -21,6 +21,8 @@ import org.checkerframework.checker.nullness.qual.Nullable; import java.lang.reflect.Method; +import java.math.BigDecimal; +import java.math.RoundingMode; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -591,6 +593,16 @@ static class ListEnumerable extends CollectionEnumerable { return new ListEnumerable<>(list.subList(count, list.size())); } + @Override public Enumerable skip(BigDecimal count) { + final List list = toList(); + BigDecimal rows = count.max(BigDecimal.ZERO) + .setScale(0, RoundingMode.CEILING); + if (rows.compareTo(BigDecimal.valueOf(list.size())) >= 0) { + return Linq4j.emptyEnumerable(); + } + return new ListEnumerable<>(list.subList(rows.intValueExact(), list.size())); + } + @Override public Enumerable take(int count) { final List list = toList(); if (count >= list.size()) { @@ -599,6 +611,16 @@ static class ListEnumerable extends CollectionEnumerable { return new ListEnumerable<>(list.subList(0, count)); } + @Override public Enumerable take(BigDecimal count) { + final List list = toList(); + BigDecimal rows = count.max(BigDecimal.ZERO) + .setScale(0, RoundingMode.CEILING); + if (rows.compareTo(BigDecimal.valueOf(list.size())) >= 0) { + return this; + } + return new ListEnumerable<>(list.subList(0, rows.intValueExact())); + } + @Override public T elementAt(int index) { return toList().get(index); } diff --git a/linq4j/src/test/java/org/apache/calcite/linq4j/test/Linq4jTest.java b/linq4j/src/test/java/org/apache/calcite/linq4j/test/Linq4jTest.java index bacb6da3c8b1..62e7dd9f5eef 100644 --- a/linq4j/src/test/java/org/apache/calcite/linq4j/test/Linq4jTest.java +++ b/linq4j/src/test/java/org/apache/calcite/linq4j/test/Linq4jTest.java @@ -1264,6 +1264,28 @@ private List contentsOf(Enumerator enumerator) { .toList(), hasSize(0)); } + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test void testTakeEnumerableDefaultsBigDecimalSize() { + assertThat( + EnumerableDefaults.take(Linq4j.asEnumerable(depts), + new BigDecimal("1.5")).toList(), hasSize(2)); + assertThat( + EnumerableDefaults.take(Linq4j.asEnumerable(depts), + BigDecimal.valueOf(-2)).toList(), hasSize(0)); + } + + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test void testTakeListEnumerableBigDecimalSize() { + assertThat(Linq4j.asEnumerable(depts).take(new BigDecimal("1.5")) + .toList(), hasSize(2)); + assertThat(Linq4j.asEnumerable(depts).take(BigDecimal.valueOf(-2)) + .toList(), hasSize(0)); + } + @Test void testTakeQueryableZeroOrNegativeSize() { assertThat(QueryableDefaults.take(Linq4j.asEnumerable(depts).asQueryable(), 0) .toList(), hasSize(0)); @@ -1424,6 +1446,28 @@ public boolean apply(Department v1, Integer v2) { || v2 == 1)).count(), is(1)); } + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test void testSkipEnumerableDefaultsBigDecimalSize() { + assertThat( + EnumerableDefaults.skip(Linq4j.asEnumerable(depts), + new BigDecimal("1.5")).count(), is(1)); + assertThat( + EnumerableDefaults.skip(Linq4j.asEnumerable(depts), + BigDecimal.valueOf(-2)).count(), is(3)); + } + + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test void testSkipListEnumerableBigDecimalSize() { + assertThat(Linq4j.asEnumerable(depts).skip(new BigDecimal("1.5")) + .count(), is(1)); + assertThat(Linq4j.asEnumerable(depts).skip(BigDecimal.valueOf(-2)) + .count(), is(3)); + } + @Test void testOrderBy() { // Note: sort is stable. Records occur Fred, Eric, Janet in input. assertThat(Linq4j.asEnumerable(emps).orderBy(EMP_DEPTNO_SELECTOR) diff --git a/site/_docs/reference.md b/site/_docs/reference.md index 90511aa40664..8ff9857cb8c0 100644 --- a/site/_docs/reference.md +++ b/site/_docs/reference.md @@ -427,8 +427,9 @@ in the order that they appear in the list; for example: "SELECT x, y FROM t ORDER BY x, y" An optional trailing ASC / DESC and NULLS FIRST / NULLS LAST applies to all keys. -In *query*, *count* and *start* may each be either an unsigned integer literal -or a dynamic parameter whose value is an integer. +In *query*, *count* and *start* may each be either an unsigned numeric literal +or a dynamic parameter whose value is numeric. +Support for decimal or non-integer values is adapter-dependent. An aggregate query is a query that contains a GROUP BY or a HAVING clause, or aggregate functions in the SELECT clause. In the SELECT, From c8d4ad548d41e26203124392ce9479682d7c9f87 Mon Sep 17 00:00:00 2001 From: Ruben Quesada Lopez Date: Fri, 10 Jul 2026 13:20:23 +0100 Subject: [PATCH 382/562] [CALCITE-6393] Byte code of SqlFunctions is invalid --- .../apache/calcite/runtime/SqlFunctions.java | 30 +++++++++++-------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java index 789fb9d6c794..5dfad1da34cf 100644 --- a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java +++ b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java @@ -210,18 +210,24 @@ public class SqlFunctions { @SuppressWarnings("unused") private static final Function1> ARRAY_CARTESIAN_PRODUCT = - lists -> { - final List> enumerators = new ArrayList<>(); - for (Object list : lists) { - enumerators.add(Linq4j.enumerator((List) list)); - } - final Enumerator> product = Linq4j.product(enumerators); - return new AbstractEnumerable<@Nullable Object[]>() { - @Override public Enumerator<@Nullable Object[]> enumerator() { - return Linq4j.transform(product, List::toArray); - } - }; - }; + SqlFunctions::arrayCartesianProduct; + + /** + * WARNING: keep this logic as a static method. JDK 8 and 11 produce invalid bytecode when + * checkerframework annotations are used on static lambdas. See CALCITE-6393. + */ + private static Enumerable<@Nullable Object[]> arrayCartesianProduct(Object[] lists) { + final List> enumerators = new ArrayList<>(); + for (Object list : lists) { + enumerators.add(Linq4j.enumerator((List) list)); + } + final Enumerator> product = Linq4j.product(enumerators); + return new AbstractEnumerable<@Nullable Object[]>() { + @Override public Enumerator<@Nullable Object[]> enumerator() { + return Linq4j.transform(product, List::toArray); + } + }; + } /** Holds, for each thread, a map from sequence name to sequence current * value. From f7982ad3554f00e9c403eee735eeac1254cef271 Mon Sep 17 00:00:00 2001 From: xuzifu666 <1206332514@qq.com> Date: Fri, 26 Jun 2026 10:45:13 +0800 Subject: [PATCH 383/562] [CALCITE-6104] Aggregate function that references outer column should be evaluated in outer query --- .../calcite/runtime/CalciteResource.java | 3 + .../sql/validate/SqlAbstractConformance.java | 4 + .../calcite/sql/validate/SqlConformance.java | 16 + .../sql/validate/SqlConformanceEnum.java | 10 + .../validate/SqlDelegatingConformance.java | 4 + .../sql/validate/SqlValidatorImpl.java | 328 ++++++++++++++++++ .../runtime/CalciteResource.properties | 1 + .../apache/calcite/test/SqlValidatorTest.java | 38 ++ core/src/test/resources/sql/agg.iq | 227 ++++++++++++ 9 files changed, 631 insertions(+) diff --git a/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java b/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java index 01ea9cbda0b1..c6e1a4dbdcc5 100644 --- a/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java +++ b/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java @@ -930,6 +930,9 @@ ExInst illegalArgumentForTableFunctionCall(String a0, @BaseMessage("Extended columns not allowed under the current SQL conformance level") ExInst extendNotAllowed(); + @BaseMessage("Aggregate function referencing outer column is not allowed under the current SQL conformance level") + ExInst correlatedAggregateNotAllowed(); + @BaseMessage("Rolled up column ''{0}'' is not allowed in {1}") ExInst rolledUpNotAllowed(String column, String context); diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlAbstractConformance.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlAbstractConformance.java index 84cf77e3c96a..7734f1b2fec5 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlAbstractConformance.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlAbstractConformance.java @@ -184,4 +184,8 @@ public abstract class SqlAbstractConformance implements SqlConformance { @Override public boolean isDistinctOnAllowed() { return SqlConformanceEnum.DEFAULT.isDistinctOnAllowed(); } + + @Override public boolean isCorrelatedAggregateAllowed() { + return SqlConformanceEnum.DEFAULT.isCorrelatedAggregateAllowed(); + } } diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlConformance.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlConformance.java index db0dd693a99c..703dfd4a1447 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlConformance.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlConformance.java @@ -731,4 +731,20 @@ default boolean isColonFieldAccessAllowed() { * false otherwise. */ boolean isDistinctOnAllowed(); + + /** + * Whether an aggregate function inside a scalar sub-query is allowed to + * reference columns from an outer query. + * + *

      This is not allowed by the SQL standard, but is supported by some + * databases, including SQLite, DuckDB and SQL Server. + * + *

      Among the built-in conformance levels, true in + * {@link SqlConformanceEnum#BABEL}, + * {@link SqlConformanceEnum#LENIENT}; + * false otherwise. + */ + default boolean isCorrelatedAggregateAllowed() { + return false; + } } diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlConformanceEnum.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlConformanceEnum.java index 4ed6cb93c285..681835724fb1 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlConformanceEnum.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlConformanceEnum.java @@ -562,4 +562,14 @@ public enum SqlConformanceEnum implements SqlConformance { return false; } } + + @Override public boolean isCorrelatedAggregateAllowed() { + switch (this) { + case BABEL: + case LENIENT: + return true; + default: + return false; + } + } } diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlDelegatingConformance.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlDelegatingConformance.java index e142893891e1..7bed86a41072 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlDelegatingConformance.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlDelegatingConformance.java @@ -189,4 +189,8 @@ protected SqlDelegatingConformance(SqlConformance delegate) { @Override public boolean isDistinctOnAllowed() { return delegate.isDistinctOnAllowed(); } + + @Override public boolean isCorrelatedAggregateAllowed() { + return delegate.isCorrelatedAggregateAllowed(); + } } diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java index 331c8849a699..5d667fe502e3 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java @@ -5618,6 +5618,17 @@ protected RelDataType validateSelectList(final SqlNodeList selectItems, // First pass, ensure that aliases are unique. "*" and "TABLE.*" items // are ignored. + // Rewrite scalar sub-queries whose single select item is an aggregate + // over outer columns. The aggregate belongs to the outer query per SQL + // standard, but only if the current conformance allows this non-standard + // correlated-aggregate construct. If the conformance does not allow it, + // validate that no such construct is present. + if (config.conformance().isCorrelatedAggregateAllowed()) { + rewriteOuterAggregatesInSelectList(select); + } else { + checkNoCorrelatedAggregatesInSelectList(select); + } + // Validate SELECT list. Expand terms of the form "*" or "TABLE.*". final SqlValidatorScope selectScope = getSelectScope(select); final List expandedSelectItems = new ArrayList<>(); @@ -5677,6 +5688,323 @@ protected RelDataType validateSelectList(final SqlNodeList selectItems, return typeFactory.createStructType(fieldList); } + /** + * Rewrites scalar sub-queries in the SELECT list whose single select item is + * an aggregate function whose arguments reference only outer columns. Per the + * SQL standard, such aggregates belong to the outer query. + * + *

      The algorithm is: + *

        + *
      1. For each item in the SELECT list, check whether it is a scalar + * sub-query, optionally wrapped in {@code AS} or {@code WITH}. + *
      2. Inside the sub-query, require the SELECT list to contain exactly one + * item, and that item to be an aggregate function call. + *
      3. Require every argument of the aggregate to reference only columns from + * the outer query (no inner columns), and to contain no nested sub-queries. + *
      4. If all conditions hold, lift the aggregate out of the sub-query: keep + * the aggregate in the outer SELECT list, and guard it with + * {@code (SELECT 1 FROM ... LIMIT 1) IS NOT NULL}. The result is equivalent + * because the aggregate is evaluated over the outer rows, but yields NULL + * whenever the inner query has no rows (as the original scalar sub-query + * would). + *
      5. If the outer query becomes an aggregate query as a result, upgrade its + * SELECT clause scope from {@link SelectScope} to + * {@link AggregatingSelectScope}. + *
      + * + *

      For example, + * 

      +   * WITH aa(a) AS (VALUES 1, 2, 3),
+   *      t(x) AS (VALUES 10, 20, 30)
+   * SELECT (SELECT sum(a) FROM t) FROM aa
+   *
      + * is rewritten to + * 
      +   * WITH aa(a) AS (VALUES 1, 2, 3),
+   *      t(x) AS (VALUES 10, 20, 30)
+   * SELECT CASE WHEN (SELECT 1 FROM t LIMIT 1) IS NOT NULL
+   *        THEN sum(a) END FROM aa
+   *
      + */ + private void rewriteOuterAggregatesInSelectList(SqlSelect select) { + final SqlNodeList selectItems = select.getSelectList(); + if (selectItems == null) { + return; + } + final SqlValidatorScope selectScope = getSelectScope(select); + final boolean wasAggregate = isAggregate(select); + for (int i = 0; i < selectItems.size(); i++) { + final SqlNode selectItem = selectItems.get(i); + final SqlNode rewrittenItem = + rewriteOuterAggregateItem(selectScope, selectItem); + if (rewrittenItem != selectItem) { + selectItems.set(i, rewrittenItem); + } + } + // The rewrite may have introduced an aggregate into a query that was not + // previously aggregate. Update the SELECT clause scope accordingly so that + // subsequent validation and conversion see an AggregatingSelectScope. + if (!wasAggregate && isAggregate(select)) { + SqlValidatorScope scope = + clauseScopes.get(IdPair.of(select, Clause.SELECT)); + if (scope instanceof AggregatingSelectScope) { + scope = ((AggregatingSelectScope) scope).getParent(); + } + clauseScopes.put(IdPair.of(select, Clause.SELECT), + new AggregatingSelectScope( + requireNonNull(scope, "scope"), select, false)); + } + } + + /** + * Validates that the SELECT list does not contain a correlated aggregate + * when the current conformance does not allow it. + */ + private void checkNoCorrelatedAggregatesInSelectList(SqlSelect select) { + final SqlNodeList selectItems = select.getSelectList(); + if (selectItems == null) { + return; + } + final SqlValidatorScope selectScope = getSelectScope(select); + for (SqlNode selectItem : selectItems) { + final SqlSelect subQuery = findScalarSubQuerySelect(selectItem); + if (subQuery == null) { + continue; + } + final SqlNode aggExpr = findCorrelatedAggregate(subQuery, selectScope); + if (aggExpr != null) { + throw newValidationError(aggExpr, + RESOURCE.correlatedAggregateNotAllowed()); + } + } + } + + /** + * Rewrites a single SELECT list item if it is a scalar sub-query containing + * an aggregate over outer columns. + * + * @return the rewritten expression, or {@code selectItem} if no rewrite applies + */ + private SqlNode rewriteOuterAggregateItem(SqlValidatorScope parentScope, + SqlNode selectItem) { + final SqlSelect subQuery = findScalarSubQuerySelect(selectItem); + if (subQuery == null) { + return selectItem; + } + // Unwrap AS. + SqlNode expr = selectItem; + @Nullable SqlIdentifier alias = null; + if (SqlUtil.isCallTo(selectItem, SqlStdOperatorTable.AS)) { + SqlCall asCall = (SqlCall) selectItem; + expr = asCall.operand(0); + alias = asCall.operand(1); + } + final SqlBasicCall scalarSubQuery = (SqlBasicCall) expr; + final SqlNode query = scalarSubQuery.operand(0); + final SqlNode rewrittenSubQuery = + rewriteOuterAggregate(query, subQuery, parentScope); + if (rewrittenSubQuery == query) { + return selectItem; + } + SqlNode result = rewrittenSubQuery; + if (alias != null) { + result = + SqlStdOperatorTable.AS.createCall(selectItem.getParserPosition(), + rewrittenSubQuery, alias); + } + return result; + } + + /** + * Extracts the {@link SqlSelect} from a SELECT list item that is a scalar + * sub-query, optionally wrapped in {@code AS}. + * + * @return the sub-query's SELECT, or {@code null} if the item is not a + * scalar sub-query + */ + private @Nullable SqlSelect findScalarSubQuerySelect(SqlNode selectItem) { + SqlNode expr = selectItem; + if (SqlUtil.isCallTo(selectItem, SqlStdOperatorTable.AS)) { + expr = ((SqlCall) selectItem).operand(0); + } + if (!SqlUtil.isCallTo(expr, SqlStdOperatorTable.SCALAR_QUERY)) { + return null; + } + final SqlNode query = ((SqlBasicCall) expr).operand(0); + return query instanceof SqlWith + ? (SqlSelect) ((SqlWith) query).body + : query instanceof SqlSelect + ? (SqlSelect) query + : null; + } + + /** + * Finds the single select item of a scalar sub-query that can be lifted to + * the outer query. Such an item contains at least one aggregate function and + * references only columns from the outer query, so per the SQL standard it + * belongs to the outer query. + * + *

      The item may be a bare aggregate call (e.g. {@code sum(a)}) or an + * expression over aggregates of outer columns (e.g. {@code sum(a) + sum(b)}). + * It is not lifted if any column reference resolves to an inner column, which + * covers mixed cases such as {@code sum(a) + sum(x)} where {@code x} is an + * inner column. + * + * @return the liftable select item, or {@code null} if none applies + */ + private @Nullable SqlNode findCorrelatedAggregate(SqlSelect subQuery, + SqlValidatorScope parentScope) { + final SqlNodeList subSelectItems = SqlNonNullableAccessors.getSelectList(subQuery); + if (subSelectItems.size() != 1) { + return null; + } + if (subQuery.getGroup() != null && !subQuery.getGroup().isEmpty()) { + return null; + } + SqlNode subSelectItem = subSelectItems.get(0); + if (SqlUtil.isCallTo(subSelectItem, SqlStdOperatorTable.AS)) { + subSelectItem = ((SqlCall) subSelectItem).operand(0); + } + if (aggFinder.findAgg(subSelectItem) == null) { + return null; + } + final SqlValidatorScope subScope = getSelectScope(subQuery); + final SqlValidatorScope operandScope = + subScope instanceof AggregatingSelectScope + ? ((AggregatingSelectScope) subScope).parent + : subScope; + if (!referencesOnlyOuterColumns(subSelectItem, parentScope, operandScope)) { + return null; + } + return subSelectItem; + } + + /** + * Rewrites a scalar sub-query whose single select item is an aggregate + * function whose arguments reference only outer columns. The aggregate is + * pulled out to the enclosing query, and the sub-query's select item is + * replaced with the constant 1. + * + * @return the rewritten sub-query expression, or {@code subQuery} if no rewrite applies + */ + private SqlNode rewriteOuterAggregate(SqlNode originalQuery, + SqlSelect subQuery, SqlValidatorScope parentScope) { + final SqlNode aggExpr = findCorrelatedAggregate(subQuery, parentScope); + if (aggExpr == null) { + return originalQuery; + } + final SqlNodeList subSelectItems = SqlNonNullableAccessors.getSelectList(subQuery); + // Rewrite: replace aggregate with 1 in a new sub-query, and multiply + // by the aggregate in the outer query. + final SqlLiteral one = SqlLiteral.createExactNumeric("1", SqlParserPos.ZERO); + final SqlNodeList newSubSelectItems = + new SqlNodeList(ImmutableList.of(one), subSelectItems.getParserPosition()); + final SqlNodeList keywordList = (SqlNodeList) subQuery.getOperandList().get(0); + final SqlSelect newSubQuery = + new SqlSelect(subQuery.getParserPosition(), + keywordList, + newSubSelectItems, + subQuery.getFrom(), + subQuery.getWhere(), + subQuery.getGroup(), + subQuery.getHaving(), + subQuery.getWindowList(), + subQuery.getQualify(), + subQuery.getOrderList(), + subQuery.getOffset(), + subQuery.getFetch(), + subQuery.getHints(), + subQuery.getDistinctOn()); + // Ensure the rewritten sub-query still returns at most one row, because + // removing the aggregate may otherwise produce multiple rows. + if (newSubQuery.getFetch() == null) { + newSubQuery.setFetch( + SqlLiteral.createExactNumeric("1", SqlParserPos.ZERO)); + } + final SqlNode newQuery = originalQuery instanceof SqlWith + ? new SqlWith(originalQuery.getParserPosition(), + ((SqlWith) originalQuery).withList, newSubQuery) + : newSubQuery; + registerQuery(parentScope, null, newQuery, newQuery, null, false); + validateQuery(newQuery, parentScope, unknownType); + final SqlNode scalarSubQuery = + SqlStdOperatorTable.SCALAR_QUERY.createCall(newQuery.getParserPosition(), newQuery); + + final SqlParserPos pos = aggExpr.getParserPosition(); + final SqlNode condition = + SqlStdOperatorTable.IS_NOT_NULL.createCall(pos, scalarSubQuery); + return new SqlCase(pos, null, + new SqlNodeList(ImmutableList.of(condition), pos), + new SqlNodeList(ImmutableList.of(aggExpr), pos), + SqlLiteral.createNull(pos)); + } + + /** + * Returns whether an expression contains only references to columns that are + * outside the current select scope, and can be resolved in the parent scope. + */ + private boolean referencesOnlyOuterColumns(SqlNode node, + SqlValidatorScope parentScope, SqlValidatorScope currentScope) { + // Do not rewrite if the aggregate argument contains a sub-query; the + // sub-query may reference inner columns, and pulling the aggregate out + // would change the semantics. + if (containsSubQuery(node)) { + return false; + } + // ok[0] is cleared if any identifier is not an outer reference; + // ok[1] is set once at least one outer column is found. + final boolean[] ok = {true, false}; + node.accept(new SqlBasicVisitor() { + @Override public Void visit(SqlIdentifier id) { + if (!isOuterReference(currentScope, id)) { + ok[0] = false; + return null; + } + // The identifier must be resolvable in the parent scope; otherwise it + // references an intermediate scope, not the immediate enclosing query. + try { + parentScope.fullyQualify(id); + ok[1] = true; + } catch (CalciteException e) { + ok[0] = false; + } + return null; + } + }); + return ok[0] && ok[1]; + } + + /** + * Returns whether an expression contains a sub-query. + */ + private static boolean containsSubQuery(SqlNode node) { + final boolean[] found = {false}; + node.accept(new SqlBasicVisitor() { + @Override public Void visit(SqlCall call) { + if (call.getKind().belongsTo(SqlKind.QUERY)) { + found[0] = true; + return null; + } + return super.visit(call); + } + }); + return found[0]; + } + + /** + * Returns whether an identifier resolves to a scope which is not the + * supplied one. + */ + private boolean isOuterReference(SqlValidatorScope scope, SqlIdentifier id) { + final SqlQualified fqId = scope.fullyQualify(id); + if (fqId.prefixLength <= 0) { + return false; + } + final SqlValidatorScope.ResolvedImpl resolved = new SqlValidatorScope.ResolvedImpl(); + scope.resolve(fqId.prefix(), catalogReader.nameMatcher(), false, resolved); + return resolved.count() == 1 && !resolved.only().scope.isWithin(scope); + } + /** * Validates an expression. * diff --git a/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties b/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties index 27e52cbce900..f4f16d73266a 100644 --- a/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties +++ b/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties @@ -303,6 +303,7 @@ IntervalMustBeNonNegative=Interval must be non-negative ''{0}'' CannotUseWithinWithoutOrderBy=Must contain an ORDER BY clause when WITHIN is used FirstColumnOfOrderByMustBeTimestamp=First column of ORDER BY must be of type TIMESTAMP ExtendNotAllowed=Extended columns not allowed under the current SQL conformance level +CorrelatedAggregateNotAllowed=Aggregate function referencing outer column is not allowed under the current SQL conformance level RolledUpNotAllowed=Rolled up column ''{0}'' is not allowed in {1} SchemaExists=Schema ''{0}'' already exists SchemaInvalidType=Invalid schema type ''{0}''; valid values: {1} diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index efa86f9c0b02..228d90026b22 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -8469,6 +8469,44 @@ void testGroupExpressionEquivalenceParams() { .columnType("INTEGER NOT NULL"); } + @Test void testCorrelatedAggregateConformance() { + final String sql = "select (select ^sum(sal)^ from dept) from emp"; + // LENIENT and BABEL allow the non-standard correlated-aggregate construct. + sql(sql).withConformance(SqlConformanceEnum.LENIENT).ok(); + sql(sql).withConformance(SqlConformanceEnum.BABEL).ok(); + // DEFAULT and STRICT_2003 do not. + final String err = + "Aggregate function referencing outer column is not allowed under the current SQL conformance level"; + sql(sql).withConformance(SqlConformanceEnum.DEFAULT).fails(err); + sql(sql).withConformance(SqlConformanceEnum.STRICT_2003).fails(err); + + // A non-numeric aggregate over an outer column (MAX of a VARCHAR) is also + // lifted to the outer query; the CASE-based rewrite is type-agnostic. + final String nonNumeric = + "select (select max(ename) from dept) from emp"; + sql(nonNumeric).withConformance(SqlConformanceEnum.LENIENT).ok(); + sql(nonNumeric).withConformance(SqlConformanceEnum.BABEL).ok(); + + // An expression over aggregates of outer columns is lifted to the outer + // query as a whole, and is subject to the same conformance rules as a bare + // aggregate. + final String twoAggs = + "select (select ^sum(sal) + sum(comm)^ from dept) from emp"; + sql(twoAggs).withConformance(SqlConformanceEnum.LENIENT).ok(); + sql(twoAggs).withConformance(SqlConformanceEnum.BABEL).ok(); + sql(twoAggs).withConformance(SqlConformanceEnum.DEFAULT).fails(err); + sql(twoAggs).withConformance(SqlConformanceEnum.STRICT_2003).fails(err); + + // Mixed case: an expression aggregating both an outer column (sal) and an + // inner column (deptno) references an inner column, so it is not lifted and + // is not rejected under any conformance level. + final String mixed = + "select (select sum(sal) + sum(deptno) from dept) from emp"; + sql(mixed).withConformance(SqlConformanceEnum.LENIENT).ok(); + sql(mixed).withConformance(SqlConformanceEnum.DEFAULT).ok(); + sql(mixed).withConformance(SqlConformanceEnum.STRICT_2003).ok(); + } + @Test void testAggregateInOrderByFails() { sql("select empno from emp order by ^sum(empno)^") .fails(ERR_AGG_IN_ORDER_BY); diff --git a/core/src/test/resources/sql/agg.iq b/core/src/test/resources/sql/agg.iq index 5478f6bb6e2c..b637a6cbc196 100644 --- a/core/src/test/resources/sql/agg.iq +++ b/core/src/test/resources/sql/agg.iq @@ -4301,4 +4301,231 @@ GROUP BY GROUPING SETS ((deptno), ()); !ok +# [CALCITE-6104] Aggregate function that references outer column should be evaluated in outer query +# Correlated aggregates are a non-standard extension. +# They are rejected under DEFAULT conformance. +!use scott + +SELECT (SELECT sum(sal) FROM dept) FROM emp; +Aggregate function referencing outer column is not allowed under the current SQL conformance level +!error + +WITH aa (a) AS (VALUES 1, 2, 3), + xx (x) AS (VALUES 10, 20, 30) +SELECT (SELECT count(a) FROM xx LIMIT 1) AS ca +FROM aa; +Aggregate function referencing outer column is not allowed under the current SQL conformance level +!error + +WITH aa (a) AS (VALUES 1, 2, 3), + xx (x) AS (VALUES 10, 20, 30) +SELECT (SELECT sum(a) FROM xx LIMIT 1) AS sa, + (SELECT count(a) FROM xx LIMIT 1) AS ca +FROM aa; +Aggregate function referencing outer column is not allowed under the current SQL conformance level +!error + +# Allowed under LENIENT conformance. +!use scott-lenient + +WITH aa (a) AS (VALUES 1, 2, 3), + xx (x) AS (VALUES 10, 20, 30) +SELECT (SELECT sum(a) FROM xx LIMIT 1) AS sa +FROM aa; ++----+ +| SA | ++----+ +| 6 | ++----+ +(1 row) + +!ok + +# A non-numeric aggregate (MAX over a VARCHAR outer column) is also lifted to +# the outer query. The CASE-based rewrite is type-agnostic, so the aggregate is +# evaluated over the outer rows and produces a single row. +SELECT (SELECT max(ename) FROM dept) AS m FROM emp; ++------+ +| M | ++------+ +| WARD | ++------+ +(1 row) + +!ok + +# Inner table is empty: the scalar sub-query contributes NULL +WITH aa (a) AS (VALUES 1, 2, 3), + empty_xx (x) AS (SELECT 1 FROM emp WHERE 1 = 0) +SELECT (SELECT sum(a) FROM empty_xx LIMIT 1) AS sa +FROM aa; ++----+ +| SA | ++----+ +| | ++----+ +(1 row) + +!ok + +# Outer table is empty: the outer aggregate has no input rows +WITH empty_aa (a) AS (SELECT 1 FROM emp WHERE 1 = 0), + xx (x) AS (VALUES 10, 20, 30) +SELECT (SELECT sum(a) FROM xx LIMIT 1) AS sa +FROM empty_aa; ++----+ +| SA | ++----+ +| | ++----+ +(1 row) + +!ok + +# Aggregated column contains a mix of values and NULLs +WITH aa (a) AS (VALUES 1, NULL, 3), + xx (x) AS (VALUES 10, 20, 30) +SELECT (SELECT sum(a) FROM xx LIMIT 1) AS sa +FROM aa; ++----+ +| SA | ++----+ +| 4 | ++----+ +(1 row) + +!ok + +# Aggregate with no column references aggregates at the innermost level +WITH aa (a) AS (VALUES 1, 2, 3), + xx (x) AS (VALUES 10, 20, 30) +SELECT (SELECT sum(1) FROM xx LIMIT 1) AS s +FROM aa; ++---+ +| S | ++---+ +| 3 | +| 3 | +| 3 | ++---+ +(3 rows) + +!ok + +# Aggregate referencing only inner column aggregates at the inner level +WITH aa (a) AS (VALUES 1, 2, 3), + xx (x) AS (VALUES 10, 20, 30) +SELECT (SELECT sum(x) FROM xx LIMIT 1) AS s +FROM aa; ++----+ +| S | ++----+ +| 60 | +| 60 | +| 60 | ++----+ +(3 rows) + +!ok + +# An expression over aggregates of different outer columns is lifted to the +# outer query as a whole: sum(a) = 1+2+3 = 6, sum(b) = 10+20+30 = 60, so the +# result is a single row 66. +WITH aa (a, b) AS (VALUES (1, 10), (2, 20), (3, 30)), + xx (x) AS (VALUES 100, 200, 300) +SELECT (SELECT sum(a) + sum(b) FROM xx LIMIT 1) AS s +FROM aa; ++----+ +| S | ++----+ +| 66 | ++----+ +(1 row) + +!ok + +# A single aggregate over an arithmetic expression of outer columns is also +# lifted: sum(a + b) = (1+10) + (2+20) + (3+30) = 66. +WITH aa (a, b) AS (VALUES (1, 10), (2, 20), (3, 30)), + xx (x) AS (VALUES 100, 200, 300) +SELECT (SELECT sum(a + b) FROM xx LIMIT 1) AS s +FROM aa; ++----+ +| S | ++----+ +| 66 | ++----+ +(1 row) + +!ok + +# Lifted expression with an empty inner table: the guard sub-query yields no +# row, so the whole expression is NULL. +WITH aa (a, b) AS (VALUES (1, 10), (2, 20), (3, 30)), + empty_xx (x) AS (SELECT 1 FROM emp WHERE 1 = 0) +SELECT (SELECT sum(a) + sum(b) FROM empty_xx LIMIT 1) AS s +FROM aa; ++---+ +| S | ++---+ +| | ++---+ +(1 row) + +!ok + +# Mixed case: the expression aggregates an outer column (a) and an inner +# column (x). It references an inner column, so it is not lifted and keeps the +# pre-existing correlated-sub-query behaviour (evaluated once per outer row). +WITH aa (a) AS (VALUES 1, 2, 3), + xx (x) AS (VALUES 10, 20, 30) +SELECT (SELECT sum(a) + sum(x) FROM xx LIMIT 1) AS s +FROM aa; ++----+ +| S | ++----+ +| 63 | +| 66 | +| 69 | ++----+ +(3 rows) + +!ok + +# Two scalar sub-queries with outer-only aggregates are both lifted to the outer query +WITH aa (a) AS (VALUES 1, 2, 3), + xx (x) AS (VALUES 10, 20, 30) +SELECT (SELECT sum(a) FROM xx LIMIT 1) AS sa, + (SELECT count(a) FROM xx LIMIT 1) AS ca +FROM aa; ++----+----+ +| SA | CA | ++----+----+ +| 6 | 3 | ++----+----+ +(1 row) + +!ok + +EnumerableCalc(expr#0..2=[{inputs}], expr#3=[IS NOT NULL($t2)], expr#4=[null:INTEGER], expr#5=[CASE($t3, $t0, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t3, $t1, $t6)], SA=[$t5], CA=[$t7]) + EnumerableNestedLoopJoin(condition=[true], joinType=[left]) + EnumerableAggregate(group=[{}], agg#0=[SUM($0)], agg#1=[COUNT()]) + EnumerableValues(tuples=[[{ 1 }, { 2 }, { 3 }]]) + EnumerableLimit(fetch=[1]) + EnumerableValues(tuples=[[{ 1 }, { 1 }, { 1 }]]) +!plan + +SELECT (SELECT sum(sal) FROM dept) AS sum_sal +FROM emp; ++----------+ +| SUM_SAL | ++----------+ +| 29025.00 | ++----------+ +(1 row) + +!ok + +!use scott + # End agg.iq From c677fff6f6574dfcdffc06f95719195047f484ef Mon Sep 17 00:00:00 2001 From: bibi samina Date: Sat, 11 Jul 2026 11:12:17 +0530 Subject: [PATCH 384/562] BigQuery dialect should escape backslashes in string literals --- .../calcite/sql/dialect/BigQuerySqlDialect.java | 10 ++++++++++ .../calcite/rel/rel2sql/RelToSqlConverterTest.java | 14 ++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/core/src/main/java/org/apache/calcite/sql/dialect/BigQuerySqlDialect.java b/core/src/main/java/org/apache/calcite/sql/dialect/BigQuerySqlDialect.java index 6e5ebeb4c09d..6683054819ea 100644 --- a/core/src/main/java/org/apache/calcite/sql/dialect/BigQuerySqlDialect.java +++ b/core/src/main/java/org/apache/calcite/sql/dialect/BigQuerySqlDialect.java @@ -117,6 +117,16 @@ public BigQuerySqlDialect(SqlDialect.Context context) { || RESERVED_KEYWORDS.contains(val.toUpperCase(Locale.ROOT)); } + @Override public void quoteStringLiteral(StringBuilder buf, + @Nullable String charsetName, String val) { + // BigQuery treats backslash as an escape character inside string literals, + // so a literal backslash must be doubled before the base method escapes the + // enclosing quote as \'. Otherwise a value containing a backslash (e.g. + // "x\" or "\'; ...") terminates the literal early and the trailing text is + // parsed as SQL rather than data. + super.quoteStringLiteral(buf, charsetName, val.replace("\\", "\\\\")); + } + @Override public boolean supportsImplicitTypeCoercion(RexCall call) { return super.supportsImplicitTypeCoercion(call) && RexUtil.isLiteral(call.getOperands().get(0), false) diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java index 95bbe98f6adf..3327d0f3687f 100644 --- a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java +++ b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java @@ -3905,6 +3905,20 @@ private SqlDialect nonOrdinalDialect() { .withBigQuery().ok(expectedBigQuery); } + /** Tests that a backslash in a character literal is escaped for BigQuery, + * which uses backslash as the escape character. Without doubling the + * backslash the generated literal is terminated early. */ + @Test void testCharLiteralWithBackslashForBigQuery() { + final String query = "select 'a\\b' from \"product\""; + final String expectedPostgresql = "SELECT 'a\\b'\n" + + "FROM \"foodmart\".\"product\""; + final String expectedBigQuery = "SELECT 'a\\\\b'\n" + + "FROM foodmart.product"; + sql(query) + .withPostgresql().ok(expectedPostgresql) + .withBigQuery().ok(expectedBigQuery); + } + @Test void testIdentifier() { // Note that IGNORE is reserved in BigQuery but not in standard SQL final String query = "select *\n" From 69caaaaf616b8401c16ca04b40f6c8455f7c4014 Mon Sep 17 00:00:00 2001 From: AlexisCubilla Date: Mon, 13 Jul 2026 13:28:19 -0300 Subject: [PATCH 385/562] [CALCITE-7652] MssqlSqlDialect unparses CAST to TIMESTAMP as "TIMESTAMP", which is invalid in SQL Server (should be DATETIME2) MssqlSqlDialect.getCastSpec did not override TIMESTAMP, so a CAST to TIMESTAMP was emitted with the ANSI type name TIMESTAMP. In SQL Server, TIMESTAMP is a deprecated synonym for ROWVERSION (a binary type), so the generated SQL was rejected ("CAST or CONVERT: invalid attributes specified for type 'timestamp'"). Map TIMESTAMP to DATETIME2 and TIMESTAMP WITH LOCAL TIME ZONE to DATETIMEOFFSET (SQL Server 2008+), preserving fractional-seconds precision. --- .../calcite/sql/dialect/MssqlSqlDialect.java | 39 ++++++++++++ .../rel/rel2sql/RelToSqlConverterTest.java | 61 +++++++++++++++++++ 2 files changed, 100 insertions(+) diff --git a/core/src/main/java/org/apache/calcite/sql/dialect/MssqlSqlDialect.java b/core/src/main/java/org/apache/calcite/sql/dialect/MssqlSqlDialect.java index a88f78b48f8e..964fa03f8f91 100644 --- a/core/src/main/java/org/apache/calcite/sql/dialect/MssqlSqlDialect.java +++ b/core/src/main/java/org/apache/calcite/sql/dialect/MssqlSqlDialect.java @@ -22,8 +22,10 @@ import org.apache.calcite.rel.type.RelDataTypeSystem; import org.apache.calcite.rel.type.RelDataTypeSystemImpl; import org.apache.calcite.sql.SqlAbstractDateTimeLiteral; +import org.apache.calcite.sql.SqlAlienSystemTypeNameSpec; import org.apache.calcite.sql.SqlBasicFunction; import org.apache.calcite.sql.SqlCall; +import org.apache.calcite.sql.SqlDataTypeSpec; import org.apache.calcite.sql.SqlDialect; import org.apache.calcite.sql.SqlFunction; import org.apache.calcite.sql.SqlFunctionCategory; @@ -80,6 +82,10 @@ public class MssqlSqlDialect extends SqlDialect { SqlBasicFunction.create("SUBSTRING", ReturnTypes.ARG0_NULLABLE_VARYING, OperandTypes.VARIADIC, SqlFunctionCategory.STRING); + /** Maximum fractional-seconds precision of SQL Server's {@code DATETIME2} + * and {@code DATETIMEOFFSET} types. */ + private static final int MAX_DATETIME_PRECISION = 7; + /** Whether to generate "SELECT TOP(fetch)" rather than * "SELECT ... FETCH NEXT fetch ROWS ONLY". */ private final boolean top; @@ -92,6 +98,39 @@ public MssqlSqlDialect(Context context) { top = context.databaseMajorVersion() < 11; } + @Override public @Nullable SqlNode getCastSpec(RelDataType type) { + switch (type.getSqlTypeName()) { + case TIMESTAMP: + // In SQL Server, TIMESTAMP is a deprecated synonym for ROWVERSION + // (a binary, auto-generated type), not a temporal type. The correct + // fixed-precision date/time type is DATETIME2 (SQL Server 2008+). + return createDatetimeCastSpec("DATETIME2", type); + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + // SQL Server's timezone-aware date/time type. + return createDatetimeCastSpec("DATETIMEOFFSET", type); + default: + return super.getCastSpec(type); + } + } + + /** Builds a SQL Server date/time cast target such as {@code DATETIME2(3)}. + * + *

      SQL Server supports a fractional-seconds precision in the range + * {@code [0, 7]}. An unspecified precision is omitted, letting SQL Server + * apply its own default (7); a higher precision is clamped to 7. A precision + * above 7 is only reachable through a custom type system, since Calcite's + * default caps TIMESTAMP precision at + * {@link org.apache.calcite.sql.type.SqlTypeName#MAX_DATETIME_PRECISION}. */ + private static SqlNode createDatetimeCastSpec(String typeAlias, RelDataType type) { + final int precision = type.getPrecision(); + final String spec = precision < 0 + ? typeAlias + : typeAlias + "(" + Math.min(precision, MAX_DATETIME_PRECISION) + ")"; + return new SqlDataTypeSpec( + new SqlAlienSystemTypeNameSpec(spec, type.getSqlTypeName(), SqlParserPos.ZERO), + SqlParserPos.ZERO); + } + /** {@inheritDoc} * *

      MSSQL does not support NULLS FIRST, so we emulate using CASE diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java index 3327d0f3687f..f056ee9d4fc8 100644 --- a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java +++ b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java @@ -1617,6 +1617,67 @@ private static String toSql(RelNode root, SqlDialect dialect, sql(query).ok(expected); } + /** Test case for + * [CALCITE-7652] + * MssqlSqlDialect unparses CAST to TIMESTAMP as "TIMESTAMP", which is invalid + * in SQL Server (should be DATETIME2). */ + @Test void testCastToTimestampMssql() { + final String query = "select cast(\"hire_date\" as timestamp(3))\n" + + "from \"employee\""; + final String expectedMssql = "SELECT CAST([hire_date] AS DATETIME2(3))\n" + + "FROM [foodmart].[employee]"; + sql(query).withMssql().ok(expectedMssql); + } + + /** Test case for + * [CALCITE-7652] + * MssqlSqlDialect unparses CAST to TIMESTAMP as "TIMESTAMP", which is invalid + * in SQL Server (should be DATETIME2). TIMESTAMP WITH LOCAL TIME ZONE maps + * to DATETIMEOFFSET. */ + @Test void testCastToTimestampWithLocalTimeZoneMssql() { + final String query = "select cast(\"hire_date\" as timestamp(3) with local time zone)\n" + + "from \"employee\""; + final String expectedMssql = "SELECT CAST([hire_date] AS DATETIMEOFFSET(3))\n" + + "FROM [foodmart].[employee]"; + sql(query).withMssql().ok(expectedMssql); + } + + /** Test case for + * [CALCITE-7652] + * MssqlSqlDialect unparses CAST to TIMESTAMP as "TIMESTAMP", which is invalid + * in SQL Server (should be DATETIME2). SQL Server's DATETIME2 and + * DATETIMEOFFSET support a fractional-seconds precision of at most 7, so a + * higher precision (only reachable through a custom type system, since + * Calcite's default caps TIMESTAMP precision at 3) is clamped to 7. */ + @Test void testCastToTimestampMssqlClampsPrecision() { + final RelDataTypeSystem typeSystem = new RelDataTypeSystemImpl() { + @Override public int getMaxPrecision(SqlTypeName typeName) { + switch (typeName) { + case TIMESTAMP: + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + return 9; + default: + return super.getMaxPrecision(typeName); + } + } + }; + final SqlTypeFactoryImpl typeFactory = new SqlTypeFactoryImpl(typeSystem); + final RelDataType timestamp9 = + typeFactory.createSqlType(SqlTypeName.TIMESTAMP, 9); + final SqlNode timestampCast = MssqlSqlDialect.DEFAULT.getCastSpec(timestamp9); + assertThat(timestampCast, notNullValue()); + assertThat(timestampCast.toSqlString(MssqlSqlDialect.DEFAULT).getSql(), + is("DATETIME2(7)")); + + final RelDataType timestampTz9 = + typeFactory.createSqlType(SqlTypeName.TIMESTAMP_WITH_LOCAL_TIME_ZONE, 9); + final SqlNode timestampTzCast = + MssqlSqlDialect.DEFAULT.getCastSpec(timestampTz9); + assertThat(timestampTzCast, notNullValue()); + assertThat(timestampTzCast.toSqlString(MssqlSqlDialect.DEFAULT).getSql(), + is("DATETIMEOFFSET(7)")); + } + /** * Test case for * [CALCITE-4706] From 77b36594ca7271564e45a1626d94317a305f1604 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Mon, 13 Jul 2026 17:37:11 -0700 Subject: [PATCH 386/562] [CALCITE-7653] ARRAY[NULL] produces uncompilable Java code Signed-off-by: Mihai Budiu --- .../enumerable/RexToLixTranslator.java | 9 ++++-- core/src/test/resources/sql/cast.iq | 32 +++++++++++++++++++ 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java index ea4ff65127c3..5a1e0fee2082 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java @@ -401,10 +401,15 @@ private Expression getConvertExpression( final RelDataType targetDataType = targetType.getComponentType(); assert sourceDataType != null; assert targetDataType != null; + final Type sourceComponentClass = typeFactory.getJavaClass(sourceDataType); final ParameterExpression parameter = - Expressions.parameter(typeFactory.getJavaClass(sourceDataType), "root"); + Expressions.parameter(sourceComponentClass, "root"); + // A NULL component type maps to java.lang.Void. Such elements are always null, + // so convert to a null constant instead of the parameter. + final Expression element = + sourceComponentClass == Void.class ? Expressions.constant(null) : parameter; Expression convert = - getConvertExpression(sourceDataType, targetDataType, parameter, format); + getConvertExpression(sourceDataType, targetDataType, element, format); return Expressions.call(BuiltInMethod.LIST_TRANSFORM.method, operand, Expressions.lambda(Function1.class, convert, parameter)); diff --git a/core/src/test/resources/sql/cast.iq b/core/src/test/resources/sql/cast.iq index 9278af60c2e3..a0ef44885409 100644 --- a/core/src/test/resources/sql/cast.iq +++ b/core/src/test/resources/sql/cast.iq @@ -1999,4 +1999,36 @@ select cast(mod(deptno, 3) as BOOLEAN), mod(deptno, 3) from emp; !ok +# Test case for [CALCITE-7653] ARRAY[NULL] produces uncompilable Java code +values (cast(array[null] as integer array)); ++--------+ +| EXPR$0 | ++--------+ +| [null] | ++--------+ +(1 row) + +!ok + +select coalesce(cast(null as integer array), array[null]) as a +from (values (1)); ++--------+ +| A | ++--------+ +| [null] | ++--------+ +(1 row) + +!ok + +values (cast(multiset[null] as integer multiset)); ++--------+ +| EXPR$0 | ++--------+ +| [null] | ++--------+ +(1 row) + +!ok + # End cast.iq From eb3a4c4bcc7df68bf197fd5cf144e720064dd2eb Mon Sep 17 00:00:00 2001 From: Thomas Rebele Date: Fri, 10 Jul 2026 17:09:01 +0200 Subject: [PATCH 387/562] [CALCITE-7650] Bytecode by JDK 11 is invalid due to Checkerframework annotations --- .../calcite/adapter/clone/ArrayTable.java | 15 +- .../calcite/adapter/enumerable/EnumUtils.java | 60 ++++- .../enumerable/EnumerableInterpretable.java | 62 +++-- .../calcite/interpreter/Interpreter.java | 18 +- .../apache/calcite/runtime/SqlFunctions.java | 20 +- .../schema/impl/ListTransientTable.java | 81 +++--- .../calcite/sql2rel/SqlToRelConverter.java | 61 +++-- .../java/org/apache/calcite/util/Holder.java | 2 +- .../apache/calcite/util/mapping/Mappings.java | 27 +- .../adapter/csv/CsvFilterableTable.java | 28 +- .../adapter/csv/CsvScannableTable.java | 26 +- .../adapter/csv/CsvStreamScannableTable.java | 26 +- .../calcite/example/maze/MazeTable.java | 31 ++- .../adapter/file/JsonScannableTable.java | 21 +- .../simple/GeodeSimpleScannableTable.java | 35 ++- .../adapter/kafka/KafkaStreamTable.java | 57 ++-- .../calcite/linq4j/EnumerableDefaults.java | 49 ++-- .../adapter/os/FilesTableFunction.java | 243 ++++++++++-------- .../adapter/os/GitCommitsTableFunction.java | 179 +++++++------ .../calcite/adapter/tpcds/TpcdsSchema.java | 67 ++--- 20 files changed, 680 insertions(+), 428 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/adapter/clone/ArrayTable.java b/core/src/main/java/org/apache/calcite/adapter/clone/ArrayTable.java index 8400ac21d621..fabbe6ea8d71 100644 --- a/core/src/main/java/org/apache/calcite/adapter/clone/ArrayTable.java +++ b/core/src/main/java/org/apache/calcite/adapter/clone/ArrayTable.java @@ -87,12 +87,15 @@ class ArrayTable extends AbstractQueryableTable implements ScannableTable { } @Override public Enumerable<@Nullable Object[]> scan(DataContext root) { - return new AbstractEnumerable<@Nullable Object[]>() { - @Override public Enumerator<@Nullable Object[]> enumerator() { - final Content content = supplier.get(); - return content.arrayEnumerator(); - } - }; + return new ArrayTableEnumerable(); + } + + /** Enumerable for {@link ArrayTable}. */ + private class ArrayTableEnumerable extends AbstractEnumerable<@Nullable Object[]> { + @Override public Enumerator<@Nullable Object[]> enumerator() { + final Content content = supplier.get(); + return content.arrayEnumerator(); + } } @Override public Queryable asQueryable(final QueryProvider queryProvider, diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java index 129180e584c5..80c46ec3f9ea 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java @@ -1086,12 +1086,29 @@ static Expression tumblingWindowSelector( public static Enumerable<@Nullable Object[]> sessionize( Enumerator<@Nullable Object[]> inputEnumerator, int indexOfWatermarkedColumn, int indexOfKeyColumn, long gap) { - return new AbstractEnumerable<@Nullable Object[]>() { - @Override public Enumerator<@Nullable Object[]> enumerator() { - return new SessionizationEnumerator(inputEnumerator, - indexOfWatermarkedColumn, indexOfKeyColumn, gap); - } - }; + return new SessionizeEnumerable(inputEnumerator, indexOfWatermarkedColumn, indexOfKeyColumn, + gap); + } + + /** Enumerable for {@link #sessionize(Enumerator, int, int, long)}. */ + private static class SessionizeEnumerable extends AbstractEnumerable<@Nullable Object[]> { + private final Enumerator<@Nullable Object[]> inputEnumerator; + private final int indexOfWatermarkedColumn; + private final int indexOfKeyColumn; + private final long gap; + + SessionizeEnumerable(Enumerator<@Nullable Object[]> inputEnumerator, + int indexOfWatermarkedColumn, int indexOfKeyColumn, long gap) { + this.inputEnumerator = inputEnumerator; + this.indexOfWatermarkedColumn = indexOfWatermarkedColumn; + this.indexOfKeyColumn = indexOfKeyColumn; + this.gap = gap; + } + + @Override public Enumerator<@Nullable Object[]> enumerator() { + return new SessionizationEnumerator(inputEnumerator, + indexOfWatermarkedColumn, indexOfKeyColumn, gap); + } } /** Enumerator that converts rows into sessions separated by gaps. */ @@ -1234,12 +1251,31 @@ private static Pair computeInitWindow(long ts, long gap) { public static Enumerable<@Nullable Object[]> hopping( Enumerator<@Nullable Object[]> inputEnumerator, int indexOfWatermarkedColumn, long emitFrequency, long windowSize, long offset) { - return new AbstractEnumerable<@Nullable Object[]>() { - @Override public Enumerator<@Nullable Object[]> enumerator() { - return new HopEnumerator(inputEnumerator, - indexOfWatermarkedColumn, emitFrequency, windowSize, offset); - } - }; + return new HoppingEnumerable(inputEnumerator, indexOfWatermarkedColumn, emitFrequency, + windowSize, offset); + } + + /** Enumerable for {@link #hopping(Enumerator, int, long, long, long)}. */ + private static class HoppingEnumerable extends AbstractEnumerable<@Nullable Object[]> { + private final Enumerator<@Nullable Object[]> inputEnumerator; + private final int indexOfWatermarkedColumn; + private final long emitFrequency; + private final long windowSize; + private final long offset; + + HoppingEnumerable(Enumerator<@Nullable Object[]> inputEnumerator, + int indexOfWatermarkedColumn, long emitFrequency, long windowSize, long offset) { + this.inputEnumerator = inputEnumerator; + this.indexOfWatermarkedColumn = indexOfWatermarkedColumn; + this.emitFrequency = emitFrequency; + this.windowSize = windowSize; + this.offset = offset; + } + + @Override public Enumerator<@Nullable Object[]> enumerator() { + return new HopEnumerator(inputEnumerator, + indexOfWatermarkedColumn, emitFrequency, windowSize, offset); + } } /** Enumerator that computes HOP. */ diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableInterpretable.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableInterpretable.java index 5f32ab1c7102..7ed858b754b5 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableInterpretable.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableInterpretable.java @@ -205,32 +205,50 @@ static ArrayBindable box(final Bindable bindable) { @Override public Enumerable<@Nullable Object[]> bind(DataContext dataContext) { final Enumerable enumerable = bindable.bind(dataContext); - return new AbstractEnumerable<@Nullable Object[]>() { - @Override public Enumerator<@Nullable Object[]> enumerator() { - final Enumerator enumerator = enumerable.enumerator(); - return new Enumerator<@Nullable Object[]>() { - @Override public @Nullable Object[] current() { - return new Object[] {enumerator.current()}; - } - - @Override public boolean moveNext() { - return enumerator.moveNext(); - } - - @Override public void reset() { - enumerator.reset(); - } - - @Override public void close() { - enumerator.close(); - } - }; - } - }; + return new BoxEnumerable(enumerable); } }; } + /** Enumerable for {@link #box(Bindable)}. */ + private static class BoxEnumerable extends AbstractEnumerable<@Nullable Object[]> { + private final Enumerable enumerable; + + BoxEnumerable(Enumerable enumerable) { + this.enumerable = enumerable; + } + + @Override public Enumerator<@Nullable Object[]> enumerator() { + final Enumerator enumerator = enumerable.enumerator(); + return new BoxEnumerator(enumerator); + } + } + + /** Enumerator for {@link #box(Bindable)}. */ + private static class BoxEnumerator implements Enumerator<@Nullable Object[]> { + private final Enumerator enumerator; + + BoxEnumerator(Enumerator enumerator) { + this.enumerator = enumerator; + } + + @Override public @Nullable Object[] current() { + return new Object[] { enumerator.current()}; + } + + @Override public boolean moveNext() { + return enumerator.moveNext(); + } + + @Override public void reset() { + enumerator.reset(); + } + + @Override public void close() { + enumerator.close(); + } + } + /** Interpreter node that reads from an {@link Enumerable}. * *

      From the interpreter's perspective, it is a leaf node. */ diff --git a/core/src/main/java/org/apache/calcite/interpreter/Interpreter.java b/core/src/main/java/org/apache/calcite/interpreter/Interpreter.java index 6bd8bde4c1fa..8bc82646a7ff 100644 --- a/core/src/main/java/org/apache/calcite/interpreter/Interpreter.java +++ b/core/src/main/java/org/apache/calcite/interpreter/Interpreter.java @@ -118,11 +118,19 @@ private static RelNode optimize(RelNode rootRel) { rows = Linq4j.iterableEnumerator(queue); } - return new TransformedEnumerator(rows) { - @Override protected @Nullable Object[] transform(Row row) { - return row.getValues(); - } - }; + return new InterpreterEnumerator(rows); + } + + /** Enumerator for {@link Interpreter}. */ + private static class InterpreterEnumerator + extends TransformedEnumerator { + InterpreterEnumerator(Enumerator rows) { + super(rows); + } + + @Override protected @Nullable Object[] transform(Row row) { + return row.getValues(); + } } @SuppressWarnings("CatchAndPrintStackTrace") diff --git a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java index 5dfad1da34cf..ed6a3ba16f09 100644 --- a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java +++ b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java @@ -222,11 +222,21 @@ public class SqlFunctions { enumerators.add(Linq4j.enumerator((List) list)); } final Enumerator> product = Linq4j.product(enumerators); - return new AbstractEnumerable<@Nullable Object[]>() { - @Override public Enumerator<@Nullable Object[]> enumerator() { - return Linq4j.transform(product, List::toArray); - } - }; + return new ArrayCartesianProductEnumerable(product); + } + + /** Enumerable for {@link #arrayCartesianProduct(Object[])}. */ + private static class ArrayCartesianProductEnumerable + extends AbstractEnumerable<@Nullable Object[]> { + private final Enumerator> product; + + ArrayCartesianProductEnumerable(Enumerator> product) { + this.product = product; + } + + @Override public Enumerator<@Nullable Object[]> enumerator() { + return Linq4j.transform(product, List::toArray); + } } /** Holds, for each thread, a map from sequence name to sequence current diff --git a/core/src/main/java/org/apache/calcite/schema/impl/ListTransientTable.java b/core/src/main/java/org/apache/calcite/schema/impl/ListTransientTable.java index a80556ceef28..f04c9372b7a0 100644 --- a/core/src/main/java/org/apache/calcite/schema/impl/ListTransientTable.java +++ b/core/src/main/java/org/apache/calcite/schema/impl/ListTransientTable.java @@ -93,38 +93,57 @@ public ListTransientTable(String name, RelDataType rowType) { final AtomicBoolean cancelFlag = DataContext.Variable.CANCEL_FLAG.get(root); - return new AbstractEnumerable<@Nullable Object[]>() { - @Override public Enumerator<@Nullable Object[]> enumerator() { - return new Enumerator<@Nullable Object[]>() { - @SuppressWarnings({"rawtypes", "unchecked"}) - private final List list = new ArrayList(rows); - private int i = -1; - - // TODO cleaner way to handle non-array objects? - @Override public Object[] current() { - Object current = list.get(i); - return current != null && current.getClass().isArray() - ? (Object[]) current - : new Object[]{current}; - } - - @Override public boolean moveNext() { - if (cancelFlag != null && cancelFlag.get()) { - return false; - } - - return ++i < list.size(); - } - - @Override public void reset() { - i = -1; - } - - @Override public void close() { - } - }; + return new ListTransientTableEnumerable(cancelFlag); + } + + /** Enumerable for {@link ListTransientTable}. */ + private class ListTransientTableEnumerable extends AbstractEnumerable<@Nullable Object[]> { + private final @Nullable AtomicBoolean cancelFlag; + + ListTransientTableEnumerable(@Nullable AtomicBoolean cancelFlag) { + this.cancelFlag = cancelFlag; + } + + @Override public Enumerator<@Nullable Object[]> enumerator() { + return new ListTransientTableEnumerator(cancelFlag); + } + } + + /** Enumerator for {@link ListTransientTable}. */ + private class ListTransientTableEnumerator implements Enumerator<@Nullable Object[]> { + @SuppressWarnings({"rawtypes", "unchecked"}) + private final List list; + private final @Nullable AtomicBoolean cancelFlag; + private int i; + + ListTransientTableEnumerator(@Nullable AtomicBoolean cancelFlag) { + this.cancelFlag = cancelFlag; + list = new ArrayList(rows); + i = -1; + } + + // TODO cleaner way to handle non-array objects? + @Override public Object[] current() { + Object current = list.get(i); + return current != null && current.getClass().isArray() + ? (Object[]) current + : new Object[]{current}; + } + + @Override public boolean moveNext() { + if (cancelFlag != null && cancelFlag.get()) { + return false; } - }; + + return ++i < list.size(); + } + + @Override public void reset() { + i = -1; + } + + @Override public void close() { + } } @Override public Expression getExpression(SchemaPlus schema, String tableName, diff --git a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java index f8b661c53fb4..182ed7a254a3 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java @@ -2967,32 +2967,7 @@ protected void convertMatchRecognize(Blackboard bb, final Set patternVarsSet = new HashSet<>(); SqlNode pattern = matchRecognize.getPattern(); final SqlBasicVisitor<@Nullable RexNode> patternVarVisitor = - new SqlBasicVisitor<@Nullable RexNode>() { - @Override public RexNode visit(SqlCall call) { - List operands = call.getOperandList(); - List newOperands = new ArrayList<>(); - for (SqlNode node : operands) { - RexNode arg = requireNonNull(node.accept(this), node::toString); - newOperands.add(arg); - } - return rexBuilder.makeCall(call.getParserPosition(), - validator().getUnknownType(), call.getOperator(), newOperands); - } - - @Override public RexNode visit(SqlIdentifier id) { - assert id.isSimple(); - patternVarsSet.add(id.getSimple()); - return rexBuilder.makeLiteral(id.getSimple()); - } - - @Override public RexNode visit(SqlLiteral literal) { - if (literal instanceof SqlNumericLiteral) { - return rexBuilder.makeExactLiteral(BigDecimal.valueOf(literal.intValue(true))); - } else { - return rexBuilder.makeLiteral(literal.booleanValue()); - } - } - }; + new PatternVarVisitor(patternVarsSet); final RexNode patternNode = pattern.accept(patternVarVisitor); if (patternNode == null) { throw new AssertionError("pattern is not found in " + pattern); @@ -7124,4 +7099,38 @@ private class MeasureBlackboard extends Blackboard { return super.lookupMeasure(identifier); } } + + /** Visitor for {@link #convertMatchRecognize(Blackboard, SqlMatchRecognize)}. */ + private class PatternVarVisitor extends SqlBasicVisitor<@Nullable RexNode> { + private final Set patternVarsSet; + + PatternVarVisitor(Set patternVarsSet) { + this.patternVarsSet = patternVarsSet; + } + + @Override public RexNode visit(SqlCall call) { + List operands = call.getOperandList(); + List newOperands = new ArrayList<>(); + for (SqlNode node : operands) { + RexNode arg = requireNonNull(node.accept(this), node::toString); + newOperands.add(arg); + } + return rexBuilder.makeCall(call.getParserPosition(), + validator().getUnknownType(), call.getOperator(), newOperands); + } + + @Override public RexNode visit(SqlIdentifier id) { + assert id.isSimple(); + patternVarsSet.add(id.getSimple()); + return rexBuilder.makeLiteral(id.getSimple()); + } + + @Override public RexNode visit(SqlLiteral literal) { + if (literal instanceof SqlNumericLiteral) { + return rexBuilder.makeExactLiteral(BigDecimal.valueOf(literal.intValue(true))); + } else { + return rexBuilder.makeLiteral(literal.booleanValue()); + } + } + } } diff --git a/core/src/main/java/org/apache/calcite/util/Holder.java b/core/src/main/java/org/apache/calcite/util/Holder.java index bc5567e8c07f..78aedcd3eebd 100644 --- a/core/src/main/java/org/apache/calcite/util/Holder.java +++ b/core/src/main/java/org/apache/calcite/util/Holder.java @@ -64,6 +64,6 @@ public static Holder of(E e) { /** Creates a holder containing null. */ @SuppressWarnings("ConstantConditions") public static Holder<@Nullable E> empty() { - return new Holder<@Nullable E>(null); + return new Holder<>(null); } } diff --git a/core/src/main/java/org/apache/calcite/util/mapping/Mappings.java b/core/src/main/java/org/apache/calcite/util/mapping/Mappings.java index 82a3c7403954..0139473bc50d 100644 --- a/core/src/main/java/org/apache/calcite/util/mapping/Mappings.java +++ b/core/src/main/java/org/apache/calcite/util/mapping/Mappings.java @@ -319,16 +319,25 @@ public static List permute(final List list, */ @CheckReturnValue public static List<@Nullable Integer> asList(final TargetMapping mapping) { - return new AbstractList<@Nullable Integer>() { - @Override public @Nullable Integer get(int source) { - int target = mapping.getTargetOpt(source); - return target < 0 ? null : target; - } + return new MappingsList(mapping); + } - @Override public int size() { - return mapping.getSourceCount(); - } - }; + /** List view of a {@link TargetMapping}. */ + private static class MappingsList extends AbstractList<@Nullable Integer> { + private final TargetMapping mapping; + + MappingsList(TargetMapping mapping) { + this.mapping = mapping; + } + + @Override public @Nullable Integer get(int source) { + int target = mapping.getTargetOpt(source); + return target < 0 ? null : target; + } + + @Override public int size() { + return mapping.getSourceCount(); + } } /** diff --git a/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvFilterableTable.java b/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvFilterableTable.java index ee621740f9b0..212aff0ce5d1 100644 --- a/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvFilterableTable.java +++ b/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvFilterableTable.java @@ -63,12 +63,28 @@ public CsvFilterableTable(Source source, filters.removeIf(filter -> addFilter(filter, filterValues)); final List fields = ImmutableIntList.identity(fieldTypes.size()); final AtomicBoolean cancelFlag = DataContext.Variable.CANCEL_FLAG.get(root); - return new AbstractEnumerable<@Nullable Object[]>() { - @Override public Enumerator<@Nullable Object[]> enumerator() { - return new CsvEnumerator<>(source, cancelFlag, false, filterValues, - CsvEnumerator.arrayConverter(fieldTypes, fields, false), ','); - } - }; + return new CsvFilterableTableEnumerable(cancelFlag, filterValues, fieldTypes, fields); + } + + /** Enumerable for {@link CsvFilterableTable}. */ + private class CsvFilterableTableEnumerable extends AbstractEnumerable<@Nullable Object[]> { + private final AtomicBoolean cancelFlag; + private final @Nullable String[] filterValues; + private final List fieldTypes; + private final List fields; + + CsvFilterableTableEnumerable(AtomicBoolean cancelFlag, @Nullable String[] filterValues, + List fieldTypes, List fields) { + this.cancelFlag = cancelFlag; + this.filterValues = filterValues; + this.fieldTypes = fieldTypes; + this.fields = fields; + } + + @Override public Enumerator<@Nullable Object[]> enumerator() { + return new CsvEnumerator<>(source, cancelFlag, false, filterValues, + CsvEnumerator.arrayConverter(fieldTypes, fields, false), ','); + } } private static boolean addFilter(RexNode filter, @Nullable Object[] filterValues) { diff --git a/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvScannableTable.java b/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvScannableTable.java index 836af81373b7..046825bd35db 100644 --- a/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvScannableTable.java +++ b/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvScannableTable.java @@ -55,11 +55,25 @@ public class CsvScannableTable extends CsvTable final List fieldTypes = getFieldTypes(typeFactory); final List fields = ImmutableIntList.identity(fieldTypes.size()); final AtomicBoolean cancelFlag = DataContext.Variable.CANCEL_FLAG.get(root); - return new AbstractEnumerable<@Nullable Object[]>() { - @Override public Enumerator<@Nullable Object[]> enumerator() { - return new CsvEnumerator<>(source, cancelFlag, false, null, - CsvEnumerator.arrayConverter(fieldTypes, fields, false), ','); - } - }; + return new CsvScannableTableEnumerable(cancelFlag, fieldTypes, fields); + } + + /** Enumerable for {@link CsvScannableTable}. */ + private class CsvScannableTableEnumerable extends AbstractEnumerable<@Nullable Object[]> { + private final AtomicBoolean cancelFlag; + private final List fieldTypes; + private final List fields; + + CsvScannableTableEnumerable(AtomicBoolean cancelFlag, List fieldTypes, + List fields) { + this.cancelFlag = cancelFlag; + this.fieldTypes = fieldTypes; + this.fields = fields; + } + + @Override public Enumerator<@Nullable Object[]> enumerator() { + return new CsvEnumerator<>(source, cancelFlag, false, null, + CsvEnumerator.arrayConverter(fieldTypes, fields, false), ','); + } } } diff --git a/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvStreamScannableTable.java b/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvStreamScannableTable.java index 7c0d574cc7af..98ab6a03bcbf 100644 --- a/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvStreamScannableTable.java +++ b/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvStreamScannableTable.java @@ -62,12 +62,26 @@ public class CsvStreamScannableTable extends CsvScannableTable final List fieldTypes = getFieldTypes(typeFactory); final List fields = ImmutableIntList.identity(fieldTypes.size()); final AtomicBoolean cancelFlag = DataContext.Variable.CANCEL_FLAG.get(root); - return new AbstractEnumerable<@Nullable Object[]>() { - @Override public Enumerator<@Nullable Object[]> enumerator() { - return new CsvEnumerator<>(source, cancelFlag, true, null, - CsvEnumerator.arrayConverter(fieldTypes, fields, true), ','); - } - }; + return new CsvStreamScannableTableEnumerable(cancelFlag, fieldTypes, fields); + } + + /** Enumerable for {@link CsvStreamScannableTable}. */ + private class CsvStreamScannableTableEnumerable extends AbstractEnumerable<@Nullable Object[]> { + private final AtomicBoolean cancelFlag; + private final List fieldTypes; + private final List fields; + + CsvStreamScannableTableEnumerable(AtomicBoolean cancelFlag, List fieldTypes, + List fields) { + this.cancelFlag = cancelFlag; + this.fieldTypes = fieldTypes; + this.fields = fields; + } + + @Override public Enumerator<@Nullable Object[]> enumerator() { + return new CsvEnumerator<>(source, cancelFlag, true, null, + CsvEnumerator.arrayConverter(fieldTypes, fields, true), ','); + } } @Override public Table stream() { diff --git a/example/function/src/main/java/org/apache/calcite/example/maze/MazeTable.java b/example/function/src/main/java/org/apache/calcite/example/maze/MazeTable.java index 0818cc0a74ea..1068bf25cbbf 100644 --- a/example/function/src/main/java/org/apache/calcite/example/maze/MazeTable.java +++ b/example/function/src/main/java/org/apache/calcite/example/maze/MazeTable.java @@ -95,17 +95,26 @@ public static ScannableTable solve(int width, int height, int seed) { if (Maze.DEBUG) { maze.print(pw, true); } - return new AbstractEnumerable<@Nullable Object[]>() { - @Override public Enumerator<@Nullable Object[]> enumerator() { - final Set solutionSet; - if (solution) { - solutionSet = maze.solve(0, 0); - } else { - solutionSet = null; - } - return Linq4j.transform(maze.enumerator(solutionSet), - s -> new Object[] {s}); + return new MazeTableEnumerable(maze); + } + + /** Enumerable for {@link MazeTable}. */ + private class MazeTableEnumerable extends AbstractEnumerable<@Nullable Object[]> { + private final Maze maze; + + MazeTableEnumerable(Maze maze) { + this.maze = maze; + } + + @Override public Enumerator<@Nullable Object[]> enumerator() { + final Set solutionSet; + if (solution) { + solutionSet = maze.solve(0, 0); + } else { + solutionSet = null; } - }; + return Linq4j.transform(maze.enumerator(solutionSet), + s -> new Object[] {s}); + } } } diff --git a/file/src/main/java/org/apache/calcite/adapter/file/JsonScannableTable.java b/file/src/main/java/org/apache/calcite/adapter/file/JsonScannableTable.java index 8f9d2bde292f..f0f23739635d 100644 --- a/file/src/main/java/org/apache/calcite/adapter/file/JsonScannableTable.java +++ b/file/src/main/java/org/apache/calcite/adapter/file/JsonScannableTable.java @@ -46,11 +46,20 @@ public JsonScannableTable(Source source) { } @Override public Enumerable<@Nullable Object[]> scan(DataContext root) { - return new AbstractEnumerable<@Nullable Object[]>() { - @Override public Enumerator<@Nullable Object[]> enumerator() { - JavaTypeFactory typeFactory = root.getTypeFactory(); - return new JsonEnumerator(getDataList(typeFactory)); - } - }; + return new JsonScannableTableEnumerable(root); + } + + /** Enumerable for {@link JsonScannableTable}. */ + private class JsonScannableTableEnumerable extends AbstractEnumerable<@Nullable Object[]> { + private final DataContext root; + + JsonScannableTableEnumerable(DataContext root) { + this.root = root; + } + + @Override public Enumerator<@Nullable Object[]> enumerator() { + JavaTypeFactory typeFactory = root.getTypeFactory(); + return new JsonEnumerator(getDataList(typeFactory)); + } } } diff --git a/geode/src/main/java/org/apache/calcite/adapter/geode/simple/GeodeSimpleScannableTable.java b/geode/src/main/java/org/apache/calcite/adapter/geode/simple/GeodeSimpleScannableTable.java index f48698c7c6c1..1594b30e27d3 100644 --- a/geode/src/main/java/org/apache/calcite/adapter/geode/simple/GeodeSimpleScannableTable.java +++ b/geode/src/main/java/org/apache/calcite/adapter/geode/simple/GeodeSimpleScannableTable.java @@ -58,18 +58,29 @@ public GeodeSimpleScannableTable(String regionName, RelDataType relDataType, } @Override public Enumerable<@Nullable Object[]> scan(DataContext root) { - return new AbstractEnumerable<@Nullable Object[]>() { - @Override public Enumerator<@Nullable Object[]> enumerator() { - return new GeodeSimpleEnumerator<@Nullable Object[]>(clientCache, regionName) { - @Override public @Nullable Object[] convert(Object obj) { - Object values = convertToRowValues(relDataType.getFieldList(), obj); - if (values instanceof Object[]) { - return (Object[]) values; - } - return new Object[]{values}; - } - }; + return new GeodeSimpleScannableTableEnumerable(); + } + + /** Enumerable for {@link GeodeSimpleScannableTable}. */ + private class GeodeSimpleScannableTableEnumerable extends AbstractEnumerable<@Nullable Object[]> { + @Override public Enumerator<@Nullable Object[]> enumerator() { + return new GeodeSimpleScannableTableEnumerator(); + } + } + + /** Enumerator for {@link GeodeSimpleScannableTable}. */ + private class GeodeSimpleScannableTableEnumerator + extends GeodeSimpleEnumerator<@Nullable Object[]> { + GeodeSimpleScannableTableEnumerator() { + super(GeodeSimpleScannableTable.this.clientCache, GeodeSimpleScannableTable.this.regionName); + } + + @Override public @Nullable Object[] convert(Object obj) { + Object values = convertToRowValues(relDataType.getFieldList(), obj); + if (values instanceof Object[]) { + return (Object[]) values; } - }; + return new Object[]{values}; + } } } diff --git a/kafka/src/main/java/org/apache/calcite/adapter/kafka/KafkaStreamTable.java b/kafka/src/main/java/org/apache/calcite/adapter/kafka/KafkaStreamTable.java index e432eb803940..f73941c9418a 100644 --- a/kafka/src/main/java/org/apache/calcite/adapter/kafka/KafkaStreamTable.java +++ b/kafka/src/main/java/org/apache/calcite/adapter/kafka/KafkaStreamTable.java @@ -60,31 +60,40 @@ public class KafkaStreamTable implements ScannableTable, StreamableTable { @Override public Enumerable<@Nullable Object[]> scan(final DataContext root) { final AtomicBoolean cancelFlag = DataContext.Variable.CANCEL_FLAG.get(root); - return new AbstractEnumerable<@Nullable Object[]>() { - @Override public Enumerator<@Nullable Object[]> enumerator() { - if (tableOptions.getConsumer() != null) { - return new KafkaMessageEnumerator(tableOptions.getConsumer(), - tableOptions.getRowConverter(), cancelFlag); - } - - Properties consumerConfig = new Properties(); - consumerConfig.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, - tableOptions.getBootstrapServers()); - // by default it's - consumerConfig.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, - "org.apache.kafka.common.serialization.ByteArrayDeserializer"); - consumerConfig.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, - "org.apache.kafka.common.serialization.ByteArrayDeserializer"); - - if (tableOptions.getConsumerParams() != null) { - consumerConfig.putAll(tableOptions.getConsumerParams()); - } - Consumer consumer = new KafkaConsumer<>(consumerConfig); - consumer.subscribe(Collections.singletonList(tableOptions.getTopicName())); - - return new KafkaMessageEnumerator(consumer, tableOptions.getRowConverter(), cancelFlag); + return new KafkaStreamTableEnumerable(cancelFlag); + } + + /** Enumerable for {@link KafkaStreamTable}. */ + private class KafkaStreamTableEnumerable extends AbstractEnumerable<@Nullable Object[]> { + private final AtomicBoolean cancelFlag; + + KafkaStreamTableEnumerable(AtomicBoolean cancelFlag) { + this.cancelFlag = cancelFlag; + } + + @Override public Enumerator<@Nullable Object[]> enumerator() { + if (tableOptions.getConsumer() != null) { + return new KafkaMessageEnumerator(tableOptions.getConsumer(), + tableOptions.getRowConverter(), cancelFlag); } - }; + + Properties consumerConfig = new Properties(); + consumerConfig.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, + tableOptions.getBootstrapServers()); + // by default it's + consumerConfig.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, + "org.apache.kafka.common.serialization.ByteArrayDeserializer"); + consumerConfig.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, + "org.apache.kafka.common.serialization.ByteArrayDeserializer"); + + if (tableOptions.getConsumerParams() != null) { + consumerConfig.putAll(tableOptions.getConsumerParams()); + } + Consumer consumer = new KafkaConsumer<>(consumerConfig); + consumer.subscribe(Collections.singletonList(tableOptions.getTopicName())); + + return new KafkaMessageEnumerator(consumer, tableOptions.getRowConverter(), cancelFlag); + } } @Override public RelDataType getRowType(final RelDataTypeFactory typeFactory) { diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java b/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java index 0ff4afddb6dd..0eca4b6d5283 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java @@ -4879,32 +4879,35 @@ protected WrapMap(Function0, V>> mapProvider, EqualityComparer } @Override public Set> entrySet() { - return new AbstractSet>() { - @SuppressWarnings("override.return.invalid") - @Override public Iterator> iterator() { - final Iterator, V>> iterator = - map.entrySet().iterator(); - - return new Iterator>() { - @Override public boolean hasNext() { - return iterator.hasNext(); - } + return new WrapMapEntrySet(); + } - @Override public Entry next() { - Entry, V> next = iterator.next(); - return new SimpleEntry<>(next.getKey().element, next.getValue()); - } + /** EntrySet for {@link WrapMap}. */ + private class WrapMapEntrySet extends AbstractSet> { + @SuppressWarnings("override.return.invalid") + @Override public Iterator> iterator() { + final Iterator, V>> iterator = + map.entrySet().iterator(); - @Override public void remove() { - iterator.remove(); - } - }; - } + return new Iterator>() { + @Override public boolean hasNext() { + return iterator.hasNext(); + } - @Override public int size() { - return map.size(); - } - }; + @Override public Entry next() { + Entry, V> next = iterator.next(); + return new SimpleEntry<>(next.getKey().element, next.getValue()); + } + + @Override public void remove() { + iterator.remove(); + } + }; + } + + @Override public int size() { + return map.size(); + } } @SuppressWarnings("contracts.conditional.postcondition.not.satisfied") diff --git a/plus/src/main/java/org/apache/calcite/adapter/os/FilesTableFunction.java b/plus/src/main/java/org/apache/calcite/adapter/os/FilesTableFunction.java index 3df9498c3532..cd7ca9e8b8d1 100644 --- a/plus/src/main/java/org/apache/calcite/adapter/os/FilesTableFunction.java +++ b/plus/src/main/java/org/apache/calcite/adapter/os/FilesTableFunction.java @@ -217,121 +217,148 @@ private boolean isGnuStat() { default: enumerable = sourceLinux(); } - return new AbstractEnumerable<@Nullable Object[]>() { - @Override public Enumerator<@Nullable Object[]> enumerator() { - final Enumerator e = enumerable.enumerator(); - return new Enumerator<@Nullable Object[]>() { - @Nullable Object @Nullable [] current; + return new FilesTableFunctionEnumerable(enumerable, fieldNames, osName); + } + }; + } - @Override public Object[] current() { - return requireNonNull(current, "current"); - } + /** Enumerable for {@link FilesTableFunction}. */ + private static class FilesTableFunctionEnumerable extends AbstractEnumerable<@Nullable Object[]> { + private final Enumerable enumerable; + private final List fieldNames; + private final String osName; - @Override public boolean moveNext() { - current = new Object[fieldNames.size()]; - for (int i = 0; i < current.length; i++) { - if (!e.moveNext()) { - return false; - } - final String v = e.current(); - try { - current[i] = field(fieldNames.get(i), v); - } catch (RuntimeException e) { - throw new RuntimeException("while parsing value [" - + v + "] of field [" + fieldNames.get(i) - + "] in line [" + Arrays.toString(current) + "]", e); - } - } - switch (osName) { - case "Mac OS X": - // post-process fields: compute filename, dir_name, depth from path - String path = requireNonNull((String) current[14]); - if (".".equals(path)) { - current[14] = path = ""; - current[3] = 0; // depth - } else if (path.startsWith("./")) { - current[14] = path = path.substring(2); - current[3] = count(path, '/') + 1; // depth - } else { - current[3] = count(path, '/'); // depth - } - final int slash = path.lastIndexOf('/'); - if (slash >= 0) { - current[5] = path.substring(slash + 1); // filename - current[9] = path.substring(0, slash); // dir_name - } else { - current[5] = path; // filename - current[9] = ""; // dir_name - } + FilesTableFunctionEnumerable(Enumerable enumerable, List fieldNames, + String osName) { + this.enumerable = enumerable; + this.fieldNames = fieldNames; + this.osName = osName; + } - // detect output format: BSD outputs single chars, GNU outputs words - final String type = requireNonNull((String) current[19]); - if (type.length() > 1) { - // GNU stat outputs descriptive types like "regular file", "directory" - current[19] = type.contains("directory") ? "d" - : type.contains("regular") ? "f" - : type.contains("symbolic") ? "l" - : type.contains("block") ? "b" - : type.contains("character") ? "c" - : type.contains("fifo") ? "p" - : type.contains("socket") ? "s" - : "?"; - } else { - // BSD stat outputs single characters: "/" "*" "@" or "" - current[19] = "/".equals(type) ? "d" - : "".equals(type) || "*".equals(type) ? "f" - : "@".equals(type) ? "l" - : type; - } - break; - default: - break; - } - return true; - } + @Override public Enumerator<@Nullable Object[]> enumerator() { + final Enumerator e = enumerable.enumerator(); + return new FilesTableFunctionEnumerator(fieldNames, e, osName); + } + } - private int count(String s, char c) { - int n = 0; - for (int i = 0, len = s.length(); i < len; i++) { - if (s.charAt(i) == c) { - ++n; - } - } - return n; - } + /** Enumerator for {@link FilesTableFunction}. */ + private static class FilesTableFunctionEnumerator implements Enumerator<@Nullable Object[]> { + private final List fieldNames; + private final Enumerator e; + private final String osName; + @Nullable Object @Nullable [] current; - @Override public void reset() { - throw new UnsupportedOperationException(); - } + FilesTableFunctionEnumerator(List fieldNames, Enumerator e, + String osName) { + this.fieldNames = fieldNames; + this.e = e; + this.osName = osName; + } - @Override public void close() { - e.close(); - } + @Override public Object[] current() { + return requireNonNull(current, "current"); + } - private Object field(String field, String value) { - switch (field) { - case "block_count": - case "depth": - case "device": - case "gid": - case "uid": - case "hard": - return Integer.valueOf(value); - case "inode": - case "size": - return Long.valueOf(value); - case "access_time": - case "change_time": - case "mod_time": - return new BigDecimal(value).multiply(THOUSAND).longValue(); - default: - return value; - } - } - }; - } - }; + @Override public boolean moveNext() { + current = new Object[fieldNames.size()]; + for (int i = 0; i < current.length; i++) { + if (!e.moveNext()) { + return false; + } + final String v = e.current(); + try { + current[i] = field(fieldNames.get(i), v); + } catch (RuntimeException e) { + throw new RuntimeException("while parsing value [" + + v + "] of field [" + fieldNames.get(i) + + "] in line [" + Arrays.toString(current) + "]", e); + } } - }; + switch (osName) { + case "Mac OS X": + // post-process fields: compute filename, dir_name, depth from path + String path = requireNonNull((String) current[14]); + if (".".equals(path)) { + current[14] = path = ""; + current[3] = 0; // depth + } else if (path.startsWith("./")) { + current[14] = path = path.substring(2); + current[3] = count(path, '/') + 1; // depth + } else { + current[3] = count(path, '/'); // depth + } + final int slash = path.lastIndexOf('/'); + if (slash >= 0) { + current[5] = path.substring(slash + 1); // filename + current[9] = path.substring(0, slash); // dir_name + } else { + current[5] = path; // filename + current[9] = ""; // dir_name + } + + // detect output format: BSD outputs single chars, GNU outputs words + final String type = requireNonNull((String) current[19]); + if (type.length() > 1) { + // GNU stat outputs descriptive types like "regular file", "directory" + current[19] = type.contains("directory") ? "d" + : type.contains("regular") ? "f" + : type.contains("symbolic") ? "l" + : type.contains("block") ? "b" + : type.contains("character") ? "c" + : type.contains("fifo") ? "p" + : type.contains("socket") ? "s" + : "?"; + } else { + // BSD stat outputs single characters: "/" "*" "@" or "" + current[19] = "/".equals(type) ? "d" + : "".equals(type) || "*".equals(type) ? "f" + : "@".equals(type) ? "l" + : type; + } + break; + default: + break; + } + return true; + } + + private int count(String s, char c) { + int n = 0; + for (int i = 0, len = s.length(); i < len; i++) { + if (s.charAt(i) == c) { + ++n; + } + } + return n; + } + + @Override public void reset() { + throw new UnsupportedOperationException(); + } + + @Override public void close() { + e.close(); + } + + private Object field(String field, String value) { + switch (field) { + case "block_count": + case "depth": + case "device": + case "gid": + case "uid": + case "hard": + return Integer.valueOf(value); + case "inode": + case "size": + return Long.valueOf(value); + case "access_time": + case "change_time": + case "mod_time": + return new BigDecimal(value).multiply(THOUSAND).longValue(); + default: + return value; + } + } } } diff --git a/plus/src/main/java/org/apache/calcite/adapter/os/GitCommitsTableFunction.java b/plus/src/main/java/org/apache/calcite/adapter/os/GitCommitsTableFunction.java index 0f31e6763cad..2e589572914b 100644 --- a/plus/src/main/java/org/apache/calcite/adapter/os/GitCommitsTableFunction.java +++ b/plus/src/main/java/org/apache/calcite/adapter/os/GitCommitsTableFunction.java @@ -51,91 +51,110 @@ public static ScannableTable eval(boolean b) { @Override public Enumerable<@Nullable Object[]> scan(DataContext root) { final Enumerable enumerable = Processes.processLines("git", "log", "--pretty=raw"); - return new AbstractEnumerable<@Nullable Object[]>() { - @Override public Enumerator<@Nullable Object[]> enumerator() { - final Enumerator e = enumerable.enumerator(); - return new Enumerator<@Nullable Object[]>() { - private @Nullable Object @Nullable [] objects; - private final StringBuilder b = new StringBuilder(); - - @Override public @Nullable Object[] current() { - if (objects == null) { - throw new NoSuchElementException(); - } - return objects; - } + return new GitCommitsTableFunctionEnumerable(enumerable); + } - @Override public boolean moveNext() { - if (!e.moveNext()) { - objects = null; - return false; - } - objects = new Object[9]; - for (;;) { - final String line = e.current(); - if (line.isEmpty()) { - break; // next line will be start of comments - } - if (line.startsWith("commit ")) { - objects[0] = line.substring("commit ".length()); - } else if (line.startsWith("tree ")) { - objects[1] = line.substring("tree ".length()); - } else if (line.startsWith("parent ")) { - if (objects[2] == null) { - objects[2] = line.substring("parent ".length()); - } else { - objects[3] = line.substring("parent ".length()); - } - } else if (line.startsWith("author ")) { - objects[4] = - line.substring("author ".length(), - line.length() - TS_OFF.length() - 1); - objects[5] = - parseLong( - line.substring(line.length() - TS_OFF.length(), - line.length() - OFF.length() - 1)) * 1000; - } else if (line.startsWith("committer ")) { - objects[6] = - line.substring("committer ".length(), - line.length() - TS_OFF.length() - 1); - objects[7] = - parseLong( - line.substring(line.length() - TS_OFF.length(), - line.length() - OFF.length() - 1)) * 1000; - } - if (!e.moveNext()) { - // We have a row, and it's the last because input is empty - return true; - } - } - for (;;) { - if (!e.moveNext()) { - // We have a row, and it's the last because input is empty - objects[8] = b.toString(); - b.setLength(0); - return true; - } - final String line = e.current(); - if (line.isEmpty()) { - // We're seeing the empty line at the end of message - objects[8] = b.toString(); - b.setLength(0); - return true; - } - b.append(line.substring(" ".length())).append("\n"); - } - } + /** Enumerable for {@link GitCommitsTableFunction}. */ + class GitCommitsTableFunctionEnumerable + extends AbstractEnumerable<@Nullable Object[]> { + private final Enumerable enumerable; - @Override public void reset() { - throw new UnsupportedOperationException(); - } + GitCommitsTableFunctionEnumerable(Enumerable enumerable) { + this.enumerable = enumerable; + } + + @Override public Enumerator<@Nullable Object[]> enumerator() { + final Enumerator e = enumerable.enumerator(); + return new GitCommitsTableFunctionEnumerator(e); + } + } + + /** Enumerator for {@link GitCommitsTableFunction}. */ + class GitCommitsTableFunctionEnumerator implements Enumerator<@Nullable Object[]> { + private final Enumerator e; + private @Nullable Object @Nullable [] objects; + private final StringBuilder b; - @Override public void close() { - e.close(); + GitCommitsTableFunctionEnumerator(Enumerator e) { + this.e = e; + b = new StringBuilder(); + } + + @Override public @Nullable Object[] current() { + if (objects == null) { + throw new NoSuchElementException(); + } + return objects; + } + + @Override public boolean moveNext() { + if (!e.moveNext()) { + objects = null; + return false; + } + objects = new Object[9]; + for (;;) { + final String line = e.current(); + if (line.isEmpty()) { + break; // next line will be start of comments + } + if (line.startsWith("commit ")) { + objects[0] = line.substring("commit ".length()); + } else if (line.startsWith("tree ")) { + objects[1] = line.substring("tree ".length()); + } else if (line.startsWith("parent ")) { + if (objects[2] == null) { + objects[2] = line.substring("parent ".length()); + } else { + objects[3] = line.substring("parent ".length()); } - }; + } else if (line.startsWith("author ")) { + objects[4] = + line.substring("author ".length(), + line.length() - TS_OFF.length() - 1); + objects[5] = + parseLong( + line.substring(line.length() - TS_OFF.length(), + line.length() - OFF.length() - 1)) * 1000; + } else if (line.startsWith("committer ")) { + objects[6] = + line.substring("committer ".length(), + line.length() - TS_OFF.length() - 1); + objects[7] = + parseLong( + line.substring(line.length() - TS_OFF.length(), + line.length() - OFF.length() - 1)) * 1000; + } + if (!e.moveNext()) { + // We have a row, and it's the last because input is empty + return true; + } } - }; + for (;;) { + if (!e.moveNext()) { + // We have a row, and it's the last because input is empty + objects[8] = b.toString(); + b.setLength(0); + return true; + } + final String line = e.current(); + if (line.isEmpty()) { + // We're seeing the empty line at the end of message + objects[8] = b.toString(); + b.setLength(0); + return true; + } + b.append(line.substring(" ".length())).append("\n"); + } + } + + @Override public void reset() { + throw new UnsupportedOperationException(); + } + + @Override public void close() { + e.close(); + } } @Override public RelDataType getRowType(RelDataTypeFactory typeFactory) { diff --git a/plus/src/main/java/org/apache/calcite/adapter/tpcds/TpcdsSchema.java b/plus/src/main/java/org/apache/calcite/adapter/tpcds/TpcdsSchema.java index 548c20f8c210..20ae7d314521 100644 --- a/plus/src/main/java/org/apache/calcite/adapter/tpcds/TpcdsSchema.java +++ b/plus/src/main/java/org/apache/calcite/adapter/tpcds/TpcdsSchema.java @@ -158,36 +158,45 @@ private class TpcdsQueryableTable @Override public Queryable asQueryable(final QueryProvider queryProvider, final SchemaPlus schema, final String tableName) { //noinspection unchecked - return (Queryable) new AbstractTableQueryable<@Nullable Object[]>(queryProvider, - schema, this, tableName) { - @Override public Enumerator<@Nullable Object[]> enumerator() { - final Session session = - Session.getDefaultSession() - .withTable(tpcdsTable) - .withScale(scaleFactor); - final Results results = Results.constructResults(tpcdsTable, session); - return Linq4j.asEnumerable(results) - .selectMany( - new Function1>, Enumerable<@Nullable Object[]>>() { - final Column[] columns = tpcdsTable.getColumns(); - - @Override public Enumerable<@Nullable Object[]> apply( - List> inRows) { - final List<@Nullable Object[]> rows = new ArrayList<>(); - for (List<@Nullable String> strings : inRows) { - final @Nullable Object[] values = new Object[columns.length]; - for (int i = 0; i < strings.size(); i++) { - values[i] = convert(strings.get(i), columns[i]); - } - rows.add(values); - } - return Linq4j.asEnumerable(rows); - } - - }) - .enumerator(); + return (Queryable) new TpcdsSchemaQueryable(queryProvider, schema, tableName); + } + + /** Queryable for {@link TpcdsSchema}. */ + private class TpcdsSchemaQueryable extends AbstractTableQueryable<@Nullable Object[]> { + TpcdsSchemaQueryable(QueryProvider queryProvider, SchemaPlus schema, + String tableName) { + super(queryProvider, schema, TpcdsQueryableTable.this, tableName); + } + + @Override public Enumerator<@Nullable Object[]> enumerator() { + final Session session = + Session.getDefaultSession() + .withTable(tpcdsTable) + .withScale(scaleFactor); + final Results results = Results.constructResults(tpcdsTable, session); + return Linq4j.asEnumerable(results) + .selectMany(new TpcdsSchemaSelector()) + .enumerator(); + } + } + + /** Selector for {@link TpcdsSchema}. */ + private class TpcdsSchemaSelector + implements Function1>, Enumerable<@Nullable Object[]>> { + final Column[] columns = tpcdsTable.getColumns(); + + @Override public Enumerable<@Nullable Object[]> apply( + List> inRows) { + final List<@Nullable Object[]> rows = new ArrayList<>(); + for (List<@Nullable String> strings : inRows) { + final @Nullable Object[] values = new Object[columns.length]; + for (int i = 0; i < strings.size(); i++) { + values[i] = convert(strings.get(i), columns[i]); + } + rows.add(values); } - }; + return Linq4j.asEnumerable(rows); + } } @Override public RelDataType getRowType(RelDataTypeFactory typeFactory) { From f020339b4925146c9e7fb4482fc67dec50f94c59 Mon Sep 17 00:00:00 2001 From: "cc.cai" <2356672992@qq.com> Date: Fri, 3 Apr 2026 11:38:01 +0800 Subject: [PATCH 388/562] [CALCITE-7460] Broken Maven Central and CI Status badges in README --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index af6a60e2dab3..adccca475b02 100644 --- a/README.md +++ b/README.md @@ -17,8 +17,8 @@ limitations under the License. {% endcomment %} --> -[![Maven Central](https://maven-badges.herokuapp.com/maven-central/org.apache.calcite/calcite-core/badge.svg)](https://maven-badges.herokuapp.com/maven-central/org.apache.calcite/calcite-core) -[![CI Status](https://github.com/apache/calcite/workflows/CI/badge.svg?branch=main)](https://github.com/apache/calcite/actions?query=branch%3Amain) +[![Maven Central](https://img.shields.io/maven-central/v/org.apache.calcite/calcite-core.svg)](https://central.sonatype.com/artifact/org.apache.calcite/calcite-core) +[![CI Status](https://github.com/apache/calcite/actions/workflows/main.yml/badge.svg?branch=main)](https://github.com/apache/calcite/actions?query=branch%3Amain) # Apache Calcite From f4a0e2558ef50bcd07c1427f1011db5d2d64d4fe Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Tue, 14 Jul 2026 14:47:53 -0700 Subject: [PATCH 389/562] Correct misspelling of 'longtitude' Signed-off-by: Mihai Budiu --- .../calcite/test/schemata/bookstore/BookstoreSchema.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testkit/src/main/java/org/apache/calcite/test/schemata/bookstore/BookstoreSchema.java b/testkit/src/main/java/org/apache/calcite/test/schemata/bookstore/BookstoreSchema.java index daf038e9d350..84c334c48bdd 100644 --- a/testkit/src/main/java/org/apache/calcite/test/schemata/bookstore/BookstoreSchema.java +++ b/testkit/src/main/java/org/apache/calcite/test/schemata/bookstore/BookstoreSchema.java @@ -104,11 +104,11 @@ public Place(@Nullable Coordinate coords, String city, String country) { /** Coordinate. */ public static class Coordinate { public final BigDecimal latitude; - public final BigDecimal longtitude; + public final BigDecimal longitude; - public Coordinate(BigDecimal latitude, BigDecimal longtitude) { + public Coordinate(BigDecimal latitude, BigDecimal longitude) { this.latitude = latitude; - this.longtitude = longtitude; + this.longitude = longitude; } } From 6c9d0ee83690387659b6ec8ecce113368241ed22 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Mon, 13 Jul 2026 20:12:12 -0700 Subject: [PATCH 390/562] [CALCITE-7654] Lambda functions handle incorrectly field accesses Signed-off-by: Mihai Budiu --- .../calcite/sql/fun/SqlDotOperator.java | 8 +++ .../calcite/sql/validate/SqlLambdaScope.java | 21 ++++---- .../sql/validate/SqlValidatorImpl.java | 8 +++ .../sql/validate/SqlValidatorUtil.java | 21 ++++++++ .../apache/calcite/test/SqlValidatorTest.java | 49 +++++++++++++++++++ 5 files changed, 97 insertions(+), 10 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlDotOperator.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlDotOperator.java index 7c9a2315797d..9d9749c3e589 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlDotOperator.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlDotOperator.java @@ -107,6 +107,14 @@ public class SqlDotOperator extends SqlSpecialOperator { return validator.getTypeFactory().createTypeWithNullability(nodeType, true); } + if (nodeType.getSqlTypeName() == SqlTypeName.ANY + && SqlValidatorUtil.inLambdaWithUntypedParameters(scope)) { + // The lambda parameters' types are not known until the enclosing call + // has inferred its operand types. Field resolution happens when the + // lambda type checker re-validates the lambda body. + return validator.getTypeFactory().createTypeWithNullability(nodeType, true); + } + if (!nodeType.isStruct()) { throw SqlUtil.newContextException(operand.getParserPosition(), Static.RESOURCE.incompatibleTypes()); diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlLambdaScope.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlLambdaScope.java index 0db4b457115b..a4ba93d56ab0 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlLambdaScope.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlLambdaScope.java @@ -62,16 +62,17 @@ public boolean isParameter(SqlIdentifier id) { } @Override public SqlQualified fullyQualify(SqlIdentifier identifier) { - if (identifier.isSimple()) { - final SqlNameMatcher nameMatcher = validator.catalogReader.nameMatcher(); - final String name = identifier.getSimple(); - boolean found = lambdaExpr.getParameters() - .stream() - .anyMatch(param -> - nameMatcher.matches(((SqlIdentifier) param).getSimple(), name)); - if (found) { - return SqlQualified.create(this, 1, null, identifier); - } + final SqlNameMatcher nameMatcher = validator.catalogReader.nameMatcher(); + final String name = identifier.names.get(0); + boolean found = lambdaExpr.getParameters() + .stream() + .anyMatch(param -> + nameMatcher.matches(((SqlIdentifier) param).getSimple(), name)); + if (found) { + // If the first component names a parameter, in a compound identifier + // such as 'x.name' the remaining components are fields of the + // parameter's struct. + return SqlQualified.create(this, 1, null, identifier); } if (!validator.config().conformance().allowLambdaClosure()) { throw validator.newValidationError(identifier, diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java index 5d667fe502e3..023de1b77f77 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java @@ -8034,6 +8034,14 @@ private class DeriveTypeVisitor implements SqlVisitor { // Resolve rest of identifier for (; i < id.names.size(); i++) { + if (type.getSqlTypeName() == SqlTypeName.ANY + && SqlValidatorUtil.inLambdaWithUntypedParameters(scope)) { + // The lambda parameters' types are not known until the enclosing + // call has inferred its operand types. Field resolution happens + // when the lambda type checker re-validates the lambda body. + return typeFactory.createTypeWithNullability( + typeFactory.createSqlType(SqlTypeName.ANY), true); + } String name = id.names.get(i); final RelDataTypeField field; if (name.isEmpty()) { diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorUtil.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorUtil.java index 1c5bfcf9a6c0..3b67e5bbc328 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorUtil.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorUtil.java @@ -57,6 +57,7 @@ import org.apache.calcite.sql.SqlUtil; import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.parser.SqlParserPos; +import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.sql.type.SqlTypeUtil; import org.apache.calcite.util.ImmutableBitSet; import org.apache.calcite.util.Litmus; @@ -1678,6 +1679,26 @@ private static class ExplicitTableSchema extends AbstractSchema { } } + /** Returns whether {@code scope} is enclosed in the scope of a lambda + * expression whose parameter types have not yet been inferred. + * + *

      Un-inferred parameters have type ANY, and so does any expression + * computed from them, whatever its shape; field resolution on such + * values must be deferred until the enclosing call has inferred the + * parameter types and re-validates the lambda body (see + * {@link SqlValidator#validateLambda}). */ + public static boolean inLambdaWithUntypedParameters(SqlValidatorScope scope) { + for (SqlValidatorScope s = scope; s instanceof DelegatingScope; + s = ((DelegatingScope) s).getParent()) { + if (s instanceof SqlLambdaScope + && ((SqlLambdaScope) s).getParameterTypes().values().stream() + .anyMatch(t -> t.getSqlTypeName() == SqlTypeName.ANY)) { + return true; + } + } + return false; + } + /** Flattens any FILTER, WITHIN DISTINCT, WITHIN GROUP surrounding a call to * an aggregate function. */ public static class FlatAggregate { diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index 228d90026b22..cecb57385e4e 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -8776,6 +8776,55 @@ void testGroupExpressionEquivalenceParams() { + "reference to 'EMP.DEPTNO' from enclosing scope"); } + /** Test case for + * [CALCITE-7654] + * Lambda functions handle incorrectly field accesses. */ + @Test void testLambdaStructFieldAccess() { + SqlOperatorTable chain = + SqlOperatorTables.chain( + SqlOperatorTables.of(SqlLibraryOperators.EXISTS), + SqlStdOperatorTable.instance()); + final SqlValidatorFixture s = fixture().withOperatorTable(chain); + // EMPLOYEES is ARRAY. + // Parenthesized form: parses as DOT(e, empno). + s.withSql("select \"EXISTS\"(employees, e -> (e).empno > 0)\n" + + "from dept_nested") + .columnType("BOOLEAN"); + // Unparenthesized form: parses as compound identifier e.empno. + s.withSql("select \"EXISTS\"(employees, e -> e.empno > 0)\n" + + "from dept_nested") + .columnType("BOOLEAN"); + // Nested field access. + s.withSql("select \"EXISTS\"(employees, e -> (e).detail.skills is not null)\n" + + "from dept_nested") + .columnType("BOOLEAN"); + // Field access through an array index: DETAIL.SKILLS is + // ARRAY. + s.withSql("select \"EXISTS\"(employees,\n" + + " e -> (e).detail.skills[1].\"TYPE\" = 'a')\n" + + "from dept_nested") + .columnType("BOOLEAN"); + // Same, starting from a compound identifier. + s.withSql("select \"EXISTS\"(employees,\n" + + " e -> e.detail.skills[1].\"TYPE\" = 'a')\n" + + "from dept_nested") + .columnType("BOOLEAN"); + // A field that does not exist in the parameter's type. + s.withSql("select \"EXISTS\"(employees, e -> e.^bad^ > 0)\n" + + "from dept_nested") + .fails("Unknown field 'BAD'"); + // An unknown field behind an array index produces an error. + s.withSql("select \"EXISTS\"(employees,\n" + + " e -> e.detail.skills[1].^bad^ = 'a')\n" + + "from dept_nested") + .fails("Unknown field 'BAD'"); + // Field access on an expression that is not a simple access chain. + s.withSql("select \"EXISTS\"(employees,\n" + + " e -> coalesce((e).detail, e.detail).skills is not null)\n" + + "from dept_nested") + .columnType("BOOLEAN"); + } + /** Test case for [CALCITE-7193] * In an aggregation validator treats lambda variable names as column names. */ @Test void testGroupByLambda() { From 657c317cfca697679752010525ac50c0bf4a2688 Mon Sep 17 00:00:00 2001 From: Stamatis Zampetakis Date: Fri, 3 May 2024 00:23:03 +0200 Subject: [PATCH 391/562] [CALCITE-7649] Add bytecode verification check using ASM in the gradle build --- build.gradle.kts | 11 ++++ buildSrc/gradle.properties | 3 ++ buildSrc/settings.gradle.kts | 1 + .../asmchecker/asmchecker.gradle.kts | 34 ++++++++++++ .../buildtools/asmchecker/AsmCheckerPlugin.kt | 26 +++++++++ .../buildtools/asmchecker/AsmCheckerTask.kt | 53 +++++++++++++++++++ gradle.properties | 1 + 7 files changed, 129 insertions(+) create mode 100644 buildSrc/subprojects/asmchecker/asmchecker.gradle.kts create mode 100644 buildSrc/subprojects/asmchecker/src/main/kotlin/org/apache/calcite/buildtools/asmchecker/AsmCheckerPlugin.kt create mode 100644 buildSrc/subprojects/asmchecker/src/main/kotlin/org/apache/calcite/buildtools/asmchecker/AsmCheckerTask.kt diff --git a/build.gradle.kts b/build.gradle.kts index 08cd2bd38989..44e00c7fa39a 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -26,6 +26,7 @@ import com.github.vlsi.gradle.release.RepositoryType import de.thetaphi.forbiddenapis.gradle.CheckForbiddenApis import de.thetaphi.forbiddenapis.gradle.CheckForbiddenApisExtension import net.ltgt.gradle.errorprone.errorprone +import org.apache.calcite.buildtools.asmchecker.AsmCheckerTask import org.apache.calcite.buildtools.buildext.dsl.ParenthesisBalancer import org.gradle.api.tasks.testing.logging.TestExceptionFormat @@ -37,6 +38,7 @@ plugins { publishing // Verification checkstyle + id("calcite.asmchecker") calcite.buildext jacoco id("jacoco-report-aggregation") @@ -930,6 +932,15 @@ allprojects { } jvmArgs("-Xmx6g") } + register("bytecodeCheck") { + group = LifecycleBasePlugin.VERIFICATION_GROUP + description = "Checks the bytecode of every .class file in the build directory using ASM." + dependsOn("classes") + } + + check { + dependsOn("bytecodeCheck") + } hepLargePlanModeTestIncludes[project.path]?.let { includes -> val hepLargePlanModeTask = register("testHepLargePlanMode") { group = LifecycleBasePlugin.VERIFICATION_GROUP diff --git a/buildSrc/gradle.properties b/buildSrc/gradle.properties index f308e6c2451a..a3ffbfb44aeb 100644 --- a/buildSrc/gradle.properties +++ b/buildSrc/gradle.properties @@ -20,3 +20,6 @@ kotlin.code.style=official # Plugins com.github.autostyle.version=3.2 com.github.vlsi.vlsi-release-plugins.version=1.52 + +# Dependencies (keep in sync with root gradle.properties) +asm.version=9.9.1 diff --git a/buildSrc/settings.gradle.kts b/buildSrc/settings.gradle.kts index e3b6a38b8049..413ea0b7e593 100644 --- a/buildSrc/settings.gradle.kts +++ b/buildSrc/settings.gradle.kts @@ -24,6 +24,7 @@ pluginManagement { } } +include("asmchecker") include("javacc") include("fmpp") include("buildext") diff --git a/buildSrc/subprojects/asmchecker/asmchecker.gradle.kts b/buildSrc/subprojects/asmchecker/asmchecker.gradle.kts new file mode 100644 index 000000000000..03f796afd589 --- /dev/null +++ b/buildSrc/subprojects/asmchecker/asmchecker.gradle.kts @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +dependencies { + val asmVersion = providers.gradleProperty("asm.version").get() + implementation("org.ow2.asm:asm:$asmVersion") + implementation("org.ow2.asm:asm-analysis:$asmVersion") + implementation("org.ow2.asm:asm-commons:$asmVersion") + implementation("org.ow2.asm:asm-tree:$asmVersion") + implementation("org.ow2.asm:asm-util:$asmVersion") +} + +gradlePlugin { + plugins { + register("asmchecker") { + id = "calcite.asmchecker" + implementationClass = "org.apache.calcite.buildtools.asmchecker.AsmCheckerPlugin" + } + } +} diff --git a/buildSrc/subprojects/asmchecker/src/main/kotlin/org/apache/calcite/buildtools/asmchecker/AsmCheckerPlugin.kt b/buildSrc/subprojects/asmchecker/src/main/kotlin/org/apache/calcite/buildtools/asmchecker/AsmCheckerPlugin.kt new file mode 100644 index 000000000000..22463b41cd0d --- /dev/null +++ b/buildSrc/subprojects/asmchecker/src/main/kotlin/org/apache/calcite/buildtools/asmchecker/AsmCheckerPlugin.kt @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.calcite.buildtools.asmchecker + +import org.gradle.api.Plugin +import org.gradle.api.Project + +open class AsmCheckerPlugin : Plugin { + override fun apply(target: Project) { + } +} diff --git a/buildSrc/subprojects/asmchecker/src/main/kotlin/org/apache/calcite/buildtools/asmchecker/AsmCheckerTask.kt b/buildSrc/subprojects/asmchecker/src/main/kotlin/org/apache/calcite/buildtools/asmchecker/AsmCheckerTask.kt new file mode 100644 index 000000000000..f1dc97e657ae --- /dev/null +++ b/buildSrc/subprojects/asmchecker/src/main/kotlin/org/apache/calcite/buildtools/asmchecker/AsmCheckerTask.kt @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.calcite.buildtools.asmchecker + +import java.nio.file.Files +import java.nio.file.Paths +import org.gradle.api.DefaultTask +import org.gradle.api.tasks.CacheableTask +import org.gradle.api.tasks.TaskAction +import org.objectweb.asm.ClassReader +import org.objectweb.asm.ClassWriter +import org.objectweb.asm.commons.ClassRemapper +import org.objectweb.asm.commons.Remapper +import org.objectweb.asm.util.CheckClassAdapter + +@CacheableTask +open class AsmCheckerTask : DefaultTask() { + + @TaskAction + fun run() { + project.layout.buildDirectory.get().asFile.walk() + .onEnter { dir -> + // the classes in spark/build/sparkServer/classes, generated by SparkHandlerImpl, + // have invalid bytecode, so exclude them + "sparkServer${java.io.File.separator}classes" !in dir.path } + .filter { file -> file.getName().lowercase().endsWith(".class") } + .forEach { + val classReader = ClassReader(Files.readAllBytes(Paths.get(it.getPath()))) + val classVisitor = CheckClassAdapter(ClassWriter(ClassWriter.COMPUTE_MAXS)) + val classRemapper = ClassRemapper(classVisitor, object : Remapper() {}) + try { + classReader.accept(classRemapper, ClassReader.EXPAND_FRAMES) + } catch (e: java.lang.RuntimeException) { + throw java.lang.RuntimeException("Invalid bytecode file:" + it, e) + } + } + } +} diff --git a/gradle.properties b/gradle.properties index 14d29dfa356d..0c5e83c5038c 100644 --- a/gradle.properties +++ b/gradle.properties @@ -82,6 +82,7 @@ jandex.version=3.5.3 aggdesigner-algorithm.version=6.1 apiguardian-api.version=1.1.2 arrow.version=16.0.0 +# must be kept in sync with buildSrc/gradle.properties asm.version=9.9.1 byte-buddy.version=1.18.8 cassandra-all.version=4.1.6 From 2f01f003e760b5c5d619aa643acf842651ea3500 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Tue, 14 Jul 2026 20:37:05 -0700 Subject: [PATCH 392/562] [CALCITE-7656] Result type inferred for VAR_SAMP is incorrect Signed-off-by: Mihai Budiu --- .../java/org/apache/calcite/sql/type/ReturnTypes.java | 7 +++++-- .../AggregateReduceFunctionsOnGroupKeysRuleTest.xml | 6 +++--- .../org/apache/calcite/test/RelOptRulesTest.xml | 4 ++-- .../java/org/apache/calcite/test/SqlOperatorTest.java | 11 ++++------- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql/type/ReturnTypes.java b/core/src/main/java/org/apache/calcite/sql/type/ReturnTypes.java index 9e32e0563d96..5e5ab2207cda 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/ReturnTypes.java +++ b/core/src/main/java/org/apache/calcite/sql/type/ReturnTypes.java @@ -1581,8 +1581,10 @@ private static RelDataType multivalentStringWithSepSumPrecision( final RelDataType relDataType = typeFactory.getTypeSystem().deriveAvgAggType(typeFactory, opBinding.getOperandType(0)); + final SqlKind kind = opBinding.getOperator().kind; if (opBinding.hasEmptyGroup() || opBinding.hasFilter() - || opBinding.getOperator().kind == SqlKind.STDDEV_SAMP) { + || kind == SqlKind.STDDEV_SAMP + || kind == SqlKind.VAR_SAMP) { return typeFactory.createTypeWithNullability(relDataType, true); } else { return relDataType; @@ -1594,7 +1596,8 @@ private static RelDataType multivalentStringWithSepSumPrecision( final RelDataType relDataType = typeFactory.getTypeSystem().deriveCovarType(typeFactory, opBinding.getOperandType(0), opBinding.getOperandType(1)); - if (opBinding.hasEmptyGroup() || opBinding.hasFilter()) { + if (opBinding.hasEmptyGroup() || opBinding.hasFilter() + || opBinding.getOperator().kind == SqlKind.COVAR_SAMP) { return typeFactory.createTypeWithNullability(relDataType, true); } else { return relDataType; diff --git a/core/src/test/resources/org/apache/calcite/test/AggregateReduceFunctionsOnGroupKeysRuleTest.xml b/core/src/test/resources/org/apache/calcite/test/AggregateReduceFunctionsOnGroupKeysRuleTest.xml index 4f7ac679939e..bead1ea3bd2b 100644 --- a/core/src/test/resources/org/apache/calcite/test/AggregateReduceFunctionsOnGroupKeysRuleTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/AggregateReduceFunctionsOnGroupKeysRuleTest.xml @@ -338,7 +338,7 @@ LogicalProject(SAL=[$0], SD=[$2], SDP=[$3], VP=[$4], VS=[$5]) Date: Sat, 4 Jul 2026 08:12:19 +0800 Subject: [PATCH 393/562] [CALCITE-6767] PERCENTILE_CONT/PERCENTILE_DISC function are not supported --- .../enumerable/EnumerableAggregateBase.java | 15 +++ .../adapter/enumerable/RexImpTable.java | 58 +++++++++ .../apache/calcite/runtime/SqlFunctions.java | 50 ++++++++ .../apache/calcite/util/BuiltInMethod.java | 2 + core/src/test/resources/sql/agg.iq | 113 ++++++++++++++++++ 5 files changed, 238 insertions(+) diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableAggregateBase.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableAggregateBase.java index 3d0ab7e89546..ae2f939988aa 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableAggregateBase.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableAggregateBase.java @@ -30,6 +30,7 @@ import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.RelCollations; +import org.apache.calcite.rel.RelFieldCollation; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Aggregate; import org.apache.calcite.rel.core.AggregateCall; @@ -264,6 +265,20 @@ protected void createAccumulatorAdders( for (int index : agg.call.getArgList()) { args.add(RexInputRef.of(index, inputTypes)); } + // Percentile functions such as PERCENTILE_CONT and + // PERCENTILE_DISC take the fraction as their only argument, but + // aggregate over the WITHIN GROUP (ORDER BY ...) column. Expose + // that collation column as an extra argument so that the + // accumulator can collect its values (already sorted by + // SourceSorter). + if (agg.call.getAggregation().isPercentile()) { + for (RelFieldCollation fieldCollation + : agg.call.collation.getFieldCollations()) { + args.add( + RexInputRef.of(fieldCollation.getFieldIndex(), + inputTypes)); + } + } return args; } diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java index acbd66cb1cc7..bf5de12e742f 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java @@ -499,6 +499,8 @@ import static org.apache.calcite.sql.fun.SqlStdOperatorTable.OCTET_LENGTH; import static org.apache.calcite.sql.fun.SqlStdOperatorTable.OR; import static org.apache.calcite.sql.fun.SqlStdOperatorTable.OVERLAY; +import static org.apache.calcite.sql.fun.SqlStdOperatorTable.PERCENTILE_CONT; +import static org.apache.calcite.sql.fun.SqlStdOperatorTable.PERCENTILE_DISC; import static org.apache.calcite.sql.fun.SqlStdOperatorTable.PI; import static org.apache.calcite.sql.fun.SqlStdOperatorTable.PLUS; import static org.apache.calcite.sql.fun.SqlStdOperatorTable.POSITION; @@ -1326,6 +1328,8 @@ void populate3() { defineAgg(SINGLE_VALUE, SingleValueImplementor.class); defineAgg(COLLECT, CollectImplementor.class); defineAgg(ARRAY_AGG, CollectImplementor.class); + defineAgg(PERCENTILE_CONT, PercentileImplementor.class); + defineAgg(PERCENTILE_DISC, PercentileImplementor.class); defineAgg(LISTAGG, ListaggImplementor.class); defineAgg(FUSION, FusionImplementor.class); defineAgg(MODE, ModeImplementor.class); @@ -1967,6 +1971,60 @@ static class CollectImplementor extends StrictAggImplementor { } } + /** Implementor for the {@code PERCENTILE_CONT} and {@code PERCENTILE_DISC} + * aggregate functions. + * + *

      The fraction is the sole argument of the aggregate call, while the + * values whose percentile is computed come from the + * {@code WITHIN GROUP (ORDER BY ...)} column, which + * {@link EnumerableAggregateBase#createAccumulatorAdders} exposes as an extra + * argument. The input rows are sorted by {@code SourceSorter} before being + * accumulated, so the collected values are already in order. */ + static class PercentileImplementor extends StrictAggImplementor { + @Override public List getNotNullState(AggContext info) { + final List types = new ArrayList<>(); + types.add(List.class); + types.add(double.class); + return types; + } + + @Override protected void implementNotNullReset(AggContext info, + AggResetContext reset) { + reset.currentBlock().add( + Expressions.statement( + Expressions.assign(reset.accumulator().get(0), + Expressions.new_(ArrayList.class)))); + reset.currentBlock().add( + Expressions.statement( + Expressions.assign(reset.accumulator().get(1), + Expressions.constant(0d)))); + } + + @Override protected void implementNotNullAdd(AggContext info, + AggAddContext add) { + add.currentBlock().add( + Expressions.statement( + Expressions.assign(add.accumulator().get(1), + EnumUtils.convert(add.arguments().get(0), double.class)))); + + add.currentBlock().add( + Expressions.statement( + Expressions.call(add.accumulator().get(0), + BuiltInMethod.COLLECTION_ADD.method, + Expressions.box(add.arguments().get(1))))); + } + + @Override protected Expression implementNotNullResult(AggContext info, + AggResultContext result) { + final BuiltInMethod method = + info.aggregation().kind == SqlKind.PERCENTILE_DISC + ? BuiltInMethod.PERCENTILE_DISC + : BuiltInMethod.PERCENTILE_CONT; + return Expressions.call(method.method, result.accumulator().get(0), + result.accumulator().get(1)); + } + } + /** Implementor for the {@code LISTAGG} aggregate function. */ static class ListaggImplementor extends StrictAggImplementor { @Override protected void implementNotNullReset(AggContext info, diff --git a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java index ed6a3ba16f09..c41ab84d71d5 100644 --- a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java +++ b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java @@ -133,6 +133,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.NoSuchElementException; import java.util.Objects; import java.util.Set; import java.util.TimeZone; @@ -4069,6 +4070,55 @@ public static BigDecimal mod(BigDecimal b0, BigDecimal b1) { return bigDecimals[1]; } + // PERCENTILE_CONT / PERCENTILE_DISC + + /** Support the PERCENTILE_CONT aggregate function. + * + *

      The {@code values} list must already be sorted according to the + * {@code WITHIN GROUP (ORDER BY ...)} clause. The fraction must be in the + * range 0 to 1 inclusive. The result is a linear interpolation between the + * two values that surround the desired position. */ + public static BigDecimal percentileCont(List values, + double fraction) { + final int n = values.size(); + if (n == 0) { + throw new NoSuchElementException( + "PERCENTILE_CONT is not defined on an empty group"); + } + final double rank = fraction * (n - 1); + final int lo = (int) Math.floor(rank); + final int hi = (int) Math.ceil(rank); + final BigDecimal loValue = toBigDecimal(values.get(lo)); + if (lo == hi) { + return loValue; + } + final BigDecimal hiValue = toBigDecimal(values.get(hi)); + final BigDecimal frac = BigDecimal.valueOf(rank - lo); + return loValue.add(hiValue.subtract(loValue).multiply(frac)); + } + + /** Support the PERCENTILE_DISC aggregate function. + * + *

      The {@code values} list must already be sorted according to the + * {@code WITHIN GROUP (ORDER BY ...)} clause. The fraction must be in the + * range 0 to 1 inclusive. The result is an actual value from the group: the + * first whose cumulative distribution is greater than or equal to the + * fraction. */ + public static Object percentileDisc(List values, double fraction) { + final int n = values.size(); + if (n == 0) { + throw new NoSuchElementException( + "PERCENTILE_DISC is not defined on an empty group"); + } + int index = (int) Math.ceil(fraction * n) - 1; + if (index < 0) { + index = 0; + } else if (index >= n) { + index = n - 1; + } + return requireNonNull(values.get(index)); + } + // FLOOR public static double floor(double b0) { diff --git a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java index 2b07069b6598..3c3a3e363e65 100644 --- a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java +++ b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java @@ -415,6 +415,8 @@ public enum BuiltInMethod { COLLECTION_ADD(Collection.class, "add", Object.class), COLLECTION_ADDALL(Collection.class, "addAll", Collection.class), COLLECTION_RETAIN_ALL(Collection.class, "retainAll", Collection.class), + PERCENTILE_CONT(SqlFunctions.class, "percentileCont", List.class, double.class), + PERCENTILE_DISC(SqlFunctions.class, "percentileDisc", List.class, double.class), LIST_CONTAINS(List.class, "contains", Object.class), LIST_GET(List.class, "get", int.class), LIST_TO_ARRAY(List.class, "toArray"), diff --git a/core/src/test/resources/sql/agg.iq b/core/src/test/resources/sql/agg.iq index b637a6cbc196..9355ffedae36 100644 --- a/core/src/test/resources/sql/agg.iq +++ b/core/src/test/resources/sql/agg.iq @@ -3844,6 +3844,119 @@ select distinct sum(deptno + '1') as deptsum from dept order by 1; !ok +# [CALCITE-6767] PERCENTILE_CONT/PERCENTILE_DISC syntax not supported. +# These sql programs were validated in PostgreSQL +select + percentile_cont(0.5) within group (order by empno) as c, + percentile_disc(0.5) within group (order by empno) as d +from emp; ++------+------+ +| C | D | ++------+------+ +| 7785 | 7782 | ++------+------+ +(1 row) + +!ok + +# PERCENTILE_CONT / PERCENTILE_DISC with GROUP BY. +select deptno, + percentile_cont(0.5) within group (order by empno) as c, + percentile_disc(0.5) within group (order by empno) as d +from emp +group by deptno +order by deptno; ++--------+------+------+ +| DEPTNO | C | D | ++--------+------+------+ +| 10 | 7839 | 7839 | +| 20 | 7788 | 7788 | +| 30 | 7676 | 7654 | ++--------+------+------+ +(3 rows) + +!ok + +# PERCENTILE_CONT / PERCENTILE_DISC honour a descending ORDER BY. +select + percentile_disc(0.25) within group (order by empno desc) as d +from emp; ++------+ +| D | ++------+ +| 7876 | ++------+ +(1 row) + +!ok + +# PERCENTILE_CONT / PERCENTILE_DISC on DOUBLE values. +select + percentile_cont(0.5) within group (order by v) as c, + percentile_disc(0.5) within group (order by v) as d +from (values (cast(1.5 as double)), + (cast(2.5 as double)), + (cast(4.5 as double)), + (cast(8.5 as double))) as t(v); ++-----+-----+ +| C | D | ++-----+-----+ +| 3.5 | 2.5 | ++-----+-----+ +(1 row) + +!ok + +# PERCENTILE_CONT / PERCENTILE_DISC on large DECIMAL values that are very +# close together. These 19-digit values exceed the ~15-16 significant digits a +# double can hold, so the linear interpolation must be done in BigDecimal; +# converting to double would round 9999999999999999990 to 1.0E19 and lose the +# trailing digits. +select + percentile_cont(0.5) within group (order by v) as c, + percentile_disc(0.5) within group (order by v) as d +from (values (cast('9999999999999999990' as decimal(19, 0))), + (cast('9999999999999999992' as decimal(19, 0)))) as t(v); ++-----------------------+---------------------+ +| C | D | ++-----------------------+---------------------+ +| 9999999999999999991.0 | 9999999999999999990 | ++-----------------------+---------------------+ +(1 row) + +!ok + +# A three-row large DECIMAL group whose 0.75 percentile interpolates between +# two adjacent, nearly-equal values. +select + percentile_cont(0.75) within group (order by v) as c +from (values (cast('9999999999999999990' as decimal(19, 0))), + (cast('9999999999999999994' as decimal(19, 0))), + (cast('9999999999999999998' as decimal(19, 0)))) as t(v); ++-----------------------+ +| C | ++-----------------------+ +| 9999999999999999996.0 | ++-----------------------+ +(1 row) + +!ok + +# PERCENTILE_CONT / PERCENTILE_DISC on UNSIGNED values. +select + percentile_cont(0.5) within group (order by v) as c, + percentile_disc(0.5) within group (order by v) as d +from (values (cast(10 as int unsigned)), + (cast(20 as int unsigned))) as t(v); ++----+----+ +| C | D | ++----+----+ +| 15 | 10 | ++----+----+ +(1 row) + +!ok + # [CALCITE-6839] The SUM function sometimes throws overflow exceptions due # to incorrect return types !use scott-spark From 732a1e71f866c2e71158f7f723559503ae356277 Mon Sep 17 00:00:00 2001 From: zzwqqq Date: Tue, 23 Jun 2026 17:05:50 +0800 Subject: [PATCH 394/562] [CALCITE-7607] DML type coercion inserts implicit narrowing casts for target-column assignments --- .../apache/calcite/rel/core/TableModify.java | 30 ++++++++++++ .../validate/implicit/TypeCoercionImpl.java | 8 +++- .../rel/rel2sql/RelToSqlConverterTest.java | 21 ++++----- .../apache/calcite/test/JdbcAdapterTest.java | 8 ++-- .../org/apache/calcite/test/JdbcTest.java | 2 +- .../apache/calcite/test/RelOptRulesTest.java | 2 +- .../test/TypeCoercionConverterTest.java | 32 +++++++++++++ .../apache/calcite/test/RelOptRulesTest.xml | 4 +- .../calcite/test/SqlToRelConverterTest.xml | 4 +- .../test/TypeCoercionConverterTest.xml | 47 +++++++++++++++++-- .../test/resources/sql/materialized_view.iq | 26 +++++----- server/src/test/resources/sql/table_as.iq | 28 +++++------ 12 files changed, 160 insertions(+), 52 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rel/core/TableModify.java b/core/src/main/java/org/apache/calcite/rel/core/TableModify.java index 60b9814dfd20..83dd8106476e 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/TableModify.java +++ b/core/src/main/java/org/apache/calcite/rel/core/TableModify.java @@ -32,7 +32,9 @@ import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexUtil; import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.type.SqlTypeUtil; @@ -261,9 +263,37 @@ public boolean isMerge() { null); } + inputRowType = expectedInputRowTypeForAssignment(inputRowType); return inputRowType; } + private RelDataType expectedInputRowTypeForAssignment( + RelDataType expectedRowType) { + final RelDataType actualRowType = getInput().getRowType(); + if (actualRowType.getFieldCount() != expectedRowType.getFieldCount()) { + return expectedRowType; + } + final RelDataTypeFactory typeFactory = getCluster().getTypeFactory(); + final RelDataTypeFactory.Builder builder = typeFactory.builder(); + boolean changed = false; + final List expectedFields = expectedRowType.getFieldList(); + final List actualFields = actualRowType.getFieldList(); + for (int i = 0; i < expectedFields.size(); i++) { + final RelDataTypeField expectedField = expectedFields.get(i); + final RelDataType actualType = actualFields.get(i).getType(); + final RelDataType expectedType = expectedField.getType(); + if (!SqlTypeUtil.equalSansNullability(typeFactory, actualType, expectedType) + && SqlTypeUtil.canAssignFrom(expectedType, actualType) + && !RexUtil.isLosslessCast(actualType, expectedType)) { + builder.add(expectedField.getName(), actualType); + changed = true; + } else { + builder.add(expectedField); + } + } + return changed ? builder.build() : expectedRowType; + } + @Override public RelWriter explainTerms(RelWriter pw) { return super.explainTerms(pw) .item("table", table.getQualifiedName()) diff --git a/core/src/main/java/org/apache/calcite/sql/validate/implicit/TypeCoercionImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/implicit/TypeCoercionImpl.java index 26812be96110..46dc3af5ed80 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/implicit/TypeCoercionImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/implicit/TypeCoercionImpl.java @@ -19,6 +19,7 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.rex.RexUtil; import org.apache.calcite.sql.SqlCall; import org.apache.calcite.sql.SqlCallBinding; import org.apache.calcite.sql.SqlFunction; @@ -830,8 +831,13 @@ && coerceOperandType(binding.getScope(), binding.getCall(), i, implicitType) } boolean coerced = false; for (int i = 0; i < sourceFields.size(); i++) { + RelDataType sourceType = sourceFields.get(i).getType(); RelDataType targetType = targetFields.get(i).getType(); - coerced = coerceSourceRowType(scope, query, i, targetType) || coerced; + if (!SqlTypeUtil.equalSansNullability(validator.getTypeFactory(), sourceType, targetType) + && (RexUtil.isLosslessCast(sourceType, targetType) + || !SqlTypeUtil.canAssignFrom(targetType, sourceType))) { + coerced = coerceSourceRowType(scope, query, i, targetType) || coerced; + } } return coerced; } diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java index f056ee9d4fc8..2ac820e83855 100644 --- a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java +++ b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java @@ -10105,7 +10105,7 @@ private void checkLiteral2(String expression, String expected) { + "ON \"DEPT\".\"DEPTNO\" = \"DEPT0\".\"DEPTNO\"\n" + "WHEN MATCHED THEN UPDATE SET \"DNAME\" = \"DEPT\".\"DNAME\"\n" + "WHEN NOT MATCHED THEN INSERT (\"DEPTNO\", \"DNAME\", \"LOC\") " - + "VALUES CAST(\"DEPT\".\"DEPTNO\" + 1 AS TINYINT),\n" + + "VALUES \"DEPT\".\"DEPTNO\" + 1,\n" + "LOWER(\"DEPT\".\"DNAME\"),\n" + "UPPER(\"DEPT\".\"LOC\")"; sql(sql1) @@ -10139,9 +10139,9 @@ private void checkLiteral2(String expression, String expected) { + "ON \"DEPT\".\"DEPTNO\" = \"DEPT0\".\"DEPTNO\"\n" + "WHEN MATCHED THEN UPDATE SET \"DNAME\" = \"DEPT\".\"DNAME\"\n" + "WHEN NOT MATCHED THEN INSERT (\"DEPTNO\", \"DNAME\", \"LOC\") " - + "VALUES CAST(\"DEPT\".\"DEPTNO\" + 1 AS TINYINT),\n" + + "VALUES \"DEPT\".\"DEPTNO\" + 1,\n" + "'abc',\n" - + "CAST(LOWER(\"DEPT\".\"DNAME\") AS VARCHAR(13) CHARACTER SET \"ISO-8859-1\")"; + + "LOWER(\"DEPT\".\"DNAME\")"; sql(sql3) .schema(CalciteAssert.SchemaSpec.JDBC_SCOTT) .ok(expected3); @@ -10171,7 +10171,7 @@ private void checkLiteral2(String expression, String expected) { + "USING \"SCOTT\".\"DEPT\"\n" + "ON \"DEPT\".\"DEPTNO\" = \"DEPT0\".\"DEPTNO\"\n" + "WHEN NOT MATCHED THEN INSERT (\"DEPTNO\", \"DNAME\", \"LOC\") " - + "VALUES CAST(\"DEPT\".\"DEPTNO\" + 1 AS TINYINT),\n" + + "VALUES \"DEPT\".\"DEPTNO\" + 1,\n" + "LOWER(\"DEPT\".\"DNAME\"),\n" + "UPPER(\"DEPT\".\"LOC\")"; sql(sql5) @@ -10191,7 +10191,7 @@ private void checkLiteral2(String expression, String expected) { + "WHERE CAST(\"DEPTNO\" AS INTEGER) <> 5) AS \"t0\"\n" + "ON \"t0\".\"DEPTNO\" = \"DEPT0\".\"DEPTNO\"\n" + "WHEN NOT MATCHED THEN INSERT (\"DEPTNO\", \"DNAME\", \"LOC\") " - + "VALUES CAST(\"t0\".\"DEPTNO\" + 1 AS TINYINT),\n" + + "VALUES \"t0\".\"DEPTNO\" + 1,\n" + "LOWER(\"t0\".\"DNAME\"),\n" + "UPPER(\"t0\".\"LOC\")"; sql(sql6) @@ -10213,7 +10213,7 @@ private void checkLiteral2(String expression, String expected) { + "ON \"t0\".\"EXPR$0\" = \"t1\".\"DEPTNO0\"\n" + "WHEN MATCHED THEN UPDATE SET \"DNAME\" = 'abc'\n" + "WHEN NOT MATCHED THEN INSERT (\"DEPTNO\", \"DNAME\", \"LOC\") " - + "VALUES CAST(\"t0\".\"EXPR$0\" + 1 AS TINYINT),\n" + + "VALUES \"t0\".\"EXPR$0\" + 1,\n" + "CAST(LOWER(\"t0\".\"EXPR$1\") AS VARCHAR(14) CHARACTER SET \"ISO-8859-1\"),\n" + "CAST(UPPER(\"t0\".\"EXPR$2\") AS VARCHAR(13) CHARACTER SET \"ISO-8859-1\")"; sql(sql7) @@ -10402,9 +10402,8 @@ private void checkLiteral2(String expression, String expected) { final String sql0 = "update \"foodmart\".\"product\" " + "set \"product_name\" = \"product_name\" || '_'\n" + "where \"product_id\" > 10"; - final String expected0 = "UPDATE \"foodmart\".\"product\" SET \"product_name\" = CAST" - + "(\"product_name\" || '_' AS VARCHAR(60) CHARACTER SET \"ISO-8859-1\")\nWHERE " - + "\"product_id\" > 10"; + final String expected0 = "UPDATE \"foodmart\".\"product\" SET \"product_name\" = " + + "\"product_name\" || '_'\nWHERE \"product_id\" > 10"; sql(sql0).ok(expected0); final String sql1 = "update \"foodmart\".\"product\"" @@ -10420,8 +10419,8 @@ private void checkLiteral2(String expression, String expected) { + " \"product_name\" = \"product_name\" || '_' \n" + "where \"product_id\" > 10"; final String expected2 = "UPDATE \"foodmart\".\"product\" SET \"product_id\" = \"product_id\"" - + " + CHAR_LENGTH(\"product_name\"), \"product_name\" = CAST(\"product_name\" || '_' AS " - + "VARCHAR(60) CHARACTER SET \"ISO-8859-1\")\nWHERE \"product_id\" > 10"; + + " + CHAR_LENGTH(\"product_name\"), \"product_name\" = \"product_name\" || '_'" + + "\nWHERE \"product_id\" > 10"; sql(sql2).ok(expected2); final String sql3 = "update \"foodmart\".\"product\"\n" diff --git a/core/src/test/java/org/apache/calcite/test/JdbcAdapterTest.java b/core/src/test/java/org/apache/calcite/test/JdbcAdapterTest.java index 7bed49bb1037..5ecf63c5ba72 100644 --- a/core/src/test/java/org/apache/calcite/test/JdbcAdapterTest.java +++ b/core/src/test/java/org/apache/calcite/test/JdbcAdapterTest.java @@ -1465,21 +1465,21 @@ private LockWrapper exclusiveCleanDb(Connection c) throws SQLException { + " JdbcTableModify(table=[[foodmart, expense_fact]], operation=[MERGE]," + " updateColumnList=[[amount]], flattened=[false])\n" + " JdbcProject(STORE_ID=[$0], $f1=[666], $f2=[1997-01-01 00:00:00], $f3=[666]," - + " $f4=['666':VARCHAR(30)], $f5=[666], AMOUNT=[CAST($1):DECIMAL(10, 4) NOT NULL]," + + " $f4=['666':VARCHAR(30)], $f5=[666], AMOUNT=[$1]," + " store_id=[$2]," + " account_id=[$3], exp_date=[$4], time_id=[$5], category_id=[$6], currency_id=[$7]," - + " amount=[$8], AMOUNT0=[CAST($1):DECIMAL(10, 4) NOT NULL])\n" + + " amount=[$8], AMOUNT0=[$1])\n" + " JdbcJoin(condition=[=($2, $0)], joinType=[left])\n" + " JdbcValues(tuples=[[{ 666, 42 }]])\n" + " JdbcTableScan(table=[[foodmart, expense_fact]])\n"; final String jdbcSql = "MERGE INTO \"foodmart\".\"expense_fact\"\n" + "USING (VALUES (666, 42)) AS \"t\" (\"STORE_ID\", \"AMOUNT\")\n" + "ON \"t\".\"STORE_ID\" = \"expense_fact\".\"store_id\"\n" - + "WHEN MATCHED THEN UPDATE SET \"amount\" = CAST(\"t\".\"AMOUNT\" AS DECIMAL(10, 4))\n" + + "WHEN MATCHED THEN UPDATE SET \"amount\" = \"t\".\"AMOUNT\"\n" + "WHEN NOT MATCHED THEN INSERT (\"store_id\", \"account_id\", \"exp_date\", \"time_id\", " + "\"category_id\", \"currency_id\", \"amount\") VALUES \"t\".\"STORE_ID\",\n" + "666,\nTIMESTAMP '1997-01-01 00:00:00',\n666,\n'666',\n666,\n" - + "CAST(\"t\".\"AMOUNT\" AS DECIMAL(10, 4))"; + + "\"t\".\"AMOUNT\""; final AssertThat that = CalciteAssert.model(FoodmartSchema.FOODMART_MODEL) .enable(CalciteAssert.DB == DatabaseInstance.HSQLDB); diff --git a/core/src/test/java/org/apache/calcite/test/JdbcTest.java b/core/src/test/java/org/apache/calcite/test/JdbcTest.java index 0c5ab19b2ec4..83ce5b1d78a6 100644 --- a/core/src/test/java/org/apache/calcite/test/JdbcTest.java +++ b/core/src/test/java/org/apache/calcite/test/JdbcTest.java @@ -390,7 +390,7 @@ static void forEachExpand(Runnable r) { + "expr#7=[null:JavaType(class java.lang.Integer)], " + "empid=[$t3], deptno=[$t4], name=[$t5], salary=[$t6], " + "commission=[$t7])\n" - + " EnumerableValues(tuples=[[{ 'Fred', 56, 123.4000015258789E0 }]])\n"; + + " EnumerableValues(tuples=[[{ 'Fred', 56, 123.4 }]])\n"; assertThat(resultSet.getString(1), isLinux(expected)); // With named columns diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index d94ae4652659..60a5c56cba02 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -6053,7 +6053,7 @@ private void checkEmptyJoin(RelOptFixture f) { @Test void testReduceCastsNullable() { HepProgram program = new HepProgramBuilder() - // Simulate the way INSERT will insert casts to the target types + // Simulate the way INSERT can insert casts to the target types. .addRuleInstance( CoerceInputsRule.Config.DEFAULT .withCoerceNames(false) diff --git a/core/src/test/java/org/apache/calcite/test/TypeCoercionConverterTest.java b/core/src/test/java/org/apache/calcite/test/TypeCoercionConverterTest.java index b31e98267189..a12a07d5e57b 100644 --- a/core/src/test/java/org/apache/calcite/test/TypeCoercionConverterTest.java +++ b/core/src/test/java/org/apache/calcite/test/TypeCoercionConverterTest.java @@ -236,4 +236,36 @@ public static void checkActualAndReferenceFiles() { String sql = "select CAST(null AS INTEGER) union select '10'"; sql(sql).ok(); } + + /** Test case for + * [CALCITE-7607] + * Avoid implicit narrowing casts in DML assignment coercion. */ + @Test void testInsertVarcharNarrowingKeepsSourceType() { + final String sql = "insert into t1 select " + + "CAST('AVeryLongLongStringValue' AS VARCHAR(64)), " + + "t2_smallint, t2_int, t2_bigint,\n" + + "t2_real, t2_double, t2_decimal, t2_timestamp, " + + "t2_date, t2_binary, t2_boolean from t2"; + sql(sql).ok(); + } + + /** Test case for + * [CALCITE-7607] + * Avoid implicit narrowing casts in DML assignment coercion. */ + @Test void testUpdateVarcharNarrowingKeepsSourceType() { + final String sql = "update t1 set t1_varchar20 = " + + "CAST('AVeryLongLongStringValue' AS VARCHAR(64))"; + sql(sql).ok(); + } + + /** Test case for + * [CALCITE-7607] + * Avoid implicit narrowing casts in DML assignment coercion. */ + @Test void testMergeVarcharNarrowingKeepsSourceType() { + final String sql = "merge into t1\n" + + "using t2 on t1.t1_smallint = t2.t2_smallint\n" + + "when matched then update set t1_varchar20 = " + + "CAST('AVeryLongLongStringValue' AS VARCHAR(64))"; + sql(sql).ok(); + } } diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index 24d6a16f3fcf..d7c03e93f507 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -16542,14 +16542,14 @@ select empno, cast(job as varchar(128)) from sales.empnullables]]> diff --git a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml index f2ab8fec37e7..c36f7ad59bfe 100644 --- a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml @@ -5597,7 +5597,7 @@ values(t.empno, t.ename, 10, t.sal * .15)]]> ($7, 5)]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) ]]> diff --git a/core/src/test/resources/org/apache/calcite/test/TypeCoercionConverterTest.xml b/core/src/test/resources/org/apache/calcite/test/TypeCoercionConverterTest.xml index 8e9d0207ca2d..15512cb61550 100644 --- a/core/src/test/resources/org/apache/calcite/test/TypeCoercionConverterTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/TypeCoercionConverterTest.xml @@ -140,7 +140,7 @@ t2_double, t2_decimal, t2_int, t2_date, t2_timestamp, t2_varchar20, t2_int from ($2, 0)]) + LogicalProject(t1_varchar20=[CAST($1):VARCHAR(20) NOT NULL], t1_smallint=[$2], t1_int=[$3], t1_bigint=[$4], t1_real=[$5], t1_double=[$6], t1_decimal=[CAST($2):DECIMAL(19, 0) NOT NULL], t1_timestamp=[CAST($8):TIMESTAMP(0) NOT NULL], t1_date=[CAST($7):DATE NOT NULL], t1_binary=[CAST($0):BINARY(1) NOT NULL], t1_boolean=[<>($2, 0)]) LogicalTableScan(table=[[CATALOG, SALES, T2]]) ]]> @@ -152,7 +152,7 @@ LogicalTableModify(table=[[CATALOG, SALES, T1]], operation=[INSERT], flattened=[ @@ -164,6 +164,19 @@ LogicalTableModify(table=[[CATALOG, SALES, T1]], operation=[INSERT], flattened=[ + + + + + + + + @@ -206,6 +219,22 @@ LogicalProject(X=[$0]) LogicalAggregate(group=[{0}]) LogicalProject(X=[$0]) LogicalValues(tuples=[[{ 3 }, { 4 }]]) +]]> + + + + + + + + @@ -282,7 +311,19 @@ LogicalUnion(all=[false]) + + + + + + + + diff --git a/server/src/test/resources/sql/materialized_view.iq b/server/src/test/resources/sql/materialized_view.iq index 8a8b12992f81..ca18a8ccf674 100644 --- a/server/src/test/resources/sql/materialized_view.iq +++ b/server/src/test/resources/sql/materialized_view.iq @@ -19,7 +19,7 @@ !set outputformat mysql # Create a source table -create table dept (deptno int not null, name varchar(10)); +create table dept (deptno int not null, name varchar(20)); (0 rows modified) !update @@ -39,12 +39,12 @@ select * from dept where deptno > 10; # Check contents select * from v; -+--------+------------+ -| DEPTNO | NAME | -+--------+------------+ -| 20 | Marketing | -| 30 | Engineerin | -+--------+------------+ ++--------+-------------+ +| DEPTNO | NAME | ++--------+-------------+ +| 20 | Marketing | +| 30 | Engineering | ++--------+-------------+ (2 rows) !ok @@ -64,12 +64,12 @@ select * from dept where deptno < 30; # Check contents are unchanged select * from v; -+--------+------------+ -| DEPTNO | NAME | -+--------+------------+ -| 20 | Marketing | -| 30 | Engineerin | -+--------+------------+ ++--------+-------------+ +| DEPTNO | NAME | ++--------+-------------+ +| 20 | Marketing | +| 30 | Engineering | ++--------+-------------+ (2 rows) !ok diff --git a/server/src/test/resources/sql/table_as.iq b/server/src/test/resources/sql/table_as.iq index cf2450991422..8e16f4c5a944 100644 --- a/server/src/test/resources/sql/table_as.iq +++ b/server/src/test/resources/sql/table_as.iq @@ -19,7 +19,7 @@ !set outputformat mysql # Create a source table -create table dept (deptno int not null, name varchar(10)); +create table dept (deptno int not null, name varchar(20)); (0 rows modified) !update @@ -37,14 +37,14 @@ select * from dept where deptno > 10; !update -# Check contents; "Engineering" is too long for varchar(10) +# Check contents select * from d; -+--------+------------+ -| DEPTNO | NAME | -+--------+------------+ -| 20 | Marketing | -| 30 | Engineerin | -+--------+------------+ ++--------+-------------+ +| DEPTNO | NAME | ++--------+-------------+ +| 20 | Marketing | +| 30 | Engineering | ++--------+-------------+ (2 rows) !ok @@ -64,12 +64,12 @@ select * from dept where deptno < 30; # Check contents are unchanged select * from d; -+--------+------------+ -| DEPTNO | NAME | -+--------+------------+ -| 20 | Marketing | -| 30 | Engineerin | -+--------+------------+ ++--------+-------------+ +| DEPTNO | NAME | ++--------+-------------+ +| 20 | Marketing | +| 30 | Engineering | ++--------+-------------+ (2 rows) !ok From cf62eba18ba81f586e82ce9f522afaa71a4ef15b Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Mon, 6 Jul 2026 14:45:57 -0700 Subject: [PATCH 395/562] [CALCITE-7646] CorrelateProjectExtractor does not handle nested field accesses cor0.field0.field1 Signed-off-by: Mihai Budiu --- .../sql2rel/CorrelateProjectExtractor.java | 77 ++++++++++++--- .../CorrelateProjectExtractorTest.java | 72 ++++++++++++++ .../calcite/sql2rel/RelDecorrelatorTest.java | 99 +++++++++++++++---- .../apache/calcite/test/RelOptRulesTest.xml | 20 ++-- .../calcite/test/SqlToRelConverterTest.xml | 40 ++++---- core/src/test/resources/sql/lateral.iq | 94 ++++++++++++++++++ 6 files changed, 341 insertions(+), 61 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql2rel/CorrelateProjectExtractor.java b/core/src/main/java/org/apache/calcite/sql2rel/CorrelateProjectExtractor.java index 50d255c3312d..127f4e487941 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/CorrelateProjectExtractor.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/CorrelateProjectExtractor.java @@ -88,6 +88,20 @@ public CorrelateProjectExtractor(RelBuilderFactory factory) { this.builderFactory = factory; } + /** Returns whether {@code node} is a direct field access on the correlation + * variable with the specified id, such as {@code $cor0.DEPTNO}. A nested + * access such as {@code $cor0.REC.DEPTNO} is not direct. */ + private static boolean isDirectFieldAccess(RexNode node, CorrelationId id) { + if (node instanceof RexFieldAccess) { + RexFieldAccess access = (RexFieldAccess) node; + if (access.getReferenceExpr() instanceof RexCorrelVariable) { + RexCorrelVariable correlVar = (RexCorrelVariable) access.getReferenceExpr(); + return correlVar.id.equals(id); + } + } + return false; + } + @Override public RelNode visit(LogicalCorrelate correlate) { RelNode left = correlate.getLeft().accept(this); RelNode right = correlate.getRight().accept(this); @@ -95,8 +109,12 @@ public CorrelateProjectExtractor(RelBuilderFactory factory) { // Find the correlated expressions from the right side that can be moved to the left Set callsWithCorrelationInRight = findCorrelationDependentCalls(correlate.getCorrelationId(), right); + // Only direct field accesses on the correlation variable, such as + // $cor0.DEPTNO, are left in place. A nested field access, such as + // $cor0.REC.DEPTNO, is extracted boolean isTrivialCorrelation = - callsWithCorrelationInRight.stream().allMatch(exp -> exp instanceof RexFieldAccess); + callsWithCorrelationInRight.stream() + .allMatch(exp -> isDirectFieldAccess(exp, correlate.getCorrelationId())); // Early exit condition if (isTrivialCorrelation) { if (correlate.getLeft().equals(left) && correlate.getRight().equals(right)) { @@ -116,27 +134,42 @@ public CorrelateProjectExtractor(RelBuilderFactory factory) { // Transform the correlated expression from the right side to an expression over the left side builder.push(left); + ImmutableBitSet.Builder requiredColumns = ImmutableBitSet.builder(); List callsWithCorrelationOverLeft = new ArrayList<>(); for (RexNode callInRight : callsWithCorrelationInRight) { - callsWithCorrelationOverLeft.add(replaceCorrelationsWithInputRef(callInRight, builder)); + if (isDirectFieldAccess(callInRight, correlate.getCorrelationId())) { + // Direct field accesses stay in the right side and keep reading their + // original left column; that column must remain a required column. + requiredColumns.set(((RexFieldAccess) callInRight).getField().getIndex()); + } else { + callsWithCorrelationOverLeft.add(replaceCorrelationsWithInputRef(callInRight, builder)); + } } builder.projectPlus(callsWithCorrelationOverLeft); // Construct the mapping to transform the expressions in the right side based on the new // projection in the left side. Map transformMapping = new HashMap<>(); + int newFieldIndex = oldLeft; for (RexNode callInRight : callsWithCorrelationInRight) { - RexBuilder xb = builder.getRexBuilder(); - RexNode v = xb.makeCorrel(builder.peek().getRowType(), correlate.getCorrelationId()); - RexNode flatCorrelationInRight = xb.makeFieldAccess(v, oldLeft + transformMapping.size()); - transformMapping.put(callInRight, flatCorrelationInRight); + if (!isDirectFieldAccess(callInRight, correlate.getCorrelationId())) { + RexBuilder xb = builder.getRexBuilder(); + RexNode v = xb.makeCorrel(builder.peek().getRowType(), correlate.getCorrelationId()); + RexNode flatCorrelationInRight = xb.makeFieldAccess(v, newFieldIndex); + transformMapping.put(callInRight, flatCorrelationInRight); + newFieldIndex++; + } } - // Select the required fields/columns from the left side of the correlation. Based on the code - // above all these fields should be at the end of the left relational expression. + // Select the required fields/columns from the left side of the correlation: the columns + // read by the direct field accesses plus the newly projected columns, which are at the + // end of the left relational expression. List requiredFields = builder.fields( - ImmutableBitSet.range(oldLeft, oldLeft + callsWithCorrelationOverLeft.size()).asList()); + requiredColumns + .set(oldLeft, oldLeft + callsWithCorrelationOverLeft.size()) + .build() + .asList()); final int newLeft = builder.fields().size(); // Transform the expressions in the right side using the mapping constructed earlier. @@ -264,8 +297,13 @@ private static boolean isSimpleCorrelatedExpression(RexNode node, CorrelationId * +(10, $cor0.DEPTNO) -> TRUE * /(100,+(10, $cor0.DEPTNO)) -> TRUE * CAST(+(10, $cor0.DEPTNO)):INTEGER NOT NULL -> TRUE + * CASE(IS NOT NULL($cor0.PATH), $cor0.PATH, ARRAY(null:INTEGER)) -> TRUE * +($0, $cor0.DEPTNO) -> FALSE * } + * + *

      A subexpression built only from literals and dynamic parameters, such + * as {@code ARRAY(null:INTEGER)} above, is neutral: it neither qualifies nor + * disqualifies the enclosing call. */ private static class SimpleCorrelationDetector extends RexVisitorImpl<@Nullable Boolean> { @@ -284,7 +322,8 @@ private SimpleCorrelationDetector(CorrelationId corrId) { return Boolean.FALSE; } - @Override public Boolean visitCall(RexCall call) { + @Override public @Nullable Boolean visitCall(RexCall call) { + // Constant operands must not disqualify the call Boolean hasSimpleCorrelation = null; for (RexNode op : call.operands) { Boolean b = op.accept(this); @@ -292,7 +331,8 @@ private SimpleCorrelationDetector(CorrelationId corrId) { hasSimpleCorrelation = hasSimpleCorrelation == null ? b : hasSimpleCorrelation && b; } } - return hasSimpleCorrelation == null ? Boolean.FALSE : hasSimpleCorrelation; + // If unsure return null; caller will decide + return hasSimpleCorrelation; } @Override public @Nullable Boolean visitFieldAccess(RexFieldAccess fieldAccess) { @@ -332,8 +372,10 @@ private static RexNode replaceCorrelationsWithInputRef(RexNode exp, RelBuilder b } /** - * A visitor traversing row expressions and replacing calls with other expressions according - * to the specified mapping. + * A visitor traversing row expressions and replacing calls and field + * accesses with other expressions according to the specified mapping. + * The mapping is consulted before recursing so that the outermost + * matching expression wins. */ private static final class CallReplacer extends RexShuttle { private final Map mapping; @@ -350,5 +392,14 @@ private static final class CallReplacer extends RexShuttle { return super.visitCall(oldCall); } } + + @Override public RexNode visitFieldAccess(RexFieldAccess fieldAccess) { + RexNode replacement = mapping.get(fieldAccess); + if (replacement != null) { + return replacement; + } else { + return super.visitFieldAccess(fieldAccess); + } + } } } diff --git a/core/src/test/java/org/apache/calcite/sql2rel/CorrelateProjectExtractorTest.java b/core/src/test/java/org/apache/calcite/sql2rel/CorrelateProjectExtractorTest.java index 72276d42c332..da0a17296b3c 100644 --- a/core/src/test/java/org/apache/calcite/sql2rel/CorrelateProjectExtractorTest.java +++ b/core/src/test/java/org/apache/calcite/sql2rel/CorrelateProjectExtractorTest.java @@ -84,6 +84,78 @@ public static Frameworks.ConfigBuilder config() { assertThat(after, hasTree(planAfter)); } + /** Test case for [CALCITE-7646] + * CorrelateProjectExtractor does not handle nested field accesses cor0.field0.field1. */ + @Test void testNestedCorrelationFieldAccessInFilter() { + final RelBuilder builder = RelBuilder.create(config().build()); + final Holder<@Nullable RexCorrelVariable> v = Holder.empty(); + RelNode before = builder.scan("EMP") + .project( + builder.alias( + builder.call(SqlStdOperatorTable.ROW, + builder.field("EMPNO"), builder.field("DEPTNO")), "R")) + .variable(v::set) + .scan("DEPT") + .filter( + builder.equals(builder.field(0), + builder.getRexBuilder().makeFieldAccess(builder.field(v.get(), "R"), 1))) + .correlate(JoinRelType.LEFT, v.get().id, builder.field(2, 0, "R")) + .build(); + + final String planBefore = "" + + "LogicalCorrelate(correlation=[$cor0], joinType=[left], requiredColumns=[{0}])\n" + + " LogicalProject(R=[ROW($0, $7)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalFilter(condition=[=($0, $cor0.R.EXPR$1)])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n"; + assertThat(before, hasTree(planBefore)); + + RelNode after = before.accept(new CorrelateProjectExtractor(RelFactories.LOGICAL_BUILDER)); + final String planAfter = "" + + "LogicalProject(R=[$0], DEPTNO=[$2], DNAME=[$3], LOC=[$4])\n" + + " LogicalCorrelate(correlation=[$cor0], joinType=[left], requiredColumns=[{1}])\n" + + " LogicalProject(R=[ROW($0, $7)], $f1=[ROW($0, $7).EXPR$1])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalFilter(condition=[=($0, $cor0.$f1)])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n"; + assertThat(after, hasTree(planAfter)); + } + + /** Tests that a constant call operand, such as {@code POWER(2, 3)}, does + * not prevent extracting the enclosing correlated call. */ + @Test void testCorrelationCallWithConstantCallOperandInFilter() { + final RelBuilder builder = RelBuilder.create(config().build()); + final Holder<@Nullable RexCorrelVariable> v = Holder.empty(); + RelNode before = builder.scan("EMP") + .variable(v::set) + .scan("DEPT") + .filter( + builder.equals(builder.field(0), + builder.call(SqlStdOperatorTable.PLUS, + builder.call(SqlStdOperatorTable.POWER, + builder.literal(2), builder.literal(3)), + builder.field(v.get(), "DEPTNO")))) + .correlate(JoinRelType.LEFT, v.get().id, builder.field(2, 0, "DEPTNO")) + .build(); + + final String planBefore = "" + + "LogicalCorrelate(correlation=[$cor0], joinType=[left], requiredColumns=[{7}])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalFilter(condition=[=($0, +(POWER(2, 3), $cor0.DEPTNO))])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n"; + assertThat(before, hasTree(planBefore)); + + RelNode after = before.accept(new CorrelateProjectExtractor(RelFactories.LOGICAL_BUILDER)); + final String planAfter = "" + + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7], DEPTNO0=[$9], DNAME=[$10], LOC=[$11])\n" + + " LogicalCorrelate(correlation=[$cor0], joinType=[left], requiredColumns=[{8}])\n" + + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7], $f8=[+(POWER(2, 3), $7)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalFilter(condition=[=($0, $cor0.$f8)])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n"; + assertThat(after, hasTree(planAfter)); + } + @Test void testDoubleCorrelationCallOverVariableInFilters() { final RelBuilder builder = RelBuilder.create(config().build()); final Holder<@Nullable RexCorrelVariable> v = Holder.empty(); diff --git a/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java b/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java index 240f9a36ae02..ac218b220afa 100644 --- a/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java +++ b/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java @@ -290,9 +290,9 @@ public static Frameworks.ConfigBuilder config() { + " LogicalJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left])\n" + " LogicalValues(tuples=[[{ 7369 }, { 7499 }]])\n" + " LogicalAggregate(group=[{0}], agg#0=[SINGLE_VALUE($1)])\n" - + " LogicalProject(EMPNO1=[$12], EXPR$0=[||(||($1, ' from dept '), $13)])\n" - + " LogicalJoin(condition=[AND(=($7, $10), =($9, $11))], joinType=[left])\n" - + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7], DEPTNO0=[$7], EMPNO0=[CAST($0):INTEGER NOT NULL])\n" + + " LogicalProject(EMPNO1=[$11], EXPR$0=[||(||($1, ' from dept '), $12)])\n" + + " LogicalJoin(condition=[AND(=($7, $9), =($8, $10))], joinType=[left])\n" + + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7], EMPNO0=[CAST($0):INTEGER NOT NULL])\n" + " LogicalTableScan(table=[[scott, EMP]])\n" + " LogicalAggregate(group=[{0, 1, 2}], agg#0=[SINGLE_VALUE($3)])\n" + " LogicalProject(DEPTNO0=[$3], EMPNO0=[$4], EMPNO=[$5], DNAME=[$1])\n" @@ -556,29 +556,29 @@ public static Frameworks.ConfigBuilder config() { // LogicalTableScan(table=[[scott, EMP]]) final String planAfter = "" + "LogicalSort(sort0=[$0], dir0=[ASC])\n" - + " LogicalProject(DNAME=[$1], C=[$7])\n" - + " LogicalJoin(condition=[AND(=($0, $5), =($4, $6))], joinType=[left])\n" - + " LogicalProject(DEPTNO=[$0], DNAME=[$1], LOC=[$2], DEPTNO0=[$0], $f4=[*($0, 100)])\n" + + " LogicalProject(DNAME=[$1], C=[$6])\n" + + " LogicalJoin(condition=[AND(=($0, $4), =($3, $5))], joinType=[left])\n" + + " LogicalProject(DEPTNO=[$0], DNAME=[$1], LOC=[$2], $f3=[*($0, 100)])\n" + " LogicalTableScan(table=[[scott, DEPT]])\n" - + " LogicalProject(DEPTNO8=[$0], $f4=[$1], EXPR$0=[CASE(IS NOT NULL($4), $4, 0)])\n" + + " LogicalProject(DEPTNO8=[$0], $f3=[$1], EXPR$0=[CASE(IS NOT NULL($4), $4, 0)])\n" + " LogicalJoin(condition=[AND(IS NOT DISTINCT FROM($0, $2), IS NOT DISTINCT FROM($1, $3))], joinType=[left])\n" - + " LogicalProject(DEPTNO=[$0], $f4=[*($0, 100)])\n" + + " LogicalProject(DEPTNO=[$0], $f3=[*($0, 100)])\n" + " LogicalTableScan(table=[[scott, DEPT]])\n" + " LogicalAggregate(group=[{0, 1}], EXPR$0=[COUNT()])\n" - + " LogicalProject(DEPTNO8=[$7], $f4=[$9])\n" + + " LogicalProject(DEPTNO8=[$7], $f3=[$9])\n" + " LogicalFilter(condition=[IS NOT NULL($7)])\n" - + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7], i=[$11], $f4=[$9])\n" + + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7], i=[$11], $f3=[$9])\n" + " LogicalJoin(condition=[=($8, $10)], joinType=[inner])\n" + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7], SAL0=[CAST($5):DECIMAL(12, 2)])\n" + " LogicalTableScan(table=[[scott, EMP]])\n" - + " LogicalProject($f4=[$0], SAL0=[$1], $f2=[true])\n" + + " LogicalProject($f3=[$0], SAL0=[$1], $f2=[true])\n" + " LogicalAggregate(group=[{0, 1}])\n" - + " LogicalProject($f4=[$1], SAL0=[$2])\n" + + " LogicalProject($f3=[$1], SAL0=[$2])\n" + " LogicalJoin(condition=[AND(>($2, CAST($0):DECIMAL(12, 2) NOT NULL), <($1, $0))], joinType=[inner])\n" + " LogicalValues(tuples=[[{ 1000 }, { 2000 }, { 3000 }]])\n" + " LogicalJoin(condition=[true], joinType=[inner])\n" + " LogicalAggregate(group=[{0}])\n" - + " LogicalProject($f4=[*($0, 100)])\n" + + " LogicalProject($f3=[*($0, 100)])\n" + " LogicalTableScan(table=[[scott, DEPT]])\n" + " LogicalAggregate(group=[{0}])\n" + " LogicalProject(SAL0=[CAST($5):DECIMAL(12, 2)])\n" @@ -1793,9 +1793,9 @@ public static Frameworks.ConfigBuilder config() { RelDecorrelator.decorrelateQuery(before, builder, RuleSets.ofList(Collections.emptyList()), RuleSets.ofList(Collections.emptyList())); final String planAfter = "" - + "LogicalProject(DEPTNO=[$0], I0=[$3], I1=[$8])\n" - + " LogicalJoin(condition=[AND(=($0, $6), =($5, $7))], joinType=[left])\n" - + " LogicalProject(DEPTNO=[$0], DNAME=[$1], LOC=[$2], EXPR$0=[$5], DEPTNO0=[$0], $f5=[>(CAST($0):INTEGER NOT NULL, 0)])\n" + + "LogicalProject(DEPTNO=[$0], I0=[$3], I1=[$7])\n" + + " LogicalJoin(condition=[AND(=($0, $5), =($4, $6))], joinType=[left])\n" + + " LogicalProject(DEPTNO=[$0], DNAME=[$1], LOC=[$2], EXPR$0=[$5], $f4=[>(CAST($0):INTEGER NOT NULL, 0)])\n" + " LogicalJoin(condition=[=($3, $4)], joinType=[left])\n" + " LogicalProject(DEPTNO=[$0], DNAME=[$1], LOC=[$2], DEPTNO0=[CAST($0):SMALLINT NOT NULL])\n" + " LogicalTableScan(table=[[scott, DEPT]])\n" @@ -1807,12 +1807,12 @@ public static Frameworks.ConfigBuilder config() { + " LogicalProject(DEPTNO0=[CAST($0):SMALLINT NOT NULL])\n" + " LogicalTableScan(table=[[scott, DEPT]])\n" + " LogicalAggregate(group=[{0, 1}], EXPR$0=[MIN($2)])\n" - + " LogicalProject(DEPTNO0=[$8], $f5=[$9], $f0=[0])\n" + + " LogicalProject(DEPTNO0=[$8], $f4=[$9], $f0=[0])\n" + " LogicalJoin(condition=[=($7, $8)], joinType=[inner])\n" + " LogicalFilter(condition=[=($1, 'SMITH')])\n" + " LogicalTableScan(table=[[scott, EMP]])\n" + " LogicalFilter(condition=[$1])\n" - + " LogicalProject(DEPTNO=[$0], $f5=[>(CAST($0):INTEGER NOT NULL, 0)])\n" + + " LogicalProject(DEPTNO=[$0], $f4=[>(CAST($0):INTEGER NOT NULL, 0)])\n" + " LogicalJoin(condition=[=($3, $4)], joinType=[left])\n" + " LogicalProject(DEPTNO=[$0], DNAME=[$1], LOC=[$2], DEPTNO0=[CAST($0):SMALLINT NOT NULL])\n" + " LogicalTableScan(table=[[scott, DEPT]])\n" @@ -2351,4 +2351,67 @@ public static Frameworks.ConfigBuilder config() { + " LogicalTableScan(table=[[scott, EMP]])\n"; assertThat(after, hasTree(planAfter)); } + + /** Test case for + * [CALCITE-7646] + * CorrelateProjectExtractor does not handle nested field accesses cor0.field0.field1. */ + @Test void testNestedCorrelatedFieldAccess() throws SqlParseException { + final String sql = "select a.\"aid\", t.lat\n" + + "from \"bookstore\".\"authors\" a,\n" + + "lateral (select b.\"aid\" as c,\n" + + " (a.\"birthPlace\").\"coords\".\"latitude\" as lat\n" + + " from \"bookstore\".\"authors\" b\n" + + " where b.\"aid\" = a.\"aid\") as t"; + SchemaPlus rootSchema = Frameworks.createRootSchema(true); + CalciteAssert.addSchema(rootSchema, CalciteAssert.SchemaSpec.BOOKSTORE); + CalciteConnectionConfig config = new CalciteConnectionConfigImpl(new Properties()); + // The Frameworks planner cannot be used here because it flattens + // structured types, and RelStructuredTypeFlattener does not support + // correlations on structured columns. + SqlTestFactory factory = SqlTestFactory.INSTANCE + .withCatalogReader((typeFactory, caseSensitive) -> + new CalciteCatalogReader( + CalciteSchema.from(rootSchema), + ImmutableList.of("bookstore"), + typeFactory, + config)); + SqlParser parser = factory.createParser(sql); + SqlNode parsed = parser.parseQuery(); + final SqlToRelConverter sqlToRelConverter = factory.createSqlToRelConverter(); + assert sqlToRelConverter.validator != null; + final SqlNode validated = sqlToRelConverter.validator.validate(parsed); + final RelNode before = sqlToRelConverter.convertQuery(validated, false, true).rel; + + final String planBefore = "" + + "LogicalProject(aid=[$0], LAT=[$5])\n" + + " LogicalCorrelate(correlation=[$cor1], joinType=[inner], requiredColumns=[{0, 2}])\n" + + " LogicalTableScan(table=[[bookstore, authors]])\n" + + " LogicalProject(C=[$0], LAT=[$cor1.birthPlace.coords.latitude])\n" + + " LogicalFilter(condition=[=($0, $cor1.aid)])\n" + + " LogicalTableScan(table=[[bookstore, authors]])\n"; + assertThat(before, hasTree(planBefore)); + + final RelBuilder relBuilder = + RelFactories.LOGICAL_BUILDER.create(before.getCluster(), null); + // Decorrelate without any rules, just "purely" decorrelation algorithm on RelDecorrelator + final RelNode after = + RelDecorrelator.decorrelateQuery(before, relBuilder, + RuleSets.ofList(Collections.emptyList()), + RuleSets.ofList(Collections.emptyList())); + + // The nested field access is extracted into the projection $f4 on the + // left side and no correlation variables remain. + final String planAfter = "" + + "LogicalProject(aid=[$0], LAT=[$6])\n" + + " LogicalJoin(condition=[AND(=($0, $7), IS NOT DISTINCT FROM($4, $8))], joinType=[inner])\n" + + " LogicalProject(aid=[$0], name=[$1], birthPlace=[$2], books=[$3], $f4=[$2.coords.latitude])\n" + + " LogicalTableScan(table=[[bookstore, authors]])\n" + + " LogicalProject(C=[$0], LAT=[$4], aid=[$0], $f4=[$4])\n" + + " LogicalJoin(condition=[true], joinType=[inner])\n" + + " LogicalTableScan(table=[[bookstore, authors]])\n" + + " LogicalAggregate(group=[{0}])\n" + + " LogicalProject($f4=[$2.coords.latitude])\n" + + " LogicalTableScan(table=[[bookstore, authors]])\n"; + assertThat(after, hasTree(planAfter)); + } } diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index d7c03e93f507..1e37b2699b3a 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -3097,9 +3097,9 @@ LogicalProject(NAME=[$1], EXPR$1=[$2]) ($5, 1000))]) LogicalTableScan(table=[[CATALOG, SALES, EMPNULLABLES]]) LogicalFilter(condition=[=($1, $0)]) LogicalAggregate(group=[{0, 1, 2}]) - LogicalProject(SAL=[$5], SAL0=[$8], $f9=[$9]) + LogicalProject(SAL=[$5], SAL0=[$8], $f8=[$9]) LogicalJoin(condition=[OR(=($8, $5), $9)], joinType=[inner]) LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5], COMM=[$6], SLACKER=[$8]) LogicalFilter(condition=[AND(=($7, 20), >($5, 1000))]) LogicalTableScan(table=[[CATALOG, SALES, EMPNULLABLES]]) LogicalAggregate(group=[{0, 1}]) - LogicalProject(SAL=[$5], $f9=[=($5, 4)]) + LogicalProject(SAL=[$5], $f8=[=($5, 4)]) LogicalFilter(condition=[AND(=($7, 20), >($5, 1000))]) LogicalTableScan(table=[[CATALOG, SALES, EMPNULLABLES]]) ]]> @@ -9212,9 +9212,9 @@ LEFT JOIN LATERAL ( @@ -1796,18 +1796,18 @@ cross join lateral @@ -5222,9 +5222,9 @@ LogicalProject(C=[$0], D=[$1], C0=[$2]) diff --git a/core/src/test/resources/sql/lateral.iq b/core/src/test/resources/sql/lateral.iq index 5c82727b8930..4c4ffbe17072 100644 --- a/core/src/test/resources/sql/lateral.iq +++ b/core/src/test/resources/sql/lateral.iq @@ -244,4 +244,98 @@ where job = 'MANAGER'; !ok +# 3 test cases for [CALCITE-7646] CorrelateProjectExtractor +# does not handle nested field accesses cor0.field0.field1. + +# All queries use LATERAL, which converts directly to a Correlate. +# The results were validated on Postgres + +!use scott + +select t.dd, t.x +from dept d, +lateral (select d.deptno as dd, u.x + from unnest(array[d.deptno + 100]) as u(x)) as t +where d.dname = 'SALES'; ++----+-----+ +| DD | X | ++----+-----+ +| 30 | 130 | ++----+-----+ +(1 row) + +!ok + +select t.dd, t.dd1, t.x +from dept d, +lateral (select d.deptno as dd, d.deptno + 1 as dd1, u.x + from unnest(array[1, 2]) as u(x)) as t +where d.dname = 'SALES'; ++----+-----+---+ +| DD | DD1 | X | ++----+-----+---+ +| 30 | 31 | 1 | +| 30 | 31 | 2 | ++----+-----+---+ +(2 rows) + +!ok +!if (use_old_decorr) { +# The correlated computation d.deptno + 1 (DD1) has been extracted into the left +# input of the EnumerableNestedLoopJoin, as $f3. The right input, +# UNNEST(ARRAY[1, 2]), references no correlation variable, so decorrelation +# replaces the Correlate with a join. +EnumerableCalc(expr#0..4=[{inputs}], proj#0..2=[{exprs}]) + EnumerableHashJoin(condition=[AND(=($3, $5), =($4, $6))], joinType=[semi]) + EnumerableCalc(expr#0..2=[{inputs}], proj#0..2=[{exprs}], DEPTNO=[$t0], $f3=[$t1]) + EnumerableNestedLoopJoin(condition=[true], joinType=[inner]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1], expr#4=[+($t0, $t3)], expr#5=['SALES':VARCHAR(14)], expr#6=[=($t1, $t5)], DEPTNO=[$t0], $f3=[$t4], $condition=[$t6]) + EnumerableTableScan(table=[[scott, DEPT]]) + EnumerableUncollect + EnumerableCalc(expr#0=[{inputs}], expr#1=[1], expr#2=[2], expr#3=[ARRAY($t1, $t2)], EXPR$0=[$t3]) + EnumerableValues(tuples=[[{ 0 }]]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1], expr#4=[+($t0, $t3)], expr#5=['SALES':VARCHAR(14)], expr#6=[=($t1, $t5)], DEPTNO=[$t0], $f3=[$t4], $condition=[$t6]) + EnumerableTableScan(table=[[scott, DEPT]]) +!plan +!} + +# COALESCE(d.path, ARRAY[CAST(NULL AS INTEGER)]) converts to +# CASE(IS NOT NULL($cor0.PATH), $cor0.PATH, ARRAY(null:INTEGER)). The constant +# ARRAY(null:INTEGER) operand must not prevent extracting the CASE to the +# left input of the Correlate operator +select d.deptno, t.x +from (select deptno, + case when deptno = 10 then array[deptno, deptno + 1] end as path + from dept) as d, +lateral (select * from unnest(coalesce(d.path, array[cast(null as integer)])) as u(x)) as t +order by d.deptno, t.x; ++--------+----+ +| DEPTNO | X | ++--------+----+ +| 10 | 10 | +| 10 | 11 | +| 20 | | +| 30 | | +| 40 | | ++--------+----+ +(5 rows) + +!ok +!if (use_old_decorr) { +# The entire CASE produced by COALESCE has been extracted into the left input of the +# EnumerableCorrelate, as $f2. The right input reads the array through $cor0.$f2. +# The query cannot be decorrelated because of the remaining Uncollect. +EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) + EnumerableCalc(expr#0..2=[{inputs}], DEPTNO=[$t0], X=[$t2]) + EnumerableCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{1}]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[CAST($t0):INTEGER NOT NULL], expr#4=[10], expr#5=[=($t3, $t4)], expr#6=[1], expr#7=[+($t0, $t6)], expr#8=[ARRAY($t3, $t7)], expr#9=[null:INTEGER NOT NULL ARRAY], expr#10=[CASE($t5, $t8, $t9)], expr#11=[IS NOT NULL($t10)], expr#12=[CAST($t10):INTEGER NOT NULL ARRAY NOT NULL], expr#13=[CAST($t12):INTEGER ARRAY NOT NULL], expr#14=[null:INTEGER], expr#15=[ARRAY($t14)], expr#16=[CASE($t11, $t13, $t15)], DEPTNO=[$t0], $f2=[$t16]) + EnumerableTableScan(table=[[scott, DEPT]]) + EnumerableUncollect + EnumerableCalc(expr#0=[{inputs}], expr#1=[$cor0], expr#2=[$t1.$f2], EXPR$0=[$t2]) + EnumerableValues(tuples=[[{ 0 }]]) +!plan +!} + +!set planner-rules original + # End lateral.iq From ab5e7f6e3903d91bc4472fb533cdfb0690088fc6 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Wed, 15 Jul 2026 15:54:25 -0700 Subject: [PATCH 396/562] [CALCITE-7658] Type checker rejects CAST(ARRAY() AS ROW(x INT) ARRAY) Signed-off-by: Mihai Budiu --- .../enumerable/RexToLixTranslator.java | 16 ++++ .../calcite/sql/fun/SqlCastFunction.java | 2 +- .../calcite/sql/type/SqlTypeFactoryImpl.java | 31 +++++- .../apache/calcite/sql/type/SqlTypeUtil.java | 15 ++- .../rel/rel2sql/RelToSqlConverterTest.java | 14 +-- .../calcite/sql/type/SqlTypeFactoryTest.java | 36 +++++++ .../apache/calcite/test/SqlValidatorTest.java | 10 +- .../apache/calcite/test/SqlOperatorTest.java | 95 ++++++++++++++++++- 8 files changed, 195 insertions(+), 24 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java index 5a1e0fee2082..a39146ac754c 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java @@ -341,6 +341,16 @@ Expression translateCast( return expressionHandlingSafe(convert3, safe, targetType); } + /** Returns whether every runtime value of {@code type} is null. + * This holds for the NULL type, which describes untyped NULL literals, + * and for the UNKNOWN type, e.g. the element type inferred for the + * no-argument array constructor ARRAY(). Such values are never + * created, but the code generated needs to typecheck. */ + private static boolean valueIsAlwaysNull(RelDataType type) { + SqlTypeName typeName = type.getSqlTypeName(); + return typeName == SqlTypeName.UNKNOWN || typeName == SqlTypeName.NULL; + } + private Expression getConvertExpression( RelDataType sourceType, RelDataType targetType, @@ -366,6 +376,9 @@ private Expression getConvertExpression( } if (targetType.getSqlTypeName() == SqlTypeName.ROW) { + if (valueIsAlwaysNull(sourceType)) { + return Expressions.constant(null); + } assert sourceType.getSqlTypeName() == SqlTypeName.ROW; List targetTypes = targetType.getFieldList(); List sourceTypes = sourceType.getFieldList(); @@ -397,6 +410,9 @@ private Expression getConvertExpression( switch (targetType.getSqlTypeName()) { case ARRAY: case MULTISET: + if (valueIsAlwaysNull(sourceType)) { + return Expressions.constant(null); + } final RelDataType sourceDataType = sourceType.getComponentType(); final RelDataType targetDataType = targetType.getComponentType(); assert sourceDataType != null; diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlCastFunction.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlCastFunction.java index b25a9b8fdd57..62ce488718ba 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlCastFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlCastFunction.java @@ -200,7 +200,7 @@ private static RelDataType createTypeWithNullabilityFromExpr(RelDataTypeFactory RelDataType valueType = createTypeWithNullabilityFromExpr( typeFactory, expressionValueType, targetValueType, safe); - SqlTypeUtil.createMapType(typeFactory, keyType, valueType, isNullable); + return SqlTypeUtil.createMapType(typeFactory, keyType, valueType, isNullable); } return typeFactory.createTypeWithNullability(targetType, isNullable); diff --git a/core/src/main/java/org/apache/calcite/sql/type/SqlTypeFactoryImpl.java b/core/src/main/java/org/apache/calcite/sql/type/SqlTypeFactoryImpl.java index d0ccab7dfd2f..544f9ca708da 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/SqlTypeFactoryImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/type/SqlTypeFactoryImpl.java @@ -28,6 +28,7 @@ import org.checkerframework.checker.nullness.qual.Nullable; import java.nio.charset.Charset; +import java.util.ArrayList; import java.util.List; import static com.google.common.base.Preconditions.checkArgument; @@ -191,11 +192,33 @@ public SqlTypeFactoryImpl(RelDataTypeSystem typeSystem) { RelDataType type0 = types.get(0); if (type0.getSqlTypeName() != null) { - RelDataType resultType = leastRestrictiveSqlType(types); - if (resultType != null) { - return resultType; + // First preprocess to filter out UNKNOWN types. + // leastRestrictive() can be thought as a form of type unification, + // and UNKNOWN behaves like an unbound type variable: it unifies with any type + // without constraining the result. + // Note that UNKNOWN can be nullable, so this information is carried over to the result. + List knownTypes = new ArrayList<>(types.size()); + // True if any UNKNOWN type is nullable + boolean anyUnknownIsNullable = false; + for (RelDataType type : types) { + if (type.getSqlTypeName() == SqlTypeName.UNKNOWN) { + anyUnknownIsNullable |= type.isNullable(); + } else { + knownTypes.add(type); + } } - return leastRestrictiveByCast(types, mappingRule); + if (knownTypes.isEmpty()) { + // All types are unknown + return createTypeWithNullability(createUnknownType(), anyUnknownIsNullable); + } + RelDataType resultType = leastRestrictiveSqlType(knownTypes); + if (resultType == null) { + resultType = leastRestrictiveByCast(knownTypes, mappingRule); + } + if (resultType != null && anyUnknownIsNullable) { + resultType = createTypeWithNullability(resultType, true); + } + return resultType; } return super.leastRestrictive(types, mappingRule); diff --git a/core/src/main/java/org/apache/calcite/sql/type/SqlTypeUtil.java b/core/src/main/java/org/apache/calcite/sql/type/SqlTypeUtil.java index 02988668287b..5de00a50b4c8 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/SqlTypeUtil.java +++ b/core/src/main/java/org/apache/calcite/sql/type/SqlTypeUtil.java @@ -1128,8 +1128,9 @@ public static boolean canCastFrom( requireNonNull(fromType.getComponentType(), "componentType"), typeMappingRule); } else if (fromType.getFamily() == SqlTypeFamily.CHARACTER - || fromType.getSqlTypeName() == SqlTypeName.NULL) { - // Cast from NULL or string to array is legal + || fromType.getSqlTypeName() == SqlTypeName.NULL + || fromType.getSqlTypeName() == SqlTypeName.UNKNOWN) { + // Cast from NULL, UNKNOWN, or string to array is legal return true; } return false; @@ -1145,8 +1146,9 @@ && canCastFrom( requireNonNull(fromType.getValueType(), "valueType"), typeMappingRule); } else if (fromType.getFamily() == SqlTypeFamily.CHARACTER - || fromType.getSqlTypeName() == SqlTypeName.NULL) { - // Cast from NULL or string to map is legal + || fromType.getSqlTypeName() == SqlTypeName.NULL + || fromType.getSqlTypeName() == SqlTypeName.UNKNOWN) { + // Cast from NULL, UNKNOWN, or string to map is legal return true; } return false; @@ -1164,7 +1166,10 @@ && canCastFrom( toType, fromType.getFieldList().get(0).getType(), typeMappingRule); } else if (toTypeName == SqlTypeName.ROW) { if (fromTypeName != SqlTypeName.ROW) { - return fromTypeName == SqlTypeName.NULL; + // UNKNOWN can arise e.g. as the element type inferred for the + // no-argument array constructor ARRAY() + return fromTypeName == SqlTypeName.NULL + || fromTypeName == SqlTypeName.UNKNOWN; } int n = toType.getFieldCount(); if (fromType.getFieldCount() != n) { diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java index 2ac820e83855..b25c213defcb 100644 --- a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java +++ b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java @@ -3032,25 +3032,25 @@ private SqlDialect nonOrdinalDialect() { + " as MAP array)"; final String expectedClickHouse2 = "SELECT CAST(array(map('a', '1'), map('b', '2'), map('c', '3'))" - + " AS Array(Map(`String`, `Nullable(String)`)))"; + + " AS Array(Map(`String`, `String`)))"; sql(query2).withClickHouse().ok(expectedClickHouse2); final String query3 = "select cast(MAP['a',ARRAY[1,2,3]]" + " as MAP)"; final String expectedClickHouse3 = - "SELECT CAST(map('a', array(1, 2, 3)) AS Map(`String`, Array(`Nullable(Int32)`)))"; + "SELECT CAST(map('a', array(1, 2, 3)) AS Map(`String`, Array(`Int32`)))"; sql(query3).withClickHouse().ok(expectedClickHouse3); final String query4 = "select cast(MAP['a',ARRAY[1.0,2.0,3.0]]" + " as MAP)"; final String expectedClickHouse4 = - "SELECT CAST(map('a', array(1.0, 2.0, 3.0)) AS Map(`String`, Array(`Nullable(Float32)`)))"; + "SELECT CAST(map('a', array(1.0, 2.0, 3.0)) AS Map(`String`, Array(`Float32`)))"; sql(query4).withClickHouse().ok(expectedClickHouse4); final String query5 = "select cast(MAP['a',MAP['b','c']]" + " as MAP>)"; final String expectedClickHouse5 = - "SELECT CAST(map('a', map('b', 'c')) AS Map(`String`, Map(`String`, `Nullable(String)`)))"; + "SELECT CAST(map('a', map('b', 'c')) AS Map(`String`, Map(`String`, `String`)))"; sql(query5).withClickHouse().ok(expectedClickHouse5); } @@ -5652,15 +5652,15 @@ private SqlDialect nonOrdinalDialect() { @Test void testCastAsMapType() { sql("SELECT CAST(MAP['A', 1.0] AS MAP)") .ok("SELECT CAST(MAP['A', 1.0] AS " - + "MAP< VARCHAR CHARACTER SET \"ISO-8859-1\", DOUBLE NULL >)\n" + + "MAP< VARCHAR CHARACTER SET \"ISO-8859-1\", DOUBLE >)\n" + "FROM (VALUES (0)) AS \"t\" (\"ZERO\")"); sql("SELECT CAST(MAP['A', ARRAY[1, 2, 3]] AS MAP)") .ok("SELECT CAST(MAP['A', ARRAY[1, 2, 3]] AS " - + "MAP< VARCHAR CHARACTER SET \"ISO-8859-1\", INTEGER ARRAY NULL >)\n" + + "MAP< VARCHAR CHARACTER SET \"ISO-8859-1\", INTEGER ARRAY >)\n" + "FROM (VALUES (0)) AS \"t\" (\"ZERO\")"); sql("SELECT CAST(MAP[ARRAY['A'], MAP[1, 2]] AS MAP>)") .ok("SELECT CAST(MAP[ARRAY['A'], MAP[1, 2]] AS " - + "MAP< VARCHAR CHARACTER SET \"ISO-8859-1\" ARRAY, MAP< INTEGER, INTEGER NULL > NULL >)\n" + + "MAP< VARCHAR CHARACTER SET \"ISO-8859-1\" ARRAY, MAP< INTEGER, INTEGER > >)\n" + "FROM (VALUES (0)) AS \"t\" (\"ZERO\")"); } diff --git a/core/src/test/java/org/apache/calcite/sql/type/SqlTypeFactoryTest.java b/core/src/test/java/org/apache/calcite/sql/type/SqlTypeFactoryTest.java index f0ff190d28c7..68a34eb5bac7 100644 --- a/core/src/test/java/org/apache/calcite/sql/type/SqlTypeFactoryTest.java +++ b/core/src/test/java/org/apache/calcite/sql/type/SqlTypeFactoryTest.java @@ -87,6 +87,42 @@ class SqlTypeFactoryTest { assertThat(leastRestrictive.isNullable(), is(true)); } + /** UNKNOWN types in leastRestrictive() affect only the result nullability. */ + @Test void testLeastRestrictiveWithUnknown() { + SqlTypeFixture f = new SqlTypeFixture(); + // UNKNOWN never constrains the result, no matter its position + checkUnknownWithType(f, f.sqlBigInt); + checkUnknownWithType(f, f.structOfInt); + checkUnknownWithType(f, f.arrayBigInt); + checkUnknownWithType(f, f.mapOfInt); + // A nullable UNKNOWN makes the result nullable + RelDataType leastRestrictive = + f.typeFactory.leastRestrictive( + Lists.newArrayList(f.structOfInt, + f.typeFactory.createTypeWithNullability(f.sqlUnknown, true))); + assertThat(leastRestrictive, notNullValue()); + assertThat(leastRestrictive.getSqlTypeName(), is(SqlTypeName.ROW)); + assertThat(leastRestrictive.isNullable(), is(true)); + // A list of UNKNOWN unifies to UNKNOWN + leastRestrictive = + f.typeFactory.leastRestrictive( + Lists.newArrayList(f.sqlUnknown, f.sqlUnknown)); + assertThat(leastRestrictive.getSqlTypeName(), is(SqlTypeName.UNKNOWN)); + } + + /** Checks that leastRestictive({@code type}, UNKNOWN) yields {@code type}, + * in either order. */ + private void checkUnknownWithType(SqlTypeFixture f, RelDataType type) { + RelDataType r1 = + f.typeFactory.leastRestrictive(Lists.newArrayList(type, f.sqlUnknown)); + RelDataType r2 = + f.typeFactory.leastRestrictive(Lists.newArrayList(f.sqlUnknown, type)); + assertThat(r1, notNullValue()); + assertThat(r2, notNullValue()); + assertThat(r1.getFullTypeString(), is(type.getFullTypeString())); + assertThat(r2.getFullTypeString(), is(type.getFullTypeString())); + } + @Test void testLeastRestrictiveStructWithNull() { SqlTypeFixture f = new SqlTypeFixture(); RelDataType leastRestrictive = diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index cecb57385e4e..5e9224364ef3 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -1527,7 +1527,7 @@ void testLikeAndSimilarFails() { expr("cast(ARRAY[1,2,3] AS VARIANT ARRAY)") .columnType("VARIANT NOT NULL ARRAY NOT NULL"); expr("cast(MAP['a','b','c','d'] AS MAP)") - .columnType("(VARCHAR NOT NULL, VARIANT) MAP NOT NULL"); + .columnType("(VARCHAR NOT NULL, VARIANT NOT NULL) MAP NOT NULL"); // Test case for [CALCITE-7293] https://issues.apache.org/jira/browse/CALCITE-7293 // MAP constructor cannot handle VARIANT values that need casts expr("MAP['a', CAST('x' AS VARIANT), 'b', CAST(NULL AS VARIANT)]") @@ -9640,16 +9640,16 @@ void testGroupExpressionEquivalenceParams() { @Test void testCastMapType() { sql("select cast(\"int2IntMapType\" as map) from COMPLEXTYPES.CTC_T1") .withExtendedCatalog() - .columnType("(INTEGER NOT NULL, INTEGER) MAP NOT NULL"); + .columnType("(INTEGER NOT NULL, INTEGER NOT NULL) MAP NOT NULL"); sql("select cast(\"int2varcharArrayMapType\" as map) " + "from COMPLEXTYPES.CTC_T1") .withExtendedCatalog() - .columnType("(INTEGER NOT NULL, VARCHAR ARRAY) MAP NOT NULL"); + .columnType("(INTEGER NOT NULL, VARCHAR NOT NULL ARRAY NOT NULL) MAP NOT NULL"); sql("select cast(\"varcharMultiset2IntIntMapType\" as map>)" + " from COMPLEXTYPES.CTC_T1") .withExtendedCatalog() - .columnType("(VARCHAR(5) MULTISET NOT NULL, " - + "(INTEGER NOT NULL, INTEGER) MAP) MAP NOT NULL"); + .columnType("(VARCHAR(5) NOT NULL MULTISET NOT NULL, " + + "(INTEGER NOT NULL, INTEGER NOT NULL) MAP NOT NULL) MAP NOT NULL"); } @Test void testCastAsRowType() { diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java index d710b516aaa7..f82222312e3a 100644 --- a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java +++ b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java @@ -1843,6 +1843,39 @@ void testCastToBoolean(CastType castType, SqlOperatorFixture f) { f.checkNull("cast(null as row(f0 varchar, f1 varchar))"); } + /** Test case for + * + * [CALCITE-7658] Type checker rejects + * CAST(ARRAY() AS ROW(x INT) ARRAY). + * + *

      The Spark {@code ARRAY()} function creates an empty array whose + * element type is UNKNOWN; such an array can be cast to any array type. */ + @Test void testCastEmptyArray() { + final SqlOperatorFixture f = fixture().withLibrary(SqlLibrary.SPARK); + f.checkScalar("cast(array() as integer array)", "[]", + "INTEGER NOT NULL ARRAY NOT NULL"); + f.checkScalar("cast(array() as row(x int) array)", "[]", + "RecordType(INTEGER NOT NULL X) NOT NULL ARRAY NOT NULL"); + f.checkScalar("cast(array() as integer array array)", "[]", + "INTEGER ARRAY NOT NULL ARRAY NOT NULL"); + f.checkScalar("cast(array() as map array)", "[]", + "(VARCHAR NOT NULL, INTEGER) MAP NOT NULL ARRAY NOT NULL"); + // A non-empty array with UNKNOWN or NULL element type contains only nulls + f.checkScalar("cast(array_append(array(), null) as row(x int) array)", + "[null]", + "RecordType(INTEGER NOT NULL X) ARRAY NOT NULL"); + f.checkScalar("cast(array(null) as row(x int) array)", + "[null]", + "RecordType(INTEGER NOT NULL X) ARRAY NOT NULL"); + // The empty MAP() has UNKNOWN key and value types + f.checkScalar("cast(map() as map)", "{}", + "(VARCHAR NOT NULL, INTEGER NOT NULL) MAP NOT NULL"); + f.checkScalar("cast(map() as map)", "{}", + "(VARCHAR NOT NULL, RecordType(INTEGER X) NOT NULL) MAP NOT NULL"); + f.checkScalar("cast(map() as map)", "{}", + "(VARCHAR NOT NULL, INTEGER ARRAY NOT NULL) MAP NOT NULL"); + } + /** Test cases for * * [CALCITE-4918] Add a VARIANT data type. */ @@ -8229,7 +8262,7 @@ void checkRegexpExtract(SqlOperatorFixture f0, FunctionAlias functionAlias) { f.checkScalar("array_append(array(null), null)", "[null, null]", "NULL ARRAY NOT NULL"); f.checkScalar("array_append(array(), null)", "[null]", - "UNKNOWN ARRAY NOT NULL"); + "NULL ARRAY NOT NULL"); f.checkScalar("array_append(array(), 1)", "[1]", "INTEGER NOT NULL ARRAY NOT NULL"); f.checkScalar("array_append(array[array[1, 2]], array[3, 4])", "[[1, 2], [3, 4]]", @@ -8568,7 +8601,7 @@ void checkRegexpExtract(SqlOperatorFixture f0, FunctionAlias functionAlias) { f.checkScalar("array_prepend(array(null), null)", "[null, null]", "NULL ARRAY NOT NULL"); f.checkScalar("array_prepend(array(), null)", "[null]", - "UNKNOWN ARRAY NOT NULL"); + "NULL ARRAY NOT NULL"); f.checkScalar("array_prepend(array(), 1)", "[1]", "INTEGER NOT NULL ARRAY NOT NULL"); f.checkScalar("array_prepend(array[array[1, 2]], array[3, 4])", "[[3, 4], [1, 2]]", @@ -13606,6 +13639,22 @@ private static void checkArrayConcatAggFuncFails(SqlOperatorFixture t) { "RecordType(INTEGER EXPR$0, INTEGER EXPR$1) NOT NULL ARRAY NOT NULL"); f2.checkScalar("array(row(1, 2), row(3, 4))", "[{1, 2}, {3, 4}]", "RecordType(INTEGER NOT NULL EXPR$0, INTEGER NOT NULL EXPR$1) NOT NULL ARRAY NOT NULL"); + // Tests for unification of UNKNOWN with other types; array() has a type + // of UNKNOWN ARRAY, yet the type of ARRAY() is inferred from other operands. + f2.checkScalar("array(array(1), array())", "[[1], []]", + "INTEGER NOT NULL ARRAY NOT NULL ARRAY NOT NULL"); + f2.checkScalar("array(array(), array(1))", "[[], [1]]", + "INTEGER NOT NULL ARRAY NOT NULL ARRAY NOT NULL"); + f2.checkScalar("array(array(row(1, 2)), array())", "[[{1, 2}], []]", + "RecordType(INTEGER NOT NULL EXPR$0, INTEGER NOT NULL EXPR$1) NOT NULL " + + "ARRAY NOT NULL ARRAY NOT NULL"); + f2.checkScalar("array(array(), array(row(1, 2)))", "[[], [{1, 2}]]", + "RecordType(INTEGER NOT NULL EXPR$0, INTEGER NOT NULL EXPR$1) NOT NULL " + + "ARRAY NOT NULL ARRAY NOT NULL"); + f2.checkScalar("array(array(array(1)), array())", "[[[1]], []]", + "INTEGER NOT NULL ARRAY NOT NULL ARRAY NOT NULL ARRAY NOT NULL"); + f2.checkScalar("array(array(), array(array(1)))", "[[], [[1]]]", + "INTEGER NOT NULL ARRAY NOT NULL ARRAY NOT NULL ARRAY NOT NULL"); // checkFails f2.checkFails("^array(row(1), row(2, 3))^", "Parameters must be of the same type", false); @@ -13624,6 +13673,32 @@ private static void checkArrayConcatAggFuncFails(SqlOperatorFixture t) { f.forEachLibrary(libraries, consumer); } + /** Tests that empty collections created by the Spark + * {@code ARRAY()} and {@code MAP()} functions, whose element + * types are UNKNOWN, unify with collections with known types. */ + @Test void testEmptyCollections() { + final SqlOperatorFixture f = fixture().withLibrary(SqlLibrary.SPARK); + f.checkScalar("array(map(1, 2), map())", "[{1=2}, {}]", + "(INTEGER NOT NULL, INTEGER NOT NULL) MAP NOT NULL ARRAY NOT NULL"); + f.checkScalar("array(map(), map(1, 2))", "[{}, {1=2}]", + "(INTEGER NOT NULL, INTEGER NOT NULL) MAP NOT NULL ARRAY NOT NULL"); + // Nested: empty collections inside a ROW unify field by field + f.checkScalar("array(row(array(), map()))", "[{[], {}}]", + "RecordType(UNKNOWN NOT NULL ARRAY NOT NULL EXPR$0, " + + "(UNKNOWN NOT NULL, UNKNOWN NOT NULL) MAP NOT NULL EXPR$1) " + + "NOT NULL ARRAY NOT NULL"); + f.checkScalar("array(row(array(1), map(1, 2)), row(array(), map()))", + "[{[1], {1=2}}, {[], {}}]", + "RecordType(INTEGER NOT NULL ARRAY NOT NULL EXPR$0, " + + "(INTEGER NOT NULL, INTEGER NOT NULL) MAP NOT NULL EXPR$1) " + + "NOT NULL ARRAY NOT NULL"); + f.checkScalar("array(row(array(), map()), row(array(1), map(1, 2)))", + "[{[], {}}, {[1], {1=2}}]", + "RecordType(INTEGER NOT NULL ARRAY NOT NULL EXPR$0, " + + "(INTEGER NOT NULL, INTEGER NOT NULL) MAP NOT NULL EXPR$1) " + + "NOT NULL ARRAY NOT NULL"); + } + @Test void testArrayQueryConstructor() { final SqlOperatorFixture f = fixture(); f.setFor(SqlStdOperatorTable.ARRAY_QUERY, SqlOperatorFixture.VmName.EXPAND); @@ -13930,6 +14005,22 @@ private static void checkArrayConcatAggFuncFails(SqlOperatorFixture t) { f1.checkScalar("map('k1', 1, 'k2', 2.0)", "{k1=1.0, k2=2.0}", "(CHAR(2) NOT NULL, DECIMAL(11, 1) NOT NULL) MAP NOT NULL"); + f1.checkScalar("map('a', array(1), 'b', array())", "{a=[1], b=[]}", + "(CHAR(1) NOT NULL, INTEGER NOT NULL ARRAY NOT NULL) MAP NOT NULL"); + f1.checkScalar("map('a', array(), 'b', array(1))", "{a=[], b=[1]}", + "(CHAR(1) NOT NULL, INTEGER NOT NULL ARRAY NOT NULL) MAP NOT NULL"); + f1.checkScalar("map('a', map(1, 2), 'b', map())", "{a={1=2}, b={}}", + "(CHAR(1) NOT NULL, (INTEGER NOT NULL, INTEGER NOT NULL) MAP NOT NULL) MAP NOT NULL"); + f1.checkScalar("map('a', map(), 'b', map(1, 2))", "{a={}, b={1=2}}", + "(CHAR(1) NOT NULL, (INTEGER NOT NULL, INTEGER NOT NULL) MAP NOT NULL) MAP NOT NULL"); + // Avatica's conversion of MAP to STRING is broken, so we only check + // the type for the following 2 tests + f1.checkType("map('a', array(row(1, 2)), 'b', array())", + "(CHAR(1) NOT NULL, RecordType(INTEGER NOT NULL EXPR$0, INTEGER NOT NULL EXPR$1) " + + "NOT NULL ARRAY NOT NULL) MAP NOT NULL"); + f1.checkType("map('a', array(), 'b', array(row(1, 2)))", + "(CHAR(1) NOT NULL, RecordType(INTEGER NOT NULL EXPR$0, INTEGER NOT NULL EXPR$1) " + + "NOT NULL ARRAY NOT NULL) MAP NOT NULL"); } @Test void testMapQueryConstructor() { From 0d92ef0eada062dc14059f48821280f4854d9bd5 Mon Sep 17 00:00:00 2001 From: Kirill Tkalenko Date: Mon, 20 Jul 2026 06:50:41 +0300 Subject: [PATCH 397/562] [CALCITE-7592] Add expression support for FETCH --- core/src/main/codegen/templates/Parser.jj | 20 +- .../calcite/adapter/enumerable/EnumUtils.java | 7 +- .../adapter/enumerable/EnumerableLimit.java | 20 +- .../enumerable/EnumerableLimitSort.java | 6 +- .../enumerable/EnumerableMergeUnionRule.java | 9 +- .../enumerable/RexToLixTranslator.java | 16 - .../apache/calcite/interpreter/SortNode.java | 102 +++++- .../rel/metadata/RelMdMaxRowCount.java | 15 +- .../rel/metadata/RelMdMinRowCount.java | 17 +- .../calcite/rel/metadata/RelMdRowCount.java | 12 +- .../calcite/rel/metadata/RelMdUtil.java | 15 +- .../rel/rel2sql/RelToSqlConverter.java | 12 +- .../calcite/rel/rules/MeasureRules.java | 4 +- .../calcite/rel/rules/PruneEmptyRules.java | 6 +- .../rel/rules/SortJoinTransposeRule.java | 4 +- .../rel/rules/SortRemoveRedundantRule.java | 3 + .../rel/rules/SortUnionTransposeRule.java | 9 +- .../java/org/apache/calcite/rex/RexUtil.java | 80 +++++ .../calcite/runtime/CalciteResource.java | 9 + .../org/apache/calcite/sql/SqlDialect.java | 34 +- .../calcite/sql/dialect/SqliteSqlDialect.java | 2 +- .../calcite/sql/fun/SqlCastFunction.java | 2 +- .../calcite/sql/type/SqlTypeFactoryImpl.java | 31 +- .../apache/calcite/sql/type/SqlTypeUtil.java | 15 +- .../sql/validate/SqlValidatorImpl.java | 29 ++ .../sql2rel/CorrelateProjectExtractor.java | 77 +--- .../calcite/sql2rel/RelDecorrelator.java | 29 +- .../sql2rel/TopDownGeneralDecorrelator.java | 35 +- .../org/apache/calcite/tools/RelBuilder.java | 52 ++- .../runtime/CalciteResource.properties | 3 + .../adapter/enumerable/EnumUtilsTest.java | 11 + .../rel/rel2sql/RelToSqlConverterTest.java | 82 ++++- .../apache/calcite/rex/RexProgramTest.java | 30 ++ .../calcite/sql/type/SqlTypeFactoryTest.java | 36 -- .../CorrelateProjectExtractorTest.java | 72 ---- .../calcite/sql2rel/RelDecorrelatorTest.java | 99 +---- .../org/apache/calcite/test/JdbcTest.java | 338 ++++++++++++++++++ .../apache/calcite/test/RelBuilderTest.java | 105 ++++++ .../apache/calcite/test/RelMetadataTest.java | 33 +- .../apache/calcite/test/RelOptRulesTest.java | 81 ++++- .../calcite/test/SqlToRelConverterTest.java | 9 + .../apache/calcite/test/SqlValidatorTest.java | 26 +- .../enumerable/EnumerableMergeUnionTest.java | 30 ++ .../apache/calcite/test/RelOptRulesTest.xml | 217 ++++++++++- .../calcite/test/SqlToRelConverterTest.xml | 52 +-- core/src/test/resources/sql/fetch.iq | 183 ++++++++++ core/src/test/resources/sql/lateral.iq | 94 ----- .../org/apache/calcite/test/ServerTest.java | 37 ++ site/_docs/reference.md | 9 +- .../calcite/sql/parser/SqlParserTest.java | 19 + .../apache/calcite/test/SqlOperatorTest.java | 95 +---- 51 files changed, 1704 insertions(+), 629 deletions(-) create mode 100644 core/src/test/resources/sql/fetch.iq diff --git a/core/src/main/codegen/templates/Parser.jj b/core/src/main/codegen/templates/Parser.jj index c5d392e2fb0c..5c403e1cedd8 100644 --- a/core/src/main/codegen/templates/Parser.jj +++ b/core/src/main/codegen/templates/Parser.jj @@ -691,7 +691,7 @@ SqlNode ExprOrJoinOrOrderedQuery(ExprContext exprContext) : * * 

        *    [ OFFSET start { ROW | ROWS } ]
- *    [ FETCH { FIRST | NEXT } [ count ] { ROW | ROWS } ONLY ]
      + * [ FETCH { FIRST | NEXT } [ count | (expression) ] { ROW | ROWS } ONLY ] *
      */ SqlNode OrderedQueryOrExpr(ExprContext exprContext) : @@ -778,10 +778,26 @@ void FetchClause(SqlNode[] offsetFetch) : { // SQL:2008-style syntax. "OFFSET ... FETCH ...". // If you specify both LIMIT and FETCH, FETCH wins. - ( | ) offsetFetch[1] = UnsignedNumericLiteralOrParam() + ( | ) offsetFetch[1] = FetchCount() ( | ) } +/** + * Parses the row count of a FETCH clause. Expressions must be parenthesized. + */ +SqlNode FetchCount() : +{ + final SqlNode e; +} +{ + ( + e = UnsignedNumericLiteralOrParam() + | + e = Expression(ExprContext.ACCEPT_NON_QUERY) + ) + { return e; } +} + /** * Parses a LIMIT clause in an ORDER BY expression. */ diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java index 80c46ec3f9ea..71dc7e602624 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java @@ -116,7 +116,7 @@ private EnumUtils() {} /** Converts a FETCH or OFFSET runtime value to {@link BigDecimal}. * *

      The value must be numeric and non-negative. */ - public static BigDecimal numberToBigDecimal(Object value, String kind) { + public static BigDecimal numberToBigDecimal(@Nullable Object value, String kind) { return numberToBigDecimal(value, kind, FetchOffsetRoundingPolicy.NONE); } @@ -124,8 +124,11 @@ public static BigDecimal numberToBigDecimal(Object value, String kind) { * *

      The value must be numeric and non-negative. The result is adjusted by * the configured rounding policy. */ - public static BigDecimal numberToBigDecimal(Object value, String kind, + public static BigDecimal numberToBigDecimal(@Nullable Object value, String kind, FetchOffsetRoundingPolicy roundingPolicy) { + if (value == null) { + throw new IllegalArgumentException(kind + " expression evaluated to NULL"); + } if (!(value instanceof Number)) { throw new IllegalArgumentException(kind + " must be a number"); } diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimit.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimit.java index 02fd54bdad86..de1f94d562d5 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimit.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimit.java @@ -106,13 +106,15 @@ public static EnumerableLimit create(final RelNode input, @Nullable RexNode offs v = builder.append("offset", Expressions.call(BuiltInMethod.SKIP_BIG_DECIMAL.method, v, - getExpression(offset, "OFFSET", roundingPolicyExp))); + getExpression(offset, "OFFSET", implementor, builder, + roundingPolicyExp, false))); } if (fetch != null) { v = builder.append("fetch", Expressions.call(BuiltInMethod.TAKE_BIG_DECIMAL.method, v, - getExpression(fetch, "FETCH", roundingPolicyExp))); + getExpression(fetch, "FETCH", implementor, builder, + roundingPolicyExp, true))); } builder.add(Expressions.return_(null, v)); @@ -120,7 +122,8 @@ public static EnumerableLimit create(final RelNode input, @Nullable RexNode offs } static Expression getExpression(RexNode rexNode, String kind, - Expression roundingPolicy) { + EnumerableRelImplementor implementor, BlockBuilder builder, + Expression roundingPolicy, boolean translateExpression) { final Expression value; if (rexNode instanceof RexDynamicParam) { final RexDynamicParam param = (RexDynamicParam) rexNode; @@ -128,8 +131,17 @@ static Expression getExpression(RexNode rexNode, String kind, Expressions.call(DataContext.ROOT, BuiltInMethod.DATA_CONTEXT_GET.method, Expressions.constant("?" + param.getIndex())); - } else { + } else if (rexNode instanceof RexLiteral) { value = Expressions.constant(RexLiteral.bigDecimalValue(rexNode)); + } else { + if (!translateExpression) { + throw new IllegalArgumentException(kind + " must be a literal or dynamic parameter"); + } + + value = + RexToLixTranslator.forAggregation(implementor.getTypeFactory(), + builder, null, implementor.getConformance()) + .translate(rexNode); } return Expressions.call( BuiltInMethod.NUMBER_TO_BIG_DECIMAL_LIMIT.method, diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimitSort.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimitSort.java index 325fe687ba4d..97d9fd8169b7 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimitSort.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimitSort.java @@ -104,14 +104,16 @@ public static EnumerableLimitSort create( if (this.fetch == null) { fetchVal = Expressions.constant(BigDecimal.valueOf(Integer.MAX_VALUE)); } else { - fetchVal = getExpression(this.fetch, "FETCH", roundingPolicyExp); + fetchVal = + getExpression(this.fetch, "FETCH", implementor, builder, roundingPolicyExp, true); } final Expression offsetVal; if (this.offset == null) { offsetVal = Expressions.constant(BigDecimal.ZERO); } else { - offsetVal = getExpression(this.offset, "OFFSET", roundingPolicyExp); + offsetVal = + getExpression(this.offset, "OFFSET", implementor, builder, roundingPolicyExp, false); } builder.add( diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeUnionRule.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeUnionRule.java index 7d47e639b78e..57f864794aa9 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeUnionRule.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeUnionRule.java @@ -29,6 +29,7 @@ import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexUtil; import org.apache.calcite.tools.RelBuilder; import org.apache.calcite.util.ImmutableBitSet; @@ -88,9 +89,13 @@ public EnumerableMergeUnionRule(Config config) { // Push down sort limit, if possible. RexNode inputFetch = null; if (sort.fetch != null) { - if (sort.offset == null) { + final boolean safeToReevaluate = + RexUtil.isDeterministic(sort.fetch); + if (sort.offset == null && safeToReevaluate) { inputFetch = sort.fetch; - } else if (sort.fetch instanceof RexLiteral && sort.offset instanceof RexLiteral) { + } else if (safeToReevaluate + && sort.fetch instanceof RexLiteral + && sort.offset instanceof RexLiteral) { inputFetch = call.builder().literal(RexLiteral.bigDecimalValue(sort.fetch) .add(RexLiteral.bigDecimalValue(sort.offset))); diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java index a39146ac754c..5a1e0fee2082 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java @@ -341,16 +341,6 @@ Expression translateCast( return expressionHandlingSafe(convert3, safe, targetType); } - /** Returns whether every runtime value of {@code type} is null. - * This holds for the NULL type, which describes untyped NULL literals, - * and for the UNKNOWN type, e.g. the element type inferred for the - * no-argument array constructor ARRAY(). Such values are never - * created, but the code generated needs to typecheck. */ - private static boolean valueIsAlwaysNull(RelDataType type) { - SqlTypeName typeName = type.getSqlTypeName(); - return typeName == SqlTypeName.UNKNOWN || typeName == SqlTypeName.NULL; - } - private Expression getConvertExpression( RelDataType sourceType, RelDataType targetType, @@ -376,9 +366,6 @@ private Expression getConvertExpression( } if (targetType.getSqlTypeName() == SqlTypeName.ROW) { - if (valueIsAlwaysNull(sourceType)) { - return Expressions.constant(null); - } assert sourceType.getSqlTypeName() == SqlTypeName.ROW; List targetTypes = targetType.getFieldList(); List sourceTypes = sourceType.getFieldList(); @@ -410,9 +397,6 @@ private Expression getConvertExpression( switch (targetType.getSqlTypeName()) { case ARRAY: case MULTISET: - if (valueIsAlwaysNull(sourceType)) { - return Expressions.constant(null); - } final RelDataType sourceDataType = sourceType.getComponentType(); final RelDataType targetDataType = targetType.getComponentType(); assert sourceDataType != null; diff --git a/core/src/main/java/org/apache/calcite/interpreter/SortNode.java b/core/src/main/java/org/apache/calcite/interpreter/SortNode.java index 71d9f2b22e42..0f393a3e68d8 100644 --- a/core/src/main/java/org/apache/calcite/interpreter/SortNode.java +++ b/core/src/main/java/org/apache/calcite/interpreter/SortNode.java @@ -16,14 +16,22 @@ */ package org.apache.calcite.interpreter; +import org.apache.calcite.adapter.enumerable.EnumUtils; +import org.apache.calcite.adapter.enumerable.EnumerableRelImplementor; +import org.apache.calcite.adapter.enumerable.FetchOffsetRoundingPolicy; import org.apache.calcite.rel.RelFieldCollation; import org.apache.calcite.rel.core.Sort; import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; import org.apache.calcite.util.Util; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Ordering; +import org.checkerframework.checker.nullness.qual.Nullable; + +import java.math.BigDecimal; +import java.math.RoundingMode; import java.util.ArrayList; import java.util.Comparator; import java.util.List; @@ -35,37 +43,57 @@ * {@link org.apache.calcite.rel.core.Sort}. */ public class SortNode extends AbstractSingleNode { + private final @Nullable Scalar offsetScalar; + private final @Nullable Context offsetContext; + private final @Nullable Scalar fetchScalar; + private final @Nullable Context fetchContext; + private final FetchOffsetRoundingPolicy fetchOffsetRoundingPolicy; + public SortNode(Compiler compiler, Sort rel) { super(compiler, rel); - } - - private static int getValueAsInt(RexNode node) { - return requireNonNull(((RexLiteral) node).getValueAs(Integer.class), - () -> "getValueAs(Integer.class) for " + node); + if (rel.offset != null && !(rel.offset instanceof RexLiteral)) { + this.offsetScalar = compiler.compile(ImmutableList.of(rel.offset), null); + this.offsetContext = compiler.createContext(); + } else { + this.offsetScalar = null; + this.offsetContext = null; + } + if (rel.fetch != null && !(rel.fetch instanceof RexLiteral)) { + this.fetchScalar = compiler.compile(ImmutableList.of(rel.fetch), null); + this.fetchContext = compiler.createContext(); + } else { + this.fetchScalar = null; + this.fetchContext = null; + } + final Object roundingPolicy = compiler.getDataContext() + .get(EnumerableRelImplementor.FETCH_OFFSET_ROUNDING_POLICY); + this.fetchOffsetRoundingPolicy = + roundingPolicy instanceof FetchOffsetRoundingPolicy + ? (FetchOffsetRoundingPolicy) roundingPolicy + : FetchOffsetRoundingPolicy.NONE; } @Override public void run() throws InterruptedException { - final int offset = - rel.offset == null - ? 0 - : getValueAsInt(rel.offset); - final int fetch = - rel.fetch == null - ? -1 - : getValueAsInt(rel.fetch); + final BigDecimal offset = getOffset(); + final @Nullable BigDecimal fetch = getFetch(); // In pure limit mode. No sort required. Row row; loop: if (rel.getCollation().getFieldCollations().isEmpty()) { - for (int i = 0; i < offset; i++) { + BigDecimal skipped = BigDecimal.ZERO; + while (skipped.compareTo(offset) < 0) { row = source.receive(); if (row == null) { break loop; } + skipped = skipped.add(BigDecimal.ONE); } - if (fetch >= 0) { - for (int i = 0; i < fetch && (row = source.receive()) != null; i++) { + if (fetch != null) { + BigDecimal fetched = BigDecimal.ZERO; + while (fetched.compareTo(fetch) < 0 + && (row = source.receive()) != null) { sink.send(row); + fetched = fetched.add(BigDecimal.ONE); } } else { while ((row = source.receive()) != null) { @@ -79,10 +107,15 @@ private static int getValueAsInt(RexNode node) { list.add(row); } list.sort(comparator()); - final int end = fetch < 0 || offset + fetch > list.size() + final int start = offset.compareTo(BigDecimal.valueOf(list.size())) >= 0 + ? list.size() + : rowCount(offset); + final int available = list.size() - start; + final int end = fetch == null + || fetch.compareTo(BigDecimal.valueOf(available)) >= 0 ? list.size() - : offset + fetch; - for (int i = offset; i < end; i++) { + : start + rowCount(fetch); + for (int i = start; i < end; i++) { sink.send(list.get(i)); } } @@ -116,4 +149,35 @@ private static Comparator comparator(RelFieldCollation fieldCollation) { }; } } + + private @Nullable BigDecimal getFetch() { + if (rel.fetch == null) { + return null; + } + return getValue(rel.fetch, fetchScalar, fetchContext, "FETCH"); + } + + private BigDecimal getOffset() { + if (rel.offset == null) { + return BigDecimal.ZERO; + } + return getValue(rel.offset, offsetScalar, offsetContext, "OFFSET"); + } + + private BigDecimal getValue(RexNode node, @Nullable Scalar scalar, + @Nullable Context context, String kind) { + final @Nullable Object value; + if (node instanceof RexLiteral) { + value = RexLiteral.bigDecimalValue(node); + } else { + value = + requireNonNull(scalar, () -> kind + " scalar") + .execute(requireNonNull(context, () -> kind + " context")); + } + return EnumUtils.numberToBigDecimal(value, kind, fetchOffsetRoundingPolicy); + } + + private static int rowCount(BigDecimal value) { + return value.setScale(0, RoundingMode.CEILING).intValueExact(); + } } diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdMaxRowCount.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdMaxRowCount.java index e728c22e1ede..869f1ad50e78 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdMaxRowCount.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdMaxRowCount.java @@ -117,10 +117,12 @@ public Double getMaxRowCount(Sort rel, RelMetadataQuery mq) { rowCount = Double.POSITIVE_INFINITY; } - final double offset = literalValueApproximatedByDouble(rel.offset, 0D); + final double offset = + literalValueApproximatedByDouble(rel.offset, 0D); rowCount = Math.max(rowCount - offset, 0D); - final double limit = literalValueApproximatedByDouble(rel.fetch, rowCount); + final double limit = + literalValueApproximatedByDouble(rel.fetch, rowCount); return limit < rowCount ? limit : rowCount; } @@ -130,10 +132,12 @@ public Double getMaxRowCount(EnumerableLimit rel, RelMetadataQuery mq) { rowCount = Double.POSITIVE_INFINITY; } - final double offset = literalValueApproximatedByDouble(rel.offset, 0D); + final double offset = + literalValueApproximatedByDouble(rel.offset, 0D); rowCount = Math.max(rowCount - offset, 0D); - final double limit = literalValueApproximatedByDouble(rel.fetch, rowCount); + final double limit = + literalValueApproximatedByDouble(rel.fetch, rowCount); return limit < rowCount ? limit : rowCount; } @@ -214,7 +218,8 @@ public Double getMaxRowCount(RelSubset rel, RelMetadataQuery mq) { if (node instanceof Sort) { Sort sort = (Sort) node; if (sort.fetch instanceof RexLiteral) { - return literalValueApproximatedByDouble(sort.fetch, Double.POSITIVE_INFINITY); + return literalValueApproximatedByDouble(sort.fetch, + Double.POSITIVE_INFINITY); } } } diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdMinRowCount.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdMinRowCount.java index 869d34333547..2cb710f39808 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdMinRowCount.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdMinRowCount.java @@ -116,10 +116,13 @@ public Double getMinRowCount(Sort rel, RelMetadataQuery mq) { rowCount = 0D; } - final double offset = literalValueApproximatedByDouble(rel.offset, 0D); + final double offset = + literalValueApproximatedByDouble(rel.offset, 0D); rowCount = Math.max(rowCount - offset, 0D); - final double limit = literalValueApproximatedByDouble(rel.fetch, rowCount); + final double limit = + literalValueApproximatedByDouble(rel.fetch, + rel.fetch == null ? rowCount : 0D); return limit < rowCount ? limit : rowCount; } @@ -129,10 +132,13 @@ public Double getMinRowCount(EnumerableLimit rel, RelMetadataQuery mq) { rowCount = 0D; } - final double offset = literalValueApproximatedByDouble(rel.offset, 0D); + final double offset = + literalValueApproximatedByDouble(rel.offset, 0D); rowCount = Math.max(rowCount - offset, 0D); - final double limit = literalValueApproximatedByDouble(rel.fetch, rowCount); + final double limit = + literalValueApproximatedByDouble(rel.fetch, + rel.fetch == null ? rowCount : 0D); return limit < rowCount ? limit : rowCount; } @@ -174,7 +180,8 @@ public Double getMinRowCount(RelSubset rel, RelMetadataQuery mq) { if (node instanceof Sort) { Sort sort = (Sort) node; if (sort.fetch instanceof RexLiteral) { - return literalValueApproximatedByDouble(sort.fetch, Double.POSITIVE_INFINITY); + return literalValueApproximatedByDouble(sort.fetch, + Double.POSITIVE_INFINITY); } } } diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdRowCount.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdRowCount.java index e83f4c1da9f4..3e7824e1aac6 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdRowCount.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdRowCount.java @@ -165,10 +165,12 @@ public Double getRowCount(Calc rel, RelMetadataQuery mq) { return null; } - final double offset = literalValueApproximatedByDouble(rel.offset, 0D); + final double offset = + literalValueApproximatedByDouble(rel.offset, 0D); rowCount = Math.max(rowCount - offset, 0D); - final double limit = literalValueApproximatedByDouble(rel.fetch, rowCount); + final double limit = + literalValueApproximatedByDouble(rel.fetch, rowCount); return limit < rowCount ? limit : rowCount; } @@ -178,10 +180,12 @@ public Double getRowCount(Calc rel, RelMetadataQuery mq) { return null; } - final double offset = literalValueApproximatedByDouble(rel.offset, 0D); + final double offset = + literalValueApproximatedByDouble(rel.offset, 0D); rowCount = Math.max(rowCount - offset, 0D); - final double limit = literalValueApproximatedByDouble(rel.fetch, rowCount); + final double limit = + literalValueApproximatedByDouble(rel.fetch, rowCount); return limit < rowCount ? limit : rowCount; } diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdUtil.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdUtil.java index 1f6502243626..5b096289382e 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdUtil.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdUtil.java @@ -25,6 +25,7 @@ import org.apache.calcite.rel.core.JoinRelType; import org.apache.calcite.rel.core.Minus; import org.apache.calcite.rel.core.Project; +import org.apache.calcite.rel.core.Sort; import org.apache.calcite.rel.core.Union; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexCall; @@ -56,6 +57,7 @@ import java.util.ArrayList; import java.util.LinkedHashSet; import java.util.List; +import java.util.Objects; import java.util.Set; import static com.google.common.base.Preconditions.checkArgument; @@ -483,6 +485,9 @@ public static double literalValueApproximatedByDouble(@Nullable RexNode node, throw new IllegalArgumentException( "literal value " + number + " cannot be converted to BigDecimal"); } + if (decimal.signum() < 0) { + return defaultValue; + } if (decimal.abs().compareTo(BigDecimal.valueOf(Double.MAX_VALUE)) > 0) { throw new IllegalArgumentException( "literal value " + decimal + " exceeds double range"); @@ -1043,8 +1048,16 @@ private static boolean alreadySmaller(RelMetadataQuery mq, RelNode input, if (fetch == null) { return true; } + final RelNode strippedInput = input.stripped(); + if (strippedInput instanceof Sort) { + final Sort sort = (Sort) strippedInput; + if (Objects.equals(offset, sort.offset) + && Objects.equals(fetch, sort.fetch)) { + return true; + } + } final Double rowCount = mq.getMaxRowCount(input); - if (rowCount == null || offset instanceof RexDynamicParam || fetch instanceof RexDynamicParam) { + if (rowCount == null || offset instanceof RexDynamicParam || !(fetch instanceof RexLiteral)) { // Cannot be determined return false; } diff --git a/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java b/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java index 0404fa7bc51c..dc91fca54ecb 100644 --- a/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java +++ b/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java @@ -59,6 +59,7 @@ import org.apache.calcite.rex.RexLocalRef; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexProgram; +import org.apache.calcite.rex.RexUtil; import org.apache.calcite.sql.JoinConditionType; import org.apache.calcite.sql.JoinType; import org.apache.calcite.sql.SqlAsofJoin; @@ -1191,7 +1192,7 @@ public Result visit(Sort e) { sqlSelect.setOffset(offset); } if (e.fetch != null) { - SqlNode fetch = builder.context.toSql(null, e.fetch); + SqlNode fetch = toSqlFetch(e, builder.context); sqlSelect.setFetch(fetch); } return result(sqlSelect, ImmutableList.of(Clause.ORDER_BY), e, null); @@ -1249,13 +1250,20 @@ public Result visit(Sort e) { * The builder must have been created with OFFSET and FETCH clauses. */ void offsetFetch(Sort e, Builder builder) { if (e.fetch != null) { - builder.setFetch(builder.context.toSql(null, e.fetch)); + builder.setFetch(toSqlFetch(e, builder.context)); } if (e.offset != null) { builder.setOffset(builder.context.toSql(null, e.offset)); } } + private static SqlNode toSqlFetch(Sort sort, Context context) { + final RexNode fetch = requireNonNull(sort.fetch, "fetch"); + final @Nullable RexLiteral reduced = + RexUtil.reduceFetchToLiteral(sort.getCluster(), fetch); + return context.toSql(null, reduced == null ? fetch : reduced); + } + public boolean hasTrickyRollup(Sort e, Aggregate aggregate) { return !dialect.supportsAggregateFunction(SqlKind.ROLLUP) && dialect.supportsGroupByWithRollup() diff --git a/core/src/main/java/org/apache/calcite/rel/rules/MeasureRules.java b/core/src/main/java/org/apache/calcite/rel/rules/MeasureRules.java index 037a4d605459..f69810a14d42 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/MeasureRules.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/MeasureRules.java @@ -30,7 +30,6 @@ import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexCorrelVariable; import org.apache.calcite.rex.RexInputRef; -import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexShuttle; import org.apache.calcite.rex.RexUtil; @@ -508,8 +507,7 @@ protected ProjectSortMeasureRule(ProjectSortMeasureRuleConfig config) { relBuilder.push(sort.getInput()) .projectPlus(map.keySet()) - .sortLimit(sort.offset == null ? 0 : RexLiteral.numberValue(sort.offset), - sort.fetch == null ? -1 : RexLiteral.numberValue(sort.fetch), + .sortLimit(sort.offset, sort.fetch, sort.getSortExps()) .project(newProjects); call.transformTo(relBuilder.build()); diff --git a/core/src/main/java/org/apache/calcite/rel/rules/PruneEmptyRules.java b/core/src/main/java/org/apache/calcite/rel/rules/PruneEmptyRules.java index 5d70c5e0dec4..5cb60bc90652 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/PruneEmptyRules.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/PruneEmptyRules.java @@ -41,7 +41,6 @@ import org.apache.calcite.rel.logical.LogicalValues; import org.apache.calcite.rel.metadata.RelMdUtil; import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.rex.RexDynamicParam; import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; import org.apache.calcite.tools.RelBuilder; @@ -519,9 +518,8 @@ public interface SortFetchZeroRuleConfig extends PruneEmptyRule.Config { return new RemoveEmptySingleRule(this) { @Override public boolean matches(final RelOptRuleCall call) { Sort sort = call.rel(0); - return sort.fetch != null - && !(sort.fetch instanceof RexDynamicParam) - && RexLiteral.bigDecimalValue(sort.fetch).equals(BigDecimal.ZERO); + return sort.fetch instanceof RexLiteral + && BigDecimal.ZERO.equals(RexLiteral.bigDecimalValue(sort.fetch)); } }; } diff --git a/core/src/main/java/org/apache/calcite/rel/rules/SortJoinTransposeRule.java b/core/src/main/java/org/apache/calcite/rel/rules/SortJoinTransposeRule.java index 4310d6d65576..df967e56aad6 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/SortJoinTransposeRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/SortJoinTransposeRule.java @@ -105,9 +105,9 @@ public SortJoinTransposeRule(Class sortClass, final Sort sort = call.rel(0); final Join join = call.rel(1); - // Do nothing if SORT contains dynamic parameters in offset or fetch + // The pushed fetch is calculated from literal offset and fetch values. if (sort.offset instanceof RexDynamicParam - || sort.fetch instanceof RexDynamicParam) { + || sort.fetch != null && !(sort.fetch instanceof RexLiteral)) { return false; } diff --git a/core/src/main/java/org/apache/calcite/rel/rules/SortRemoveRedundantRule.java b/core/src/main/java/org/apache/calcite/rel/rules/SortRemoveRedundantRule.java index 9bcf026fc656..08563cdcdb86 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/SortRemoveRedundantRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/SortRemoveRedundantRule.java @@ -133,6 +133,9 @@ protected SortRemoveRedundantRule(final SortRemoveRedundantRule.Config config) { private static Optional getRowCountThreshold(Sort sort) { if (RelOptUtil.isLimit(sort)) { assert sort.fetch != null; + if (!(sort.fetch instanceof RexLiteral)) { + return Optional.empty(); + } final BigDecimal fetch = RexLiteral.bigDecimalValue(sort.fetch); // We don't need to deal with fetch is 0. diff --git a/core/src/main/java/org/apache/calcite/rel/rules/SortUnionTransposeRule.java b/core/src/main/java/org/apache/calcite/rel/rules/SortUnionTransposeRule.java index 416825ee926d..93b6af657c43 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/SortUnionTransposeRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/SortUnionTransposeRule.java @@ -23,7 +23,7 @@ import org.apache.calcite.rel.core.Union; import org.apache.calcite.rel.metadata.RelMdUtil; import org.apache.calcite.rel.metadata.RelMetadataQuery; -import org.apache.calcite.rex.RexDynamicParam; +import org.apache.calcite.rex.RexUtil; import org.apache.calcite.tools.RelBuilderFactory; import org.immutables.value.Value; @@ -67,13 +67,14 @@ public SortUnionTransposeRule( @Override public boolean matches(RelOptRuleCall call) { final Sort sort = call.rel(0); final Union union = call.rel(1); - // We only apply this rule if Union.all is true, Sort.offset is null and Sort.fetch is not - // a dynamic param. + // Re-evaluating a non-deterministic FETCH in every branch can produce a + // different limit from the top Sort. // There is a flag indicating if this rule should be applied when // Sort.fetch is null. return union.all && sort.offset == null - && !(sort.fetch instanceof RexDynamicParam) + && (sort.fetch == null + || RexUtil.isDeterministic(sort.fetch)) && (config.matchNullFetch() || sort.fetch != null); } diff --git a/core/src/main/java/org/apache/calcite/rex/RexUtil.java b/core/src/main/java/org/apache/calcite/rex/RexUtil.java index 3604e98dfd5b..b592093a5aef 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexUtil.java +++ b/core/src/main/java/org/apache/calcite/rex/RexUtil.java @@ -19,6 +19,7 @@ import org.apache.calcite.DataContexts; import org.apache.calcite.linq4j.function.Predicate1; import org.apache.calcite.plan.PlanTooComplexError; +import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.plan.RelOptPredicateList; import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.rel.RelCollation; @@ -48,6 +49,7 @@ import org.apache.calcite.util.ControlFlowException; import org.apache.calcite.util.ImmutableBitSet; import org.apache.calcite.util.Litmus; +import org.apache.calcite.util.NumberUtil; import org.apache.calcite.util.Pair; import org.apache.calcite.util.RangeSets; import org.apache.calcite.util.Sarg; @@ -63,9 +65,11 @@ import org.apiguardian.api.API; import org.checkerframework.checker.nullness.qual.Nullable; +import java.math.BigDecimal; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -840,6 +844,82 @@ public static boolean isDeterministic(RexNode e) { } } + /** Returns whether an expression contains a dynamic function. */ + public static boolean containsDynamicFunction(RexNode e) { + try { + e.accept( + new RexVisitorImpl(true) { + @Override public Void visitCall(RexCall call) { + if (call.getOperator().isDynamicFunction()) { + throw Util.FoundOne.NULL; + } + return super.visitCall(call); + } + }); + return false; + } catch (Util.FoundOne ex) { + Util.swallow(ex, null); + return true; + } + } + + /** Returns whether an expression contains a dynamic parameter. */ + public static boolean containsDynamicParam(RexNode e) { + try { + e.accept( + new RexVisitorImpl(true) { + @Override public Void visitDynamicParam(RexDynamicParam dynamicParam) { + throw Util.FoundOne.NULL; + } + }); + return false; + } catch (Util.FoundOne ex) { + Util.swallow(ex, null); + return true; + } + } + + /** Converts a FETCH expression result to its validated canonical representation. */ + public static BigDecimal validateFetchValue(@Nullable Number value) { + if (value == null) { + throw new IllegalArgumentException("FETCH expression evaluated to NULL"); + } + final BigDecimal decimal = NumberUtil.toBigDecimal(value); + if (decimal.signum() < 0) { + throw new IllegalArgumentException("FETCH value " + value + + " is out of range; expected a non-negative value"); + } + return decimal; + } + + /** Reduces a constant FETCH expression to a validated literal. */ + public static @Nullable RexLiteral reduceFetchToLiteral( + RelOptCluster cluster, RexNode fetch) { + final RexLiteral literal; + if (fetch instanceof RexLiteral) { + literal = (RexLiteral) fetch; + } else { + if (!isConstant(fetch) + || !isDeterministic(fetch) + || containsDynamicFunction(fetch) + || containsDynamicParam(fetch)) { + return null; + } + final RexExecutor executor = + Util.first(cluster.getPlanner().getExecutor(), EXECUTOR); + final List reducedValues = new ArrayList<>(1); + executor.reduce(cluster.getRexBuilder(), + Collections.singletonList(fetch), reducedValues); + final RexNode reduced = reducedValues.get(0); + if (!(reduced instanceof RexLiteral)) { + return null; + } + literal = (RexLiteral) reduced; + } + validateFetchValue(literal.getValueAs(Number.class)); + return literal; + } + public static List retainDeterministic(List list) { List conjunctions = new ArrayList<>(); for (RexNode x : list) { diff --git a/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java b/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java index c6e1a4dbdcc5..e5932b63768d 100644 --- a/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java +++ b/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java @@ -164,6 +164,15 @@ ExInstWithCause validatorContext(int a0, int a1, @BaseMessage("Values passed to {0} operator must have compatible types") ExInst incompatibleValueType(String a0); + @BaseMessage("FETCH expression must have a numeric type; actual type is ''{0}''") + ExInst fetchExpressionMustBeNumeric(String type); + + @BaseMessage("FETCH expression cannot reference table column ''{0}''") + ExInst fetchExpressionCannotReferenceColumn(String column); + + @BaseMessage("FETCH expression evaluated to NULL") + ExInst fetchExpressionEvaluatedToNull(); + @BaseMessage("Values in expression list must have compatible types") ExInst incompatibleTypesInList(); diff --git a/core/src/main/java/org/apache/calcite/sql/SqlDialect.java b/core/src/main/java/org/apache/calcite/sql/SqlDialect.java index 869976f0d42a..663aee63b0db 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlDialect.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlDialect.java @@ -1081,7 +1081,18 @@ protected static void unparseFetchUsingAnsi(SqlWriter writer, @Nullable SqlNode writer.startList(SqlWriter.FrameTypeEnum.FETCH); writer.keyword("FETCH"); writer.keyword("NEXT"); - fetch.unparse(writer, -1, -1); + if (fetch instanceof SqlLiteral + || fetch instanceof SqlDynamicParam) { + fetch.unparse(writer, -1, -1); + } else { + final SqlWriter.Frame expressionFrame = writer.startList("(", ")"); + if (fetch instanceof SqlCall) { + writer.getDialect().unparseCall(writer, (SqlCall) fetch, 0, 0); + } else { + fetch.unparse(writer, 0, 0); + } + writer.endList(expressionFrame); + } writer.keyword("ROWS"); writer.keyword("ONLY"); writer.endList(fetchFrame); @@ -1091,13 +1102,32 @@ protected static void unparseFetchUsingAnsi(SqlWriter writer, @Nullable SqlNode /** Unparses offset/fetch using "LIMIT fetch OFFSET offset" syntax. */ protected static void unparseFetchUsingLimit(SqlWriter writer, @Nullable SqlNode offset, @Nullable SqlNode fetch) { + unparseFetchUsingLimit(writer, offset, fetch, false); + } + + /** Unparses offset/fetch using "LIMIT fetch OFFSET offset" syntax, + * optionally allowing a scalar expression as fetch. */ + protected static void unparseFetchUsingLimit(SqlWriter writer, @Nullable SqlNode offset, + @Nullable SqlNode fetch, boolean allowExpression) { checkArgument(fetch != null || offset != null); - unparseLimit(writer, fetch); + unparseLimit(writer, fetch, allowExpression); unparseOffset(writer, offset); } protected static void unparseLimit(SqlWriter writer, @Nullable SqlNode fetch) { + unparseLimit(writer, fetch, false); + } + + private static void unparseLimit(SqlWriter writer, @Nullable SqlNode fetch, + boolean allowExpression) { if (fetch != null) { + if (!allowExpression + && !(fetch instanceof SqlLiteral) + && !(fetch instanceof SqlDynamicParam)) { + throw new IllegalArgumentException( + "LIMIT dialect does not support FETCH expressions that cannot " + + "be reduced to a literal"); + } writer.newlineAndIndent(); final SqlWriter.Frame fetchFrame = writer.startList(SqlWriter.FrameTypeEnum.FETCH); diff --git a/core/src/main/java/org/apache/calcite/sql/dialect/SqliteSqlDialect.java b/core/src/main/java/org/apache/calcite/sql/dialect/SqliteSqlDialect.java index 82376ae576ab..f31276413600 100644 --- a/core/src/main/java/org/apache/calcite/sql/dialect/SqliteSqlDialect.java +++ b/core/src/main/java/org/apache/calcite/sql/dialect/SqliteSqlDialect.java @@ -90,7 +90,7 @@ public SqliteSqlDialect(SqlDialect.Context context) { @Override public void unparseOffsetFetch(SqlWriter writer, @Nullable SqlNode offset, @Nullable SqlNode fetch) { - unparseFetchUsingLimit(writer, offset, fetch); + unparseFetchUsingLimit(writer, offset, fetch, true); } @Override public void unparseCall(SqlWriter writer, SqlCall call, diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlCastFunction.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlCastFunction.java index 62ce488718ba..b25a9b8fdd57 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlCastFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlCastFunction.java @@ -200,7 +200,7 @@ private static RelDataType createTypeWithNullabilityFromExpr(RelDataTypeFactory RelDataType valueType = createTypeWithNullabilityFromExpr( typeFactory, expressionValueType, targetValueType, safe); - return SqlTypeUtil.createMapType(typeFactory, keyType, valueType, isNullable); + SqlTypeUtil.createMapType(typeFactory, keyType, valueType, isNullable); } return typeFactory.createTypeWithNullability(targetType, isNullable); diff --git a/core/src/main/java/org/apache/calcite/sql/type/SqlTypeFactoryImpl.java b/core/src/main/java/org/apache/calcite/sql/type/SqlTypeFactoryImpl.java index 544f9ca708da..d0ccab7dfd2f 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/SqlTypeFactoryImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/type/SqlTypeFactoryImpl.java @@ -28,7 +28,6 @@ import org.checkerframework.checker.nullness.qual.Nullable; import java.nio.charset.Charset; -import java.util.ArrayList; import java.util.List; import static com.google.common.base.Preconditions.checkArgument; @@ -192,33 +191,11 @@ public SqlTypeFactoryImpl(RelDataTypeSystem typeSystem) { RelDataType type0 = types.get(0); if (type0.getSqlTypeName() != null) { - // First preprocess to filter out UNKNOWN types. - // leastRestrictive() can be thought as a form of type unification, - // and UNKNOWN behaves like an unbound type variable: it unifies with any type - // without constraining the result. - // Note that UNKNOWN can be nullable, so this information is carried over to the result. - List knownTypes = new ArrayList<>(types.size()); - // True if any UNKNOWN type is nullable - boolean anyUnknownIsNullable = false; - for (RelDataType type : types) { - if (type.getSqlTypeName() == SqlTypeName.UNKNOWN) { - anyUnknownIsNullable |= type.isNullable(); - } else { - knownTypes.add(type); - } + RelDataType resultType = leastRestrictiveSqlType(types); + if (resultType != null) { + return resultType; } - if (knownTypes.isEmpty()) { - // All types are unknown - return createTypeWithNullability(createUnknownType(), anyUnknownIsNullable); - } - RelDataType resultType = leastRestrictiveSqlType(knownTypes); - if (resultType == null) { - resultType = leastRestrictiveByCast(knownTypes, mappingRule); - } - if (resultType != null && anyUnknownIsNullable) { - resultType = createTypeWithNullability(resultType, true); - } - return resultType; + return leastRestrictiveByCast(types, mappingRule); } return super.leastRestrictive(types, mappingRule); diff --git a/core/src/main/java/org/apache/calcite/sql/type/SqlTypeUtil.java b/core/src/main/java/org/apache/calcite/sql/type/SqlTypeUtil.java index 5de00a50b4c8..02988668287b 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/SqlTypeUtil.java +++ b/core/src/main/java/org/apache/calcite/sql/type/SqlTypeUtil.java @@ -1128,9 +1128,8 @@ public static boolean canCastFrom( requireNonNull(fromType.getComponentType(), "componentType"), typeMappingRule); } else if (fromType.getFamily() == SqlTypeFamily.CHARACTER - || fromType.getSqlTypeName() == SqlTypeName.NULL - || fromType.getSqlTypeName() == SqlTypeName.UNKNOWN) { - // Cast from NULL, UNKNOWN, or string to array is legal + || fromType.getSqlTypeName() == SqlTypeName.NULL) { + // Cast from NULL or string to array is legal return true; } return false; @@ -1146,9 +1145,8 @@ && canCastFrom( requireNonNull(fromType.getValueType(), "valueType"), typeMappingRule); } else if (fromType.getFamily() == SqlTypeFamily.CHARACTER - || fromType.getSqlTypeName() == SqlTypeName.NULL - || fromType.getSqlTypeName() == SqlTypeName.UNKNOWN) { - // Cast from NULL, UNKNOWN, or string to map is legal + || fromType.getSqlTypeName() == SqlTypeName.NULL) { + // Cast from NULL or string to map is legal return true; } return false; @@ -1166,10 +1164,7 @@ && canCastFrom( toType, fromType.getFieldList().get(0).getType(), typeMappingRule); } else if (toTypeName == SqlTypeName.ROW) { if (fromTypeName != SqlTypeName.ROW) { - // UNKNOWN can arise e.g. as the element type inferred for the - // no-argument array constructor ARRAY() - return fromTypeName == SqlTypeName.NULL - || fromTypeName == SqlTypeName.UNKNOWN; + return fromTypeName == SqlTypeName.NULL; } int n = toType.getFieldCount(); if (fromType.getFieldCount() != n) { diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java index 023de1b77f77..02a044820f49 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java @@ -1771,6 +1771,34 @@ private void handleOffsetFetch(@Nullable SqlNode offset, @Nullable SqlNode fetch } } + private void validateFetchExpression(@Nullable SqlNode fetch) { + if (fetch == null || fetch instanceof SqlDynamicParam) { + return; + } + if (SqlUtil.isNullLiteral(fetch, true)) { + throw newValidationError(fetch, + RESOURCE.fetchExpressionEvaluatedToNull()); + } + validateNoAggs(aggOrOverFinder, fetch, "FETCH"); + fetch.accept(new SqlBasicVisitor() { + @Override public Void visit(SqlIdentifier id) { + if (makeNullaryCall(id) != null) { + return null; + } + throw newValidationError(id, + RESOURCE.fetchExpressionCannotReferenceColumn(id.toString())); + } + }); + final SqlValidatorScope scope = getEmptyScope(); + inferUnknownTypes(typeFactory.createSqlType(SqlTypeName.DECIMAL), scope, fetch); + validateExpr(fetch, scope); + final RelDataType type = getValidatedNodeType(fetch); + if (!SqlTypeUtil.isNumeric(type)) { + throw newValidationError(fetch, + RESOURCE.fetchExpressionMustBeNumeric(type.getFullTypeString())); + } + } + /** * Performs expression rewrites which are always used unconditionally. These * rewrites massage the expression tree into a standard form so that the @@ -4469,6 +4497,7 @@ protected void validateSelect( validateWindowClause(select); validateQualifyClause(select); handleOffsetFetch(select.getOffset(), select.getFetch()); + validateFetchExpression(select.getFetch()); // Validate the SELECT clause late, because a select item might // depend on the GROUP BY list, or the window function might reference diff --git a/core/src/main/java/org/apache/calcite/sql2rel/CorrelateProjectExtractor.java b/core/src/main/java/org/apache/calcite/sql2rel/CorrelateProjectExtractor.java index 127f4e487941..50d255c3312d 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/CorrelateProjectExtractor.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/CorrelateProjectExtractor.java @@ -88,20 +88,6 @@ public CorrelateProjectExtractor(RelBuilderFactory factory) { this.builderFactory = factory; } - /** Returns whether {@code node} is a direct field access on the correlation - * variable with the specified id, such as {@code $cor0.DEPTNO}. A nested - * access such as {@code $cor0.REC.DEPTNO} is not direct. */ - private static boolean isDirectFieldAccess(RexNode node, CorrelationId id) { - if (node instanceof RexFieldAccess) { - RexFieldAccess access = (RexFieldAccess) node; - if (access.getReferenceExpr() instanceof RexCorrelVariable) { - RexCorrelVariable correlVar = (RexCorrelVariable) access.getReferenceExpr(); - return correlVar.id.equals(id); - } - } - return false; - } - @Override public RelNode visit(LogicalCorrelate correlate) { RelNode left = correlate.getLeft().accept(this); RelNode right = correlate.getRight().accept(this); @@ -109,12 +95,8 @@ private static boolean isDirectFieldAccess(RexNode node, CorrelationId id) { // Find the correlated expressions from the right side that can be moved to the left Set callsWithCorrelationInRight = findCorrelationDependentCalls(correlate.getCorrelationId(), right); - // Only direct field accesses on the correlation variable, such as - // $cor0.DEPTNO, are left in place. A nested field access, such as - // $cor0.REC.DEPTNO, is extracted boolean isTrivialCorrelation = - callsWithCorrelationInRight.stream() - .allMatch(exp -> isDirectFieldAccess(exp, correlate.getCorrelationId())); + callsWithCorrelationInRight.stream().allMatch(exp -> exp instanceof RexFieldAccess); // Early exit condition if (isTrivialCorrelation) { if (correlate.getLeft().equals(left) && correlate.getRight().equals(right)) { @@ -134,42 +116,27 @@ private static boolean isDirectFieldAccess(RexNode node, CorrelationId id) { // Transform the correlated expression from the right side to an expression over the left side builder.push(left); - ImmutableBitSet.Builder requiredColumns = ImmutableBitSet.builder(); List callsWithCorrelationOverLeft = new ArrayList<>(); for (RexNode callInRight : callsWithCorrelationInRight) { - if (isDirectFieldAccess(callInRight, correlate.getCorrelationId())) { - // Direct field accesses stay in the right side and keep reading their - // original left column; that column must remain a required column. - requiredColumns.set(((RexFieldAccess) callInRight).getField().getIndex()); - } else { - callsWithCorrelationOverLeft.add(replaceCorrelationsWithInputRef(callInRight, builder)); - } + callsWithCorrelationOverLeft.add(replaceCorrelationsWithInputRef(callInRight, builder)); } builder.projectPlus(callsWithCorrelationOverLeft); // Construct the mapping to transform the expressions in the right side based on the new // projection in the left side. Map transformMapping = new HashMap<>(); - int newFieldIndex = oldLeft; for (RexNode callInRight : callsWithCorrelationInRight) { - if (!isDirectFieldAccess(callInRight, correlate.getCorrelationId())) { - RexBuilder xb = builder.getRexBuilder(); - RexNode v = xb.makeCorrel(builder.peek().getRowType(), correlate.getCorrelationId()); - RexNode flatCorrelationInRight = xb.makeFieldAccess(v, newFieldIndex); - transformMapping.put(callInRight, flatCorrelationInRight); - newFieldIndex++; - } + RexBuilder xb = builder.getRexBuilder(); + RexNode v = xb.makeCorrel(builder.peek().getRowType(), correlate.getCorrelationId()); + RexNode flatCorrelationInRight = xb.makeFieldAccess(v, oldLeft + transformMapping.size()); + transformMapping.put(callInRight, flatCorrelationInRight); } - // Select the required fields/columns from the left side of the correlation: the columns - // read by the direct field accesses plus the newly projected columns, which are at the - // end of the left relational expression. + // Select the required fields/columns from the left side of the correlation. Based on the code + // above all these fields should be at the end of the left relational expression. List requiredFields = builder.fields( - requiredColumns - .set(oldLeft, oldLeft + callsWithCorrelationOverLeft.size()) - .build() - .asList()); + ImmutableBitSet.range(oldLeft, oldLeft + callsWithCorrelationOverLeft.size()).asList()); final int newLeft = builder.fields().size(); // Transform the expressions in the right side using the mapping constructed earlier. @@ -297,13 +264,8 @@ private static boolean isSimpleCorrelatedExpression(RexNode node, CorrelationId * +(10, $cor0.DEPTNO) -> TRUE * /(100,+(10, $cor0.DEPTNO)) -> TRUE * CAST(+(10, $cor0.DEPTNO)):INTEGER NOT NULL -> TRUE - * CASE(IS NOT NULL($cor0.PATH), $cor0.PATH, ARRAY(null:INTEGER)) -> TRUE * +($0, $cor0.DEPTNO) -> FALSE * } - * - *

      A subexpression built only from literals and dynamic parameters, such - * as {@code ARRAY(null:INTEGER)} above, is neutral: it neither qualifies nor - * disqualifies the enclosing call. */ private static class SimpleCorrelationDetector extends RexVisitorImpl<@Nullable Boolean> { @@ -322,8 +284,7 @@ private SimpleCorrelationDetector(CorrelationId corrId) { return Boolean.FALSE; } - @Override public @Nullable Boolean visitCall(RexCall call) { - // Constant operands must not disqualify the call + @Override public Boolean visitCall(RexCall call) { Boolean hasSimpleCorrelation = null; for (RexNode op : call.operands) { Boolean b = op.accept(this); @@ -331,8 +292,7 @@ private SimpleCorrelationDetector(CorrelationId corrId) { hasSimpleCorrelation = hasSimpleCorrelation == null ? b : hasSimpleCorrelation && b; } } - // If unsure return null; caller will decide - return hasSimpleCorrelation; + return hasSimpleCorrelation == null ? Boolean.FALSE : hasSimpleCorrelation; } @Override public @Nullable Boolean visitFieldAccess(RexFieldAccess fieldAccess) { @@ -372,10 +332,8 @@ private static RexNode replaceCorrelationsWithInputRef(RexNode exp, RelBuilder b } /** - * A visitor traversing row expressions and replacing calls and field - * accesses with other expressions according to the specified mapping. - * The mapping is consulted before recursing so that the outermost - * matching expression wins. + * A visitor traversing row expressions and replacing calls with other expressions according + * to the specified mapping. */ private static final class CallReplacer extends RexShuttle { private final Map mapping; @@ -392,14 +350,5 @@ private static final class CallReplacer extends RexShuttle { return super.visitCall(oldCall); } } - - @Override public RexNode visitFieldAccess(RexFieldAccess fieldAccess) { - RexNode replacement = mapping.get(fieldAccess); - if (replacement != null) { - return replacement; - } else { - return super.visitFieldAccess(fieldAccess); - } - } } } diff --git a/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java b/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java index a5a817579372..b4999b455bef 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java @@ -575,6 +575,10 @@ protected RexNode removeCorrelationExpr( // Its output does not change the input ordering, so there's no // need to call propagateExpr. + if (isCorVarDefined && !canDecorrelateOffsetFetch(rel)) { + return null; + } + final RelNode oldInput = rel.getInput(); final Frame frame = getInvoke(oldInput, isCorVarDefined, rel, true); if (frame == null) { @@ -1137,8 +1141,31 @@ private static void shiftMapping(Map mapping, int startIndex, return register(sort, result, mapOldToNewOutputs, corDefOutputs); } + static boolean canDecorrelateOffsetFetch(Sort sort) { + final @Nullable RexLiteral fetch = sort.fetch == null + ? null + : RexUtil.reduceFetchToLiteral(sort.getCluster(), sort.fetch); + return isNonNegativeIntegralLiteral(sort.offset) + && (sort.fetch == null + || fetch != null && isNonNegativeIntegralLiteral(fetch)); + } + + private static boolean isNonNegativeIntegralLiteral(@Nullable RexNode node) { + if (node == null) { + return true; + } + if (!(node instanceof RexLiteral)) { + return false; + } + final @Nullable BigDecimal value = + ((RexLiteral) node).getValueAs(BigDecimal.class); + return value != null + && value.signum() >= 0 + && value.stripTrailingZeros().scale() <= 0; + } + protected @Nullable Frame decorrelateSortAsAggregate(Sort sort, final Frame frame) { - if (sort.offset != null || sort.fetch == null) { + if (sort.offset != null || !(sort.fetch instanceof RexLiteral)) { return null; } diff --git a/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java b/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java index 291eb619d0ed..4139cf4b2b99 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java @@ -233,12 +233,14 @@ public static RelNode decorrelateQuery(RelNode rel, RelBuilder builder) { RelNode preparedRel = prePlanner.findBestExp(); // start decorrelating - TopDownGeneralDecorrelator decorrelator = createEmptyDecorrelator(builder); RelNode decorrelateNode = rel; - try { - decorrelateNode = decorrelator.correlateElimination(preparedRel, true); - } catch (UnsupportedOperationException e) { - // if the correlation exists in an unsupported operator, retain the original plan. + if (canDecorrelateOffsetFetch(preparedRel, false)) { + TopDownGeneralDecorrelator decorrelator = createEmptyDecorrelator(builder); + try { + decorrelateNode = decorrelator.correlateElimination(preparedRel, true); + } catch (UnsupportedOperationException e) { + // if the correlation exists in an unsupported operator, retain the original plan. + } } HepProgram postProgram = HepProgram.builder() @@ -255,6 +257,29 @@ public static RelNode decorrelateQuery(RelNode rel, RelBuilder builder) { return postPlanner.findBestExp(); } + /** Returns whether correlated Sorts in a tree have OFFSET and FETCH values + * that can be decorrelated without changing their row-count semantics. */ + private static boolean canDecorrelateOffsetFetch(RelNode rel, + boolean isCorVarDefined) { + if (isCorVarDefined && rel instanceof Sort + && !RelDecorrelator.canDecorrelateOffsetFetch((Sort) rel)) { + return false; + } + if (rel instanceof Correlate) { + final Correlate correlate = (Correlate) rel; + if (!canDecorrelateOffsetFetch(correlate.getLeft(), isCorVarDefined)) { + return false; + } + return canDecorrelateOffsetFetch(correlate.getRight(), true); + } + for (RelNode input : rel.getInputs()) { + if (!canDecorrelateOffsetFetch(input, isCorVarDefined)) { + return false; + } + } + return true; + } + /** * Eliminates Correlate. * diff --git a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java index 36d56a9f0488..2309102ff826 100644 --- a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java +++ b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java @@ -86,6 +86,7 @@ import org.apache.calcite.rex.RexSubQuery; import org.apache.calcite.rex.RexUnknownAs; import org.apache.calcite.rex.RexUtil; +import org.apache.calcite.rex.RexVisitorImpl; import org.apache.calcite.rex.RexWindowBound; import org.apache.calcite.rex.RexWindowBounds; import org.apache.calcite.rex.RexWindowExclusion; @@ -108,6 +109,7 @@ import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.sql.type.SqlTypeUtil; import org.apache.calcite.sql.type.TableFunctionReturnTypeInference; import org.apache.calcite.sql.validate.SqlValidatorUtil; import org.apache.calcite.sql2rel.SqlToRelConverter; @@ -3801,8 +3803,7 @@ public RelBuilder sortLimit(Number offset, Number fetch, * * @param offsetNode RexLiteral means number of rows to skip is deterministic, * RexDynamicParam means number of rows to skip is dynamic. - * @param fetchNode RexLiteral means maximum number of rows to fetch is deterministic, - * RexDynamicParam mean maximum number is dynamic. + * @param fetchNode Maximum number of rows to fetch * @param nodes Sort expressions */ public RelBuilder sortLimit(@Nullable RexNode offsetNode, @Nullable RexNode fetchNode, @@ -3812,12 +3813,17 @@ public RelBuilder sortLimit(@Nullable RexNode offsetNode, @Nullable RexNode fetc throw new IllegalArgumentException("OFFSET node must be RexLiteral or RexDynamicParam"); } } - if (fetchNode != null) { - if (!(fetchNode instanceof RexLiteral || fetchNode instanceof RexDynamicParam)) { - throw new IllegalArgumentException("FETCH node must be RexLiteral or RexDynamicParam"); - } + if (fetchNode != null && !isValidFetchExpression(fetchNode)) { + throw new IllegalArgumentException( + "FETCH node must not reference input fields or contain aggregate functions, " + + "window functions, or subqueries"); + } + if (fetchNode != null + && !SqlTypeUtil.isNumeric(fetchNode.getType())) { + throw new IllegalArgumentException( + "FETCH node must have a numeric type; actual type is " + + fetchNode.getType().getFullTypeString()); } - final Registrar registrar = new Registrar(fields(), ImmutableList.of()); final List fieldCollations = registrar.registerFieldCollations(nodes); @@ -3884,6 +3890,38 @@ public RelBuilder sortLimit(@Nullable RexNode offsetNode, @Nullable RexNode fetc return this; } + private static boolean isValidFetchExpression(RexNode node) { + return Boolean.TRUE.equals(node.accept(new FetchExpressionVisitor())); + } + + /** Visitor that validates FETCH expressions. */ + private static class FetchExpressionVisitor + extends RexVisitorImpl<@Nullable Boolean> { + FetchExpressionVisitor() { + super(false); + } + + @Override public Boolean visitLiteral(RexLiteral literal) { + return true; + } + + @Override public Boolean visitDynamicParam(RexDynamicParam dynamicParam) { + return true; + } + + @Override public Boolean visitCall(RexCall call) { + if (call.getOperator().isAggregator()) { + return false; + } + for (RexNode operand : call.getOperands()) { + if (!Boolean.TRUE.equals(operand.accept(this))) { + return false; + } + } + return true; + } + } + private static RelFieldCollation collation(RexNode node, RelFieldCollation.Direction direction, RelFieldCollation.@Nullable NullDirection nullDirection, diff --git a/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties b/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties index f4f16d73266a..19f3ef47a8d3 100644 --- a/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties +++ b/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties @@ -61,6 +61,9 @@ ValidatorContext=From line {0,number,#}, column {1,number,#} to line {2,number,# CannotCastValue=Cast function cannot convert value of type {0} to type {1} UnknownDatatypeName=Unknown datatype name ''{0}'' IncompatibleValueType=Values passed to {0} operator must have compatible types +FetchExpressionMustBeNumeric=FETCH expression must have a numeric type; actual type is ''{0}'' +FetchExpressionCannotReferenceColumn=FETCH expression cannot reference table column ''{0}'' +FetchExpressionEvaluatedToNull=FETCH expression evaluated to NULL IncompatibleTypesInList=Values in expression list must have compatible types IncompatibleCharset=Cannot apply operation ''{0}'' to strings with different charsets ''{1}'' and ''{2}'' InvalidOrderByPos=ORDER BY is only allowed on top-level SELECT diff --git a/core/src/test/java/org/apache/calcite/adapter/enumerable/EnumUtilsTest.java b/core/src/test/java/org/apache/calcite/adapter/enumerable/EnumUtilsTest.java index 40b825939df9..70370d7bb1ab 100644 --- a/core/src/test/java/org/apache/calcite/adapter/enumerable/EnumUtilsTest.java +++ b/core/src/test/java/org/apache/calcite/adapter/enumerable/EnumUtilsTest.java @@ -34,6 +34,7 @@ import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; /** * Tests for {@link EnumUtils}. @@ -186,6 +187,16 @@ public final class EnumUtilsTest { is(BigDecimal.valueOf(2))); } + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testNumberToBigDecimalRejectsNull() { + final IllegalArgumentException e = + assertThrows(IllegalArgumentException.class, + () -> EnumUtils.numberToBigDecimal(null, "FETCH")); + assertThat(e.getMessage(), is("FETCH expression evaluated to NULL")); + } + @Test void testMethodCallExpression() { // test for Object.class method parameter type final ConstantExpression arg0 = Expressions.constant(1, int.class); diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java index b25c213defcb..44dc09078d02 100644 --- a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java +++ b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java @@ -3032,25 +3032,25 @@ private SqlDialect nonOrdinalDialect() { + " as MAP array)"; final String expectedClickHouse2 = "SELECT CAST(array(map('a', '1'), map('b', '2'), map('c', '3'))" - + " AS Array(Map(`String`, `String`)))"; + + " AS Array(Map(`String`, `Nullable(String)`)))"; sql(query2).withClickHouse().ok(expectedClickHouse2); final String query3 = "select cast(MAP['a',ARRAY[1,2,3]]" + " as MAP)"; final String expectedClickHouse3 = - "SELECT CAST(map('a', array(1, 2, 3)) AS Map(`String`, Array(`Int32`)))"; + "SELECT CAST(map('a', array(1, 2, 3)) AS Map(`String`, Array(`Nullable(Int32)`)))"; sql(query3).withClickHouse().ok(expectedClickHouse3); final String query4 = "select cast(MAP['a',ARRAY[1.0,2.0,3.0]]" + " as MAP)"; final String expectedClickHouse4 = - "SELECT CAST(map('a', array(1.0, 2.0, 3.0)) AS Map(`String`, Array(`Float32`)))"; + "SELECT CAST(map('a', array(1.0, 2.0, 3.0)) AS Map(`String`, Array(`Nullable(Float32)`)))"; sql(query4).withClickHouse().ok(expectedClickHouse4); final String query5 = "select cast(MAP['a',MAP['b','c']]" + " as MAP>)"; final String expectedClickHouse5 = - "SELECT CAST(map('a', map('b', 'c')) AS Map(`String`, Map(`String`, `String`)))"; + "SELECT CAST(map('a', map('b', 'c')) AS Map(`String`, Map(`String`, `Nullable(String)`)))"; sql(query5).withClickHouse().ok(expectedClickHouse5); } @@ -4918,6 +4918,74 @@ private SqlDialect nonOrdinalDialect() { .withSybase().ok(expectedSybase); } + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testFetchExpressionWithLimitDialect() { + final String query = "select \"product_id\"\n" + + "from \"product\"\n" + + "fetch next (1 + 2) rows only"; + final String expected = "SELECT `product_id`\n" + + "FROM `foodmart`.`product`\n" + + "LIMIT 3"; + sql(query).withMysql().ok(expected); + } + + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testNegativeFetchExpressionIsRejectedBeforeSqlGeneration() { + final String query = "select \"product_id\"\n" + + "from \"product\"\n" + + "fetch next (0 - 1) rows only"; + final String error = + "FETCH value -1 is out of range; expected a non-negative value"; + sql(query).throws_(error); + sql(query).withMysql().throws_(error); + sql(query).withSQLite().throws_(error); + } + + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testParameterizedFetchExpressionWithLimitDialect() { + final String query = "select \"product_id\"\n" + + "from \"product\"\n" + + "fetch next (? + 1) rows only"; + sql(query).withMysql().throws_( + "LIMIT dialect does not support FETCH expressions that cannot " + + "be reduced to a literal"); + } + + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testParameterizedFetchExpressionWithSQLite() { + final String query = "select \"product_id\"\n" + + "from \"product\"\n" + + "fetch next (? + 1) rows only"; + final String expected = "SELECT \"product_id\"\n" + + "FROM \"foodmart\".\"product\"\n" + + "LIMIT ? + 1"; + sql(query).withSQLite().ok(expected); + } + + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testDynamicFetchExpressionIsNotReduced() { + final String query = "select \"product_id\"\n" + + "from \"product\"\n" + + "fetch next (extract(day from current_date)) rows only"; + final String expected = "SELECT \"product_id\"\n" + + "FROM \"foodmart\".\"product\"\n" + + "FETCH NEXT (EXTRACT(DAY FROM CURRENT_DATE)) ROWS ONLY"; + sql(query).ok(expected); + sql(query).withMysql().throws_( + "LIMIT dialect does not support FETCH expressions that cannot " + + "be reduced to a literal"); + } + @Test void testSelectQueryComplex() { String query = "select count(*), \"units_per_case\" from \"product\" where \"cases_per_pallet\" > 100 " @@ -5652,15 +5720,15 @@ private SqlDialect nonOrdinalDialect() { @Test void testCastAsMapType() { sql("SELECT CAST(MAP['A', 1.0] AS MAP)") .ok("SELECT CAST(MAP['A', 1.0] AS " - + "MAP< VARCHAR CHARACTER SET \"ISO-8859-1\", DOUBLE >)\n" + + "MAP< VARCHAR CHARACTER SET \"ISO-8859-1\", DOUBLE NULL >)\n" + "FROM (VALUES (0)) AS \"t\" (\"ZERO\")"); sql("SELECT CAST(MAP['A', ARRAY[1, 2, 3]] AS MAP)") .ok("SELECT CAST(MAP['A', ARRAY[1, 2, 3]] AS " - + "MAP< VARCHAR CHARACTER SET \"ISO-8859-1\", INTEGER ARRAY >)\n" + + "MAP< VARCHAR CHARACTER SET \"ISO-8859-1\", INTEGER ARRAY NULL >)\n" + "FROM (VALUES (0)) AS \"t\" (\"ZERO\")"); sql("SELECT CAST(MAP[ARRAY['A'], MAP[1, 2]] AS MAP>)") .ok("SELECT CAST(MAP[ARRAY['A'], MAP[1, 2]] AS " - + "MAP< VARCHAR CHARACTER SET \"ISO-8859-1\" ARRAY, MAP< INTEGER, INTEGER > >)\n" + + "MAP< VARCHAR CHARACTER SET \"ISO-8859-1\" ARRAY, MAP< INTEGER, INTEGER NULL > NULL >)\n" + "FROM (VALUES (0)) AS \"t\" (\"ZERO\")"); } diff --git a/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java b/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java index 60fd60a83751..bf506ceb121a 100644 --- a/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java +++ b/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java @@ -83,6 +83,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.fail; import static java.util.Objects.requireNonNull; @@ -3593,6 +3594,35 @@ private void assertTypeAndToString( hasSize(0)); } + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testContainsDynamicParam() { + final RelDataType intType = typeFactory.createSqlType(SqlTypeName.INTEGER); + final RexNode literal = rexBuilder.makeExactLiteral(BigDecimal.ONE, intType); + final RexNode dynamicParam = rexBuilder.makeDynamicParam(intType, 0); + final RexNode expression = + rexBuilder.makeCall(SqlStdOperatorTable.PLUS, literal, dynamicParam); + + assertThat(RexUtil.containsDynamicParam(literal), is(false)); + assertThat(RexUtil.containsDynamicParam(dynamicParam), is(true)); + assertThat(RexUtil.containsDynamicParam(expression), is(true)); + } + + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testValidateFetchValueAllowsFractionalBigDecimal() { + assertThat(RexUtil.validateFetchValue(new BigDecimal("1.5")), + is(new BigDecimal("1.5"))); + + final IllegalArgumentException e = + assertThrows(IllegalArgumentException.class, + () -> RexUtil.validateFetchValue(new BigDecimal("-1.5"))); + assertThat(e.getMessage(), + containsString("FETCH value -1.5 is out of range")); + } + @Test void testConstantMap() { final RelDataType intType = typeFactory.createSqlType(SqlTypeName.INTEGER); final RelDataType bigintType = typeFactory.createSqlType(SqlTypeName.BIGINT); diff --git a/core/src/test/java/org/apache/calcite/sql/type/SqlTypeFactoryTest.java b/core/src/test/java/org/apache/calcite/sql/type/SqlTypeFactoryTest.java index 68a34eb5bac7..f0ff190d28c7 100644 --- a/core/src/test/java/org/apache/calcite/sql/type/SqlTypeFactoryTest.java +++ b/core/src/test/java/org/apache/calcite/sql/type/SqlTypeFactoryTest.java @@ -87,42 +87,6 @@ class SqlTypeFactoryTest { assertThat(leastRestrictive.isNullable(), is(true)); } - /** UNKNOWN types in leastRestrictive() affect only the result nullability. */ - @Test void testLeastRestrictiveWithUnknown() { - SqlTypeFixture f = new SqlTypeFixture(); - // UNKNOWN never constrains the result, no matter its position - checkUnknownWithType(f, f.sqlBigInt); - checkUnknownWithType(f, f.structOfInt); - checkUnknownWithType(f, f.arrayBigInt); - checkUnknownWithType(f, f.mapOfInt); - // A nullable UNKNOWN makes the result nullable - RelDataType leastRestrictive = - f.typeFactory.leastRestrictive( - Lists.newArrayList(f.structOfInt, - f.typeFactory.createTypeWithNullability(f.sqlUnknown, true))); - assertThat(leastRestrictive, notNullValue()); - assertThat(leastRestrictive.getSqlTypeName(), is(SqlTypeName.ROW)); - assertThat(leastRestrictive.isNullable(), is(true)); - // A list of UNKNOWN unifies to UNKNOWN - leastRestrictive = - f.typeFactory.leastRestrictive( - Lists.newArrayList(f.sqlUnknown, f.sqlUnknown)); - assertThat(leastRestrictive.getSqlTypeName(), is(SqlTypeName.UNKNOWN)); - } - - /** Checks that leastRestictive({@code type}, UNKNOWN) yields {@code type}, - * in either order. */ - private void checkUnknownWithType(SqlTypeFixture f, RelDataType type) { - RelDataType r1 = - f.typeFactory.leastRestrictive(Lists.newArrayList(type, f.sqlUnknown)); - RelDataType r2 = - f.typeFactory.leastRestrictive(Lists.newArrayList(f.sqlUnknown, type)); - assertThat(r1, notNullValue()); - assertThat(r2, notNullValue()); - assertThat(r1.getFullTypeString(), is(type.getFullTypeString())); - assertThat(r2.getFullTypeString(), is(type.getFullTypeString())); - } - @Test void testLeastRestrictiveStructWithNull() { SqlTypeFixture f = new SqlTypeFixture(); RelDataType leastRestrictive = diff --git a/core/src/test/java/org/apache/calcite/sql2rel/CorrelateProjectExtractorTest.java b/core/src/test/java/org/apache/calcite/sql2rel/CorrelateProjectExtractorTest.java index da0a17296b3c..72276d42c332 100644 --- a/core/src/test/java/org/apache/calcite/sql2rel/CorrelateProjectExtractorTest.java +++ b/core/src/test/java/org/apache/calcite/sql2rel/CorrelateProjectExtractorTest.java @@ -84,78 +84,6 @@ public static Frameworks.ConfigBuilder config() { assertThat(after, hasTree(planAfter)); } - /** Test case for [CALCITE-7646] - * CorrelateProjectExtractor does not handle nested field accesses cor0.field0.field1. */ - @Test void testNestedCorrelationFieldAccessInFilter() { - final RelBuilder builder = RelBuilder.create(config().build()); - final Holder<@Nullable RexCorrelVariable> v = Holder.empty(); - RelNode before = builder.scan("EMP") - .project( - builder.alias( - builder.call(SqlStdOperatorTable.ROW, - builder.field("EMPNO"), builder.field("DEPTNO")), "R")) - .variable(v::set) - .scan("DEPT") - .filter( - builder.equals(builder.field(0), - builder.getRexBuilder().makeFieldAccess(builder.field(v.get(), "R"), 1))) - .correlate(JoinRelType.LEFT, v.get().id, builder.field(2, 0, "R")) - .build(); - - final String planBefore = "" - + "LogicalCorrelate(correlation=[$cor0], joinType=[left], requiredColumns=[{0}])\n" - + " LogicalProject(R=[ROW($0, $7)])\n" - + " LogicalTableScan(table=[[scott, EMP]])\n" - + " LogicalFilter(condition=[=($0, $cor0.R.EXPR$1)])\n" - + " LogicalTableScan(table=[[scott, DEPT]])\n"; - assertThat(before, hasTree(planBefore)); - - RelNode after = before.accept(new CorrelateProjectExtractor(RelFactories.LOGICAL_BUILDER)); - final String planAfter = "" - + "LogicalProject(R=[$0], DEPTNO=[$2], DNAME=[$3], LOC=[$4])\n" - + " LogicalCorrelate(correlation=[$cor0], joinType=[left], requiredColumns=[{1}])\n" - + " LogicalProject(R=[ROW($0, $7)], $f1=[ROW($0, $7).EXPR$1])\n" - + " LogicalTableScan(table=[[scott, EMP]])\n" - + " LogicalFilter(condition=[=($0, $cor0.$f1)])\n" - + " LogicalTableScan(table=[[scott, DEPT]])\n"; - assertThat(after, hasTree(planAfter)); - } - - /** Tests that a constant call operand, such as {@code POWER(2, 3)}, does - * not prevent extracting the enclosing correlated call. */ - @Test void testCorrelationCallWithConstantCallOperandInFilter() { - final RelBuilder builder = RelBuilder.create(config().build()); - final Holder<@Nullable RexCorrelVariable> v = Holder.empty(); - RelNode before = builder.scan("EMP") - .variable(v::set) - .scan("DEPT") - .filter( - builder.equals(builder.field(0), - builder.call(SqlStdOperatorTable.PLUS, - builder.call(SqlStdOperatorTable.POWER, - builder.literal(2), builder.literal(3)), - builder.field(v.get(), "DEPTNO")))) - .correlate(JoinRelType.LEFT, v.get().id, builder.field(2, 0, "DEPTNO")) - .build(); - - final String planBefore = "" - + "LogicalCorrelate(correlation=[$cor0], joinType=[left], requiredColumns=[{7}])\n" - + " LogicalTableScan(table=[[scott, EMP]])\n" - + " LogicalFilter(condition=[=($0, +(POWER(2, 3), $cor0.DEPTNO))])\n" - + " LogicalTableScan(table=[[scott, DEPT]])\n"; - assertThat(before, hasTree(planBefore)); - - RelNode after = before.accept(new CorrelateProjectExtractor(RelFactories.LOGICAL_BUILDER)); - final String planAfter = "" - + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7], DEPTNO0=[$9], DNAME=[$10], LOC=[$11])\n" - + " LogicalCorrelate(correlation=[$cor0], joinType=[left], requiredColumns=[{8}])\n" - + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7], $f8=[+(POWER(2, 3), $7)])\n" - + " LogicalTableScan(table=[[scott, EMP]])\n" - + " LogicalFilter(condition=[=($0, $cor0.$f8)])\n" - + " LogicalTableScan(table=[[scott, DEPT]])\n"; - assertThat(after, hasTree(planAfter)); - } - @Test void testDoubleCorrelationCallOverVariableInFilters() { final RelBuilder builder = RelBuilder.create(config().build()); final Holder<@Nullable RexCorrelVariable> v = Holder.empty(); diff --git a/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java b/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java index ac218b220afa..240f9a36ae02 100644 --- a/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java +++ b/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java @@ -290,9 +290,9 @@ public static Frameworks.ConfigBuilder config() { + " LogicalJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left])\n" + " LogicalValues(tuples=[[{ 7369 }, { 7499 }]])\n" + " LogicalAggregate(group=[{0}], agg#0=[SINGLE_VALUE($1)])\n" - + " LogicalProject(EMPNO1=[$11], EXPR$0=[||(||($1, ' from dept '), $12)])\n" - + " LogicalJoin(condition=[AND(=($7, $9), =($8, $10))], joinType=[left])\n" - + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7], EMPNO0=[CAST($0):INTEGER NOT NULL])\n" + + " LogicalProject(EMPNO1=[$12], EXPR$0=[||(||($1, ' from dept '), $13)])\n" + + " LogicalJoin(condition=[AND(=($7, $10), =($9, $11))], joinType=[left])\n" + + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7], DEPTNO0=[$7], EMPNO0=[CAST($0):INTEGER NOT NULL])\n" + " LogicalTableScan(table=[[scott, EMP]])\n" + " LogicalAggregate(group=[{0, 1, 2}], agg#0=[SINGLE_VALUE($3)])\n" + " LogicalProject(DEPTNO0=[$3], EMPNO0=[$4], EMPNO=[$5], DNAME=[$1])\n" @@ -556,29 +556,29 @@ public static Frameworks.ConfigBuilder config() { // LogicalTableScan(table=[[scott, EMP]]) final String planAfter = "" + "LogicalSort(sort0=[$0], dir0=[ASC])\n" - + " LogicalProject(DNAME=[$1], C=[$6])\n" - + " LogicalJoin(condition=[AND(=($0, $4), =($3, $5))], joinType=[left])\n" - + " LogicalProject(DEPTNO=[$0], DNAME=[$1], LOC=[$2], $f3=[*($0, 100)])\n" + + " LogicalProject(DNAME=[$1], C=[$7])\n" + + " LogicalJoin(condition=[AND(=($0, $5), =($4, $6))], joinType=[left])\n" + + " LogicalProject(DEPTNO=[$0], DNAME=[$1], LOC=[$2], DEPTNO0=[$0], $f4=[*($0, 100)])\n" + " LogicalTableScan(table=[[scott, DEPT]])\n" - + " LogicalProject(DEPTNO8=[$0], $f3=[$1], EXPR$0=[CASE(IS NOT NULL($4), $4, 0)])\n" + + " LogicalProject(DEPTNO8=[$0], $f4=[$1], EXPR$0=[CASE(IS NOT NULL($4), $4, 0)])\n" + " LogicalJoin(condition=[AND(IS NOT DISTINCT FROM($0, $2), IS NOT DISTINCT FROM($1, $3))], joinType=[left])\n" - + " LogicalProject(DEPTNO=[$0], $f3=[*($0, 100)])\n" + + " LogicalProject(DEPTNO=[$0], $f4=[*($0, 100)])\n" + " LogicalTableScan(table=[[scott, DEPT]])\n" + " LogicalAggregate(group=[{0, 1}], EXPR$0=[COUNT()])\n" - + " LogicalProject(DEPTNO8=[$7], $f3=[$9])\n" + + " LogicalProject(DEPTNO8=[$7], $f4=[$9])\n" + " LogicalFilter(condition=[IS NOT NULL($7)])\n" - + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7], i=[$11], $f3=[$9])\n" + + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7], i=[$11], $f4=[$9])\n" + " LogicalJoin(condition=[=($8, $10)], joinType=[inner])\n" + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7], SAL0=[CAST($5):DECIMAL(12, 2)])\n" + " LogicalTableScan(table=[[scott, EMP]])\n" - + " LogicalProject($f3=[$0], SAL0=[$1], $f2=[true])\n" + + " LogicalProject($f4=[$0], SAL0=[$1], $f2=[true])\n" + " LogicalAggregate(group=[{0, 1}])\n" - + " LogicalProject($f3=[$1], SAL0=[$2])\n" + + " LogicalProject($f4=[$1], SAL0=[$2])\n" + " LogicalJoin(condition=[AND(>($2, CAST($0):DECIMAL(12, 2) NOT NULL), <($1, $0))], joinType=[inner])\n" + " LogicalValues(tuples=[[{ 1000 }, { 2000 }, { 3000 }]])\n" + " LogicalJoin(condition=[true], joinType=[inner])\n" + " LogicalAggregate(group=[{0}])\n" - + " LogicalProject($f3=[*($0, 100)])\n" + + " LogicalProject($f4=[*($0, 100)])\n" + " LogicalTableScan(table=[[scott, DEPT]])\n" + " LogicalAggregate(group=[{0}])\n" + " LogicalProject(SAL0=[CAST($5):DECIMAL(12, 2)])\n" @@ -1793,9 +1793,9 @@ public static Frameworks.ConfigBuilder config() { RelDecorrelator.decorrelateQuery(before, builder, RuleSets.ofList(Collections.emptyList()), RuleSets.ofList(Collections.emptyList())); final String planAfter = "" - + "LogicalProject(DEPTNO=[$0], I0=[$3], I1=[$7])\n" - + " LogicalJoin(condition=[AND(=($0, $5), =($4, $6))], joinType=[left])\n" - + " LogicalProject(DEPTNO=[$0], DNAME=[$1], LOC=[$2], EXPR$0=[$5], $f4=[>(CAST($0):INTEGER NOT NULL, 0)])\n" + + "LogicalProject(DEPTNO=[$0], I0=[$3], I1=[$8])\n" + + " LogicalJoin(condition=[AND(=($0, $6), =($5, $7))], joinType=[left])\n" + + " LogicalProject(DEPTNO=[$0], DNAME=[$1], LOC=[$2], EXPR$0=[$5], DEPTNO0=[$0], $f5=[>(CAST($0):INTEGER NOT NULL, 0)])\n" + " LogicalJoin(condition=[=($3, $4)], joinType=[left])\n" + " LogicalProject(DEPTNO=[$0], DNAME=[$1], LOC=[$2], DEPTNO0=[CAST($0):SMALLINT NOT NULL])\n" + " LogicalTableScan(table=[[scott, DEPT]])\n" @@ -1807,12 +1807,12 @@ public static Frameworks.ConfigBuilder config() { + " LogicalProject(DEPTNO0=[CAST($0):SMALLINT NOT NULL])\n" + " LogicalTableScan(table=[[scott, DEPT]])\n" + " LogicalAggregate(group=[{0, 1}], EXPR$0=[MIN($2)])\n" - + " LogicalProject(DEPTNO0=[$8], $f4=[$9], $f0=[0])\n" + + " LogicalProject(DEPTNO0=[$8], $f5=[$9], $f0=[0])\n" + " LogicalJoin(condition=[=($7, $8)], joinType=[inner])\n" + " LogicalFilter(condition=[=($1, 'SMITH')])\n" + " LogicalTableScan(table=[[scott, EMP]])\n" + " LogicalFilter(condition=[$1])\n" - + " LogicalProject(DEPTNO=[$0], $f4=[>(CAST($0):INTEGER NOT NULL, 0)])\n" + + " LogicalProject(DEPTNO=[$0], $f5=[>(CAST($0):INTEGER NOT NULL, 0)])\n" + " LogicalJoin(condition=[=($3, $4)], joinType=[left])\n" + " LogicalProject(DEPTNO=[$0], DNAME=[$1], LOC=[$2], DEPTNO0=[CAST($0):SMALLINT NOT NULL])\n" + " LogicalTableScan(table=[[scott, DEPT]])\n" @@ -2351,67 +2351,4 @@ public static Frameworks.ConfigBuilder config() { + " LogicalTableScan(table=[[scott, EMP]])\n"; assertThat(after, hasTree(planAfter)); } - - /** Test case for - * [CALCITE-7646] - * CorrelateProjectExtractor does not handle nested field accesses cor0.field0.field1. */ - @Test void testNestedCorrelatedFieldAccess() throws SqlParseException { - final String sql = "select a.\"aid\", t.lat\n" - + "from \"bookstore\".\"authors\" a,\n" - + "lateral (select b.\"aid\" as c,\n" - + " (a.\"birthPlace\").\"coords\".\"latitude\" as lat\n" - + " from \"bookstore\".\"authors\" b\n" - + " where b.\"aid\" = a.\"aid\") as t"; - SchemaPlus rootSchema = Frameworks.createRootSchema(true); - CalciteAssert.addSchema(rootSchema, CalciteAssert.SchemaSpec.BOOKSTORE); - CalciteConnectionConfig config = new CalciteConnectionConfigImpl(new Properties()); - // The Frameworks planner cannot be used here because it flattens - // structured types, and RelStructuredTypeFlattener does not support - // correlations on structured columns. - SqlTestFactory factory = SqlTestFactory.INSTANCE - .withCatalogReader((typeFactory, caseSensitive) -> - new CalciteCatalogReader( - CalciteSchema.from(rootSchema), - ImmutableList.of("bookstore"), - typeFactory, - config)); - SqlParser parser = factory.createParser(sql); - SqlNode parsed = parser.parseQuery(); - final SqlToRelConverter sqlToRelConverter = factory.createSqlToRelConverter(); - assert sqlToRelConverter.validator != null; - final SqlNode validated = sqlToRelConverter.validator.validate(parsed); - final RelNode before = sqlToRelConverter.convertQuery(validated, false, true).rel; - - final String planBefore = "" - + "LogicalProject(aid=[$0], LAT=[$5])\n" - + " LogicalCorrelate(correlation=[$cor1], joinType=[inner], requiredColumns=[{0, 2}])\n" - + " LogicalTableScan(table=[[bookstore, authors]])\n" - + " LogicalProject(C=[$0], LAT=[$cor1.birthPlace.coords.latitude])\n" - + " LogicalFilter(condition=[=($0, $cor1.aid)])\n" - + " LogicalTableScan(table=[[bookstore, authors]])\n"; - assertThat(before, hasTree(planBefore)); - - final RelBuilder relBuilder = - RelFactories.LOGICAL_BUILDER.create(before.getCluster(), null); - // Decorrelate without any rules, just "purely" decorrelation algorithm on RelDecorrelator - final RelNode after = - RelDecorrelator.decorrelateQuery(before, relBuilder, - RuleSets.ofList(Collections.emptyList()), - RuleSets.ofList(Collections.emptyList())); - - // The nested field access is extracted into the projection $f4 on the - // left side and no correlation variables remain. - final String planAfter = "" - + "LogicalProject(aid=[$0], LAT=[$6])\n" - + " LogicalJoin(condition=[AND(=($0, $7), IS NOT DISTINCT FROM($4, $8))], joinType=[inner])\n" - + " LogicalProject(aid=[$0], name=[$1], birthPlace=[$2], books=[$3], $f4=[$2.coords.latitude])\n" - + " LogicalTableScan(table=[[bookstore, authors]])\n" - + " LogicalProject(C=[$0], LAT=[$4], aid=[$0], $f4=[$4])\n" - + " LogicalJoin(condition=[true], joinType=[inner])\n" - + " LogicalTableScan(table=[[bookstore, authors]])\n" - + " LogicalAggregate(group=[{0}])\n" - + " LogicalProject($f4=[$2.coords.latitude])\n" - + " LogicalTableScan(table=[[bookstore, authors]])\n"; - assertThat(after, hasTree(planAfter)); - } } diff --git a/core/src/test/java/org/apache/calcite/test/JdbcTest.java b/core/src/test/java/org/apache/calcite/test/JdbcTest.java index 83ce5b1d78a6..f213b12dcb7e 100644 --- a/core/src/test/java/org/apache/calcite/test/JdbcTest.java +++ b/core/src/test/java/org/apache/calcite/test/JdbcTest.java @@ -3581,6 +3581,181 @@ public void checkOrderBy(final boolean desc, + "store_id=4; grocery_sqft=16844\n"); } + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testFetchExpression() { + CalciteAssert.that() + .query("select * from (values (1), (2), (3), (4)) as t(x)\n" + + "fetch next (1 + abs(-2)) rows only") + .returns("X=1\n" + + "X=2\n" + + "X=3\n"); + } + + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testBindableFetchExpression() { + try (Hook.Closeable ignored = Hook.ENABLE_BINDABLE.addThread(Hook.propertyJ(true))) { + final CalciteAssert.AssertThat with = CalciteAssert.that(); + with + .query("select * from (values (1), (2), (3), (4)) as t(x)\n" + + "fetch next (rand_integer(1) + 2) rows only") + .explainContains("BindableSort(fetch=[+(RAND_INTEGER(1), 2)])") + .returns("X=1\n" + + "X=2\n"); + with.query("select * from (values (1), (2), (3), (4)) as t(x)\n" + + "fetch next (cast(9223372036854775808 as decimal(20, 0))) rows only") + .returns("X=1\nX=2\nX=3\nX=4\n"); + with.query("select * from (values (1), (2), (3), (4)) as t(x)\n" + + "order by x fetch next ? rows only") + .explainContains("BindableSort(sort0=[$0], dir0=[ASC], fetch=[?0])") + .consumesPreparedStatement(p -> + p.setBigDecimal(1, new BigDecimal("1.5"))) + .returns("X=1\n" + + "X=2\n"); + } + } + + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testFetchExpressionFunctionArguments() { + final CalciteAssert.AssertThat with = CalciteAssert.that(); + final String values = "select * from (values (1), (2), (3)) as t(x)\n"; + with.query(values + "fetch next (abs(2)) rows only") + .returns("X=1\n" + + "X=2\n"); + with.query(values + "fetch next (abs(-2)) rows only") + .returns("X=1\n" + + "X=2\n"); + } + + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testFetchExpressionInvalidValue() { + final CalciteAssert.AssertThat with = CalciteAssert.that(); + final String values = "select * from (values (1), (2), (3)) as t(x)\n"; + with.query(values + "fetch next (0 - 1) rows only") + .throws_("FETCH must not be negative"); + with.query(values + "fetch next (-1) rows only") + .throws_("FETCH must not be negative"); + with.query(values + + "fetch next (cast(null as integer)) rows only") + .throws_("FETCH expression evaluated to NULL"); + } + + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testCorrelatedFetchExpressionInvalidValue() { + final String sqlPrefix = "select d.\"name\", e.\"name\"\n" + + "from \"hr\".\"depts\" d,\n" + + "lateral (select \"name\" from \"hr\".\"emps\"\n" + + " where \"deptno\" = d.\"deptno\"\n"; + for (String fetch : new String[] {"(0 - 1)", "(-1)"}) { + for (boolean topDown : new boolean[] {false, true}) { + CalciteAssert.hr() + .with(CalciteConnectionProperty.TOPDOWN_GENERAL_DECORRELATION_ENABLED, topDown) + .query(sqlPrefix + " fetch next " + fetch + " rows only) e") + .throws_("FETCH value -1 is out of range"); + } + } + } + + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testCorrelatedFractionalOffsetFetch() { + final String sqlPrefix = "select d.\"name\" as dname, e.\"name\" as ename\n" + + "from \"hr\".\"depts\" d,\n" + + "lateral (select \"empid\", \"name\" from \"hr\".\"emps\"\n" + + " where \"deptno\" = d.\"deptno\"\n" + + " order by \"empid\" "; + final String sqlSuffix = ") e\norder by e.\"empid\""; + for (boolean topDown : new boolean[] {false, true}) { + final CalciteAssert.AssertThat with = CalciteAssert.hr() + .with(CalciteConnectionProperty.TOPDOWN_GENERAL_DECORRELATION_ENABLED, topDown); + with.query(sqlPrefix + "fetch next (0.5 + 1) rows only" + sqlSuffix) + .returns("DNAME=Sales; ENAME=Bill\n" + + "DNAME=Sales; ENAME=Theodore\n"); + with.query(sqlPrefix + "offset 1.5 rows fetch next 1 row only" + sqlSuffix) + .returns("DNAME=Sales; ENAME=Sebastian\n"); + } + } + + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testCorrelatedPreparedFractionalOffset() throws Exception { + final String sql = "select d.\"name\" as dname, e.\"name\" as ename\n" + + "from \"hr\".\"depts\" d,\n" + + "lateral (select \"empid\", \"name\" from \"hr\".\"emps\"\n" + + " where \"deptno\" = d.\"deptno\"\n" + + " order by \"empid\" offset ? rows fetch next 1 row only) e\n" + + "order by e.\"empid\""; + for (boolean topDown : new boolean[] {false, true}) { + CalciteAssert.hr() + .with(CalciteConnectionProperty.TOPDOWN_GENERAL_DECORRELATION_ENABLED, topDown) + .doWithConnection(connection -> { + checkPreparedBigDecimalParameter(connection, sql, + new BigDecimal("1.5"), + "DNAME=Sales; ENAME=Sebastian\n"); + }); + } + } + + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testCorrelatedPreparedFetchExpression() throws Exception { + for (String fetch : new String[] {"?", "(? + 0)"}) { + final String sql = "select d.\"name\" as dname, e.\"name\" as ename\n" + + "from \"hr\".\"depts\" d,\n" + + "lateral (select \"empid\", \"name\" from \"hr\".\"emps\"\n" + + " where \"deptno\" = d.\"deptno\"\n" + + " order by \"empid\" fetch next " + fetch + " rows only) e\n" + + "order by e.\"empid\""; + for (boolean topDown : new boolean[] {false, true}) { + CalciteAssert.hr() + .with(CalciteConnectionProperty.TOPDOWN_GENERAL_DECORRELATION_ENABLED, topDown) + .doWithConnection(connection -> { + checkPreparedFetchRepeated(connection, sql, + new int[] {1, 3}, + new String[] { + "DNAME=Sales; ENAME=Bill\n", + "DNAME=Sales; ENAME=Bill\n" + + "DNAME=Sales; ENAME=Theodore\n" + + "DNAME=Sales; ENAME=Sebastian\n" + }); + checkPreparedParameterFails(connection, sql, -1, + "FETCH must not be negative"); + checkPreparedParameterNullFails(connection, sql, + "FETCH expression evaluated to NULL"); + }); + } + } + } + + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testFetchExpressionBeyondLong() { + final CalciteAssert.AssertThat with = CalciteAssert.that(); + final String values = "select * from (values (1), (2), (3), (4)) as t(x)\n"; + final String expected = "X=1\nX=2\nX=3\nX=4\n"; + with.query(values + "fetch next 9223372036854775808 rows only") + .returns(expected); + with.query(values + "fetch next " + + "(cast(9223372036854775808 as decimal(20, 0)) + 1) rows only") + .returns(expected); + with.query(values + "order by x fetch next " + + "(cast(9223372036854775808 as decimal(20, 0)) + 1) rows only") + .returns(expected); + } + /** Tests ORDER BY ... OFFSET ... FETCH. */ @Test void testOrderByOffsetFetch() { CalciteAssert.that() @@ -6058,6 +6233,169 @@ private CalciteAssert.AssertQuery withEmpDept(String sql) { "name=Theodore"); } + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testPreparedFetchExpression() throws Exception { + CalciteAssert.that() + .doWithConnection(connection -> { + final String values = + "select * from (values (1), (2), (3), (4)) as t(x)\n"; + checkPreparedFetch(connection, values + "fetch next (?) rows only", + 2, "X=1\nX=2\n"); + checkPreparedFetch(connection, values + "fetch next (? + 1) rows only", + 2, "X=1\nX=2\nX=3\n"); + checkPreparedFetch(connection, + values + "fetch next (abs(cast(? as integer))) rows only", + 2, "X=1\nX=2\n"); + checkPreparedFetch(connection, + values + "fetch next (abs(cast(? as integer))) rows only", + -2, "X=1\nX=2\n"); + checkPreparedFetchRepeated(connection, + values + "fetch next (?) rows only", + new int[] {1, 3}, + new String[] {"X=1\n", "X=1\nX=2\nX=3\n"}); + checkPreparedFetchRepeated(connection, + values + "fetch next (? + 1) rows only", + new int[] {0, 2, 3}, + new String[] {"X=1\n", "X=1\nX=2\nX=3\n", + "X=1\nX=2\nX=3\nX=4\n"}); + checkPreparedFetch(connection, + values + "fetch next (? + abs(2)) rows only", + 1, "X=1\nX=2\nX=3\n"); + checkPreparedBigDecimalParameter(connection, + values + "fetch next (cast(? as decimal(20, 0))) rows only", + new BigDecimal("9223372036854775808"), + "X=1\nX=2\nX=3\nX=4\n"); + + checkPreparedParameterFails(connection, + values + "fetch next (?) rows only", -1, + "FETCH must not be negative"); + checkPreparedParameterFails(connection, + values + "fetch next (? + 1) rows only", -2, + "FETCH must not be negative"); + }); + } + + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testBindablePreparedFetchExpression() throws Exception { + try (Hook.Closeable ignored = Hook.ENABLE_BINDABLE.addThread(Hook.propertyJ(true))) { + CalciteAssert.that() + .doWithConnection(connection -> { + final String values = + "select * from (values (1), (2), (3), (4)) as t(x)\n"; + checkPreparedFetch(connection, + values + "fetch next (? + 1) rows only", + 2, "X=1\nX=2\nX=3\n"); + checkPreparedFetchRepeated(connection, + values + "fetch next (? + 1) rows only", + new int[] {0, 2, 3}, + new String[] {"X=1\n", "X=1\nX=2\nX=3\n", + "X=1\nX=2\nX=3\nX=4\n"}); + checkPreparedBigDecimalParameter(connection, + values + "fetch next (cast(? as decimal(20, 0))) rows only", + new BigDecimal("9223372036854775808"), + "X=1\nX=2\nX=3\nX=4\n"); + }); + } + } + + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testBindablePreparedOffset() throws Exception { + try (Hook.Closeable ignored = Hook.ENABLE_BINDABLE.addThread(Hook.propertyJ(true))) { + CalciteAssert.that() + .doWithConnection(connection -> { + final String values = + "select * from (values (1), (2), (3), (4)) as t(x)\n"; + final String offset = values + "offset ? rows"; + checkPreparedBigDecimalParameter(connection, offset, + new BigDecimal("1.5"), + "X=3\nX=4\n"); + checkPreparedBigDecimalParameter(connection, offset, + BigDecimal.valueOf(Integer.MAX_VALUE).add(BigDecimal.ONE), ""); + + final String sortedOffset = values + "order by x desc offset ? rows"; + checkPreparedBigDecimalParameter(connection, sortedOffset, + new BigDecimal("1.5"), + "X=2\nX=1\n"); + checkPreparedParameterFails(connection, offset, -1, + "OFFSET must not be negative"); + checkPreparedParameterNullFails(connection, offset, + "OFFSET expression evaluated to NULL"); + }); + } + } + + private static void checkPreparedFetch(Connection connection, String sql, + int value, String expected) { + try (PreparedStatement p = connection.prepareStatement(sql)) { + p.setInt(1, value); + try (ResultSet r = p.executeQuery()) { + assertThat(CalciteAssert.toString(r), is(expected)); + } + } catch (SQLException e) { + throw TestUtil.rethrow(e); + } + } + + private static void checkPreparedBigDecimalParameter(Connection connection, String sql, + BigDecimal value, String expected) { + try (PreparedStatement p = connection.prepareStatement(sql)) { + p.setBigDecimal(1, value); + try (ResultSet r = p.executeQuery()) { + assertThat(CalciteAssert.toString(r), is(expected)); + } + } catch (SQLException e) { + throw TestUtil.rethrow(e); + } + } + + private static void checkPreparedFetchRepeated(Connection connection, String sql, + int[] values, String[] expected) { + try (PreparedStatement p = connection.prepareStatement(sql)) { + for (int i = 0; i < values.length; i++) { + p.setInt(1, values[i]); + try (ResultSet r = p.executeQuery()) { + assertThat(CalciteAssert.toString(r), is(expected[i])); + } + } + } catch (SQLException e) { + throw TestUtil.rethrow(e); + } + } + + private static void checkPreparedParameterFails(Connection connection, String sql, + long value, String expectedMessage) { + try (PreparedStatement p = connection.prepareStatement(sql)) { + if (value >= Integer.MIN_VALUE && value <= Integer.MAX_VALUE) { + p.setInt(1, (int) value); + } else { + p.setLong(1, value); + } + final SQLException e = + assertThrows(SQLException.class, p::executeQuery); + assertThat(e.getMessage(), containsString(expectedMessage)); + } catch (SQLException e) { + throw TestUtil.rethrow(e); + } + } + + private static void checkPreparedParameterNullFails(Connection connection, String sql, + String expectedMessage) { + try (PreparedStatement p = connection.prepareStatement(sql)) { + p.setNull(1, Types.INTEGER); + final SQLException e = + assertThrows(SQLException.class, p::executeQuery); + assertThat(e.getMessage(), containsString(expectedMessage)); + } catch (SQLException e) { + throw TestUtil.rethrow(e); + } + } + private void checkPreparedOffsetFetch(final int offset, final int fetch, final Matcher matcher) throws Exception { CalciteAssert.hr() diff --git a/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java b/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java index e53f1027f78d..30aa2ff77fdf 100644 --- a/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java @@ -52,7 +52,10 @@ import org.apache.calcite.rex.RexCorrelVariable; import org.apache.calcite.rex.RexFieldCollation; import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexLambdaRef; import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexNodeAndFieldIndex; +import org.apache.calcite.rex.RexSubQuery; import org.apache.calcite.rex.RexWindowBounds; import org.apache.calcite.runtime.CalciteException; import org.apache.calcite.schema.SchemaPlus; @@ -5577,6 +5580,108 @@ private static RelNode buildCorrelateWithJoin(JoinRelType type, RelBuilder build assertThat(mq.getMaxRowCount(planAfter), is(Double.POSITIVE_INFINITY)); } + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testFetchExpressionCannotReferenceInputField() { + final RelBuilder builder = RelBuilder.create(config().build()); + builder.scan("DEPT"); + final RexNode field = builder.field("DEPTNO"); + + assertThrows(IllegalArgumentException.class, + () -> builder.sortLimit(null, field, ImmutableList.of())); + assertThrows(IllegalArgumentException.class, + () -> builder.sortLimit(null, + builder.call(SqlStdOperatorTable.PLUS, builder.literal(1), field), + ImmutableList.of())); + assertThrows(IllegalArgumentException.class, + () -> builder.sortLimit(null, + new RexNodeAndFieldIndex(0, 0, "DEPTNO", field.getType()), + ImmutableList.of())); + } + + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testFetchExpressionMustHaveNumericType() { + final RelBuilder builder = RelBuilder.create(config().build()); + builder.scan("DEPT"); + + assertThrows(IllegalArgumentException.class, + () -> builder.sortLimit(null, builder.literal("x"), ImmutableList.of())); + builder.sortLimit(null, builder.literal(new BigDecimal("1.5")), + ImmutableList.of()); + } + + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testFetchExpressionAllowsScalarCallAndDynamicParameter() { + final RelBuilder builder = RelBuilder.create(config().build()); + final RelDataType intType = + builder.getTypeFactory().createSqlType(SqlTypeName.INTEGER); + builder.scan("DEPT") + .sortLimit(null, + builder.call(SqlStdOperatorTable.PLUS, + builder.getRexBuilder().makeDynamicParam(intType, 0), + builder.literal(1)), + ImmutableList.of()); + + assertThat( + builder.build(), hasTree("LogicalSort(fetch=[+(?0, 1)])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n")); + } + + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testFetchExpressionCannotContainAggregateWindowOrSubQuery() { + final RelBuilder builder = RelBuilder.create(config().build()); + final RelDataType intType = + builder.getTypeFactory().createSqlType(SqlTypeName.INTEGER); + builder.scan("DEPT"); + final RexNode aggregate = + builder.call(SqlStdOperatorTable.SUM, builder.literal(1)); + assertThrows(IllegalArgumentException.class, + () -> builder.sortLimit(null, aggregate, ImmutableList.of())); + + final RexNode over = + builder.getRexBuilder().makeOver(intType, + SqlStdOperatorTable.ROW_NUMBER, ImmutableList.of(), + ImmutableList.of(), ImmutableList.of(), + RexWindowBounds.UNBOUNDED_PRECEDING, + RexWindowBounds.UNBOUNDED_FOLLOWING, + true, true, false, false, false); + assertThrows(IllegalArgumentException.class, + () -> builder.sortLimit(null, over, ImmutableList.of())); + + final RelBuilder subQueryBuilder = RelBuilder.create(config().build()); + final RexNode subQuery = + RexSubQuery.scalar(subQueryBuilder.values(new String[] {"N"}, 1).build()); + assertThrows(IllegalArgumentException.class, + () -> builder.sortLimit(null, subQuery, ImmutableList.of())); + } + + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testFetchExpressionCannotContainLambda() { + final RelBuilder builder = RelBuilder.create(config().build()); + final RelDataType intType = + builder.getTypeFactory().createSqlType(SqlTypeName.INTEGER); + builder.scan("DEPT"); + final RexLambdaRef lambdaRef = new RexLambdaRef(0, "x", intType); + final RexNode lambda = + builder.getRexBuilder().makeLambdaCall( + builder.call(SqlStdOperatorTable.PLUS, lambdaRef, builder.literal(1)), + ImmutableList.of(lambdaRef)); + + assertThrows(IllegalArgumentException.class, + () -> builder.sortLimit(null, lambda, ImmutableList.of())); + assertThrows(IllegalArgumentException.class, + () -> builder.sortLimit(null, lambdaRef, ImmutableList.of())); + } + @Test void testAdoptConventionEnumerable() { final RelBuilder builder = RelBuilder.create(config().build()); RelNode root = builder diff --git a/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java b/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java index 9c3c30e3448e..460e066051fe 100644 --- a/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java @@ -1467,7 +1467,7 @@ void testColumnOriginsUnion() { @Test void testRowCountSortLimitBeyondLong() { final BigDecimal fetch = BigDecimal.valueOf(Long.MAX_VALUE).add(BigDecimal.ONE); final double fetchDouble = fetch.doubleValue(); - final String sql = "select * from emp order by ename limit " + fetchDouble; + final String sql = "select * from emp order by ename limit " + fetch.toPlainString(); final RelMetadataFixture fixture = sql(sql); fixture.assertThatRowCount(is(EMP_SIZE), is(0D), is(fetchDouble)); } @@ -1496,6 +1496,37 @@ void testColumnOriginsUnion() { fixture.assertThatRowCount(is(1d), is(0D), is(0d)); } + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testMinRowCountFetchExpression() { + final String sql = "select * from (values (1), (2)) as t(x)\n" + + "fetch next (2 - 2) rows only"; + final RelMetadataFixture fixture = sql(sql); + fixture.assertThatRowCount(is(2D), is(0D), is(2D)); + + fixture + .withCluster(cluster -> { + final RelOptPlanner planner = new VolcanoPlanner(); + planner.addRule(EnumerableRules.ENUMERABLE_VALUES_RULE); + planner.addRule(EnumerableRules.ENUMERABLE_PROJECT_RULE); + planner.addRule(EnumerableRules.ENUMERABLE_LIMIT_RULE); + planner.addRelTraitDef(ConventionTraitDef.INSTANCE); + return RelOptCluster.create(planner, cluster.getRexBuilder()); + }) + .withRelTransform(rel -> { + final RelOptPlanner planner = rel.getCluster().getPlanner(); + planner.setRoot(rel); + final RelTraitSet requiredOutputTraits = + rel.getCluster().traitSet().replace(EnumerableConvention.INSTANCE); + final RelNode root = planner.changeTraits(rel, requiredOutputTraits); + planner.setRoot(root); + return planner.findBestExp(); + }) + .assertThatRel(is(instanceOf(EnumerableLimit.class))) + .assertThatRowCount(is(2D), is(0D), is(2D)); + } + @Test void testRowCountSortLimitOffset() { final String sql = "select * from emp order by ename limit 10 offset 5"; /* 14 - 5 */ diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index 60a5c56cba02..52c1875e3696 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -1738,6 +1738,34 @@ private void checkJoinProjectTransposeDoesNotMatch(JoinRelType type) { .check(); } + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testSortUnionTransposeWithNonDeterministicFetch() { + final String sql = "select a.name from dept a\n" + + "union all\n" + + "select b.name from dept b\n" + + "order by name fetch next (rand_integer(10)) rows only"; + sql(sql) + .withPreRule(CoreRules.PROJECT_SET_OP_TRANSPOSE) + .withRule(CoreRules.SORT_UNION_TRANSPOSE) + .checkUnchanged(); + } + + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testSortUnionTransposePushesParameterizedFetchExpression() { + final String sql = "select a.name from dept a\n" + + "union all\n" + + "select b.name from dept b\n" + + "order by name fetch next (? + 1) rows only"; + sql(sql) + .withPreRule(CoreRules.PROJECT_SET_OP_TRANSPOSE) + .withRule(CoreRules.SORT_UNION_TRANSPOSE) + .check(); + } + @Test void testSortRemovalAllKeysConstant() { final String sql = "select count(*) as c\n" + "from sales.emp\n" @@ -5981,10 +6009,9 @@ private void checkEmptyJoin(RelOptFixture f) { } /** Test case for - * [CALCITE-6647] - * SortUnionTransposeRule should not push SORT past a UNION when SORT's fetch is DynamicParam - . */ - @Test void testSortWithDynamicParam() { + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testSortWithDynamicParamPushesOnce() { HepProgramBuilder builder = new HepProgramBuilder(); builder.addRuleClass(SortProjectTransposeRule.class); builder.addRuleClass(SortUnionTransposeRule.class); @@ -9714,6 +9741,19 @@ private void checkSemiJoinRuleOnAntiJoin(RelOptRule rule) { .check(); } + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testDecorrelateProjectWithFetchExpression() { + final String query = "SELECT name, " + + "(SELECT sal FROM emp where dept.deptno = emp.deptno order by sal " + + "fetch next (1 + 0) rows only) " + + "FROM dept"; + sql(query).withRule(CoreRules.PROJECT_SUB_QUERY_TO_CORRELATE) + .withLateDecorrelate(true) + .check(); + } + /** Test case for [CALCITE-7289] * Select NULL subquery throwing exception. */ @Test void testNullSelect() { @@ -12202,6 +12242,39 @@ private static RelNode applyAggregateRemoveLiteralAggRule(RelNode rel) { .check(); } + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testNondeterministicFetchPreventsDecorrelation() { + checkNondeterministicFetchPreventsDecorrelation(false); + } + + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testNondeterministicFetchPreventsTopDownDecorrelation() { + checkNondeterministicFetchPreventsDecorrelation(true); + } + + private void checkNondeterministicFetchPreventsDecorrelation(boolean enableTopDown) { + final String sql = "select t.deptno, e.ename\n" + + "from (select distinct deptno from emp) t,\n" + + "lateral (select ename from emp\n" + + " where emp.deptno = t.deptno\n" + + " order by sal\n" + + " fetch next (rand_integer(2) + 1) rows only) e"; + + final RelOptFixture fixture = sql(sql) + .withRule() // empty program + .withLateDecorrelate(true) + .withTopDownGeneralDecorrelate(enableTopDown); + if (enableTopDown) { + fixture.check(); + } else { + fixture.checkUnchanged(); + } + } + @Test void testTopDownGeneralDecorrelateForFilterSome() { final String sql = "select empno from emp where " + "empno > SOME(select empno from emp_b where emp.ename = emp_b.ename)"; diff --git a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java index 6b3255e653c7..6ce401502c93 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java @@ -1263,6 +1263,15 @@ public static void checkActualAndReferenceFiles() { sql(sql).ok(); } + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testFetchWithExpression() { + final String sql = + "select empno from emp fetch next (1 + abs(-2)) rows only"; + sql(sql).ok(); + } + /** Test case for * [CALCITE-439] * SqlValidatorUtil.uniquify() may not terminate under some conditions. */ diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index 5e9224364ef3..9b0d0de70221 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -1527,7 +1527,7 @@ void testLikeAndSimilarFails() { expr("cast(ARRAY[1,2,3] AS VARIANT ARRAY)") .columnType("VARIANT NOT NULL ARRAY NOT NULL"); expr("cast(MAP['a','b','c','d'] AS MAP)") - .columnType("(VARCHAR NOT NULL, VARIANT NOT NULL) MAP NOT NULL"); + .columnType("(VARCHAR NOT NULL, VARIANT) MAP NOT NULL"); // Test case for [CALCITE-7293] https://issues.apache.org/jira/browse/CALCITE-7293 // MAP constructor cannot handle VARIANT values that need casts expr("MAP['a', CAST('x' AS VARIANT), 'b', CAST(NULL AS VARIANT)]") @@ -9640,16 +9640,16 @@ void testGroupExpressionEquivalenceParams() { @Test void testCastMapType() { sql("select cast(\"int2IntMapType\" as map) from COMPLEXTYPES.CTC_T1") .withExtendedCatalog() - .columnType("(INTEGER NOT NULL, INTEGER NOT NULL) MAP NOT NULL"); + .columnType("(INTEGER NOT NULL, INTEGER) MAP NOT NULL"); sql("select cast(\"int2varcharArrayMapType\" as map) " + "from COMPLEXTYPES.CTC_T1") .withExtendedCatalog() - .columnType("(INTEGER NOT NULL, VARCHAR NOT NULL ARRAY NOT NULL) MAP NOT NULL"); + .columnType("(INTEGER NOT NULL, VARCHAR ARRAY) MAP NOT NULL"); sql("select cast(\"varcharMultiset2IntIntMapType\" as map>)" + " from COMPLEXTYPES.CTC_T1") .withExtendedCatalog() - .columnType("(VARCHAR(5) NOT NULL MULTISET NOT NULL, " - + "(INTEGER NOT NULL, INTEGER NOT NULL) MAP NOT NULL) MAP NOT NULL"); + .columnType("(VARCHAR(5) MULTISET NOT NULL, " + + "(INTEGER NOT NULL, INTEGER) MAP) MAP NOT NULL"); } @Test void testCastAsRowType() { @@ -10707,6 +10707,22 @@ void testGroupExpressionEquivalenceParams() { .rewritesTo(expected); } + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testFetchExpressionType() { + sql("select name from dept fetch next (^upper('x')^) rows only") + .fails("FETCH expression must have a numeric type; " + + "actual type is 'CHAR\\(1\\) NOT NULL'"); + sql("select name from dept fetch next (^'x'^) rows only") + .fails("FETCH expression must have a numeric type; " + + "actual type is 'CHAR\\(1\\) NOT NULL'"); + sql("select name from dept fetch next 1.5 rows only").ok(); + sql("select name from dept " + + "fetch next (^row_number() over ()^) rows only") + .fails("Windowed aggregate expression is illegal in FETCH clause"); + } + @Test void testRewriteWithOffsetWithoutOrderBy() { final String sql = "select name from dept offset 2"; final String expected = "SELECT `NAME`\n" diff --git a/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableMergeUnionTest.java b/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableMergeUnionTest.java index 68bb56cf366d..44055f707462 100644 --- a/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableMergeUnionTest.java +++ b/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableMergeUnionTest.java @@ -78,6 +78,36 @@ class EnumerableMergeUnionTest { "empid=45; name=Pascal"); } + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void mergeUnionDoesNotPushNonDeterministicFetch() { + tester(false, + new HrSchemaBig(), + "select * from (select empid, name from emps " + + "union all select empid, name from emps) " + + "order by empid fetch next (rand_integer(10)) rows only") + .explainContains("EnumerableLimitSort(sort0=[$0], dir0=[ASC], " + + "fetch=[RAND_INTEGER(10)])\n" + + " EnumerableMergeUnion(all=[true])\n" + + " EnumerableSort(sort0=[$0], dir0=[ASC])\n"); + } + + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void mergeUnionPushesParameterizedFetchExpression() { + tester(false, + new HrSchemaBig(), + "select * from (select empid, name from emps " + + "union all select empid, name from emps) " + + "order by empid fetch next (? + 1) rows only") + .explainContains("EnumerableLimit(fetch=[+(?0, 1)])\n" + + " EnumerableMergeUnion(all=[true])\n" + + " EnumerableLimitSort(sort0=[$0], dir0=[ASC], " + + "fetch=[+(?0, 1)])\n"); + } + @Test void mergeUnionAllOrderByName() { tester(false, new HrSchemaBig(), diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index 1e37b2699b3a..e2b725a7d66d 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -2950,6 +2950,46 @@ LogicalProject(NAME=[$1]) LogicalFilter(condition=[<=($3, 1)]) LogicalProject(SAL=[$5], EXPR$1=[EXTRACT(FLAG(YEAR), $4)], DEPTNO=[$7], rn=[ROW_NUMBER() OVER (PARTITION BY $7 ORDER BY EXTRACT(FLAG(YEAR), $4) NULLS LAST, $5 DESC NULLS FIRST)]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +]]> + + + + + + + + + + + + + + @@ -3097,9 +3137,9 @@ LogicalProject(NAME=[$1], EXPR$1=[$2]) ($5, 1000))]) LogicalTableScan(table=[[CATALOG, SALES, EMPNULLABLES]]) LogicalFilter(condition=[=($1, $0)]) LogicalAggregate(group=[{0, 1, 2}]) - LogicalProject(SAL=[$5], SAL0=[$8], $f8=[$9]) + LogicalProject(SAL=[$5], SAL0=[$8], $f9=[$9]) LogicalJoin(condition=[OR(=($8, $5), $9)], joinType=[inner]) LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5], COMM=[$6], SLACKER=[$8]) LogicalFilter(condition=[AND(=($7, 20), >($5, 1000))]) LogicalTableScan(table=[[CATALOG, SALES, EMPNULLABLES]]) LogicalAggregate(group=[{0, 1}]) - LogicalProject(SAL=[$5], $f8=[=($5, 4)]) + LogicalProject(SAL=[$5], $f9=[=($5, 4)]) LogicalFilter(condition=[AND(=($7, 20), >($5, 1000))]) LogicalTableScan(table=[[CATALOG, SALES, EMPNULLABLES]]) ]]> @@ -9212,9 +9252,9 @@ LEFT JOIN LATERAL ( + + + + + + + + + + + + + + + + + + + + + + + + + @@ -19903,7 +20032,55 @@ LogicalSort(sort0=[$0], dir0=[ASC], fetch=[0]) ]]> - + + + + + + + + + + + + + + + + + + + + diff --git a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml index aafc9c11efd3..1aa3c3f66134 100644 --- a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml @@ -1770,15 +1770,15 @@ LogicalAggregate(group=[{}], EXPR$0=[COUNT()], EXPR$1=[SUM($0)]) @@ -1796,18 +1796,18 @@ cross join lateral @@ -2601,6 +2601,18 @@ LogicalSort(fetch=[5]) LogicalSort(fetch=[?0]) LogicalProject(EMPNO=[$0]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +]]> + + + + + + + + @@ -5222,9 +5234,9 @@ LogicalProject(C=[$0], D=[$1], C0=[$2]) diff --git a/core/src/test/resources/sql/fetch.iq b/core/src/test/resources/sql/fetch.iq new file mode 100644 index 000000000000..8f4b0dd53d58 --- /dev/null +++ b/core/src/test/resources/sql/fetch.iq @@ -0,0 +1,183 @@ +# fetch.iq +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +!use post +!set outputformat mysql + +# FETCH accepts a parenthesized arithmetic expression. +select * +from (values (1), (2), (3), (4)) as t(x) +fetch next (1 + abs(-2)) rows only; ++---+ +| X | ++---+ +| 1 | +| 2 | +| 3 | ++---+ +(3 rows) + +!ok + +# FETCH accepts a parenthesized scalar expression. +select * +from (values (1), (2), (3), (4)) as t(x) +fetch next (abs(2)) rows only; ++---+ +| X | ++---+ +| 1 | +| 2 | ++---+ +(2 rows) + +!ok + +# FETCH values are not restricted to the BIGINT range. +select * +from (values (1), (2), (3), (4)) as t(x) +fetch next (cast(9223372036854775808 as decimal(20, 0)) + 1) rows only; ++---+ +| X | ++---+ +| 1 | +| 2 | +| 3 | +| 4 | ++---+ +(4 rows) + +!ok + +# FETCH expression cannot be negative. +select * +from (values (1), (2), (3)) as t(x) +fetch next (0 - 1) rows only; +FETCH must not be negative +!error + +# FETCH expression cannot evaluate to NULL. +select * +from (values (1), (2), (3)) as t(x) +fetch next (cast(null as integer)) rows only; +FETCH expression evaluated to NULL +!error + +# FETCH expression may have a fractional numeric type. +select * +from (values (1), (2), (3)) as t(x) +fetch next (1.5) rows only; ++---+ +| X | ++---+ +| 1 | +| 2 | ++---+ +(2 rows) + +!ok + +# FETCH expression cannot reference input columns. +select * +from (values (1), (2), (3)) as t(x) +fetch next (x) rows only; +FETCH expression cannot reference table column 'X' +!error + +# Expressions without parentheses are not allowed in FETCH. +select * +from (values (1), (2), (3)) as t(x) +fetch next 1 + 2 rows only; +Encountered "+" +!error + +# FETCH expression works with a table source. +select deptno, dname +from dept +order by deptno +fetch next (1 + 1) rows only; ++--------+-------------+ +| DEPTNO | DNAME | ++--------+-------------+ +| 10 | Sales | +| 20 | Marketing | ++--------+-------------+ +(2 rows) + +!ok + +# FETCH expression works together with OFFSET on a table source. +select deptno, dname +from dept +order by deptno +offset 1 rows +fetch next (1 + 1) rows only; ++--------+-------------+ +| DEPTNO | DNAME | ++--------+-------------+ +| 20 | Marketing | +| 30 | Engineering | ++--------+-------------+ +(2 rows) + +!ok + +# FETCH expression may contain a scalar function on a table source. +select deptno +from dept +order by deptno +fetch next (abs(-3)) rows only; ++--------+ +| DEPTNO | ++--------+ +| 10 | +| 20 | +| 30 | ++--------+ +(3 rows) + +!ok + +# FETCH expression cannot reference columns of a table source. +select deptno, dname +from dept +order by deptno +fetch next (deptno) rows only; +FETCH expression cannot reference table column 'DEPTNO' +!error + +# FETCH expression cannot reference columns even inside a larger expression. +select deptno, dname +from dept +order by deptno +fetch next (deptno + 1) rows only; +FETCH expression cannot reference table column 'DEPTNO' +!error + +# FETCH expression may be zero on a table source. +select deptno +from dept +order by deptno +fetch next (2 - 2) rows only; ++--------+ +| DEPTNO | ++--------+ ++--------+ +(0 rows) + +!ok diff --git a/core/src/test/resources/sql/lateral.iq b/core/src/test/resources/sql/lateral.iq index 4c4ffbe17072..5c82727b8930 100644 --- a/core/src/test/resources/sql/lateral.iq +++ b/core/src/test/resources/sql/lateral.iq @@ -244,98 +244,4 @@ where job = 'MANAGER'; !ok -# 3 test cases for [CALCITE-7646] CorrelateProjectExtractor -# does not handle nested field accesses cor0.field0.field1. - -# All queries use LATERAL, which converts directly to a Correlate. -# The results were validated on Postgres - -!use scott - -select t.dd, t.x -from dept d, -lateral (select d.deptno as dd, u.x - from unnest(array[d.deptno + 100]) as u(x)) as t -where d.dname = 'SALES'; -+----+-----+ -| DD | X | -+----+-----+ -| 30 | 130 | -+----+-----+ -(1 row) - -!ok - -select t.dd, t.dd1, t.x -from dept d, -lateral (select d.deptno as dd, d.deptno + 1 as dd1, u.x - from unnest(array[1, 2]) as u(x)) as t -where d.dname = 'SALES'; -+----+-----+---+ -| DD | DD1 | X | -+----+-----+---+ -| 30 | 31 | 1 | -| 30 | 31 | 2 | -+----+-----+---+ -(2 rows) - -!ok -!if (use_old_decorr) { -# The correlated computation d.deptno + 1 (DD1) has been extracted into the left -# input of the EnumerableNestedLoopJoin, as $f3. The right input, -# UNNEST(ARRAY[1, 2]), references no correlation variable, so decorrelation -# replaces the Correlate with a join. -EnumerableCalc(expr#0..4=[{inputs}], proj#0..2=[{exprs}]) - EnumerableHashJoin(condition=[AND(=($3, $5), =($4, $6))], joinType=[semi]) - EnumerableCalc(expr#0..2=[{inputs}], proj#0..2=[{exprs}], DEPTNO=[$t0], $f3=[$t1]) - EnumerableNestedLoopJoin(condition=[true], joinType=[inner]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1], expr#4=[+($t0, $t3)], expr#5=['SALES':VARCHAR(14)], expr#6=[=($t1, $t5)], DEPTNO=[$t0], $f3=[$t4], $condition=[$t6]) - EnumerableTableScan(table=[[scott, DEPT]]) - EnumerableUncollect - EnumerableCalc(expr#0=[{inputs}], expr#1=[1], expr#2=[2], expr#3=[ARRAY($t1, $t2)], EXPR$0=[$t3]) - EnumerableValues(tuples=[[{ 0 }]]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1], expr#4=[+($t0, $t3)], expr#5=['SALES':VARCHAR(14)], expr#6=[=($t1, $t5)], DEPTNO=[$t0], $f3=[$t4], $condition=[$t6]) - EnumerableTableScan(table=[[scott, DEPT]]) -!plan -!} - -# COALESCE(d.path, ARRAY[CAST(NULL AS INTEGER)]) converts to -# CASE(IS NOT NULL($cor0.PATH), $cor0.PATH, ARRAY(null:INTEGER)). The constant -# ARRAY(null:INTEGER) operand must not prevent extracting the CASE to the -# left input of the Correlate operator -select d.deptno, t.x -from (select deptno, - case when deptno = 10 then array[deptno, deptno + 1] end as path - from dept) as d, -lateral (select * from unnest(coalesce(d.path, array[cast(null as integer)])) as u(x)) as t -order by d.deptno, t.x; -+--------+----+ -| DEPTNO | X | -+--------+----+ -| 10 | 10 | -| 10 | 11 | -| 20 | | -| 30 | | -| 40 | | -+--------+----+ -(5 rows) - -!ok -!if (use_old_decorr) { -# The entire CASE produced by COALESCE has been extracted into the left input of the -# EnumerableCorrelate, as $f2. The right input reads the array through $cor0.$f2. -# The query cannot be decorrelated because of the remaining Uncollect. -EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) - EnumerableCalc(expr#0..2=[{inputs}], DEPTNO=[$t0], X=[$t2]) - EnumerableCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{1}]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[CAST($t0):INTEGER NOT NULL], expr#4=[10], expr#5=[=($t3, $t4)], expr#6=[1], expr#7=[+($t0, $t6)], expr#8=[ARRAY($t3, $t7)], expr#9=[null:INTEGER NOT NULL ARRAY], expr#10=[CASE($t5, $t8, $t9)], expr#11=[IS NOT NULL($t10)], expr#12=[CAST($t10):INTEGER NOT NULL ARRAY NOT NULL], expr#13=[CAST($t12):INTEGER ARRAY NOT NULL], expr#14=[null:INTEGER], expr#15=[ARRAY($t14)], expr#16=[CASE($t11, $t13, $t15)], DEPTNO=[$t0], $f2=[$t16]) - EnumerableTableScan(table=[[scott, DEPT]]) - EnumerableUncollect - EnumerableCalc(expr#0=[{inputs}], expr#1=[$cor0], expr#2=[$t1.$f2], EXPR$0=[$t2]) - EnumerableValues(tuples=[[{ 0 }]]) -!plan -!} - -!set planner-rules original - # End lateral.iq diff --git a/server/src/test/java/org/apache/calcite/test/ServerTest.java b/server/src/test/java/org/apache/calcite/test/ServerTest.java index 355d39de7d63..39d434f23ae9 100644 --- a/server/src/test/java/org/apache/calcite/test/ServerTest.java +++ b/server/src/test/java/org/apache/calcite/test/ServerTest.java @@ -43,6 +43,7 @@ import java.math.BigDecimal; import java.sql.Connection; import java.sql.DriverManager; +import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; @@ -452,6 +453,42 @@ static Connection connect() throws SQLException { } } + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testFetchExpressionCannotReferenceInputColumn() throws Exception { + try (Connection c = connect(); + Statement s = c.createStatement()) { + s.execute("create table person (id int not null, name varchar(20))"); + try (PreparedStatement p = + c.prepareStatement("insert into person (id, name) values (?, ?)")) { + p.setInt(1, 1); + p.setString(2, "foo"); + assertThat(p.executeUpdate(), is(1)); + } + + SQLException e = + assertThrows( + SQLException.class, () -> s.executeQuery("select * from person " + + "fetch next id rows only")); + assertThat(e.getMessage(), containsString("Encountered \"id\"")); + + e = + assertThrows( + SQLException.class, () -> s.executeQuery("select * from person " + + "fetch next (id) rows only")); + assertThat(e.getMessage(), + containsString("FETCH expression cannot reference table column 'ID'")); + + e = + assertThrows( + SQLException.class, () -> s.executeQuery("select * from person " + + "fetch next (1 + id) rows only")); + assertThat(e.getMessage(), + containsString("FETCH expression cannot reference table column 'ID'")); + } + } + /** Test case for * [CALCITE-6022] * Support "CREATE TABLE ... LIKE" DDL in server module. */ diff --git a/site/_docs/reference.md b/site/_docs/reference.md index 8ff9857cb8c0..b1f67eeef1ee 100644 --- a/site/_docs/reference.md +++ b/site/_docs/reference.md @@ -427,8 +427,13 @@ in the order that they appear in the list; for example: "SELECT x, y FROM t ORDER BY x, y" An optional trailing ASC / DESC and NULLS FIRST / NULLS LAST applies to all keys. -In *query*, *count* and *start* may each be either an unsigned numeric literal -or a dynamic parameter whose value is numeric. +In *query*, *start* may be either an unsigned numeric literal or a dynamic +parameter whose value is numeric. The *count* in a LIMIT clause may be either +an unsigned numeric literal or a dynamic parameter whose value is numeric. The +*count* in a FETCH clause may be an unsigned numeric literal, a dynamic +parameter whose value is numeric, or a scalar expression enclosed in +parentheses. A FETCH *count* expression cannot reference columns from the query +input, and cannot contain aggregate functions, window functions, or sub-queries. Support for decimal or non-integer values is adapter-dependent. An aggregate query is a query that contains a GROUP BY or a HAVING diff --git a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java index e61cc66d5662..372cd1189d28 100644 --- a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java +++ b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java @@ -4075,12 +4075,31 @@ void checkPeriodPredicate(Checker checker) { + "FROM `FOO`\n" + "OFFSET ? ROWS\n" + "FETCH NEXT ? ROWS ONLY"); + // CALCITE-7592: Arithmetic and scalar expressions are allowed within parentheses. + sql("select a from foo fetch next (1 + abs(-2)) rows only") + .ok("SELECT `A`\n" + + "FROM `FOO`\n" + + "FETCH NEXT (1 + ABS(-2)) ROWS ONLY"); + // Expressions without parentheses are not allowed. + sql("select a from foo fetch next 1 ^+^ 2 rows only") + .fails("(?s).*Encountered \"\\+\" at .*"); + sql("select a from foo fetch next ? ^+^ abs(2) rows only") + .fails("(?s).*Encountered \"\\+\" at .*"); // missing ROWS after FETCH sql("select a from foo offset 1 fetch next 3 ^only^") .fails("(?s).*Encountered \"only\" at .*"); // FETCH before OFFSET is illegal sql("select a from foo fetch next 3 rows only ^offset^ 1") .fails("(?s).*Encountered \"offset\" at .*"); + // Subqueries are not allowed in FETCH + sql("select a from foo fetch next ^select^ 2 rows only") + .fails("(?s).*Encountered \"select\" at .*"); + sql("select a from foo fetch next (^select^ 2) rows only") + .fails("(?s).*Encountered \"select\" at .*"); + sql("select a from foo fetch next (^select^ ?) rows only") + .fails("(?s).*Encountered \"select\" at .*"); + sql("select a from foo fetch next (^select^ max(a) from foo) rows only") + .fails("(?s).*Encountered \"select\" at .*"); } /** diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java index f82222312e3a..d710b516aaa7 100644 --- a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java +++ b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java @@ -1843,39 +1843,6 @@ void testCastToBoolean(CastType castType, SqlOperatorFixture f) { f.checkNull("cast(null as row(f0 varchar, f1 varchar))"); } - /** Test case for - * - * [CALCITE-7658] Type checker rejects - * CAST(ARRAY() AS ROW(x INT) ARRAY). - * - *

      The Spark {@code ARRAY()} function creates an empty array whose - * element type is UNKNOWN; such an array can be cast to any array type. */ - @Test void testCastEmptyArray() { - final SqlOperatorFixture f = fixture().withLibrary(SqlLibrary.SPARK); - f.checkScalar("cast(array() as integer array)", "[]", - "INTEGER NOT NULL ARRAY NOT NULL"); - f.checkScalar("cast(array() as row(x int) array)", "[]", - "RecordType(INTEGER NOT NULL X) NOT NULL ARRAY NOT NULL"); - f.checkScalar("cast(array() as integer array array)", "[]", - "INTEGER ARRAY NOT NULL ARRAY NOT NULL"); - f.checkScalar("cast(array() as map array)", "[]", - "(VARCHAR NOT NULL, INTEGER) MAP NOT NULL ARRAY NOT NULL"); - // A non-empty array with UNKNOWN or NULL element type contains only nulls - f.checkScalar("cast(array_append(array(), null) as row(x int) array)", - "[null]", - "RecordType(INTEGER NOT NULL X) ARRAY NOT NULL"); - f.checkScalar("cast(array(null) as row(x int) array)", - "[null]", - "RecordType(INTEGER NOT NULL X) ARRAY NOT NULL"); - // The empty MAP() has UNKNOWN key and value types - f.checkScalar("cast(map() as map)", "{}", - "(VARCHAR NOT NULL, INTEGER NOT NULL) MAP NOT NULL"); - f.checkScalar("cast(map() as map)", "{}", - "(VARCHAR NOT NULL, RecordType(INTEGER X) NOT NULL) MAP NOT NULL"); - f.checkScalar("cast(map() as map)", "{}", - "(VARCHAR NOT NULL, INTEGER ARRAY NOT NULL) MAP NOT NULL"); - } - /** Test cases for * * [CALCITE-4918] Add a VARIANT data type. */ @@ -8262,7 +8229,7 @@ void checkRegexpExtract(SqlOperatorFixture f0, FunctionAlias functionAlias) { f.checkScalar("array_append(array(null), null)", "[null, null]", "NULL ARRAY NOT NULL"); f.checkScalar("array_append(array(), null)", "[null]", - "NULL ARRAY NOT NULL"); + "UNKNOWN ARRAY NOT NULL"); f.checkScalar("array_append(array(), 1)", "[1]", "INTEGER NOT NULL ARRAY NOT NULL"); f.checkScalar("array_append(array[array[1, 2]], array[3, 4])", "[[1, 2], [3, 4]]", @@ -8601,7 +8568,7 @@ void checkRegexpExtract(SqlOperatorFixture f0, FunctionAlias functionAlias) { f.checkScalar("array_prepend(array(null), null)", "[null, null]", "NULL ARRAY NOT NULL"); f.checkScalar("array_prepend(array(), null)", "[null]", - "NULL ARRAY NOT NULL"); + "UNKNOWN ARRAY NOT NULL"); f.checkScalar("array_prepend(array(), 1)", "[1]", "INTEGER NOT NULL ARRAY NOT NULL"); f.checkScalar("array_prepend(array[array[1, 2]], array[3, 4])", "[[3, 4], [1, 2]]", @@ -13639,22 +13606,6 @@ private static void checkArrayConcatAggFuncFails(SqlOperatorFixture t) { "RecordType(INTEGER EXPR$0, INTEGER EXPR$1) NOT NULL ARRAY NOT NULL"); f2.checkScalar("array(row(1, 2), row(3, 4))", "[{1, 2}, {3, 4}]", "RecordType(INTEGER NOT NULL EXPR$0, INTEGER NOT NULL EXPR$1) NOT NULL ARRAY NOT NULL"); - // Tests for unification of UNKNOWN with other types; array() has a type - // of UNKNOWN ARRAY, yet the type of ARRAY() is inferred from other operands. - f2.checkScalar("array(array(1), array())", "[[1], []]", - "INTEGER NOT NULL ARRAY NOT NULL ARRAY NOT NULL"); - f2.checkScalar("array(array(), array(1))", "[[], [1]]", - "INTEGER NOT NULL ARRAY NOT NULL ARRAY NOT NULL"); - f2.checkScalar("array(array(row(1, 2)), array())", "[[{1, 2}], []]", - "RecordType(INTEGER NOT NULL EXPR$0, INTEGER NOT NULL EXPR$1) NOT NULL " - + "ARRAY NOT NULL ARRAY NOT NULL"); - f2.checkScalar("array(array(), array(row(1, 2)))", "[[], [{1, 2}]]", - "RecordType(INTEGER NOT NULL EXPR$0, INTEGER NOT NULL EXPR$1) NOT NULL " - + "ARRAY NOT NULL ARRAY NOT NULL"); - f2.checkScalar("array(array(array(1)), array())", "[[[1]], []]", - "INTEGER NOT NULL ARRAY NOT NULL ARRAY NOT NULL ARRAY NOT NULL"); - f2.checkScalar("array(array(), array(array(1)))", "[[], [[1]]]", - "INTEGER NOT NULL ARRAY NOT NULL ARRAY NOT NULL ARRAY NOT NULL"); // checkFails f2.checkFails("^array(row(1), row(2, 3))^", "Parameters must be of the same type", false); @@ -13673,32 +13624,6 @@ private static void checkArrayConcatAggFuncFails(SqlOperatorFixture t) { f.forEachLibrary(libraries, consumer); } - /** Tests that empty collections created by the Spark - * {@code ARRAY()} and {@code MAP()} functions, whose element - * types are UNKNOWN, unify with collections with known types. */ - @Test void testEmptyCollections() { - final SqlOperatorFixture f = fixture().withLibrary(SqlLibrary.SPARK); - f.checkScalar("array(map(1, 2), map())", "[{1=2}, {}]", - "(INTEGER NOT NULL, INTEGER NOT NULL) MAP NOT NULL ARRAY NOT NULL"); - f.checkScalar("array(map(), map(1, 2))", "[{}, {1=2}]", - "(INTEGER NOT NULL, INTEGER NOT NULL) MAP NOT NULL ARRAY NOT NULL"); - // Nested: empty collections inside a ROW unify field by field - f.checkScalar("array(row(array(), map()))", "[{[], {}}]", - "RecordType(UNKNOWN NOT NULL ARRAY NOT NULL EXPR$0, " - + "(UNKNOWN NOT NULL, UNKNOWN NOT NULL) MAP NOT NULL EXPR$1) " - + "NOT NULL ARRAY NOT NULL"); - f.checkScalar("array(row(array(1), map(1, 2)), row(array(), map()))", - "[{[1], {1=2}}, {[], {}}]", - "RecordType(INTEGER NOT NULL ARRAY NOT NULL EXPR$0, " - + "(INTEGER NOT NULL, INTEGER NOT NULL) MAP NOT NULL EXPR$1) " - + "NOT NULL ARRAY NOT NULL"); - f.checkScalar("array(row(array(), map()), row(array(1), map(1, 2)))", - "[{[], {}}, {[1], {1=2}}]", - "RecordType(INTEGER NOT NULL ARRAY NOT NULL EXPR$0, " - + "(INTEGER NOT NULL, INTEGER NOT NULL) MAP NOT NULL EXPR$1) " - + "NOT NULL ARRAY NOT NULL"); - } - @Test void testArrayQueryConstructor() { final SqlOperatorFixture f = fixture(); f.setFor(SqlStdOperatorTable.ARRAY_QUERY, SqlOperatorFixture.VmName.EXPAND); @@ -14005,22 +13930,6 @@ private static void checkArrayConcatAggFuncFails(SqlOperatorFixture t) { f1.checkScalar("map('k1', 1, 'k2', 2.0)", "{k1=1.0, k2=2.0}", "(CHAR(2) NOT NULL, DECIMAL(11, 1) NOT NULL) MAP NOT NULL"); - f1.checkScalar("map('a', array(1), 'b', array())", "{a=[1], b=[]}", - "(CHAR(1) NOT NULL, INTEGER NOT NULL ARRAY NOT NULL) MAP NOT NULL"); - f1.checkScalar("map('a', array(), 'b', array(1))", "{a=[], b=[1]}", - "(CHAR(1) NOT NULL, INTEGER NOT NULL ARRAY NOT NULL) MAP NOT NULL"); - f1.checkScalar("map('a', map(1, 2), 'b', map())", "{a={1=2}, b={}}", - "(CHAR(1) NOT NULL, (INTEGER NOT NULL, INTEGER NOT NULL) MAP NOT NULL) MAP NOT NULL"); - f1.checkScalar("map('a', map(), 'b', map(1, 2))", "{a={}, b={1=2}}", - "(CHAR(1) NOT NULL, (INTEGER NOT NULL, INTEGER NOT NULL) MAP NOT NULL) MAP NOT NULL"); - // Avatica's conversion of MAP to STRING is broken, so we only check - // the type for the following 2 tests - f1.checkType("map('a', array(row(1, 2)), 'b', array())", - "(CHAR(1) NOT NULL, RecordType(INTEGER NOT NULL EXPR$0, INTEGER NOT NULL EXPR$1) " - + "NOT NULL ARRAY NOT NULL) MAP NOT NULL"); - f1.checkType("map('a', array(), 'b', array(row(1, 2)))", - "(CHAR(1) NOT NULL, RecordType(INTEGER NOT NULL EXPR$0, INTEGER NOT NULL EXPR$1) " - + "NOT NULL ARRAY NOT NULL) MAP NOT NULL"); } @Test void testMapQueryConstructor() { From 9816f59e17331dd9b4d05c7f69bbf85a00dcf0fe Mon Sep 17 00:00:00 2001 From: zzwqqq Date: Fri, 17 Jul 2026 10:47:41 +0800 Subject: [PATCH 398/562] [CALCITE-7655] RelToSqlConverter incorrectly removes a subquery when grouping by a window function result --- .../calcite/rel/rel2sql/SqlImplementor.java | 19 +++++++++++ .../rel/rel2sql/RelToSqlConverterTest.java | 34 +++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java b/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java index d8c02a56f3f4..c74da741d7c3 100644 --- a/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java +++ b/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java @@ -626,6 +626,21 @@ private static boolean isWindowedAggregate(SqlNode node) { && ((SqlCall) node).getOperator() instanceof SqlOverOperator; } + /** Returns whether one of an aggregate's group keys contains an OVER expression. */ + private static boolean groupKeysContainOver(Aggregate aggregate) { + final RelNode input = aggregate.getInput(); + if (!(input instanceof Project)) { + return false; + } + final Project project = (Project) input; + for (int group : aggregate.getGroupSet()) { + if (RexOver.containsOver(project.getProjects().get(group))) { + return true; + } + } + return false; + } + /** Context for translating a {@link RexNode} expression (within a * {@link RelNode}) into a {@link SqlNode} expression (within a SQL parse * tree). */ @@ -2172,6 +2187,10 @@ && hasSortByOrdinal(node)) { // Avoid losing the distinct attribute of inner aggregate. return !hasNestedAgg || Aggregate.isNotGrandTotal(agg); } + + if (groupKeysContainOver(agg)) { + return true; + } } return false; diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java index 44dc09078d02..801612976b00 100644 --- a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java +++ b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java @@ -2016,6 +2016,40 @@ private static String toSql(RelNode root, SqlDialect dialect, relFn(relFn).withOracle().ok(expectedOracle); } + /** Test case for + * [CALCITE-7655] + * RelToSqlConverter incorrectly removes a subquery when grouping by a window function + * result. */ + @Test void testGroupByWindowFunction() { + final String query = "SELECT \"EMPNO\", \"row_number\", COUNT(*) AS \"c\"\n" + + "FROM (\n" + + " SELECT \"EMPNO\",\n" + + " ROW_NUMBER() OVER (ORDER BY \"EMPNO\" NULLS FIRST) AS \"row_number\"\n" + + " FROM \"EMP\") AS \"t\"\n" + + "GROUP BY \"EMPNO\", \"row_number\""; + + final String expectedMysql = "SELECT `EMPNO`, `row_number`, COUNT(*) AS `c`\n" + + "FROM (SELECT `EMPNO`, ROW_NUMBER() OVER (ORDER BY `EMPNO`) AS `row_number`\n" + + "FROM `SCOTT`.`EMP`) AS `t`\n" + + "GROUP BY `EMPNO`, `row_number`"; + final String expectedOracle = "SELECT \"EMPNO\", \"row_number\", COUNT(*) \"c\"\n" + + "FROM (SELECT \"EMPNO\", ROW_NUMBER() OVER (ORDER BY \"EMPNO\" NULLS FIRST)" + + " \"row_number\"\n" + + "FROM \"SCOTT\".\"EMP\") \"t\"\n" + + "GROUP BY \"EMPNO\", \"row_number\""; + final String expectedPostgresql = + "SELECT \"EMPNO\", \"row_number\", COUNT(*) AS \"c\"\n" + + "FROM (SELECT \"EMPNO\", ROW_NUMBER() OVER (ORDER BY \"EMPNO\" NULLS FIRST)" + + " AS \"row_number\"\n" + + "FROM \"SCOTT\".\"EMP\") AS \"t\"\n" + + "GROUP BY \"EMPNO\", \"row_number\""; + sql(query) + .schema(CalciteAssert.SchemaSpec.JDBC_SCOTT) + .withMysql().ok(expectedMysql) + .withOracle().ok(expectedOracle) + .withPostgresql().ok(expectedPostgresql); + } + @Test void testSemiJoin() { final RelBuilder builder = relBuilder(); final RelNode root = builder From b2a335421dda61a0b53c8eec492f0ebb65a1be70 Mon Sep 17 00:00:00 2001 From: xuzifu666 <1206332514@qq.com> Date: Thu, 16 Jul 2026 11:24:05 +0800 Subject: [PATCH 399/562] Test case for [CALCITE-5216] Cannot parse parenthesized nested WITH clause --- core/src/test/resources/sql/sub-query.iq | 12 ++++++++++++ .../org/apache/calcite/sql/parser/SqlParserTest.java | 10 ++++++++++ 2 files changed, 22 insertions(+) diff --git a/core/src/test/resources/sql/sub-query.iq b/core/src/test/resources/sql/sub-query.iq index 84f9aa5183f5..daa85b96538b 100644 --- a/core/src/test/resources/sql/sub-query.iq +++ b/core/src/test/resources/sql/sub-query.iq @@ -937,6 +937,18 @@ where sal + 100 not in ( !ok !} +# [CALCITE-5216] Cannot parse parenthesized nested WITH clause +# This sql program was validated in PostgreSQL +with a as (with b as (select 1)(select 1)) select * from a; ++--------+ +| EXPR$0 | ++--------+ +| 1 | ++--------+ +(1 row) + +!ok + # [CALCITE-356] AssertionError while translating query with WITH and correlated sub-query !if (use_old_decorr) { with t (a, b) as (select * from (values (1, 2))) diff --git a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java index 372cd1189d28..59bd1a5d2030 100644 --- a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java +++ b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java @@ -2858,6 +2858,16 @@ void checkPeriodPredicate(Checker checker) { sql(sql).ok(expected); } + /** Test case for + * [CALCITE-5216] + * Cannot parse parenthesized nested WITH clause. */ + @Test void testNestedWithParenthesized() { + final String sql = "with a as (with b as (select 1)(select 1)) select * from a"; + final String expected = "WITH `A` AS (WITH `B` AS (SELECT 1) SELECT 1) SELECT *\n" + + "FROM `A`"; + sql(sql).ok(expected); + } + /** Test case for * [CALCITE-5252] * JDBC adapter sometimes miss parentheses around SELECT in WITH_ITEM body. */ From 692907b70bca3139a5a33bf4693e457fa2415dbb Mon Sep 17 00:00:00 2001 From: Sergey Nuyanzin Date: Fri, 17 Jul 2026 00:41:37 +0200 Subject: [PATCH 400/562] [CALCITE-7660] `SqlSetSemanticsTableOperator` might produce invalid SQL while unparse --- core/src/main/codegen/templates/Parser.jj | 3 ++- .../sql/SqlSetSemanticsTableOperator.java | 13 ++++++++----- .../apache/calcite/test/SqlValidatorTest.java | 16 ++++++++++++++++ .../apache/calcite/sql/parser/SqlParserTest.java | 9 ++++++--- 4 files changed, 32 insertions(+), 9 deletions(-) diff --git a/core/src/main/codegen/templates/Parser.jj b/core/src/main/codegen/templates/Parser.jj index 5c403e1cedd8..d3c115974180 100644 --- a/core/src/main/codegen/templates/Parser.jj +++ b/core/src/main/codegen/templates/Parser.jj @@ -1744,7 +1744,8 @@ SqlNode PartitionedQueryOrQueryOrExpr(ExprContext exprContext) : SqlNode e; } { - e = OrderedQueryOrExpr(exprContext) + // QueryOrExpr, not OrderedQueryOrExpr: ORDER BY is handled by PartitionedByAndOrderBy below. + e = QueryOrExpr(exprContext) e = PartitionedByAndOrderBy(e) { return e; } diff --git a/core/src/main/java/org/apache/calcite/sql/SqlSetSemanticsTableOperator.java b/core/src/main/java/org/apache/calcite/sql/SqlSetSemanticsTableOperator.java index 84e35075f110..f33003c74318 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlSetSemanticsTableOperator.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlSetSemanticsTableOperator.java @@ -67,17 +67,20 @@ public SqlSetSemanticsTableOperator() { SqlNodeList partitionList = call.operand(1); if (!partitionList.isEmpty()) { writer.sep("PARTITION BY"); - final SqlWriter.Frame partitionFrame = writer.startList("", ""); + final SqlWriter.Frame partitionFrame = partitionList.size() == 1 + ? writer.startList("", "") + : writer.startList("(", ")"); partitionList.unparse(writer, 0, 0); writer.endList(partitionFrame); } SqlNodeList orderList = call.operand(2); if (!orderList.isEmpty()) { writer.sep("ORDER BY"); - writer.list( - SqlWriter.FrameTypeEnum.ORDER_BY_LIST, - SqlWriter.COMMA, - orderList); + final SqlWriter.Frame orderFrame = orderList.size() == 1 + ? writer.startList("", "") + : writer.startList("(", ")"); + orderList.unparse(writer, 0, 0); + writer.endList(orderFrame); } } diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index 9b0d0de70221..08667c036b98 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -1979,12 +1979,28 @@ void testLikeAndSimilarFails() { .ok(); // test multiple partition keys for input table with set semantic sql("select * from table(topn(table orders partition by (orderId, productid), 3))") + .rewritesTo("SELECT *\n" + + "FROM TABLE(TOPN((SELECT *\n" + + "FROM `ORDERS`) PARTITION BY (`ORDERID`, `PRODUCTID`), 3))") + .ok(); + sql("select * from table(topn(table orders partition by (orderId), 3))") + .rewritesTo("SELECT *\n" + + "FROM TABLE(TOPN((SELECT *\n" + + "FROM `ORDERS`) PARTITION BY `ORDERID`, 3))") .ok(); // test one order key for input table with set semantic sql("select * from table(topn(table orders order by orderId, 3))") .ok(); + sql("select * from table(topn(table orders order by (orderId), 3))") + .rewritesTo("SELECT *\n" + + "FROM TABLE(TOPN((SELECT *\n" + + "FROM `ORDERS`) ORDER BY `ORDERID`, 3))") + .ok(); // test multiple order keys for input table with set semantic sql("select * from table(topn(table orders order by (orderId, productid), 3))") + .rewritesTo("SELECT *\n" + + "FROM TABLE(TOPN((SELECT *\n" + + "FROM `ORDERS`) ORDER BY (`ORDERID`, `PRODUCTID`), 3))") .ok(); // test complex order-by clause for input table with set semantic sql("select * from table(topn(table orders order by (orderId desc, productid asc), 3))") diff --git a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java index 59bd1a5d2030..9916e0f25e7c 100644 --- a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java +++ b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java @@ -5007,8 +5007,9 @@ void checkPeriodPredicate(Checker checker) { final String sql = "select * from table(topn(table orders partition by (orderId, productid), 3))"; final String expected = "SELECT *\n" - + "FROM TABLE(`TOPN`((TABLE `ORDERS`) PARTITION BY `ORDERID`, `PRODUCTID`, 3))"; + + "FROM TABLE(`TOPN`((TABLE `ORDERS`) PARTITION BY (`ORDERID`, `PRODUCTID`), 3))"; sql(sql).ok(expected); + sql(expected).withConfig(c -> c.withQuoting(Quoting.BACK_TICK)).same(); } @Test void testTableFunctionWithOrderKey() { @@ -5025,8 +5026,9 @@ void checkPeriodPredicate(Checker checker) { final String sql = "select * from table(topn(table orders order by (orderId, productid), 3))"; final String expected = "SELECT *\n" - + "FROM TABLE(`TOPN`((TABLE `ORDERS`) ORDER BY `ORDERID`, `PRODUCTID`, 3))"; + + "FROM TABLE(`TOPN`((TABLE `ORDERS`) ORDER BY (`ORDERID`, `PRODUCTID`), 3))"; sql(sql).ok(expected); + sql(expected).withConfig(c -> c.withQuoting(Quoting.BACK_TICK)).same(); } @Test void testTableFunctionWithComplexOrderBy() { @@ -5034,8 +5036,9 @@ void checkPeriodPredicate(Checker checker) { final String sql = "select * from table(topn(table orders order by (orderId desc, productid asc), 3))"; final String expected = "SELECT *\n" - + "FROM TABLE(`TOPN`((TABLE `ORDERS`) ORDER BY `ORDERID` DESC, `PRODUCTID`, 3))"; + + "FROM TABLE(`TOPN`((TABLE `ORDERS`) ORDER BY (`ORDERID` DESC, `PRODUCTID`), 3))"; sql(sql).ok(expected); + sql(expected).withConfig(c -> c.withQuoting(Quoting.BACK_TICK)).same(); } @Test void testTableFunctionWithPartitionKeyAndOrderKey() { From 178a59ad473223bc30eeaf4e3e2055495fc0af51 Mon Sep 17 00:00:00 2001 From: Ruben Quesada Lopez Date: Mon, 20 Jul 2026 16:31:34 +0100 Subject: [PATCH 401/562] [CALCITE-7659] Add AggregateCall#withFunction API --- .../java/org/apache/calcite/rel/core/AggregateCall.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/core/src/main/java/org/apache/calcite/rel/core/AggregateCall.java b/core/src/main/java/org/apache/calcite/rel/core/AggregateCall.java index 95ca99e13a6b..47164f94def9 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/AggregateCall.java +++ b/core/src/main/java/org/apache/calcite/rel/core/AggregateCall.java @@ -388,6 +388,13 @@ public final SqlAggFunction getAggregation() { return aggFunction; } + /** Withs {@link #aggFunction}. */ + public AggregateCall withFunction(SqlAggFunction aggFunction) { + return aggFunction.equals(this.aggFunction) ? this + : new AggregateCall(pos, aggFunction, distinct, approximate, ignoreNulls, + rexList, argList, filterArg, distinctKeys, collation, type, name); + } + /** * Returns the aggregate ordering definition (the {@code WITHIN GROUP} clause * in SQL), or the empty list if not specified. From 2df91e99205cd62e5dc075e982c71b43e820f4f9 Mon Sep 17 00:00:00 2001 From: Diveyam Mishra Date: Tue, 23 Jun 2026 00:33:23 +0530 Subject: [PATCH 402/562] [CALCITE-7618] Add filter pushdown support to the file adapter's CSV table implementation Implement filter pushdown rules for the file adapter's CSV table, add support for arbitrary filter predicates, and refactor sameValue with improved test coverage. --- .../java/org/apache/calcite/test/CsvTest.java | 16 + .../calcite/adapter/file/CsvEnumerator.java | 73 +++- .../adapter/file/CsvFilterTableScanRule.java | 85 ++++ .../file/CsvProjectFilterTableScanRule.java | 141 +++++++ .../adapter/file/CsvProjectTableScanRule.java | 19 +- .../calcite/adapter/file/CsvTableScan.java | 68 ++- .../calcite/adapter/file/FileRules.java | 15 + .../adapter/file/CsvEnumeratorTest.java | 14 + .../calcite/adapter/file/FileAdapterTest.java | 389 +++++++++++++++++- 9 files changed, 785 insertions(+), 35 deletions(-) create mode 100644 file/src/main/java/org/apache/calcite/adapter/file/CsvFilterTableScanRule.java create mode 100644 file/src/main/java/org/apache/calcite/adapter/file/CsvProjectFilterTableScanRule.java diff --git a/example/csv/src/test/java/org/apache/calcite/test/CsvTest.java b/example/csv/src/test/java/org/apache/calcite/test/CsvTest.java index 7616fcef26a6..2236345f06ab 100644 --- a/example/csv/src/test/java/org/apache/calcite/test/CsvTest.java +++ b/example/csv/src/test/java/org/apache/calcite/test/CsvTest.java @@ -383,6 +383,22 @@ void testPushDownProjectAggregateNested(String format) { .ok(); } + @Test void testFilterableWhereAge() { + // age column has nulls in the data — make sure they're excluded under objectsEqual + final String sql = "select name from EMPS where age = 25"; + sql("filterable-model", sql) + .returns("NAME=Fred") + .ok(); + } + + @Test void testFilterableWhereSlacker() { + // slacker column has nulls in the data — make sure they're excluded under objectsEqual + final String sql = "select name from EMPS where slacker = false"; + sql("filterable-model", sql) + .returns("NAME=John", "NAME=Alice") + .ok(); + } + /** Test case for * [CALCITE-2272] * Incorrect result for {@code name like '%E%' and city not like '%W%'}. diff --git a/file/src/main/java/org/apache/calcite/adapter/file/CsvEnumerator.java b/file/src/main/java/org/apache/calcite/adapter/file/CsvEnumerator.java index f62433beab47..99951e85cdbe 100644 --- a/file/src/main/java/org/apache/calcite/adapter/file/CsvEnumerator.java +++ b/file/src/main/java/org/apache/calcite/adapter/file/CsvEnumerator.java @@ -139,7 +139,7 @@ public CsvEnumerator(Source source, AtomicBoolean cancelFlag, boolean stream, } } - private static RowConverter converter(List fieldTypes, + static RowConverter converter(List fieldTypes, List fields) { if (fields.size() == 1) { final int field = fields.get(0); @@ -254,6 +254,31 @@ static CSVReader openCsv(Source source, char separator) throws IOException { return new CSVReader(source.reader(), separator); } + /** + * Evaluates equality between 2 Comparable objects, conforming to SQL WHERE filter '=' semantics. + * + *

      Returns {@code false} if either operand is null. Because of this, it cannot be + * directly used for {@code IS NOT DISTINCT FROM} comparisons without additional null handling. + * + *

      When both operands are of the same class (like BigDecimal), it utilizes + * {@code compareTo()} to ignore differences in representation (e.g. scale) + * that would cause standard {@code equals()} to fail. Otherwise, falls back to + * {@code equals()}. + */ + @SuppressWarnings("unchecked") + static boolean sameValue(@Nullable Comparable o1, @Nullable Comparable o2) { + if (o1 == null || o2 == null) { + return false; + } + if (o1 == o2) { + return true; + } + if (o1.getClass().isInstance(o2)) { + return o1.compareTo(o2) == 0; + } + return o1.equals(o2); + } + @Override public E current() { return castNonNull(current); } @@ -284,11 +309,26 @@ static CSVReader openCsv(Source source, char separator) throws IOException { return false; } if (filterValues != null) { - for (int i = 0; i < strings.length; i++) { + for (int i = 0; i < filterValues.size(); i++) { String filterValue = filterValues.get(i); if (filterValue != null) { - if (!filterValue.equals(strings[i])) { - continue outer; + final String rowValueStr = field(strings, i); + final RelDataType fieldType = rowConverter.getFieldType(i); + if (fieldType != null && fieldType.getSqlTypeName() != SqlTypeName.VARCHAR + && fieldType.getSqlTypeName() != SqlTypeName.CHAR) { + final Object filterValObj = RowConverter.convert(fieldType, filterValue); + final Object rowValObj = RowConverter.convert(fieldType, rowValueStr); + if (filterValObj instanceof Comparable && rowValObj instanceof Comparable) { + if (!sameValue((Comparable) filterValObj, (Comparable) rowValObj)) { + continue outer; + } + } else if (!java.util.Objects.equals(filterValObj, rowValObj)) { + continue outer; + } + } else { + if (!filterValue.equals(rowValueStr)) { + continue outer; + } } } } @@ -329,14 +369,23 @@ private static RelDataType toNullableRelDataType(JavaTypeFactory typeFactory, return typeFactory.createTypeWithNullability(typeFactory.createSqlType(sqlTypeName), true); } + /** Returns a field from a CSV row, or null if the row is too short. */ + private static @Nullable String field(String[] strings, int index) { + return index < strings.length ? strings[index] : null; + } + /** Row converter. * * @param element type */ abstract static class RowConverter { abstract E convertRow(@Nullable String[] rows); + @Nullable RelDataType getFieldType(int index) { + return null; + } + @SuppressWarnings("JavaUtilDate") - protected @Nullable Object convert(@Nullable RelDataType fieldType, @Nullable String string) { + static @Nullable Object convert(@Nullable RelDataType fieldType, @Nullable String string) { if (fieldType == null || string == null) { return string; } @@ -468,6 +517,10 @@ static class ArrayRowConverter extends RowConverter<@Nullable Object[]> { this.stream = stream; } + @Override @Nullable RelDataType getFieldType(int index) { + return index < fieldTypes.size() ? fieldTypes.get(index) : null; + } + @Override public @Nullable Object[] convertRow(@Nullable String[] strings) { if (stream) { return convertStreamRow(strings); @@ -480,7 +533,7 @@ static class ArrayRowConverter extends RowConverter<@Nullable Object[]> { final @Nullable Object[] objects = new Object[fields.size()]; for (int i = 0; i < fields.size(); i++) { int field = fields.get(i); - objects[i] = convert(fieldTypes.get(field), strings[field]); + objects[i] = convert(fieldTypes.get(field), field(strings, field)); } return objects; } @@ -490,7 +543,7 @@ static class ArrayRowConverter extends RowConverter<@Nullable Object[]> { objects[0] = System.currentTimeMillis(); for (int i = 0; i < fields.size(); i++) { int field = fields.get(i); - objects[i + 1] = convert(fieldTypes.get(field), strings[field]); + objects[i + 1] = convert(fieldTypes.get(field), field(strings, field)); } return objects; } @@ -506,8 +559,12 @@ private SingleColumnRowConverter(RelDataType fieldType, int fieldIndex) { this.fieldIndex = fieldIndex; } + @Override @Nullable RelDataType getFieldType(int index) { + return index == fieldIndex ? fieldType : null; + } + @Override public @Nullable Object convertRow(@Nullable String[] strings) { - return convert(fieldType, strings[fieldIndex]); + return convert(fieldType, field(strings, fieldIndex)); } } } diff --git a/file/src/main/java/org/apache/calcite/adapter/file/CsvFilterTableScanRule.java b/file/src/main/java/org/apache/calcite/adapter/file/CsvFilterTableScanRule.java new file mode 100644 index 000000000000..4ed61b2bcb58 --- /dev/null +++ b/file/src/main/java/org/apache/calcite/adapter/file/CsvFilterTableScanRule.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.adapter.file; + +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelRule; +import org.apache.calcite.rel.logical.LogicalFilter; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexUtil; + +import org.immutables.value.Value; + +/** + * Planner rule that pushes filter predicates into a + * {@link CsvTableScan}. + * + *

      Any predicate expressible as a {@link org.apache.calcite.rex.RexNode} + * (including AND, OR, NOT, IS NULL, comparisons, LIKE, etc.) can be pushed + * down. The condition is compiled at plan time via + * {@link org.apache.calcite.adapter.enumerable.RexToLixTranslator} into a + * Java {@link org.apache.calcite.linq4j.function.Predicate1} and applied + * directly on the enumerable produced by the scan, so no rows that fail the + * predicate are ever materialised. + * + * @see FileRules#FILTER_SCAN + */ +@Value.Enclosing +public class CsvFilterTableScanRule + extends RelRule { + + /** Creates a CsvFilterTableScanRule. */ + protected CsvFilterTableScanRule(Config config) { + super(config); + } + + @Override public void onMatch(RelOptRuleCall call) { + final LogicalFilter filter = call.rel(0); + final CsvTableScan scan = call.rel(1); + + // Compose a conjunction of the existing condition and the new one. + final RexNode newCondition; + if (scan.condition == null) { + newCondition = filter.getCondition(); + } else { + newCondition = + RexUtil.composeConjunction(scan.getCluster().getRexBuilder(), + java.util.Arrays.asList(scan.condition, filter.getCondition())); + } + + // Build a new scan that carries the pushed-down filter condition. + final CsvTableScan newScan = + new CsvTableScan(scan.getCluster(), scan.getTable(), scan.csvTable, + scan.fields, newCondition); + + call.transformTo(newScan); + } + + /** Rule configuration. */ + @Value.Immutable(singleton = false) + public interface Config extends RelRule.Config { + Config DEFAULT = ImmutableCsvFilterTableScanRule.Config.builder() + .withOperandSupplier(b0 -> + b0.operand(LogicalFilter.class).oneInput(b1 -> + b1.operand(CsvTableScan.class).noInputs())) + .build(); + + @Override default CsvFilterTableScanRule toRule() { + return new CsvFilterTableScanRule(this); + } + } +} diff --git a/file/src/main/java/org/apache/calcite/adapter/file/CsvProjectFilterTableScanRule.java b/file/src/main/java/org/apache/calcite/adapter/file/CsvProjectFilterTableScanRule.java new file mode 100644 index 000000000000..3bbcb5037ffb --- /dev/null +++ b/file/src/main/java/org/apache/calcite/adapter/file/CsvProjectFilterTableScanRule.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.adapter.file; + +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelRule; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.logical.LogicalFilter; +import org.apache.calcite.rel.logical.LogicalProject; +import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexUtil; + +import org.immutables.value.Value; + +import java.util.List; + +/** + * Planner rule that matches a {@link LogicalProject} on a {@link LogicalFilter} + * on a {@link CsvTableScan}, and pushes filter predicates into the scan. + * + * @see FileRules#PROJECT_FILTER_SCAN + */ +@Value.Enclosing +public class CsvProjectFilterTableScanRule + extends RelRule { + + /** Creates a CsvProjectFilterTableScanRule. */ + protected CsvProjectFilterTableScanRule(Config config) { + super(config); + } + + @Override public void onMatch(RelOptRuleCall call) { + final LogicalProject project = call.rel(0); + final LogicalFilter filter = call.rel(1); + final CsvTableScan scan = call.rel(2); + + // Find all input fields referenced by the project expressions + final java.util.Set projectInputFields = new java.util.HashSet<>(); + for (RexNode proj : project.getProjects()) { + proj.accept(new org.apache.calcite.rex.RexVisitorImpl(true) { + @Override public Void visitInputRef(RexInputRef inputRef) { + projectInputFields.add(inputRef.getIndex()); + return null; + } + }); + } + + // Find all input fields referenced by the filter condition + final java.util.Set filterInputFields = new java.util.HashSet<>(); + filter.getCondition().accept(new org.apache.calcite.rex.RexVisitorImpl(true) { + @Override public Void visitInputRef(RexInputRef inputRef) { + filterInputFields.add(inputRef.getIndex()); + return null; + } + }); + + // Union the projected/referenced indices + final java.util.Set neededProjectedIndices = new java.util.TreeSet<>(); + neededProjectedIndices.addAll(projectInputFields); + neededProjectedIndices.addAll(filterInputFields); + + // Map needed scan projected indices to full-table indices + final int[] newFields = new int[neededProjectedIndices.size()]; + int k = 0; + for (int idx : neededProjectedIndices) { + newFields[k++] = scan.fields[idx]; + } + + // Build index map from old projected index to new index in newFields + final java.util.Map indexMap = new java.util.HashMap<>(); + int newIdx = 0; + for (int idx : neededProjectedIndices) { + indexMap.put(idx, newIdx++); + } + + // Create shuttle to map RexInputRef indices + final org.apache.calcite.rex.RexShuttle shuttle = new org.apache.calcite.rex.RexShuttle() { + @Override public RexNode visitInputRef(RexInputRef inputRef) { + final Integer mapped = indexMap.get(inputRef.getIndex()); + if (mapped == null) { + return inputRef; + } + return scan.getCluster().getRexBuilder().makeInputRef(inputRef.getType(), mapped); + } + }; + + final RexNode mappedCondition = filter.getCondition().accept(shuttle); + final List mappedProjects = new java.util.ArrayList<>(); + for (RexNode proj : project.getProjects()) { + mappedProjects.add(proj.accept(shuttle)); + } + + final RexNode finalCondition; + if (scan.condition == null) { + finalCondition = mappedCondition; + } else { + finalCondition = + RexUtil.composeConjunction(scan.getCluster().getRexBuilder(), + java.util.Arrays.asList(scan.condition.accept(shuttle), mappedCondition)); + } + + final CsvTableScan newScan = + new CsvTableScan(scan.getCluster(), scan.getTable(), scan.csvTable, + newFields, finalCondition); + + final RelNode result = + project.copy(project.getTraitSet(), newScan, mappedProjects, project.getRowType()); + + call.transformTo(result); + } + + /** Rule configuration. */ + @Value.Immutable(singleton = false) + public interface Config extends RelRule.Config { + Config DEFAULT = ImmutableCsvProjectFilterTableScanRule.Config.builder() + .withOperandSupplier(b0 -> + b0.operand(LogicalProject.class).oneInput(b1 -> + b1.operand(LogicalFilter.class).oneInput(b2 -> + b2.operand(CsvTableScan.class).noInputs()))) + .build(); + + @Override default CsvProjectFilterTableScanRule toRule() { + return new CsvProjectFilterTableScanRule(this); + } + } +} diff --git a/file/src/main/java/org/apache/calcite/adapter/file/CsvProjectTableScanRule.java b/file/src/main/java/org/apache/calcite/adapter/file/CsvProjectTableScanRule.java index a0e006ae4ca8..79ba80722e40 100644 --- a/file/src/main/java/org/apache/calcite/adapter/file/CsvProjectTableScanRule.java +++ b/file/src/main/java/org/apache/calcite/adapter/file/CsvProjectTableScanRule.java @@ -45,17 +45,30 @@ protected CsvProjectTableScanRule(Config config) { @Override public void onMatch(RelOptRuleCall call) { final LogicalProject project = call.rel(0); final CsvTableScan scan = call.rel(1); - int[] fields = getProjectFields(project.getProjects()); - if (fields == null) { + int[] projectFieldIndices = getProjectFields(project.getProjects()); + if (projectFieldIndices == null) { // Project contains expressions more complex than just field references. return; } + if (scan.condition != null) { + // If the scan already has a condition, we cannot push the project down + // because the condition references the scan's current row type. + return; + } + // The project field indices are into the scan's *current* row type (which + // may already be a subset of the full table due to a prior projection). + // Map through scan.fields to get the original full-table column indices. + final int[] newFields = new int[projectFieldIndices.length]; + for (int i = 0; i < projectFieldIndices.length; i++) { + newFields[i] = scan.fields[projectFieldIndices[i]]; + } call.transformTo( new CsvTableScan( scan.getCluster(), scan.getTable(), scan.csvTable, - fields)); + newFields, + scan.condition)); } private static int[] getProjectFields(List exps) { diff --git a/file/src/main/java/org/apache/calcite/adapter/file/CsvTableScan.java b/file/src/main/java/org/apache/calcite/adapter/file/CsvTableScan.java index 8d7e80c6ee71..36b2ab332f34 100644 --- a/file/src/main/java/org/apache/calcite/adapter/file/CsvTableScan.java +++ b/file/src/main/java/org/apache/calcite/adapter/file/CsvTableScan.java @@ -16,6 +16,7 @@ */ package org.apache.calcite.adapter.file; +import org.apache.calcite.adapter.enumerable.EnumerableCalc; import org.apache.calcite.adapter.enumerable.EnumerableConvention; import org.apache.calcite.adapter.enumerable.EnumerableRel; import org.apache.calcite.adapter.enumerable.EnumerableRelImplementor; @@ -37,11 +38,14 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexProgram; import com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.Nullable; +import java.util.ArrayList; import java.util.List; import static java.util.Objects.requireNonNull; @@ -53,23 +57,32 @@ */ public class CsvTableScan extends TableScan implements EnumerableRel { final CsvTranslatableTable csvTable; - private final int[] fields; + final int[] fields; + final @Nullable RexNode condition; protected CsvTableScan(RelOptCluster cluster, RelOptTable table, CsvTranslatableTable csvTable, int[] fields) { + this(cluster, table, csvTable, fields, null); + } + + protected CsvTableScan(RelOptCluster cluster, RelOptTable table, + CsvTranslatableTable csvTable, int[] fields, + @Nullable RexNode condition) { super(cluster, cluster.traitSetOf(EnumerableConvention.INSTANCE), ImmutableList.of(), table); this.csvTable = requireNonNull(csvTable, "csvTable"); this.fields = fields; + this.condition = condition; } @Override public RelNode copy(RelTraitSet traitSet, List inputs) { assert inputs.isEmpty(); - return new CsvTableScan(getCluster(), table, csvTable, fields); + return new CsvTableScan(getCluster(), table, csvTable, fields, condition); } @Override public RelWriter explainTerms(RelWriter pw) { return super.explainTerms(pw) - .item("fields", Primitive.asList(fields)); + .item("fields", Primitive.asList(fields)) + .itemIf("condition", condition, condition != null); } @Override public RelDataType deriveRowType() { @@ -84,6 +97,8 @@ protected CsvTableScan(RelOptCluster cluster, RelOptTable table, @Override public void register(RelOptPlanner planner) { planner.addRule(FileRules.PROJECT_SCAN); + planner.addRule(FileRules.FILTER_SCAN); + planner.addRule(FileRules.PROJECT_FILTER_SCAN); } @Override public @Nullable RelOptCost computeSelfCost(RelOptPlanner planner, @@ -93,12 +108,14 @@ protected CsvTableScan(RelOptCluster cluster, RelOptTable table, // // The "+ 2D" on top and bottom keeps the function fairly smooth. // - // For example, if table has 3 fields, project has 1 field, - // then factor = (1 + 2) / (3 + 2) = 0.6 - final RelOptCost cost = requireNonNull(super.computeSelfCost(planner, mq)); - return cost - .multiplyBy(((double) fields.length + 2D) - / ((double) table.getRowType().getFieldCount() + 2D)); + // For example, if the table has 3 fields and the scan has 1 field, + // then factor = (1 + 2) / (3 + 2) = 0.6. + final RelOptCost cost = + requireNonNull(super.computeSelfCost(planner, mq)); + final double factor = + (fields.length + 2D) + / (table.getRowType().getFieldCount() + 2D); + return cost.multiplyBy(factor); } @Override public Result implement(EnumerableRelImplementor implementor, Prefer pref) { @@ -110,11 +127,32 @@ protected CsvTableScan(RelOptCluster cluster, RelOptTable table, final Expression expression = requireNonNull(table.getExpression(CsvTranslatableTable.class)); - return implementor.result( - physType, - Blocks.toBlock( - Expressions.call(expression, - "project", implementor.getRootExpression(), - Expressions.constant(fields)))); + + // Call CsvTranslatableTable.project(root, fields) to get the base enumerable. + Expression enumerable = + Expressions.call(expression, + "project", implementor.getRootExpression(), + Expressions.constant(fields)); + + if (condition != null) { + final List projects = new ArrayList<>(); + for (int i = 0; i < getRowType().getFieldCount(); i++) { + projects.add( + getCluster().getRexBuilder().makeInputRef( + getRowType().getFieldList().get(i).getType(), i)); + } + final RexProgram program = + RexProgram.create(getRowType(), projects, condition, + getRowType(), getCluster().getRexBuilder()); + + // Create a scan node without the condition so EnumerableCalc sees a plain + // enumerable input, then wrap it with EnumerableCalc to apply the filter. + final CsvTableScan plainScan = + new CsvTableScan(getCluster(), table, csvTable, fields); + final EnumerableCalc calc = EnumerableCalc.create(plainScan, program); + return calc.implement(implementor, pref); + } + + return implementor.result(physType, Blocks.toBlock(enumerable)); } } diff --git a/file/src/main/java/org/apache/calcite/adapter/file/FileRules.java b/file/src/main/java/org/apache/calcite/adapter/file/FileRules.java index 9c7e228c746d..15468c2328d8 100644 --- a/file/src/main/java/org/apache/calcite/adapter/file/FileRules.java +++ b/file/src/main/java/org/apache/calcite/adapter/file/FileRules.java @@ -24,4 +24,19 @@ private FileRules() {} * a {@link CsvTableScan} and pushes down projects if possible. */ public static final CsvProjectTableScanRule PROJECT_SCAN = CsvProjectTableScanRule.Config.DEFAULT.toRule(); + + /** Rule that matches a {@link org.apache.calcite.rel.core.Filter} on + * a {@link CsvTableScan} and pushes arbitrary predicates into the scan. + * Any {@link org.apache.calcite.rex.RexNode} condition is compiled at plan + * time via {@link org.apache.calcite.adapter.enumerable.RexToLixTranslator} + * into a {@link org.apache.calcite.linq4j.function.Predicate1}. */ + public static final CsvFilterTableScanRule FILTER_SCAN = + CsvFilterTableScanRule.Config.DEFAULT.toRule(); + + /** Rule that matches a {@link org.apache.calcite.rel.core.Project} on + * a {@link org.apache.calcite.rel.core.Filter} on a {@link CsvTableScan}, + * pushes the filter condition into the scan, and remaps project and filter + * input references to match the scan's new projection. */ + public static final CsvProjectFilterTableScanRule PROJECT_FILTER_SCAN = + CsvProjectFilterTableScanRule.Config.DEFAULT.toRule(); } diff --git a/file/src/test/java/org/apache/calcite/adapter/file/CsvEnumeratorTest.java b/file/src/test/java/org/apache/calcite/adapter/file/CsvEnumeratorTest.java index 43f4a6b24f9f..92566afbb546 100644 --- a/file/src/test/java/org/apache/calcite/adapter/file/CsvEnumeratorTest.java +++ b/file/src/test/java/org/apache/calcite/adapter/file/CsvEnumeratorTest.java @@ -16,12 +16,16 @@ */ package org.apache.calcite.adapter.file; +import org.apache.calcite.rel.type.RelDataType; + import org.junit.jupiter.api.Test; import java.math.BigDecimal; +import java.util.Arrays; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertThrows; /** @@ -57,4 +61,14 @@ private static void checkThrows(int precision, int scale, String s) { assertThrows(IllegalArgumentException.class, () -> CsvEnumerator.parseDecimal(precision, scale, s)); } + + @Test void testConvertRowWithMissingFields() { + final CsvEnumerator.RowConverter converter = + CsvEnumerator.arrayConverter( + Arrays.asList(null, null, null, null), + Arrays.asList(0, 1, 2, 3), false); + + assertArrayEquals(new Object[] {"a", "b", "c", null}, + converter.convertRow(new String[] {"a", "b", "c"})); + } } diff --git a/file/src/test/java/org/apache/calcite/adapter/file/FileAdapterTest.java b/file/src/test/java/org/apache/calcite/adapter/file/FileAdapterTest.java index ad774f3964e6..f818c8a156db 100644 --- a/file/src/test/java/org/apache/calcite/adapter/file/FileAdapterTest.java +++ b/file/src/test/java/org/apache/calcite/adapter/file/FileAdapterTest.java @@ -17,8 +17,18 @@ package org.apache.calcite.adapter.file; import org.apache.calcite.jdbc.CalciteConnection; +import org.apache.calcite.plan.RelOptRule; +import org.apache.calcite.plan.RelOptUtil; +import org.apache.calcite.plan.hep.HepPlanner; +import org.apache.calcite.plan.hep.HepProgramBuilder; +import org.apache.calcite.rel.RelNode; import org.apache.calcite.schema.Schema; +import org.apache.calcite.schema.SchemaPlus; +import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql2rel.SqlToRelConverter; +import org.apache.calcite.tools.FrameworkConfig; +import org.apache.calcite.tools.Frameworks; +import org.apache.calcite.tools.Planner; import org.apache.calcite.util.TestUtil; import com.google.common.collect.ImmutableMap; @@ -47,11 +57,13 @@ import static org.apache.calcite.adapter.file.FileAdapterTests.sql; +import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.isA; import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; /** * System test of the Calcite file adapter, which can read and parse @@ -417,6 +429,158 @@ private static void checkEmpty(ResultSet resultSet) { sql("model-with-custom-table", sql).ok(); } + /** Test case for + * [CALCITE-7618] + * Add filter pushdown support to file adapter's CSV implementation. + * + *

      Verifies that a simple equality filter is pushed into {@link CsvTableScan}, + * eliminating the {@code EnumerableCalc} that would otherwise evaluate it. */ + @Test void testFilterPushDown() { + final String sql = "explain plan for select * from EMPS where deptno = 20"; + final String expected = "PLAN=CsvTableScan(table=[[SALES, EMPS]], " + + "fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]], condition=[=($2, 20)])\n"; + sql("smart", sql).returns(expected).ok(); + } + + @Test void testFilterPushDownWithProject() { + final String sql = "explain plan for select name, empno from EMPS where deptno = 20"; + final String expected = "PLAN=EnumerableCalc(expr#0..2=[{inputs}]," + + " expr#3=[20], expr#4=[=($t2, $t3)], NAME=[$t1], EMPNO=[$t0], $condition=[$t4])\n" + + " CsvTableScan(table=[[SALES, EMPS]], fields=[[0, 1, 2]])\n"; + sql("smart", sql).returns(expected).ok(); + } + + /** Test case for + * [CALCITE-7618] + * Add filter pushdown support to file adapter's CSV implementation. + * + *

      Verifies that filter pushdown returns correct query results. */ + @Test void testFilterPushDownResult() { + final String sql = "select name, empno from EMPS where deptno = 20"; + sql("smart", sql) + .returns("NAME=Eric; EMPNO=110", + "NAME=Wilma; EMPNO=120") + .ok(); + } + + /** Test case for + * [CALCITE-7618] + * Add filter pushdown support to file adapter's CSV implementation. + * + *

      Verifies that range filters are evaluated correctly under the new compiled-filter + * pushdown mechanism. */ + @Test void testRangeFilterPushDown() { + // empno > 110 is a range filter; the compiler-based pushdown handles it + // like any other predicate, pushing it into the scan via EnumerableCalc. + final String sql = "select name from EMPS where empno > 110"; + sql("smart", sql) + .returns("NAME=Wilma", + "NAME=Alice") + .ok(); + } + + @Test void testFilterOnNullValues() { + final String sql = "select name, age from long_emps where age is null"; + sql("bug", sql) + .returns("NAME=John; AGE=null", + "NAME=Alice; AGE=null") + .ok(); + } + + @Test void testFilterPushDownLong() { + final String sql = "select name from long_emps where empno = 130"; + sql("bug", sql) + .returns("NAME=Alice") + .ok(); + final String plan = "explain plan for " + sql; + sql("bug", plan) + .returns("PLAN=EnumerableCalc(expr#0..1=[{inputs}]," + + " expr#2=[130:BIGINT], expr#3=[=($t0, $t2)], NAME=[$t1], $condition=[$t3])\n" + + " CsvTableScan(table=[[BUG, LONG_EMPS]], fields=[[0, 1]])\n") + .ok(); + } + + @Test void testFilterPushDownBoolean() { + final String sql = "select name from long_emps where slacker = true"; + sql("bug", sql) + .returns("NAME=Fred") + .ok(); + final String plan = "explain plan for " + sql; + sql("bug", plan) + .returns("PLAN=EnumerableCalc(expr#0..1=[{inputs}], NAME=[$t0], $condition=[$t1])\n" + + " CsvTableScan(table=[[BUG, LONG_EMPS]], fields=[[1, 7]])\n") + .ok(); + } + + @Test void testFilterPushDownString() { + final String sql = "select empno from long_emps where gender = 'F'"; + sql("bug", sql) + .returns("EMPNO=120", "EMPNO=130") + .ok(); + final String plan = "explain plan for " + sql; + sql("bug", plan) + .returns("PLAN=EnumerableCalc(expr#0..1=[{inputs}], expr#2=['F':VARCHAR]," + + " expr#3=[=($t1, $t2)], EMPNO=[$t0], $condition=[$t3])\n" + + " CsvTableScan(table=[[BUG, LONG_EMPS]], fields=[[0, 3]])\n") + .ok(); + } + + @Test void testFilterPushDownDecimal() { + final String sql = "select deptno from sales.\"DECIMAL\" where budget = 100.01"; + sql("sales-csv", sql) + .returns("DEPTNO=20") + .ok(); + final String plan = "explain plan for " + sql; + sql("sales-csv", plan) + .returns("PLAN=EnumerableCalc(expr#0..1=[{inputs}], DEPTNO=[$t0])\n" + + " CsvTableScan(table=[[SALES, DECIMAL]], fields=[[0, 1]], condition=[=($1, 100.01)])\n") + .ok(); + } + + @Test void testFilterPushDownDate() { + final String sql = "select name from long_emps where joinedat = DATE '2001-01-01'"; + sql("bug", sql) + .returns("NAME=Eric") + .ok(); + final String plan = "explain plan for " + sql; + sql("bug", plan) + .returns("PLAN=EnumerableCalc(expr#0..1=[{inputs}], expr#2=[2001-01-01]," + + " expr#3=[=($t1, $t2)], NAME=[$t0], $condition=[$t3])\n" + + " CsvTableScan(table=[[BUG, LONG_EMPS]], fields=[[1, 9]])\n") + .ok(); + } + + @Test void testFilterPushDownTime() { + final String sql = "select empno from \"DATE\" where jointime = TIME '07:15:56'"; + sql("bug", sql) + .returns("EMPNO=140") + .ok(); + final String plan = "explain plan for " + sql; + sql("bug", plan) + .returns("PLAN=EnumerableCalc(expr#0..1=[{inputs}], expr#2=[07:15:56]," + + " expr#3=[=($t1, $t2)], EMPNO=[$t0], $condition=[$t3])\n" + + " CsvTableScan(table=[[BUG, DATE]], fields=[[0, 2]])\n") + .ok(); + } + + @Test void testFilterPushDownTimestamp() { + final String sql = "select empno from \"DATE\" where" + + " jointimes = TIMESTAMP '2015-12-31 07:15:56'"; + sql("bug", sql) + .returns("EMPNO=140") + .ok(); + final String plan = "explain plan for " + sql; + sql("bug", plan) + .returns("PLAN=EnumerableCalc(expr#0..1=[{inputs}]," + + " expr#2=[2015-12-31 07:15:56], expr#3=[=($t1, $t2)]," + + " EMPNO=[$t0], $condition=[$t3])\n" + + " CsvTableScan(table=[[BUG, DATE]], fields=[[0, 3]])\n") + .ok(); + } + + + + @Test void testPushDownProject() { final String sql = "explain plan for select * from EMPS"; final String expected = "PLAN=CsvTableScan(table=[[SALES, EMPS]], " @@ -438,6 +602,52 @@ private static void checkEmpty(ResultSet resultSet) { .ok(); } + @Test void testFilterPushDownOr() { + final String sql = "select name from EMPS where deptno = 20 or empno = 100"; + sql("smart", sql) + .returns("NAME=Fred", "NAME=Eric", "NAME=Wilma") + .ok(); + final String plan = "explain plan for " + sql; + sql("smart", plan) + .returns("PLAN=EnumerableCalc(expr#0..2=[{inputs}], expr#3=[20]," + + " expr#4=[=($t2, $t3)], expr#5=[100], expr#6=[=($t0, $t5)]," + + " expr#7=[OR($t4, $t6)], NAME=[$t1], $condition=[$t7])\n" + + " CsvTableScan(table=[[SALES, EMPS]], fields=[[0, 1, 2]])\n") + .ok(); + } + + @Test void testFilterPushDownNotEquals() { + sql("smart", "select name from EMPS where deptno <> 20") + .returns("NAME=Fred", "NAME=John", "NAME=Alice") + .ok(); + } + + @Test void testFilterPushDownNotEqualsPlan() throws Exception { + final String plan = + applyRule("select name from EMPS where deptno <> 20", + FileRules.PROJECT_FILTER_SCAN); + + assertThat(plan, + containsString("CsvTableScan(table=[[SALES, EMPS]], " + + "fields=[[1, 2]], condition=[<>($1, 20)])")); + } + + @Test void testFilterPushDownRange() { + sql("smart", "select name from EMPS where empno >= 120") + .returns("NAME=Wilma", "NAME=Alice") + .ok(); + } + + @Test void testFilterPushDownRangePlan() throws Exception { + final String plan = + applyRule("select name from EMPS where empno >= 120", + FileRules.PROJECT_FILTER_SCAN); + + assertThat(plan, + containsString("CsvTableScan(table=[[SALES, EMPS]], " + + "fields=[[0, 1]], condition=[>=($0, 120)])")); + } + @ParameterizedTest @MethodSource("explainFormats") void testPushDownProjectAggregate(String format) { @@ -471,21 +681,15 @@ void testPushDownProjectAggregateWithFilter(String format) { switch (format) { case "dot": expected = "PLAN=digraph {\n" - + "\"EnumerableCalc\\nexpr#0..1 = {inputs}\\nexpr#2 = 'F':VARCHAR\\nexpr#3 = =($t1, $t2)" - + "\\nproj#0..1 = {exprs}\\n$condition = $t3\" -> \"EnumerableAggregate\\ngroup = " - + "{}\\nEXPR$0 = MAX($0)\\n\" [label=\"0\"]\n" - + "\"CsvTableScan\\ntable = [SALES, EMPS\\n]\\nfields = [0, 3]\\n\" -> " - + "\"EnumerableCalc\\nexpr#0..1 = {inputs}\\nexpr#2 = 'F':VARCHAR\\nexpr#3 = =($t1, $t2)" - + "\\nproj#0..1 = {exprs}\\n$condition = $t3\" [label=\"0\"]\n" + + "\"CsvTableScan\\ntable = [SALES, EMPS\\n]\\nfields = [0, 3]\\ncondition = =($1, 'F\\n')\\n\" " + + "-> \"EnumerableAggregate\\ngroup = {}\\nEXPR$0 = MAX($0)\\n\" [label=\"0\"]\n" + "}\n"; extra = " as dot "; break; case "text": expected = "PLAN=" + "EnumerableAggregate(group=[{}], EXPR$0=[MAX($0)])\n" - + " EnumerableCalc(expr#0..1=[{inputs}], expr#2=['F':VARCHAR], " - + "expr#3=[=($t1, $t2)], proj#0..1=[{exprs}], $condition=[$t3])\n" - + " CsvTableScan(table=[[SALES, EMPS]], fields=[[0, 3]])\n"; + + " CsvTableScan(table=[[SALES, EMPS]], fields=[[0, 3]], condition=[=($1, 'F')])\n"; extra = ""; break; } @@ -1105,4 +1309,171 @@ private String range(int first, int count) { is(Timestamp.valueOf("1996-08-03 00:01:02"))); } } + + @Test void testFilterPushDownDoesNotReturnNullRows() { + // age column has nulls in the data — make sure they're excluded, not included + final String sql = "select name from long_emps where age = 25"; + sql("bug", sql) + .returns("NAME=Fred") // only Fred has age=25, null-age rows must not appear + .ok(); + } + + @Test void testFilterPushDownNullColumnExcluded() { + // slacker has null values — null rows must not match true or false + final String sql = "select name from long_emps where slacker = false"; + sql("bug", sql) + .returns("NAME=John", "NAME=Alice") // Eric and Wilma have null slacker — excluded + .ok(); + } + + @Test void testSameValueBehavior() { + // Basic null behavior + assertFalse(CsvEnumerator.sameValue(null, null)); + assertFalse(CsvEnumerator.sameValue(null, new BigDecimal("1.0"))); + assertFalse(CsvEnumerator.sameValue(new BigDecimal("1.0"), null)); + assertFalse(CsvEnumerator.sameValue(null, "hello")); + assertFalse(CsvEnumerator.sameValue("hello", null)); + + // Mixed null and zero + assertFalse(CsvEnumerator.sameValue(null, 0)); + assertFalse(CsvEnumerator.sameValue(null, BigDecimal.ZERO)); + assertFalse(CsvEnumerator.sameValue(null, "")); + + // NULL IS NOT DISTINCT FROM NULL → should be TRUE under IS NOT DISTINCT FROM semantics, + // but sameValue implements SQL WHERE filter '=' semantics (where null = null evaluates + // to UNKNOWN, which behaves as false). + assertFalse(CsvEnumerator.sameValue(null, 1)); + assertFalse(CsvEnumerator.sameValue(1, null)); + + // Large scale differences + assertTrue(CsvEnumerator.sameValue(new BigDecimal("1.000000"), new BigDecimal("1"))); + + // Negative zero edge case + assertTrue(CsvEnumerator.sameValue(new BigDecimal("0.0"), new BigDecimal("-0.0"))); + + // Very large numbers with scale + assertTrue( + CsvEnumerator.sameValue( + new BigDecimal("999999999.9"), new BigDecimal("999999999.90"))); + + // Strings + assertTrue(CsvEnumerator.sameValue("hello", "hello")); + assertFalse(CsvEnumerator.sameValue("hello", "world")); + + // Integers / Longs + assertTrue(CsvEnumerator.sameValue(42, 42)); + assertFalse(CsvEnumerator.sameValue(42, 43)); + assertTrue(CsvEnumerator.sameValue(1L, 1L)); + + // Cross-type comparison (implicit type promotion is not handled by sameValue, returns false) + assertFalse(CsvEnumerator.sameValue(42, 42L)); + + // BigDecimal scale differences & symmetry + BigDecimal val = new BigDecimal("2.0"); + assertTrue(CsvEnumerator.sameValue(val, val)); + assertTrue(CsvEnumerator.sameValue(new BigDecimal("2.0"), new BigDecimal("2.00"))); + assertTrue(CsvEnumerator.sameValue(new BigDecimal("2.00"), new BigDecimal("2.0"))); + assertFalse(CsvEnumerator.sameValue(new BigDecimal("1.0"), new BigDecimal("2.0"))); + assertTrue(CsvEnumerator.sameValue(new BigDecimal("0.0"), new BigDecimal("0.00"))); + assertTrue(CsvEnumerator.sameValue(new BigDecimal("-1.0"), new BigDecimal("-1.00"))); + + // Objects.equals() performs exact class/structure comparison (including scale + // for BigDecimal), which incorrectly returns false for semantically equal numbers. + // Confirm Objects.equals fails here. + assertFalse(java.util.Objects.equals(new BigDecimal("2.0"), new BigDecimal("2.00"))); + } + + @SuppressWarnings("deprecation") + private static String applyRule(String sql, RelOptRule rule) + throws Exception { + final Properties info = new Properties(); + info.put("model", FileAdapterTests.jsonPath("smart")); + + try (Connection connection = + DriverManager.getConnection("jdbc:calcite:", info)) { + final CalciteConnection calciteConnection = + connection.unwrap(CalciteConnection.class); + final SchemaPlus salesSchema = + calciteConnection.getRootSchema().getSubSchema("SALES"); + + final FrameworkConfig config = Frameworks.newConfigBuilder() + .defaultSchema(salesSchema) + .build(); + + final Planner planner = Frameworks.getPlanner(config); + final SqlNode parsed = planner.parse(sql); + final SqlNode validated = planner.validate(parsed); + final RelNode rel = planner.rel(validated).project(); + + final HepProgramBuilder programBuilder = new HepProgramBuilder(); + programBuilder.addRuleInstance(rule); + + final HepPlanner hepPlanner = + new HepPlanner(programBuilder.build()); + hepPlanner.setRoot(rel); + + return RelOptUtil.toString(hepPlanner.findBestExp()); + } + } + + @Test void testFilterPushDownRule() throws Exception { + final String plan = + applyRule("select * from EMPS where deptno = 20", + FileRules.FILTER_SCAN); + + assertThat(plan, + containsString("CsvTableScan(table=[[SALES, EMPS]], " + + "fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]], " + + "condition=[=($2, 20)])")); + } + + @Test void testProjectFilterPushDownRule() throws Exception { + final String plan = + applyRule("select name, empno from EMPS where deptno = 20", + FileRules.PROJECT_FILTER_SCAN); + + assertThat(plan, + containsString("CsvTableScan(table=[[SALES, EMPS]], " + + "fields=[[0, 1, 2]], condition=[=($2, 20)])")); + } + + @SuppressWarnings("deprecation") + @Test void testFilterProjectTransposeWithProjectFilterScan() throws Exception { + final String sql = "select name from (select name, deptno from EMPS) where deptno = 20"; + + final Properties info = new Properties(); + info.put("model", FileAdapterTests.jsonPath("smart")); + + try (Connection connection = + DriverManager.getConnection("jdbc:calcite:", info)) { + final CalciteConnection calciteConnection = + connection.unwrap(CalciteConnection.class); + final SchemaPlus salesSchema = + calciteConnection.getRootSchema().getSubSchema("SALES"); + + final FrameworkConfig config = Frameworks.newConfigBuilder() + .defaultSchema(salesSchema) + .build(); + + final Planner planner = Frameworks.getPlanner(config); + final SqlNode parsed = planner.parse(sql); + final SqlNode validated = planner.validate(parsed); + final RelNode rel = planner.rel(validated).project(); + + final HepProgramBuilder programBuilder = new HepProgramBuilder(); + programBuilder.addRuleInstance( + org.apache.calcite.rel.rules.CoreRules.FILTER_PROJECT_TRANSPOSE); + programBuilder.addRuleInstance(FileRules.PROJECT_FILTER_SCAN); + + final HepPlanner hepPlanner = + new HepPlanner(programBuilder.build()); + hepPlanner.setRoot(rel); + + final String plan = RelOptUtil.toString(hepPlanner.findBestExp()); + + assertThat(plan, + containsString("CsvTableScan(table=[[SALES, EMPS]], " + + "fields=[[1, 2]], condition=[=($1, 20)])")); + } + } } From b014fbe8e4e9c2b58f52a96680501eb0a635e164 Mon Sep 17 00:00:00 2001 From: AlexisCubilla Date: Tue, 21 Jul 2026 23:07:11 -0300 Subject: [PATCH 403/562] [CALCITE-7663] RelToSqlConverter generates ambiguous column references when expanding SELECT * over a join with duplicate field names When a dialect's supportGenerateSelectStar() returns false for a join with duplicate field names, the SELECT * expansion in SqlImplementor did not alias the expanded columns to their unique row-type field names. A sub-query wrapping such a join then exposed two identically named columns, making outer references ambiguous (e.g. PostgreSQL: column reference "id" is ambiguous). Alias each expanded column to its unique row-type field name, mirroring the validator path. --- .../calcite/rel/rel2sql/SqlImplementor.java | 32 +++++++++++++- .../rel/rel2sql/RelToSqlConverterTest.java | 43 ++++++++++++++++++- 2 files changed, 72 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java b/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java index c74da741d7c3..4b42df2eeca1 100644 --- a/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java +++ b/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java @@ -297,6 +297,18 @@ protected SqlCall as(SqlNode e, String alias, String... fieldNames) { return SqlStdOperatorTable.AS.createCall(POS, operandList); } + /** Wraps a column reference in an {@code AS} alias when its intrinsic name + * differs from {@code name}, so that the column is emitted with + * {@code name}. */ + private SqlNode renameAs(SqlNode fieldNode, String name) { + final String currentName = fieldNode instanceof SqlIdentifier + ? Util.last(((SqlIdentifier) fieldNode).names) + : null; + return name.equals(currentName) + ? fieldNode + : as(fieldNode, name); + } + /** Returns whether a list of expressions projects all fields, in order, * from the input, with the same names. */ public static boolean isStar(List exps, RelDataType inputRowType, @@ -2082,9 +2094,17 @@ private Builder builder(RelNode rel, Set clauses) { newContext = aliasContext(aliases, qualified); } if (!dialect.supportGenerateSelectStar(rel.getInput(0))) { + // Rename each expanded column to its (unique) row-type field name. + // Otherwise a sub-query that wraps a join with duplicate field names + // (e.g. two columns named DEPTNO) would expose two identically named + // columns, which is ambiguous when referenced from an outer query. + final List fieldNames = rel.getRowType().getFieldNames(); final List expandedSelectList = new ArrayList<>(); for (int i = 0; i < newContext.fieldCount; i++) { - expandedSelectList.add(newContext.field(i)); + final SqlNode field = newContext.field(i); + expandedSelectList.add(i < fieldNames.size() + ? renameAs(field, fieldNames.get(i)) + : field); } select.setSelectList(new SqlNodeList(expandedSelectList, POS)); } @@ -2449,9 +2469,17 @@ SqlSelect maybeExpandStar(SqlSelect select) { boolean qualified = !dialect.hasImplicitTableAlias() || aliases.size() > 1; final Context ctx = aliasContext(aliases, qualified); + // Rename each expanded column to its (unique) row-type field name. + // Otherwise a sub-query that wraps a join with duplicate field names + // (e.g. two columns named DEPTNO) would expose two identically named + // columns, which is ambiguous when referenced from an outer query. + final List fieldNames = expectedRel.getRowType().getFieldNames(); final List expandedList = new ArrayList<>(); for (int i = 0; i < ctx.fieldCount; i++) { - expandedList.add(ctx.field(i)); + final SqlNode field = ctx.field(i); + expandedList.add(i < fieldNames.size() + ? renameAs(field, fieldNames.get(i)) + : field); } return new SqlSelect(select.getParserPosition(), (SqlNodeList) select.getOperandList().get(0), diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java index 801612976b00..3fe96782339a 100644 --- a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java +++ b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java @@ -9991,11 +9991,14 @@ private void checkLiteral2(String expression, String expected) { b.equals(b.field(2, 0, "DEPTNO"), b.field(2, 1, "DEPTNO"))) .build(); + // The join has two columns named DEPTNO; the second is aliased to its + // unique row-type field name (DEPTNO0) so the result never exposes two + // identically named columns (CALCITE-7663). final String expected = "SELECT" + " \"EMP\".\"EMPNO\", \"EMP\".\"ENAME\", \"EMP\".\"JOB\"," + " \"EMP\".\"MGR\", \"EMP\".\"HIREDATE\", \"EMP\".\"SAL\"," + " \"EMP\".\"COMM\", \"EMP\".\"DEPTNO\"," - + " \"DEPT\".\"DEPTNO\"," + + " \"DEPT\".\"DEPTNO\" AS \"DEPTNO0\"," + " \"DEPT\".\"DNAME\", \"DEPT\".\"LOC\"\n" + "FROM \"scott\".\"EMP\"\n" + "INNER JOIN \"scott\".\"DEPT\"" @@ -10003,6 +10006,44 @@ private void checkLiteral2(String expression, String expected) { relFn(relFn).dialect(NO_STAR_DIALECT).ok(expected); } + /** Test case for + * [CALCITE-7663]. + * A join with duplicate field names (two DEPTNO) wrapped by a FETCH becomes a + * sub-query; when it is joined again, the sub-query must not expose two + * columns with the same name, otherwise the outer references to them are + * ambiguous (e.g. PostgreSQL: {@code column reference "deptno" is ambiguous}). + * Each expanded column is aliased to its unique row-type field name. */ + @Test void testNoSelectStarJoinWithDuplicateNamesAndFetchIsNotAmbiguous() { + final Function relFn = b -> b + .scan("EMP") + .scan("DEPT") + .join(JoinRelType.INNER, + b.equals(b.field(2, 0, "DEPTNO"), b.field(2, 1, "DEPTNO"))) + .limit(0, 10) + .scan("DEPT") + .join(JoinRelType.INNER, + b.equals(b.field(2, 0, "EMPNO"), b.field(2, 1, "DEPTNO"))) + .limit(0, 5) + .build(); + final String expected = "SELECT \"t\".\"EMPNO\", \"t\".\"ENAME\"," + + " \"t\".\"JOB\", \"t\".\"MGR\", \"t\".\"HIREDATE\", \"t\".\"SAL\"," + + " \"t\".\"COMM\", \"t\".\"DEPTNO\", \"t\".\"DEPTNO0\"," + + " \"t\".\"DNAME\", \"t\".\"LOC\"," + + " \"DEPT0\".\"DEPTNO\" AS \"DEPTNO1\"," + + " \"DEPT0\".\"DNAME\" AS \"DNAME0\", \"DEPT0\".\"LOC\" AS \"LOC0\"\n" + + "FROM (SELECT \"EMP\".\"EMPNO\", \"EMP\".\"ENAME\", \"EMP\".\"JOB\"," + + " \"EMP\".\"MGR\", \"EMP\".\"HIREDATE\", \"EMP\".\"SAL\"," + + " \"EMP\".\"COMM\", \"EMP\".\"DEPTNO\"," + + " \"DEPT\".\"DEPTNO\" AS \"DEPTNO0\", \"DEPT\".\"DNAME\", \"DEPT\".\"LOC\"\n" + + "FROM \"scott\".\"EMP\"\n" + + "INNER JOIN \"scott\".\"DEPT\" ON \"EMP\".\"DEPTNO\" = \"DEPT\".\"DEPTNO\"\n" + + "FETCH NEXT 10 ROWS ONLY) AS \"t\"\n" + + "INNER JOIN \"scott\".\"DEPT\" AS \"DEPT0\"" + + " ON \"t\".\"EMPNO\" = \"DEPT0\".\"DEPTNO\"\n" + + "FETCH NEXT 5 ROWS ONLY"; + relFn(relFn).withPostgresql().ok(expected); + } + /** Test case for * [CALCITE-7483] * RelToSqlConverter generates SELECT * despite supportGenerateSelectStar. From aaaaa38f6a03f79e56755bf194923b60dc18a910 Mon Sep 17 00:00:00 2001 From: Darpan Date: Wed, 22 Jul 2026 12:19:38 +0530 Subject: [PATCH 404/562] [CALCITE-7661] RelDecorrelator loses shared correlation constraint across inner join inputs --- .../calcite/sql2rel/RelDecorrelator.java | 18 +++-- .../calcite/sql2rel/RelDecorrelatorTest.java | 77 +++++++++++++++++++ core/src/test/resources/sql/sub-query.iq | 30 ++++++++ 3 files changed, 119 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java b/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java index b4999b455bef..4e4104ad4876 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java @@ -2074,12 +2074,18 @@ private static boolean isWidening(RelDataType type, RelDataType type1) { joinConditions.add(originalCond); } - if (generatesNullsOnLeft || generatesNullsOnRight) { - List conds = - buildCorDefJoinConditions(leftCorDefOutputs, rightCorDefOutputs, - newLeftFrame.r, newRightFrame.r, relBuilder); - joinConditions.addAll(conds); - } + // Decorrelation propagates references to outer columns as columns in the + // rewritten inputs. If both join inputs propagate the same reference, add + // a condition to ensure that they still represent the same outer value. + // This applies to every join type, regardless of whether it generates nulls; + // if the inputs have no references in common, no condition is added. + joinConditions.addAll( + buildCorDefJoinConditions( + newLeftFrame.corDefOutputs, + newRightFrame.corDefOutputs, + newLeftFrame.r, + newRightFrame.r, + relBuilder)); RexNode finalCondition = joinConditions.isEmpty() ? relBuilder.literal(true) diff --git a/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java b/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java index 240f9a36ae02..51c1e89d2d9d 100644 --- a/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java +++ b/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java @@ -1737,6 +1737,83 @@ public static Frameworks.ConfigBuilder config() { assertThat(after, hasTree(planAfter)); } + /** + * Test case for [CALCITE-7661] + * RelDecorrelator loses shared correlation constraint across inner join inputs. + */ + @Test void testDecorrelateInnerJoinWithSharedCorrelation() { + final FrameworkConfig frameworkConfig = config().build(); + final RelBuilder builder = RelBuilder.create(frameworkConfig); + final RelOptCluster cluster = builder.getCluster(); + final Planner planner = Frameworks.getPlanner(frameworkConfig); + final String sql = "" + + "SELECT d.deptno FROM dept d WHERE EXISTS (\n" + + " SELECT *\n" + + " FROM (SELECT * FROM emp e WHERE e.deptno = d.deptno) l\n" + + " JOIN (SELECT * FROM dept d2 WHERE d2.deptno = d.deptno) r\n" + + " ON TRUE)"; + final RelNode originalRel; + try { + final SqlNode parse = planner.parse(sql); + final SqlNode validate = planner.validate(parse); + originalRel = planner.rel(validate).rel; + } catch (Exception e) { + throw TestUtil.rethrow(e); + } + + final HepProgram hepProgram = HepProgram.builder() + .addRuleCollection( + ImmutableList.of( + CoreRules.FILTER_SUB_QUERY_TO_CORRELATE, + CoreRules.PROJECT_SUB_QUERY_TO_CORRELATE, + CoreRules.JOIN_SUB_QUERY_TO_CORRELATE)) + .build(); + final Program program = + Programs.of(hepProgram, true, + requireNonNull(cluster.getMetadataProvider())); + final RelNode before = + program.run(cluster.getPlanner(), originalRel, cluster.traitSet(), + Collections.emptyList(), Collections.emptyList()); + final String planBefore = "" + + "LogicalProject(DEPTNO=[$0])\n" + + " LogicalProject(DEPTNO=[$0], DNAME=[$1], LOC=[$2])\n" + + " LogicalCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{0}])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n" + + " LogicalAggregate(group=[{0}])\n" + + " LogicalProject(i=[true])\n" + + " LogicalJoin(condition=[true], joinType=[inner])\n" + + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], " + + "HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7])\n" + + " LogicalFilter(condition=[=($7, $cor0.DEPTNO)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalProject(DEPTNO=[$0], DNAME=[$1], LOC=[$2])\n" + + " LogicalFilter(condition=[=($0, $cor0.DEPTNO)])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n"; + assertThat(before, hasTree(planBefore)); + + // Without the shared-carrier condition, the inner join becomes + // LogicalJoin(condition=[true]) and can join an employee to a different + // department. + final RelNode after = + RelDecorrelator.decorrelateQuery(before, builder, RuleSets.ofList(Collections.emptyList()), + RuleSets.ofList(Collections.emptyList())); + final String planAfter = "" + + "LogicalProject(DEPTNO=[$0])\n" + + " LogicalJoin(condition=[=($0, $3)], joinType=[inner])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n" + + " LogicalProject(DEPTNO3=[$0], $f1=[true])\n" + + " LogicalAggregate(group=[{0}])\n" + + " LogicalProject(DEPTNO3=[$12])\n" + + " LogicalJoin(condition=[IS NOT DISTINCT FROM($8, $12)], joinType=[inner])\n" + + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], " + + "HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7], DEPTNO8=[$7])\n" + + " LogicalFilter(condition=[IS NOT NULL($7)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalProject(DEPTNO=[$0], DNAME=[$1], LOC=[$2], DEPTNO3=[$0])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n"; + assertThat(after, hasTree(planAfter)); + } + /** Test case for [CALCITE-5390] * RelDecorrelator throws NullPointerException. */ @Test void testCorrelationLexicalScoping() { diff --git a/core/src/test/resources/sql/sub-query.iq b/core/src/test/resources/sql/sub-query.iq index daa85b96538b..c8719eefe064 100644 --- a/core/src/test/resources/sql/sub-query.iq +++ b/core/src/test/resources/sql/sub-query.iq @@ -9624,6 +9624,36 @@ ON t2.a = foo.a !ok +# [CALCITE-7661] RelDecorrelator loses shared correlation constraint across inner join inputs +# Both join inputs reference d.deptno; decorrelation must keep their correlation carriers equal. +SELECT d.deptno +FROM dept d +WHERE EXISTS ( + SELECT * + FROM ( + SELECT * + FROM emp e + WHERE e.deptno = d.deptno + ) l + JOIN ( + SELECT * + FROM dept d2 + WHERE d2.deptno = d.deptno + ) r + ON TRUE +) +ORDER BY d.deptno; ++--------+ +| DEPTNO | ++--------+ +| 10 | +| 20 | +| 30 | ++--------+ +(3 rows) + +!ok + # [CALCITE-7320] AggregateProjectMergeRule throws AssertionError when Project maps multiple grouping keys to the same field SELECT deptno, (SELECT SUM(cnt) From a05b55cf41f73f4e8b2f15c1bc3c65ef19fcb25e Mon Sep 17 00:00:00 2001 From: zzwqqq Date: Tue, 21 Jul 2026 11:18:07 +0800 Subject: [PATCH 405/562] [CALCITE-7642] RelToSqlConverter may generate duplicate aliases for internal derived relations in case-insensitive dialects --- .../rel/rel2sql/RelToSqlConverter.java | 11 - .../calcite/rel/rel2sql/SqlImplementor.java | 192 +++++++++++++++--- .../rel/rel2sql/RelToSqlConverterTest.java | 96 +++++++++ 3 files changed, 263 insertions(+), 36 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java b/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java index dc91fca54ecb..44eec4feb9f3 100644 --- a/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java +++ b/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java @@ -574,7 +574,6 @@ public Result visit(Filter e) { final Context context = x.qualifiedContext(); if (selectListRequired(context)) { final ImmutableList.Builder selectList = ImmutableList.builder(); - // Fieldnames are unique since they are created by SqlValidatorUtil.deriveJoinRowType() final List uniqueFieldNames = input.getRowType().getFieldNames(); for (int i = 0; i < context.fieldCount; i++) { final SqlNode field = context.field(i); @@ -1584,16 +1583,6 @@ public List createAsFullOperands(RelDataType rowType, SqlNode leftOpera return result; } - @Override public void addSelect(List selectList, SqlNode node, - RelDataType rowType) { - String name = rowType.getFieldNames().get(selectList.size()); - @Nullable String alias = SqlValidatorUtil.alias(node); - if (alias == null || !alias.equals(name)) { - node = as(node, name); - } - selectList.add(node); - } - private void parseCorrelTable(RelNode relNode, Result x) { for (CorrelationId id : relNode.getVariablesSet()) { correlTableMap.put(id, x.qualifiedContext()); diff --git a/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java b/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java index 4b42df2eeca1..b65a797f14fe 100644 --- a/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java +++ b/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java @@ -40,7 +40,9 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.rel.type.RelDataTypeFieldImpl; import org.apache.calcite.rel.type.RelDataTypeSystemImpl; +import org.apache.calcite.rel.type.RelRecordType; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexCorrelVariable; @@ -276,6 +278,11 @@ public abstract Result visitInput(RelNode e, int i, boolean anon, public void addSelect(List selectList, SqlNode node, RelDataType rowType) { String name = rowType.getFieldNames().get(selectList.size()); + addSelect(selectList, node, name); + } + + private void addSelect(List selectList, SqlNode node, + String name) { @Nullable String alias = SqlValidatorUtil.alias(node); if (alias == null || !alias.equals(name)) { node = as(node, name); @@ -283,6 +290,20 @@ public void addSelect(List selectList, SqlNode node, selectList.add(node); } + /** Returns a copy of a row type with different field names. */ + private static RelDataType renameRowTypeFields(RelDataType rowType, + List fieldNames) { + assert fieldNames.size() == rowType.getFieldCount(); + final List fields = new ArrayList<>(); + final List oldFields = rowType.getFieldList(); + for (int i = 0; i < oldFields.size(); i++) { + fields.add( + new RelDataTypeFieldImpl(fieldNames.get(i), i, + oldFields.get(i).getType())); + } + return new RelRecordType(rowType.getStructKind(), fields, rowType.isNullable()); + } + /** Convenience method for creating column and table aliases. * *

      {@code AS(e, "c")} creates "e AS c"; @@ -2061,13 +2082,16 @@ private Builder builder(RelNode rel, Set clauses) { final Set clauses2 = ignoreClauses ? ImmutableSet.of() : clauses; final boolean needNew = needNewSubQuery(rel, this.clauses, clauses2); assert needNew == this.needNew; + final Result input = needNew || node.getKind() != SqlKind.SELECT + ? forDerivedRelation() + : this; SqlSelect select; Expressions.FluentList clauseList = Expressions.list(); if (needNew) { - select = subSelect(); + select = input.subSelect(); } else { - select = asSelect(); - clauseList.addAll(this.clauses); + select = input.asSelect(); + clauseList.addAll(input.clauses); } clauseList.appendAll(clauses); final Context newContext; @@ -2079,19 +2103,19 @@ private Builder builder(RelNode rel, Set clauses) { newContext = selectListContext(selectList, aliasRef); } else { boolean qualified = - !dialect.hasImplicitTableAlias() || aliases.size() > 1; + !dialect.hasImplicitTableAlias() || input.aliases.size() > 1; // basically, we did a subSelect() since needNew is set and neededAlias is not null // now, we need to make sure that we need to update the alias context. // if our aliases map has a single element: , // then we don't need to rewrite the alias but otherwise, it should be updated. if (needNew && neededAlias != null - && (aliases.size() != 1 || !aliases.containsKey(neededAlias))) { + && (input.aliases.size() != 1 || !input.aliases.containsKey(neededAlias))) { newAliases = ImmutableMap.of(neededAlias, rel.getInput(0).getRowType()); newContext = aliasContext(newAliases, qualified); } else { - newContext = aliasContext(aliases, qualified); + newContext = aliasContext(input.aliases, qualified); } if (!dialect.supportGenerateSelectStar(rel.getInput(0))) { // Rename each expanded column to its (unique) row-type field name. @@ -2109,8 +2133,34 @@ private Builder builder(RelNode rel, Set clauses) { select.setSelectList(new SqlNodeList(expandedSelectList, POS)); } } + if (input != this) { + restoreOutputFieldNames(rel, select, newContext); + } return new Builder(rel, clauseList, select, newContext, isAnon(), - needNew && !aliases.containsKey(neededAlias) ? newAliases : aliases); + needNew && !input.aliases.containsKey(neededAlias) ? newAliases : input.aliases); + } + + /** Restores field names after an input was renamed for a derived relation. */ + private void restoreOutputFieldNames(RelNode rel, SqlSelect select, + Context context) { + final RelDataType rowType = rel.getRowType(); + if (!select.getSelectList().equals(SqlNodeList.SINGLETON_STAR) + || context.fieldCount != rowType.getFieldCount()) { + return; + } + final List fieldNames = rowType.getFieldNames(); + // Project internal aliases back to the row type field names. + for (int i = 0; i < context.fieldCount; i++) { + final @Nullable String name = SqlValidatorUtil.alias(context.field(i)); + if (name == null || !name.equals(fieldNames.get(i))) { + final List selectList = new ArrayList<>(); + for (int j = 0; j < context.fieldCount; j++) { + addSelect(selectList, context.field(j), rowType); + } + select.setSelectList(new SqlNodeList(selectList, POS)); + return; + } + } } /** Returns whether a new sub-query is required. */ @@ -2519,18 +2569,104 @@ public Context qualifiedContext() { return aliasContext(aliases, true); } + /** Returns a result for use as a derived relation in the FROM clause of an + * enclosing query. Field names are made unique according to the dialect so + * that the enclosing query can reference them. */ + private Result forDerivedRelation() { + if (neededType == null) { + return this; + } + final List fieldNames = + SqlValidatorUtil.uniquify(neededType.getFieldNames(), + dialect.isCaseSensitive()); + if (fieldNames.equals(neededType.getFieldNames())) { + return this; + } + final RelDataType type = renameRowTypeFields(neededType, fieldNames); + final SqlNode newNode = withOutputFieldNames(fieldNames); + final ImmutableMap.Builder aliasBuilder = + ImmutableMap.builder(); + for (Map.Entry alias : aliases.entrySet()) { + aliasBuilder.put(alias.getKey(), + alias.getValue() == neededType ? type : alias.getValue()); + } + return new Result(newNode, clauses, neededAlias, type, aliasBuilder.build(), anon, + ignoreClauses, expectedClauses, expectedRel, forceExplicitAlias); + } + + /** Returns this result's SQL node with {@code fieldNames} as its output + * field names. + * + * @param fieldNames Output field names + */ + private SqlNode withOutputFieldNames(List fieldNames) { + if (node.getKind() == SqlKind.AS) { + final SqlCall call = (SqlCall) node; + final List operands = call.getOperandList(); + // AS operands have the form [relation, relationAlias, fieldAlias0, ...], + // so field aliases start at index 2. If there is one alias per output field, + // replace the field aliases with fieldNames. + final int fieldAliasStart = 2; + if (operands.size() == fieldNames.size() + fieldAliasStart) { + final List newOperands = new ArrayList<>(operands.size()); + newOperands.add(call.operand(0)); + newOperands.add(call.operand(1)); + for (String fieldName : fieldNames) { + newOperands.add(new SqlIdentifier(fieldName, POS)); + } + return SqlStdOperatorTable.AS.createCall(POS, newOperands); + } + } + return withSelectFieldNames(fieldNames); + } + + /** Returns this result as a SELECT whose items use {@code fieldNames}. + * + * @param fieldNames Names for the SELECT items + */ + private SqlNode withSelectFieldNames(List fieldNames) { + final SqlSelect select = asSelect(); + final SqlNodeList selectList = select.getSelectList(); + assert selectList.equals(SqlNodeList.SINGLETON_STAR) + || selectList.size() == fieldNames.size(); + final List newSelectList = new ArrayList<>(); + final Context context = + aliasContext(aliases, !dialect.hasImplicitTableAlias() || aliases.size() > 1); + if (selectList.equals(SqlNodeList.SINGLETON_STAR)) { + for (int i = 0; i < fieldNames.size(); i++) { + addSelect(newSelectList, context.field(i), fieldNames.get(i)); + } + } else { + for (int i = 0; i < fieldNames.size(); i++) { + SqlNode selectItem = selectList.get(i); + if (selectItem.getKind() == SqlKind.AS) { + selectItem = ((SqlCall) selectItem).operand(0); + } + if (selectItem instanceof SqlIdentifier + && ((SqlIdentifier) selectItem).isSimple() + && aliases.size() > 1) { + selectItem = context.field(i); + } + addSelect(newSelectList, selectItem, fieldNames.get(i)); + } + } + select.setSelectList(new SqlNodeList(newSelectList, POS)); + return select; + } + /** * In join, when the left and right nodes have been generated, * update their alias with 'neededAlias' if not null. */ public Result resetAlias() { - if (neededAlias == null) { - return this; - } else { - return new Result(node, clauses, neededAlias, neededType, - ImmutableMap.of(neededAlias, castNonNull(neededType)), anon, ignoreClauses, - expectedClauses, expectedRel, false); + final Result input = forDerivedRelation(); + if (input.neededAlias == null) { + return input; } + return new Result(input.node, input.clauses, input.neededAlias, input.neededType, + ImmutableMap.of(input.neededAlias, castNonNull(input.neededType)), input.anon, + input.ignoreClauses, input.expectedClauses, input.expectedRel, + input.forceExplicitAlias); } /** @@ -2540,9 +2676,12 @@ public Result resetAlias() { * @param type type of the node associated with the alias */ public Result resetAlias(String alias, RelDataType type) { - return new Result(node, clauses, alias, neededType, - ImmutableMap.of(alias, type), anon, ignoreClauses, - expectedClauses, expectedRel, false); + final Result input = forDerivedRelation(); + final RelDataType aliasType = + input.neededType != null && neededType == type ? input.neededType : type; + return new Result(input.node, input.clauses, alias, input.neededType, + ImmutableMap.of(alias, aliasType), input.anon, input.ignoreClauses, + input.expectedClauses, input.expectedRel, input.forceExplicitAlias); } /** @@ -2554,17 +2693,20 @@ public Result resetAlias(String alias, RelDataType type) { * @return New Result with forced explicit alias */ public Result resetAliasForCorrelation(String alias, RelDataType type) { + final Result input = forDerivedRelation(); + final RelDataType aliasType = + input.neededType != null && neededType == type ? input.neededType : type; return new Result( - node, - clauses, + input.node, + input.clauses, alias, - neededType, - ImmutableMap.of(alias, type), - anon, - ignoreClauses, - expectedClauses, - expectedRel, - true); // Force explicit alias + input.neededType, + ImmutableMap.of(alias, aliasType), + input.anon, + input.ignoreClauses, + input.expectedClauses, + input.expectedRel, + true); } /** Returns a copy of this Result, overriding the value of {@code anon}. */ diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java index 3fe96782339a..a534d8c2066d 100644 --- a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java +++ b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java @@ -9862,6 +9862,102 @@ private void checkLiteral2(String expression, String expected) { } }; + /** Test cases for + * [CALCITE-7642] + * RelToSqlConverter may generate duplicate aliases for internal derived relations + * in case-insensitive dialects. */ + @Test void testCaseInsensitiveRootAliases() { + final SqlDialect mysqlDialect = + new MysqlSqlDialect( + MysqlSqlDialect.DEFAULT_CONTEXT.withCaseSensitive(false)); + relFn(b -> b.values(new String[]{"id", "ID"}, 1, 2).build()) + .dialect(mysqlDialect) + .ok("SELECT 1 AS `id`, 2 AS `ID`"); + } + + @Test void testCaseInsensitiveDerivedValuesAliases() { + final SqlDialect postgresqlDialect = + new PostgresqlSqlDialect( + PostgresqlSqlDialect.DEFAULT_CONTEXT.withCaseSensitive(false)); + relFn(b -> b.values(new String[]{"id", "ID"}, 1, 2) + .filter(b.equals(b.field(1), b.literal(2))) + .build()) + .dialect(postgresqlDialect) + .ok("SELECT \"id\", \"ID0\" AS \"ID\"\n" + + "FROM (VALUES (1, 2)) AS \"t\" (\"id\", \"ID0\")\n" + + "WHERE \"ID0\" = 2"); + } + + @Test void testCaseInsensitiveJoinAliases() { + final SqlDialect mysqlDialect = + new MysqlSqlDialect( + MysqlSqlDialect.DEFAULT_CONTEXT.withCaseSensitive(false)); + relFn(b -> { + b.values(new String[]{"id"}, 1); + b.values(new String[]{"ID"}, 2); + final RelNode left = b.join(JoinRelType.INNER) + .project(b.fields(), ImmutableList.of(), true) + .build(); + return b.push(left) + .values(new String[]{"x"}, 3) + .join(JoinRelType.INNER) + .project(ImmutableList.of(b.field(0), b.field(1)), + ImmutableList.of(), true) + .build(); + }).dialect(mysqlDialect).ok("SELECT `t1`.`id`, `t1`.`ID0` AS `ID`\n" + + "FROM (SELECT `t`.`id`, `t0`.`ID` AS `ID0`\n" + + "FROM (SELECT 1 AS `id`) AS `t`,\n" + + "(SELECT 2 AS `ID`) AS `t0`) AS `t1`,\n" + + "(SELECT 3 AS `x`) AS `t2`"); + } + + @Test void testCaseInsensitiveCorrelateAliases() { + final SqlDialect postgresqlDialect = + new PostgresqlSqlDialect( + PostgresqlSqlDialect.DEFAULT_CONTEXT.withCaseSensitive(false)); + relFn(b -> { + final Holder v = Holder.empty(); + return b.values(new String[]{"id", "ID"}, 1, 2) + .variable(v::set) + .values(new String[]{"x"}, 2) + .filter( + b.equals(b.field("x"), + b.getRexBuilder().makeFieldAccess(v.get(), 1))) + .correlate(JoinRelType.INNER, v.get().id, b.field(2, 0, 1)) + .build(); + }).dialect(postgresqlDialect).ok("SELECT *\n" + + "FROM (VALUES (1, 2)) AS \"$cor0\" (\"id\", \"ID0\"),\n" + + "LATERAL (SELECT *\n" + + "FROM (VALUES (2)) AS \"t0\" (\"x\")\n" + + "WHERE \"x\" = \"$cor0\".\"ID0\") AS \"t1\""); + } + + @Test void testCaseInsensitiveCorrelatedProjectAliases() { + final SqlDialect postgresqlDialect = + new PostgresqlSqlDialect( + PostgresqlSqlDialect.DEFAULT_CONTEXT.withCaseSensitive(false)); + relFn(b -> { + final Holder v = Holder.empty(); + return b.values(new String[]{"id", "ID"}, 1, 2) + .variable(v::set) + .project( + ImmutableList.of( + b.field(0), + b.scalarQuery(unused -> + b.values(new String[]{"x"}, 2) + .filter( + b.equals(b.field("x"), + b.getRexBuilder().makeFieldAccess(v.get(), 1))) + .project(b.field("x")) + .build())), + ImmutableList.of(), false, ImmutableList.of(v.get().id)) + .build(); + }).dialect(postgresqlDialect).ok("SELECT \"id\", (SELECT *\n" + + "FROM (VALUES (2)) AS \"t0\" (\"x\")\n" + + "WHERE \"x\" = \"t\".\"ID0\") AS \"$f1\"\n" + + "FROM (VALUES (1, 2)) AS \"t\" (\"id\", \"ID0\")"); + } + /** Test case for * [CALCITE-7483] * RelToSqlConverter generates SELECT * despite supportGenerateSelectStar. From 5042482c9906be4a3e1c6fc0246b407bcaf76f37 Mon Sep 17 00:00:00 2001 From: bibi samina Date: Thu, 16 Jul 2026 15:26:40 +0530 Subject: [PATCH 406/562] Firebolt dialect should quote identifiers that need quoting --- .../sql/dialect/FireboltSqlDialect.java | 2 +- .../rel/rel2sql/RelToSqlConverterTest.java | 32 +++++++++---------- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql/dialect/FireboltSqlDialect.java b/core/src/main/java/org/apache/calcite/sql/dialect/FireboltSqlDialect.java index 49b74d07edaa..0b2d9e0e3490 100644 --- a/core/src/main/java/org/apache/calcite/sql/dialect/FireboltSqlDialect.java +++ b/core/src/main/java/org/apache/calcite/sql/dialect/FireboltSqlDialect.java @@ -122,7 +122,7 @@ public FireboltSqlDialect(Context context) { } @Override protected boolean identifierNeedsQuote(String val) { - return IDENTIFIER_REGEX.matcher(val).matches() + return !IDENTIFIER_REGEX.matcher(val).matches() || RESERVED_KEYWORDS.contains(val.toUpperCase(Locale.ROOT)); } diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java index a534d8c2066d..affb7093f03e 100644 --- a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java +++ b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java @@ -463,13 +463,7 @@ private static String toSql(RelNode root, SqlDialect dialect, + "FROM foodmart.product\n" + "WHERE product_id > 0\n" + "GROUP BY product_id"; - final String expectedFirebolt = "SELECT" - + " SUM(CASE WHEN \"net_weight\" > 0E0 IS TRUE" - + " THEN \"shelf_width\" ELSE NULL END), " - + "SUM(\"shelf_width\")\n" - + "FROM \"foodmart\".\"product\"\n" - + "WHERE \"product_id\" > 0\n" - + "GROUP BY \"product_id\""; + final String expectedFirebolt = expectedBigQuery; final String expectedMysql = "SELECT" + " SUM(CASE WHEN `net_weight` > 0E0 IS TRUE" + " THEN `shelf_width` ELSE NULL END), SUM(`shelf_width`)\n" @@ -4043,12 +4037,16 @@ private SqlDialect nonOrdinalDialect() { + " 4 AS \"fo$ur\", 5 AS \"ignore\", 6 AS \"si`x\"\n" + "FROM foodmart.days) AS t\n" + "WHERE one < tWo AND THREE < \"fo$ur\""; + // Firebolt quotes like Exasol, except that IGNORE is not reserved + final String expectedFirebolt = + expectedExasol.replace("\"ignore\"", "ignore"); sql(query) .withBigQuery().ok(expectedBigQuery) .withMysql().ok(expectedMysql) .withOracle().ok(expectedOracle) .withPostgresql().ok(expectedPostgresql) - .withExasol().ok(expectedExasol); + .withExasol().ok(expectedExasol) + .withFirebolt().ok(expectedFirebolt); } @Test void testModFunctionForHive() { @@ -6210,7 +6208,7 @@ private void checkLiteral2(String expression, String expected) { String expectedPresto = "SELECT DATE_TRUNC('MINUTE', \"hire_date\")\n" + "FROM \"foodmart\".\"employee\""; String expectedTrino = expectedPresto; - String expectedFirebolt = expectedPostgresql; + String expectedFirebolt = expectedPostgresql.replace("\"", ""); String expectedStarRocks = "SELECT DATE_TRUNC('MINUTE', `hire_date`)\n" + "FROM `foodmart`.`employee`"; String expectedDoris = "SELECT DATE_TRUNC(`hire_date`, 'MINUTE')\n" @@ -6510,16 +6508,16 @@ private void checkLiteral2(String expression, String expected) { final String sql0 = "select * from \"employee\" where \"hire_date\" - " + "INTERVAL '19800' SECOND(5) > TIMESTAMP '2005-10-17 00:00:00' "; final String expect0 = "SELECT *\n" - + "FROM \"foodmart\".\"employee\"\n" - + "WHERE (\"hire_date\" - INTERVAL '19800 SECOND ')" + + "FROM foodmart.employee\n" + + "WHERE (hire_date - INTERVAL '19800 SECOND ')" + " > TIMESTAMP '2005-10-17 00:00:00'"; sql(sql0).withFirebolt().ok(expect0); final String sql1 = "select * from \"employee\" where \"hire_date\" + " + "INTERVAL '10' HOUR > TIMESTAMP '2005-10-17 00:00:00' "; final String expect1 = "SELECT *\n" - + "FROM \"foodmart\".\"employee\"\n" - + "WHERE (\"hire_date\" + INTERVAL '10 HOUR ')" + + "FROM foodmart.employee\n" + + "WHERE (hire_date + INTERVAL '10 HOUR ')" + " > TIMESTAMP '2005-10-17 00:00:00'"; sql(sql1).withFirebolt().ok(expect1); @@ -6617,7 +6615,7 @@ private void checkLiteral2(String expression, String expected) { + " DATE_FORMAT(`hire_date`, '%Y-%m-%d %H:%i:00')\n" + "FROM `foodmart`.`employee`\n" + "GROUP BY DATE_FORMAT(`hire_date`, '%Y-%m-%d %H:%i:00')"; - final String expectedFirebolt = expectedPostgresql; + final String expectedFirebolt = expectedPostgresql.replace("\"", ""); sql(query) .withClickHouse().ok(expectedClickHouse) .withFirebolt().ok(expectedFirebolt) @@ -6644,7 +6642,7 @@ private void checkLiteral2(String expression, String expected) { + "FROM \"foodmart\".\"product\""; final String expectedSnowflake = expectedPostgresql; final String expectedRedshift = expectedPostgresql; - final String expectedFirebolt = expectedPresto; + final String expectedFirebolt = expectedPresto.replace("\"", ""); final String expectedMysql = "SELECT SUBSTRING(`brand_name`, 2)\n" + "FROM `foodmart`.`product`"; final String expectedStarRocks = "SELECT SUBSTRING(`brand_name`, 2)\n" @@ -6684,7 +6682,7 @@ private void checkLiteral2(String expression, String expected) { + "FROM \"foodmart\".\"product\""; final String expectedSnowflake = expectedPostgresql; final String expectedRedshift = expectedPostgresql; - final String expectedFirebolt = expectedPresto; + final String expectedFirebolt = expectedPresto.replace("\"", ""); final String expectedMysql = "SELECT SUBSTRING(`brand_name`, 2, 3)\n" + "FROM `foodmart`.`product`"; final String expectedMssql = "SELECT SUBSTRING([brand_name], 2, 3)\n" @@ -8098,7 +8096,7 @@ private void checkLiteral2(String expression, String expected) { + "FROM (SELECT 1 AS a, 'x' AS b\n" + "UNION ALL\n" + "SELECT 2 AS a, 'yy' AS b)"; - final String expectedFirebolt = expectedPostgresql; + final String expectedFirebolt = expectedPostgresql.replace("\"", ""); final String expectedSnowflake = expectedPostgresql; final String expectedRedshift = "SELECT \"a\"\n" + "FROM (SELECT 1 AS \"a\", 'x ' AS \"b\"\n" From 575be035ada85db3f8eb3dcf8f8c1fdc396a21dd Mon Sep 17 00:00:00 2001 From: xuzifu666 <1206332514@qq.com> Date: Thu, 16 Jul 2026 10:06:30 +0800 Subject: [PATCH 407/562] [CALCITE-7657] Apply the absorption law to simplify boolean expressions --- .../org/apache/calcite/rex/RexSimplify.java | 49 +++++++++++++ .../apache/calcite/rex/RexProgramTest.java | 48 +++++++++++-- .../apache/calcite/test/JdbcAdapterTest.java | 5 +- ...terializedViewSubstitutionVisitorTest.java | 2 +- .../apache/calcite/test/RelBuilderTest.java | 70 +++++++++++++++++++ core/src/test/resources/sql/sub-query.iq | 12 ++-- 6 files changed, 171 insertions(+), 15 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rex/RexSimplify.java b/core/src/main/java/org/apache/calcite/rex/RexSimplify.java index 58979c0f5c60..b275c8b6d97b 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexSimplify.java +++ b/core/src/main/java/org/apache/calcite/rex/RexSimplify.java @@ -93,6 +93,9 @@ public class RexSimplify { private static final Strong STRONG = new Strong(); + /** Maximum number of terms for which to apply the absorption law. */ + private static final int MAX_TERMS_FOR_ABSORPTION = 20; + /** * Creates a RexSimplify. * @@ -1843,6 +1846,9 @@ RexNode simplifyAnd2(List terms, List notTerms) { SqlStdOperatorTable.IS_NULL, notSatisfiableNullable), UNKNOWN)); } } + // Absorption law: a AND (a OR b) => a + absorb(terms, SqlKind.OR); + // Add the NOT disjunctions back in. for (RexNode notDisjunction : notTerms) { terms.add(simplify(not(notDisjunction), UNKNOWN)); @@ -2087,6 +2093,9 @@ private > RexNode simplifyAnd2ForUnknownAsFalse( if (!Collections.disjoint(nullOperands, strongOperands)) { return rexBuilder.makeLiteral(false); } + // Absorption law: a AND (a OR b) => a + absorb(terms, SqlKind.OR); + // Remove not necessary IS NOT NULL expressions. // Example. IS NOT NULL(x) AND x < 5 : x < 5 for (RexNode operand : notNullOperands) { @@ -2367,9 +2376,49 @@ private RexNode simplifyOrs(List terms, RexUnknownAs unknownAs) { break; } } + + // Absorption law: a OR (a AND b) => a + absorb(terms, SqlKind.AND); + return RexUtil.composeDisjunction(rexBuilder, terms); } + /** + * Applies the absorption law to a list of terms, removing any composite term + * that is absorbed by a sibling term. + * + *

      When {@code compositeKind} is {@link SqlKind#OR}, removes any + * {@code (a OR b)} term whose disjunctions contain a sibling {@code a}, so + * {@code a AND (a OR b) => a}. When it is {@link SqlKind#AND}, removes any + * {@code (a AND b)} term whose conjunctions contain a sibling {@code a}, so + * {@code a OR (a AND b) => a}. + * + *

      The absorbing sibling {@code a} must be deterministic; otherwise its two + * occurrences might evaluate differently and the rewrite would not be + * equivalence-preserving. + */ + private static void absorb(List terms, SqlKind compositeKind) { + if (terms.size() > MAX_TERMS_FOR_ABSORPTION) { + return; + } + for (int i = 0; i < terms.size(); i++) { + final RexNode term = terms.get(i); + if (term.getKind() == compositeKind) { + final List components = compositeKind == SqlKind.OR + ? RelOptUtil.disjunctions(term) + : RelOptUtil.conjunctions(term); + for (RexNode other : terms) { + if (other != term && components.contains(other) + && RexUtil.isDeterministic(other)) { + terms.remove(i); + i--; + break; + } + } + } + } + } + private Pair evaluate(RexNode e, Map map) { Comparable c = null; RuntimeException ex = null; diff --git a/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java b/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java index bf506ceb121a..5f8b8edfb1db 100644 --- a/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java +++ b/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java @@ -513,6 +513,45 @@ private RexProgramBuilder createProg(int variant) { "false"); } + /** Test case for + * [CALCITE-7657] + * Apply the absorption law to simplify boolean expressions. */ + @Test void testAbsorptionLaw() { + // AND absorption: a AND (a OR b) => a + checkSimplify(and(vBool(), or(vBool(), vBool(1))), "?0.bool0"); + checkSimplify(and(or(vBool(), vBool(1)), vBool()), "?0.bool0"); + + // OR absorption: a OR (a AND b) => a + checkSimplify(or(vBool(), and(vBool(), vBool(1))), "?0.bool0"); + checkSimplify(or(and(vBool(), vBool(1)), vBool()), "?0.bool0"); + + // with not-null booleans + checkSimplify(and(vBoolNotNull(), or(vBoolNotNull(), vBoolNotNull(1))), "?0.notNullBool0"); + checkSimplify(or(vBoolNotNull(), and(vBoolNotNull(), vBoolNotNull(1))), "?0.notNullBool0"); + + // filter mode (unknownAsFalse) + checkSimplifyFilter(and(vBool(), or(vBool(), vBool(1))), "?0.bool0"); + checkSimplifyFilter(or(vBool(), and(vBool(), vBool(1))), "?0.bool0"); + } + + @Test void testAbsorptionLawWithNonDeterministic() { + // a is a non-deterministic boolean ("NDC()") + final SqlOperator ndc = getNoDeterministicOperator(); + final RexNode a = rexBuilder.makeCall(ndc); + final RexNode b = gt(vInt(1), literal(1)); + + // a AND (a OR b) must NOT be simplified to a + checkSimplifyUnchanged(and(a, or(a, b))); + // a OR (a AND b) must NOT be simplified to a + checkSimplifyUnchanged(or(a, and(a, b))); + + // Sanity check: when a is deterministic, absorption does apply. + final SqlOperator dc = getDeterministicOperator(); + final RexNode da = rexBuilder.makeCall(dc); + checkSimplify(and(da, or(da, b)), "DC()"); + checkSimplify(or(da, and(da, b)), "DC()"); + } + @Disabled("CALCITE-3457: AssertionError in RexSimplify.validateStrongPolicy") @Test void reproducerFor3457() { // Identified with RexProgramFuzzyTest#testFuzzy, seed=4887662474363391810L @@ -3064,10 +3103,9 @@ trueLiteral, literal(1), // ==> // "A IS NOT NULL" SqlOperator dc = getDeterministicOperator(); - checkSimplify2( + checkSimplify( and(or(isNotNull(rexBuilder.makeCall(dc)), gt(vInt(2), literal(2))), isNotNull(rexBuilder.makeCall(dc))), - "AND(OR(IS NOT NULL(DC()), >(?0.int2, 2)), IS NOT NULL(DC()))", "IS NOT NULL(DC())"); } @@ -3962,7 +4000,7 @@ private static String getString(Map map) { // -> "x = x AND y < y" (treating unknown as unknown) // -> false (treating unknown as false) checkSimplify3(and(eq(vInt(1), vInt(1)), not(ge(vInt(2), vInt(2)))), - "AND(OR(null, IS NOT NULL(?0.int1)), null, IS NULL(?0.int2))", + "AND(null, IS NULL(?0.int2))", "false", "IS NULL(?0.int2)"); @@ -3970,7 +4008,7 @@ private static String getString(Map map) { // -> "OR(x <> x, y >= y)" (treating unknown as unknown) // -> "y IS NOT NULL" (treating unknown as false) checkSimplify3(not(and(eq(vInt(1), vInt(1)), not(ge(vInt(2), vInt(2))))), - "OR(AND(null, IS NULL(?0.int1)), null, IS NOT NULL(?0.int2))", + "OR(null, IS NOT NULL(?0.int2))", "IS NOT NULL(?0.int2)", "true"); } @@ -4028,7 +4066,7 @@ private static String getString(Map map) { // -> "AND(x <> x, y >= y)" (treating unknown as unknown) // -> "FALSE" (treating unknown as false) checkSimplify3(not(or(eq(vInt(1), vInt(1)), not(ge(vInt(2), vInt(2))))), - "AND(null, IS NULL(?0.int1), OR(null, IS NOT NULL(?0.int2)))", + "AND(null, IS NULL(?0.int1))", "false", "IS NULL(?0.int1)"); } diff --git a/core/src/test/java/org/apache/calcite/test/JdbcAdapterTest.java b/core/src/test/java/org/apache/calcite/test/JdbcAdapterTest.java index 5ecf63c5ba72..fbf502169d45 100644 --- a/core/src/test/java/org/apache/calcite/test/JdbcAdapterTest.java +++ b/core/src/test/java/org/apache/calcite/test/JdbcAdapterTest.java @@ -322,9 +322,8 @@ class JdbcAdapterTest { + " JdbcProject($f2=[OR(IS NOT NULL($7), IS NOT NULL($1))])\n" + " JdbcTableScan(table=[[SCOTT, EMP]])\n" + " JdbcToEnumerableConverter\n" - + " JdbcAggregate(group=[{0, 1}], i=[LITERAL_AGG(true)], em=[MAX($2)])\n" - + " JdbcProject(DEPTNO=[$7], ENAME=[CAST($1):VARCHAR(14)]," - + " $f2=[AND(IS NOT NULL($7), IS NOT NULL($1))])\n" + + " JdbcAggregate(group=[{0, 1}], i=[LITERAL_AGG(true)])\n" + + " JdbcProject(DEPTNO=[$7], ENAME=[CAST($1):VARCHAR(14)])\n" + " JdbcFilter(condition=[OR(IS NOT NULL($7), IS NOT NULL($1))])\n" + " JdbcTableScan(table=[[SCOTT, EMP]])\n\n"); } diff --git a/core/src/test/java/org/apache/calcite/test/MaterializedViewSubstitutionVisitorTest.java b/core/src/test/java/org/apache/calcite/test/MaterializedViewSubstitutionVisitorTest.java index a19cd683d21b..285c1cf9a3b2 100644 --- a/core/src/test/java/org/apache/calcite/test/MaterializedViewSubstitutionVisitorTest.java +++ b/core/src/test/java/org/apache/calcite/test/MaterializedViewSubstitutionVisitorTest.java @@ -1653,7 +1653,7 @@ protected final MaterializedViewFixture sql(String materialize, SqlStdOperatorTable.NOT, i4)))); f.checkSatisfiable(e8, - "AND(=($0, 0), $2, $3, OR(NOT($2), NOT($3), NOT($4)), NOT($4))"); + "AND(=($0, 0), $2, $3, NOT($4))"); } @Test void testSplitFilter() { diff --git a/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java b/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java index 30aa2ff77fdf..7ad9a4d733fc 100644 --- a/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java @@ -752,6 +752,76 @@ private void checkSimplify(UnaryOperator transform, assertThat(f.apply(createBuilder()), hasTree(expected)); } + /** Test case for + * [CALCITE-7657] + * Apply the absorption law to simplify boolean expressions. */ + @Test void testFilterAndAbsorptionLaw() { + // Equivalent SQL: + // SELECT * + // FROM emp + // WHERE deptno = 10 AND (deptno = 10 OR sal > 100) + // Should be simplified to: + // SELECT * + // FROM emp + // WHERE deptno = 10 + final Function f = b -> + b.scan("EMP") + .filter( + b.and( + b.equals(b.field("DEPTNO"), b.literal(10)), + b.or( + b.equals(b.field("DEPTNO"), b.literal(10)), + b.greaterThan(b.field("SAL"), b.literal(100))))) + .build(); + + final String expected = "LogicalFilter(condition=[=($7, 10)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + assertThat(f.apply(createBuilder()), hasTree(expected)); + } + + @Test void testFilterOrAbsorptionLaw() { + // Equivalent SQL: + // SELECT * + // FROM emp + // WHERE deptno = 10 OR (deptno = 10 AND sal > 100) + // Should be simplified to: + // SELECT * + // FROM emp + // WHERE deptno = 10 + final Function f = b -> + b.scan("EMP") + .filter( + b.or( + b.equals(b.field("DEPTNO"), b.literal(10)), + b.and( + b.equals(b.field("DEPTNO"), b.literal(10)), + b.greaterThan(b.field("SAL"), b.literal(100))))) + .build(); + + final String expected = "LogicalFilter(condition=[=($7, 10)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + assertThat(f.apply(createBuilder()), hasTree(expected)); + } + + @Test void testFilterAbsorptionLawWithNonDeterministic() { + final Function f = b -> { + final RexNode rand = + b.greaterThan( + b.call(SqlStdOperatorTable.RAND), b.literal(0.5)); + return b.scan("EMP") + .filter( + b.and(rand, + b.or(rand, + b.greaterThan(b.field("SAL"), b.literal(100))))) + .build(); + }; + + final String expected = "LogicalFilter(condition=[AND(>(RAND(), 0.5E0)," + + " OR(>(RAND(), 0.5E0), >($5, 100)))])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + assertThat(f.apply(createBuilder()), hasTree(expected)); + } + @Test void testBadFieldName() { final RelBuilder builder = RelBuilder.create(config().build()); try { diff --git a/core/src/test/resources/sql/sub-query.iq b/core/src/test/resources/sql/sub-query.iq index c8719eefe064..847cdb98d949 100644 --- a/core/src/test/resources/sql/sub-query.iq +++ b/core/src/test/resources/sql/sub-query.iq @@ -4327,7 +4327,7 @@ select * from "scott".emp where (empno, deptno) not in ((1, 2), (3, null)); !ok !if (use_old_decorr) { -EnumerableCalc(expr#0..14=[{inputs}], expr#15=[0], expr#16=[=($t8, $t15)], expr#17=[IS NULL($t7)], expr#18=[IS NOT NULL($t13)], expr#19=[AND($t14, $t18)], expr#20=[<($t9, $t8)], expr#21=[OR($t17, $t19, $t18, $t20)], expr#22=[IS NOT TRUE($t21)], expr#23=[OR($t16, $t22)], proj#0..7=[{exprs}], $condition=[$t23]) +EnumerableCalc(expr#0..13=[{inputs}], expr#14=[0], expr#15=[=($t8, $t14)], expr#16=[IS NULL($t13)], expr#17=[>=($t9, $t8)], expr#18=[IS NOT NULL($t7)], expr#19=[AND($t16, $t17, $t18)], expr#20=[OR($t15, $t19)], proj#0..7=[{exprs}], $condition=[$t20]) EnumerableMergeJoin(condition=[AND(=($10, $11), OR(IS NULL($12), =(CAST($7):INTEGER, $12)))], joinType=[left]) EnumerableSort(sort0=[$10], dir0=[ASC]) EnumerableCalc(expr#0..9=[{inputs}], expr#10=[CAST($t0):INTEGER NOT NULL], proj#0..10=[{exprs}]) @@ -4336,7 +4336,7 @@ EnumerableCalc(expr#0..14=[{inputs}], expr#15=[0], expr#16=[=($t8, $t15)], expr# EnumerableAggregate(group=[{}], c=[COUNT()], ck=[COUNT() FILTER $0]) EnumerableValues(tuples=[[{ true }, { true }]]) EnumerableSort(sort0=[$0], dir0=[ASC]) - EnumerableCalc(expr#0..1=[{inputs}], expr#2=[true], expr#3=[IS NOT NULL($t1)], proj#0..3=[{exprs}]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[true], proj#0..2=[{exprs}]) EnumerableValues(tuples=[[{ 3, null }, { 1, 2 }]]) !plan !} @@ -4351,14 +4351,14 @@ select * from "scott".emp where (mgr, deptno) not in ((1, 2), (3, null), (cast(n !ok !if (use_old_decorr) { -EnumerableCalc(expr#0..13=[{inputs}], expr#14=[0], expr#15=[=($t8, $t14)], expr#16=[IS NULL($t3)], expr#17=[IS NULL($t7)], expr#18=[IS NOT NULL($t12)], expr#19=[AND($t13, $t18)], expr#20=[<($t9, $t8)], expr#21=[OR($t16, $t17, $t19, $t18, $t20)], expr#22=[IS NOT TRUE($t21)], expr#23=[OR($t15, $t22)], proj#0..7=[{exprs}], $condition=[$t23]) +EnumerableCalc(expr#0..12=[{inputs}], expr#13=[0], expr#14=[=($t8, $t13)], expr#15=[IS NULL($t12)], expr#16=[>=($t9, $t8)], expr#17=[IS NOT NULL($t3)], expr#18=[IS NOT NULL($t7)], expr#19=[AND($t15, $t16, $t17, $t18)], expr#20=[OR($t14, $t19)], proj#0..7=[{exprs}], $condition=[$t20]) EnumerableNestedLoopJoin(condition=[AND(OR(IS NULL($10), =(CAST($3):INTEGER, $10)), OR(IS NULL($11), =(CAST($7):INTEGER, $11)))], joinType=[left]) EnumerableNestedLoopJoin(condition=[true], joinType=[inner]) EnumerableTableScan(table=[[scott, EMP]]) EnumerableAggregate(group=[{}], c=[COUNT()], ck=[COUNT() FILTER $0]) EnumerableCalc(expr#0..1=[{inputs}], expr#2=[IS NOT NULL($t0)], expr#3=[IS NOT NULL($t1)], expr#4=[OR($t2, $t3)], $f2=[$t4]) EnumerableValues(tuples=[[{ 3, null }, { null, null }, { 1, 2 }]]) - EnumerableCalc(expr#0..1=[{inputs}], expr#2=[true], expr#3=[IS NOT NULL($t0)], expr#4=[IS NOT NULL($t1)], expr#5=[AND($t3, $t4)], expr#6=[OR($t3, $t4)], proj#0..2=[{exprs}], $f20=[$t5], $condition=[$t6]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[true], expr#3=[IS NOT NULL($t0)], expr#4=[IS NOT NULL($t1)], expr#5=[OR($t3, $t4)], proj#0..2=[{exprs}], $condition=[$t5]) EnumerableValues(tuples=[[{ 3, null }, { null, null }, { 1, 2 }]]) !plan !} @@ -4393,7 +4393,7 @@ select * from "scott".emp where (empno, deptno) not in ((7369, 20), (7499, 30)); !ok !if (use_old_decorr) { -EnumerableCalc(expr#0..15=[{inputs}], expr#16=[0], expr#17=[=($t8, $t16)], expr#18=[IS NULL($t7)], expr#19=[IS NOT NULL($t14)], expr#20=[AND($t15, $t19)], expr#21=[<($t9, $t8)], expr#22=[OR($t18, $t20, $t19, $t21)], expr#23=[IS NOT TRUE($t22)], expr#24=[OR($t17, $t23)], proj#0..7=[{exprs}], $condition=[$t24]) +EnumerableCalc(expr#0..14=[{inputs}], expr#15=[0], expr#16=[=($t8, $t15)], expr#17=[IS NULL($t14)], expr#18=[>=($t9, $t8)], expr#19=[IS NOT NULL($t7)], expr#20=[AND($t17, $t18, $t19)], expr#21=[OR($t16, $t20)], proj#0..7=[{exprs}], $condition=[$t21]) EnumerableMergeJoin(condition=[AND(=($10, $12), =($11, $13))], joinType=[left]) EnumerableSort(sort0=[$10], sort1=[$11], dir0=[ASC], dir1=[ASC]) EnumerableCalc(expr#0..9=[{inputs}], expr#10=[CAST($t0):INTEGER NOT NULL], expr#11=[CAST($t7):INTEGER], proj#0..11=[{exprs}]) @@ -4403,7 +4403,7 @@ EnumerableCalc(expr#0..15=[{inputs}], expr#16=[0], expr#17=[=($t8, $t16)], expr# EnumerableCalc(expr#0..1=[{inputs}], expr#2=[true], $f2=[$t2]) EnumerableValues(tuples=[[{ 7369, 20 }, { 7499, 30 }]]) EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) - EnumerableCalc(expr#0..1=[{inputs}], expr#2=[true], proj#0..2=[{exprs}], $f20=[$t2]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[true], proj#0..2=[{exprs}]) EnumerableValues(tuples=[[{ 7369, 20 }, { 7499, 30 }]]) !plan !} From d6b4e3202f5e8c67f3e01f7e358518da53d851c4 Mon Sep 17 00:00:00 2001 From: Niels Pardon Date: Fri, 24 Jul 2026 07:59:07 +0200 Subject: [PATCH 408/562] [CALCITE-7639] Support bitwise right shift (>>) operator and RIGHTSHIFT function Mirrors the left-shift work in CALCITE-7109 to close a gap in the umbrella issue CALCITE-5087. Adds: * `>>` (SqlStdOperatorTable.BIT_RIGHT_SHIFT), a signed/arithmetic right shift (Java `>>`), symmetric to `<<` (precedence 32, left-assoc, ReturnTypes.ARG0_NULLABLE, InferTypes.FIRST_KNOWN). * `RIGHTSHIFT(x, n)` scalar function, mirroring `LEFTSHIFT`. * SqlFunctions.rightShift(...) runtime overloads (int, long, and the joou unsigned types), plus BuiltInMethod.RIGHT_SHIFT and the RexImpTable registrations for both operator and function. Operands are limited to integer and unsigned numeric types. Unlike `<<`, binary (BINARY/VARBINARY) right shift is intentionally rejected at validation until the endianness of bitwise shifts on binary is settled; that follow-up is tracked in CALCITE-7651. A greedy `>>` token cannot be added: the lexer would also match the two `>` that close nested angle-bracket types (e.g. MAP>), breaking type parsing. `>>` is therefore recognized in expression context as two adjacent `>` tokens via LOOKAHEAD(2) in BinaryRowOperator, leaving type parsing unchanged. Because there is no dedicated token, the SQL advisor advertises `>` rather than `>>`, so SqlAdvisorTest is not modified. Scope is limited to `>>` (arithmetic). The logical/fill-zero `>>>` (RIGHT_SHIFT_FILL_ZERO) remains a possible follow-up. Tests: SqlOperatorTest (operator + function forms), SqlFunctionsTest, operator.iq, and the operator-precedence dump in SqlValidatorTest; docs in site/_docs/reference.md. --- core/src/main/codegen/templates/Parser.jj | 29 +++ .../adapter/enumerable/RexImpTable.java | 14 ++ .../apache/calcite/runtime/SqlFunctions.java | 135 +++++++++--- .../calcite/sql/fun/SqlStdOperatorTable.java | 57 ++++- .../calcite/sql/parser/SqlParserPos.java | 14 ++ .../apache/calcite/util/BuiltInMethod.java | 3 +- .../calcite/sql/parser/SqlParserPosTest.java | 57 +++++ .../apache/calcite/test/SqlFunctionsTest.java | 25 ++- .../apache/calcite/test/SqlValidatorTest.java | 1 + core/src/test/resources/sql/operator.iq | 31 +++ site/_docs/reference.md | 3 +- .../calcite/sql/parser/SqlParserTest.java | 19 ++ .../apache/calcite/test/SqlOperatorTest.java | 197 ++++++++++++++++++ 13 files changed, 544 insertions(+), 41 deletions(-) create mode 100644 core/src/test/java/org/apache/calcite/sql/parser/SqlParserPosTest.java diff --git a/core/src/main/codegen/templates/Parser.jj b/core/src/main/codegen/templates/Parser.jj index d3c115974180..ce69124c4b5c 100644 --- a/core/src/main/codegen/templates/Parser.jj +++ b/core/src/main/codegen/templates/Parser.jj @@ -279,6 +279,24 @@ public class ${parser.class} extends SqlAbstractParserImpl Span.of(table, extendList).pos(), table, extendList); } + /** + * Returns the parser position of the given token. + * + *

      Declared as a plain method (rather than JAVACODE) so it can be called + * from generated lookahead code, which does not declare + * {@code throws ParseException}. + * + * @param token token whose position to return + * @return parser position spanning the given token + */ + private static SqlParserPos pos(Token token) { + return new SqlParserPos( + token.beginLine, + token.beginColumn, + token.endLine, + token.endColumn); + } + /** Adds a warning that a token such as "HOURS" was used, * whereas the SQL standard only allows "HOUR". * @@ -8487,6 +8505,17 @@ SqlBinaryOperator BinaryRowOperator() : // is handled as a special case { return SqlStdOperatorTable.EQUALS; } | { return SqlStdOperatorTable.BIT_LEFT_SHIFT; } + // The right shift operator ">>" is matched as two ">" tokens rather than a + // single ">>" token. A greedy ">>" token would be produced by the lexer even + // when the two ">" characters close nested angle-bracket types (e.g. + // MAP>), breaking type parsing. Matching two ">" tokens here + // keeps right shift confined to expression context. The semantic lookahead + // requires the two ">" to be immediately adjacent (no intervening whitespace), + // so "a > > b" is not treated as a right shift; otherwise we fall through to + // the single ">" (greater-than) alternative below. +| LOOKAHEAD({ getToken(1).kind == GT && getToken(2).kind == GT + && pos(getToken(1)).endsImmediatelyBefore(pos(getToken(2))) }) + { return SqlStdOperatorTable.BIT_RIGHT_SHIFT; } | { return SqlStdOperatorTable.GREATER_THAN; } | { return SqlStdOperatorTable.LESS_THAN; } | { return SqlStdOperatorTable.LESS_THAN_OR_EQUAL; } diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java index bf5de12e742f..2ad2be289378 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java @@ -377,6 +377,7 @@ import static org.apache.calcite.sql.fun.SqlStdOperatorTable.BIT_AND; import static org.apache.calcite.sql.fun.SqlStdOperatorTable.BIT_LEFT_SHIFT; import static org.apache.calcite.sql.fun.SqlStdOperatorTable.BIT_OR; +import static org.apache.calcite.sql.fun.SqlStdOperatorTable.BIT_RIGHT_SHIFT; import static org.apache.calcite.sql.fun.SqlStdOperatorTable.BIT_XOR; import static org.apache.calcite.sql.fun.SqlStdOperatorTable.CARDINALITY; import static org.apache.calcite.sql.fun.SqlStdOperatorTable.CAST; @@ -514,6 +515,7 @@ import static org.apache.calcite.sql.fun.SqlStdOperatorTable.REGR_COUNT; import static org.apache.calcite.sql.fun.SqlStdOperatorTable.REINTERPRET; import static org.apache.calcite.sql.fun.SqlStdOperatorTable.REPLACE; +import static org.apache.calcite.sql.fun.SqlStdOperatorTable.RIGHTSHIFT; import static org.apache.calcite.sql.fun.SqlStdOperatorTable.ROUND; import static org.apache.calcite.sql.fun.SqlStdOperatorTable.ROW; import static org.apache.calcite.sql.fun.SqlStdOperatorTable.ROW_NUMBER; @@ -922,6 +924,18 @@ void populate1() { // (e.g., x << y) defineMethod(BIT_LEFT_SHIFT, BuiltInMethod.LEFT_SHIFT.method, NullPolicy.STRICT); + // Right shift operations: shift bits to the right by specified amount. + // Supports integer and unsigned integer data types. Binary right shift is + // intentionally not supported; see [CALCITE-7651]. + // Shift amount is normalized using modulo arithmetic based on data type bit width. + + // RIGHTSHIFT: Function call syntax for bitwise right shift operation (e.g., RIGHTSHIFT(x, y)) + defineMethod(RIGHTSHIFT, BuiltInMethod.RIGHT_SHIFT.method, NullPolicy.STRICT); + + // BIT_RIGHT_SHIFT: Operator syntax for bitwise right shift in SQL expressions + // (e.g., x >> y) + defineMethod(BIT_RIGHT_SHIFT, BuiltInMethod.RIGHT_SHIFT.method, NullPolicy.STRICT); + define(SAFE_ADD, new SafeArithmeticImplementor(BuiltInMethod.SAFE_ADD.method)); define(SAFE_DIVIDE, diff --git a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java index c41ab84d71d5..fd4101b5a42b 100644 --- a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java +++ b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java @@ -3789,6 +3789,21 @@ private static ByteString binaryOperator( return new ByteString(result); } + /** + * Returns {@code x} modulo {@code m}, normalized to the range {@code [0, m)} + * (unlike {@code %}, the result is never negative). Used to normalize a shift + * amount to the bit width of the value being shifted. + * + * @param x the value (typically a shift amount, which may be negative) + * @param m the modulus, which must be positive (typically a bit width) + * @return {@code x} modulo {@code m}, in the range {@code [0, m)} + */ + private static int positiveModulo(long x, int m) { + // Math.floorMod(long, int) is only available since JDK 9, so widen to + // Math.floorMod(long, long) and narrow the result (always in [0, m)) to int. + return (int) Math.floorMod(x, (long) m); + } + /** * Performs PostgresSQL-style bitwise shift on a 32-bit integer. * @@ -3796,8 +3811,8 @@ private static ByteString binaryOperator( * @param y the shift amount (positive: left shift, negative: right shift) * @return the shifted integer */ - public static int leftShift(int x, int y) { - int shift = ((y % 32) + 32) % 32; // normalize to 0~31 + public static int leftShift(int x, long y) { + int shift = positiveModulo(y, 32); // normalize to 0~31 return y >= 0 ? x << shift : x >> shift; // arithmetic right shift } @@ -3810,23 +3825,11 @@ public static int leftShift(int x, int y) { * @param y the shift amount * @return the shifted long value */ - public static long leftShift(long x, int y) { - int shift = ((y % 64) + 64) % 64; // normalize to 0~63 + public static long leftShift(long x, long y) { + int shift = positiveModulo(y, 64); // normalize to 0~63 return y >= 0 ? x << shift : x >> shift; } - /** - * Performs PostgresSQL-style bitwise shift on an int value with a long shift amount. - * - * @param x the int value to shift - * @param y the long shift amount - * @return the shifted value as long - */ - public static long leftShift(int x, long y) { - int shift = (int) (((y % 32) + 32) % 32); // normalize to 0~31 - return y >= 0 ? (long) x << shift : (long) x >> shift; - } - /** * Performs PostgresSQL-style bitwise shift on a byte array. * Positive shift: left shift. @@ -3836,7 +3839,7 @@ public static long leftShift(int x, long y) { * @param y the shift amount in bits * @return the shifted byte array */ - public static byte[] leftShift(byte[] bytes, int y) { + public static byte[] leftShift(byte[] bytes, long y) { if (bytes.length == 0) { return new byte[0]; } @@ -3845,7 +3848,7 @@ public static byte[] leftShift(byte[] bytes, int y) { // PostgreSQL behavior: always treat as left shift with modulo arithmetic // Negative y becomes equivalent positive shift - int shift = ((y % bitLen) + bitLen) % bitLen; + int shift = positiveModulo(y, bitLen); if (shift == 0) { return bytes.clone(); @@ -3883,7 +3886,7 @@ public static byte[] leftShift(byte[] bytes, int y) { * @param y the shift amount in bits * @return shifted ByteString */ - public static ByteString leftShift(ByteString bytes, int y) { + public static ByteString leftShift(ByteString bytes, long y) { return new ByteString(leftShift(bytes.getBytes(), y)); } @@ -3891,8 +3894,8 @@ public static ByteString leftShift(ByteString bytes, int y) { * Performs PostgresSQL-style bitwise shift on UByte. * Overflow bits are masked to 8 bits. */ - public static UByte leftShift(UByte x, int y) { - int shift = ((y % 8) + 8) % 8; + public static UByte leftShift(UByte x, long y) { + int shift = positiveModulo(y, 8); int val = x.byteValue() & 0xFF; val = (y >= 0) ? (val << shift) & 0xFF : (val >> shift) & 0xFF; return UByte.valueOf((byte) val); @@ -3902,8 +3905,8 @@ public static UByte leftShift(UByte x, int y) { * Performs PostgresSQL-style bitwise shift on UShort. * Overflow bits are masked to 16 bits. */ - public static UShort leftShift(UShort x, int y) { - int shift = ((y % 16) + 16) % 16; + public static UShort leftShift(UShort x, long y) { + int shift = positiveModulo(y, 16); int val = x.shortValue() & 0xFFFF; val = (y >= 0) ? (val << shift) & 0xFFFF : (val >> shift) & 0xFFFF; return UShort.valueOf((short) val); @@ -3913,8 +3916,8 @@ public static UShort leftShift(UShort x, int y) { * Performs PostgresSQL-style bitwise shift on UInteger. * Overflow bits are masked to 32 bits. */ - public static UInteger leftShift(UInteger x, int y) { - int shift = ((y % 32) + 32) % 32; + public static UInteger leftShift(UInteger x, long y) { + int shift = positiveModulo(y, 32); long val = x.longValue() & 0xFFFFFFFFL; val = (y >= 0) ? (val << shift) & 0xFFFFFFFFL : (val >> shift) & 0xFFFFFFFFL; return UInteger.valueOf(val); @@ -3924,10 +3927,86 @@ public static UInteger leftShift(UInteger x, int y) { * Performs PostgresSQL-style bitwise shift on ULong. * Overflow bits are masked to 64 bits (long shifts naturally truncate). */ - public static ULong leftShift(ULong x, int y) { - int shift = ((y % 64) + 64) % 64; + public static ULong leftShift(ULong x, long y) { + int shift = positiveModulo(y, 64); + long val = x.longValue(); + // A negative shift amount shifts right; use a logical (unsigned) shift so + // the full-width ULong value is not sign-extended. + val = (y >= 0) ? val << shift : val >>> shift; + return ULong.valueOf(val); + } + + /** + * Performs PostgresSQL-style bitwise shift on a 32-bit integer. + * + * @param x the integer value to shift + * @param y the shift amount (positive: right shift, negative: left shift) + * @return the shifted integer + */ + public static int rightShift(int x, long y) { + int shift = positiveModulo(y, 32); // normalize to 0~31 + return y >= 0 ? x >> shift : x << shift; // arithmetic right shift + } + + /** + * Performs PostgresSQL-style bitwise shift on a 64-bit long value. + * + * @param x the long value to shift + * @param y the shift amount + * @return the shifted long value + */ + public static long rightShift(long x, long y) { + int shift = positiveModulo(y, 64); // normalize to 0~63 + return y >= 0 ? x >> shift : x << shift; + } + + // Right shift on binary (byte[]/ByteString) is intentionally not implemented: + // BINARY/VARBINARY operands are rejected for >> and RIGHTSHIFT until the + // endianness of bitwise shifts on binary is settled. See [CALCITE-7651]. + + /** + * Performs PostgresSQL-style bitwise shift on UByte. + * Overflow bits are masked to 8 bits. + */ + public static UByte rightShift(UByte x, long y) { + int shift = positiveModulo(y, 8); + int val = x.byteValue() & 0xFF; + val = (y >= 0) ? (val >> shift) & 0xFF : (val << shift) & 0xFF; + return UByte.valueOf((byte) val); + } + + /** + * Performs PostgresSQL-style bitwise shift on UShort. + * Overflow bits are masked to 16 bits. + */ + public static UShort rightShift(UShort x, long y) { + int shift = positiveModulo(y, 16); + int val = x.shortValue() & 0xFFFF; + val = (y >= 0) ? (val >> shift) & 0xFFFF : (val << shift) & 0xFFFF; + return UShort.valueOf((short) val); + } + + /** + * Performs PostgresSQL-style bitwise shift on UInteger. + * Overflow bits are masked to 32 bits. + */ + public static UInteger rightShift(UInteger x, long y) { + int shift = positiveModulo(y, 32); + long val = x.longValue() & 0xFFFFFFFFL; + val = (y >= 0) ? (val >> shift) & 0xFFFFFFFFL : (val << shift) & 0xFFFFFFFFL; + return UInteger.valueOf(val); + } + + /** + * Performs PostgresSQL-style bitwise shift on ULong. + * Overflow bits are masked to 64 bits (long shifts naturally truncate). + */ + public static ULong rightShift(ULong x, long y) { + int shift = positiveModulo(y, 64); long val = x.longValue(); - val = (y >= 0) ? val << shift : val >> shift; + // Use a logical (unsigned) right shift: ULong holds the full 64 bits, so + // the raw long may be negative and an arithmetic '>>' would sign-extend. + val = (y >= 0) ? val >>> shift : val << shift; return ULong.valueOf(val); } diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java index b1f0c0d5044d..20a5e690ffe6 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java @@ -64,6 +64,7 @@ import org.apache.calcite.sql.type.OperandTypes; import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.type.SqlOperandCountRanges; +import org.apache.calcite.sql.type.SqlOperandTypeChecker; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.apache.calcite.sql.type.SqlTypeFamily; import org.apache.calcite.sql.type.SqlTypeName; @@ -1367,6 +1368,29 @@ public class SqlStdOperatorTable extends ReflectiveSqlOperatorTable { public static final SqlAggFunction BIT_XOR = new SqlBitOpAggFunction(SqlKind.BIT_XOR); + /** + * Operand type checker shared by the left shift operator ({@code <<}) and its + * function form ({@code LEFTSHIFT}). The first operand is the value being + * shifted (integer, binary or unsigned numeric) and the second is the integer + * shift amount. + */ + private static final SqlOperandTypeChecker SHIFT_OPERAND_TYPE_CHECKER = + OperandTypes.INTEGER_INTEGER + .or(OperandTypes.family(SqlTypeFamily.BINARY, SqlTypeFamily.INTEGER)) + .or(OperandTypes.family(SqlTypeFamily.UNSIGNED_NUMERIC, SqlTypeFamily.INTEGER)); + + /** + * Operand type checker for the right shift operator ({@code >>}) and its + * function form ({@code RIGHTSHIFT}). Like {@link #SHIFT_OPERAND_TYPE_CHECKER} + * but the value being shifted may only be integer or unsigned numeric, not + * binary. Binary right shift is intentionally excluded until the endianness of + * bitwise shifts on {@code BINARY}/{@code VARBINARY} is settled; see + * [CALCITE-7651]. + */ + private static final SqlOperandTypeChecker NON_BINARY_SHIFT_OPERAND_TYPE_CHECKER = + OperandTypes.INTEGER_INTEGER + .or(OperandTypes.family(SqlTypeFamily.UNSIGNED_NUMERIC, SqlTypeFamily.INTEGER)); + /** * {@code <<} (left shift) operator. */ @@ -1378,10 +1402,7 @@ public class SqlStdOperatorTable extends ReflectiveSqlOperatorTable { true, ReturnTypes.ARG0_NULLABLE, InferTypes.FIRST_KNOWN, - OperandTypes.or( - OperandTypes.family(SqlTypeFamily.INTEGER, SqlTypeFamily.INTEGER), - OperandTypes.family(SqlTypeFamily.BINARY, SqlTypeFamily.INTEGER), - OperandTypes.family(SqlTypeFamily.UNSIGNED_NUMERIC, SqlTypeFamily.INTEGER))); + SHIFT_OPERAND_TYPE_CHECKER); /** * left shift function. @@ -1391,10 +1412,30 @@ public class SqlStdOperatorTable extends ReflectiveSqlOperatorTable { "LEFTSHIFT", SqlKind.OTHER_FUNCTION, ReturnTypes.ARG0_NULLABLE, - OperandTypes.or( - OperandTypes.family(SqlTypeFamily.INTEGER, SqlTypeFamily.INTEGER), - OperandTypes.family(SqlTypeFamily.BINARY, SqlTypeFamily.INTEGER), - OperandTypes.family(SqlTypeFamily.UNSIGNED_NUMERIC, SqlTypeFamily.INTEGER))); + SHIFT_OPERAND_TYPE_CHECKER); + + /** + * {@code >>} (right shift) operator. + */ + public static final SqlBinaryOperator BIT_RIGHT_SHIFT = + new SqlBinaryOperator( + ">>", + SqlKind.OTHER, + 32, // Standard shift operator precedence + true, + ReturnTypes.ARG0_NULLABLE, + InferTypes.FIRST_KNOWN, + NON_BINARY_SHIFT_OPERAND_TYPE_CHECKER); + + /** + * right shift function. + */ + public static final SqlFunction RIGHTSHIFT = + SqlBasicFunction.create( + "RIGHTSHIFT", + SqlKind.OTHER_FUNCTION, + ReturnTypes.ARG0_NULLABLE, + NON_BINARY_SHIFT_OPERAND_TYPE_CHECKER); //------------------------------------------------------------- // WINDOW Aggregate Functions diff --git a/core/src/main/java/org/apache/calcite/sql/parser/SqlParserPos.java b/core/src/main/java/org/apache/calcite/sql/parser/SqlParserPos.java index 97fb72cac001..ed74719ab332 100644 --- a/core/src/main/java/org/apache/calcite/sql/parser/SqlParserPos.java +++ b/core/src/main/java/org/apache/calcite/sql/parser/SqlParserPos.java @@ -264,6 +264,20 @@ public boolean startsAt(SqlParserPos pos) { && columnNumber == pos.columnNumber; } + /** + * Returns whether this position ends exactly one column before another + * position begins, on the same line, with no characters (such as whitespace) + * in between. + * + * @param pos position that may immediately follow this one + * @return whether this position ends exactly one column before {@code pos} + * begins, on the same line + */ + public boolean endsImmediatelyBefore(SqlParserPos pos) { + return endLineNumber == pos.lineNumber + && endColumnNumber + 1 == pos.columnNumber; + } + /** Parser position for an identifier segment that is quoted. */ private static class QuotedParserPos extends SqlParserPos { QuotedParserPos(int startLineNumber, int startColumnNumber, diff --git a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java index 3c3a3e363e65..629357b4e07e 100644 --- a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java +++ b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java @@ -714,7 +714,8 @@ public enum BuiltInMethod { BIT_OR(SqlFunctions.class, "bitOr", long.class, long.class), BIT_XOR(SqlFunctions.class, "bitXor", long.class, long.class), BIT_NOT(SqlFunctions.class, "bitNot", long.class), - LEFT_SHIFT(SqlFunctions.class, "leftShift", int.class, int.class), + LEFT_SHIFT(SqlFunctions.class, "leftShift", int.class, long.class), + RIGHT_SHIFT(SqlFunctions.class, "rightShift", int.class, long.class), MODIFIABLE_TABLE_GET_MODIFIABLE_COLLECTION(ModifiableTable.class, "getModifiableCollection"), SCANNABLE_TABLE_SCAN(ScannableTable.class, "scan", DataContext.class), diff --git a/core/src/test/java/org/apache/calcite/sql/parser/SqlParserPosTest.java b/core/src/test/java/org/apache/calcite/sql/parser/SqlParserPosTest.java new file mode 100644 index 000000000000..ed6d07e77c6b --- /dev/null +++ b/core/src/test/java/org/apache/calcite/sql/parser/SqlParserPosTest.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.sql.parser; + +import org.junit.jupiter.api.Test; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; + +/** + * Tests for {@link SqlParserPos}. + */ +public class SqlParserPosTest { + /** Tests {@link SqlParserPos#endsImmediatelyBefore(SqlParserPos)}. */ + @Test void testEndsImmediatelyBefore() { + // A single-character position ends immediately before the one in the next + // column, like the two '>' of a '>>' token. + final SqlParserPos col1 = new SqlParserPos(1, 1); + final SqlParserPos col2 = new SqlParserPos(1, 2); + assertThat(col1.endsImmediatelyBefore(col2), is(true)); + + // The relation is directional, not symmetric. + assertThat(col2.endsImmediatelyBefore(col1), is(false)); + + // A gap between the positions (e.g. whitespace, like '> >') does not + // qualify. + final SqlParserPos col3 = new SqlParserPos(1, 3); + assertThat(col1.endsImmediatelyBefore(col3), is(false)); + + // A position does not end immediately before itself. + assertThat(col1.endsImmediatelyBefore(col1), is(false)); + + // A multi-column position ends immediately before the position that starts + // one column after it ends. + final SqlParserPos cols1To2 = new SqlParserPos(1, 1, 1, 2); + assertThat(cols1To2.endsImmediatelyBefore(col3), is(true)); + assertThat(cols1To2.endsImmediatelyBefore(col2), is(false)); + + // Positions on different lines never qualify. + final SqlParserPos line2 = new SqlParserPos(2, 1); + assertThat(new SqlParserPos(1, 5).endsImmediatelyBefore(line2), is(false)); + } +} diff --git a/core/src/test/java/org/apache/calcite/test/SqlFunctionsTest.java b/core/src/test/java/org/apache/calcite/test/SqlFunctionsTest.java index 959e2fabc258..9231437cf8ea 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlFunctionsTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlFunctionsTest.java @@ -2045,19 +2045,30 @@ private long sqlTimestamp(String str) { return toLong(java.sql.Timestamp.valueOf(str)); } @Test void testLeftShift() { - // Test 1-byte array + // For every shift amount, cross-check the byte-array shift against the + // equivalent shift computed on the value's integer interpretation. The byte + // array is treated as a little-endian bit string of width 8 * length (index + // 0 is the least-significant byte), and leftShift always shifts left by the + // amount normalized modulo that width (so a negative amount wraps into range + // rather than reversing direction). byte[] data1 = {(byte) 0x0F}; // 00001111 for (int shift = -10; shift <= 10; shift++) { byte[] result = SqlFunctions.leftShift(data1.clone(), shift); - // Just verify it doesn't crash and returns correct length assertEquals(1, result.length); + int norm = Math.floorMod(shift, 8); + int expected = ((data1[0] & 0xFF) << norm) & 0xFF; + assertEquals((byte) expected, result[0]); } - // Test 2-byte array byte[] data2 = {(byte) 0x12, (byte) 0x34}; for (int shift = -18; shift <= 18; shift++) { byte[] result = SqlFunctions.leftShift(data2.clone(), shift); assertEquals(2, result.length); + int value = (data2[0] & 0xFF) | ((data2[1] & 0xFF) << 8); // little-endian + int norm = Math.floorMod(shift, 16); + int expected = (value << norm) & 0xFFFF; + assertEquals((byte) expected, result[0]); + assertEquals((byte) (expected >>> 8), result[1]); } // Verify specific known cases @@ -2068,6 +2079,14 @@ private long sqlTimestamp(String str) { SqlFunctions.leftShift(new byte[]{(byte) 0x40, (byte) 0x00}, 1)); } + @Test void testRightShift() { + // Scalar arithmetic (sign-preserving) right shift. Binary right shift is + // intentionally not supported (see [CALCITE-7651]), so there is no + // byte-array overload to exercise here. + assertEquals(2, SqlFunctions.rightShift(8, 2)); + assertEquals(-5, SqlFunctions.rightShift(-20, 2)); + } + @Test void testCombineQueryResults() { // Test combining two equal-length lists List list1 = Arrays.asList(1, 2, 3); diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index 08667c036b98..7bfc58eafe64 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -11467,6 +11467,7 @@ private static int prec(SqlOperator op) { + "> SOME left\n" + ">= ALL left\n" + ">= SOME left\n" + + ">> left\n" + "BETWEEN ASYMMETRIC -\n" + "BETWEEN SYMMETRIC -\n" + "IN left\n" diff --git a/core/src/test/resources/sql/operator.iq b/core/src/test/resources/sql/operator.iq index 33731a08c4d2..41a470e5f934 100644 --- a/core/src/test/resources/sql/operator.iq +++ b/core/src/test/resources/sql/operator.iq @@ -121,6 +121,21 @@ WHERE comm IS NOT NULL LIMIT 4; !ok +# [CALCITE-7639] Add support for >> operator in Calcite +SELECT CAST(comm AS INTEGER) >> 2 AS foo FROM "scott".emp +WHERE comm IS NOT NULL LIMIT 4; ++-----+ +| FOO | ++-----+ +| 75 | +| 0 | +| 125 | +| 350 | ++-----+ +(4 rows) + +!ok + # [CALCITE-5531] COALESCE throws ClassCastException SELECT COALESCE(DATE '2021-07-08', DATE '2020-01-01') as d; +------------+ @@ -811,4 +826,20 @@ SELECT !ok +-- Bitwise RIGHT SHIFT operator `>>` +SELECT + 1 >> 0 AS shift1, + 2 >> 1 AS shift2, + 8 >> 2 AS shift3, + 32 >> 3 AS shift4, + CAST(16 AS SMALLINT) >> CAST(3 AS INTEGER) AS cast_shift; ++--------+--------+--------+--------+------------+ +| SHIFT1 | SHIFT2 | SHIFT3 | SHIFT4 | CAST_SHIFT | ++--------+--------+--------+--------+------------+ +| 1 | 1 | 2 | 4 | 2 | ++--------+--------+--------+--------+------------+ +(1 row) + +!ok + # End operator.iq diff --git a/site/_docs/reference.md b/site/_docs/reference.md index b1f67eeef1ee..56fbf8a86ca4 100644 --- a/site/_docs/reference.md +++ b/site/_docs/reference.md @@ -3001,7 +3001,8 @@ In the following: | * | BITAND(value1, value2) | Returns the bitwise AND of *value1* and *value2*. *value1* and *value2* must both be integer or binary values. Binary values must be of the same length. | * | BITOR(value1, value2) | Returns the bitwise OR of *value1* and *value2*. *value1* and *value2* must both be integer or binary values. Binary values must be of the same length. | * | BITXOR(value1, value2) | Returns the bitwise XOR of *value1* and *value2*. *value1* and *value2* must both be integer or binary values. Binary values must be of the same length. -| * | LEFTSHIFT(value1, value2) | Returns the result of left-shifting *value1* by *value2* bits. *value1* can be integer, unsigned integer, or binary. For binary, the result has the same length as *value1*. The shift amount *value2* is normalized using modulo arithmetic based on the bit width of *value1*. For integers, this uses modulo 32; for binary types, it uses modulo (8 × byte_length). Negative shift amounts are converted to equivalent positive shifts through this modulo operation. For example, `LEFTSHIFT(1, -2)` returns `1073741824` (equivalent to `1 << 30`), and `LEFTSHIFT(8, -1)` returns `0` due to overflow. +| * | LEFTSHIFT(value1, value2) | Returns the result of left-shifting *value1* by *value2* bits. *value1* can be integer, unsigned integer, or binary. For binary, the result has the same length as *value1*. The shift amount *value2* is normalized using modulo arithmetic: for signed integer types the modulus is 32 for `TINYINT`, `SMALLINT` and `INTEGER` (all backed by a 32-bit representation) and 64 for `BIGINT`; for unsigned integer types it matches the type's bit width (modulo 8, 16, 32 or 64); for binary types it is modulo (8 × N), where N is the actual length in bytes of the *value1* value — for a variable-length `VARBINARY` value this is the length of the value itself, not its declared maximum. For integer and unsigned types the sign of *value2* selects the direction: a non-negative amount shifts left and a negative amount shifts right by the normalized magnitude (for example, `LEFTSHIFT(1, -2)` returns `0`, a right shift by 30, and `LEFTSHIFT(8, -1)` returns `0`). For binary the shift is always to the left; a negative *value2* is simply folded into the range [0, 8 × N) by the same modulo. +| * | RIGHTSHIFT(value1, value2) | Returns the result of right-shifting *value1* by *value2* bits. For signed integers the shift is arithmetic (the sign bit is preserved). *value1* can be integer or unsigned integer (binary right shift is not yet supported). The shift amount *value2* is normalized using modulo arithmetic: for signed integer types the modulus is 32 for `TINYINT`, `SMALLINT` and `INTEGER` (all backed by a 32-bit representation) and 64 for `BIGINT`; for unsigned integer types it matches the type's bit width (modulo 8, 16, 32 or 64). The sign of *value2* selects the direction: a non-negative amount shifts right and a negative amount shifts left by the normalized magnitude (for example, `RIGHTSHIFT(1024, 2)` returns `256`, `RIGHTSHIFT(-20, 2)` returns `-5`, and `RIGHTSHIFT(1, -2)` returns `1073741824`, a left shift by 30). | * | BITNOT(value) | Returns the bitwise NOT of *value*. *value* must be either an integer type or a binary value. | f | BITAND_AGG(value) | Equivalent to `BIT_AND(value)` | f | BITOR_AGG(value) | Equivalent to `BIT_OR(value)` diff --git a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java index 9916e0f25e7c..394aecf26f2a 100644 --- a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java +++ b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java @@ -1254,6 +1254,25 @@ private void checkLarge(int n) { .ok("((NOT (NOT (`A` = `B`))) OR (NOT (NOT (`C` = `D`))))"); } + @Test void testShiftOperators() { + expr("1 << 2") + .ok("(1 << 2)"); + // '>>' is recognized as two adjacent '>' tokens. + expr("1 >> 2") + .ok("(1 >> 2)"); + + // '<<' and '>>' have the same precedence and are left-associative. + expr("a << b >> c") + .ok("((`A` << `B`) >> `C`)"); + expr("a >> b >> c") + .ok("((`A` >> `B`) >> `C`)"); + + // The two '>' of a right shift must be adjacent, so "a > > b" (with a space + // between the '>' characters) is not parsed as a right shift. + expr("a ^>^ > b") + .fails("(?s).*Encountered \"> >\" at line 1, column 3\\..*"); + } + @Test void testIsBooleans() { String[] inOuts = {"NULL", "TRUE", "FALSE", "UNKNOWN"}; diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java index d710b516aaa7..8ddefaa32bf1 100644 --- a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java +++ b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java @@ -16955,6 +16955,7 @@ private static void checkLogicalOrFunc(SqlOperatorFixture f) { f.checkType("CAST(2 AS SMALLINT) << CAST(3 AS SMALLINT)", "SMALLINT NOT NULL"); f.checkType("CAST(2 AS INTEGER) << CAST(3 AS INTEGER)", "INTEGER NOT NULL"); f.checkType("CAST(2 AS BIGINT) << CAST(3 AS BIGINT)", "BIGINT NOT NULL"); + f.checkScalar("CAST(2 AS BIGINT) << CAST(3 AS BIGINT)", "16", "BIGINT NOT NULL"); // === BigInt shifts with explicit BIGINT inputs === f.checkScalar("CAST(1 AS BIGINT) << 62", BigInteger.ONE.shiftLeft(62).toString(), @@ -17002,8 +17003,16 @@ private static void checkLogicalOrFunc(SqlOperatorFixture f) { "INTEGER UNSIGNED NOT NULL"); f.checkScalar("CAST(1 AS INTEGER UNSIGNED) << 31", "2147483648", "INTEGER UNSIGNED NOT NULL"); f.checkScalar("CAST(1 AS INTEGER UNSIGNED) << -1", "0", "INTEGER UNSIGNED NOT NULL"); + // BIGINT UNSIGNED with the high bit set (2^63, built via 1 << 63), shifted + // left by a negative amount (i.e. right by 60): the implied right shift must + // be logical, not arithmetic (the raw long is negative). + f.checkScalar("CAST(1 AS BIGINT UNSIGNED) << 63 << -4", + "8", "BIGINT UNSIGNED NOT NULL"); // === Negative shift counts === + // A negative left shift shifts right by the normalized magnitude (here + // 32 - 2 = 30); it is not the same as a right shift by the given amount. + f.checkScalar("1 << -2", "0", "INTEGER NOT NULL"); // 1 >> 30 f.checkScalar("8 << -1", "0", "INTEGER NOT NULL"); f.checkScalar("16 << -2", "0", "INTEGER NOT NULL"); @@ -17075,6 +17084,7 @@ private static void checkLogicalOrFunc(SqlOperatorFixture f) { f.checkType("LEFTSHIFT(CAST(2 AS SMALLINT), CAST(3 AS SMALLINT))", "SMALLINT NOT NULL"); f.checkType("LEFTSHIFT(CAST(2 AS INTEGER), CAST(3 AS INTEGER))", "INTEGER NOT NULL"); f.checkType("LEFTSHIFT(CAST(2 AS BIGINT), CAST(3 AS BIGINT))", "BIGINT NOT NULL"); + f.checkScalar("LEFTSHIFT(CAST(2 AS BIGINT), CAST(3 AS BIGINT))", "16", "BIGINT NOT NULL"); // === BigInt shifts with explicit BIGINT inputs === f.checkScalar("LEFTSHIFT(CAST(1 AS BIGINT), 62)", @@ -17155,6 +17165,193 @@ private static void checkLogicalOrFunc(SqlOperatorFixture f) { f.checkNull("LEFTSHIFT(CAST(NULL AS INTEGER UNSIGNED), 2)"); } + /** + * Test cases for + * [CALCITE-7639] + * Support bitwise right shift (>>) operator and RIGHTSHIFT function. + */ + @Test void testRightShiftScalarFunc() { + final SqlOperatorFixture f = fixture(); + f.setFor(SqlStdOperatorTable.BIT_RIGHT_SHIFT, VmName.EXPAND); + + // === Basic functionality === + f.checkScalar("8 >> 2", "2", "INTEGER NOT NULL"); + f.checkScalar("1024 >> 10", "1", "INTEGER NOT NULL"); + f.checkScalar("0 >> 5", "0", "INTEGER NOT NULL"); + + // === Type coercion and signed (arithmetic) behavior === + f.checkScalar("CAST(16 AS INTEGER) >> CAST(3 AS BIGINT)", "2", "INTEGER NOT NULL"); + f.checkScalar("-20 >> 2", "-5", "INTEGER NOT NULL"); + f.checkScalar("-40 >> 3", "-5", "INTEGER NOT NULL"); + f.checkScalar("CAST(-20 AS TINYINT) >> CAST(2 AS TINYINT)", "-5", "TINYINT NOT NULL"); + + // === Verify return type matches first argument type === + f.checkType("CAST(8 AS TINYINT) >> CAST(2 AS TINYINT)", "TINYINT NOT NULL"); + f.checkType("CAST(8 AS SMALLINT) >> CAST(2 AS SMALLINT)", "SMALLINT NOT NULL"); + f.checkType("CAST(8 AS INTEGER) >> CAST(2 AS INTEGER)", "INTEGER NOT NULL"); + f.checkType("CAST(8 AS BIGINT) >> CAST(2 AS BIGINT)", "BIGINT NOT NULL"); + f.checkScalar("CAST(8 AS BIGINT) >> CAST(2 AS BIGINT)", "2", "BIGINT NOT NULL"); + + // === BigInt shifts with explicit BIGINT inputs (arithmetic/sign-preserving) === + f.checkScalar("CAST(4611686018427387904 AS BIGINT) >> 62", "1", "BIGINT NOT NULL"); // 2^62 + f.checkScalar("CAST(9223372036854775807 AS BIGINT) >> 1", + BigInteger.valueOf(Long.MAX_VALUE).shiftRight(1).toString(), "BIGINT NOT NULL"); + f.checkScalar("CAST(-1 AS BIGINT) >> 63", "-1", "BIGINT NOT NULL"); // sign bit preserved + f.checkScalar("CAST(-1 AS BIGINT) >> 1", "-1", "BIGINT NOT NULL"); + f.checkScalar("CAST(1000000000 AS BIGINT) >> 5", "31250000", "BIGINT NOT NULL"); + + // === Shift amount normalized using modulo of the bit width === + f.checkScalar("CAST(1024 AS BIGINT) >> 64", "1024", "BIGINT NOT NULL"); // 64 % 64 = 0 + f.checkScalar("CAST(1024 AS BIGINT) >> 74", "1", "BIGINT NOT NULL"); // 74 % 64 = 10 + f.checkScalar("1 >> 32", "1", "INTEGER NOT NULL"); // 32 % 32 = 0 + f.checkScalar("123 >> 60", "0", "INTEGER NOT NULL"); // 60 % 32 = 28 + + // === Unsigned types === + f.checkScalar("CAST(252 AS TINYINT UNSIGNED) >> 2", "63", "TINYINT UNSIGNED NOT NULL"); + f.checkScalar("CAST(65280 AS SMALLINT UNSIGNED) >> 8", "255", "SMALLINT UNSIGNED NOT NULL"); + f.checkScalar("CAST(4294901760 AS INTEGER UNSIGNED) >> 16", "65535", + "INTEGER UNSIGNED NOT NULL"); + f.checkScalar("CAST(2147483648 AS INTEGER UNSIGNED) >> 31", "1", "INTEGER UNSIGNED NOT NULL"); + f.checkScalar("CAST(1 AS INTEGER UNSIGNED) >> -1", "2147483648", "INTEGER UNSIGNED NOT NULL"); + // BIGINT UNSIGNED with the high bit set (2^63, built via 1 << 63): the + // right shift must be logical, not arithmetic (the raw long is negative). + f.checkScalar("CAST(1 AS BIGINT UNSIGNED) << 63 >> 4", + "576460752303423488", "BIGINT UNSIGNED NOT NULL"); + + // A BIGINT shift amount is accepted (INTEGER family), but an unsigned shift + // amount is not: the second operand must be a signed integer type. + f.checkScalar("CAST(8 AS INTEGER) >> CAST(2 AS BIGINT)", "2", "INTEGER NOT NULL"); + f.checkFails("^8 >> CAST(2 AS INTEGER UNSIGNED)^", + "Cannot apply '>>' to arguments of type ' >> '\\. " + + "Supported form\\(s\\): ' >> '\\n" + + "' >> '", + false); + + // === Negative shift counts (normalized via modulo, then shifted the other way) === + // A negative right shift shifts left by the normalized magnitude (here + // 32 - 2 = 30); it is not the same as a left shift by the given amount. + f.checkScalar("1 >> -2", "1073741824", "INTEGER NOT NULL"); // 1 << 30 + f.checkScalar("8 >> -1", "0", "INTEGER NOT NULL"); + f.checkScalar("16 >> -2", "0", "INTEGER NOT NULL"); + + // === Shift by zero and large shifts === + f.checkScalar("0 >> 32", "0", "INTEGER NOT NULL"); + f.checkScalar("0 >> 100", "0", "INTEGER NOT NULL"); + + // === Binary operands are not supported === + // Unlike '<<', binary right shift is intentionally rejected until the + // endianness of bitwise shifts on binary is settled (see [CALCITE-7651]). + // A binary literal such as X'FF' already has type BINARY(1). + f.checkFails("^X'FF' >> 1^", + "Cannot apply '>>' to arguments of type ' >> '\\. " + + "Supported form\\(s\\): ' >> '\\n" + + "' >> '", + false); + f.checkFails("^CAST(X'FF' AS VARBINARY) >> 1^", + "Cannot apply '>>' to arguments of type ' >> '\\. " + + "Supported form\\(s\\): ' >> '\\n" + + "' >> '", + false); + + // === Invalid argument types === + f.checkFails("^1.2 >> 2^", + "Cannot apply '>>' to arguments of type ' >> '\\. Supported " + + "form\\(s\\): ' >> '\\n' " + + ">> '", + false); + + // === Null propagation === + f.checkNull("CAST(NULL AS INTEGER) >> 5"); + f.checkNull("10 >> CAST(NULL AS INTEGER)"); + f.checkNull("CAST(NULL AS INTEGER) >> CAST(NULL AS INTEGER)"); + f.checkNull("CAST(NULL AS INTEGER UNSIGNED) >> 2"); + } + + @Test void testRightShiftFunctionCall() { + final SqlOperatorFixture f = fixture(); + f.setFor(SqlStdOperatorTable.BIT_RIGHT_SHIFT, VmName.EXPAND); + + // === Basic functionality === + f.checkScalar("RIGHTSHIFT(8, 2)", "2", "INTEGER NOT NULL"); + f.checkScalar("RIGHTSHIFT(1024, 10)", "1", "INTEGER NOT NULL"); + f.checkScalar("RIGHTSHIFT(0, 5)", "0", "INTEGER NOT NULL"); + + // === Type coercion and signed (arithmetic) behavior === + f.checkScalar("RIGHTSHIFT(CAST(16 AS INTEGER), CAST(3 AS BIGINT))", "2", "INTEGER NOT NULL"); + f.checkScalar("RIGHTSHIFT(-20, 2)", "-5", "INTEGER NOT NULL"); + f.checkScalar("RIGHTSHIFT(-40, 3)", "-5", "INTEGER NOT NULL"); + f.checkScalar("RIGHTSHIFT(CAST(-20 AS TINYINT), CAST(2 AS TINYINT))", "-5", "TINYINT NOT NULL"); + + // === Verify return type matches first argument type === + f.checkType("RIGHTSHIFT(CAST(8 AS TINYINT), CAST(2 AS TINYINT))", "TINYINT NOT NULL"); + f.checkType("RIGHTSHIFT(CAST(8 AS SMALLINT), CAST(2 AS SMALLINT))", "SMALLINT NOT NULL"); + f.checkType("RIGHTSHIFT(CAST(8 AS INTEGER), CAST(2 AS INTEGER))", "INTEGER NOT NULL"); + f.checkType("RIGHTSHIFT(CAST(8 AS BIGINT), CAST(2 AS BIGINT))", "BIGINT NOT NULL"); + f.checkScalar("RIGHTSHIFT(CAST(8 AS BIGINT), CAST(2 AS BIGINT))", "2", "BIGINT NOT NULL"); + + // === BigInt shifts with explicit BIGINT inputs === + f.checkScalar("RIGHTSHIFT(CAST(4611686018427387904 AS BIGINT), 62)", "1", "BIGINT NOT NULL"); + f.checkScalar("RIGHTSHIFT(CAST(9223372036854775807 AS BIGINT), 1)", + BigInteger.valueOf(Long.MAX_VALUE).shiftRight(1).toString(), "BIGINT NOT NULL"); + f.checkScalar("RIGHTSHIFT(CAST(-1 AS BIGINT), 63)", "-1", "BIGINT NOT NULL"); + f.checkScalar("RIGHTSHIFT(CAST(-1 AS BIGINT), 1)", "-1", "BIGINT NOT NULL"); + f.checkScalar("RIGHTSHIFT(CAST(1000000000 AS BIGINT), 5)", "31250000", "BIGINT NOT NULL"); + + // === Shift amount normalized using modulo of the bit width === + f.checkScalar("RIGHTSHIFT(CAST(1024 AS BIGINT), 64)", "1024", "BIGINT NOT NULL"); + f.checkScalar("RIGHTSHIFT(CAST(1024 AS BIGINT), 74)", "1", "BIGINT NOT NULL"); + f.checkScalar("RIGHTSHIFT(1, 32)", "1", "INTEGER NOT NULL"); + f.checkScalar("RIGHTSHIFT(123, 60)", "0", "INTEGER NOT NULL"); + + // === Unsigned types === + f.checkScalar("RIGHTSHIFT(CAST(252 AS TINYINT UNSIGNED), 2)", "63", + "TINYINT UNSIGNED NOT NULL"); + f.checkScalar("RIGHTSHIFT(CAST(65280 AS SMALLINT UNSIGNED), 8)", "255", + "SMALLINT UNSIGNED NOT NULL"); + f.checkScalar("RIGHTSHIFT(CAST(4294901760 AS INTEGER UNSIGNED), 16)", "65535", + "INTEGER UNSIGNED NOT NULL"); + f.checkScalar("RIGHTSHIFT(CAST(2147483648 AS INTEGER UNSIGNED), 31)", "1", + "INTEGER UNSIGNED NOT NULL"); + f.checkScalar("RIGHTSHIFT(CAST(1 AS INTEGER UNSIGNED), -1)", "2147483648", + "INTEGER UNSIGNED NOT NULL"); + + // === Negative shifts === + f.checkScalar("RIGHTSHIFT(8, -1)", "0", "INTEGER NOT NULL"); + f.checkScalar("RIGHTSHIFT(16, -2)", "0", "INTEGER NOT NULL"); + + // === Large shifts === + f.checkScalar("RIGHTSHIFT(0, 32)", "0", "INTEGER NOT NULL"); + f.checkScalar("RIGHTSHIFT(0, 100)", "0", "INTEGER NOT NULL"); + + // === Binary operands are not supported === + // Unlike LEFTSHIFT, binary right shift is intentionally rejected until the + // endianness of bitwise shifts on binary is settled (see [CALCITE-7651]). + // A binary literal such as X'FF' already has type BINARY(1). + f.checkFails("^RIGHTSHIFT(X'FF', 1)^", + "Cannot apply 'RIGHTSHIFT' to arguments of type 'RIGHTSHIFT\\(, \\)'\\. Supported form\\(s\\): 'RIGHTSHIFT\\(, \\)'\\n'RIGHTSHIFT\\(, \\)'", + false); + f.checkFails("^RIGHTSHIFT(CAST(X'FF' AS VARBINARY), 1)^", + "Cannot apply 'RIGHTSHIFT' to arguments of type 'RIGHTSHIFT\\(, \\)'\\. Supported form\\(s\\): 'RIGHTSHIFT\\(, \\)'\\n'RIGHTSHIFT\\(, \\)'", + false); + + // === Invalid types === + f.checkFails("^RIGHTSHIFT(1.2, 2)^", + "Cannot apply 'RIGHTSHIFT' to arguments of type 'RIGHTSHIFT\\(, \\)'\\. Supported form\\(s\\): 'RIGHTSHIFT\\(, \\)'\\n'RIGHTSHIFT\\(, \\)'", + false); + // A BIGINT shift amount is accepted, but an unsigned shift amount is not: + // the second argument must be a signed integer type. + f.checkScalar("RIGHTSHIFT(8, CAST(2 AS BIGINT))", "2", "INTEGER NOT NULL"); + f.checkFails("^RIGHTSHIFT(8, CAST(2 AS INTEGER UNSIGNED))^", + "Cannot apply 'RIGHTSHIFT' to arguments of type 'RIGHTSHIFT\\(, \\)'\\. Supported form\\(s\\): 'RIGHTSHIFT\\(, \\)'\\n'RIGHTSHIFT\\(, \\)'", + false); + + // === Nulls === + f.checkNull("RIGHTSHIFT(CAST(NULL AS INTEGER), 5)"); + f.checkNull("RIGHTSHIFT(10, CAST(NULL AS INTEGER))"); + f.checkNull("RIGHTSHIFT(CAST(NULL AS INTEGER), CAST(NULL AS INTEGER))"); + f.checkNull("RIGHTSHIFT(CAST(NULL AS INTEGER UNSIGNED), 2)"); + } + /** * Test cases for * [CALCITE-7184] From 2827244f73ab48d8171619e1235fb13e247359d7 Mon Sep 17 00:00:00 2001 From: Darpan Date: Fri, 3 Jul 2026 21:53:06 +0530 Subject: [PATCH 409/562] [CALCITE-7640] Enumerable engine should execute aggregates whose implementor comes from a custom RexImplementorTable CALCITE-7631 made RexImplementorTable composable for scalar code generation and constant reduction, but the aggregate path still resolves implementors through the RexImpTable singleton: EnumerableAggregate's constructor rejects any aggregate the built-ins do not know, and AggImpState generates code against the singleton. A custom aggregate implementor supplied through a chained table is therefore never used. Resolve aggregate implementors through the injected table, defaulting to the built-ins. Add RexImplementorTables.of(RelOptCluster) (reads the planner context, else the built-in table), move the availability check from the EnumerableAggregate constructor into EnumerableAggregateRule (keeping only the table-independent structural checks in the node), and thread the resolved table into AggImpState and its callers. Behaviour is unchanged when no table is registered. --- .../adapter/enumerable/AggImpState.java | 14 +- .../enumerable/EnumerableAggregate.java | 10 +- .../enumerable/EnumerableAggregateRule.java | 8 + .../enumerable/EnumerableSortedAggregate.java | 4 +- .../adapter/enumerable/EnumerableWindow.java | 4 +- .../enumerable/RexImplementorTables.java | 10 + .../calcite/interpreter/AggregateNode.java | 5 +- .../EnumerableCustomAggregateTest.java | 214 ++++++++++++++++++ 8 files changed, 258 insertions(+), 11 deletions(-) create mode 100644 core/src/test/java/org/apache/calcite/adapter/enumerable/EnumerableCustomAggregateTest.java diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/AggImpState.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/AggImpState.java index 0edc7351f03f..a22583778871 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/AggImpState.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/AggImpState.java @@ -35,10 +35,22 @@ public class AggImpState { public @MonotonicNonNull List state; public @MonotonicNonNull Expression accumulatorAdder; + /** + * Creates an AggImpState, resolving the implementor from the built-in table. + * + * @deprecated Use {@link #AggImpState(int, AggregateCall, boolean, RexImplementorTable)}. + */ + @Deprecated // to be removed before 2.0 public AggImpState(int aggIdx, AggregateCall call, boolean windowContext) { + this(aggIdx, call, windowContext, RexImpTable.INSTANCE); + } + + public AggImpState(int aggIdx, AggregateCall call, boolean windowContext, + RexImplementorTable implementorTable) { this.aggIdx = aggIdx; this.call = call; - AggImplementor implementor = RexImpTable.INSTANCE.get(call.getAggregation(), windowContext); + AggImplementor implementor = + implementorTable.get(call.getAggregation(), windowContext); if (implementor == null) { throw new IllegalArgumentException( "Unable to get aggregate implementation for aggregate " diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableAggregate.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableAggregate.java index d38b26c333cd..0429168a8fd5 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableAggregate.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableAggregate.java @@ -67,12 +67,6 @@ public EnumerableAggregate( throw new InvalidRelException( "within-distinct aggregation not supported"); } - AggImplementor implementor2 = - RexImpTable.INSTANCE.get(aggCall.getAggregation(), false); - if (implementor2 == null) { - throw new InvalidRelException( - "aggregation " + aggCall.getAggregation() + " not supported"); - } } } @@ -182,8 +176,10 @@ public EnumerableAggregate(RelOptCluster cluster, RelTraitSet traitSet, final int groupCount = getGroupCount(); final List aggs = new ArrayList<>(aggCalls.size()); + final RexImplementorTable implementorTable = + RexImplementorTables.of(getCluster()); for (Ord call : Ord.zip(aggCalls)) { - aggs.add(new AggImpState(call.i, call.e, false)); + aggs.add(new AggImpState(call.i, call.e, false, implementorTable)); } // Function0 accumulatorInitializer = diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableAggregateRule.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableAggregateRule.java index db06fea5582d..55474083886f 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableAggregateRule.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableAggregateRule.java @@ -22,6 +22,7 @@ import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.convert.ConverterRule; import org.apache.calcite.rel.core.Aggregate; +import org.apache.calcite.rel.core.AggregateCall; import org.apache.calcite.rel.logical.LogicalAggregate; import org.checkerframework.checker.nullness.qual.Nullable; @@ -48,6 +49,13 @@ protected EnumerableAggregateRule(Config config) { final Aggregate agg = (Aggregate) rel; final RelTraitSet traitSet = rel.getCluster() .traitSet().replace(EnumerableConvention.INSTANCE); + final RexImplementorTable implementorTable = + RexImplementorTables.of(rel.getCluster()); + for (AggregateCall aggCall : agg.getAggCallList()) { + if (implementorTable.get(aggCall.getAggregation(), false) == null) { + return null; + } + } try { return new EnumerableAggregate( rel.getCluster(), diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableSortedAggregate.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableSortedAggregate.java index 73a452511235..29997dd3b720 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableSortedAggregate.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableSortedAggregate.java @@ -136,8 +136,10 @@ public EnumerableSortedAggregate( final int groupCount = getGroupCount(); final List aggs = new ArrayList<>(aggCalls.size()); + final RexImplementorTable implementorTable = + RexImplementorTables.of(getCluster()); for (Ord call : Ord.zip(aggCalls)) { - aggs.add(new AggImpState(call.i, call.e, false)); + aggs.add(new AggImpState(call.i, call.e, false, implementorTable)); } // Function0 accumulatorInitializer = diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java index 78ecce8821d7..1b60c561261e 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java @@ -224,12 +224,14 @@ private static void sampleOfTheGeneratedWindowedAggregate() { List aggs = new ArrayList<>(); List aggregateCalls = group.getAggregateCalls(this); + final RexImplementorTable implementorTable = + RexImplementorTables.of(getCluster()); for (int aggIdx = 0; aggIdx < aggregateCalls.size(); aggIdx++) { AggregateCall call = aggregateCalls.get(aggIdx); if (call.ignoreNulls()) { throw new UnsupportedOperationException("IGNORE NULLS not supported"); } - aggs.add(new AggImpState(aggIdx, call, true)); + aggs.add(new AggImpState(aggIdx, call, true, implementorTable)); } // The output from this stage is the input plus the aggregate functions. diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImplementorTables.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImplementorTables.java index 22b3340ee88e..d5ff234a0e19 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImplementorTables.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImplementorTables.java @@ -17,6 +17,7 @@ package org.apache.calcite.adapter.enumerable; import org.apache.calcite.adapter.enumerable.RexImpTable.RexCallImplementor; +import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.sql.SqlAggFunction; import org.apache.calcite.sql.SqlMatchFunction; import org.apache.calcite.sql.SqlOperator; @@ -36,6 +37,15 @@ public abstract class RexImplementorTables { private RexImplementorTables() { } + /** Returns the implementor table registered on the {@code cluster}'s planner + * {@link org.apache.calcite.plan.Context}, or the built-in + * {@link RexImpTable#instance()} when none is registered. */ + public static RexImplementorTable of(RelOptCluster cluster) { + return cluster.getPlanner().getContext() + .maybeUnwrap(RexImplementorTable.class) + .orElse(RexImpTable.instance()); + } + /** Creates a table that consults each of the given tables in turn, returning * the first non-null implementor. * diff --git a/core/src/main/java/org/apache/calcite/interpreter/AggregateNode.java b/core/src/main/java/org/apache/calcite/interpreter/AggregateNode.java index 7c908cc0e8dc..684baad8d493 100644 --- a/core/src/main/java/org/apache/calcite/interpreter/AggregateNode.java +++ b/core/src/main/java/org/apache/calcite/interpreter/AggregateNode.java @@ -22,6 +22,7 @@ import org.apache.calcite.adapter.enumerable.JavaRowFormat; import org.apache.calcite.adapter.enumerable.PhysType; import org.apache.calcite.adapter.enumerable.PhysTypeImpl; +import org.apache.calcite.adapter.enumerable.RexImplementorTables; import org.apache.calcite.adapter.enumerable.RexToLixTranslator; import org.apache.calcite.adapter.enumerable.impl.AggAddContextImpl; import org.apache.calcite.adapter.java.JavaTypeFactory; @@ -143,7 +144,9 @@ private AccumulatorFactory getAccumulator(Compiler compiler, final JavaTypeFactory typeFactory = (JavaTypeFactory) rel.getCluster().getTypeFactory(); int stateOffset = 0; - final AggImpState agg = new AggImpState(0, call, false); + final AggImpState agg = + new AggImpState(0, call, false, + RexImplementorTables.of(rel.getCluster())); int stateSize = requireNonNull(agg.state, "agg.state").size(); final BlockBuilder builder2 = new BlockBuilder(); diff --git a/core/src/test/java/org/apache/calcite/adapter/enumerable/EnumerableCustomAggregateTest.java b/core/src/test/java/org/apache/calcite/adapter/enumerable/EnumerableCustomAggregateTest.java new file mode 100644 index 000000000000..8e68b547574e --- /dev/null +++ b/core/src/test/java/org/apache/calcite/adapter/enumerable/EnumerableCustomAggregateTest.java @@ -0,0 +1,214 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.adapter.enumerable; + +import org.apache.calcite.DataContexts; +import org.apache.calcite.adapter.enumerable.RexImpTable.RexCallImplementor; +import org.apache.calcite.jdbc.CalcitePrepare; +import org.apache.calcite.linq4j.Enumerable; +import org.apache.calcite.linq4j.Enumerator; +import org.apache.calcite.plan.Contexts; +import org.apache.calcite.plan.ConventionTraitDef; +import org.apache.calcite.plan.RelOptRule; +import org.apache.calcite.plan.RelOptRules; +import org.apache.calcite.plan.RelOptUtil; +import org.apache.calcite.plan.RelTraitSet; +import org.apache.calcite.plan.volcano.VolcanoPlanner; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.RelRoot; +import org.apache.calcite.runtime.Bindable; +import org.apache.calcite.sql.SqlAggFunction; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlMatchFunction; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.SqlWindowTableFunction; +import org.apache.calcite.sql.fun.SqlBasicAggFunction; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.parser.SqlParser; +import org.apache.calcite.sql.test.SqlTestFactory; +import org.apache.calcite.sql.type.OperandTypes; +import org.apache.calcite.sql.type.ReturnTypes; +import org.apache.calcite.sql.util.SqlOperatorTables; +import org.apache.calcite.sql.validate.SqlValidator; +import org.apache.calcite.sql2rel.SqlToRelConverter; +import org.apache.calcite.tools.FrameworkConfig; +import org.apache.calcite.tools.Frameworks; +import org.apache.calcite.tools.Planner; +import org.apache.calcite.tools.Programs; + +import com.google.common.collect.ImmutableMap; + +import org.checkerframework.checker.nullness.qual.Nullable; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import static java.util.Objects.requireNonNull; + +/** + * Tests that an aggregate implementor supplied by a custom + * {@link RexImplementorTable} registered on the planner + * {@link org.apache.calcite.plan.Context} is used both to plan and to execute + * an {@link EnumerableAggregate}. + */ +class EnumerableCustomAggregateTest { + /** An aggregate function with no built-in implementor. */ + private static final SqlAggFunction MY_AGG = + SqlBasicAggFunction.create("MY_CUSTOM_AGG", SqlKind.OTHER_FUNCTION, + ReturnTypes.BIGINT, OperandTypes.ANY); + + private static final String SQL = "SELECT g, MY_CUSTOM_AGG(x) AS c\n" + + "FROM (VALUES ('a', 1), ('a', 2), ('b', 3)) AS t (g, x)\n" + + "GROUP BY g"; + + /** Returns a table that maps {@link #MY_AGG} to the built-in {@code COUNT} + * implementor, so the custom aggregate counts its (non-null) input. */ + private static RexImplementorTable myAggTable() { + final AggImplementor countImplementor = + requireNonNull( + RexImpTable.instance().get(SqlStdOperatorTable.COUNT, false), + "COUNT implementor"); + return new RexImplementorTable() { + @Override public @Nullable RexCallImplementor get(SqlOperator operator) { + return null; + } + + @Override public @Nullable AggImplementor get(SqlAggFunction aggregation, + boolean forWindowAggregate) { + return aggregation == MY_AGG ? countImplementor : null; + } + + @Override public @Nullable MatchImplementor get(SqlMatchFunction function) { + return null; + } + + @Override public @Nullable TableFunctionCallImplementor get( + SqlWindowTableFunction operator) { + return null; + } + }; + } + + /** Plans a {@code GROUP BY} query through the {@link Frameworks} API. */ + private static RelNode planWithFrameworks( + @Nullable RexImplementorTable implementorTable) throws Exception { + final List rules = new ArrayList<>(EnumerableRules.rules()); + rules.addAll(RelOptRules.CALC_RULES); + rules.remove(EnumerableRules.ENUMERABLE_PROJECT_RULE); + Frameworks.ConfigBuilder configBuilder = Frameworks.newConfigBuilder() + .parserConfig(SqlParser.Config.DEFAULT) + .defaultSchema(Frameworks.createRootSchema(true)) + .operatorTable( + SqlOperatorTables.chain(SqlStdOperatorTable.instance(), + SqlOperatorTables.of(MY_AGG))) + .programs(Programs.ofRules(rules)); + if (implementorTable != null) { + configBuilder = configBuilder.context(Contexts.of(implementorTable)); + } + final FrameworkConfig config = configBuilder.build(); + + final Planner planner = Frameworks.getPlanner(config); + final SqlNode parse = planner.parse(SQL); + final RelNode logical = planner.rel(planner.validate(parse)).project(); + final RelTraitSet traitSet = + logical.getTraitSet().replace(EnumerableConvention.INSTANCE); + return planner.transform(0, traitSet, logical); + } + + private static EnumerableRel planDirectly(RexImplementorTable implementorTable) + throws Exception { + final VolcanoPlanner planner = new VolcanoPlanner(Contexts.of(implementorTable)); + planner.addRelTraitDef(ConventionTraitDef.INSTANCE); + for (RelOptRule rule : EnumerableRules.rules()) { + planner.addRule(rule); + } + for (RelOptRule rule : RelOptRules.CALC_RULES) { + planner.addRule(rule); + } + + final SqlTestFactory factory = SqlTestFactory.INSTANCE + .withPlannerFactory(context -> planner) + .withOperatorTable(opTab -> + SqlOperatorTables.chain(opTab, SqlOperatorTables.of(MY_AGG))); + final SqlNode parse = factory.createParser(SQL).parseQuery(); + final SqlToRelConverter converter = factory.createSqlToRelConverter(); + final SqlValidator validator = converter.validator; + final RelRoot root = + converter.convertQuery(validator.validate(parse), false, true); + final RelTraitSet traitSet = + root.rel.getTraitSet().replace(EnumerableConvention.INSTANCE); + final RelNode rel = planner.changeTraits(root.rel, traitSet); + planner.setRoot(rel); + return (EnumerableRel) planner.findBestExp(); + } + + private static Map execute(EnumerableRel physical) { + final Bindable bindable = + EnumerableInterpretable.toBindable(Collections.emptyMap(), + CalcitePrepare.Dummy.getSparkHandler(false), + physical, EnumerableRel.Prefer.ARRAY); + final Enumerable result = bindable.bind(DataContexts.EMPTY); + final Map actual = new HashMap<>(); + try (Enumerator enumerator = result.enumerator()) { + while (enumerator.moveNext()) { + final Object[] row = enumerator.current(); + actual.put((String) row[0], ((Number) row[1]).longValue()); + } + } + return actual; + } + + /** With the custom table on the planner context, the aggregate is planned as + * an {@link EnumerableAggregate} and executes to the expected values. */ + @Test void customAggregateImplementorIsUsedEndToEnd() throws Exception { + final RelNode physical = + planWithFrameworks( + RexImplementorTables.chain(myAggTable(), RexImpTable.instance())); + assertThat(RelOptUtil.toString(physical), + containsString("EnumerableAggregate")); + + final Map actual = execute((EnumerableRel) physical); + assertThat(actual, is(ImmutableMap.of("a", 2L, "b", 1L))); + } + + @Test void directPlannerUsesCustomTable() throws Exception { + final RexImplementorTable implementors = + RexImplementorTables.chain(myAggTable(), RexImpTable.instance()); + final EnumerableRel physical = planDirectly(implementors); + + assertThat(RelOptUtil.toString(physical), + containsString("EnumerableAggregate")); + assertThat(execute(physical), is(ImmutableMap.of("a", 2L, "b", 1L))); + } + + /** Without the custom table the built-in table has no implementor for the + * aggregate, so {@link EnumerableAggregateRule} declines and planning fails. */ + @Test void unknownAggregateIsRejectedWithoutCustomTable() { + assertThrows(RuntimeException.class, + () -> planWithFrameworks(null)); + } +} From 0a4e9c774e940e3e4a166eb3fd6604c461d30228 Mon Sep 17 00:00:00 2001 From: krooswu Date: Sat, 20 Jun 2026 21:06:26 +0800 Subject: [PATCH 410/562] [CALCITE-7505] RelToSqlConverter fails to alias outer relation for correlated sub-queries in Filter --- .../rel/rel2sql/RelToSqlConverter.java | 45 ++++++++++-- .../rel/rel2sql/RelToSqlConverterTest.java | 68 +++++++++++++++++-- 2 files changed, 103 insertions(+), 10 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java b/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java index 44eec4feb9f3..718e6b1cb965 100644 --- a/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java +++ b/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java @@ -547,17 +547,38 @@ public Result visit(Correlate e) { null); return result(join, leftResult, rightResult); } - /** Visits a Filter; called by {@link #dispatch} via reflection. */ public Result visit(Filter e) { final RelNode input = e.getInput(); + final Set definedHere = e.getVariablesSet(); + if (input instanceof Aggregate) { final Aggregate aggregate = (Aggregate) input; final boolean ignoreClauses = aggregate.getInput() instanceof Project; - final Result x = + Result x = visitInput(e, 0, isAnon(), ignoreClauses, ImmutableSet.of(Clause.HAVING)); + // Only rebind the correlation alias when e.getInput() renders as a single + // addressable relation (<=1 input): TableScan, Project, Filter, etc. + // Join/Correlate (2+ inputs) already expose a multi-alias context via + // joinContext(); collapsing it here would break field resolution. + final boolean pushed = !definedHere.isEmpty() + && e.getInput().getInputs().size() <= 1; + if (pushed) { + String alias = x.neededAlias; + if (alias != null) { + x = x.resetAliasForCorrelation(alias, e.getInput().getRowType()); + } else { + alias = unqualifiedName(x.node); + if (alias == null) { + alias = "t"; + } + x = x.resetAliasForCorrelation + (alias, e.getInput().getRowType()); + } + } parseCorrelTable(e, x); + final Builder builder = x.builder(e); x.asSelect().setHaving( SqlUtil.andExpressions(x.asSelect().getHaving(), @@ -565,8 +586,24 @@ public Result visit(Filter e) { return builder.result(); } else { Result x = visitInput(e, 0, Clause.WHERE); - if (!e.getVariablesSet().isEmpty()) { - x = x.resetAlias(); + // Only rebind the correlation alias when e.getInput() renders as a single + // addressable relation (<=1 input): TableScan, Project, Filter, etc. + // Join/Correlate (2+ inputs) already expose a multi-alias context via + // joinContext(); collapsing it here would break field resolution. + final boolean pushed = !definedHere.isEmpty() + && e.getInput().getInputs().size() <= 1; + if (pushed) { + String alias = x.neededAlias; + if (alias != null) { + x = x.resetAliasForCorrelation(alias, e.getInput().getRowType()); + } else { + alias = unqualifiedName(x.node); + if (alias == null) { + alias = "t"; + } + x = x.resetAliasForCorrelation + (alias, e.getInput().getRowType()); + } } parseCorrelTable(e, x); final Builder builder = x.builder(e); diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java index affb7093f03e..279454f6b311 100644 --- a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java +++ b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java @@ -6714,7 +6714,7 @@ private void checkLiteral2(String expression, String expected) { + "from \"sales_fact_1997\"b " + "where b.\"product_id\" = a.\"product_id\")"; String expected = "SELECT \"product_name\"\n" - + "FROM \"foodmart\".\"product\"\n" + + "FROM \"foodmart\".\"product\" AS \"product\"\n" + "WHERE EXISTS (SELECT COUNT(*)\n" + "FROM \"foodmart\".\"sales_fact_1997\"\n" + "WHERE \"product_id\" = \"product\".\"product_id\")"; @@ -6727,7 +6727,7 @@ private void checkLiteral2(String expression, String expected) { + "from \"sales_fact_1997\"b " + "where b.\"product_id\" = a.\"product_id\")"; String expected = "SELECT \"product_name\"\n" - + "FROM \"foodmart\".\"product\"\n" + + "FROM \"foodmart\".\"product\" AS \"product\"\n" + "WHERE NOT EXISTS (SELECT COUNT(*)\n" + "FROM \"foodmart\".\"sales_fact_1997\"\n" + "WHERE \"product_id\" = \"product\".\"product_id\")"; @@ -6740,7 +6740,7 @@ private void checkLiteral2(String expression, String expected) { + "from \"sales_fact_1997\"b " + "where b.\"product_id\" = a.\"product_id\")"; String expected = "SELECT \"product_name\"\n" - + "FROM \"foodmart\".\"product\"\n" + + "FROM \"foodmart\".\"product\" AS \"product\"\n" + "WHERE \"product_id\" IN (SELECT \"product_id\"\n" + "FROM \"foodmart\".\"sales_fact_1997\"\n" + "WHERE \"product_id\" = \"product\".\"product_id\")"; @@ -6762,7 +6762,7 @@ private void checkLiteral2(String expression, String expected) { + "from \"sales_fact_1997\"b " + "where b.\"product_id\" = a.\"product_id\")"; String expected = "SELECT \"product_name\"\n" - + "FROM \"foodmart\".\"product\"\n" + + "FROM \"foodmart\".\"product\" AS \"product\"\n" + "WHERE \"product_id\" NOT IN (SELECT \"product_id\"\n" + "FROM \"foodmart\".\"sales_fact_1997\"\n" + "WHERE \"product_id\" = \"product\".\"product_id\")"; @@ -6780,7 +6780,7 @@ private void checkLiteral2(String expression, String expected) { + "where t2.\"product_id\" = t1.\"product_id\" " + "and t1.\"product_id\" = 2 and t2.\"product_id\" = 1)"; String expected = "SELECT \"product_name\"\n" - + "FROM \"foodmart\".\"product\"\n" + + "FROM \"foodmart\".\"product\" AS \"product\"\n" + "WHERE \"product_id\" NOT IN (SELECT \"product_id\"\n" + "FROM \"foodmart\".\"product\" AS \"product0\"\n" + "WHERE \"product_id\" = \"product\".\"product_id\" " @@ -9165,7 +9165,7 @@ private void checkLiteral2(String expression, String expected) { + "GROUP BY \"t1\".\"department_id\"\n" + "HAVING \"t1\".\"department_id\" = MIN(\"t1\".\"department_id\")) \"t4\" ON \"employee\".\"department_id\" = \"t4\".\"department_id0\""; final String expectedNoExpand = "SELECT \"department_id\"\n" - + "FROM \"foodmart\".\"employee\"\n" + + "FROM \"foodmart\".\"employee\" AS \"employee\"\n" + "WHERE \"department_id\" = (SELECT MIN(\"employee\".\"department_id\")\n" + "FROM \"foodmart\".\"department\"\n" + "WHERE 1 = 2)"; @@ -12733,4 +12733,60 @@ public Sql schema(CalciteAssert.SchemaSpec schemaSpec) { sql(query).withLibrary(SqlLibrary.HIVE).withHive().ok(expectedHive); sql(query).withLibrary(SqlLibrary.SPARK).withSpark().ok(expectedSpark); } + + /** Test case for + * [CALCITE-7505] + * RelToSqlConverter fails to alias outer relation for correlated sub-queries in Filter. */ + @Test void testExistsSubQueryAliasConflict() { + final String sql = + "select deptno, sum(sal) as total\n" + + "from emp t\n" + + "where exists (\n" + + " select * from dept t0\n" + + " where deptno = t.deptno\n" + + ")\n" + + "group by deptno"; + + + sql(sql) + .schema(CalciteAssert.SchemaSpec.JDBC_SCOTT) + .ok( + "SELECT \"DEPTNO\", SUM(\"SAL\") AS \"TOTAL\"\n" + + "FROM \"SCOTT\".\"EMP\" AS \"EMP\"\n" + + "WHERE EXISTS (SELECT *\n" + + "FROM \"SCOTT\".\"DEPT\"\n" + + "WHERE \"DEPTNO\" = \"EMP\".\"DEPTNO\")\n" + + "GROUP BY \"DEPTNO\""); + } + + /** Test case for + * [CALCITE-7505] + * RelToSqlConverter fails to alias outer relation for correlated sub-queries in Filter. */ + @Test void testExistsSubQueryOverUnion() { + final String sql = + "SELECT *\n" + + "FROM (\n" + + " SELECT deptno FROM emp\n" + + " UNION ALL\n" + + " SELECT deptno FROM dept\n" + + ") u\n" + + "WHERE EXISTS (\n" + + " SELECT 1\n" + + " FROM emp e\n" + + " WHERE e.deptno = u.deptno\n" + + ")"; + + sql(sql) + .schema(CalciteAssert.SchemaSpec.JDBC_SCOTT) + .ok( + "SELECT *\n" + + "FROM (SELECT \"DEPTNO\"\n" + + "FROM \"SCOTT\".\"EMP\"\n" + + "UNION ALL\n" + + "SELECT \"DEPTNO\"\n" + + "FROM \"SCOTT\".\"DEPT\") AS \"t1\"\n" + + "WHERE EXISTS (SELECT *\n" + + "FROM \"SCOTT\".\"EMP\"\n" + + "WHERE \"DEPTNO\" = \"t1\".\"DEPTNO\")"); + } } From bbb5dbd39ae97c1dd0c452d6e931adbab52be4ea Mon Sep 17 00:00:00 2001 From: Vladimir Sitnikov Date: Tue, 21 Jul 2026 15:48:29 +0300 Subject: [PATCH 411/562] [CALCITE-7605] Add threat model Document what Calcite treats as a security vulnerability: the attacker model, the properties it provides and disclaims (P1-P4), the surprising-vs-unsurprising class-loading rule, and denial of service as a hardening goal. Add SECURITY.md with the private-reporting channel. This is an alternative draft for the same Jira as #5020, reorganizing the same normative content into a structure that is easier to read and to maintain per-CVE. Co-Authored-By: Claude Opus 4.8 --- .ratignore | 1 + SECURITY.md | 14 ++ site/_data/docs.yml | 4 + site/_docs/security_threat_model.md | 202 ++++++++++++++++++++++++++++ 4 files changed, 221 insertions(+) create mode 100644 SECURITY.md create mode 100644 site/_docs/security_threat_model.md diff --git a/.ratignore b/.ratignore index 71048d405c5c..93e7e93e7a17 100644 --- a/.ratignore +++ b/.ratignore @@ -13,6 +13,7 @@ **/data.txt **/data2.txt .idea/vcs.xml +SECURITY.md example/csv/src/test/resources/smoke_test.sql # TODO: remove when pom.xml files are removed diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 000000000000..4d14a5b793ca --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,14 @@ +# Security Policy + +## Reporting a Vulnerability + +`apache/calcite` follows the [Apache Software Foundation security process](https://www.apache.org/security/). +Please report suspected vulnerabilities privately to `security@apache.org`; do not open public +GitHub issues or pull requests for security reports. + +## Threat Model + +What the project treats as in scope and out of scope, the security +properties it provides and disclaims, the adversary model, and how +findings are triaged are documented in the +[threat model](site/_docs/security_threat_model.md). diff --git a/site/_data/docs.yml b/site/_data/docs.yml index 5a6a71491485..fc1bac1f612b 100644 --- a/site/_data/docs.yml +++ b/site/_data/docs.yml @@ -44,6 +44,10 @@ - model - howto +- title: Security + docs: + - security_threat_model + - title: Meta docs: - history diff --git a/site/_docs/security_threat_model.md b/site/_docs/security_threat_model.md new file mode 100644 index 000000000000..5f5deb7b9a67 --- /dev/null +++ b/site/_docs/security_threat_model.md @@ -0,0 +1,202 @@ +--- +layout: docs +title: Security threat model +permalink: /docs/security_threat_model.html +--- + + +Calcite is an embedded library: it runs inside a host application's JVM and +exposes no network port of its own. This threat model covers what an attacker +who reaches that embedded engine over a JDBC connection can and cannot do. + +Calcite treats the behaviors below as security vulnerabilities, so that +reporters and committers triage them the same way. A report that +contradicts this model is a feature request or a documentation gap, not a +vulnerability. + +**Status:** draft for PMC discussion; not yet ratified. + +* TOC +{:toc} + +## Attacker and trust boundary + +One attacker profile: a *query author* who reaches Calcite over a JDBC +connection. + +The attacker can: + +* set any connection property to any value: `model`, `parserFactory`, + `schemaFactory`, `fun`, `typeSystem`, `dataSource`, `jdbcUrl`, and the + rest; +* execute any SQL, including DDL. + +The attacker cannot: + +* change JVM system properties; +* change the classpath (add or replace classes or JARs). + +Out of scope by definition: the configuration and behavior of a +third-party driver or service that a [model]({{ site.baseurl }}/docs/model.html) +points at. If a model references h2, h2's own settings and behavior are +h2's concern, not Calcite's. + +Because Calcite listens on no socket of its own, network transport, TLS, and +authentication belong to the host application, not to this model. A host that +lets an untrusted principal set connection properties has handed that principal +the attacker's full capability above. + +## Assets + +* the host running Calcite: no code execution, and no file access beyond + what an adapter is configured to perform; +* the internal network reachable from that host: no attacker-directed + outbound requests. + +## Security properties + +* **P1: no code execution.** Neither connecting nor running SQL may + execute code outside Calcite's query-processing semantics. This covers + `Runtime.exec` and `ProcessBuilder`, and the weaker primitive of loading + an attacker-named class so that its static initializer, constructor, or + an accessed static field runs. Exception: the os-adapter. +* **P2: no incidental file access.** Neither connecting nor running + SQL may read or create a file, except where a file-oriented adapter or + table function reads the local path it was explicitly configured with. The + carve-out covers local filesystem paths only; a file adapter that fetches a + URL (`http://`, `https://`) is making a network request and falls under P3. +* **P3: no server-side request forgery.** Neither connecting nor + running SQL may open a network connection to an attacker-chosen host. +* **P4: no escape from the configured schemas.** Neither connecting nor + running SQL may read data outside the schemas the connection exposes. A query + that reaches another schema, a file, or a catalog that the connection's root + schema does not make visible is a vulnerability. + +## Always a vulnerability + +1. **Code execution** that results from connecting or running SQL, except + through the os-adapter. The bar is the primitive, not a full chain: a + reachable sink that loads an attacker-named class qualifies, because + class loading runs the static initializer before any type check. +2. **Arbitrary file read or write** that no explicitly-configured file + adapter was asked to perform. +3. **Server-side request forgery**, forcing Calcite to connect to a + host the attacker chooses (internal services, cloud metadata endpoints, + port scans). +4. **Reading beyond the configured schemas**, reaching another schema, + a file, or a catalog that the connection's root schema does not make + visible. + +## Not a vulnerability + +* The os-adapter running OS commands. It exists to do that, and an operator + must add it on purpose. +* A file, CSV, or JSON adapter reading the local path it was configured + with. Opt-in, by the same reasoning as the os-adapter. +* Anything that needs a changed system property or classpath. Both are + outside the attacker's reach by assumption. +* The behavior of a third-party driver once Calcite has connected to the + endpoint it was configured with. +* Cross-tenant reads that follow from the embedder exposing more than one + principal's schemas on a single connection. Calcite has no authentication or + authorization; scoping each connection's root schema to what its principal may + see is the embedder's job. P4 applies where the user submits only SQL; a user + who also sets connection properties configures their own schema visibility. + +## Surprising vs unsurprising class loading + +Calcite loads a class named in a connection property or in SQL only to use it +through a specific SPI: a schema factory, table factory, function, operator, +data source, or driver. The security boundary follows that contract, not a +blanket trust of the classpath or of SQL. + +* **Unsurprising.** The class implements the SPI interface for the position it + was named in, and Calcite invokes it through that interface. This is working + as designed, even when the class is otherwise dangerous. The operator who + placed the class on the classpath, and the author of the class, own that + contract. +* **Surprising.** Naming a class runs the class's own code (a static + initializer, constructor, method, or static-field read) even though it + does not implement the SPI for that position. `java.lang.Runtime`, + `org.springframework.boot.SpringApplication`, and `javax.naming.InitialContext` + are surprising in a `SchemaFactory` slot. Surprising class loading is always a + vulnerability. + +This boundary lets three goals hold at once: + +* untrusted classes may sit on the classpath; a dangerous class that does + not implement a Calcite SPI is never instantiated by name, so the operator + need not audit every class; +* SQL may be arbitrary; it can name SPI classes, but only SPI + implementations run, and only through their SPI; +* a reasonable-looking query cannot trigger an unexpected process launch, file + read, or network call, because the interface gate rejects the classes that + would cause one. + +The same rule governs any path that reconstructs objects or loads classes from +attacker-controlled input, including JSON plan deserialization. + +**Mechanism.** Load with `Class.forName(name, false, loader)`, check +`pluginClass.isAssignableFrom(clazz)`, and only then initialize and instantiate. +A class that fails the check never runs its static initializer. + +**Standard-Java SPIs.** The gate is tightest when the SPI is a Calcite interface +(`SchemaFactory`, `TableFactory`, `Function`, `SqlOperator`): only a class +written to be a Calcite plugin passes. Two positions name a standard Java +interface instead, `dataSource` (`javax.sql.DataSource`) and `jdbcDriver` +(`java.sql.Driver`), which many unrelated libraries implement. The +interface gate still blocks the surprising case, since `java.lang.Runtime` +implements neither, so naming a `DataSource` or `Driver` implementation is the +documented feature, not a vulnerability. An allowlist of permitted +implementations is optional hardening for these positions, not a security +boundary. The host a driver then connects to is governed by P3, independently of +which class is named. + +### Triage rule for class-loading sinks + +A reachable sink that loads an attacker-named class is a vulnerability on +its own. A reporter need not demonstrate end-to-end remote code execution +on a specific classpath: the demonstrated primitive (a static +initializer, a constructor, or a static-field read on an attacker-named +class) is enough to require a fix. The fix loads with +`initialize=false`, gates on the expected interface (or an allowlist), and +then instantiates. + +## Denial of service + +A single query should not be able to exhaust the host. This is in scope as a +hardening goal. The controls are not all in place yet, so treat the gaps below +as known limitations rather than per-report vulnerabilities until the controls +land. + +* **Planning.** A crafted query can drive the planner into a combinatorial + blow-up. The fix is a set of bounds: a planning deadline, a cap on rule + firings, and a cap on the number of explored alternatives. Calcite already + carries a `CancelFlag` in the planner context, so a deadline can build on it; + the firing and size caps are new. +* **Execution.** Catastrophic regex backtracking in `LIKE`, `SIMILAR TO`, or + `RLIKE`, or an unbounded join, exhausts resources at run time. Planning bounds + do not help here; the mitigation is a match-time limit or a backtracking-free + regex engine. +* **Parsing.** Deeply nested expressions can overflow the parser stack. The + mitigation is a nesting-depth limit. + +Once the planning bounds exist, a single reasonably-sized query that exceeds +them is a configuration choice, not a vulnerability. From 1ddaa3318aa6368ec1dde00ea4d2842743555312 Mon Sep 17 00:00:00 2001 From: Vladimir Sitnikov Date: Tue, 21 Jul 2026 18:22:04 +0300 Subject: [PATCH 412/562] [CALCITE-7605] Threat model: add Inputs, Downstream responsibilities, Triage dispositions Fold in the three additive sections proposed in vlsi/calcite#3, corrected against the code: - Inputs: map every attacker-controlled input to its governing rule. Add the metaTableFactory/metaColumnFactory PLUGIN properties, place operator under the RelJson row (SqlOperator loads only via RelJson), and note that fun selects built-in libraries rather than loading a class. - Downstream responsibilities: collect the host/operator duties in one place and drop the now-duplicated transport paragraph from the attacker section. - Triage dispositions: a closed outcome set a human or agent can route against. - Not a vulnerability: state that pushed-down SQL is by design, while a pushdown bug that reads beyond the configured schemas is P4. Also remove the draft-status line ahead of merge. Co-authored-by: Jarek Potiuk Co-Authored-By: Claude Opus 4.8 --- site/_docs/security_threat_model.md | 79 ++++++++++++++++++++++++++--- 1 file changed, 72 insertions(+), 7 deletions(-) diff --git a/site/_docs/security_threat_model.md b/site/_docs/security_threat_model.md index 5f5deb7b9a67..d9936197d473 100644 --- a/site/_docs/security_threat_model.md +++ b/site/_docs/security_threat_model.md @@ -31,8 +31,6 @@ reporters and committers triage them the same way. A report that contradicts this model is a feature request or a documentation gap, not a vulnerability. -**Status:** draft for PMC discussion; not yet ratified. - * TOC {:toc} @@ -58,11 +56,6 @@ third-party driver or service that a [model]({{ site.baseurl }}/docs/model.html) points at. If a model references h2, h2's own settings and behavior are h2's concern, not Calcite's. -Because Calcite listens on no socket of its own, network transport, TLS, and -authentication belong to the host application, not to this model. A host that -lets an untrusted principal set connection properties has handed that principal -the attacker's full capability above. - ## Assets * the host running Calcite: no code execution, and no file access beyond @@ -70,6 +63,23 @@ the attacker's full capability above. * the internal network reachable from that host: no attacker-directed outbound requests. +## Inputs + +Everything the attacker controls resolves to one of P1–P4 or to an explicit +carve-out below. A report that reaches a sink not covered here is a model gap +(see [Triage dispositions](#triage-dispositions)). + +| Input | How it is supplied | What it feeds | Governing rule | +| --- | --- | --- | --- | +| SQL text, including DDL | any statement on the connection | parser → validator → planner → generated code | P1–P4; parser nesting depth is a DoS surface (see [Denial of service](#denial-of-service)) | +| Class-naming connection properties — `schemaFactory`, `parserFactory`, `typeSystem`, `metaTableFactory`, `metaColumnFactory` | connection property or `model` | a class loaded through a Calcite SPI | [Surprising vs unsurprising class loading](#surprising-vs-unsurprising-class-loading) (P1) | +| `tableFactory` and function classes | `model` | a class loaded through a Calcite table or function SPI | Surprising vs unsurprising class loading (P1) | +| `dataSource`, `jdbcDriver` | connection property or `model` | a class loaded through a standard-Java SPI (`javax.sql.DataSource`, `java.sql.Driver`) | Surprising vs unsurprising class loading (P1); the host it then dials is P3 | +| `fun` | connection property | selects built-in function libraries by name | no class loading; ordinary SQL semantics under P1–P4 | +| `model` — inline JSON, a `file:` path, or a URL | connection property | schema/table factories and adapter operands | P1 (factories via SPI), P2 (local-file operands), P3 (a URL model, or a URL-fetching adapter) | +| A serialized RelNode plan (`RelJson`) — types and operators | any path that reconstructs a plan from attacker input | type and operator class resolution | Surprising vs unsurprising class loading (P1) | +| Adapter operands — e.g. a file/CSV/JSON path, or the os-adapter | `model` or SQL | the adapter's configured resource | P2 for a configured local path (opt-in ⇒ not a vulnerability); the os-adapter is opt-in (not a vulnerability) | + ## Security properties * **P1: no code execution.** Neither connecting nor running SQL may @@ -114,12 +124,39 @@ the attacker's full capability above. outside the attacker's reach by assumption. * The behavior of a third-party driver once Calcite has connected to the endpoint it was configured with. +* SQL that Calcite pushes down to a configured backend. Calcite generates the + text and sends it to the endpoint the operator configured, and the query + author can already reach that endpoint's data through the visible schemas. A + pushdown bug that reads beyond the configured schemas is P4 and a + vulnerability; the generated SQL reaching the configured backend is not. * Cross-tenant reads that follow from the embedder exposing more than one principal's schemas on a single connection. Calcite has no authentication or authorization; scoping each connection's root schema to what its principal may see is the embedder's job. P4 applies where the user submits only SQL; a user who also sets connection properties configures their own schema visibility. +## Downstream responsibilities + +Calcite is embedded, so several controls belong to the host or the operator, +not to the library. A finding that lands in one of these is not a Calcite +vulnerability. + +* **Transport and identity.** Calcite opens no socket. TLS, the network + perimeter, authentication, and authorization live in the host. A host that + lets an untrusted principal set connection properties hands that principal the + full capability in [Attacker and trust boundary](#attacker-and-trust-boundary). +* **Schema scoping.** Scope each connection's root schema to what its principal + may see; Calcite has no authentication or authorization. Cross-tenant reads + across schemas exposed on one connection are the embedder's to prevent (see + [Not a vulnerability](#not-a-vulnerability)). +* **Adapter selection.** Add the os-adapter and the file, CSV, or JSON adapters + only where the query author is trusted to reach what they expose. +* **Classpath.** The operator owns the classpath. Calcite gates class loading by + SPI; which classes are present is the operator's trust decision. +* **What a `model` points at.** A third-party driver or service a `model` + references is configured and patched by the operator; its behavior past the + connection boundary is out of this model. + ## Surprising vs unsurprising class loading Calcite loads a class named in a connection property or in SQL only to use it @@ -200,3 +237,31 @@ land. Once the planning bounds exist, a single reasonably-sized query that exceeds them is a configuration choice, not a vulnerability. + +## Triage dispositions + +Every security report against Calcite resolves to exactly one of: + +* **Valid** — violates P1–P4, or matches an item in + [Always a vulnerability](#always-a-vulnerability). Gets a fix. A demonstrated + class-loading primitive qualifies on its own (see + [Triage rule for class-loading sinks](#triage-rule-for-class-loading-sinks)); + it need not be chained to end-to-end RCE. +* **Not a vulnerability (by design)** — matches an item in + [Not a vulnerability](#not-a-vulnerability): the os-adapter, an opt-in + file/CSV/JSON adapter reading its configured path, third-party driver or + pushed-down SQL behavior past the connection, or cross-tenant reads that + follow from the embedder's schema exposure. Close with a pointer to this + model. +* **Out of model** — requires a capability the attacker does not have (changing + a JVM system property or the classpath), or lands in a layer this model + assigns to the host (network transport, TLS, authentication, authorization). + Close; redirect to the operator or embedder. +* **Known limitation** — a [Denial of service](#denial-of-service) gap whose + control has not landed yet. Tracked as hardening, not a per-report + vulnerability, until the bound exists. +* **Duplicate** — the same sink or root cause is already tracked in an open + Jira. Link and close. +* **Model gap** — plausible, but this model does not clearly place it in or out. + Escalate to the PMC to decide, then update this document with the ruling so + the next report of its kind is no longer a gap. From eeb2e60f0917d3c63f7d99e573305f334462eddd Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Fri, 24 Jul 2026 16:50:48 -0700 Subject: [PATCH 413/562] [CALCITE-7671] EXISTS fails typechecking nested lambda EXIST(a -> EXISTS(a, b -> b > 2)) Signed-off-by: Mihai Budiu --- .../apache/calcite/sql/type/OperandTypes.java | 57 ++++++++++++++++++- .../apache/calcite/test/SqlOperatorTest.java | 17 ++++++ 2 files changed, 71 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql/type/OperandTypes.java b/core/src/main/java/org/apache/calcite/sql/type/OperandTypes.java index 99c623865cb1..114b11cf169d 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/OperandTypes.java +++ b/core/src/main/java/org/apache/calcite/sql/type/OperandTypes.java @@ -1417,11 +1417,28 @@ public static SqlSingleOperandTypeChecker same(int operandCount, SqlCallBinding callBinding, boolean throwOnFailure) { // The first operand must be an array type - ARRAY.checkSingleOperandType(callBinding, callBinding.operand(0), 0, throwOnFailure); + if (!ARRAY.checkSingleOperandType(callBinding, callBinding.operand(0), 0, + throwOnFailure)) { + return false; + } final RelDataType arrayType = SqlTypeUtil.deriveType(callBinding, callBinding.operand(0)); - final RelDataType componentType = - requireNonNull(arrayType.getComponentType(), "componentType"); + RelDataType componentType = arrayType.getComponentType(); + if (componentType == null) { + // The ARRAY family check above accepts operands of type ANY and + // untyped NULL literals, which have no component type. + if (arrayType.getSqlTypeName() == SqlTypeName.ANY) { + // This is probably a parameter of an enclosing lambda, whose type has + // not been inferred yet. Accept for now; the enclosing function's checker + // will re-validated the lambda body with concrete parameter types, which runs + // this checker again. + return true; + } + // Untyped NULL literal: an unknown array whose elements are also + // NULL; the call returns NULL. + componentType = + callBinding.getTypeFactory().createSqlType(SqlTypeName.NULL); + } // The second operand is a function(array_element_type)->boolean type LambdaRelOperandTypeChecker lambdaChecker = @@ -1948,6 +1965,40 @@ private static class LambdaRelOperandTypeChecker /** * Abstract base class for type-checking strategies involving lambda expressions. * This class provides common functionality for checking the type of lambda expression. + * + *

      Lambda expressions are validated in two passes, and operand checkers of + * functions that accept lambda operands (higher-order functions) must be + * written with both passes in mind. For example, consider validating + * + *

      {@code + * EXISTS(array[array[1, 2]], a -> EXISTS(a, b -> b > 1)) + * }
      + * + *
        + *
      1. Deriving the type of the outer call starts by deriving the types of + * its operands. The type of the outer lambda is derived by validating its + * body with parameter {@code a} set to the nullable ANY type, the default + * that {@link org.apache.calcite.sql.validate.SqlLambdaScope} assigns while + * the parameter types are still unknown. During this pass the checker of + * the inner {@code EXISTS} call runs and sees its array operand + * {@code a} typed as ANY. It cannot check anything meaningful yet, so it + * must accept the call provisionally instead of failing; definitive checking + * happens in the second pass. + * + *
      2. The checker of the outer {@code EXISTS} then computes the + * concrete type of {@code a} from the component type of its array operand + * {@code array[array[1, 2]]}, namely {@code INTEGER ARRAY}; stores it in the + * lambda's {@code SqlLambdaScope}; discards the types derived during the + * first pass (see {@code TypeRemover}); and calls + * {@link org.apache.calcite.sql.validate.SqlValidator#validateLambda} to + * re-validate the body. This re-runs the checker of the inner + * {@code EXISTS}, which now sees {@code a} as {@code INTEGER ARRAY} and + * repeats the same protocol for the inner lambda, typing {@code b} as + * {@code INTEGER}. + *
      + * + *

      See {@link OperandTypes#EXISTS} for a checker that follows this + * protocol. */ private abstract static class LambdaOperandTypeChecker implements SqlSingleOperandTypeChecker { diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java index 8ddefaa32bf1..91f20af68167 100644 --- a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java +++ b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java @@ -9208,6 +9208,23 @@ void checkArrayReverseFunc(SqlOperatorFixture f0, SqlFunction function, f.checkNull("\"EXISTS\"(array[null, 3], x -> cast(null as boolean))"); f.checkNull("\"EXISTS\"(array[null, 3], x -> x = null)"); f.checkNull("\"EXISTS\"(cast(null as integer array), x -> x > 2)"); + f.checkNull("\"EXISTS\"(null, x -> x > 2)"); + } + + /** Test case for + * [CALCITE-7671] + * EXISTS fails typechecking nested lambda + * EXIST(a -> EXISTS(a, b -> b > 2)). */ + @Test void testNestedExistsFunc() { + final SqlOperatorFixture f = fixture() + .setFor(SqlLibraryOperators.EXISTS) + .withLibrary(SqlLibrary.SPARK); + f.checkScalar("\"EXISTS\"(array[array[1, 2], array[3, 4]]," + + " a -> \"EXISTS\"(a, b -> b > 3))", true, "BOOLEAN"); + f.checkScalar("\"EXISTS\"(array[array[1, 2], array[3, 4]]," + + " a -> \"EXISTS\"(a, b -> b > 4))", false, "BOOLEAN"); + f.checkNull("\"EXISTS\"(cast(null as integer array array)," + + " a -> \"EXISTS\"(a, b -> b > 3))"); } /** Tests {@code MAP_CONCAT} function from Spark. */ From 3b2d58f0f003012a8c2696897f8f8d0919a35759 Mon Sep 17 00:00:00 2001 From: xuzifu666 <1206332514@qq.com> Date: Fri, 24 Jul 2026 16:46:14 +0800 Subject: [PATCH 414/562] [CALCITE-7666] Make regex pattern caches static to avoid repeated allocation in RexInterpreter --- .../apache/calcite/runtime/SqlFunctions.java | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java index fd4101b5a42b..56e6bdb42298 100644 --- a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java +++ b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java @@ -2095,7 +2095,7 @@ Pattern toPattern() { } } - private final LoadingCache cache = + private static final LoadingCache CACHE = CacheBuilder.newBuilder() .maximumSize(FUNCTION_LEVEL_CACHE_MAX_SIZE.value()) .build(CacheLoader.from(Key::toPattern)); @@ -2103,32 +2103,32 @@ Pattern toPattern() { /** SQL {@code LIKE} function. */ public boolean like(String s, String pattern) { final Key key = new Key(pattern, null, 0); - return cache.getUnchecked(key).matcher(s).matches(); + return CACHE.getUnchecked(key).matcher(s).matches(); } /** SQL {@code LIKE} function with escape. */ public boolean like(String s, String pattern, String escape) { final Key key = new Key(pattern, escape, 0); - return cache.getUnchecked(key).matcher(s).matches(); + return CACHE.getUnchecked(key).matcher(s).matches(); } /** SQL {@code ILIKE} function. */ public boolean ilike(String s, String pattern) { final Key key = new Key(pattern, null, Pattern.CASE_INSENSITIVE); - return cache.getUnchecked(key).matcher(s).matches(); + return CACHE.getUnchecked(key).matcher(s).matches(); } /** SQL {@code ILIKE} function with escape. */ public boolean ilike(String s, String pattern, String escape) { final Key key = new Key(pattern, escape, Pattern.CASE_INSENSITIVE); - return cache.getUnchecked(key).matcher(s).matches(); + return CACHE.getUnchecked(key).matcher(s).matches(); } } /** State for {@code SIMILAR} function. */ @Deterministic public static class SimilarFunction { - private final LoadingCache cache = + private static final LoadingCache CACHE = CacheBuilder.newBuilder() .maximumSize(FUNCTION_LEVEL_CACHE_MAX_SIZE.value()) .build( @@ -2137,7 +2137,7 @@ public static class SimilarFunction { /** SQL {@code SIMILAR} function. */ public boolean similar(String s, String pattern) { - return cache.getUnchecked(pattern).matcher(s).matches(); + return CACHE.getUnchecked(pattern).matcher(s).matches(); } } @@ -2154,14 +2154,14 @@ Pattern toPattern() { } } - private final LoadingCache cache = + private static final LoadingCache CACHE = CacheBuilder.newBuilder() .maximumSize(FUNCTION_LEVEL_CACHE_MAX_SIZE.value()) .build(CacheLoader.from(Key::toPattern)); /** SQL {@code SIMILAR} function with escape. */ public boolean similar(String s, String pattern, String escape) { - return cache.getUnchecked(new Key(pattern, escape)) + return CACHE.getUnchecked(new Key(pattern, escape)) .matcher(s).matches(); } } From 9e18414d140e9752b4942ef4053627e1480e8416 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Jul 2026 00:45:53 +0000 Subject: [PATCH 415/562] Bump json from 2.10.2 to 2.19.9 in /site Bumps [json](https://github.com/ruby/json) from 2.10.2 to 2.19.9. - [Release notes](https://github.com/ruby/json/releases) - [Changelog](https://github.com/ruby/json/blob/master/CHANGES.md) - [Commits](https://github.com/ruby/json/compare/v2.10.2...v2.19.9) --- updated-dependencies: - dependency-name: json dependency-version: 2.19.9 dependency-type: indirect ... Signed-off-by: dependabot[bot] --- site/Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site/Gemfile.lock b/site/Gemfile.lock index 7fc66d79a4b4..0a2b05da29cf 100644 --- a/site/Gemfile.lock +++ b/site/Gemfile.lock @@ -64,7 +64,7 @@ GEM sass-embedded (~> 1.75) jekyll-watch (2.2.1) listen (~> 3.0) - json (2.10.2) + json (2.19.9) kramdown (2.5.1) rexml (>= 3.3.9) kramdown-parser-gfm (1.1.0) From 3a1996c40de16ef1e29aa24a492a8f6682a99b6a Mon Sep 17 00:00:00 2001 From: microbluey Date: Mon, 27 Jul 2026 11:51:47 +0800 Subject: [PATCH 416/562] [CALCITE-7557] Linq4j.ListEnumerable.take(int) / skip(int) diverge from EnumerableDefaults on negative counts ListEnumerable specializes take(int) and skip(int) for lists, but unlike the adjacent BigDecimal overloads, which clamp via count.max(BigDecimal.ZERO), the int versions pass the count straight to List.subList. A negative count therefore threw IllegalArgumentException from take and IndexOutOfBoundsException from skip, while the generic EnumerableDefaults path returns an empty enumerable and the original sequence respectively. Clamp the count to zero in both methods so that the optimized list path agrees with the generic path. Math.max is used rather than negating the count so that Integer.MIN_VALUE does not overflow. --- .../java/org/apache/calcite/linq4j/Linq4j.java | 16 ++++++++++++---- .../apache/calcite/linq4j/test/Linq4jTest.java | 16 ++++++++++++++++ 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/Linq4j.java b/linq4j/src/main/java/org/apache/calcite/linq4j/Linq4j.java index df9e39e35bd2..c59303d4fdc0 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/Linq4j.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/Linq4j.java @@ -587,10 +587,14 @@ static class ListEnumerable extends CollectionEnumerable { @Override public Enumerable skip(int count) { final List list = toList(); - if (count >= list.size()) { + // Clamp to zero, as the BigDecimal overload does, so that a negative + // count skips nothing and matches EnumerableDefaults.skip rather than + // throwing from List.subList. + final int rows = Math.max(count, 0); + if (rows >= list.size()) { return Linq4j.emptyEnumerable(); } - return new ListEnumerable<>(list.subList(count, list.size())); + return new ListEnumerable<>(list.subList(rows, list.size())); } @Override public Enumerable skip(BigDecimal count) { @@ -605,10 +609,14 @@ static class ListEnumerable extends CollectionEnumerable { @Override public Enumerable take(int count) { final List list = toList(); - if (count >= list.size()) { + // Clamp to zero, as the BigDecimal overload does, so that a negative + // count yields an empty enumerable and matches EnumerableDefaults.take + // rather than throwing from List.subList. + final int rows = Math.max(count, 0); + if (rows >= list.size()) { return this; } - return new ListEnumerable<>(list.subList(0, count)); + return new ListEnumerable<>(list.subList(0, rows)); } @Override public Enumerable take(BigDecimal count) { diff --git a/linq4j/src/test/java/org/apache/calcite/linq4j/test/Linq4jTest.java b/linq4j/src/test/java/org/apache/calcite/linq4j/test/Linq4jTest.java index 62e7dd9f5eef..cd894f964b83 100644 --- a/linq4j/src/test/java/org/apache/calcite/linq4j/test/Linq4jTest.java +++ b/linq4j/src/test/java/org/apache/calcite/linq4j/test/Linq4jTest.java @@ -2238,4 +2238,20 @@ public String toString() { new Department("HR", 20, ImmutableList.of()), new Department("Marketing", 30, ImmutableList.of(emps[1])), }; + + @Test void testTakeListEnumerableNegativeSize() { + final List values = Arrays.asList(1, 2, 3); + + assertThat(EnumerableDefaults.take(Linq4j.asEnumerable(values), -1).toList(), + is(empty())); + assertThat(Linq4j.asEnumerable(values).take(-1).toList(), is(empty())); + } + + @Test void testSkipListEnumerableNegativeSize() { + final List values = Arrays.asList(1, 2, 3); + + assertThat(EnumerableDefaults.skip(Linq4j.asEnumerable(values), -1).toList(), + is(values)); + assertThat(Linq4j.asEnumerable(values).skip(-1).toList(), is(values)); + } } From 27c0411971d8b9db4f0198573801d0f4db88713e Mon Sep 17 00:00:00 2001 From: xuzifu666 <1206332514@qq.com> Date: Mon, 27 Jul 2026 11:38:50 +0800 Subject: [PATCH 417/562] [CALCITE-7672] PruneEmptyRules should support pruning empty Calc --- .../org/apache/calcite/plan/RelOptRules.java | 1 + .../calcite/rel/rules/PruneEmptyRules.java | 17 ++++++++++++++++ .../apache/calcite/test/RelOptRulesTest.java | 16 +++++++++++++++ .../apache/calcite/test/RelOptRulesTest.xml | 20 +++++++++++++++++++ 4 files changed, 54 insertions(+) diff --git a/core/src/main/java/org/apache/calcite/plan/RelOptRules.java b/core/src/main/java/org/apache/calcite/plan/RelOptRules.java index 2abce5f327ec..434e278dcb85 100644 --- a/core/src/main/java/org/apache/calcite/plan/RelOptRules.java +++ b/core/src/main/java/org/apache/calcite/plan/RelOptRules.java @@ -101,6 +101,7 @@ private RelOptRules() { PruneEmptyRules.MINUS_INSTANCE, PruneEmptyRules.PROJECT_INSTANCE, PruneEmptyRules.FILTER_INSTANCE, + PruneEmptyRules.CALC_INSTANCE, PruneEmptyRules.SORT_INSTANCE, PruneEmptyRules.AGGREGATE_INSTANCE, PruneEmptyRules.WINDOW_INSTANCE, diff --git a/core/src/main/java/org/apache/calcite/rel/rules/PruneEmptyRules.java b/core/src/main/java/org/apache/calcite/rel/rules/PruneEmptyRules.java index 5cb60bc90652..95331c69a414 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/PruneEmptyRules.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/PruneEmptyRules.java @@ -26,6 +26,7 @@ import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.SingleRel; import org.apache.calcite.rel.core.Aggregate; +import org.apache.calcite.rel.core.Calc; import org.apache.calcite.rel.core.Correlate; import org.apache.calcite.rel.core.Filter; import org.apache.calcite.rel.core.Intersect; @@ -191,6 +192,19 @@ private static boolean isEmpty(RelNode node) { public static final RelOptRule FILTER_INSTANCE = RemoveEmptySingleRule.RemoveEmptySingleRuleConfig.FILTER.toRule(); + /** + * Rule that converts a {@link org.apache.calcite.rel.core.Calc} + * to empty if its child is empty. + * + *

      Examples: + * + *

        + *
      • Calc(Empty) becomes Empty + *
      + */ + public static final RelOptRule CALC_INSTANCE = + RemoveEmptySingleRule.RemoveEmptySingleRuleConfig.CALC.toRule(); + /** * Rule that converts a {@link org.apache.calcite.rel.core.Sort} * to empty if its child is empty. @@ -373,6 +387,9 @@ public interface RemoveEmptySingleRuleConfig extends PruneEmptyRule.Config { RemoveEmptySingleRuleConfig FILTER = ImmutableRemoveEmptySingleRuleConfig.of() .withDescription("PruneEmptyFilter") .withOperandFor(Filter.class, singleRel -> true); + RemoveEmptySingleRuleConfig CALC = ImmutableRemoveEmptySingleRuleConfig.of() + .withDescription("PruneEmptyCalc") + .withOperandFor(Calc.class, singleRel -> true); RemoveEmptySingleRuleConfig SORT = ImmutableRemoveEmptySingleRuleConfig.of() .withDescription("PruneEmptySort") .withOperandFor(Sort.class, singleRel -> true); diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index 52c1875e3696..5c687519daa0 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -5349,6 +5349,22 @@ RelOptFixture checkDynamicFunctions(boolean treatDynamicCallsAsConstant) { .check(); } + /** Test case for + * [CALCITE-7672] + * PruneEmptyRules should support pruning empty Calc. + */ + @Test void testEmptyCalc() { + final String sql = "select z + x from (\n" + + " select x + y as z, x from (\n" + + " select * from (values (10, 1), (30, 3)) as t (x, y)\n" + + " where x + y > 50))"; + sql(sql) + .withRule(CoreRules.FILTER_VALUES_MERGE, + CoreRules.PROJECT_TO_CALC, + PruneEmptyRules.CALC_INSTANCE) + .check(); + } + @Test void testEmptyIntersect() { final String sql = "select * from (values (30, 3))" + "intersect\n" diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index e2b725a7d66d..f1bb4248e528 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -4212,6 +4212,26 @@ LogicalAggregate(group=[{}], EXPR$0=[COUNT()], EXPR$1=[SUM($0)]) + + + + + 50))]]> + + + (+($0, $1), 50)]) + LogicalValues(tuples=[[{ 10, 1 }, { 30, 3 }]]) +]]> + + + From 0c80f142ee15802a52f677ae0796cdeecdbc11a5 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Mon, 27 Jul 2026 18:34:03 +0200 Subject: [PATCH 418/562] Add AGENTS.md so the threat model is discoverable by agents MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CALCITE-7605 landed the threat model at site/_docs/security_threat_model.md and a SECURITY.md that links to it. The remaining gap is the entry point: automated agents locate a project's security model by following AGENTS.md -> SECURITY.md -> the model, and this repository has no AGENTS.md, so the first hop is missing and the chain cannot be walked mechanically. This adds one, pointing at SECURITY.md and summarising the two rules that carry most of the triage weight — the surprising-vs-unsurprising class-loading gate over the SPI positions, and the pushed-down-SQL split (generated SQL reaching a configured backend is not a vulnerability; a pushdown bug reading beyond the configured schemas is P4 and is one) — plus the "Not a vulnerability" and "Downstream responsibilities" lists. No new claims: everything here restates the merged model. AGENTS.md is added to .ratignore alongside SECURITY.md, matching what CALCITE-7605 did for that file, since it carries no license header. Docs only; no production code touched. Generated-by: Claude Opus 5 (1M context) --- .ratignore | 1 + AGENTS.md | 43 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 AGENTS.md diff --git a/.ratignore b/.ratignore index 93e7e93e7a17..d42e5ca0fd74 100644 --- a/.ratignore +++ b/.ratignore @@ -14,6 +14,7 @@ **/data2.txt .idea/vcs.xml SECURITY.md +AGENTS.md example/csv/src/test/resources/smoke_test.sql # TODO: remove when pom.xml files are removed diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 000000000000..67870438a738 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,43 @@ +# Agent guidance + +This file is read by automated agents (security scanners, code analyzers, +AI assistants) operating on this repository. It points them at the +human-authored references they should consult before producing output. + +## Security + +Security model: [SECURITY.md](./SECURITY.md), which links to the project's +threat model at +[site/_docs/security_threat_model.md](./site/_docs/security_threat_model.md). + +Calcite is an embedded SQL framework, not a server. It opens no socket and +has no authentication or authorization of its own; the host application +owns transport, identity, and schema scoping. Read the threat model before +reporting anything — it is explicit about what it does and does not treat +as a vulnerability. + +Two rules carry most of the triage weight: + +- **Surprising vs unsurprising class loading.** A class named through a + Calcite SPI position — `schemaFactory`, `parserFactory`, `typeSystem`, + `metaTableFactory`, `metaColumnFactory`, `tableFactory`, function + classes, `dataSource`, `jdbcDriver`, `model` — is loaded only through + that SPI, gated by `Class.forName(name, false, loader)` plus an + `isAssignableFrom` check. A class that does not implement the SPI for + its position is never instantiated by name. SQL may name SPI classes, + but only SPI implementations run, and only through their SPI. +- **Pushed-down SQL.** The SQL Calcite generates and sends to a backend + the operator configured is *not* a vulnerability — the query author can + already reach that endpoint through the visible schemas. A pushdown bug + that reads *beyond* the configured schemas is P4 and *is* one. + +Explicitly not vulnerabilities (see the model's "Not a vulnerability" +section): the os-adapter running OS commands, the file/CSV/JSON adapters +reading paths they were configured with, anything requiring a changed +system property or classpath, a third-party driver's behaviour past the +connection boundary, and cross-tenant reads that follow from the embedder +exposing several principals' schemas on one connection. + +The model also lists what belongs to the host rather than the library — +transport and identity, schema scoping, adapter selection, the classpath, +and whatever a `model` points at — under "Downstream responsibilities". From 500ae3c3943f47dfe7109513c58b489829ddc2ee Mon Sep 17 00:00:00 2001 From: Ruben Quesada Lopez Date: Fri, 24 Jul 2026 12:53:21 +0100 Subject: [PATCH 419/562] [CALCITE-7667] Improve string-literal encoding in pushdown translators Co-authored-by: bibi samina --- .../adapter/cassandra/CassandraFilter.java | 22 ++- .../CassandraFilterTranslatorTest.java | 138 ++++++++++++++++++ .../calcite/test/CassandraAdapterTest.java | 9 ++ .../org/apache/calcite/sql/SqlDialect.java | 7 + .../sql/dialect/BigQuerySqlDialect.java | 2 +- .../calcite/sql/dialect/MysqlSqlDialect.java | 10 ++ .../rel/rel2sql/RelToSqlConverterTest.java | 32 +++- .../adapter/geode/rel/GeodeFilter.java | 4 +- .../geode/rel/GeodeFilterTranslatorTest.java | 98 +++++++++++++ .../adapter/geode/rel/GeodeZipsTest.java | 11 ++ .../apache/calcite/adapter/pig/PigFilter.java | 7 +- .../pig/PigFilterLiteralEscapeTest.java | 79 ++++++++++ .../apache/calcite/test/PigAdapterTest.java | 15 ++ 13 files changed, 426 insertions(+), 8 deletions(-) create mode 100644 cassandra/src/test/java/org/apache/calcite/adapter/cassandra/CassandraFilterTranslatorTest.java create mode 100644 geode/src/test/java/org/apache/calcite/adapter/geode/rel/GeodeFilterTranslatorTest.java create mode 100644 pig/src/test/java/org/apache/calcite/adapter/pig/PigFilterLiteralEscapeTest.java diff --git a/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraFilter.java b/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraFilter.java index f357079d61f1..8da8cff99186 100644 --- a/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraFilter.java +++ b/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraFilter.java @@ -47,6 +47,7 @@ import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.regex.Pattern; import static org.apache.calcite.util.DateTimeStringUtils.ISO_DATETIME_FRACTIONAL_SECOND_FORMAT; import static org.apache.calcite.util.DateTimeStringUtils.getDateFormatter; @@ -126,6 +127,11 @@ public RelCollation getImplicitCollation() { /** Translates {@link RexNode} expressions into Cassandra expression strings. */ static class Translator { + /** Canonical UUID form: 8-4-4-4-12 hex digits (case-insensitive). */ + private static final Pattern UUID_PATTERN = + Pattern.compile("[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}" + + "-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}"); + private final RelDataType rowType; private final List fieldNames; private final Set partitionKeys; @@ -301,8 +307,20 @@ private String translateOp2(String op, String name, RexLiteral right) { RelDataTypeField field = requireNonNull(rowType.getField(name, true, false)); SqlTypeName typeName = field.getType().getSqlTypeName(); - if (typeName != SqlTypeName.CHAR) { - valueString = "'" + valueString + "'"; + if (typeName == SqlTypeName.CHAR) { + // Cassandra UUID and TIMEUUID columns are mapped to + // SqlTypeName.CHAR (see CqlToSqlTypeConversionRules), + // CQL accepts UUIDs as bare 8-4-4-4-12 hex literals + if (!UUID_PATTERN.matcher(valueString).matches()) { + throw new IllegalArgumentException( + "Cannot push down filter on Cassandra uuid/timeuuid column '" + + name + "': value is not a well-formed UUID"); + } + // valueString is a validated UUID; safe to emit unquoted. + } else { + // CQL string literals use `''` to represent a single `'` inside a + // `'...'` literal, so double any embedded `'` before wrapping + valueString = "'" + valueString.replace("'", "''") + "'"; } } return name + " " + op + " " + valueString; diff --git a/cassandra/src/test/java/org/apache/calcite/adapter/cassandra/CassandraFilterTranslatorTest.java b/cassandra/src/test/java/org/apache/calcite/adapter/cassandra/CassandraFilterTranslatorTest.java new file mode 100644 index 000000000000..49c177b84709 --- /dev/null +++ b/cassandra/src/test/java/org/apache/calcite/adapter/cassandra/CassandraFilterTranslatorTest.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.adapter.cassandra; + +import org.apache.calcite.jdbc.JavaTypeFactoryImpl; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.type.SqlTypeName; + +import org.junit.jupiter.api.Test; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.Collections; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Unit tests for {@link CassandraFilter.Translator} covering CQL literal serialization. + * + *

      The tests exercise the translator directly against a synthetic row + * type; they do not require a running Cassandra cluster. + */ +class CassandraFilterTranslatorTest { + + private static final RelDataTypeFactory TYPE_FACTORY = new JavaTypeFactoryImpl(); + private static final RexBuilder REX_BUILDER = new RexBuilder(TYPE_FACTORY); + + /** Two-column row type mirroring how CqlToSqlTypeConversionRules + * maps Cassandra types: a CHAR column stands in for `uuid`/`timeuuid` + * and a VARCHAR column stands in for `text`. */ + private static RelDataType rowType() { + return TYPE_FACTORY.builder() + .add("id", SqlTypeName.CHAR, 36) + .add("name", SqlTypeName.VARCHAR) + .build(); + } + + /** Invokes the (private) translator's translateMatch entry point via reflection. + * Any exception is unwrapped from InvocationTargetException so callers can + * see the real cause. */ + private static String translate(RexNode condition) throws Throwable { + CassandraFilter.Translator t = + new CassandraFilter.Translator(rowType(), + Collections.singletonList("id"), + Collections.emptyList(), + Collections.emptyList()); + Method m = CassandraFilter.Translator.class + .getDeclaredMethod("translateMatch", RexNode.class); + m.setAccessible(true); + try { + return (String) m.invoke(t, condition); + } catch (InvocationTargetException e) { + throw e.getCause(); + } + } + + private static RexNode eqLit(int fieldIndex, SqlTypeName fieldType, + int precision, String value) { + RelDataType t = precision > 0 + ? TYPE_FACTORY.createSqlType(fieldType, precision) + : TYPE_FACTORY.createSqlType(fieldType); + RexInputRef ref = REX_BUILDER.makeInputRef(t, fieldIndex); + RexNode lit = REX_BUILDER.makeLiteral(value, t, false); + return REX_BUILDER.makeCall(SqlStdOperatorTable.EQUALS, ref, lit); + } + + @Test void uuidColumnValidUuidEmittedUnquoted() throws Throwable { + String cql = + translate(eqLit(0, SqlTypeName.CHAR, 36, "037f7c30-abcd-11ee-8000-000000000001")); + assertThat(cql, is("id = 037f7c30-abcd-11ee-8000-000000000001")); + } + + @Test void uuidColumnNonUuidValueRejected() { + IllegalArgumentException e = + assertThrows( + IllegalArgumentException.class, () -> translate( + eqLit(0, SqlTypeName.CHAR, 36, "not-a-uuid"))); + assertThat(e.getMessage(), containsString("not a well-formed UUID")); + } + + @Test void uuidColumnEmptyRejected() { + assertThrows(IllegalArgumentException.class, + () -> translate(eqLit(0, SqlTypeName.CHAR, 36, ""))); + } + + @Test void uuidColumnAlmostUuidRejected() { + assertThrows(IllegalArgumentException.class, + () -> translate( + eqLit(0, SqlTypeName.CHAR, 36, "037f7c30-abcd-11ee-8000-000000000001x"))); + } + + @Test void varcharColumnPlainValueQuoted() throws Throwable { + String cql = translate(eqLit(1, SqlTypeName.VARCHAR, -1, "alice")); + assertThat(cql, is("name = 'alice'")); + } + + @Test void varcharColumnValueWithApostrophe() throws Throwable { + String cql = translate(eqLit(1, SqlTypeName.VARCHAR, -1, "O'Brien")); + assertThat(cql, is("name = 'O''Brien'")); + } + + @Test void varcharColumnValueWithMultipleApostrophes() throws Throwable { + String cql = translate(eqLit(1, SqlTypeName.VARCHAR, -1, "a''b")); + assertThat(cql, is("name = 'a''''b'")); + } + + @Test void varcharColumnValueWithApostropheAtTheStart() throws Throwable { + String cql = translate(eqLit(1, SqlTypeName.VARCHAR, -1, "'a")); + assertThat(cql, is("name = '''a'")); + } + + @Test void varcharColumnValueWithApostropheAtTheEnd() throws Throwable { + String cql = translate(eqLit(1, SqlTypeName.VARCHAR, -1, "a'")); + assertThat(cql, is("name = 'a'''")); + } +} diff --git a/cassandra/src/test/java/org/apache/calcite/test/CassandraAdapterTest.java b/cassandra/src/test/java/org/apache/calcite/test/CassandraAdapterTest.java index 411ffdbb65be..88898e6a0455 100644 --- a/cassandra/src/test/java/org/apache/calcite/test/CassandraAdapterTest.java +++ b/cassandra/src/test/java/org/apache/calcite/test/CassandraAdapterTest.java @@ -71,6 +71,15 @@ static void load(CqlSession session) { + " CassandraTableScan(table=[[twissandra, userline]]"); } + @Test void testFilterWithSingleQuote() { + // A string literal containing a single quote must be escaped so it does not + // break out of the CQL string literal in the generated query. + CalciteAssert.that() + .with(TWISSANDRA) + .query("select * from \"userline\" where \"username\" = 'a''b'") + .returnsCount(0); + } + @Test void testFilterUUID() { CalciteAssert.that() .with(TWISSANDRA) diff --git a/core/src/main/java/org/apache/calcite/sql/SqlDialect.java b/core/src/main/java/org/apache/calcite/sql/SqlDialect.java index 663aee63b0db..164f212c6c7a 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlDialect.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlDialect.java @@ -445,6 +445,13 @@ public void quoteStringLiteral(StringBuilder buf, @Nullable String charsetName, buf.append(literalEndQuoteString); } + /** Doubles every backslash in {@code val}, for dialects whose backend + * treats {@code \} as an in-string escape character (e.g. MySQL, + * MariaDB, BigQuery). */ + protected static String escapeBackslash(String val) { + return val.replace("\\", "\\\\"); + } + public void unparseCall(SqlWriter writer, SqlCall call, int leftPrec, int rightPrec) { SqlOperator operator = call.getOperator(); diff --git a/core/src/main/java/org/apache/calcite/sql/dialect/BigQuerySqlDialect.java b/core/src/main/java/org/apache/calcite/sql/dialect/BigQuerySqlDialect.java index 6683054819ea..13314a1aeda1 100644 --- a/core/src/main/java/org/apache/calcite/sql/dialect/BigQuerySqlDialect.java +++ b/core/src/main/java/org/apache/calcite/sql/dialect/BigQuerySqlDialect.java @@ -124,7 +124,7 @@ public BigQuerySqlDialect(SqlDialect.Context context) { // enclosing quote as \'. Otherwise a value containing a backslash (e.g. // "x\" or "\'; ...") terminates the literal early and the trailing text is // parsed as SQL rather than data. - super.quoteStringLiteral(buf, charsetName, val.replace("\\", "\\\\")); + super.quoteStringLiteral(buf, charsetName, escapeBackslash(val)); } @Override public boolean supportsImplicitTypeCoercion(RexCall call) { diff --git a/core/src/main/java/org/apache/calcite/sql/dialect/MysqlSqlDialect.java b/core/src/main/java/org/apache/calcite/sql/dialect/MysqlSqlDialect.java index 7b4475283ea0..03d4d2c50454 100644 --- a/core/src/main/java/org/apache/calcite/sql/dialect/MysqlSqlDialect.java +++ b/core/src/main/java/org/apache/calcite/sql/dialect/MysqlSqlDialect.java @@ -126,6 +126,16 @@ public MysqlSqlDialect(Context context) { return false; } + @Override public void quoteStringLiteral(StringBuilder buf, + @Nullable String charsetName, String val) { + // MySQL treats backslash as an escape character inside string literals, + // so a literal backslash must be doubled before the base method escapes the + // enclosing quote as \'. Otherwise a value containing a backslash (e.g. + // "x\" or "\'; ...") terminates the literal early and the trailing text is + // parsed as SQL rather than data. + super.quoteStringLiteral(buf, charsetName, escapeBackslash(val)); + } + @Override public boolean requiresAliasForFromItems() { return true; } diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java index 279454f6b311..c04da0a26bd6 100644 --- a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java +++ b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java @@ -9510,6 +9510,36 @@ private void checkLiteral2(String expression, String expected) { }); } + /** Test case for the MySQL-family backslash-escape bypass: + * dialects whose backend treats {@code \} as an in-string escape + * character must double it in + * {@link SqlDialect#quoteStringLiteral(StringBuilder, String, String)}. */ + @Test void testDialectQuoteStringLiteralWithBackslash() { + dialects().forEach((dialect, databaseProduct) -> { + final boolean escapesBackslash = + databaseProduct == DatabaseProduct.BIG_QUERY + || databaseProduct == DatabaseProduct.MYSQL + || databaseProduct == DatabaseProduct.STARROCKS + || databaseProduct == DatabaseProduct.DORIS; + + // Trailing backslash + assertThat(dialect.quoteStringLiteral("x\\"), + escapesBackslash ? is("'x\\\\'") : is("'x\\'")); + + // Leading backslash + assertThat(dialect.quoteStringLiteral("\\x"), + escapesBackslash ? is("'\\\\x'") : is("'\\x'")); + + // Backslash followed by content + assertThat(dialect.quoteStringLiteral("x\\y"), + escapesBackslash ? is("'x\\\\y'") : is("'x\\y'")); + + // Two consecutive backslashes must both be doubled + assertThat(dialect.quoteStringLiteral("x\\\\"), + escapesBackslash ? is("'x\\\\\\\\'") : is("'x\\\\'")); + }); + } + @Test void testSelectCountStar() { final String query = "select count(*) from \"product\""; final String expected = "SELECT COUNT(*)\n" @@ -11166,7 +11196,7 @@ private void checkLiteral2(String expression, String expected) { final String query = "SELECT TRIM(BOTH '$@*A' from '$@*AABC$@*AADCAA$@*A')\n" + "from \"foodmart\".\"reserve_employee\""; final String expectedStarRocks = "SELECT REGEXP_REPLACE('$@*AABC$@*AADCAA$@*A'," - + " '^(\\$\\@\\*A)*|(\\$\\@\\*A)*$', '')\n" + + " '^(\\\\$\\\\@\\\\*A)*|(\\\\$\\\\@\\\\*A)*$', '')\n" + "FROM `foodmart`.`reserve_employee`"; sql(query).withStarRocks().ok(expectedStarRocks) .withDoris().ok(expectedStarRocks); diff --git a/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeFilter.java b/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeFilter.java index dafc0122d139..47d43728c620 100644 --- a/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeFilter.java +++ b/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeFilter.java @@ -395,7 +395,9 @@ private String translateBinary2(String op, RexNode left, private static String quoteCharLiteral(RexLiteral literal) { String value = literalValue(literal); if (literal.getTypeName() == CHAR) { - value = "'" + value + "'"; + // OQL string literals use `''` to represent a single `'` inside + // a `'...'` literal, so double any embedded `'` before wrapping + value = "'" + value.replace("'", "''") + "'"; } return value; } diff --git a/geode/src/test/java/org/apache/calcite/adapter/geode/rel/GeodeFilterTranslatorTest.java b/geode/src/test/java/org/apache/calcite/adapter/geode/rel/GeodeFilterTranslatorTest.java new file mode 100644 index 000000000000..35f21d4f736b --- /dev/null +++ b/geode/src/test/java/org/apache/calcite/adapter/geode/rel/GeodeFilterTranslatorTest.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.adapter.geode.rel; + +import org.apache.calcite.jdbc.JavaTypeFactoryImpl; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.type.SqlTypeName; + +import org.junit.jupiter.api.Test; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; + +/** + * Unit tests for {@link GeodeFilter.Translator} covering OQL literal serialization. + * + *

      The tests exercise the translator directly against a synthetic + * row type; they do not require a running Geode cluster. + */ +class GeodeFilterTranslatorTest { + + private static final RelDataTypeFactory TYPE_FACTORY = new JavaTypeFactoryImpl(); + private static final RexBuilder REX_BUILDER = new RexBuilder(TYPE_FACTORY); + + private static RelDataType rowType() { + return TYPE_FACTORY.builder() + .add("code", SqlTypeName.CHAR, 32) + .build(); + } + + /** Invokes the (private) translator's translateMatch entry + * point via reflection so this test can live in the same package + * without widening the Translator's visibility. Any exception the + * target throws is unwrapped from InvocationTargetException. */ + private static String translate(RexNode condition) throws Throwable { + GeodeFilter.Translator t = + new GeodeFilter.Translator(rowType(), REX_BUILDER); + Method m = GeodeFilter.Translator.class + .getDeclaredMethod("translateMatch", RexNode.class); + m.setAccessible(true); + try { + return (String) m.invoke(t, condition); + } catch (InvocationTargetException e) { + throw e.getCause(); + } + } + + /** Builds a {@code code = } predicate where the literal is + * typed as {@code CHAR(N)} matching {@code value.length()} exactly. */ + private static RexNode eqLit(String value) { + RelDataType t = TYPE_FACTORY.createSqlType(SqlTypeName.CHAR, value.length()); + RexInputRef ref = REX_BUILDER.makeInputRef(t, 0); + RexNode lit = REX_BUILDER.makeLiteral(value, t, false); + return REX_BUILDER.makeCall(SqlStdOperatorTable.EQUALS, ref, lit); + } + + @Test void charColumnPlainValueQuoted() throws Throwable { + assertThat(translate(eqLit("alpha")), is("code = 'alpha'")); + } + + @Test void charColumnValueWithApostrophe() throws Throwable { + assertThat(translate(eqLit("O'Brien")), is("code = 'O''Brien'")); + } + + @Test void charColumnValueWithMultipleApostrophes() throws Throwable { + assertThat(translate(eqLit("a''b")), is("code = 'a''''b'")); + } + + @Test void charColumnValueWithApostropheAtTheStart() throws Throwable { + assertThat(translate(eqLit("'a")), is("code = '''a'")); + } + + @Test void charColumnValueWithApostropheAtTheEnd() throws Throwable { + assertThat(translate(eqLit("a'")), is("code = 'a'''")); + } +} diff --git a/geode/src/test/java/org/apache/calcite/adapter/geode/rel/GeodeZipsTest.java b/geode/src/test/java/org/apache/calcite/adapter/geode/rel/GeodeZipsTest.java index bf42258edabc..12053da96f12 100644 --- a/geode/src/test/java/org/apache/calcite/adapter/geode/rel/GeodeZipsTest.java +++ b/geode/src/test/java/org/apache/calcite/adapter/geode/rel/GeodeZipsTest.java @@ -253,6 +253,17 @@ public void testJoin() { GeodeAssertions.query(expectedQuery)); } + @Test void testFilterWithSingleQuoteLiteral() { + String expectedQuery = "SELECT city AS city FROM /zips " + + "WHERE city = 'a''b'"; + calciteAssert() + .query("SELECT city as city " + + "FROM view WHERE city = 'a''b'") + .returnsCount(0) + .queryContains( + GeodeAssertions.query(expectedQuery)); + } + @Test void testSqlSingleStringWhereFilter() { String expectedQuery = "SELECT state AS state FROM /zips " + "WHERE state = 'NY'"; diff --git a/pig/src/main/java/org/apache/calcite/adapter/pig/PigFilter.java b/pig/src/main/java/org/apache/calcite/adapter/pig/PigFilter.java index 9c720aeb8442..f36b0dd6f5d3 100644 --- a/pig/src/main/java/org/apache/calcite/adapter/pig/PigFilter.java +++ b/pig/src/main/java/org/apache/calcite/adapter/pig/PigFilter.java @@ -133,10 +133,11 @@ private static boolean containsOnlyConjunctions(RexNode condition) { /** * Converts a literal to a Pig Latin string literal. - * - *

      TODO: do proper literal to string conversion + escaping */ private static String getLiteralAsString(RexLiteral literal) { - return '\'' + RexLiteral.stringValue(literal) + '\''; + // Pig Latin string literals use `''` to represent a single `'` inside + // a `'...'` literal, so double any embedded `'` before wrapping + final String raw = RexLiteral.stringValue(literal); + return '\'' + (raw != null ? raw.replace("'", "''") : null) + '\''; } } diff --git a/pig/src/test/java/org/apache/calcite/adapter/pig/PigFilterLiteralEscapeTest.java b/pig/src/test/java/org/apache/calcite/adapter/pig/PigFilterLiteralEscapeTest.java new file mode 100644 index 000000000000..f90bbb3cbd70 --- /dev/null +++ b/pig/src/test/java/org/apache/calcite/adapter/pig/PigFilterLiteralEscapeTest.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.adapter.pig; + +import org.apache.calcite.jdbc.JavaTypeFactoryImpl; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexLiteral; +import org.apache.calcite.sql.type.SqlTypeName; + +import org.junit.jupiter.api.Test; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; + +/** + * Unit tests for {@link PigFilter}'s literal-to-Pig-Latin serialization. + */ +class PigFilterLiteralEscapeTest { + + private static final RelDataTypeFactory TYPE_FACTORY = new JavaTypeFactoryImpl(); + private static final RexBuilder REX_BUILDER = new RexBuilder(TYPE_FACTORY); + + private static RexLiteral charLiteral(String value) { + return (RexLiteral) REX_BUILDER.makeLiteral(value, + TYPE_FACTORY.createSqlType(SqlTypeName.CHAR, value.length()), false); + } + + private static String call(RexLiteral literal) throws Throwable { + Method m = PigFilter.class.getDeclaredMethod("getLiteralAsString", RexLiteral.class); + m.setAccessible(true); + try { + return (String) m.invoke(null, literal); + } catch (InvocationTargetException e) { + throw e.getCause(); + } + } + + @Test void plainValueQuoted() throws Throwable { + assertThat(call(charLiteral("alice")), is("'alice'")); + } + + @Test void valueWithApostrophe() throws Throwable { + assertThat(call(charLiteral("O'Brien")), is("'O''Brien'")); + } + + @Test void valueWithApostropheAtTheEnd() throws Throwable { + assertThat(call(charLiteral("a'")), is("'a'''")); + } + + @Test void valueWithApostropheAtTheStart() throws Throwable { + assertThat(call(charLiteral("'a")), is("'''a'")); + } + + @Test void valueWithMultipleApostrophes() throws Throwable { + assertThat(call(charLiteral("a''b")), is("'a''''b'")); + } + + @Test void emptyValue() throws Throwable { + assertThat(call(charLiteral("")), is("''")); + } +} diff --git a/pig/src/test/java/org/apache/calcite/test/PigAdapterTest.java b/pig/src/test/java/org/apache/calcite/test/PigAdapterTest.java index f18cb80ae87c..62b247a756b8 100644 --- a/pig/src/test/java/org/apache/calcite/test/PigAdapterTest.java +++ b/pig/src/test/java/org/apache/calcite/test/PigAdapterTest.java @@ -57,6 +57,21 @@ class PigAdapterTest extends AbstractPigTest { + "t = FILTER t BY (tc0 > 'abc');")); } + @Test void testFilterWithSingleQuote() { + // A string literal containing a single quote must be doubled per Pig Latin + // string-literal rules so it does not break out of the '...' literal in + // the generated FILTER statement. + CalciteAssert.that() + .with(MODEL) + .query("select * from \"t\" where \"tc0\" = 'a''b'") + .runs() + .queryContains( + pigScriptChecker("t = LOAD '" + + getFullPathForTestDataFile("data.txt") + + "' USING PigStorage() AS (tc0:chararray, tc1:chararray);\n" + + "t = FILTER t BY (tc0 == 'a''b');")); + } + @Test void testImplWithMultipleFilters() { CalciteAssert.that() .with(MODEL) From c757d98a6e2ade0a4ddff845de0c6d79ddb05d8b Mon Sep 17 00:00:00 2001 From: bibi samina Date: Wed, 29 Jul 2026 14:14:39 +0530 Subject: [PATCH 420/562] [CALCITE-7668] MongoDB adapter should escape embedded quotes in field references --- .../calcite/adapter/mongodb/MongoRules.java | 6 ++-- .../adapter/mongodb/MongoAdapterTest.java | 33 +++++++++++++++++++ 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoRules.java b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoRules.java index 48e46e610d55..fe7c65db1059 100644 --- a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoRules.java +++ b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoRules.java @@ -110,7 +110,9 @@ static String maybeQuote(String s) { } static String quote(String s) { - return "'" + s + "'"; // TODO: handle embedded quotes + // Escape backslash and the single-quote delimiter so that s cannot break + // out of the quoted token when the string is parsed by BsonDocument.parse. + return "'" + s.replace("\\", "\\\\").replace("'", "\\'") + "'"; } private static boolean needsQuote(String s) { @@ -181,7 +183,7 @@ protected RexToMongoTranslator(JavaTypeFactory typeFactory, @Override public String visitCall(RexCall call) { String name = isItem(call); if (name != null) { - return "'$" + name + "'"; + return quote("$" + name); } final List strings = visitList(call.operands); if (call.getKind() == SqlKind.CAST) { diff --git a/mongodb/src/test/java/org/apache/calcite/adapter/mongodb/MongoAdapterTest.java b/mongodb/src/test/java/org/apache/calcite/adapter/mongodb/MongoAdapterTest.java index 7c79718e7984..c4fef104f8b5 100644 --- a/mongodb/src/test/java/org/apache/calcite/adapter/mongodb/MongoAdapterTest.java +++ b/mongodb/src/test/java/org/apache/calcite/adapter/mongodb/MongoAdapterTest.java @@ -86,6 +86,13 @@ public class MongoAdapterTest implements SchemaFactory { /** Number of records in local file. */ protected static final int ZIPS_SIZE = 149; + /** Field of the "datatypes" collection whose name contains a single quote + * and characters that would be pipeline syntax if it were not escaped. */ + private static final String QUOTED_FIELD = "x', injected: {$literal: 1}, y: 'z"; + + /** Field of the "datatypes" collection whose name ends with a backslash. */ + private static final String BACKSLASH_FIELD = "a\\"; + @RegisterExtension public static final MongoDatabasePolicy POLICY = MongoDatabasePolicy.create(); @@ -119,6 +126,8 @@ public static void setUp() throws Exception { doc.put("ownerId", new BsonString("531e7789e4b0853ddb861313")); doc.put("arr", new BsonArray(Arrays.asList(new BsonString("a"), new BsonString("b")))); doc.put("binaryData", new BsonBinary("binaryData".getBytes(StandardCharsets.UTF_8))); + doc.put(QUOTED_FIELD, new BsonString("quoted")); + doc.put(BACKSLASH_FIELD, new BsonString("backslash")); datatypes.insertOne(doc); schema = new MongoSchema(database); @@ -743,6 +752,30 @@ private void checkPredicate(int expected, String q) { .returnsUnordered("EXPR$0=[a, b]"); } + /** A field name that contains a single quote or a backslash must not be able + * to break out of the quoted token in the generated pipeline and add stage + * fields of its own. + * + *

      The expected values were validated against a real MongoDB instance: + * without the escaping the first query parses as + * {@code {$project: {C: '$x', injected: {$literal: 1}, y: 'z'}}} and the + * injected field breaks the pipeline, so the query fails. */ + @Test void testItemKeyWithEmbeddedQuote() { + assertModel(MODEL) + .query("select cast(_MAP['x'', injected: {$literal: 1}, y: ''z'] as varchar) as c\n" + + "from \"mongo_raw\".\"datatypes\"") + .returnsUnordered("C=quoted") + .queryContains( + mongoChecker("{$project: {C: '$x\\', injected: {$literal: 1}, y: \\'z'}}")); + + assertModel(MODEL) + .query("select cast(_MAP['a\\'] as varchar) as c\n" + + "from \"mongo_raw\".\"datatypes\"") + .returnsUnordered("C=backslash") + .queryContains( + mongoChecker("{$project: {C: '$a\\\\'}}")); + } + /** Test case for * [CALCITE-665] * ClassCastException in MongoDB adapter. */ From 843db1e0fc1fdd7751f353bee4eb1dc2ed9cabdb Mon Sep 17 00:00:00 2001 From: Jensen Date: Tue, 28 Jul 2026 10:31:06 +0800 Subject: [PATCH 421/562] Revert "[CALCITE-7592] Add expression support for FETCH" This reverts commit 711d46417d31b3370947dc63f0c793e0973673c9. --- core/src/main/codegen/templates/Parser.jj | 20 +- .../calcite/adapter/enumerable/EnumUtils.java | 7 +- .../adapter/enumerable/EnumerableLimit.java | 20 +- .../enumerable/EnumerableLimitSort.java | 6 +- .../enumerable/EnumerableMergeUnionRule.java | 9 +- .../enumerable/RexToLixTranslator.java | 16 + .../apache/calcite/interpreter/SortNode.java | 102 +----- .../rel/metadata/RelMdMaxRowCount.java | 15 +- .../rel/metadata/RelMdMinRowCount.java | 17 +- .../calcite/rel/metadata/RelMdRowCount.java | 12 +- .../calcite/rel/metadata/RelMdUtil.java | 15 +- .../rel/rel2sql/RelToSqlConverter.java | 12 +- .../calcite/rel/rules/MeasureRules.java | 4 +- .../calcite/rel/rules/PruneEmptyRules.java | 6 +- .../rel/rules/SortJoinTransposeRule.java | 4 +- .../rel/rules/SortRemoveRedundantRule.java | 3 - .../rel/rules/SortUnionTransposeRule.java | 9 +- .../java/org/apache/calcite/rex/RexUtil.java | 80 ----- .../calcite/runtime/CalciteResource.java | 9 - .../org/apache/calcite/sql/SqlDialect.java | 34 +- .../calcite/sql/dialect/SqliteSqlDialect.java | 2 +- .../calcite/sql/fun/SqlCastFunction.java | 2 +- .../calcite/sql/type/SqlTypeFactoryImpl.java | 31 +- .../apache/calcite/sql/type/SqlTypeUtil.java | 15 +- .../sql/validate/SqlValidatorImpl.java | 29 -- .../sql2rel/CorrelateProjectExtractor.java | 77 +++- .../calcite/sql2rel/RelDecorrelator.java | 29 +- .../sql2rel/TopDownGeneralDecorrelator.java | 35 +- .../org/apache/calcite/tools/RelBuilder.java | 52 +-- .../runtime/CalciteResource.properties | 3 - .../adapter/enumerable/EnumUtilsTest.java | 11 - .../rel/rel2sql/RelToSqlConverterTest.java | 82 +---- .../apache/calcite/rex/RexProgramTest.java | 30 -- .../calcite/sql/type/SqlTypeFactoryTest.java | 36 ++ .../CorrelateProjectExtractorTest.java | 72 ++++ .../calcite/sql2rel/RelDecorrelatorTest.java | 99 ++++- .../org/apache/calcite/test/JdbcTest.java | 338 ------------------ .../apache/calcite/test/RelBuilderTest.java | 105 ------ .../apache/calcite/test/RelMetadataTest.java | 33 +- .../apache/calcite/test/RelOptRulesTest.java | 81 +---- .../calcite/test/SqlToRelConverterTest.java | 9 - .../apache/calcite/test/SqlValidatorTest.java | 26 +- .../enumerable/EnumerableMergeUnionTest.java | 30 -- .../apache/calcite/test/RelOptRulesTest.xml | 217 +---------- .../calcite/test/SqlToRelConverterTest.xml | 52 ++- core/src/test/resources/sql/fetch.iq | 183 ---------- core/src/test/resources/sql/lateral.iq | 94 +++++ .../org/apache/calcite/test/ServerTest.java | 37 -- site/_docs/reference.md | 9 +- .../calcite/sql/parser/SqlParserTest.java | 19 - .../apache/calcite/test/SqlOperatorTest.java | 95 ++++- 51 files changed, 629 insertions(+), 1704 deletions(-) delete mode 100644 core/src/test/resources/sql/fetch.iq diff --git a/core/src/main/codegen/templates/Parser.jj b/core/src/main/codegen/templates/Parser.jj index ce69124c4b5c..7246e6084211 100644 --- a/core/src/main/codegen/templates/Parser.jj +++ b/core/src/main/codegen/templates/Parser.jj @@ -709,7 +709,7 @@ SqlNode ExprOrJoinOrOrderedQuery(ExprContext exprContext) : * * 

        *    [ OFFSET start { ROW | ROWS } ]
- *    [ FETCH { FIRST | NEXT } [ count | (expression) ] { ROW | ROWS } ONLY ]
      + * [ FETCH { FIRST | NEXT } [ count ] { ROW | ROWS } ONLY ] *
      */ SqlNode OrderedQueryOrExpr(ExprContext exprContext) : @@ -796,26 +796,10 @@ void FetchClause(SqlNode[] offsetFetch) : { // SQL:2008-style syntax. "OFFSET ... FETCH ...". // If you specify both LIMIT and FETCH, FETCH wins. - ( | ) offsetFetch[1] = FetchCount() + ( | ) offsetFetch[1] = UnsignedNumericLiteralOrParam() ( | ) } -/** - * Parses the row count of a FETCH clause. Expressions must be parenthesized. - */ -SqlNode FetchCount() : -{ - final SqlNode e; -} -{ - ( - e = UnsignedNumericLiteralOrParam() - | - e = Expression(ExprContext.ACCEPT_NON_QUERY) - ) - { return e; } -} - /** * Parses a LIMIT clause in an ORDER BY expression. */ diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java index 71dc7e602624..80c46ec3f9ea 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java @@ -116,7 +116,7 @@ private EnumUtils() {} /** Converts a FETCH or OFFSET runtime value to {@link BigDecimal}. * *

      The value must be numeric and non-negative. */ - public static BigDecimal numberToBigDecimal(@Nullable Object value, String kind) { + public static BigDecimal numberToBigDecimal(Object value, String kind) { return numberToBigDecimal(value, kind, FetchOffsetRoundingPolicy.NONE); } @@ -124,11 +124,8 @@ public static BigDecimal numberToBigDecimal(@Nullable Object value, String kind) * *

      The value must be numeric and non-negative. The result is adjusted by * the configured rounding policy. */ - public static BigDecimal numberToBigDecimal(@Nullable Object value, String kind, + public static BigDecimal numberToBigDecimal(Object value, String kind, FetchOffsetRoundingPolicy roundingPolicy) { - if (value == null) { - throw new IllegalArgumentException(kind + " expression evaluated to NULL"); - } if (!(value instanceof Number)) { throw new IllegalArgumentException(kind + " must be a number"); } diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimit.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimit.java index de1f94d562d5..02fd54bdad86 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimit.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimit.java @@ -106,15 +106,13 @@ public static EnumerableLimit create(final RelNode input, @Nullable RexNode offs v = builder.append("offset", Expressions.call(BuiltInMethod.SKIP_BIG_DECIMAL.method, v, - getExpression(offset, "OFFSET", implementor, builder, - roundingPolicyExp, false))); + getExpression(offset, "OFFSET", roundingPolicyExp))); } if (fetch != null) { v = builder.append("fetch", Expressions.call(BuiltInMethod.TAKE_BIG_DECIMAL.method, v, - getExpression(fetch, "FETCH", implementor, builder, - roundingPolicyExp, true))); + getExpression(fetch, "FETCH", roundingPolicyExp))); } builder.add(Expressions.return_(null, v)); @@ -122,8 +120,7 @@ public static EnumerableLimit create(final RelNode input, @Nullable RexNode offs } static Expression getExpression(RexNode rexNode, String kind, - EnumerableRelImplementor implementor, BlockBuilder builder, - Expression roundingPolicy, boolean translateExpression) { + Expression roundingPolicy) { final Expression value; if (rexNode instanceof RexDynamicParam) { final RexDynamicParam param = (RexDynamicParam) rexNode; @@ -131,17 +128,8 @@ static Expression getExpression(RexNode rexNode, String kind, Expressions.call(DataContext.ROOT, BuiltInMethod.DATA_CONTEXT_GET.method, Expressions.constant("?" + param.getIndex())); - } else if (rexNode instanceof RexLiteral) { - value = Expressions.constant(RexLiteral.bigDecimalValue(rexNode)); } else { - if (!translateExpression) { - throw new IllegalArgumentException(kind + " must be a literal or dynamic parameter"); - } - - value = - RexToLixTranslator.forAggregation(implementor.getTypeFactory(), - builder, null, implementor.getConformance()) - .translate(rexNode); + value = Expressions.constant(RexLiteral.bigDecimalValue(rexNode)); } return Expressions.call( BuiltInMethod.NUMBER_TO_BIG_DECIMAL_LIMIT.method, diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimitSort.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimitSort.java index 97d9fd8169b7..325fe687ba4d 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimitSort.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimitSort.java @@ -104,16 +104,14 @@ public static EnumerableLimitSort create( if (this.fetch == null) { fetchVal = Expressions.constant(BigDecimal.valueOf(Integer.MAX_VALUE)); } else { - fetchVal = - getExpression(this.fetch, "FETCH", implementor, builder, roundingPolicyExp, true); + fetchVal = getExpression(this.fetch, "FETCH", roundingPolicyExp); } final Expression offsetVal; if (this.offset == null) { offsetVal = Expressions.constant(BigDecimal.ZERO); } else { - offsetVal = - getExpression(this.offset, "OFFSET", implementor, builder, roundingPolicyExp, false); + offsetVal = getExpression(this.offset, "OFFSET", roundingPolicyExp); } builder.add( diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeUnionRule.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeUnionRule.java index 57f864794aa9..7d47e639b78e 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeUnionRule.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeUnionRule.java @@ -29,7 +29,6 @@ import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; -import org.apache.calcite.rex.RexUtil; import org.apache.calcite.tools.RelBuilder; import org.apache.calcite.util.ImmutableBitSet; @@ -89,13 +88,9 @@ public EnumerableMergeUnionRule(Config config) { // Push down sort limit, if possible. RexNode inputFetch = null; if (sort.fetch != null) { - final boolean safeToReevaluate = - RexUtil.isDeterministic(sort.fetch); - if (sort.offset == null && safeToReevaluate) { + if (sort.offset == null) { inputFetch = sort.fetch; - } else if (safeToReevaluate - && sort.fetch instanceof RexLiteral - && sort.offset instanceof RexLiteral) { + } else if (sort.fetch instanceof RexLiteral && sort.offset instanceof RexLiteral) { inputFetch = call.builder().literal(RexLiteral.bigDecimalValue(sort.fetch) .add(RexLiteral.bigDecimalValue(sort.offset))); diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java index 5a1e0fee2082..a39146ac754c 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java @@ -341,6 +341,16 @@ Expression translateCast( return expressionHandlingSafe(convert3, safe, targetType); } + /** Returns whether every runtime value of {@code type} is null. + * This holds for the NULL type, which describes untyped NULL literals, + * and for the UNKNOWN type, e.g. the element type inferred for the + * no-argument array constructor ARRAY(). Such values are never + * created, but the code generated needs to typecheck. */ + private static boolean valueIsAlwaysNull(RelDataType type) { + SqlTypeName typeName = type.getSqlTypeName(); + return typeName == SqlTypeName.UNKNOWN || typeName == SqlTypeName.NULL; + } + private Expression getConvertExpression( RelDataType sourceType, RelDataType targetType, @@ -366,6 +376,9 @@ private Expression getConvertExpression( } if (targetType.getSqlTypeName() == SqlTypeName.ROW) { + if (valueIsAlwaysNull(sourceType)) { + return Expressions.constant(null); + } assert sourceType.getSqlTypeName() == SqlTypeName.ROW; List targetTypes = targetType.getFieldList(); List sourceTypes = sourceType.getFieldList(); @@ -397,6 +410,9 @@ private Expression getConvertExpression( switch (targetType.getSqlTypeName()) { case ARRAY: case MULTISET: + if (valueIsAlwaysNull(sourceType)) { + return Expressions.constant(null); + } final RelDataType sourceDataType = sourceType.getComponentType(); final RelDataType targetDataType = targetType.getComponentType(); assert sourceDataType != null; diff --git a/core/src/main/java/org/apache/calcite/interpreter/SortNode.java b/core/src/main/java/org/apache/calcite/interpreter/SortNode.java index 0f393a3e68d8..71d9f2b22e42 100644 --- a/core/src/main/java/org/apache/calcite/interpreter/SortNode.java +++ b/core/src/main/java/org/apache/calcite/interpreter/SortNode.java @@ -16,22 +16,14 @@ */ package org.apache.calcite.interpreter; -import org.apache.calcite.adapter.enumerable.EnumUtils; -import org.apache.calcite.adapter.enumerable.EnumerableRelImplementor; -import org.apache.calcite.adapter.enumerable.FetchOffsetRoundingPolicy; import org.apache.calcite.rel.RelFieldCollation; import org.apache.calcite.rel.core.Sort; import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; import org.apache.calcite.util.Util; -import com.google.common.collect.ImmutableList; import com.google.common.collect.Ordering; -import org.checkerframework.checker.nullness.qual.Nullable; - -import java.math.BigDecimal; -import java.math.RoundingMode; import java.util.ArrayList; import java.util.Comparator; import java.util.List; @@ -43,57 +35,37 @@ * {@link org.apache.calcite.rel.core.Sort}. */ public class SortNode extends AbstractSingleNode { - private final @Nullable Scalar offsetScalar; - private final @Nullable Context offsetContext; - private final @Nullable Scalar fetchScalar; - private final @Nullable Context fetchContext; - private final FetchOffsetRoundingPolicy fetchOffsetRoundingPolicy; - public SortNode(Compiler compiler, Sort rel) { super(compiler, rel); - if (rel.offset != null && !(rel.offset instanceof RexLiteral)) { - this.offsetScalar = compiler.compile(ImmutableList.of(rel.offset), null); - this.offsetContext = compiler.createContext(); - } else { - this.offsetScalar = null; - this.offsetContext = null; - } - if (rel.fetch != null && !(rel.fetch instanceof RexLiteral)) { - this.fetchScalar = compiler.compile(ImmutableList.of(rel.fetch), null); - this.fetchContext = compiler.createContext(); - } else { - this.fetchScalar = null; - this.fetchContext = null; - } - final Object roundingPolicy = compiler.getDataContext() - .get(EnumerableRelImplementor.FETCH_OFFSET_ROUNDING_POLICY); - this.fetchOffsetRoundingPolicy = - roundingPolicy instanceof FetchOffsetRoundingPolicy - ? (FetchOffsetRoundingPolicy) roundingPolicy - : FetchOffsetRoundingPolicy.NONE; + } + + private static int getValueAsInt(RexNode node) { + return requireNonNull(((RexLiteral) node).getValueAs(Integer.class), + () -> "getValueAs(Integer.class) for " + node); } @Override public void run() throws InterruptedException { - final BigDecimal offset = getOffset(); - final @Nullable BigDecimal fetch = getFetch(); + final int offset = + rel.offset == null + ? 0 + : getValueAsInt(rel.offset); + final int fetch = + rel.fetch == null + ? -1 + : getValueAsInt(rel.fetch); // In pure limit mode. No sort required. Row row; loop: if (rel.getCollation().getFieldCollations().isEmpty()) { - BigDecimal skipped = BigDecimal.ZERO; - while (skipped.compareTo(offset) < 0) { + for (int i = 0; i < offset; i++) { row = source.receive(); if (row == null) { break loop; } - skipped = skipped.add(BigDecimal.ONE); } - if (fetch != null) { - BigDecimal fetched = BigDecimal.ZERO; - while (fetched.compareTo(fetch) < 0 - && (row = source.receive()) != null) { + if (fetch >= 0) { + for (int i = 0; i < fetch && (row = source.receive()) != null; i++) { sink.send(row); - fetched = fetched.add(BigDecimal.ONE); } } else { while ((row = source.receive()) != null) { @@ -107,15 +79,10 @@ public SortNode(Compiler compiler, Sort rel) { list.add(row); } list.sort(comparator()); - final int start = offset.compareTo(BigDecimal.valueOf(list.size())) >= 0 - ? list.size() - : rowCount(offset); - final int available = list.size() - start; - final int end = fetch == null - || fetch.compareTo(BigDecimal.valueOf(available)) >= 0 + final int end = fetch < 0 || offset + fetch > list.size() ? list.size() - : start + rowCount(fetch); - for (int i = start; i < end; i++) { + : offset + fetch; + for (int i = offset; i < end; i++) { sink.send(list.get(i)); } } @@ -149,35 +116,4 @@ private static Comparator comparator(RelFieldCollation fieldCollation) { }; } } - - private @Nullable BigDecimal getFetch() { - if (rel.fetch == null) { - return null; - } - return getValue(rel.fetch, fetchScalar, fetchContext, "FETCH"); - } - - private BigDecimal getOffset() { - if (rel.offset == null) { - return BigDecimal.ZERO; - } - return getValue(rel.offset, offsetScalar, offsetContext, "OFFSET"); - } - - private BigDecimal getValue(RexNode node, @Nullable Scalar scalar, - @Nullable Context context, String kind) { - final @Nullable Object value; - if (node instanceof RexLiteral) { - value = RexLiteral.bigDecimalValue(node); - } else { - value = - requireNonNull(scalar, () -> kind + " scalar") - .execute(requireNonNull(context, () -> kind + " context")); - } - return EnumUtils.numberToBigDecimal(value, kind, fetchOffsetRoundingPolicy); - } - - private static int rowCount(BigDecimal value) { - return value.setScale(0, RoundingMode.CEILING).intValueExact(); - } } diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdMaxRowCount.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdMaxRowCount.java index 869f1ad50e78..e728c22e1ede 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdMaxRowCount.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdMaxRowCount.java @@ -117,12 +117,10 @@ public Double getMaxRowCount(Sort rel, RelMetadataQuery mq) { rowCount = Double.POSITIVE_INFINITY; } - final double offset = - literalValueApproximatedByDouble(rel.offset, 0D); + final double offset = literalValueApproximatedByDouble(rel.offset, 0D); rowCount = Math.max(rowCount - offset, 0D); - final double limit = - literalValueApproximatedByDouble(rel.fetch, rowCount); + final double limit = literalValueApproximatedByDouble(rel.fetch, rowCount); return limit < rowCount ? limit : rowCount; } @@ -132,12 +130,10 @@ public Double getMaxRowCount(EnumerableLimit rel, RelMetadataQuery mq) { rowCount = Double.POSITIVE_INFINITY; } - final double offset = - literalValueApproximatedByDouble(rel.offset, 0D); + final double offset = literalValueApproximatedByDouble(rel.offset, 0D); rowCount = Math.max(rowCount - offset, 0D); - final double limit = - literalValueApproximatedByDouble(rel.fetch, rowCount); + final double limit = literalValueApproximatedByDouble(rel.fetch, rowCount); return limit < rowCount ? limit : rowCount; } @@ -218,8 +214,7 @@ public Double getMaxRowCount(RelSubset rel, RelMetadataQuery mq) { if (node instanceof Sort) { Sort sort = (Sort) node; if (sort.fetch instanceof RexLiteral) { - return literalValueApproximatedByDouble(sort.fetch, - Double.POSITIVE_INFINITY); + return literalValueApproximatedByDouble(sort.fetch, Double.POSITIVE_INFINITY); } } } diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdMinRowCount.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdMinRowCount.java index 2cb710f39808..869d34333547 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdMinRowCount.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdMinRowCount.java @@ -116,13 +116,10 @@ public Double getMinRowCount(Sort rel, RelMetadataQuery mq) { rowCount = 0D; } - final double offset = - literalValueApproximatedByDouble(rel.offset, 0D); + final double offset = literalValueApproximatedByDouble(rel.offset, 0D); rowCount = Math.max(rowCount - offset, 0D); - final double limit = - literalValueApproximatedByDouble(rel.fetch, - rel.fetch == null ? rowCount : 0D); + final double limit = literalValueApproximatedByDouble(rel.fetch, rowCount); return limit < rowCount ? limit : rowCount; } @@ -132,13 +129,10 @@ public Double getMinRowCount(EnumerableLimit rel, RelMetadataQuery mq) { rowCount = 0D; } - final double offset = - literalValueApproximatedByDouble(rel.offset, 0D); + final double offset = literalValueApproximatedByDouble(rel.offset, 0D); rowCount = Math.max(rowCount - offset, 0D); - final double limit = - literalValueApproximatedByDouble(rel.fetch, - rel.fetch == null ? rowCount : 0D); + final double limit = literalValueApproximatedByDouble(rel.fetch, rowCount); return limit < rowCount ? limit : rowCount; } @@ -180,8 +174,7 @@ public Double getMinRowCount(RelSubset rel, RelMetadataQuery mq) { if (node instanceof Sort) { Sort sort = (Sort) node; if (sort.fetch instanceof RexLiteral) { - return literalValueApproximatedByDouble(sort.fetch, - Double.POSITIVE_INFINITY); + return literalValueApproximatedByDouble(sort.fetch, Double.POSITIVE_INFINITY); } } } diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdRowCount.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdRowCount.java index 3e7824e1aac6..e83f4c1da9f4 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdRowCount.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdRowCount.java @@ -165,12 +165,10 @@ public Double getRowCount(Calc rel, RelMetadataQuery mq) { return null; } - final double offset = - literalValueApproximatedByDouble(rel.offset, 0D); + final double offset = literalValueApproximatedByDouble(rel.offset, 0D); rowCount = Math.max(rowCount - offset, 0D); - final double limit = - literalValueApproximatedByDouble(rel.fetch, rowCount); + final double limit = literalValueApproximatedByDouble(rel.fetch, rowCount); return limit < rowCount ? limit : rowCount; } @@ -180,12 +178,10 @@ public Double getRowCount(Calc rel, RelMetadataQuery mq) { return null; } - final double offset = - literalValueApproximatedByDouble(rel.offset, 0D); + final double offset = literalValueApproximatedByDouble(rel.offset, 0D); rowCount = Math.max(rowCount - offset, 0D); - final double limit = - literalValueApproximatedByDouble(rel.fetch, rowCount); + final double limit = literalValueApproximatedByDouble(rel.fetch, rowCount); return limit < rowCount ? limit : rowCount; } diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdUtil.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdUtil.java index 5b096289382e..1f6502243626 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdUtil.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdUtil.java @@ -25,7 +25,6 @@ import org.apache.calcite.rel.core.JoinRelType; import org.apache.calcite.rel.core.Minus; import org.apache.calcite.rel.core.Project; -import org.apache.calcite.rel.core.Sort; import org.apache.calcite.rel.core.Union; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexCall; @@ -57,7 +56,6 @@ import java.util.ArrayList; import java.util.LinkedHashSet; import java.util.List; -import java.util.Objects; import java.util.Set; import static com.google.common.base.Preconditions.checkArgument; @@ -485,9 +483,6 @@ public static double literalValueApproximatedByDouble(@Nullable RexNode node, throw new IllegalArgumentException( "literal value " + number + " cannot be converted to BigDecimal"); } - if (decimal.signum() < 0) { - return defaultValue; - } if (decimal.abs().compareTo(BigDecimal.valueOf(Double.MAX_VALUE)) > 0) { throw new IllegalArgumentException( "literal value " + decimal + " exceeds double range"); @@ -1048,16 +1043,8 @@ private static boolean alreadySmaller(RelMetadataQuery mq, RelNode input, if (fetch == null) { return true; } - final RelNode strippedInput = input.stripped(); - if (strippedInput instanceof Sort) { - final Sort sort = (Sort) strippedInput; - if (Objects.equals(offset, sort.offset) - && Objects.equals(fetch, sort.fetch)) { - return true; - } - } final Double rowCount = mq.getMaxRowCount(input); - if (rowCount == null || offset instanceof RexDynamicParam || !(fetch instanceof RexLiteral)) { + if (rowCount == null || offset instanceof RexDynamicParam || fetch instanceof RexDynamicParam) { // Cannot be determined return false; } diff --git a/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java b/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java index 718e6b1cb965..8871f24d3e86 100644 --- a/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java +++ b/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java @@ -59,7 +59,6 @@ import org.apache.calcite.rex.RexLocalRef; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexProgram; -import org.apache.calcite.rex.RexUtil; import org.apache.calcite.sql.JoinConditionType; import org.apache.calcite.sql.JoinType; import org.apache.calcite.sql.SqlAsofJoin; @@ -1228,7 +1227,7 @@ public Result visit(Sort e) { sqlSelect.setOffset(offset); } if (e.fetch != null) { - SqlNode fetch = toSqlFetch(e, builder.context); + SqlNode fetch = builder.context.toSql(null, e.fetch); sqlSelect.setFetch(fetch); } return result(sqlSelect, ImmutableList.of(Clause.ORDER_BY), e, null); @@ -1286,20 +1285,13 @@ public Result visit(Sort e) { * The builder must have been created with OFFSET and FETCH clauses. */ void offsetFetch(Sort e, Builder builder) { if (e.fetch != null) { - builder.setFetch(toSqlFetch(e, builder.context)); + builder.setFetch(builder.context.toSql(null, e.fetch)); } if (e.offset != null) { builder.setOffset(builder.context.toSql(null, e.offset)); } } - private static SqlNode toSqlFetch(Sort sort, Context context) { - final RexNode fetch = requireNonNull(sort.fetch, "fetch"); - final @Nullable RexLiteral reduced = - RexUtil.reduceFetchToLiteral(sort.getCluster(), fetch); - return context.toSql(null, reduced == null ? fetch : reduced); - } - public boolean hasTrickyRollup(Sort e, Aggregate aggregate) { return !dialect.supportsAggregateFunction(SqlKind.ROLLUP) && dialect.supportsGroupByWithRollup() diff --git a/core/src/main/java/org/apache/calcite/rel/rules/MeasureRules.java b/core/src/main/java/org/apache/calcite/rel/rules/MeasureRules.java index f69810a14d42..037a4d605459 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/MeasureRules.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/MeasureRules.java @@ -30,6 +30,7 @@ import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexCorrelVariable; import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexShuttle; import org.apache.calcite.rex.RexUtil; @@ -507,7 +508,8 @@ protected ProjectSortMeasureRule(ProjectSortMeasureRuleConfig config) { relBuilder.push(sort.getInput()) .projectPlus(map.keySet()) - .sortLimit(sort.offset, sort.fetch, + .sortLimit(sort.offset == null ? 0 : RexLiteral.numberValue(sort.offset), + sort.fetch == null ? -1 : RexLiteral.numberValue(sort.fetch), sort.getSortExps()) .project(newProjects); call.transformTo(relBuilder.build()); diff --git a/core/src/main/java/org/apache/calcite/rel/rules/PruneEmptyRules.java b/core/src/main/java/org/apache/calcite/rel/rules/PruneEmptyRules.java index 95331c69a414..02b0bd8af1b2 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/PruneEmptyRules.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/PruneEmptyRules.java @@ -42,6 +42,7 @@ import org.apache.calcite.rel.logical.LogicalValues; import org.apache.calcite.rel.metadata.RelMdUtil; import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rex.RexDynamicParam; import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; import org.apache.calcite.tools.RelBuilder; @@ -535,8 +536,9 @@ public interface SortFetchZeroRuleConfig extends PruneEmptyRule.Config { return new RemoveEmptySingleRule(this) { @Override public boolean matches(final RelOptRuleCall call) { Sort sort = call.rel(0); - return sort.fetch instanceof RexLiteral - && BigDecimal.ZERO.equals(RexLiteral.bigDecimalValue(sort.fetch)); + return sort.fetch != null + && !(sort.fetch instanceof RexDynamicParam) + && RexLiteral.bigDecimalValue(sort.fetch).equals(BigDecimal.ZERO); } }; } diff --git a/core/src/main/java/org/apache/calcite/rel/rules/SortJoinTransposeRule.java b/core/src/main/java/org/apache/calcite/rel/rules/SortJoinTransposeRule.java index df967e56aad6..4310d6d65576 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/SortJoinTransposeRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/SortJoinTransposeRule.java @@ -105,9 +105,9 @@ public SortJoinTransposeRule(Class sortClass, final Sort sort = call.rel(0); final Join join = call.rel(1); - // The pushed fetch is calculated from literal offset and fetch values. + // Do nothing if SORT contains dynamic parameters in offset or fetch if (sort.offset instanceof RexDynamicParam - || sort.fetch != null && !(sort.fetch instanceof RexLiteral)) { + || sort.fetch instanceof RexDynamicParam) { return false; } diff --git a/core/src/main/java/org/apache/calcite/rel/rules/SortRemoveRedundantRule.java b/core/src/main/java/org/apache/calcite/rel/rules/SortRemoveRedundantRule.java index 08563cdcdb86..9bcf026fc656 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/SortRemoveRedundantRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/SortRemoveRedundantRule.java @@ -133,9 +133,6 @@ protected SortRemoveRedundantRule(final SortRemoveRedundantRule.Config config) { private static Optional getRowCountThreshold(Sort sort) { if (RelOptUtil.isLimit(sort)) { assert sort.fetch != null; - if (!(sort.fetch instanceof RexLiteral)) { - return Optional.empty(); - } final BigDecimal fetch = RexLiteral.bigDecimalValue(sort.fetch); // We don't need to deal with fetch is 0. diff --git a/core/src/main/java/org/apache/calcite/rel/rules/SortUnionTransposeRule.java b/core/src/main/java/org/apache/calcite/rel/rules/SortUnionTransposeRule.java index 93b6af657c43..416825ee926d 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/SortUnionTransposeRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/SortUnionTransposeRule.java @@ -23,7 +23,7 @@ import org.apache.calcite.rel.core.Union; import org.apache.calcite.rel.metadata.RelMdUtil; import org.apache.calcite.rel.metadata.RelMetadataQuery; -import org.apache.calcite.rex.RexUtil; +import org.apache.calcite.rex.RexDynamicParam; import org.apache.calcite.tools.RelBuilderFactory; import org.immutables.value.Value; @@ -67,14 +67,13 @@ public SortUnionTransposeRule( @Override public boolean matches(RelOptRuleCall call) { final Sort sort = call.rel(0); final Union union = call.rel(1); - // Re-evaluating a non-deterministic FETCH in every branch can produce a - // different limit from the top Sort. + // We only apply this rule if Union.all is true, Sort.offset is null and Sort.fetch is not + // a dynamic param. // There is a flag indicating if this rule should be applied when // Sort.fetch is null. return union.all && sort.offset == null - && (sort.fetch == null - || RexUtil.isDeterministic(sort.fetch)) + && !(sort.fetch instanceof RexDynamicParam) && (config.matchNullFetch() || sort.fetch != null); } diff --git a/core/src/main/java/org/apache/calcite/rex/RexUtil.java b/core/src/main/java/org/apache/calcite/rex/RexUtil.java index b592093a5aef..3604e98dfd5b 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexUtil.java +++ b/core/src/main/java/org/apache/calcite/rex/RexUtil.java @@ -19,7 +19,6 @@ import org.apache.calcite.DataContexts; import org.apache.calcite.linq4j.function.Predicate1; import org.apache.calcite.plan.PlanTooComplexError; -import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.plan.RelOptPredicateList; import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.rel.RelCollation; @@ -49,7 +48,6 @@ import org.apache.calcite.util.ControlFlowException; import org.apache.calcite.util.ImmutableBitSet; import org.apache.calcite.util.Litmus; -import org.apache.calcite.util.NumberUtil; import org.apache.calcite.util.Pair; import org.apache.calcite.util.RangeSets; import org.apache.calcite.util.Sarg; @@ -65,11 +63,9 @@ import org.apiguardian.api.API; import org.checkerframework.checker.nullness.qual.Nullable; -import java.math.BigDecimal; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -844,82 +840,6 @@ public static boolean isDeterministic(RexNode e) { } } - /** Returns whether an expression contains a dynamic function. */ - public static boolean containsDynamicFunction(RexNode e) { - try { - e.accept( - new RexVisitorImpl(true) { - @Override public Void visitCall(RexCall call) { - if (call.getOperator().isDynamicFunction()) { - throw Util.FoundOne.NULL; - } - return super.visitCall(call); - } - }); - return false; - } catch (Util.FoundOne ex) { - Util.swallow(ex, null); - return true; - } - } - - /** Returns whether an expression contains a dynamic parameter. */ - public static boolean containsDynamicParam(RexNode e) { - try { - e.accept( - new RexVisitorImpl(true) { - @Override public Void visitDynamicParam(RexDynamicParam dynamicParam) { - throw Util.FoundOne.NULL; - } - }); - return false; - } catch (Util.FoundOne ex) { - Util.swallow(ex, null); - return true; - } - } - - /** Converts a FETCH expression result to its validated canonical representation. */ - public static BigDecimal validateFetchValue(@Nullable Number value) { - if (value == null) { - throw new IllegalArgumentException("FETCH expression evaluated to NULL"); - } - final BigDecimal decimal = NumberUtil.toBigDecimal(value); - if (decimal.signum() < 0) { - throw new IllegalArgumentException("FETCH value " + value - + " is out of range; expected a non-negative value"); - } - return decimal; - } - - /** Reduces a constant FETCH expression to a validated literal. */ - public static @Nullable RexLiteral reduceFetchToLiteral( - RelOptCluster cluster, RexNode fetch) { - final RexLiteral literal; - if (fetch instanceof RexLiteral) { - literal = (RexLiteral) fetch; - } else { - if (!isConstant(fetch) - || !isDeterministic(fetch) - || containsDynamicFunction(fetch) - || containsDynamicParam(fetch)) { - return null; - } - final RexExecutor executor = - Util.first(cluster.getPlanner().getExecutor(), EXECUTOR); - final List reducedValues = new ArrayList<>(1); - executor.reduce(cluster.getRexBuilder(), - Collections.singletonList(fetch), reducedValues); - final RexNode reduced = reducedValues.get(0); - if (!(reduced instanceof RexLiteral)) { - return null; - } - literal = (RexLiteral) reduced; - } - validateFetchValue(literal.getValueAs(Number.class)); - return literal; - } - public static List retainDeterministic(List list) { List conjunctions = new ArrayList<>(); for (RexNode x : list) { diff --git a/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java b/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java index e5932b63768d..c6e1a4dbdcc5 100644 --- a/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java +++ b/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java @@ -164,15 +164,6 @@ ExInstWithCause validatorContext(int a0, int a1, @BaseMessage("Values passed to {0} operator must have compatible types") ExInst incompatibleValueType(String a0); - @BaseMessage("FETCH expression must have a numeric type; actual type is ''{0}''") - ExInst fetchExpressionMustBeNumeric(String type); - - @BaseMessage("FETCH expression cannot reference table column ''{0}''") - ExInst fetchExpressionCannotReferenceColumn(String column); - - @BaseMessage("FETCH expression evaluated to NULL") - ExInst fetchExpressionEvaluatedToNull(); - @BaseMessage("Values in expression list must have compatible types") ExInst incompatibleTypesInList(); diff --git a/core/src/main/java/org/apache/calcite/sql/SqlDialect.java b/core/src/main/java/org/apache/calcite/sql/SqlDialect.java index 164f212c6c7a..e659d1d17eb2 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlDialect.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlDialect.java @@ -1088,18 +1088,7 @@ protected static void unparseFetchUsingAnsi(SqlWriter writer, @Nullable SqlNode writer.startList(SqlWriter.FrameTypeEnum.FETCH); writer.keyword("FETCH"); writer.keyword("NEXT"); - if (fetch instanceof SqlLiteral - || fetch instanceof SqlDynamicParam) { - fetch.unparse(writer, -1, -1); - } else { - final SqlWriter.Frame expressionFrame = writer.startList("(", ")"); - if (fetch instanceof SqlCall) { - writer.getDialect().unparseCall(writer, (SqlCall) fetch, 0, 0); - } else { - fetch.unparse(writer, 0, 0); - } - writer.endList(expressionFrame); - } + fetch.unparse(writer, -1, -1); writer.keyword("ROWS"); writer.keyword("ONLY"); writer.endList(fetchFrame); @@ -1109,32 +1098,13 @@ protected static void unparseFetchUsingAnsi(SqlWriter writer, @Nullable SqlNode /** Unparses offset/fetch using "LIMIT fetch OFFSET offset" syntax. */ protected static void unparseFetchUsingLimit(SqlWriter writer, @Nullable SqlNode offset, @Nullable SqlNode fetch) { - unparseFetchUsingLimit(writer, offset, fetch, false); - } - - /** Unparses offset/fetch using "LIMIT fetch OFFSET offset" syntax, - * optionally allowing a scalar expression as fetch. */ - protected static void unparseFetchUsingLimit(SqlWriter writer, @Nullable SqlNode offset, - @Nullable SqlNode fetch, boolean allowExpression) { checkArgument(fetch != null || offset != null); - unparseLimit(writer, fetch, allowExpression); + unparseLimit(writer, fetch); unparseOffset(writer, offset); } protected static void unparseLimit(SqlWriter writer, @Nullable SqlNode fetch) { - unparseLimit(writer, fetch, false); - } - - private static void unparseLimit(SqlWriter writer, @Nullable SqlNode fetch, - boolean allowExpression) { if (fetch != null) { - if (!allowExpression - && !(fetch instanceof SqlLiteral) - && !(fetch instanceof SqlDynamicParam)) { - throw new IllegalArgumentException( - "LIMIT dialect does not support FETCH expressions that cannot " - + "be reduced to a literal"); - } writer.newlineAndIndent(); final SqlWriter.Frame fetchFrame = writer.startList(SqlWriter.FrameTypeEnum.FETCH); diff --git a/core/src/main/java/org/apache/calcite/sql/dialect/SqliteSqlDialect.java b/core/src/main/java/org/apache/calcite/sql/dialect/SqliteSqlDialect.java index f31276413600..82376ae576ab 100644 --- a/core/src/main/java/org/apache/calcite/sql/dialect/SqliteSqlDialect.java +++ b/core/src/main/java/org/apache/calcite/sql/dialect/SqliteSqlDialect.java @@ -90,7 +90,7 @@ public SqliteSqlDialect(SqlDialect.Context context) { @Override public void unparseOffsetFetch(SqlWriter writer, @Nullable SqlNode offset, @Nullable SqlNode fetch) { - unparseFetchUsingLimit(writer, offset, fetch, true); + unparseFetchUsingLimit(writer, offset, fetch); } @Override public void unparseCall(SqlWriter writer, SqlCall call, diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlCastFunction.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlCastFunction.java index b25a9b8fdd57..62ce488718ba 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlCastFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlCastFunction.java @@ -200,7 +200,7 @@ private static RelDataType createTypeWithNullabilityFromExpr(RelDataTypeFactory RelDataType valueType = createTypeWithNullabilityFromExpr( typeFactory, expressionValueType, targetValueType, safe); - SqlTypeUtil.createMapType(typeFactory, keyType, valueType, isNullable); + return SqlTypeUtil.createMapType(typeFactory, keyType, valueType, isNullable); } return typeFactory.createTypeWithNullability(targetType, isNullable); diff --git a/core/src/main/java/org/apache/calcite/sql/type/SqlTypeFactoryImpl.java b/core/src/main/java/org/apache/calcite/sql/type/SqlTypeFactoryImpl.java index d0ccab7dfd2f..544f9ca708da 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/SqlTypeFactoryImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/type/SqlTypeFactoryImpl.java @@ -28,6 +28,7 @@ import org.checkerframework.checker.nullness.qual.Nullable; import java.nio.charset.Charset; +import java.util.ArrayList; import java.util.List; import static com.google.common.base.Preconditions.checkArgument; @@ -191,11 +192,33 @@ public SqlTypeFactoryImpl(RelDataTypeSystem typeSystem) { RelDataType type0 = types.get(0); if (type0.getSqlTypeName() != null) { - RelDataType resultType = leastRestrictiveSqlType(types); - if (resultType != null) { - return resultType; + // First preprocess to filter out UNKNOWN types. + // leastRestrictive() can be thought as a form of type unification, + // and UNKNOWN behaves like an unbound type variable: it unifies with any type + // without constraining the result. + // Note that UNKNOWN can be nullable, so this information is carried over to the result. + List knownTypes = new ArrayList<>(types.size()); + // True if any UNKNOWN type is nullable + boolean anyUnknownIsNullable = false; + for (RelDataType type : types) { + if (type.getSqlTypeName() == SqlTypeName.UNKNOWN) { + anyUnknownIsNullable |= type.isNullable(); + } else { + knownTypes.add(type); + } } - return leastRestrictiveByCast(types, mappingRule); + if (knownTypes.isEmpty()) { + // All types are unknown + return createTypeWithNullability(createUnknownType(), anyUnknownIsNullable); + } + RelDataType resultType = leastRestrictiveSqlType(knownTypes); + if (resultType == null) { + resultType = leastRestrictiveByCast(knownTypes, mappingRule); + } + if (resultType != null && anyUnknownIsNullable) { + resultType = createTypeWithNullability(resultType, true); + } + return resultType; } return super.leastRestrictive(types, mappingRule); diff --git a/core/src/main/java/org/apache/calcite/sql/type/SqlTypeUtil.java b/core/src/main/java/org/apache/calcite/sql/type/SqlTypeUtil.java index 02988668287b..5de00a50b4c8 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/SqlTypeUtil.java +++ b/core/src/main/java/org/apache/calcite/sql/type/SqlTypeUtil.java @@ -1128,8 +1128,9 @@ public static boolean canCastFrom( requireNonNull(fromType.getComponentType(), "componentType"), typeMappingRule); } else if (fromType.getFamily() == SqlTypeFamily.CHARACTER - || fromType.getSqlTypeName() == SqlTypeName.NULL) { - // Cast from NULL or string to array is legal + || fromType.getSqlTypeName() == SqlTypeName.NULL + || fromType.getSqlTypeName() == SqlTypeName.UNKNOWN) { + // Cast from NULL, UNKNOWN, or string to array is legal return true; } return false; @@ -1145,8 +1146,9 @@ && canCastFrom( requireNonNull(fromType.getValueType(), "valueType"), typeMappingRule); } else if (fromType.getFamily() == SqlTypeFamily.CHARACTER - || fromType.getSqlTypeName() == SqlTypeName.NULL) { - // Cast from NULL or string to map is legal + || fromType.getSqlTypeName() == SqlTypeName.NULL + || fromType.getSqlTypeName() == SqlTypeName.UNKNOWN) { + // Cast from NULL, UNKNOWN, or string to map is legal return true; } return false; @@ -1164,7 +1166,10 @@ && canCastFrom( toType, fromType.getFieldList().get(0).getType(), typeMappingRule); } else if (toTypeName == SqlTypeName.ROW) { if (fromTypeName != SqlTypeName.ROW) { - return fromTypeName == SqlTypeName.NULL; + // UNKNOWN can arise e.g. as the element type inferred for the + // no-argument array constructor ARRAY() + return fromTypeName == SqlTypeName.NULL + || fromTypeName == SqlTypeName.UNKNOWN; } int n = toType.getFieldCount(); if (fromType.getFieldCount() != n) { diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java index 02a044820f49..023de1b77f77 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java @@ -1771,34 +1771,6 @@ private void handleOffsetFetch(@Nullable SqlNode offset, @Nullable SqlNode fetch } } - private void validateFetchExpression(@Nullable SqlNode fetch) { - if (fetch == null || fetch instanceof SqlDynamicParam) { - return; - } - if (SqlUtil.isNullLiteral(fetch, true)) { - throw newValidationError(fetch, - RESOURCE.fetchExpressionEvaluatedToNull()); - } - validateNoAggs(aggOrOverFinder, fetch, "FETCH"); - fetch.accept(new SqlBasicVisitor() { - @Override public Void visit(SqlIdentifier id) { - if (makeNullaryCall(id) != null) { - return null; - } - throw newValidationError(id, - RESOURCE.fetchExpressionCannotReferenceColumn(id.toString())); - } - }); - final SqlValidatorScope scope = getEmptyScope(); - inferUnknownTypes(typeFactory.createSqlType(SqlTypeName.DECIMAL), scope, fetch); - validateExpr(fetch, scope); - final RelDataType type = getValidatedNodeType(fetch); - if (!SqlTypeUtil.isNumeric(type)) { - throw newValidationError(fetch, - RESOURCE.fetchExpressionMustBeNumeric(type.getFullTypeString())); - } - } - /** * Performs expression rewrites which are always used unconditionally. These * rewrites massage the expression tree into a standard form so that the @@ -4497,7 +4469,6 @@ protected void validateSelect( validateWindowClause(select); validateQualifyClause(select); handleOffsetFetch(select.getOffset(), select.getFetch()); - validateFetchExpression(select.getFetch()); // Validate the SELECT clause late, because a select item might // depend on the GROUP BY list, or the window function might reference diff --git a/core/src/main/java/org/apache/calcite/sql2rel/CorrelateProjectExtractor.java b/core/src/main/java/org/apache/calcite/sql2rel/CorrelateProjectExtractor.java index 50d255c3312d..127f4e487941 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/CorrelateProjectExtractor.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/CorrelateProjectExtractor.java @@ -88,6 +88,20 @@ public CorrelateProjectExtractor(RelBuilderFactory factory) { this.builderFactory = factory; } + /** Returns whether {@code node} is a direct field access on the correlation + * variable with the specified id, such as {@code $cor0.DEPTNO}. A nested + * access such as {@code $cor0.REC.DEPTNO} is not direct. */ + private static boolean isDirectFieldAccess(RexNode node, CorrelationId id) { + if (node instanceof RexFieldAccess) { + RexFieldAccess access = (RexFieldAccess) node; + if (access.getReferenceExpr() instanceof RexCorrelVariable) { + RexCorrelVariable correlVar = (RexCorrelVariable) access.getReferenceExpr(); + return correlVar.id.equals(id); + } + } + return false; + } + @Override public RelNode visit(LogicalCorrelate correlate) { RelNode left = correlate.getLeft().accept(this); RelNode right = correlate.getRight().accept(this); @@ -95,8 +109,12 @@ public CorrelateProjectExtractor(RelBuilderFactory factory) { // Find the correlated expressions from the right side that can be moved to the left Set callsWithCorrelationInRight = findCorrelationDependentCalls(correlate.getCorrelationId(), right); + // Only direct field accesses on the correlation variable, such as + // $cor0.DEPTNO, are left in place. A nested field access, such as + // $cor0.REC.DEPTNO, is extracted boolean isTrivialCorrelation = - callsWithCorrelationInRight.stream().allMatch(exp -> exp instanceof RexFieldAccess); + callsWithCorrelationInRight.stream() + .allMatch(exp -> isDirectFieldAccess(exp, correlate.getCorrelationId())); // Early exit condition if (isTrivialCorrelation) { if (correlate.getLeft().equals(left) && correlate.getRight().equals(right)) { @@ -116,27 +134,42 @@ public CorrelateProjectExtractor(RelBuilderFactory factory) { // Transform the correlated expression from the right side to an expression over the left side builder.push(left); + ImmutableBitSet.Builder requiredColumns = ImmutableBitSet.builder(); List callsWithCorrelationOverLeft = new ArrayList<>(); for (RexNode callInRight : callsWithCorrelationInRight) { - callsWithCorrelationOverLeft.add(replaceCorrelationsWithInputRef(callInRight, builder)); + if (isDirectFieldAccess(callInRight, correlate.getCorrelationId())) { + // Direct field accesses stay in the right side and keep reading their + // original left column; that column must remain a required column. + requiredColumns.set(((RexFieldAccess) callInRight).getField().getIndex()); + } else { + callsWithCorrelationOverLeft.add(replaceCorrelationsWithInputRef(callInRight, builder)); + } } builder.projectPlus(callsWithCorrelationOverLeft); // Construct the mapping to transform the expressions in the right side based on the new // projection in the left side. Map transformMapping = new HashMap<>(); + int newFieldIndex = oldLeft; for (RexNode callInRight : callsWithCorrelationInRight) { - RexBuilder xb = builder.getRexBuilder(); - RexNode v = xb.makeCorrel(builder.peek().getRowType(), correlate.getCorrelationId()); - RexNode flatCorrelationInRight = xb.makeFieldAccess(v, oldLeft + transformMapping.size()); - transformMapping.put(callInRight, flatCorrelationInRight); + if (!isDirectFieldAccess(callInRight, correlate.getCorrelationId())) { + RexBuilder xb = builder.getRexBuilder(); + RexNode v = xb.makeCorrel(builder.peek().getRowType(), correlate.getCorrelationId()); + RexNode flatCorrelationInRight = xb.makeFieldAccess(v, newFieldIndex); + transformMapping.put(callInRight, flatCorrelationInRight); + newFieldIndex++; + } } - // Select the required fields/columns from the left side of the correlation. Based on the code - // above all these fields should be at the end of the left relational expression. + // Select the required fields/columns from the left side of the correlation: the columns + // read by the direct field accesses plus the newly projected columns, which are at the + // end of the left relational expression. List requiredFields = builder.fields( - ImmutableBitSet.range(oldLeft, oldLeft + callsWithCorrelationOverLeft.size()).asList()); + requiredColumns + .set(oldLeft, oldLeft + callsWithCorrelationOverLeft.size()) + .build() + .asList()); final int newLeft = builder.fields().size(); // Transform the expressions in the right side using the mapping constructed earlier. @@ -264,8 +297,13 @@ private static boolean isSimpleCorrelatedExpression(RexNode node, CorrelationId * +(10, $cor0.DEPTNO) -> TRUE * /(100,+(10, $cor0.DEPTNO)) -> TRUE * CAST(+(10, $cor0.DEPTNO)):INTEGER NOT NULL -> TRUE + * CASE(IS NOT NULL($cor0.PATH), $cor0.PATH, ARRAY(null:INTEGER)) -> TRUE * +($0, $cor0.DEPTNO) -> FALSE * } + * + *

      A subexpression built only from literals and dynamic parameters, such + * as {@code ARRAY(null:INTEGER)} above, is neutral: it neither qualifies nor + * disqualifies the enclosing call. */ private static class SimpleCorrelationDetector extends RexVisitorImpl<@Nullable Boolean> { @@ -284,7 +322,8 @@ private SimpleCorrelationDetector(CorrelationId corrId) { return Boolean.FALSE; } - @Override public Boolean visitCall(RexCall call) { + @Override public @Nullable Boolean visitCall(RexCall call) { + // Constant operands must not disqualify the call Boolean hasSimpleCorrelation = null; for (RexNode op : call.operands) { Boolean b = op.accept(this); @@ -292,7 +331,8 @@ private SimpleCorrelationDetector(CorrelationId corrId) { hasSimpleCorrelation = hasSimpleCorrelation == null ? b : hasSimpleCorrelation && b; } } - return hasSimpleCorrelation == null ? Boolean.FALSE : hasSimpleCorrelation; + // If unsure return null; caller will decide + return hasSimpleCorrelation; } @Override public @Nullable Boolean visitFieldAccess(RexFieldAccess fieldAccess) { @@ -332,8 +372,10 @@ private static RexNode replaceCorrelationsWithInputRef(RexNode exp, RelBuilder b } /** - * A visitor traversing row expressions and replacing calls with other expressions according - * to the specified mapping. + * A visitor traversing row expressions and replacing calls and field + * accesses with other expressions according to the specified mapping. + * The mapping is consulted before recursing so that the outermost + * matching expression wins. */ private static final class CallReplacer extends RexShuttle { private final Map mapping; @@ -350,5 +392,14 @@ private static final class CallReplacer extends RexShuttle { return super.visitCall(oldCall); } } + + @Override public RexNode visitFieldAccess(RexFieldAccess fieldAccess) { + RexNode replacement = mapping.get(fieldAccess); + if (replacement != null) { + return replacement; + } else { + return super.visitFieldAccess(fieldAccess); + } + } } } diff --git a/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java b/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java index 4e4104ad4876..300ff959bc74 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java @@ -575,10 +575,6 @@ protected RexNode removeCorrelationExpr( // Its output does not change the input ordering, so there's no // need to call propagateExpr. - if (isCorVarDefined && !canDecorrelateOffsetFetch(rel)) { - return null; - } - final RelNode oldInput = rel.getInput(); final Frame frame = getInvoke(oldInput, isCorVarDefined, rel, true); if (frame == null) { @@ -1141,31 +1137,8 @@ private static void shiftMapping(Map mapping, int startIndex, return register(sort, result, mapOldToNewOutputs, corDefOutputs); } - static boolean canDecorrelateOffsetFetch(Sort sort) { - final @Nullable RexLiteral fetch = sort.fetch == null - ? null - : RexUtil.reduceFetchToLiteral(sort.getCluster(), sort.fetch); - return isNonNegativeIntegralLiteral(sort.offset) - && (sort.fetch == null - || fetch != null && isNonNegativeIntegralLiteral(fetch)); - } - - private static boolean isNonNegativeIntegralLiteral(@Nullable RexNode node) { - if (node == null) { - return true; - } - if (!(node instanceof RexLiteral)) { - return false; - } - final @Nullable BigDecimal value = - ((RexLiteral) node).getValueAs(BigDecimal.class); - return value != null - && value.signum() >= 0 - && value.stripTrailingZeros().scale() <= 0; - } - protected @Nullable Frame decorrelateSortAsAggregate(Sort sort, final Frame frame) { - if (sort.offset != null || !(sort.fetch instanceof RexLiteral)) { + if (sort.offset != null || sort.fetch == null) { return null; } diff --git a/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java b/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java index 4139cf4b2b99..291eb619d0ed 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java @@ -233,14 +233,12 @@ public static RelNode decorrelateQuery(RelNode rel, RelBuilder builder) { RelNode preparedRel = prePlanner.findBestExp(); // start decorrelating + TopDownGeneralDecorrelator decorrelator = createEmptyDecorrelator(builder); RelNode decorrelateNode = rel; - if (canDecorrelateOffsetFetch(preparedRel, false)) { - TopDownGeneralDecorrelator decorrelator = createEmptyDecorrelator(builder); - try { - decorrelateNode = decorrelator.correlateElimination(preparedRel, true); - } catch (UnsupportedOperationException e) { - // if the correlation exists in an unsupported operator, retain the original plan. - } + try { + decorrelateNode = decorrelator.correlateElimination(preparedRel, true); + } catch (UnsupportedOperationException e) { + // if the correlation exists in an unsupported operator, retain the original plan. } HepProgram postProgram = HepProgram.builder() @@ -257,29 +255,6 @@ public static RelNode decorrelateQuery(RelNode rel, RelBuilder builder) { return postPlanner.findBestExp(); } - /** Returns whether correlated Sorts in a tree have OFFSET and FETCH values - * that can be decorrelated without changing their row-count semantics. */ - private static boolean canDecorrelateOffsetFetch(RelNode rel, - boolean isCorVarDefined) { - if (isCorVarDefined && rel instanceof Sort - && !RelDecorrelator.canDecorrelateOffsetFetch((Sort) rel)) { - return false; - } - if (rel instanceof Correlate) { - final Correlate correlate = (Correlate) rel; - if (!canDecorrelateOffsetFetch(correlate.getLeft(), isCorVarDefined)) { - return false; - } - return canDecorrelateOffsetFetch(correlate.getRight(), true); - } - for (RelNode input : rel.getInputs()) { - if (!canDecorrelateOffsetFetch(input, isCorVarDefined)) { - return false; - } - } - return true; - } - /** * Eliminates Correlate. * diff --git a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java index 2309102ff826..36d56a9f0488 100644 --- a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java +++ b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java @@ -86,7 +86,6 @@ import org.apache.calcite.rex.RexSubQuery; import org.apache.calcite.rex.RexUnknownAs; import org.apache.calcite.rex.RexUtil; -import org.apache.calcite.rex.RexVisitorImpl; import org.apache.calcite.rex.RexWindowBound; import org.apache.calcite.rex.RexWindowBounds; import org.apache.calcite.rex.RexWindowExclusion; @@ -109,7 +108,6 @@ import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.apache.calcite.sql.type.SqlTypeName; -import org.apache.calcite.sql.type.SqlTypeUtil; import org.apache.calcite.sql.type.TableFunctionReturnTypeInference; import org.apache.calcite.sql.validate.SqlValidatorUtil; import org.apache.calcite.sql2rel.SqlToRelConverter; @@ -3803,7 +3801,8 @@ public RelBuilder sortLimit(Number offset, Number fetch, * * @param offsetNode RexLiteral means number of rows to skip is deterministic, * RexDynamicParam means number of rows to skip is dynamic. - * @param fetchNode Maximum number of rows to fetch + * @param fetchNode RexLiteral means maximum number of rows to fetch is deterministic, + * RexDynamicParam mean maximum number is dynamic. * @param nodes Sort expressions */ public RelBuilder sortLimit(@Nullable RexNode offsetNode, @Nullable RexNode fetchNode, @@ -3813,17 +3812,12 @@ public RelBuilder sortLimit(@Nullable RexNode offsetNode, @Nullable RexNode fetc throw new IllegalArgumentException("OFFSET node must be RexLiteral or RexDynamicParam"); } } - if (fetchNode != null && !isValidFetchExpression(fetchNode)) { - throw new IllegalArgumentException( - "FETCH node must not reference input fields or contain aggregate functions, " - + "window functions, or subqueries"); - } - if (fetchNode != null - && !SqlTypeUtil.isNumeric(fetchNode.getType())) { - throw new IllegalArgumentException( - "FETCH node must have a numeric type; actual type is " - + fetchNode.getType().getFullTypeString()); + if (fetchNode != null) { + if (!(fetchNode instanceof RexLiteral || fetchNode instanceof RexDynamicParam)) { + throw new IllegalArgumentException("FETCH node must be RexLiteral or RexDynamicParam"); + } } + final Registrar registrar = new Registrar(fields(), ImmutableList.of()); final List fieldCollations = registrar.registerFieldCollations(nodes); @@ -3890,38 +3884,6 @@ public RelBuilder sortLimit(@Nullable RexNode offsetNode, @Nullable RexNode fetc return this; } - private static boolean isValidFetchExpression(RexNode node) { - return Boolean.TRUE.equals(node.accept(new FetchExpressionVisitor())); - } - - /** Visitor that validates FETCH expressions. */ - private static class FetchExpressionVisitor - extends RexVisitorImpl<@Nullable Boolean> { - FetchExpressionVisitor() { - super(false); - } - - @Override public Boolean visitLiteral(RexLiteral literal) { - return true; - } - - @Override public Boolean visitDynamicParam(RexDynamicParam dynamicParam) { - return true; - } - - @Override public Boolean visitCall(RexCall call) { - if (call.getOperator().isAggregator()) { - return false; - } - for (RexNode operand : call.getOperands()) { - if (!Boolean.TRUE.equals(operand.accept(this))) { - return false; - } - } - return true; - } - } - private static RelFieldCollation collation(RexNode node, RelFieldCollation.Direction direction, RelFieldCollation.@Nullable NullDirection nullDirection, diff --git a/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties b/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties index 19f3ef47a8d3..f4f16d73266a 100644 --- a/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties +++ b/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties @@ -61,9 +61,6 @@ ValidatorContext=From line {0,number,#}, column {1,number,#} to line {2,number,# CannotCastValue=Cast function cannot convert value of type {0} to type {1} UnknownDatatypeName=Unknown datatype name ''{0}'' IncompatibleValueType=Values passed to {0} operator must have compatible types -FetchExpressionMustBeNumeric=FETCH expression must have a numeric type; actual type is ''{0}'' -FetchExpressionCannotReferenceColumn=FETCH expression cannot reference table column ''{0}'' -FetchExpressionEvaluatedToNull=FETCH expression evaluated to NULL IncompatibleTypesInList=Values in expression list must have compatible types IncompatibleCharset=Cannot apply operation ''{0}'' to strings with different charsets ''{1}'' and ''{2}'' InvalidOrderByPos=ORDER BY is only allowed on top-level SELECT diff --git a/core/src/test/java/org/apache/calcite/adapter/enumerable/EnumUtilsTest.java b/core/src/test/java/org/apache/calcite/adapter/enumerable/EnumUtilsTest.java index 70370d7bb1ab..40b825939df9 100644 --- a/core/src/test/java/org/apache/calcite/adapter/enumerable/EnumUtilsTest.java +++ b/core/src/test/java/org/apache/calcite/adapter/enumerable/EnumUtilsTest.java @@ -34,7 +34,6 @@ import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; -import static org.junit.jupiter.api.Assertions.assertThrows; /** * Tests for {@link EnumUtils}. @@ -187,16 +186,6 @@ public final class EnumUtilsTest { is(BigDecimal.valueOf(2))); } - /** Test case for - * [CALCITE-7592] - * Add expression support for FETCH. */ - @Test void testNumberToBigDecimalRejectsNull() { - final IllegalArgumentException e = - assertThrows(IllegalArgumentException.class, - () -> EnumUtils.numberToBigDecimal(null, "FETCH")); - assertThat(e.getMessage(), is("FETCH expression evaluated to NULL")); - } - @Test void testMethodCallExpression() { // test for Object.class method parameter type final ConstantExpression arg0 = Expressions.constant(1, int.class); diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java index c04da0a26bd6..772c409692be 100644 --- a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java +++ b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java @@ -3060,25 +3060,25 @@ private SqlDialect nonOrdinalDialect() { + " as MAP array)"; final String expectedClickHouse2 = "SELECT CAST(array(map('a', '1'), map('b', '2'), map('c', '3'))" - + " AS Array(Map(`String`, `Nullable(String)`)))"; + + " AS Array(Map(`String`, `String`)))"; sql(query2).withClickHouse().ok(expectedClickHouse2); final String query3 = "select cast(MAP['a',ARRAY[1,2,3]]" + " as MAP)"; final String expectedClickHouse3 = - "SELECT CAST(map('a', array(1, 2, 3)) AS Map(`String`, Array(`Nullable(Int32)`)))"; + "SELECT CAST(map('a', array(1, 2, 3)) AS Map(`String`, Array(`Int32`)))"; sql(query3).withClickHouse().ok(expectedClickHouse3); final String query4 = "select cast(MAP['a',ARRAY[1.0,2.0,3.0]]" + " as MAP)"; final String expectedClickHouse4 = - "SELECT CAST(map('a', array(1.0, 2.0, 3.0)) AS Map(`String`, Array(`Nullable(Float32)`)))"; + "SELECT CAST(map('a', array(1.0, 2.0, 3.0)) AS Map(`String`, Array(`Float32`)))"; sql(query4).withClickHouse().ok(expectedClickHouse4); final String query5 = "select cast(MAP['a',MAP['b','c']]" + " as MAP>)"; final String expectedClickHouse5 = - "SELECT CAST(map('a', map('b', 'c')) AS Map(`String`, Map(`String`, `Nullable(String)`)))"; + "SELECT CAST(map('a', map('b', 'c')) AS Map(`String`, Map(`String`, `String`)))"; sql(query5).withClickHouse().ok(expectedClickHouse5); } @@ -4950,74 +4950,6 @@ private SqlDialect nonOrdinalDialect() { .withSybase().ok(expectedSybase); } - /** Test case for - * [CALCITE-7592] - * Add expression support for FETCH. */ - @Test void testFetchExpressionWithLimitDialect() { - final String query = "select \"product_id\"\n" - + "from \"product\"\n" - + "fetch next (1 + 2) rows only"; - final String expected = "SELECT `product_id`\n" - + "FROM `foodmart`.`product`\n" - + "LIMIT 3"; - sql(query).withMysql().ok(expected); - } - - /** Test case for - * [CALCITE-7592] - * Add expression support for FETCH. */ - @Test void testNegativeFetchExpressionIsRejectedBeforeSqlGeneration() { - final String query = "select \"product_id\"\n" - + "from \"product\"\n" - + "fetch next (0 - 1) rows only"; - final String error = - "FETCH value -1 is out of range; expected a non-negative value"; - sql(query).throws_(error); - sql(query).withMysql().throws_(error); - sql(query).withSQLite().throws_(error); - } - - /** Test case for - * [CALCITE-7592] - * Add expression support for FETCH. */ - @Test void testParameterizedFetchExpressionWithLimitDialect() { - final String query = "select \"product_id\"\n" - + "from \"product\"\n" - + "fetch next (? + 1) rows only"; - sql(query).withMysql().throws_( - "LIMIT dialect does not support FETCH expressions that cannot " - + "be reduced to a literal"); - } - - /** Test case for - * [CALCITE-7592] - * Add expression support for FETCH. */ - @Test void testParameterizedFetchExpressionWithSQLite() { - final String query = "select \"product_id\"\n" - + "from \"product\"\n" - + "fetch next (? + 1) rows only"; - final String expected = "SELECT \"product_id\"\n" - + "FROM \"foodmart\".\"product\"\n" - + "LIMIT ? + 1"; - sql(query).withSQLite().ok(expected); - } - - /** Test case for - * [CALCITE-7592] - * Add expression support for FETCH. */ - @Test void testDynamicFetchExpressionIsNotReduced() { - final String query = "select \"product_id\"\n" - + "from \"product\"\n" - + "fetch next (extract(day from current_date)) rows only"; - final String expected = "SELECT \"product_id\"\n" - + "FROM \"foodmart\".\"product\"\n" - + "FETCH NEXT (EXTRACT(DAY FROM CURRENT_DATE)) ROWS ONLY"; - sql(query).ok(expected); - sql(query).withMysql().throws_( - "LIMIT dialect does not support FETCH expressions that cannot " - + "be reduced to a literal"); - } - @Test void testSelectQueryComplex() { String query = "select count(*), \"units_per_case\" from \"product\" where \"cases_per_pallet\" > 100 " @@ -5752,15 +5684,15 @@ private SqlDialect nonOrdinalDialect() { @Test void testCastAsMapType() { sql("SELECT CAST(MAP['A', 1.0] AS MAP)") .ok("SELECT CAST(MAP['A', 1.0] AS " - + "MAP< VARCHAR CHARACTER SET \"ISO-8859-1\", DOUBLE NULL >)\n" + + "MAP< VARCHAR CHARACTER SET \"ISO-8859-1\", DOUBLE >)\n" + "FROM (VALUES (0)) AS \"t\" (\"ZERO\")"); sql("SELECT CAST(MAP['A', ARRAY[1, 2, 3]] AS MAP)") .ok("SELECT CAST(MAP['A', ARRAY[1, 2, 3]] AS " - + "MAP< VARCHAR CHARACTER SET \"ISO-8859-1\", INTEGER ARRAY NULL >)\n" + + "MAP< VARCHAR CHARACTER SET \"ISO-8859-1\", INTEGER ARRAY >)\n" + "FROM (VALUES (0)) AS \"t\" (\"ZERO\")"); sql("SELECT CAST(MAP[ARRAY['A'], MAP[1, 2]] AS MAP>)") .ok("SELECT CAST(MAP[ARRAY['A'], MAP[1, 2]] AS " - + "MAP< VARCHAR CHARACTER SET \"ISO-8859-1\" ARRAY, MAP< INTEGER, INTEGER NULL > NULL >)\n" + + "MAP< VARCHAR CHARACTER SET \"ISO-8859-1\" ARRAY, MAP< INTEGER, INTEGER > >)\n" + "FROM (VALUES (0)) AS \"t\" (\"ZERO\")"); } diff --git a/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java b/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java index 5f8b8edfb1db..b22218b95560 100644 --- a/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java +++ b/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java @@ -83,7 +83,6 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotSame; -import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.fail; import static java.util.Objects.requireNonNull; @@ -3632,35 +3631,6 @@ private void assertTypeAndToString( hasSize(0)); } - /** Test case for - * [CALCITE-7592] - * Add expression support for FETCH. */ - @Test void testContainsDynamicParam() { - final RelDataType intType = typeFactory.createSqlType(SqlTypeName.INTEGER); - final RexNode literal = rexBuilder.makeExactLiteral(BigDecimal.ONE, intType); - final RexNode dynamicParam = rexBuilder.makeDynamicParam(intType, 0); - final RexNode expression = - rexBuilder.makeCall(SqlStdOperatorTable.PLUS, literal, dynamicParam); - - assertThat(RexUtil.containsDynamicParam(literal), is(false)); - assertThat(RexUtil.containsDynamicParam(dynamicParam), is(true)); - assertThat(RexUtil.containsDynamicParam(expression), is(true)); - } - - /** Test case for - * [CALCITE-7592] - * Add expression support for FETCH. */ - @Test void testValidateFetchValueAllowsFractionalBigDecimal() { - assertThat(RexUtil.validateFetchValue(new BigDecimal("1.5")), - is(new BigDecimal("1.5"))); - - final IllegalArgumentException e = - assertThrows(IllegalArgumentException.class, - () -> RexUtil.validateFetchValue(new BigDecimal("-1.5"))); - assertThat(e.getMessage(), - containsString("FETCH value -1.5 is out of range")); - } - @Test void testConstantMap() { final RelDataType intType = typeFactory.createSqlType(SqlTypeName.INTEGER); final RelDataType bigintType = typeFactory.createSqlType(SqlTypeName.BIGINT); diff --git a/core/src/test/java/org/apache/calcite/sql/type/SqlTypeFactoryTest.java b/core/src/test/java/org/apache/calcite/sql/type/SqlTypeFactoryTest.java index f0ff190d28c7..68a34eb5bac7 100644 --- a/core/src/test/java/org/apache/calcite/sql/type/SqlTypeFactoryTest.java +++ b/core/src/test/java/org/apache/calcite/sql/type/SqlTypeFactoryTest.java @@ -87,6 +87,42 @@ class SqlTypeFactoryTest { assertThat(leastRestrictive.isNullable(), is(true)); } + /** UNKNOWN types in leastRestrictive() affect only the result nullability. */ + @Test void testLeastRestrictiveWithUnknown() { + SqlTypeFixture f = new SqlTypeFixture(); + // UNKNOWN never constrains the result, no matter its position + checkUnknownWithType(f, f.sqlBigInt); + checkUnknownWithType(f, f.structOfInt); + checkUnknownWithType(f, f.arrayBigInt); + checkUnknownWithType(f, f.mapOfInt); + // A nullable UNKNOWN makes the result nullable + RelDataType leastRestrictive = + f.typeFactory.leastRestrictive( + Lists.newArrayList(f.structOfInt, + f.typeFactory.createTypeWithNullability(f.sqlUnknown, true))); + assertThat(leastRestrictive, notNullValue()); + assertThat(leastRestrictive.getSqlTypeName(), is(SqlTypeName.ROW)); + assertThat(leastRestrictive.isNullable(), is(true)); + // A list of UNKNOWN unifies to UNKNOWN + leastRestrictive = + f.typeFactory.leastRestrictive( + Lists.newArrayList(f.sqlUnknown, f.sqlUnknown)); + assertThat(leastRestrictive.getSqlTypeName(), is(SqlTypeName.UNKNOWN)); + } + + /** Checks that leastRestictive({@code type}, UNKNOWN) yields {@code type}, + * in either order. */ + private void checkUnknownWithType(SqlTypeFixture f, RelDataType type) { + RelDataType r1 = + f.typeFactory.leastRestrictive(Lists.newArrayList(type, f.sqlUnknown)); + RelDataType r2 = + f.typeFactory.leastRestrictive(Lists.newArrayList(f.sqlUnknown, type)); + assertThat(r1, notNullValue()); + assertThat(r2, notNullValue()); + assertThat(r1.getFullTypeString(), is(type.getFullTypeString())); + assertThat(r2.getFullTypeString(), is(type.getFullTypeString())); + } + @Test void testLeastRestrictiveStructWithNull() { SqlTypeFixture f = new SqlTypeFixture(); RelDataType leastRestrictive = diff --git a/core/src/test/java/org/apache/calcite/sql2rel/CorrelateProjectExtractorTest.java b/core/src/test/java/org/apache/calcite/sql2rel/CorrelateProjectExtractorTest.java index 72276d42c332..da0a17296b3c 100644 --- a/core/src/test/java/org/apache/calcite/sql2rel/CorrelateProjectExtractorTest.java +++ b/core/src/test/java/org/apache/calcite/sql2rel/CorrelateProjectExtractorTest.java @@ -84,6 +84,78 @@ public static Frameworks.ConfigBuilder config() { assertThat(after, hasTree(planAfter)); } + /** Test case for [CALCITE-7646] + * CorrelateProjectExtractor does not handle nested field accesses cor0.field0.field1. */ + @Test void testNestedCorrelationFieldAccessInFilter() { + final RelBuilder builder = RelBuilder.create(config().build()); + final Holder<@Nullable RexCorrelVariable> v = Holder.empty(); + RelNode before = builder.scan("EMP") + .project( + builder.alias( + builder.call(SqlStdOperatorTable.ROW, + builder.field("EMPNO"), builder.field("DEPTNO")), "R")) + .variable(v::set) + .scan("DEPT") + .filter( + builder.equals(builder.field(0), + builder.getRexBuilder().makeFieldAccess(builder.field(v.get(), "R"), 1))) + .correlate(JoinRelType.LEFT, v.get().id, builder.field(2, 0, "R")) + .build(); + + final String planBefore = "" + + "LogicalCorrelate(correlation=[$cor0], joinType=[left], requiredColumns=[{0}])\n" + + " LogicalProject(R=[ROW($0, $7)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalFilter(condition=[=($0, $cor0.R.EXPR$1)])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n"; + assertThat(before, hasTree(planBefore)); + + RelNode after = before.accept(new CorrelateProjectExtractor(RelFactories.LOGICAL_BUILDER)); + final String planAfter = "" + + "LogicalProject(R=[$0], DEPTNO=[$2], DNAME=[$3], LOC=[$4])\n" + + " LogicalCorrelate(correlation=[$cor0], joinType=[left], requiredColumns=[{1}])\n" + + " LogicalProject(R=[ROW($0, $7)], $f1=[ROW($0, $7).EXPR$1])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalFilter(condition=[=($0, $cor0.$f1)])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n"; + assertThat(after, hasTree(planAfter)); + } + + /** Tests that a constant call operand, such as {@code POWER(2, 3)}, does + * not prevent extracting the enclosing correlated call. */ + @Test void testCorrelationCallWithConstantCallOperandInFilter() { + final RelBuilder builder = RelBuilder.create(config().build()); + final Holder<@Nullable RexCorrelVariable> v = Holder.empty(); + RelNode before = builder.scan("EMP") + .variable(v::set) + .scan("DEPT") + .filter( + builder.equals(builder.field(0), + builder.call(SqlStdOperatorTable.PLUS, + builder.call(SqlStdOperatorTable.POWER, + builder.literal(2), builder.literal(3)), + builder.field(v.get(), "DEPTNO")))) + .correlate(JoinRelType.LEFT, v.get().id, builder.field(2, 0, "DEPTNO")) + .build(); + + final String planBefore = "" + + "LogicalCorrelate(correlation=[$cor0], joinType=[left], requiredColumns=[{7}])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalFilter(condition=[=($0, +(POWER(2, 3), $cor0.DEPTNO))])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n"; + assertThat(before, hasTree(planBefore)); + + RelNode after = before.accept(new CorrelateProjectExtractor(RelFactories.LOGICAL_BUILDER)); + final String planAfter = "" + + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7], DEPTNO0=[$9], DNAME=[$10], LOC=[$11])\n" + + " LogicalCorrelate(correlation=[$cor0], joinType=[left], requiredColumns=[{8}])\n" + + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7], $f8=[+(POWER(2, 3), $7)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalFilter(condition=[=($0, $cor0.$f8)])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n"; + assertThat(after, hasTree(planAfter)); + } + @Test void testDoubleCorrelationCallOverVariableInFilters() { final RelBuilder builder = RelBuilder.create(config().build()); final Holder<@Nullable RexCorrelVariable> v = Holder.empty(); diff --git a/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java b/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java index 51c1e89d2d9d..a15d741dc947 100644 --- a/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java +++ b/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java @@ -290,9 +290,9 @@ public static Frameworks.ConfigBuilder config() { + " LogicalJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left])\n" + " LogicalValues(tuples=[[{ 7369 }, { 7499 }]])\n" + " LogicalAggregate(group=[{0}], agg#0=[SINGLE_VALUE($1)])\n" - + " LogicalProject(EMPNO1=[$12], EXPR$0=[||(||($1, ' from dept '), $13)])\n" - + " LogicalJoin(condition=[AND(=($7, $10), =($9, $11))], joinType=[left])\n" - + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7], DEPTNO0=[$7], EMPNO0=[CAST($0):INTEGER NOT NULL])\n" + + " LogicalProject(EMPNO1=[$11], EXPR$0=[||(||($1, ' from dept '), $12)])\n" + + " LogicalJoin(condition=[AND(=($7, $9), =($8, $10))], joinType=[left])\n" + + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7], EMPNO0=[CAST($0):INTEGER NOT NULL])\n" + " LogicalTableScan(table=[[scott, EMP]])\n" + " LogicalAggregate(group=[{0, 1, 2}], agg#0=[SINGLE_VALUE($3)])\n" + " LogicalProject(DEPTNO0=[$3], EMPNO0=[$4], EMPNO=[$5], DNAME=[$1])\n" @@ -556,29 +556,29 @@ public static Frameworks.ConfigBuilder config() { // LogicalTableScan(table=[[scott, EMP]]) final String planAfter = "" + "LogicalSort(sort0=[$0], dir0=[ASC])\n" - + " LogicalProject(DNAME=[$1], C=[$7])\n" - + " LogicalJoin(condition=[AND(=($0, $5), =($4, $6))], joinType=[left])\n" - + " LogicalProject(DEPTNO=[$0], DNAME=[$1], LOC=[$2], DEPTNO0=[$0], $f4=[*($0, 100)])\n" + + " LogicalProject(DNAME=[$1], C=[$6])\n" + + " LogicalJoin(condition=[AND(=($0, $4), =($3, $5))], joinType=[left])\n" + + " LogicalProject(DEPTNO=[$0], DNAME=[$1], LOC=[$2], $f3=[*($0, 100)])\n" + " LogicalTableScan(table=[[scott, DEPT]])\n" - + " LogicalProject(DEPTNO8=[$0], $f4=[$1], EXPR$0=[CASE(IS NOT NULL($4), $4, 0)])\n" + + " LogicalProject(DEPTNO8=[$0], $f3=[$1], EXPR$0=[CASE(IS NOT NULL($4), $4, 0)])\n" + " LogicalJoin(condition=[AND(IS NOT DISTINCT FROM($0, $2), IS NOT DISTINCT FROM($1, $3))], joinType=[left])\n" - + " LogicalProject(DEPTNO=[$0], $f4=[*($0, 100)])\n" + + " LogicalProject(DEPTNO=[$0], $f3=[*($0, 100)])\n" + " LogicalTableScan(table=[[scott, DEPT]])\n" + " LogicalAggregate(group=[{0, 1}], EXPR$0=[COUNT()])\n" - + " LogicalProject(DEPTNO8=[$7], $f4=[$9])\n" + + " LogicalProject(DEPTNO8=[$7], $f3=[$9])\n" + " LogicalFilter(condition=[IS NOT NULL($7)])\n" - + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7], i=[$11], $f4=[$9])\n" + + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7], i=[$11], $f3=[$9])\n" + " LogicalJoin(condition=[=($8, $10)], joinType=[inner])\n" + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7], SAL0=[CAST($5):DECIMAL(12, 2)])\n" + " LogicalTableScan(table=[[scott, EMP]])\n" - + " LogicalProject($f4=[$0], SAL0=[$1], $f2=[true])\n" + + " LogicalProject($f3=[$0], SAL0=[$1], $f2=[true])\n" + " LogicalAggregate(group=[{0, 1}])\n" - + " LogicalProject($f4=[$1], SAL0=[$2])\n" + + " LogicalProject($f3=[$1], SAL0=[$2])\n" + " LogicalJoin(condition=[AND(>($2, CAST($0):DECIMAL(12, 2) NOT NULL), <($1, $0))], joinType=[inner])\n" + " LogicalValues(tuples=[[{ 1000 }, { 2000 }, { 3000 }]])\n" + " LogicalJoin(condition=[true], joinType=[inner])\n" + " LogicalAggregate(group=[{0}])\n" - + " LogicalProject($f4=[*($0, 100)])\n" + + " LogicalProject($f3=[*($0, 100)])\n" + " LogicalTableScan(table=[[scott, DEPT]])\n" + " LogicalAggregate(group=[{0}])\n" + " LogicalProject(SAL0=[CAST($5):DECIMAL(12, 2)])\n" @@ -1870,9 +1870,9 @@ public static Frameworks.ConfigBuilder config() { RelDecorrelator.decorrelateQuery(before, builder, RuleSets.ofList(Collections.emptyList()), RuleSets.ofList(Collections.emptyList())); final String planAfter = "" - + "LogicalProject(DEPTNO=[$0], I0=[$3], I1=[$8])\n" - + " LogicalJoin(condition=[AND(=($0, $6), =($5, $7))], joinType=[left])\n" - + " LogicalProject(DEPTNO=[$0], DNAME=[$1], LOC=[$2], EXPR$0=[$5], DEPTNO0=[$0], $f5=[>(CAST($0):INTEGER NOT NULL, 0)])\n" + + "LogicalProject(DEPTNO=[$0], I0=[$3], I1=[$7])\n" + + " LogicalJoin(condition=[AND(=($0, $5), =($4, $6))], joinType=[left])\n" + + " LogicalProject(DEPTNO=[$0], DNAME=[$1], LOC=[$2], EXPR$0=[$5], $f4=[>(CAST($0):INTEGER NOT NULL, 0)])\n" + " LogicalJoin(condition=[=($3, $4)], joinType=[left])\n" + " LogicalProject(DEPTNO=[$0], DNAME=[$1], LOC=[$2], DEPTNO0=[CAST($0):SMALLINT NOT NULL])\n" + " LogicalTableScan(table=[[scott, DEPT]])\n" @@ -1884,12 +1884,12 @@ public static Frameworks.ConfigBuilder config() { + " LogicalProject(DEPTNO0=[CAST($0):SMALLINT NOT NULL])\n" + " LogicalTableScan(table=[[scott, DEPT]])\n" + " LogicalAggregate(group=[{0, 1}], EXPR$0=[MIN($2)])\n" - + " LogicalProject(DEPTNO0=[$8], $f5=[$9], $f0=[0])\n" + + " LogicalProject(DEPTNO0=[$8], $f4=[$9], $f0=[0])\n" + " LogicalJoin(condition=[=($7, $8)], joinType=[inner])\n" + " LogicalFilter(condition=[=($1, 'SMITH')])\n" + " LogicalTableScan(table=[[scott, EMP]])\n" + " LogicalFilter(condition=[$1])\n" - + " LogicalProject(DEPTNO=[$0], $f5=[>(CAST($0):INTEGER NOT NULL, 0)])\n" + + " LogicalProject(DEPTNO=[$0], $f4=[>(CAST($0):INTEGER NOT NULL, 0)])\n" + " LogicalJoin(condition=[=($3, $4)], joinType=[left])\n" + " LogicalProject(DEPTNO=[$0], DNAME=[$1], LOC=[$2], DEPTNO0=[CAST($0):SMALLINT NOT NULL])\n" + " LogicalTableScan(table=[[scott, DEPT]])\n" @@ -2428,4 +2428,67 @@ public static Frameworks.ConfigBuilder config() { + " LogicalTableScan(table=[[scott, EMP]])\n"; assertThat(after, hasTree(planAfter)); } + + /** Test case for + * [CALCITE-7646] + * CorrelateProjectExtractor does not handle nested field accesses cor0.field0.field1. */ + @Test void testNestedCorrelatedFieldAccess() throws SqlParseException { + final String sql = "select a.\"aid\", t.lat\n" + + "from \"bookstore\".\"authors\" a,\n" + + "lateral (select b.\"aid\" as c,\n" + + " (a.\"birthPlace\").\"coords\".\"latitude\" as lat\n" + + " from \"bookstore\".\"authors\" b\n" + + " where b.\"aid\" = a.\"aid\") as t"; + SchemaPlus rootSchema = Frameworks.createRootSchema(true); + CalciteAssert.addSchema(rootSchema, CalciteAssert.SchemaSpec.BOOKSTORE); + CalciteConnectionConfig config = new CalciteConnectionConfigImpl(new Properties()); + // The Frameworks planner cannot be used here because it flattens + // structured types, and RelStructuredTypeFlattener does not support + // correlations on structured columns. + SqlTestFactory factory = SqlTestFactory.INSTANCE + .withCatalogReader((typeFactory, caseSensitive) -> + new CalciteCatalogReader( + CalciteSchema.from(rootSchema), + ImmutableList.of("bookstore"), + typeFactory, + config)); + SqlParser parser = factory.createParser(sql); + SqlNode parsed = parser.parseQuery(); + final SqlToRelConverter sqlToRelConverter = factory.createSqlToRelConverter(); + assert sqlToRelConverter.validator != null; + final SqlNode validated = sqlToRelConverter.validator.validate(parsed); + final RelNode before = sqlToRelConverter.convertQuery(validated, false, true).rel; + + final String planBefore = "" + + "LogicalProject(aid=[$0], LAT=[$5])\n" + + " LogicalCorrelate(correlation=[$cor1], joinType=[inner], requiredColumns=[{0, 2}])\n" + + " LogicalTableScan(table=[[bookstore, authors]])\n" + + " LogicalProject(C=[$0], LAT=[$cor1.birthPlace.coords.latitude])\n" + + " LogicalFilter(condition=[=($0, $cor1.aid)])\n" + + " LogicalTableScan(table=[[bookstore, authors]])\n"; + assertThat(before, hasTree(planBefore)); + + final RelBuilder relBuilder = + RelFactories.LOGICAL_BUILDER.create(before.getCluster(), null); + // Decorrelate without any rules, just "purely" decorrelation algorithm on RelDecorrelator + final RelNode after = + RelDecorrelator.decorrelateQuery(before, relBuilder, + RuleSets.ofList(Collections.emptyList()), + RuleSets.ofList(Collections.emptyList())); + + // The nested field access is extracted into the projection $f4 on the + // left side and no correlation variables remain. + final String planAfter = "" + + "LogicalProject(aid=[$0], LAT=[$6])\n" + + " LogicalJoin(condition=[AND(=($0, $7), IS NOT DISTINCT FROM($4, $8))], joinType=[inner])\n" + + " LogicalProject(aid=[$0], name=[$1], birthPlace=[$2], books=[$3], $f4=[$2.coords.latitude])\n" + + " LogicalTableScan(table=[[bookstore, authors]])\n" + + " LogicalProject(C=[$0], LAT=[$4], aid=[$0], $f4=[$4])\n" + + " LogicalJoin(condition=[true], joinType=[inner])\n" + + " LogicalTableScan(table=[[bookstore, authors]])\n" + + " LogicalAggregate(group=[{0}])\n" + + " LogicalProject($f4=[$2.coords.latitude])\n" + + " LogicalTableScan(table=[[bookstore, authors]])\n"; + assertThat(after, hasTree(planAfter)); + } } diff --git a/core/src/test/java/org/apache/calcite/test/JdbcTest.java b/core/src/test/java/org/apache/calcite/test/JdbcTest.java index f213b12dcb7e..83ce5b1d78a6 100644 --- a/core/src/test/java/org/apache/calcite/test/JdbcTest.java +++ b/core/src/test/java/org/apache/calcite/test/JdbcTest.java @@ -3581,181 +3581,6 @@ public void checkOrderBy(final boolean desc, + "store_id=4; grocery_sqft=16844\n"); } - /** Test case for - * [CALCITE-7592] - * Add expression support for FETCH. */ - @Test void testFetchExpression() { - CalciteAssert.that() - .query("select * from (values (1), (2), (3), (4)) as t(x)\n" - + "fetch next (1 + abs(-2)) rows only") - .returns("X=1\n" - + "X=2\n" - + "X=3\n"); - } - - /** Test case for - * [CALCITE-7592] - * Add expression support for FETCH. */ - @Test void testBindableFetchExpression() { - try (Hook.Closeable ignored = Hook.ENABLE_BINDABLE.addThread(Hook.propertyJ(true))) { - final CalciteAssert.AssertThat with = CalciteAssert.that(); - with - .query("select * from (values (1), (2), (3), (4)) as t(x)\n" - + "fetch next (rand_integer(1) + 2) rows only") - .explainContains("BindableSort(fetch=[+(RAND_INTEGER(1), 2)])") - .returns("X=1\n" - + "X=2\n"); - with.query("select * from (values (1), (2), (3), (4)) as t(x)\n" - + "fetch next (cast(9223372036854775808 as decimal(20, 0))) rows only") - .returns("X=1\nX=2\nX=3\nX=4\n"); - with.query("select * from (values (1), (2), (3), (4)) as t(x)\n" - + "order by x fetch next ? rows only") - .explainContains("BindableSort(sort0=[$0], dir0=[ASC], fetch=[?0])") - .consumesPreparedStatement(p -> - p.setBigDecimal(1, new BigDecimal("1.5"))) - .returns("X=1\n" - + "X=2\n"); - } - } - - /** Test case for - * [CALCITE-7592] - * Add expression support for FETCH. */ - @Test void testFetchExpressionFunctionArguments() { - final CalciteAssert.AssertThat with = CalciteAssert.that(); - final String values = "select * from (values (1), (2), (3)) as t(x)\n"; - with.query(values + "fetch next (abs(2)) rows only") - .returns("X=1\n" - + "X=2\n"); - with.query(values + "fetch next (abs(-2)) rows only") - .returns("X=1\n" - + "X=2\n"); - } - - /** Test case for - * [CALCITE-7592] - * Add expression support for FETCH. */ - @Test void testFetchExpressionInvalidValue() { - final CalciteAssert.AssertThat with = CalciteAssert.that(); - final String values = "select * from (values (1), (2), (3)) as t(x)\n"; - with.query(values + "fetch next (0 - 1) rows only") - .throws_("FETCH must not be negative"); - with.query(values + "fetch next (-1) rows only") - .throws_("FETCH must not be negative"); - with.query(values - + "fetch next (cast(null as integer)) rows only") - .throws_("FETCH expression evaluated to NULL"); - } - - /** Test case for - * [CALCITE-7592] - * Add expression support for FETCH. */ - @Test void testCorrelatedFetchExpressionInvalidValue() { - final String sqlPrefix = "select d.\"name\", e.\"name\"\n" - + "from \"hr\".\"depts\" d,\n" - + "lateral (select \"name\" from \"hr\".\"emps\"\n" - + " where \"deptno\" = d.\"deptno\"\n"; - for (String fetch : new String[] {"(0 - 1)", "(-1)"}) { - for (boolean topDown : new boolean[] {false, true}) { - CalciteAssert.hr() - .with(CalciteConnectionProperty.TOPDOWN_GENERAL_DECORRELATION_ENABLED, topDown) - .query(sqlPrefix + " fetch next " + fetch + " rows only) e") - .throws_("FETCH value -1 is out of range"); - } - } - } - - /** Test case for - * [CALCITE-7592] - * Add expression support for FETCH. */ - @Test void testCorrelatedFractionalOffsetFetch() { - final String sqlPrefix = "select d.\"name\" as dname, e.\"name\" as ename\n" - + "from \"hr\".\"depts\" d,\n" - + "lateral (select \"empid\", \"name\" from \"hr\".\"emps\"\n" - + " where \"deptno\" = d.\"deptno\"\n" - + " order by \"empid\" "; - final String sqlSuffix = ") e\norder by e.\"empid\""; - for (boolean topDown : new boolean[] {false, true}) { - final CalciteAssert.AssertThat with = CalciteAssert.hr() - .with(CalciteConnectionProperty.TOPDOWN_GENERAL_DECORRELATION_ENABLED, topDown); - with.query(sqlPrefix + "fetch next (0.5 + 1) rows only" + sqlSuffix) - .returns("DNAME=Sales; ENAME=Bill\n" - + "DNAME=Sales; ENAME=Theodore\n"); - with.query(sqlPrefix + "offset 1.5 rows fetch next 1 row only" + sqlSuffix) - .returns("DNAME=Sales; ENAME=Sebastian\n"); - } - } - - /** Test case for - * [CALCITE-7592] - * Add expression support for FETCH. */ - @Test void testCorrelatedPreparedFractionalOffset() throws Exception { - final String sql = "select d.\"name\" as dname, e.\"name\" as ename\n" - + "from \"hr\".\"depts\" d,\n" - + "lateral (select \"empid\", \"name\" from \"hr\".\"emps\"\n" - + " where \"deptno\" = d.\"deptno\"\n" - + " order by \"empid\" offset ? rows fetch next 1 row only) e\n" - + "order by e.\"empid\""; - for (boolean topDown : new boolean[] {false, true}) { - CalciteAssert.hr() - .with(CalciteConnectionProperty.TOPDOWN_GENERAL_DECORRELATION_ENABLED, topDown) - .doWithConnection(connection -> { - checkPreparedBigDecimalParameter(connection, sql, - new BigDecimal("1.5"), - "DNAME=Sales; ENAME=Sebastian\n"); - }); - } - } - - /** Test case for - * [CALCITE-7592] - * Add expression support for FETCH. */ - @Test void testCorrelatedPreparedFetchExpression() throws Exception { - for (String fetch : new String[] {"?", "(? + 0)"}) { - final String sql = "select d.\"name\" as dname, e.\"name\" as ename\n" - + "from \"hr\".\"depts\" d,\n" - + "lateral (select \"empid\", \"name\" from \"hr\".\"emps\"\n" - + " where \"deptno\" = d.\"deptno\"\n" - + " order by \"empid\" fetch next " + fetch + " rows only) e\n" - + "order by e.\"empid\""; - for (boolean topDown : new boolean[] {false, true}) { - CalciteAssert.hr() - .with(CalciteConnectionProperty.TOPDOWN_GENERAL_DECORRELATION_ENABLED, topDown) - .doWithConnection(connection -> { - checkPreparedFetchRepeated(connection, sql, - new int[] {1, 3}, - new String[] { - "DNAME=Sales; ENAME=Bill\n", - "DNAME=Sales; ENAME=Bill\n" - + "DNAME=Sales; ENAME=Theodore\n" - + "DNAME=Sales; ENAME=Sebastian\n" - }); - checkPreparedParameterFails(connection, sql, -1, - "FETCH must not be negative"); - checkPreparedParameterNullFails(connection, sql, - "FETCH expression evaluated to NULL"); - }); - } - } - } - - /** Test case for - * [CALCITE-7592] - * Add expression support for FETCH. */ - @Test void testFetchExpressionBeyondLong() { - final CalciteAssert.AssertThat with = CalciteAssert.that(); - final String values = "select * from (values (1), (2), (3), (4)) as t(x)\n"; - final String expected = "X=1\nX=2\nX=3\nX=4\n"; - with.query(values + "fetch next 9223372036854775808 rows only") - .returns(expected); - with.query(values + "fetch next " - + "(cast(9223372036854775808 as decimal(20, 0)) + 1) rows only") - .returns(expected); - with.query(values + "order by x fetch next " - + "(cast(9223372036854775808 as decimal(20, 0)) + 1) rows only") - .returns(expected); - } - /** Tests ORDER BY ... OFFSET ... FETCH. */ @Test void testOrderByOffsetFetch() { CalciteAssert.that() @@ -6233,169 +6058,6 @@ private CalciteAssert.AssertQuery withEmpDept(String sql) { "name=Theodore"); } - /** Test case for - * [CALCITE-7592] - * Add expression support for FETCH. */ - @Test void testPreparedFetchExpression() throws Exception { - CalciteAssert.that() - .doWithConnection(connection -> { - final String values = - "select * from (values (1), (2), (3), (4)) as t(x)\n"; - checkPreparedFetch(connection, values + "fetch next (?) rows only", - 2, "X=1\nX=2\n"); - checkPreparedFetch(connection, values + "fetch next (? + 1) rows only", - 2, "X=1\nX=2\nX=3\n"); - checkPreparedFetch(connection, - values + "fetch next (abs(cast(? as integer))) rows only", - 2, "X=1\nX=2\n"); - checkPreparedFetch(connection, - values + "fetch next (abs(cast(? as integer))) rows only", - -2, "X=1\nX=2\n"); - checkPreparedFetchRepeated(connection, - values + "fetch next (?) rows only", - new int[] {1, 3}, - new String[] {"X=1\n", "X=1\nX=2\nX=3\n"}); - checkPreparedFetchRepeated(connection, - values + "fetch next (? + 1) rows only", - new int[] {0, 2, 3}, - new String[] {"X=1\n", "X=1\nX=2\nX=3\n", - "X=1\nX=2\nX=3\nX=4\n"}); - checkPreparedFetch(connection, - values + "fetch next (? + abs(2)) rows only", - 1, "X=1\nX=2\nX=3\n"); - checkPreparedBigDecimalParameter(connection, - values + "fetch next (cast(? as decimal(20, 0))) rows only", - new BigDecimal("9223372036854775808"), - "X=1\nX=2\nX=3\nX=4\n"); - - checkPreparedParameterFails(connection, - values + "fetch next (?) rows only", -1, - "FETCH must not be negative"); - checkPreparedParameterFails(connection, - values + "fetch next (? + 1) rows only", -2, - "FETCH must not be negative"); - }); - } - - /** Test case for - * [CALCITE-7592] - * Add expression support for FETCH. */ - @Test void testBindablePreparedFetchExpression() throws Exception { - try (Hook.Closeable ignored = Hook.ENABLE_BINDABLE.addThread(Hook.propertyJ(true))) { - CalciteAssert.that() - .doWithConnection(connection -> { - final String values = - "select * from (values (1), (2), (3), (4)) as t(x)\n"; - checkPreparedFetch(connection, - values + "fetch next (? + 1) rows only", - 2, "X=1\nX=2\nX=3\n"); - checkPreparedFetchRepeated(connection, - values + "fetch next (? + 1) rows only", - new int[] {0, 2, 3}, - new String[] {"X=1\n", "X=1\nX=2\nX=3\n", - "X=1\nX=2\nX=3\nX=4\n"}); - checkPreparedBigDecimalParameter(connection, - values + "fetch next (cast(? as decimal(20, 0))) rows only", - new BigDecimal("9223372036854775808"), - "X=1\nX=2\nX=3\nX=4\n"); - }); - } - } - - /** Test case for - * [CALCITE-7592] - * Add expression support for FETCH. */ - @Test void testBindablePreparedOffset() throws Exception { - try (Hook.Closeable ignored = Hook.ENABLE_BINDABLE.addThread(Hook.propertyJ(true))) { - CalciteAssert.that() - .doWithConnection(connection -> { - final String values = - "select * from (values (1), (2), (3), (4)) as t(x)\n"; - final String offset = values + "offset ? rows"; - checkPreparedBigDecimalParameter(connection, offset, - new BigDecimal("1.5"), - "X=3\nX=4\n"); - checkPreparedBigDecimalParameter(connection, offset, - BigDecimal.valueOf(Integer.MAX_VALUE).add(BigDecimal.ONE), ""); - - final String sortedOffset = values + "order by x desc offset ? rows"; - checkPreparedBigDecimalParameter(connection, sortedOffset, - new BigDecimal("1.5"), - "X=2\nX=1\n"); - checkPreparedParameterFails(connection, offset, -1, - "OFFSET must not be negative"); - checkPreparedParameterNullFails(connection, offset, - "OFFSET expression evaluated to NULL"); - }); - } - } - - private static void checkPreparedFetch(Connection connection, String sql, - int value, String expected) { - try (PreparedStatement p = connection.prepareStatement(sql)) { - p.setInt(1, value); - try (ResultSet r = p.executeQuery()) { - assertThat(CalciteAssert.toString(r), is(expected)); - } - } catch (SQLException e) { - throw TestUtil.rethrow(e); - } - } - - private static void checkPreparedBigDecimalParameter(Connection connection, String sql, - BigDecimal value, String expected) { - try (PreparedStatement p = connection.prepareStatement(sql)) { - p.setBigDecimal(1, value); - try (ResultSet r = p.executeQuery()) { - assertThat(CalciteAssert.toString(r), is(expected)); - } - } catch (SQLException e) { - throw TestUtil.rethrow(e); - } - } - - private static void checkPreparedFetchRepeated(Connection connection, String sql, - int[] values, String[] expected) { - try (PreparedStatement p = connection.prepareStatement(sql)) { - for (int i = 0; i < values.length; i++) { - p.setInt(1, values[i]); - try (ResultSet r = p.executeQuery()) { - assertThat(CalciteAssert.toString(r), is(expected[i])); - } - } - } catch (SQLException e) { - throw TestUtil.rethrow(e); - } - } - - private static void checkPreparedParameterFails(Connection connection, String sql, - long value, String expectedMessage) { - try (PreparedStatement p = connection.prepareStatement(sql)) { - if (value >= Integer.MIN_VALUE && value <= Integer.MAX_VALUE) { - p.setInt(1, (int) value); - } else { - p.setLong(1, value); - } - final SQLException e = - assertThrows(SQLException.class, p::executeQuery); - assertThat(e.getMessage(), containsString(expectedMessage)); - } catch (SQLException e) { - throw TestUtil.rethrow(e); - } - } - - private static void checkPreparedParameterNullFails(Connection connection, String sql, - String expectedMessage) { - try (PreparedStatement p = connection.prepareStatement(sql)) { - p.setNull(1, Types.INTEGER); - final SQLException e = - assertThrows(SQLException.class, p::executeQuery); - assertThat(e.getMessage(), containsString(expectedMessage)); - } catch (SQLException e) { - throw TestUtil.rethrow(e); - } - } - private void checkPreparedOffsetFetch(final int offset, final int fetch, final Matcher matcher) throws Exception { CalciteAssert.hr() diff --git a/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java b/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java index 7ad9a4d733fc..493df7c30b03 100644 --- a/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java @@ -52,10 +52,7 @@ import org.apache.calcite.rex.RexCorrelVariable; import org.apache.calcite.rex.RexFieldCollation; import org.apache.calcite.rex.RexInputRef; -import org.apache.calcite.rex.RexLambdaRef; import org.apache.calcite.rex.RexNode; -import org.apache.calcite.rex.RexNodeAndFieldIndex; -import org.apache.calcite.rex.RexSubQuery; import org.apache.calcite.rex.RexWindowBounds; import org.apache.calcite.runtime.CalciteException; import org.apache.calcite.schema.SchemaPlus; @@ -5650,108 +5647,6 @@ private static RelNode buildCorrelateWithJoin(JoinRelType type, RelBuilder build assertThat(mq.getMaxRowCount(planAfter), is(Double.POSITIVE_INFINITY)); } - /** Test case for - * [CALCITE-7592] - * Add expression support for FETCH. */ - @Test void testFetchExpressionCannotReferenceInputField() { - final RelBuilder builder = RelBuilder.create(config().build()); - builder.scan("DEPT"); - final RexNode field = builder.field("DEPTNO"); - - assertThrows(IllegalArgumentException.class, - () -> builder.sortLimit(null, field, ImmutableList.of())); - assertThrows(IllegalArgumentException.class, - () -> builder.sortLimit(null, - builder.call(SqlStdOperatorTable.PLUS, builder.literal(1), field), - ImmutableList.of())); - assertThrows(IllegalArgumentException.class, - () -> builder.sortLimit(null, - new RexNodeAndFieldIndex(0, 0, "DEPTNO", field.getType()), - ImmutableList.of())); - } - - /** Test case for - * [CALCITE-7592] - * Add expression support for FETCH. */ - @Test void testFetchExpressionMustHaveNumericType() { - final RelBuilder builder = RelBuilder.create(config().build()); - builder.scan("DEPT"); - - assertThrows(IllegalArgumentException.class, - () -> builder.sortLimit(null, builder.literal("x"), ImmutableList.of())); - builder.sortLimit(null, builder.literal(new BigDecimal("1.5")), - ImmutableList.of()); - } - - /** Test case for - * [CALCITE-7592] - * Add expression support for FETCH. */ - @Test void testFetchExpressionAllowsScalarCallAndDynamicParameter() { - final RelBuilder builder = RelBuilder.create(config().build()); - final RelDataType intType = - builder.getTypeFactory().createSqlType(SqlTypeName.INTEGER); - builder.scan("DEPT") - .sortLimit(null, - builder.call(SqlStdOperatorTable.PLUS, - builder.getRexBuilder().makeDynamicParam(intType, 0), - builder.literal(1)), - ImmutableList.of()); - - assertThat( - builder.build(), hasTree("LogicalSort(fetch=[+(?0, 1)])\n" - + " LogicalTableScan(table=[[scott, DEPT]])\n")); - } - - /** Test case for - * [CALCITE-7592] - * Add expression support for FETCH. */ - @Test void testFetchExpressionCannotContainAggregateWindowOrSubQuery() { - final RelBuilder builder = RelBuilder.create(config().build()); - final RelDataType intType = - builder.getTypeFactory().createSqlType(SqlTypeName.INTEGER); - builder.scan("DEPT"); - final RexNode aggregate = - builder.call(SqlStdOperatorTable.SUM, builder.literal(1)); - assertThrows(IllegalArgumentException.class, - () -> builder.sortLimit(null, aggregate, ImmutableList.of())); - - final RexNode over = - builder.getRexBuilder().makeOver(intType, - SqlStdOperatorTable.ROW_NUMBER, ImmutableList.of(), - ImmutableList.of(), ImmutableList.of(), - RexWindowBounds.UNBOUNDED_PRECEDING, - RexWindowBounds.UNBOUNDED_FOLLOWING, - true, true, false, false, false); - assertThrows(IllegalArgumentException.class, - () -> builder.sortLimit(null, over, ImmutableList.of())); - - final RelBuilder subQueryBuilder = RelBuilder.create(config().build()); - final RexNode subQuery = - RexSubQuery.scalar(subQueryBuilder.values(new String[] {"N"}, 1).build()); - assertThrows(IllegalArgumentException.class, - () -> builder.sortLimit(null, subQuery, ImmutableList.of())); - } - - /** Test case for - * [CALCITE-7592] - * Add expression support for FETCH. */ - @Test void testFetchExpressionCannotContainLambda() { - final RelBuilder builder = RelBuilder.create(config().build()); - final RelDataType intType = - builder.getTypeFactory().createSqlType(SqlTypeName.INTEGER); - builder.scan("DEPT"); - final RexLambdaRef lambdaRef = new RexLambdaRef(0, "x", intType); - final RexNode lambda = - builder.getRexBuilder().makeLambdaCall( - builder.call(SqlStdOperatorTable.PLUS, lambdaRef, builder.literal(1)), - ImmutableList.of(lambdaRef)); - - assertThrows(IllegalArgumentException.class, - () -> builder.sortLimit(null, lambda, ImmutableList.of())); - assertThrows(IllegalArgumentException.class, - () -> builder.sortLimit(null, lambdaRef, ImmutableList.of())); - } - @Test void testAdoptConventionEnumerable() { final RelBuilder builder = RelBuilder.create(config().build()); RelNode root = builder diff --git a/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java b/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java index 460e066051fe..9c3c30e3448e 100644 --- a/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java @@ -1467,7 +1467,7 @@ void testColumnOriginsUnion() { @Test void testRowCountSortLimitBeyondLong() { final BigDecimal fetch = BigDecimal.valueOf(Long.MAX_VALUE).add(BigDecimal.ONE); final double fetchDouble = fetch.doubleValue(); - final String sql = "select * from emp order by ename limit " + fetch.toPlainString(); + final String sql = "select * from emp order by ename limit " + fetchDouble; final RelMetadataFixture fixture = sql(sql); fixture.assertThatRowCount(is(EMP_SIZE), is(0D), is(fetchDouble)); } @@ -1496,37 +1496,6 @@ void testColumnOriginsUnion() { fixture.assertThatRowCount(is(1d), is(0D), is(0d)); } - /** Test case for - * [CALCITE-7592] - * Add expression support for FETCH. */ - @Test void testMinRowCountFetchExpression() { - final String sql = "select * from (values (1), (2)) as t(x)\n" - + "fetch next (2 - 2) rows only"; - final RelMetadataFixture fixture = sql(sql); - fixture.assertThatRowCount(is(2D), is(0D), is(2D)); - - fixture - .withCluster(cluster -> { - final RelOptPlanner planner = new VolcanoPlanner(); - planner.addRule(EnumerableRules.ENUMERABLE_VALUES_RULE); - planner.addRule(EnumerableRules.ENUMERABLE_PROJECT_RULE); - planner.addRule(EnumerableRules.ENUMERABLE_LIMIT_RULE); - planner.addRelTraitDef(ConventionTraitDef.INSTANCE); - return RelOptCluster.create(planner, cluster.getRexBuilder()); - }) - .withRelTransform(rel -> { - final RelOptPlanner planner = rel.getCluster().getPlanner(); - planner.setRoot(rel); - final RelTraitSet requiredOutputTraits = - rel.getCluster().traitSet().replace(EnumerableConvention.INSTANCE); - final RelNode root = planner.changeTraits(rel, requiredOutputTraits); - planner.setRoot(root); - return planner.findBestExp(); - }) - .assertThatRel(is(instanceOf(EnumerableLimit.class))) - .assertThatRowCount(is(2D), is(0D), is(2D)); - } - @Test void testRowCountSortLimitOffset() { final String sql = "select * from emp order by ename limit 10 offset 5"; /* 14 - 5 */ diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index 5c687519daa0..4ba2f4aa9623 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -1738,34 +1738,6 @@ private void checkJoinProjectTransposeDoesNotMatch(JoinRelType type) { .check(); } - /** Test case for - * [CALCITE-7592] - * Add expression support for FETCH. */ - @Test void testSortUnionTransposeWithNonDeterministicFetch() { - final String sql = "select a.name from dept a\n" - + "union all\n" - + "select b.name from dept b\n" - + "order by name fetch next (rand_integer(10)) rows only"; - sql(sql) - .withPreRule(CoreRules.PROJECT_SET_OP_TRANSPOSE) - .withRule(CoreRules.SORT_UNION_TRANSPOSE) - .checkUnchanged(); - } - - /** Test case for - * [CALCITE-7592] - * Add expression support for FETCH. */ - @Test void testSortUnionTransposePushesParameterizedFetchExpression() { - final String sql = "select a.name from dept a\n" - + "union all\n" - + "select b.name from dept b\n" - + "order by name fetch next (? + 1) rows only"; - sql(sql) - .withPreRule(CoreRules.PROJECT_SET_OP_TRANSPOSE) - .withRule(CoreRules.SORT_UNION_TRANSPOSE) - .check(); - } - @Test void testSortRemovalAllKeysConstant() { final String sql = "select count(*) as c\n" + "from sales.emp\n" @@ -6025,9 +5997,10 @@ private void checkEmptyJoin(RelOptFixture f) { } /** Test case for - * [CALCITE-7592] - * Add expression support for FETCH. */ - @Test void testSortWithDynamicParamPushesOnce() { + * [CALCITE-6647] + * SortUnionTransposeRule should not push SORT past a UNION when SORT's fetch is DynamicParam + . */ + @Test void testSortWithDynamicParam() { HepProgramBuilder builder = new HepProgramBuilder(); builder.addRuleClass(SortProjectTransposeRule.class); builder.addRuleClass(SortUnionTransposeRule.class); @@ -9757,19 +9730,6 @@ private void checkSemiJoinRuleOnAntiJoin(RelOptRule rule) { .check(); } - /** Test case for - * [CALCITE-7592] - * Add expression support for FETCH. */ - @Test void testDecorrelateProjectWithFetchExpression() { - final String query = "SELECT name, " - + "(SELECT sal FROM emp where dept.deptno = emp.deptno order by sal " - + "fetch next (1 + 0) rows only) " - + "FROM dept"; - sql(query).withRule(CoreRules.PROJECT_SUB_QUERY_TO_CORRELATE) - .withLateDecorrelate(true) - .check(); - } - /** Test case for [CALCITE-7289] * Select NULL subquery throwing exception. */ @Test void testNullSelect() { @@ -12258,39 +12218,6 @@ private static RelNode applyAggregateRemoveLiteralAggRule(RelNode rel) { .check(); } - /** Test case for - * [CALCITE-7592] - * Add expression support for FETCH. */ - @Test void testNondeterministicFetchPreventsDecorrelation() { - checkNondeterministicFetchPreventsDecorrelation(false); - } - - /** Test case for - * [CALCITE-7592] - * Add expression support for FETCH. */ - @Test void testNondeterministicFetchPreventsTopDownDecorrelation() { - checkNondeterministicFetchPreventsDecorrelation(true); - } - - private void checkNondeterministicFetchPreventsDecorrelation(boolean enableTopDown) { - final String sql = "select t.deptno, e.ename\n" - + "from (select distinct deptno from emp) t,\n" - + "lateral (select ename from emp\n" - + " where emp.deptno = t.deptno\n" - + " order by sal\n" - + " fetch next (rand_integer(2) + 1) rows only) e"; - - final RelOptFixture fixture = sql(sql) - .withRule() // empty program - .withLateDecorrelate(true) - .withTopDownGeneralDecorrelate(enableTopDown); - if (enableTopDown) { - fixture.check(); - } else { - fixture.checkUnchanged(); - } - } - @Test void testTopDownGeneralDecorrelateForFilterSome() { final String sql = "select empno from emp where " + "empno > SOME(select empno from emp_b where emp.ename = emp_b.ename)"; diff --git a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java index 6ce401502c93..6b3255e653c7 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java @@ -1263,15 +1263,6 @@ public static void checkActualAndReferenceFiles() { sql(sql).ok(); } - /** Test case for - * [CALCITE-7592] - * Add expression support for FETCH. */ - @Test void testFetchWithExpression() { - final String sql = - "select empno from emp fetch next (1 + abs(-2)) rows only"; - sql(sql).ok(); - } - /** Test case for * [CALCITE-439] * SqlValidatorUtil.uniquify() may not terminate under some conditions. */ diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index 7bfc58eafe64..8f95099ab952 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -1527,7 +1527,7 @@ void testLikeAndSimilarFails() { expr("cast(ARRAY[1,2,3] AS VARIANT ARRAY)") .columnType("VARIANT NOT NULL ARRAY NOT NULL"); expr("cast(MAP['a','b','c','d'] AS MAP)") - .columnType("(VARCHAR NOT NULL, VARIANT) MAP NOT NULL"); + .columnType("(VARCHAR NOT NULL, VARIANT NOT NULL) MAP NOT NULL"); // Test case for [CALCITE-7293] https://issues.apache.org/jira/browse/CALCITE-7293 // MAP constructor cannot handle VARIANT values that need casts expr("MAP['a', CAST('x' AS VARIANT), 'b', CAST(NULL AS VARIANT)]") @@ -9656,16 +9656,16 @@ void testGroupExpressionEquivalenceParams() { @Test void testCastMapType() { sql("select cast(\"int2IntMapType\" as map) from COMPLEXTYPES.CTC_T1") .withExtendedCatalog() - .columnType("(INTEGER NOT NULL, INTEGER) MAP NOT NULL"); + .columnType("(INTEGER NOT NULL, INTEGER NOT NULL) MAP NOT NULL"); sql("select cast(\"int2varcharArrayMapType\" as map) " + "from COMPLEXTYPES.CTC_T1") .withExtendedCatalog() - .columnType("(INTEGER NOT NULL, VARCHAR ARRAY) MAP NOT NULL"); + .columnType("(INTEGER NOT NULL, VARCHAR NOT NULL ARRAY NOT NULL) MAP NOT NULL"); sql("select cast(\"varcharMultiset2IntIntMapType\" as map>)" + " from COMPLEXTYPES.CTC_T1") .withExtendedCatalog() - .columnType("(VARCHAR(5) MULTISET NOT NULL, " - + "(INTEGER NOT NULL, INTEGER) MAP) MAP NOT NULL"); + .columnType("(VARCHAR(5) NOT NULL MULTISET NOT NULL, " + + "(INTEGER NOT NULL, INTEGER NOT NULL) MAP NOT NULL) MAP NOT NULL"); } @Test void testCastAsRowType() { @@ -10723,22 +10723,6 @@ void testGroupExpressionEquivalenceParams() { .rewritesTo(expected); } - /** Test case for - * [CALCITE-7592] - * Add expression support for FETCH. */ - @Test void testFetchExpressionType() { - sql("select name from dept fetch next (^upper('x')^) rows only") - .fails("FETCH expression must have a numeric type; " - + "actual type is 'CHAR\\(1\\) NOT NULL'"); - sql("select name from dept fetch next (^'x'^) rows only") - .fails("FETCH expression must have a numeric type; " - + "actual type is 'CHAR\\(1\\) NOT NULL'"); - sql("select name from dept fetch next 1.5 rows only").ok(); - sql("select name from dept " - + "fetch next (^row_number() over ()^) rows only") - .fails("Windowed aggregate expression is illegal in FETCH clause"); - } - @Test void testRewriteWithOffsetWithoutOrderBy() { final String sql = "select name from dept offset 2"; final String expected = "SELECT `NAME`\n" diff --git a/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableMergeUnionTest.java b/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableMergeUnionTest.java index 44055f707462..68bb56cf366d 100644 --- a/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableMergeUnionTest.java +++ b/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableMergeUnionTest.java @@ -78,36 +78,6 @@ class EnumerableMergeUnionTest { "empid=45; name=Pascal"); } - /** Test case for - * [CALCITE-7592] - * Add expression support for FETCH. */ - @Test void mergeUnionDoesNotPushNonDeterministicFetch() { - tester(false, - new HrSchemaBig(), - "select * from (select empid, name from emps " - + "union all select empid, name from emps) " - + "order by empid fetch next (rand_integer(10)) rows only") - .explainContains("EnumerableLimitSort(sort0=[$0], dir0=[ASC], " - + "fetch=[RAND_INTEGER(10)])\n" - + " EnumerableMergeUnion(all=[true])\n" - + " EnumerableSort(sort0=[$0], dir0=[ASC])\n"); - } - - /** Test case for - * [CALCITE-7592] - * Add expression support for FETCH. */ - @Test void mergeUnionPushesParameterizedFetchExpression() { - tester(false, - new HrSchemaBig(), - "select * from (select empid, name from emps " - + "union all select empid, name from emps) " - + "order by empid fetch next (? + 1) rows only") - .explainContains("EnumerableLimit(fetch=[+(?0, 1)])\n" - + " EnumerableMergeUnion(all=[true])\n" - + " EnumerableLimitSort(sort0=[$0], dir0=[ASC], " - + "fetch=[+(?0, 1)])\n"); - } - @Test void mergeUnionAllOrderByName() { tester(false, new HrSchemaBig(), diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index f1bb4248e528..ab26ca8524bc 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -2950,46 +2950,6 @@ LogicalProject(NAME=[$1]) LogicalFilter(condition=[<=($3, 1)]) LogicalProject(SAL=[$5], EXPR$1=[EXTRACT(FLAG(YEAR), $4)], DEPTNO=[$7], rn=[ROW_NUMBER() OVER (PARTITION BY $7 ORDER BY EXTRACT(FLAG(YEAR), $4) NULLS LAST, $5 DESC NULLS FIRST)]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) -]]> - - - - - - - - - - - - - - @@ -3137,9 +3097,9 @@ LogicalProject(NAME=[$1], EXPR$1=[$2]) ($5, 1000))]) LogicalTableScan(table=[[CATALOG, SALES, EMPNULLABLES]]) LogicalFilter(condition=[=($1, $0)]) LogicalAggregate(group=[{0, 1, 2}]) - LogicalProject(SAL=[$5], SAL0=[$8], $f9=[$9]) + LogicalProject(SAL=[$5], SAL0=[$8], $f8=[$9]) LogicalJoin(condition=[OR(=($8, $5), $9)], joinType=[inner]) LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5], COMM=[$6], SLACKER=[$8]) LogicalFilter(condition=[AND(=($7, 20), >($5, 1000))]) LogicalTableScan(table=[[CATALOG, SALES, EMPNULLABLES]]) LogicalAggregate(group=[{0, 1}]) - LogicalProject(SAL=[$5], $f9=[=($5, 4)]) + LogicalProject(SAL=[$5], $f8=[=($5, 4)]) LogicalFilter(condition=[AND(=($7, 20), >($5, 1000))]) LogicalTableScan(table=[[CATALOG, SALES, EMPNULLABLES]]) ]]> @@ -9272,9 +9232,9 @@ LEFT JOIN LATERAL ( - - - - - - - - - - - - - - - - - - - - - - - - - @@ -20052,55 +19923,7 @@ LogicalSort(sort0=[$0], dir0=[ASC], fetch=[0]) ]]> - - - - - - - - - - - - - - - - - - - - + diff --git a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml index 1aa3c3f66134..aafc9c11efd3 100644 --- a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml @@ -1770,15 +1770,15 @@ LogicalAggregate(group=[{}], EXPR$0=[COUNT()], EXPR$1=[SUM($0)]) @@ -1796,18 +1796,18 @@ cross join lateral @@ -2601,18 +2601,6 @@ LogicalSort(fetch=[5]) LogicalSort(fetch=[?0]) LogicalProject(EMPNO=[$0]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) -]]> - - - - - - - - @@ -5234,9 +5222,9 @@ LogicalProject(C=[$0], D=[$1], C0=[$2]) diff --git a/core/src/test/resources/sql/fetch.iq b/core/src/test/resources/sql/fetch.iq deleted file mode 100644 index 8f4b0dd53d58..000000000000 --- a/core/src/test/resources/sql/fetch.iq +++ /dev/null @@ -1,183 +0,0 @@ -# fetch.iq -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to you under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -!use post -!set outputformat mysql - -# FETCH accepts a parenthesized arithmetic expression. -select * -from (values (1), (2), (3), (4)) as t(x) -fetch next (1 + abs(-2)) rows only; -+---+ -| X | -+---+ -| 1 | -| 2 | -| 3 | -+---+ -(3 rows) - -!ok - -# FETCH accepts a parenthesized scalar expression. -select * -from (values (1), (2), (3), (4)) as t(x) -fetch next (abs(2)) rows only; -+---+ -| X | -+---+ -| 1 | -| 2 | -+---+ -(2 rows) - -!ok - -# FETCH values are not restricted to the BIGINT range. -select * -from (values (1), (2), (3), (4)) as t(x) -fetch next (cast(9223372036854775808 as decimal(20, 0)) + 1) rows only; -+---+ -| X | -+---+ -| 1 | -| 2 | -| 3 | -| 4 | -+---+ -(4 rows) - -!ok - -# FETCH expression cannot be negative. -select * -from (values (1), (2), (3)) as t(x) -fetch next (0 - 1) rows only; -FETCH must not be negative -!error - -# FETCH expression cannot evaluate to NULL. -select * -from (values (1), (2), (3)) as t(x) -fetch next (cast(null as integer)) rows only; -FETCH expression evaluated to NULL -!error - -# FETCH expression may have a fractional numeric type. -select * -from (values (1), (2), (3)) as t(x) -fetch next (1.5) rows only; -+---+ -| X | -+---+ -| 1 | -| 2 | -+---+ -(2 rows) - -!ok - -# FETCH expression cannot reference input columns. -select * -from (values (1), (2), (3)) as t(x) -fetch next (x) rows only; -FETCH expression cannot reference table column 'X' -!error - -# Expressions without parentheses are not allowed in FETCH. -select * -from (values (1), (2), (3)) as t(x) -fetch next 1 + 2 rows only; -Encountered "+" -!error - -# FETCH expression works with a table source. -select deptno, dname -from dept -order by deptno -fetch next (1 + 1) rows only; -+--------+-------------+ -| DEPTNO | DNAME | -+--------+-------------+ -| 10 | Sales | -| 20 | Marketing | -+--------+-------------+ -(2 rows) - -!ok - -# FETCH expression works together with OFFSET on a table source. -select deptno, dname -from dept -order by deptno -offset 1 rows -fetch next (1 + 1) rows only; -+--------+-------------+ -| DEPTNO | DNAME | -+--------+-------------+ -| 20 | Marketing | -| 30 | Engineering | -+--------+-------------+ -(2 rows) - -!ok - -# FETCH expression may contain a scalar function on a table source. -select deptno -from dept -order by deptno -fetch next (abs(-3)) rows only; -+--------+ -| DEPTNO | -+--------+ -| 10 | -| 20 | -| 30 | -+--------+ -(3 rows) - -!ok - -# FETCH expression cannot reference columns of a table source. -select deptno, dname -from dept -order by deptno -fetch next (deptno) rows only; -FETCH expression cannot reference table column 'DEPTNO' -!error - -# FETCH expression cannot reference columns even inside a larger expression. -select deptno, dname -from dept -order by deptno -fetch next (deptno + 1) rows only; -FETCH expression cannot reference table column 'DEPTNO' -!error - -# FETCH expression may be zero on a table source. -select deptno -from dept -order by deptno -fetch next (2 - 2) rows only; -+--------+ -| DEPTNO | -+--------+ -+--------+ -(0 rows) - -!ok diff --git a/core/src/test/resources/sql/lateral.iq b/core/src/test/resources/sql/lateral.iq index 5c82727b8930..4c4ffbe17072 100644 --- a/core/src/test/resources/sql/lateral.iq +++ b/core/src/test/resources/sql/lateral.iq @@ -244,4 +244,98 @@ where job = 'MANAGER'; !ok +# 3 test cases for [CALCITE-7646] CorrelateProjectExtractor +# does not handle nested field accesses cor0.field0.field1. + +# All queries use LATERAL, which converts directly to a Correlate. +# The results were validated on Postgres + +!use scott + +select t.dd, t.x +from dept d, +lateral (select d.deptno as dd, u.x + from unnest(array[d.deptno + 100]) as u(x)) as t +where d.dname = 'SALES'; ++----+-----+ +| DD | X | ++----+-----+ +| 30 | 130 | ++----+-----+ +(1 row) + +!ok + +select t.dd, t.dd1, t.x +from dept d, +lateral (select d.deptno as dd, d.deptno + 1 as dd1, u.x + from unnest(array[1, 2]) as u(x)) as t +where d.dname = 'SALES'; ++----+-----+---+ +| DD | DD1 | X | ++----+-----+---+ +| 30 | 31 | 1 | +| 30 | 31 | 2 | ++----+-----+---+ +(2 rows) + +!ok +!if (use_old_decorr) { +# The correlated computation d.deptno + 1 (DD1) has been extracted into the left +# input of the EnumerableNestedLoopJoin, as $f3. The right input, +# UNNEST(ARRAY[1, 2]), references no correlation variable, so decorrelation +# replaces the Correlate with a join. +EnumerableCalc(expr#0..4=[{inputs}], proj#0..2=[{exprs}]) + EnumerableHashJoin(condition=[AND(=($3, $5), =($4, $6))], joinType=[semi]) + EnumerableCalc(expr#0..2=[{inputs}], proj#0..2=[{exprs}], DEPTNO=[$t0], $f3=[$t1]) + EnumerableNestedLoopJoin(condition=[true], joinType=[inner]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1], expr#4=[+($t0, $t3)], expr#5=['SALES':VARCHAR(14)], expr#6=[=($t1, $t5)], DEPTNO=[$t0], $f3=[$t4], $condition=[$t6]) + EnumerableTableScan(table=[[scott, DEPT]]) + EnumerableUncollect + EnumerableCalc(expr#0=[{inputs}], expr#1=[1], expr#2=[2], expr#3=[ARRAY($t1, $t2)], EXPR$0=[$t3]) + EnumerableValues(tuples=[[{ 0 }]]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1], expr#4=[+($t0, $t3)], expr#5=['SALES':VARCHAR(14)], expr#6=[=($t1, $t5)], DEPTNO=[$t0], $f3=[$t4], $condition=[$t6]) + EnumerableTableScan(table=[[scott, DEPT]]) +!plan +!} + +# COALESCE(d.path, ARRAY[CAST(NULL AS INTEGER)]) converts to +# CASE(IS NOT NULL($cor0.PATH), $cor0.PATH, ARRAY(null:INTEGER)). The constant +# ARRAY(null:INTEGER) operand must not prevent extracting the CASE to the +# left input of the Correlate operator +select d.deptno, t.x +from (select deptno, + case when deptno = 10 then array[deptno, deptno + 1] end as path + from dept) as d, +lateral (select * from unnest(coalesce(d.path, array[cast(null as integer)])) as u(x)) as t +order by d.deptno, t.x; ++--------+----+ +| DEPTNO | X | ++--------+----+ +| 10 | 10 | +| 10 | 11 | +| 20 | | +| 30 | | +| 40 | | ++--------+----+ +(5 rows) + +!ok +!if (use_old_decorr) { +# The entire CASE produced by COALESCE has been extracted into the left input of the +# EnumerableCorrelate, as $f2. The right input reads the array through $cor0.$f2. +# The query cannot be decorrelated because of the remaining Uncollect. +EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) + EnumerableCalc(expr#0..2=[{inputs}], DEPTNO=[$t0], X=[$t2]) + EnumerableCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{1}]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[CAST($t0):INTEGER NOT NULL], expr#4=[10], expr#5=[=($t3, $t4)], expr#6=[1], expr#7=[+($t0, $t6)], expr#8=[ARRAY($t3, $t7)], expr#9=[null:INTEGER NOT NULL ARRAY], expr#10=[CASE($t5, $t8, $t9)], expr#11=[IS NOT NULL($t10)], expr#12=[CAST($t10):INTEGER NOT NULL ARRAY NOT NULL], expr#13=[CAST($t12):INTEGER ARRAY NOT NULL], expr#14=[null:INTEGER], expr#15=[ARRAY($t14)], expr#16=[CASE($t11, $t13, $t15)], DEPTNO=[$t0], $f2=[$t16]) + EnumerableTableScan(table=[[scott, DEPT]]) + EnumerableUncollect + EnumerableCalc(expr#0=[{inputs}], expr#1=[$cor0], expr#2=[$t1.$f2], EXPR$0=[$t2]) + EnumerableValues(tuples=[[{ 0 }]]) +!plan +!} + +!set planner-rules original + # End lateral.iq diff --git a/server/src/test/java/org/apache/calcite/test/ServerTest.java b/server/src/test/java/org/apache/calcite/test/ServerTest.java index 39d434f23ae9..355d39de7d63 100644 --- a/server/src/test/java/org/apache/calcite/test/ServerTest.java +++ b/server/src/test/java/org/apache/calcite/test/ServerTest.java @@ -43,7 +43,6 @@ import java.math.BigDecimal; import java.sql.Connection; import java.sql.DriverManager; -import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; @@ -453,42 +452,6 @@ static Connection connect() throws SQLException { } } - /** Test case for - * [CALCITE-7592] - * Add expression support for FETCH. */ - @Test void testFetchExpressionCannotReferenceInputColumn() throws Exception { - try (Connection c = connect(); - Statement s = c.createStatement()) { - s.execute("create table person (id int not null, name varchar(20))"); - try (PreparedStatement p = - c.prepareStatement("insert into person (id, name) values (?, ?)")) { - p.setInt(1, 1); - p.setString(2, "foo"); - assertThat(p.executeUpdate(), is(1)); - } - - SQLException e = - assertThrows( - SQLException.class, () -> s.executeQuery("select * from person " - + "fetch next id rows only")); - assertThat(e.getMessage(), containsString("Encountered \"id\"")); - - e = - assertThrows( - SQLException.class, () -> s.executeQuery("select * from person " - + "fetch next (id) rows only")); - assertThat(e.getMessage(), - containsString("FETCH expression cannot reference table column 'ID'")); - - e = - assertThrows( - SQLException.class, () -> s.executeQuery("select * from person " - + "fetch next (1 + id) rows only")); - assertThat(e.getMessage(), - containsString("FETCH expression cannot reference table column 'ID'")); - } - } - /** Test case for * [CALCITE-6022] * Support "CREATE TABLE ... LIKE" DDL in server module. */ diff --git a/site/_docs/reference.md b/site/_docs/reference.md index 56fbf8a86ca4..904ab0461967 100644 --- a/site/_docs/reference.md +++ b/site/_docs/reference.md @@ -427,13 +427,8 @@ in the order that they appear in the list; for example: "SELECT x, y FROM t ORDER BY x, y" An optional trailing ASC / DESC and NULLS FIRST / NULLS LAST applies to all keys. -In *query*, *start* may be either an unsigned numeric literal or a dynamic -parameter whose value is numeric. The *count* in a LIMIT clause may be either -an unsigned numeric literal or a dynamic parameter whose value is numeric. The -*count* in a FETCH clause may be an unsigned numeric literal, a dynamic -parameter whose value is numeric, or a scalar expression enclosed in -parentheses. A FETCH *count* expression cannot reference columns from the query -input, and cannot contain aggregate functions, window functions, or sub-queries. +In *query*, *count* and *start* may each be either an unsigned numeric literal +or a dynamic parameter whose value is numeric. Support for decimal or non-integer values is adapter-dependent. An aggregate query is a query that contains a GROUP BY or a HAVING diff --git a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java index 394aecf26f2a..4058d716bfb9 100644 --- a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java +++ b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java @@ -4104,31 +4104,12 @@ void checkPeriodPredicate(Checker checker) { + "FROM `FOO`\n" + "OFFSET ? ROWS\n" + "FETCH NEXT ? ROWS ONLY"); - // CALCITE-7592: Arithmetic and scalar expressions are allowed within parentheses. - sql("select a from foo fetch next (1 + abs(-2)) rows only") - .ok("SELECT `A`\n" - + "FROM `FOO`\n" - + "FETCH NEXT (1 + ABS(-2)) ROWS ONLY"); - // Expressions without parentheses are not allowed. - sql("select a from foo fetch next 1 ^+^ 2 rows only") - .fails("(?s).*Encountered \"\\+\" at .*"); - sql("select a from foo fetch next ? ^+^ abs(2) rows only") - .fails("(?s).*Encountered \"\\+\" at .*"); // missing ROWS after FETCH sql("select a from foo offset 1 fetch next 3 ^only^") .fails("(?s).*Encountered \"only\" at .*"); // FETCH before OFFSET is illegal sql("select a from foo fetch next 3 rows only ^offset^ 1") .fails("(?s).*Encountered \"offset\" at .*"); - // Subqueries are not allowed in FETCH - sql("select a from foo fetch next ^select^ 2 rows only") - .fails("(?s).*Encountered \"select\" at .*"); - sql("select a from foo fetch next (^select^ 2) rows only") - .fails("(?s).*Encountered \"select\" at .*"); - sql("select a from foo fetch next (^select^ ?) rows only") - .fails("(?s).*Encountered \"select\" at .*"); - sql("select a from foo fetch next (^select^ max(a) from foo) rows only") - .fails("(?s).*Encountered \"select\" at .*"); } /** diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java index 91f20af68167..387915bca6b5 100644 --- a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java +++ b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java @@ -1843,6 +1843,39 @@ void testCastToBoolean(CastType castType, SqlOperatorFixture f) { f.checkNull("cast(null as row(f0 varchar, f1 varchar))"); } + /** Test case for + * + * [CALCITE-7658] Type checker rejects + * CAST(ARRAY() AS ROW(x INT) ARRAY). + * + *

      The Spark {@code ARRAY()} function creates an empty array whose + * element type is UNKNOWN; such an array can be cast to any array type. */ + @Test void testCastEmptyArray() { + final SqlOperatorFixture f = fixture().withLibrary(SqlLibrary.SPARK); + f.checkScalar("cast(array() as integer array)", "[]", + "INTEGER NOT NULL ARRAY NOT NULL"); + f.checkScalar("cast(array() as row(x int) array)", "[]", + "RecordType(INTEGER NOT NULL X) NOT NULL ARRAY NOT NULL"); + f.checkScalar("cast(array() as integer array array)", "[]", + "INTEGER ARRAY NOT NULL ARRAY NOT NULL"); + f.checkScalar("cast(array() as map array)", "[]", + "(VARCHAR NOT NULL, INTEGER) MAP NOT NULL ARRAY NOT NULL"); + // A non-empty array with UNKNOWN or NULL element type contains only nulls + f.checkScalar("cast(array_append(array(), null) as row(x int) array)", + "[null]", + "RecordType(INTEGER NOT NULL X) ARRAY NOT NULL"); + f.checkScalar("cast(array(null) as row(x int) array)", + "[null]", + "RecordType(INTEGER NOT NULL X) ARRAY NOT NULL"); + // The empty MAP() has UNKNOWN key and value types + f.checkScalar("cast(map() as map)", "{}", + "(VARCHAR NOT NULL, INTEGER NOT NULL) MAP NOT NULL"); + f.checkScalar("cast(map() as map)", "{}", + "(VARCHAR NOT NULL, RecordType(INTEGER X) NOT NULL) MAP NOT NULL"); + f.checkScalar("cast(map() as map)", "{}", + "(VARCHAR NOT NULL, INTEGER ARRAY NOT NULL) MAP NOT NULL"); + } + /** Test cases for * * [CALCITE-4918] Add a VARIANT data type. */ @@ -8229,7 +8262,7 @@ void checkRegexpExtract(SqlOperatorFixture f0, FunctionAlias functionAlias) { f.checkScalar("array_append(array(null), null)", "[null, null]", "NULL ARRAY NOT NULL"); f.checkScalar("array_append(array(), null)", "[null]", - "UNKNOWN ARRAY NOT NULL"); + "NULL ARRAY NOT NULL"); f.checkScalar("array_append(array(), 1)", "[1]", "INTEGER NOT NULL ARRAY NOT NULL"); f.checkScalar("array_append(array[array[1, 2]], array[3, 4])", "[[1, 2], [3, 4]]", @@ -8568,7 +8601,7 @@ void checkRegexpExtract(SqlOperatorFixture f0, FunctionAlias functionAlias) { f.checkScalar("array_prepend(array(null), null)", "[null, null]", "NULL ARRAY NOT NULL"); f.checkScalar("array_prepend(array(), null)", "[null]", - "UNKNOWN ARRAY NOT NULL"); + "NULL ARRAY NOT NULL"); f.checkScalar("array_prepend(array(), 1)", "[1]", "INTEGER NOT NULL ARRAY NOT NULL"); f.checkScalar("array_prepend(array[array[1, 2]], array[3, 4])", "[[3, 4], [1, 2]]", @@ -13623,6 +13656,22 @@ private static void checkArrayConcatAggFuncFails(SqlOperatorFixture t) { "RecordType(INTEGER EXPR$0, INTEGER EXPR$1) NOT NULL ARRAY NOT NULL"); f2.checkScalar("array(row(1, 2), row(3, 4))", "[{1, 2}, {3, 4}]", "RecordType(INTEGER NOT NULL EXPR$0, INTEGER NOT NULL EXPR$1) NOT NULL ARRAY NOT NULL"); + // Tests for unification of UNKNOWN with other types; array() has a type + // of UNKNOWN ARRAY, yet the type of ARRAY() is inferred from other operands. + f2.checkScalar("array(array(1), array())", "[[1], []]", + "INTEGER NOT NULL ARRAY NOT NULL ARRAY NOT NULL"); + f2.checkScalar("array(array(), array(1))", "[[], [1]]", + "INTEGER NOT NULL ARRAY NOT NULL ARRAY NOT NULL"); + f2.checkScalar("array(array(row(1, 2)), array())", "[[{1, 2}], []]", + "RecordType(INTEGER NOT NULL EXPR$0, INTEGER NOT NULL EXPR$1) NOT NULL " + + "ARRAY NOT NULL ARRAY NOT NULL"); + f2.checkScalar("array(array(), array(row(1, 2)))", "[[], [{1, 2}]]", + "RecordType(INTEGER NOT NULL EXPR$0, INTEGER NOT NULL EXPR$1) NOT NULL " + + "ARRAY NOT NULL ARRAY NOT NULL"); + f2.checkScalar("array(array(array(1)), array())", "[[[1]], []]", + "INTEGER NOT NULL ARRAY NOT NULL ARRAY NOT NULL ARRAY NOT NULL"); + f2.checkScalar("array(array(), array(array(1)))", "[[], [[1]]]", + "INTEGER NOT NULL ARRAY NOT NULL ARRAY NOT NULL ARRAY NOT NULL"); // checkFails f2.checkFails("^array(row(1), row(2, 3))^", "Parameters must be of the same type", false); @@ -13641,6 +13690,32 @@ private static void checkArrayConcatAggFuncFails(SqlOperatorFixture t) { f.forEachLibrary(libraries, consumer); } + /** Tests that empty collections created by the Spark + * {@code ARRAY()} and {@code MAP()} functions, whose element + * types are UNKNOWN, unify with collections with known types. */ + @Test void testEmptyCollections() { + final SqlOperatorFixture f = fixture().withLibrary(SqlLibrary.SPARK); + f.checkScalar("array(map(1, 2), map())", "[{1=2}, {}]", + "(INTEGER NOT NULL, INTEGER NOT NULL) MAP NOT NULL ARRAY NOT NULL"); + f.checkScalar("array(map(), map(1, 2))", "[{}, {1=2}]", + "(INTEGER NOT NULL, INTEGER NOT NULL) MAP NOT NULL ARRAY NOT NULL"); + // Nested: empty collections inside a ROW unify field by field + f.checkScalar("array(row(array(), map()))", "[{[], {}}]", + "RecordType(UNKNOWN NOT NULL ARRAY NOT NULL EXPR$0, " + + "(UNKNOWN NOT NULL, UNKNOWN NOT NULL) MAP NOT NULL EXPR$1) " + + "NOT NULL ARRAY NOT NULL"); + f.checkScalar("array(row(array(1), map(1, 2)), row(array(), map()))", + "[{[1], {1=2}}, {[], {}}]", + "RecordType(INTEGER NOT NULL ARRAY NOT NULL EXPR$0, " + + "(INTEGER NOT NULL, INTEGER NOT NULL) MAP NOT NULL EXPR$1) " + + "NOT NULL ARRAY NOT NULL"); + f.checkScalar("array(row(array(), map()), row(array(1), map(1, 2)))", + "[{[], {}}, {[1], {1=2}}]", + "RecordType(INTEGER NOT NULL ARRAY NOT NULL EXPR$0, " + + "(INTEGER NOT NULL, INTEGER NOT NULL) MAP NOT NULL EXPR$1) " + + "NOT NULL ARRAY NOT NULL"); + } + @Test void testArrayQueryConstructor() { final SqlOperatorFixture f = fixture(); f.setFor(SqlStdOperatorTable.ARRAY_QUERY, SqlOperatorFixture.VmName.EXPAND); @@ -13947,6 +14022,22 @@ private static void checkArrayConcatAggFuncFails(SqlOperatorFixture t) { f1.checkScalar("map('k1', 1, 'k2', 2.0)", "{k1=1.0, k2=2.0}", "(CHAR(2) NOT NULL, DECIMAL(11, 1) NOT NULL) MAP NOT NULL"); + f1.checkScalar("map('a', array(1), 'b', array())", "{a=[1], b=[]}", + "(CHAR(1) NOT NULL, INTEGER NOT NULL ARRAY NOT NULL) MAP NOT NULL"); + f1.checkScalar("map('a', array(), 'b', array(1))", "{a=[], b=[1]}", + "(CHAR(1) NOT NULL, INTEGER NOT NULL ARRAY NOT NULL) MAP NOT NULL"); + f1.checkScalar("map('a', map(1, 2), 'b', map())", "{a={1=2}, b={}}", + "(CHAR(1) NOT NULL, (INTEGER NOT NULL, INTEGER NOT NULL) MAP NOT NULL) MAP NOT NULL"); + f1.checkScalar("map('a', map(), 'b', map(1, 2))", "{a={}, b={1=2}}", + "(CHAR(1) NOT NULL, (INTEGER NOT NULL, INTEGER NOT NULL) MAP NOT NULL) MAP NOT NULL"); + // Avatica's conversion of MAP to STRING is broken, so we only check + // the type for the following 2 tests + f1.checkType("map('a', array(row(1, 2)), 'b', array())", + "(CHAR(1) NOT NULL, RecordType(INTEGER NOT NULL EXPR$0, INTEGER NOT NULL EXPR$1) " + + "NOT NULL ARRAY NOT NULL) MAP NOT NULL"); + f1.checkType("map('a', array(), 'b', array(row(1, 2)))", + "(CHAR(1) NOT NULL, RecordType(INTEGER NOT NULL EXPR$0, INTEGER NOT NULL EXPR$1) " + + "NOT NULL ARRAY NOT NULL) MAP NOT NULL"); } @Test void testMapQueryConstructor() { From 51a27073c26c32c57bcf89deb1f63fbcceae14d1 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Mon, 27 Jul 2026 17:53:02 -0700 Subject: [PATCH 422/562] [CALCITE-7677] CAST between ROW types fails at runtime Signed-off-by: Mihai Budiu --- .../enumerable/RexToLixTranslator.java | 78 ++++++++++++------- .../calcite/jdbc/JavaTypeFactoryImpl.java | 28 ++++++- core/src/test/resources/sql/cast.iq | 48 ++++++++++++ 3 files changed, 124 insertions(+), 30 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java index a39146ac754c..9bad3dc3f5ff 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java @@ -351,6 +351,54 @@ private static boolean valueIsAlwaysNull(RelDataType type) { return typeName == SqlTypeName.UNKNOWN || typeName == SqlTypeName.NULL; } + /** Converts a ROW value to another ROW type, field by field. */ + private Expression getRowConvertExpression( + RelDataType sourceType, + RelDataType targetType, + Expression operand, + ConstantExpression format) { + if (valueIsAlwaysNull(sourceType)) { + return Expressions.constant(null); + } + assert sourceType.getSqlTypeName() == SqlTypeName.ROW; + List targetTypes = targetType.getFieldList(); + List sourceTypes = sourceType.getFieldList(); + assert targetTypes.size() == sourceTypes.size(); + List fields = new ArrayList<>(); + for (int i = 0; i < targetTypes.size(); i++) { + RelDataTypeField targetField = targetTypes.get(i); + RelDataTypeField sourceField = sourceTypes.get(i); + Expression field = Expressions.arrayIndex(operand, Expressions.constant(i)); + // In the generated Java code 'field' is an Object, + // we need to also cast it to the correct type to enable correct method dispatch in Java. + // We force the type to be nullable; this way, instead of (int) we get (Integer). + // Casting an object to an int is not legal. + RelDataType nullableSourceFieldType = + typeFactory.createTypeWithNullability(sourceField.getType(), true); + Type javaType = typeFactory.getJavaClass(nullableSourceFieldType); + if (nullableSourceFieldType.isStruct()) { + // A struct field is represented as Object[] at runtime; + // the recursive conversion below indexes into the field, which + // requires an array-typed operand. + field = Expressions.convert_(field, Object[].class); + } else if (!javaType.getTypeName().equals("java.lang.Void")) { + // Cannot cast to Void - this is the type of NULL literals. + field = Expressions.convert_(field, javaType); + } + Expression convert = + getConvertExpression(sourceField.getType(), targetField.getType(), field, format); + if (sourceField.getType().isNullable()) { + // field == null ? field : convert + convert = + Expressions.condition( + Expressions.equal(field, Expressions.constant(null)), + Expressions.constant(null), convert); + } + fields.add(convert); + } + return Expressions.call(BuiltInMethod.ARRAY.method, fields); + } + private Expression getConvertExpression( RelDataType sourceType, RelDataType targetType, @@ -376,35 +424,7 @@ private Expression getConvertExpression( } if (targetType.getSqlTypeName() == SqlTypeName.ROW) { - if (valueIsAlwaysNull(sourceType)) { - return Expressions.constant(null); - } - assert sourceType.getSqlTypeName() == SqlTypeName.ROW; - List targetTypes = targetType.getFieldList(); - List sourceTypes = sourceType.getFieldList(); - assert targetTypes.size() == sourceTypes.size(); - List fields = new ArrayList<>(); - for (int i = 0; i < targetTypes.size(); i++) { - RelDataTypeField targetField = targetTypes.get(i); - RelDataTypeField sourceField = sourceTypes.get(i); - Expression field = Expressions.arrayIndex(operand, Expressions.constant(i)); - // In the generated Java code 'field' is an Object, - // we need to also cast it to the correct type to enable correct method dispatch in Java. - // We force the type to be nullable; this way, instead of (int) we get (Integer). - // Casting an object ot an int is not legal. - RelDataType nullableSourceFieldType = - typeFactory.createTypeWithNullability(sourceField.getType(), true); - Type javaType = typeFactory.getJavaClass(nullableSourceFieldType); - if (!javaType.getTypeName().equals("java.lang.Void") - && !nullableSourceFieldType.isStruct()) { - // Cannot cast to Void - this is the type of NULL literals. - field = Expressions.convert_(field, javaType); - } - Expression convert = - getConvertExpression(sourceField.getType(), targetField.getType(), field, format); - fields.add(convert); - } - return Expressions.call(BuiltInMethod.ARRAY.method, fields); + return getRowConvertExpression(sourceType, targetType, operand, format); } switch (targetType.getSqlTypeName()) { diff --git a/core/src/main/java/org/apache/calcite/jdbc/JavaTypeFactoryImpl.java b/core/src/main/java/org/apache/calcite/jdbc/JavaTypeFactoryImpl.java index 3f276b41e6b2..58882f6a1b2d 100644 --- a/core/src/main/java/org/apache/calcite/jdbc/JavaTypeFactoryImpl.java +++ b/core/src/main/java/org/apache/calcite/jdbc/JavaTypeFactoryImpl.java @@ -365,7 +365,33 @@ private Type createSyntheticType(RelRecordType type) { final SyntheticRecordType syntheticType = new SyntheticRecordType(type, name); for (final RelDataTypeField recordField : type.getFieldList()) { - final Type javaClass = getJavaClass(recordField.getType()); + final Type fieldClass = getJavaClass(recordField.getType()); + // A field whose type has no real Java class is stored as Object[] at + // runtime, like all rows in enumerable convention. For example, the + // element type of ARRAY[ROW(ROW(1, 'a'), 10), NULL] becomes a "synthetic" type + // named Record2_0 + // public static class Record2_0 implements java.io.Serializable { + // public Object[] EXPR$0; // the nested row, e.g. {1, 'a'} + // public Integer EXPR$1; + // ...equals, hashCode, compareTo, toString... + // } + // If EXPR$0 would also have a synthetic type, + // this would generate nested synthetic classes, which + // EnumerableRelImplementor#classDecl cannot emit. + // + // A field whose type maps to a real Java class (e.g. a bean from a + // ReflectiveSchema) uses its own Java class. + // + // 'instanceof Class' distinguishes the two cases: getJavaClass + // returns a java.lang.reflect.Type, which is a loaded + // java.lang.Class for most SQL types. For a record type with no + // Java class (here the nested row's type, which maps to its own + // Record2_N), it is a SyntheticRecordType: a description of a class + // that is only generated and compiled together with the query, so no + // Class object exists for it. + final Type javaClass = fieldClass instanceof Class + ? fieldClass + : Object[].class; syntheticType.fields.add( new RecordFieldImpl( syntheticType, diff --git a/core/src/test/resources/sql/cast.iq b/core/src/test/resources/sql/cast.iq index a0ef44885409..ce7b13b8b9f1 100644 --- a/core/src/test/resources/sql/cast.iq +++ b/core/src/test/resources/sql/cast.iq @@ -2031,4 +2031,52 @@ values (cast(multiset[null] as integer multiset)); !ok +# Tests for [CALCITE-7677] CAST between ROW types fails at runtime +# https://issues.apache.org/jira/browse/CALCITE-7677 +!use scott + +SELECT ARRAY[ROW(1, 'Alice'), ROW(NULL, 'Dan')] AS people +FROM (VALUES (0)) AS t(zero); ++---------------------------+ +| PEOPLE | ++---------------------------+ +| [{1, Alice}, {null, Dan}] | ++---------------------------+ +(1 row) + +!ok + +SELECT CAST(ROW(ROW(2, 'b'), 20) AS ROW(a ROW(x INTEGER, y CHAR(1)), b INTEGER)) AS r +FROM (VALUES (0)) AS t(zero); ++--------------+ +| R | ++--------------+ +| {{2, b}, 20} | ++--------------+ +(1 row) + +!ok + +SELECT CAST(ROW(NULL, 30) AS ROW(a ROW(x INTEGER, y CHAR(1)), b INTEGER)) AS r +FROM (VALUES (0)) AS t(zero); ++------------+ +| R | ++------------+ +| {null, 30} | ++------------+ +(1 row) + +!ok + +SELECT ARRAY[ROW(ROW(1, 'a'), 10), NULL] AS xs +FROM (VALUES (0)) AS t(zero); ++----------------------+ +| XS | ++----------------------+ +| [{{1, a}, 10}, null] | ++----------------------+ +(1 row) + +!ok + # End cast.iq From bb0cb2bd07a5e73ccef8a18d4c15b1fa9ea44de2 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Fri, 31 Jul 2026 13:27:11 -0700 Subject: [PATCH 423/562] [CALCITE-7684] HOPPING and TUMBLING window queries crash at runtime for NULL timestamps Signed-off-by: Mihai Budiu --- .../calcite/adapter/enumerable/EnumUtils.java | 66 +++++++++++++----- .../adapter/enumerable/RexImpTable.java | 12 +--- core/src/test/resources/sql/stream.iq | 69 +++++++++++++++++++ 3 files changed, 121 insertions(+), 26 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java index 80c46ec3f9ea..d3da43466da7 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java @@ -1036,7 +1036,7 @@ static Expression generatePredicate( static Expression tumblingWindowSelector( PhysType inputPhysType, PhysType outputPhysType, - Expression wmColExpr, + int wmColIndex, Expression windowSizeExpr, Expression offsetExpr) { // Generate all fields. @@ -1053,6 +1053,9 @@ static Expression tumblingWindowSelector( outputPhysType.getJavaFieldType(expressions.size())); expressions.add(expression); } + final Expression wmColExpr = + inputPhysType.fieldReference(parameter, wmColIndex, + outputPhysType.getJavaFieldType(fieldCount)); final Expression wmColExprToLong = EnumUtils.convert(wmColExpr, long.class); // Find the fixed window for a timestamp given a window size and an offset, and return the @@ -1074,8 +1077,20 @@ static Expression tumblingWindowSelector( expressions.add(windowEndExpr); - return Expressions.lambda(Function1.class, - outputPhysType.record(expressions), parameter); + Expression body = outputPhysType.record(expressions); + if (inputPhysType.getRowType().getFieldList().get(wmColIndex).getType() + .isNullable()) { + // A row whose timestamp is NULL belongs to no window, since window_start + // and window_end are declared NOT NULL. Return null for such a row + body = + Expressions.condition( + Expressions.equal( + inputPhysType.fieldReference(parameter, wmColIndex), + Expressions.constant(null)), + Expressions.constant(null, body.getType()), + body); + } + return Expressions.lambda(Function1.class, body, parameter); } /** @@ -1307,13 +1322,22 @@ private static class HopEnumerator implements Enumerator<@Nullable Object[]> { } @Override public @Nullable Object[] current() { - if (!list.isEmpty()) { - return takeOne(); - } else { + return takeOne(); + } + + @Override public boolean moveNext() { + // Expand input rows until one of them yields a window. A row whose + // timestamp is NULL belongs to no window, and window_start and + // window_end are declared NOT NULL, so such a row is discarded. + while (list.isEmpty()) { + if (!inputEnumerator.moveNext()) { + return false; + } @Nullable Object[] current = inputEnumerator.current(); - Object watermark = - requireNonNull(current[indexOfWatermarkedColumn], - "element[indexOfWatermarkedColumn]"); + Object watermark = current[indexOfWatermarkedColumn]; + if (watermark == null) { + continue; + } PairList windows = hopWindows(SqlFunctions.toLong(watermark), emitFrequency, windowSize, offset); @@ -1324,12 +1348,8 @@ private static class HopEnumerator implements Enumerator<@Nullable Object[]> { curWithWindow[current.length + 1] = right; list.offer(curWithWindow); }); - return takeOne(); } - } - - @Override public boolean moveNext() { - return !list.isEmpty() || inputEnumerator.moveNext(); + return true; } @Override public void reset() { @@ -1367,17 +1387,29 @@ public static Enumerable tumbling( Function1 outSelector) { return new AbstractEnumerable() { // Applies tumbling on each element from the input enumerator and produces - // exactly one element for each input element. + // at most one element for each input element. @Override public Enumerator enumerator() { return new Enumerator() { final Enumerator inputs = inputEnumerable.enumerator(); + @Nullable TResult current; @Override public TResult current() { - return outSelector.apply(inputs.current()); + return requireNonNull(current, "current"); } @Override public boolean moveNext() { - return inputs.moveNext(); + // The selector returns null for a row whose timestamp is NULL: + // such a row belongs to no window, and window_start and window_end + // are declared NOT NULL, so the row is discarded. + while (inputs.moveNext()) { + TResult result = outSelector.apply(inputs.current()); + if (result != null) { + current = result; + return true; + } + } + current = null; + return false; } @Override public void reset() { diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java index 2ad2be289378..2ca2528d5c0f 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java @@ -4970,14 +4970,8 @@ private static class TumbleImplementor implements TableFunctionCallImplementor { // represents the input, see StandardConvertletTable#convertWindowFunction. Expression intervalExpression = translator.translate(call.getOperands().get(1)); RexCall descriptor = (RexCall) call.getOperands().get(0); - final ParameterExpression parameter = - Expressions.parameter(Primitive.box(inputPhysType.getJavaRowType()), - "_input"); - Expression wmColExpr = - inputPhysType.fieldReference(parameter, - ((RexInputRef) descriptor.getOperands().get(0)).getIndex(), - outputPhysType.getJavaFieldType( - inputPhysType.getRowType().getFieldCount())); + final int wmColIndex = + ((RexInputRef) descriptor.getOperands().get(0)).getIndex(); // handle the optional offset parameter. Use 0 for the default value when offset // parameter is not set. @@ -4991,7 +4985,7 @@ private static class TumbleImplementor implements TableFunctionCallImplementor { EnumUtils.tumblingWindowSelector( inputPhysType, outputPhysType, - wmColExpr, + wmColIndex, intervalExpression, offsetExpr)); } diff --git a/core/src/test/resources/sql/stream.iq b/core/src/test/resources/sql/stream.iq index f20a7fe6407b..394114376265 100644 --- a/core/src/test/resources/sql/stream.iq +++ b/core/src/test/resources/sql/stream.iq @@ -95,6 +95,40 @@ SELECT * FROM TABLE(TUMBLE((SELECT * FROM ORDERS), DESCRIPTOR(ROWTIME), INTERVAL !ok +# Test case for [CALCITE-7684] HOPPING and TUMBLING window queries crash at +# runtime for NULL timestamps. +# A row whose timestamp is NULL belongs to no window; window_start and +# window_end are declared NOT NULL, so such a row is discarded. +SELECT * FROM TABLE( + TUMBLE( + (SELECT * FROM (VALUES + (TIMESTAMP '2020-01-01 10:00:00', 'a'), + (CAST(NULL AS TIMESTAMP), 'b')) AS T(TS, UID)), + DESCRIPTOR(TS), INTERVAL '1' HOUR)); ++---------------------+-----+---------------------+---------------------+ +| TS | UID | window_start | window_end | ++---------------------+-----+---------------------+---------------------+ +| 2020-01-01 10:00:00 | a | 2020-01-01 10:00:00 | 2020-01-01 11:00:00 | ++---------------------+-----+---------------------+---------------------+ +(1 row) + +!ok + +# As above, but every row is discarded, so the result is empty. +SELECT * FROM TABLE( + TUMBLE( + (SELECT * FROM (VALUES + (CAST(NULL AS TIMESTAMP), 'a'), + (CAST(NULL AS TIMESTAMP), 'b')) AS T(TS, UID)), + DESCRIPTOR(TS), INTERVAL '1' HOUR)); ++----+-----+--------------+------------+ +| TS | UID | window_start | window_end | ++----+-----+--------------+------------+ ++----+-----+--------------+------------+ +(0 rows) + +!ok + SELECT * FROM TABLE(HOP(TABLE ORDERS, DESCRIPTOR(ROWTIME), INTERVAL '5' MINUTE, INTERVAL '10' MINUTE)); +---------------------+----+---------+-------+---------------------+---------------------+ | ROWTIME | ID | PRODUCT | UNITS | window_start | window_end | @@ -176,6 +210,41 @@ SELECT * FROM TABLE(HOP((SELECT * FROM ORDERS), DESCRIPTOR(ROWTIME), INTERVAL '5 !ok +# Test case for [CALCITE-7684] HOPPING and TUMBLING window queries crash at +# runtime for NULL timestamps. +# A row whose timestamp is NULL belongs to no window, not even to the first +# of the windows that HOP would otherwise produce for it. +SELECT * FROM TABLE( + HOP( + (SELECT * FROM (VALUES + (TIMESTAMP '2020-01-01 10:00:00', 'a'), + (CAST(NULL AS TIMESTAMP), 'b')) AS T(TS, UID)), + DESCRIPTOR(TS), INTERVAL '30' MINUTE, INTERVAL '1' HOUR)); ++---------------------+-----+---------------------+---------------------+ +| TS | UID | window_start | window_end | ++---------------------+-----+---------------------+---------------------+ +| 2020-01-01 10:00:00 | a | 2020-01-01 09:30:00 | 2020-01-01 10:30:00 | +| 2020-01-01 10:00:00 | a | 2020-01-01 10:00:00 | 2020-01-01 11:00:00 | ++---------------------+-----+---------------------+---------------------+ +(2 rows) + +!ok + +# As above, but every row is discarded, so the result is empty. +SELECT * FROM TABLE( + HOP( + (SELECT * FROM (VALUES + (CAST(NULL AS TIMESTAMP), 'a'), + (CAST(NULL AS TIMESTAMP), 'b')) AS T(TS, UID)), + DESCRIPTOR(TS), INTERVAL '30' MINUTE, INTERVAL '1' HOUR)); ++----+-----+--------------+------------+ +| TS | UID | window_start | window_end | ++----+-----+--------------+------------+ ++----+-----+--------------+------------+ +(0 rows) + +!ok + SELECT * FROM TABLE(SESSION(TABLE ORDERS, DESCRIPTOR(ROWTIME), DESCRIPTOR(PRODUCT), INTERVAL '20' MINUTE)); +---------------------+----+---------+-------+---------------------+---------------------+ | ROWTIME | ID | PRODUCT | UNITS | window_start | window_end | From a41adf58f9cb4f3e5ce1ef42dc88584c33fd3293 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Fri, 19 Jun 2026 00:01:30 -0700 Subject: [PATCH 424/562] Stronger validation for JOIN UNNEST Signed-off-by: Mihai Budiu --- .../calcite/runtime/CalciteResource.java | 3 ++ .../sql/validate/SqlValidatorImpl.java | 30 +++++++++++++++++++ .../runtime/CalciteResource.properties | 1 + .../apache/calcite/test/SqlValidatorTest.java | 24 +++++++++++++++ 4 files changed, 58 insertions(+) diff --git a/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java b/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java index c6e1a4dbdcc5..c5047574a3d0 100644 --- a/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java +++ b/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java @@ -337,6 +337,9 @@ ExInst invalidCompare(String a0, String a1, String a2, @BaseMessage("Cannot specify condition (NATURAL keyword, or ON or USING clause) following CROSS JOIN") ExInst crossJoinDisallowsCondition(); + @BaseMessage("UNNEST is only supported with INNER, LEFT, CROSS, or COMMA join, not ''{0}''") + ExInst unnestInvalidJoinType(String a0); + @BaseMessage("Cannot specify NATURAL keyword with ON or USING clause") ExInst naturalDisallowsOnOrUsing(); diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java index 023de1b77f77..7021696b679c 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java @@ -4116,6 +4116,20 @@ protected void validateJoin(SqlJoin join, SqlValidatorScope scope) { } } + // UNNEST on the right side is only meaningful with INNER, LEFT, CROSS, or COMMA. + if (isUnnestNode(right)) { + switch (joinType) { + case INNER: + case LEFT: + case CROSS: + case COMMA: + break; + default: + throw newValidationError(join.getJoinTypeNode(), + RESOURCE.unnestInvalidJoinType(joinType.name())); + } + } + // Which join types require/allow a ON/USING condition, or allow // a NATURAL keyword? switch (joinType) { @@ -4192,6 +4206,22 @@ protected void validateJoin(SqlJoin join, SqlValidatorScope scope) { } } + /** + * Returns whether {@code node} is (or wraps, via AS or LATERAL) an + * {@code UNNEST} call. + */ + private static boolean isUnnestNode(SqlNode node) { + switch (node.getKind()) { + case UNNEST: + return true; + case AS: + case LATERAL: + return isUnnestNode(((SqlCall) node).operand(0)); + default: + return false; + } + } + /** * Shuttle which determines whether all SqlCalls that are * comparisons are comparing columns from both namespaces. diff --git a/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties b/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties index f4f16d73266a..a90099d7cb91 100644 --- a/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties +++ b/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties @@ -115,6 +115,7 @@ AliasListDuplicate=Duplicate name ''{0}'' in column alias list JoinRequiresCondition=INNER, LEFT, RIGHT, FULL, or ASOF join requires a condition (NATURAL keyword or ON or USING clause) DisallowsQualifyingCommonColumn=Cannot qualify common column ''{0}'' CrossJoinDisallowsCondition=Cannot specify condition (NATURAL keyword, or ON or USING clause) following CROSS JOIN +UnnestInvalidJoinType=UNNEST is only supported with INNER, LEFT, CROSS, or COMMA join, not ''{0}'' NaturalDisallowsOnOrUsing=Cannot specify NATURAL keyword with ON or USING clause ColumnInUsingNotUnique=Column name ''{0}'' in NATURAL join or USING clause is not unique on one side of join NaturalOrUsingColumnNotCompatible=Column ''{0}'' matched using NATURAL keyword or USING clause has incompatible types: cannot compare ''{1}'' to ''{2}'' diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index 8f95099ab952..018c27e898f1 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -9855,6 +9855,30 @@ void testGroupExpressionEquivalenceParams() { .fails("Column 'ORDINALITY' not found in any table"); } + /** UNNEST is valid with INNER, LEFT, CROSS, and COMMA joins; + * all other join kinds must be rejected by the validator. */ + @Test void testUnnestJoinType() { + // Allowed join kinds — these must all validate without error. + sql("select * from dept inner join unnest(array[1, 2]) as u(x) on true").ok(); + sql("select * from dept left join unnest(array[1, 2]) as u(x) on true").ok(); + sql("select * from dept cross join unnest(array[1, 2]) as u(x)").ok(); + sql("select * from dept, unnest(array[1, 2]) as u(x)").ok(); + + // LATERAL wrapping must also be allowed for valid join kinds. + sql("select * from dept cross join lateral unnest(array[1, 2]) as u(x)").ok(); + sql("select * from dept left join lateral unnest(array[1, 2]) as u(x) on true").ok(); + + // Disallowed join kinds — validator must reject these. + sql("select * from dept right ^join^ unnest(array[1, 2]) as u(x) on true") + .fails("UNNEST is only supported with INNER, LEFT, CROSS, or COMMA join, not 'RIGHT'"); + sql("select * from dept full ^join^ unnest(array[1, 2]) as u(x) on true") + .fails("UNNEST is only supported with INNER, LEFT, CROSS, or COMMA join, not 'FULL'"); + + // LATERAL wrapping must also be rejected for invalid join kinds. + sql("select * from dept right ^join^ lateral unnest(array[1, 2]) as u(x) on true") + .fails("UNNEST is only supported with INNER, LEFT, CROSS, or COMMA join, not 'RIGHT'"); + } + @Test void unnestMapMustNameColumnsKeyAndValueWhenNotAliased() { sql("select * from unnest(map[1, 12, 2, 22])") .type("RecordType(INTEGER NOT NULL KEY, INTEGER NOT NULL VALUE) NOT NULL"); From f5c022920acf4811177d77e06c837ba5cd1ae951 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Sat, 1 Aug 2026 21:47:17 -0700 Subject: [PATCH 425/562] [CALCITE-7669] Uncollect should support the Trino semantics of UNNEST Signed-off-by: Mihai Budiu --- .../enumerable/EnumerableUncollect.java | 38 +++- .../enumerable/EnumerableUncollectRule.java | 2 +- .../apache/calcite/rel/core/Uncollect.java | 120 ++++++++--- .../rel/logical/ToLogicalConverter.java | 5 +- .../calcite/rel/mutable/MutableRels.java | 5 +- .../calcite/rel/mutable/MutableUncollect.java | 37 +++- .../apache/calcite/runtime/SqlFunctions.java | 49 ++++- .../apache/calcite/sql/SqlUnnestOperator.java | 13 +- .../calcite/sql2rel/SqlToRelConverter.java | 4 +- .../apache/calcite/test/CoreQuidemTest.java | 7 + .../apache/calcite/test/SqlFunctionsTest.java | 82 ++++++++ .../calcite/test/SqlToRelConverterTest.xml | 14 +- core/src/test/resources/sql/unnest.iq | 189 ++++++++++++++++++ 13 files changed, 508 insertions(+), 57 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableUncollect.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableUncollect.java index de167cd08e4f..8f193a4acede 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableUncollect.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableUncollect.java @@ -51,7 +51,16 @@ public EnumerableUncollect(RelOptCluster cluster, RelTraitSet traitSet, *

      Use {@link #create} unless you know what you're doing. */ public EnumerableUncollect(RelOptCluster cluster, RelTraitSet traitSet, RelNode child, boolean withOrdinality) { - super(cluster, traitSet, child, withOrdinality, Collections.emptyList()); + this(cluster, traitSet, child, withOrdinality, true); + } + + /** Creates an EnumerableUncollect. + * + *

      Use {@link #create} unless you know what you're doing. */ + public EnumerableUncollect(RelOptCluster cluster, RelTraitSet traitSet, + RelNode child, boolean withOrdinality, boolean expandStructFields) { + super(cluster, traitSet, child, withOrdinality, Collections.emptyList(), + expandStructFields); assert getConvention() instanceof EnumerableConvention; assert getConvention() == child.getConvention(); } @@ -72,10 +81,27 @@ public static EnumerableUncollect create(RelTraitSet traitSet, RelNode input, return new EnumerableUncollect(cluster, traitSet, input, withOrdinality); } + /** + * Creates an EnumerableUncollect. + * + * @param traitSet Trait set + * @param input Input relational expression + * @param withOrdinality Whether output should contain an ORDINALITY column + * @param expandStructFields If true, a collection whose element type is a struct + * produces one output column per struct field; if false, + * a single column typed as the whole element + */ + public static EnumerableUncollect create(RelTraitSet traitSet, RelNode input, + boolean withOrdinality, boolean expandStructFields) { + final RelOptCluster cluster = input.getCluster(); + return new EnumerableUncollect(cluster, traitSet, input, withOrdinality, + expandStructFields); + } + @Override public EnumerableUncollect copy(RelTraitSet traitSet, RelNode newInput) { return new EnumerableUncollect(getCluster(), traitSet, newInput, - withOrdinality); + withOrdinality, expandStructFields); } @Override public Result implement(EnumerableRelImplementor implementor, Prefer pref) { @@ -105,7 +131,7 @@ public static EnumerableUncollect create(RelTraitSet traitSet, RelNode input, inputTypes.add(FlatProductInputType.MAP); } else { final RelDataType elementType = getComponentTypeOrThrow(type); - if (elementType.isStruct()) { + if (elementType.isStruct() && expandStructFields) { if (elementType.getFieldCount() == 1 && child.getRowType().getFieldList().size() == 1 && !withOrdinality) { // Solves CALCITE-4063: if we are processing a single field, which is a struct with a @@ -116,6 +142,12 @@ public static EnumerableUncollect create(RelTraitSet traitSet, RelNode input, fieldCounts.add(elementType.getFieldCount()); inputTypes.add(FlatProductInputType.LIST); } + } else if (elementType.isStruct()) { + // A struct element kept whole occupies a single output column, + // like a scalar element, but its row value must be converted from + // the collection's internal list representation to Object[]. + fieldCounts.add(-1); + inputTypes.add(FlatProductInputType.STRUCT); } else { fieldCounts.add(-1); inputTypes.add(FlatProductInputType.SCALAR); diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableUncollectRule.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableUncollectRule.java index 9079964897dc..95a9237c2222 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableUncollectRule.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableUncollectRule.java @@ -49,6 +49,6 @@ protected EnumerableUncollectRule(Config config) { convert(input, input.getTraitSet().replace(EnumerableConvention.INSTANCE)); return EnumerableUncollect.create(traitSet, newInput, - uncollect.withOrdinality); + uncollect.withOrdinality, uncollect.expandStructFields); } } diff --git a/core/src/main/java/org/apache/calcite/rel/core/Uncollect.java b/core/src/main/java/org/apache/calcite/rel/core/Uncollect.java index 2d4c3620a74c..e607509ddc18 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/Uncollect.java +++ b/core/src/main/java/org/apache/calcite/rel/core/Uncollect.java @@ -50,10 +50,21 @@ *

      Like its inverse operation {@link Collect}, Uncollect is generally * invoked in a nested loop, driven by * {@link org.apache.calcite.rel.logical.LogicalCorrelate} or similar. + * + *

      {@code expandStructFields} controls the shape of the element columns: + * if {@code true} a collection whose element type is a struct produces one + * output column per struct field; if {@code false} it produces a single + * column typed as the whole element (Trino semantics). Maps always expand + * into a key and a value column, regardless of this flag. */ public class Uncollect extends SingleRel { public final boolean withOrdinality; + /** If true, a collection whose element type is a struct expands into one + * output column per struct field; if false, it produces a single column + * typed as the whole element. */ + public final boolean expandStructFields; + // To alias the items in Uncollect list, // i.e., "UNNEST(a, b, c) as T(d, e, f)" // outputs as row type Record(d, e, f) where the field "d" has element type of "a", @@ -74,12 +85,30 @@ public Uncollect(RelOptCluster cluster, RelTraitSet traitSet, /** Creates an Uncollect. * *

      Use {@link #create} unless you know what you're doing. */ - @SuppressWarnings("method.invocation.invalid") public Uncollect(RelOptCluster cluster, RelTraitSet traitSet, RelNode input, boolean withOrdinality, List itemAliases) { + // Non-empty item aliases historically implied that struct elements are not + // expanded (Presto dialect), so this constructor derives + // {@code expandStructFields} from their absence. + this(cluster, traitSet, input, withOrdinality, itemAliases, itemAliases.isEmpty()); + } + + /** Creates an Uncollect. + * + * @param input Input relational expression + * @param withOrdinality Whether output should contain an ORDINALITY column + * @param itemAliases Aliases for the operand items + * @param expandStructFields If true, a collection whose element type is a struct + * produces one output column per struct field; if false, + * a single column typed as the whole element + */ + @SuppressWarnings("method.invocation.invalid") + public Uncollect(RelOptCluster cluster, RelTraitSet traitSet, RelNode input, + boolean withOrdinality, List itemAliases, boolean expandStructFields) { super(cluster, traitSet, input); this.withOrdinality = withOrdinality; this.itemAliases = ImmutableList.copyOf(itemAliases); + this.expandStructFields = expandStructFields; requireNonNull(deriveRowType(), "invalid child rowType"); } @@ -88,7 +117,8 @@ public Uncollect(RelOptCluster cluster, RelTraitSet traitSet, RelNode input, */ public Uncollect(RelInput input) { this(input.getCluster(), input.getTraitSet(), input.getInput(), - input.getBoolean("withOrdinality", false), Collections.emptyList()); + input.getBoolean("withOrdinality", false), Collections.emptyList(), + input.getBoolean("expandStructFields", true)); } /** @@ -111,6 +141,28 @@ public static Uncollect create( return new Uncollect(cluster, traitSet, input, withOrdinality, itemAliases); } + /** + * Creates an Uncollect. + * + * @param traitSet Trait set + * @param input Input relational expression + * @param withOrdinality Whether output should contain an ORDINALITY column + * @param itemAliases Aliases for the operand items + * @param expandStructFields If true, a collection whose element type is a struct + * produces one output column per struct field; if false, + * a single column typed as the whole element + */ + public static Uncollect create( + RelTraitSet traitSet, + RelNode input, + boolean withOrdinality, + List itemAliases, + boolean expandStructFields) { + final RelOptCluster cluster = input.getCluster(); + return new Uncollect(cluster, traitSet, input, withOrdinality, itemAliases, + expandStructFields); + } + //~ Methods ---------------------------------------------------------------- @Override public RelNode accept(RelShuttle shuttle) { @@ -119,7 +171,8 @@ public static Uncollect create( @Override public RelWriter explainTerms(RelWriter pw) { return super.explainTerms(pw) - .itemIf("withOrdinality", withOrdinality, withOrdinality); + .itemIf("withOrdinality", withOrdinality, withOrdinality) + .itemIf("expandStructFields", expandStructFields, !expandStructFields); } @Override public final RelNode copy(RelTraitSet traitSet, @@ -129,34 +182,47 @@ public static Uncollect create( public RelNode copy(RelTraitSet traitSet, RelNode input) { assert traitSet.containsIfApplicable(Convention.NONE); - return new Uncollect(getCluster(), traitSet, input, withOrdinality, itemAliases); - } - - @Override protected RelDataType deriveRowType() { - return deriveUncollectRowType(input, withOrdinality, itemAliases); + return new Uncollect(getCluster(), traitSet, input, withOrdinality, itemAliases, + expandStructFields); } /** * Returns the row type returned by applying the 'UNNEST' operation to a * relational expression. * - *

      Each column in the relational expression must be a multiset of - * structs or an array. The return type is the combination of expanding - * element types from each column, plus an ORDINALITY column if {@code - * withOrdinality}. If {@code itemAliases} is not empty, the element types - * would not expand, each column element outputs as a whole (the return - * type has same column types as input type). + * @deprecated Construct an {@link Uncollect} and call + * {@link #getRowType()} instead. */ + @Deprecated // to be removed before 2.0 public static RelDataType deriveUncollectRowType(RelNode rel, boolean withOrdinality, List itemAliases) { - RelDataType inputType = rel.getRowType(); + return new Uncollect(rel.getCluster(), rel.getTraitSet(), rel, + withOrdinality, itemAliases).getRowType(); + } + + /** + * Returns the row type of the 'UNNEST' operation. + * + *

      Each column in the input relational expression must be a multiset of + * structs or an array. The return type is the combination of expanding + * element types from each column, plus an ORDINALITY column if {@code + * withOrdinality}. + * + *

      {@code expandStructFields} controls the expansion of struct element + * types: if {@code true}, one output column per struct field; if {@code + * false}, a single column typed as the whole element. Maps always expand + * into a key and a value column. {@code itemAliases}, when not empty, + * names the non-expanded element columns. + */ + @Override protected RelDataType deriveRowType() { + RelDataType inputType = input.getRowType(); assert inputType.isStruct() : inputType + " is not a struct"; boolean requireAlias = !itemAliases.isEmpty(); assert !requireAlias || itemAliases.size() == inputType.getFieldCount(); final List fields = inputType.getFieldList(); - final RelDataTypeFactory typeFactory = rel.getCluster().getTypeFactory(); + final RelDataTypeFactory typeFactory = getCluster().getTypeFactory(); final RelDataTypeFactory.Builder builder = typeFactory.builder(); if (fields.size() == 1 @@ -192,12 +258,7 @@ public static RelDataType deriveUncollectRowType(RelNode rel, throw RESOURCE.unnestArgument().ex(); } boolean isNullable = componentType.isNullable() || padNullable; - if (requireAlias) { - RelDataType colType = padNullable - ? typeFactory.enforceTypeWithNullability(componentType, true) - : componentType; - builder.add(itemAliases.get(i), colType); - } else if (componentType.isStruct()) { + if (expandStructFields && componentType.isStruct()) { for (RelDataTypeField fieldInfo : componentType.getFieldList()) { RelDataType fieldType = fieldInfo.getType(); if (isNullable) { @@ -206,11 +267,18 @@ public static RelDataType deriveUncollectRowType(RelNode rel, builder.add(fieldInfo.getName(), fieldType); } } else { - // Element type is not a record, use the field name of the element directly - RelDataType colType = padNullable - ? typeFactory.enforceTypeWithNullability(componentType, true) + // A single column typed as the whole element, named by the item + // alias when present, otherwise by the collection field's name. + RelDataType elementType = componentType.isStruct() + ? typeFactory.builder().kind(componentType.getStructKind()) + .addAll(componentType.getFieldList()).build() : componentType; - builder.add(field.getName(), colType); + // A NULL collection element becomes a NULL value in this column, so + // the column is nullable whenever the element type is. + RelDataType colType = isNullable + ? typeFactory.enforceTypeWithNullability(elementType, true) + : elementType; + builder.add(requireAlias ? itemAliases.get(i) : field.getName(), colType); } } } diff --git a/core/src/main/java/org/apache/calcite/rel/logical/ToLogicalConverter.java b/core/src/main/java/org/apache/calcite/rel/logical/ToLogicalConverter.java index d7cee2dae7ac..4ff564f1fd54 100644 --- a/core/src/main/java/org/apache/calcite/rel/logical/ToLogicalConverter.java +++ b/core/src/main/java/org/apache/calcite/rel/logical/ToLogicalConverter.java @@ -41,8 +41,6 @@ import org.apache.calcite.rel.core.Window; import org.apache.calcite.tools.RelBuilder; -import java.util.Collections; - /** * Shuttle to convert any rel plan to a plan with all logical nodes. */ @@ -191,7 +189,8 @@ public ToLogicalConverter(RelBuilder relBuilder) { final Uncollect uncollect = (Uncollect) relNode; final RelNode input = visit(uncollect.getInput()); return Uncollect.create(input.getTraitSet(), input, - uncollect.withOrdinality, Collections.emptyList()); + uncollect.withOrdinality, uncollect.getItemAliases(), + uncollect.expandStructFields); } throw new AssertionError("Need to implement logical converter for " diff --git a/core/src/main/java/org/apache/calcite/rel/mutable/MutableRels.java b/core/src/main/java/org/apache/calcite/rel/mutable/MutableRels.java index ed509b6d60b6..176be5cfec66 100644 --- a/core/src/main/java/org/apache/calcite/rel/mutable/MutableRels.java +++ b/core/src/main/java/org/apache/calcite/rel/mutable/MutableRels.java @@ -257,7 +257,7 @@ public static RelNode fromMutable(MutableRel node, RelBuilder relBuilder) { final MutableUncollect uncollect = (MutableUncollect) node; final RelNode child = fromMutable(uncollect.getInput(), relBuilder); return Uncollect.create(child.getTraitSet(), child, uncollect.withOrdinality, - Collections.emptyList()); + Collections.emptyList(), uncollect.expandStructFields); } case WINDOW: { final MutableWindow window = (MutableWindow) node; @@ -378,7 +378,8 @@ public static MutableRel toMutable(RelNode rel) { if (rel instanceof Uncollect) { final Uncollect uncollect = (Uncollect) rel; final MutableRel input = toMutable(uncollect.getInput()); - return MutableUncollect.of(uncollect.getRowType(), input, uncollect.withOrdinality); + return MutableUncollect.of(uncollect.getRowType(), input, + uncollect.withOrdinality, uncollect.expandStructFields); } if (rel instanceof Window) { final Window window = (Window) rel; diff --git a/core/src/main/java/org/apache/calcite/rel/mutable/MutableUncollect.java b/core/src/main/java/org/apache/calcite/rel/mutable/MutableUncollect.java index 594d109b5d69..bae3854f6948 100644 --- a/core/src/main/java/org/apache/calcite/rel/mutable/MutableUncollect.java +++ b/core/src/main/java/org/apache/calcite/rel/mutable/MutableUncollect.java @@ -25,15 +25,17 @@ /** Mutable equivalent of {@link org.apache.calcite.rel.core.Uncollect}. */ public class MutableUncollect extends MutableSingleRel { public final boolean withOrdinality; + public final boolean expandStructFields; private MutableUncollect(RelDataType rowType, - MutableRel input, boolean withOrdinality) { + MutableRel input, boolean withOrdinality, boolean expandStructFields) { super(MutableRelType.UNCOLLECT, rowType, input); this.withOrdinality = withOrdinality; + this.expandStructFields = expandStructFields; } /** - * Creates a MutableUncollect. + * Creates a MutableUncollect that expands struct elements. * * @param rowType Row type * @param input Input relational expression @@ -42,26 +44,47 @@ private MutableUncollect(RelDataType rowType, */ public static MutableUncollect of(RelDataType rowType, MutableRel input, boolean withOrdinality) { - return new MutableUncollect(rowType, input, withOrdinality); + return of(rowType, input, withOrdinality, true); + } + + /** + * Creates a MutableUncollect. + * + * @param rowType Row type + * @param input Input relational expression + * @param withOrdinality Whether the output contains an extra + * {@code ORDINALITY} column + * @param expandStructFields If true, a collection whose element type + * is a struct produces one output column per + * struct field; if false, a single column + * typed as the whole element + */ + public static MutableUncollect of(RelDataType rowType, + MutableRel input, boolean withOrdinality, boolean expandStructFields) { + return new MutableUncollect(rowType, input, withOrdinality, + expandStructFields); } @Override public boolean equals(@Nullable Object obj) { return obj == this || obj instanceof MutableUncollect && withOrdinality == ((MutableUncollect) obj).withOrdinality + && expandStructFields == ((MutableUncollect) obj).expandStructFields && input.equals(((MutableUncollect) obj).input); } @Override public int hashCode() { - return Objects.hash(input, withOrdinality); + return Objects.hash(input, withOrdinality, expandStructFields); } @Override public StringBuilder digest(StringBuilder buf) { - return buf.append("Uncollect(withOrdinality: ") - .append(withOrdinality).append(")"); + return buf.append("Uncollect(withOrdinality: ").append(withOrdinality) + .append(", expandStructFields: ").append(expandStructFields) + .append(")"); } @Override public MutableRel clone() { - return MutableUncollect.of(rowType, input.clone(), withOrdinality); + return MutableUncollect.of(rowType, input.clone(), withOrdinality, + expandStructFields); } } diff --git a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java index 56e6bdb42298..4b9b48041fad 100644 --- a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java +++ b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java @@ -209,6 +209,23 @@ public class SqlFunctions { private static final Function1, Enumerable> LIST_AS_ENUMERABLE = a0 -> a0 == null ? Linq4j.emptyEnumerable() : Linq4j.asEnumerable(a0); + /** Like {@link #LIST_AS_ENUMERABLE}, for a collection whose struct elements + * are kept whole: each element is converted to an Object[] struct value. */ + private static final Function1, Enumerable<@Nullable Object>> + STRUCT_LIST_AS_ENUMERABLE = + a0 -> a0 == null ? Linq4j.emptyEnumerable() + : Linq4j.asEnumerable(a0).<@Nullable Object>select(SqlFunctions::structValue); + + /** Converts one element of a collection of structs to its Object[] struct + * value. Elements arrive as List or as Object[]; null elements stay null. */ + @SuppressWarnings("rawtypes") + private static @Nullable Object structValue(@Nullable Object element) { + if (element == null || element instanceof Object[]) { + return element; + } + return ((List) element).toArray(); + } + @SuppressWarnings("unused") private static final Function1> ARRAY_CARTESIAN_PRODUCT = SqlFunctions::arrayCartesianProduct; @@ -7590,8 +7607,15 @@ public static Function1>> flatZip( // Simple unnest without ordinality //noinspection unchecked return (Function1) LIST_AS_ENUMERABLE; + } else if (!withOrdinality && inputTypes[0] == FlatProductInputType.STRUCT) { + // A single collection of structs kept whole, without ordinality: the + // output row type has a single (ROW-typed) column, so PhysTypeImpl + // optimizes the row format down to SCALAR, under which rows are bare + // struct values rather than singleton lists. + //noinspection unchecked + return (Function1) STRUCT_LIST_AS_ENUMERABLE; } else { - // unnest with ordinality for a single scalar column + // unnest with ordinality for a single column return row -> z2(new Object[] { row }, fieldCounts, withOrdinality, inputTypes); } } @@ -7604,9 +7628,10 @@ public static Function1>> flatZip( * padding shorter collections with {@code NULL}. * * @param lists one element per collection (scalar list, struct list, or map) - * @param fieldCounts output column count for each collection (-1 for a collection of scalars) + * @param fieldCounts output column count for each collection (-1 for a collection + * of scalars or of structs kept whole) * @param withOrdinality whether to append a 1-based ordinality column - * @param inputTypes type of elements in each collection (SCALAR, LIST, or MAP) + * @param inputTypes type of elements in each collection (SCALAR, LIST, STRUCT, or MAP) */ @SuppressWarnings("rawtypes") private static Enumerable> z2( @@ -7626,6 +7651,17 @@ private static Enumerable> z2( enumerators.add(Linq4j.transform(Linq4j.enumerator(list), FlatLists::of)); widths[i] = 1; break; + case STRUCT: + // A struct element kept whole occupies a single output column, like a + // scalar element, but its value must be converted to Object[]. + @SuppressWarnings("unchecked") List structList = + (List) inputObject; + @SuppressWarnings("unchecked") Enumerator> structEnumerator = + (Enumerator) Linq4j.transform(Linq4j.enumerator(structList), + (Object e) -> FlatLists.ofSingle(structValue(e))); + enumerators.add(structEnumerator); + widths[i] = 1; + break; case LIST: @SuppressWarnings("unchecked") List> listList = (List>) inputObject; @@ -7766,7 +7802,10 @@ private static class ZipPaddedEnumerator int width = widths[i]; if (!endOfCollection[i]) { final Object elemRow = enumerators.get(i).current(); - if (elemRow instanceof Object[]) { + if (elemRow == null) { + // A NULL struct element expands to a row of NULLs, one per field. + Arrays.fill(flatElements, column, column + width, null); + } else if (elemRow instanceof Object[]) { final Object[] arr = (Object[]) elemRow; for (int p = 0; p < width; p++) { flatElements[column + p] = p < arr.length ? arr[p] : null; @@ -7900,7 +7939,7 @@ public enum JsonScope { /** Type of argument passed into {@link #flatZip}. */ public enum FlatProductInputType { - SCALAR, LIST, MAP + SCALAR, LIST, MAP, STRUCT } /** Type of part to extract passed into {@link ParseUrlFunction#parseUrl}. */ diff --git a/core/src/main/java/org/apache/calcite/sql/SqlUnnestOperator.java b/core/src/main/java/org/apache/calcite/sql/SqlUnnestOperator.java index af1753f8e8b5..61ada978234f 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlUnnestOperator.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlUnnestOperator.java @@ -99,6 +99,9 @@ public SqlUnnestOperator(boolean withOrdinality) { } else { RelDataType componentType = requireNonNull(type.getComponentType(), "componentType"); boolean isNullable = componentType.isNullable() || padNullable; + // Whether a struct element expands into one column per field depends + // on the SQL conformance; allowAliasUnnestItems describes how + // collections of ROW values are expanded. if (!allowAliasUnnestItems(opBinding) && componentType.isStruct()) { for (RelDataTypeField field : componentType.getFieldList()) { RelDataType fieldType = field.getType(); @@ -108,9 +111,15 @@ public SqlUnnestOperator(boolean withOrdinality) { builder.add(field.getName(), fieldType); } } else { - RelDataType colType = padNullable - ? typeFactory.enforceTypeWithNullability(componentType, true) + RelDataType elementType = componentType.isStruct() + ? typeFactory.builder().kind(componentType.getStructKind()) + .addAll(componentType.getFieldList()).build() : componentType; + // A NULL collection element becomes a NULL value in this column, so + // the column is nullable whenever the element type is. + RelDataType colType = isNullable + ? typeFactory.enforceTypeWithNullability(elementType, true) + : elementType; builder.add(SqlUtil.deriveAliasFromOrdinal(operand), colType); } } diff --git a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java index 182ed7a254a3..3ca7ae44f854 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java @@ -2888,7 +2888,8 @@ private void convertUnnest(Blackboard bb, SqlCall call, @Nullable List f // so Uncollect's row type stays aligned with the validator. List itemAliases; if (fieldNames != null) { - itemAliases = fieldNames; + // do not include the ordinality column name + itemAliases = fieldNames.subList(0, nodes.size()); } else { itemAliases = new ArrayList<>(nodes.size()); for (int i = 0; i < nodes.size(); i++) { @@ -2899,6 +2900,7 @@ private void convertUnnest(Blackboard bb, SqlCall call, @Nullable List f .push(child) .project(exprs) .uncollect(itemAliases, operator.withOrdinality) + .let(r -> fieldNames == null ? r : r.rename(fieldNames)) .build(); } else { // REVIEW danny 2020-04-26: should we unify the normal field aliases and diff --git a/core/src/test/java/org/apache/calcite/test/CoreQuidemTest.java b/core/src/test/java/org/apache/calcite/test/CoreQuidemTest.java index b878bfb085e0..bbb547b85833 100644 --- a/core/src/test/java/org/apache/calcite/test/CoreQuidemTest.java +++ b/core/src/test/java/org/apache/calcite/test/CoreQuidemTest.java @@ -185,6 +185,13 @@ protected Collection data() { .with(CalciteAssert.SchemaSpec.STEELWHEELS) .with(Lex.BIG_QUERY)) .connect(); + case "hr-presto": + // Same as "hr", but uses PRESTO conformance, under which + // UNNEST(array) AS alias does not expand struct elements. + return customize(CalciteAssert.hr() + .with(CalciteConnectionProperty.CONFORMANCE, + SqlConformanceEnum.PRESTO)) + .connect(); default: return super.connect(name, reference); } diff --git a/core/src/test/java/org/apache/calcite/test/SqlFunctionsTest.java b/core/src/test/java/org/apache/calcite/test/SqlFunctionsTest.java index 9231437cf8ea..e3827c12530e 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlFunctionsTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlFunctionsTest.java @@ -46,6 +46,7 @@ import static org.apache.calcite.avatica.util.DateTimeUtils.timestampStringToUnixDate; import static org.apache.calcite.runtime.SqlFunctions.FlatProductInputType.LIST; import static org.apache.calcite.runtime.SqlFunctions.FlatProductInputType.SCALAR; +import static org.apache.calcite.runtime.SqlFunctions.FlatProductInputType.STRUCT; import static org.apache.calcite.runtime.SqlFunctions.arraysOverlap; import static org.apache.calcite.runtime.SqlFunctions.charLength; import static org.apache.calcite.runtime.SqlFunctions.concat; @@ -2238,4 +2239,85 @@ private static List> zipScalars( assertThat(rows.get(0), is(list(1, 2, 10, 20))); assertThat(rows.get(1), is(Arrays.asList(3, 4, null, null))); } + + /** The runtime representation of {@code ARRAY[ROW(1, 'x'), ROW(2, 'y')]}: + * a list whose elements are the field lists of each ROW. */ + private static List> rowArray() { + return Arrays.asList(FlatLists.of(1, "x"), FlatLists.of(2, "y")); + } + + @Test void testZipPaddedWholeStructElements() { + // Models the Trino semantics of + // UNNEST(ARRAY[ROW(1, 'x'), ROW(2, 'y')], ARRAY[10, 20]) AS t(s, i): + // the STRUCT collection keeps each ROW element whole, so column s holds + // the element as an Object[]; the scalar column i zips alongside. + @SuppressWarnings({"rawtypes", "unchecked"}) + final Function1>> fn = + SqlFunctions.flatZip( + new int[]{-1, -1}, // one output column per collection + false, // no ordinality + new SqlFunctions.FlatProductInputType[]{STRUCT, SCALAR}); + + final List> rows = new ArrayList<>(); + for (FlatLists.ComparableList row + : fn.apply(new Object[]{rowArray(), Arrays.asList(10, 20)})) { + rows.add(new ArrayList<>(row)); + } + + // Expected rows: ({1, 'x'}, 10) and ({2, 'y'}, 20). + assertThat(rows, hasSize(2)); + assertArrayEquals(new Object[]{1, "x"}, (Object[]) rows.get(0).get(0)); + assertThat(rows.get(0).get(1), is(10)); + assertArrayEquals(new Object[]{2, "y"}, (Object[]) rows.get(1).get(0)); + assertThat(rows.get(1).get(1), is(20)); + } + + @Test void testZipPaddedNullStructElement() { + // A null element of an expanded ROW ARRAY is a null List, which + // must be expanded to one null per ROW field rather than dereferencing the list. + @SuppressWarnings({"rawtypes", "unchecked"}) + final Function1>> fn = + SqlFunctions.flatZip( + new int[]{2, -1}, // two columns from the struct, one scalar column + false, // no ordinality + new SqlFunctions.FlatProductInputType[]{LIST, SCALAR}); + + final List> rows = new ArrayList<>(); + for (FlatLists.ComparableList row + : fn.apply(new Object[]{ + Arrays.asList(FlatLists.of(1, "x"), null), + Arrays.asList(10, 20)})) { + rows.add(new ArrayList<>(row)); + } + + assertThat(rows, hasSize(2)); + assertThat(rows.get(0), is(Arrays.asList(1, "x", 10))); + assertThat(rows.get(1), is(Arrays.asList(null, null, 20))); + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + @Test void testFlatZipSingleWholeStructCollection() { + // Models the Trino semantics of + // UNNEST(ARRAY[ROW(1, 'x'), ROW(2, 'y')]) AS t(s): + // the output has the single ROW-typed column s, which PhysTypeImpl stores + // in SCALAR row format, so each output row is the bare Object[] struct + // value rather than a singleton list. + final Function1>> fn = + SqlFunctions.flatZip( + new int[]{-1}, false, + new SqlFunctions.FlatProductInputType[]{STRUCT}); + + final List rows = new ArrayList<>(); + for (Object row : (Enumerable) fn.apply(rowArray())) { + rows.add(row); + } + + // Expected rows: {1, 'x'} and {2, 'y'}. + assertThat(rows, hasSize(2)); + assertArrayEquals(new Object[]{1, "x"}, (Object[]) rows.get(0)); + assertArrayEquals(new Object[]{2, "y"}, (Object[]) rows.get(1)); + + // UNNEST of a null array yields no rows. + assertThat(((Enumerable) fn.apply(null)).any(), is(false)); + } } diff --git a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml index aafc9c11efd3..217b2bbc03b0 100644 --- a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml @@ -336,7 +336,7 @@ from UNNEST(ARRAY[1, 2, 3]) as t]]> @@ -348,7 +348,7 @@ LogicalProject(T=[$0]) LogicalProject(DEPTNO=[$0], E=[$5], EMPNO=[$6.EMPNO]) LogicalCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{2, 3}]) LogicalTableScan(table=[[CATALOG, SALES, DEPT_NESTED_EXPANDED]]) - Uncollect + Uncollect(expandStructFields=[false]) LogicalProject(ADMINS=[$cor0.ADMINS], EMPLOYEES=[$cor0.EMPLOYEES]) LogicalValues(tuples=[[{ 0 }]]) ]]> @@ -365,7 +365,7 @@ from dept_nested_expanded as d CROSS JOIN LogicalProject(DEPTNO=[$0], E=[$5], EMPNO=[$6.EMPNO]) LogicalCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{2, 3}]) LogicalTableScan(table=[[CATALOG, SALES, DEPT_NESTED_EXPANDED]]) - Uncollect + Uncollect(expandStructFields=[false]) LogicalProject(ADMINS=[$cor0.ADMINS], EMPLOYEES=[$cor0.EMPLOYEES]) LogicalValues(tuples=[[{ 0 }]]) ]]> @@ -382,7 +382,7 @@ from dept_nested_expanded as d CROSS JOIN LogicalProject(DEPTNO=[$0], EMPNO=[$5.EMPNO]) LogicalCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{2}]) LogicalTableScan(table=[[CATALOG, SALES, DEPT_NESTED_EXPANDED]]) - Uncollect + Uncollect(expandStructFields=[false]) LogicalProject(EMPLOYEES=[$cor0.EMPLOYEES]) LogicalValues(tuples=[[{ 0 }]]) ]]> @@ -399,7 +399,7 @@ from dept_nested_expanded as d, LogicalProject(DEPTNO=[$0], EMPNO=[$5.EMPNO]) LogicalCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{2}]) LogicalTableScan(table=[[CATALOG, SALES, DEPT_NESTED_EXPANDED]]) - Uncollect + Uncollect(expandStructFields=[false]) LogicalProject(EMPLOYEES=[$cor0.EMPLOYEES]) LogicalValues(tuples=[[{ 0 }]]) ]]> @@ -421,7 +421,7 @@ from dept_nested_expanded as d, LogicalProject(DEPTNO=[$0], EMPNO=[$5.EMPNO]) LogicalCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{2}]) LogicalTableScan(table=[[CATALOG, SALES, DEPT_NESTED_EXPANDED]]) - Uncollect + Uncollect(expandStructFields=[false]) LogicalProject(EMPLOYEES=[$cor0.EMPLOYEES]) LogicalValues(tuples=[[{ 0 }]]) ]]> @@ -438,7 +438,7 @@ from dept_nested_expanded as d, LogicalProject(DEPTNO=[$0], A=[$5]) LogicalCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{3}]) LogicalTableScan(table=[[CATALOG, SALES, DEPT_NESTED_EXPANDED]]) - Uncollect + Uncollect(expandStructFields=[false]) LogicalProject(ADMINS=[$cor0.ADMINS]) LogicalValues(tuples=[[{ 0 }]]) ]]> diff --git a/core/src/test/resources/sql/unnest.iq b/core/src/test/resources/sql/unnest.iq index 8defd3b01ce6..054234fcbed3 100644 --- a/core/src/test/resources/sql/unnest.iq +++ b/core/src/test/resources/sql/unnest.iq @@ -626,4 +626,193 @@ WHERE ( !ok +!use scott + +# Standard UNNEST semantics: struct elements expand into one column per field, so a +# NULL element yields a row of NULLs. +SELECT * FROM UNNEST(ARRAY[ + ROW(1, 'x'), + CAST(NULL AS ROW(a INTEGER, b CHAR(1)))]) AS t(a, b); ++---+---+ +| A | B | ++---+---+ +| 1 | x | +| | | ++---+---+ +(2 rows) + +!ok + +# Same as previous WITH ORDINALITY +SELECT * FROM UNNEST(ARRAY[ + ROW(1, 'x'), + CAST(NULL AS ROW(a INTEGER, b CHAR(1)))]) WITH ORDINALITY AS t(a, b, o); ++---+---+---+ +| A | B | O | ++---+---+---+ +| 1 | x | 1 | +| | | 2 | ++---+---+---+ +(2 rows) + +!ok + +# A result column is nullable whenever the ROW is nullable or the field is. +# Here no element is NULL, so the ROW is NOT NULL and each column keeps its +# own field nullability. +SELECT * FROM UNNEST(ARRAY[ROW(1, CAST(NULL AS INTEGER))]) AS t(a, b); ++---+---+ +| A | B | ++---+---+ +| 1 | | ++---+---+ +(1 row) + +!ok +A INTEGER(10) NOT NULL +B INTEGER(10) +!type + +# Same fields, but a NULL element makes the ROW nullable, so column A is also nullable +SELECT * FROM UNNEST(ARRAY[ + ROW(1, CAST(NULL AS INTEGER)), + CAST(NULL AS ROW(a INTEGER, b INTEGER))]) AS t(a, b); ++---+---+ +| A | B | ++---+---+ +| 1 | | +| | | ++---+---+ +(2 rows) + +!ok +A INTEGER(10) +B INTEGER(10) +!type + +# Tests for [CALCITE-7669] Uncollect should support the Trino semantics of UNNEST +# PRESTO conformance: UNNEST(array) AS t(col) does not expand a struct +# element into its fields; col holds the whole struct, accessed by dot +# notation. +!use hr-presto + +# INNER comma-join: Marketing (0 employees) is dropped. +select d."name" as dept, e.emp."name" as ename, e.emp."empid" as empid +from "hr"."depts" as d, +UNNEST(d."employees") as e(emp); ++-------+-----------+-------+ +| DEPT | ENAME | EMPID | ++-------+-----------+-------+ +| HR | Eric | 200 | +| Sales | Bill | 100 | +| Sales | Sebastian | 150 | ++-------+-----------+-------+ +(3 rows) + +!ok + +# WITH ORDINALITY: the struct column stays whole; ordinality still expands. +select e.emp."name" as ename, e.rn +from "hr"."depts" as d, +UNNEST(d."employees") WITH ORDINALITY as e(emp, rn); ++-----------+----+ +| ENAME | RN | ++-----------+----+ +| Eric | 1 | +| Bill | 1 | +| Sebastian | 2 | ++-----------+----+ +(3 rows) + +!ok + +# Output a whole struct column +select d."name" as dept, e.emp as emp +from "hr"."depts" as d, +UNNEST(d."employees") as e(emp); ++-------+------------------------------------+ +| DEPT | EMP | ++-------+------------------------------------+ +| HR | {200, 20, Eric, 8000.0, 500} | +| Sales | {100, 10, Bill, 10000.0, 1000} | +| Sales | {150, 10, Sebastian, 7000.0, null} | ++-------+------------------------------------+ +(3 rows) + +!ok + +WITH data AS ( + SELECT ARRAY[ + ROW(1, 'Alice'), + ROW(2, 'Bob'), + ROW(3, 'Carol'), + ROW(NULL, 'Dan'), + NULL + ] AS people +) +SELECT p.* +FROM data, UNNEST(people) AS p(p); ++-------------+ +| P | ++-------------+ +| {1, Alice} | +| {2, Bob} | +| {3, Carol} | +| {null, Dan} | +| | ++-------------+ +(5 rows) + +!ok + +# Nested records: a NULL nested-record field, and a bare NULL array element. +SELECT * FROM UNNEST(ARRAY[ + ROW(ROW(1, 'a'), 10), + ROW(ROW(NULL, 'b'), 20), + ROW(NULL, 30), + NULL]) AS p(p); ++-----------------+ +| P | ++-----------------+ +| {null, 30} | +| {{1, a}, 10} | +| {{null, b}, 20} | +| | ++-----------------+ +(4 rows) + +!ok + +# Only one value is emitted per element, so a NULL element is a single NULL +# rather than a row of NULLs. +SELECT * FROM UNNEST(ARRAY[ + ROW(1, 'x'), + CAST(NULL AS ROW(a INTEGER, b CHAR(1)))]) AS t(r); ++--------+ +| R | ++--------+ +| {1, x} | +| | ++--------+ +(2 rows) + +!ok +# The column is nullable: a NULL element yields NULL here. +R STRUCT +!type + +# Same, WITH ORDINALITY: exercises z2 rather than the single-collection path. +SELECT * FROM UNNEST(ARRAY[ + ROW(1, 'x'), + CAST(NULL AS ROW(a INTEGER, b CHAR(1)))]) WITH ORDINALITY AS t(r, o); ++--------+---+ +| R | O | ++--------+---+ +| {1, x} | 1 | +| | 2 | ++--------+---+ +(2 rows) + +!ok + # End unnest.iq From f38163eaeac3f17982e1b0205d32ac83c03338d9 Mon Sep 17 00:00:00 2001 From: Kirill Tkalenko Date: Fri, 17 Jul 2026 14:33:04 +0300 Subject: [PATCH 426/562] [CALCITE-7592] Add expression support for FETCH --- core/src/main/codegen/templates/Parser.jj | 20 +- .../calcite/adapter/enumerable/EnumUtils.java | 7 +- .../adapter/enumerable/EnumerableLimit.java | 20 +- .../enumerable/EnumerableLimitSort.java | 6 +- .../enumerable/EnumerableMergeUnionRule.java | 9 +- .../apache/calcite/interpreter/SortNode.java | 102 +++++- .../rel/metadata/RelMdMaxRowCount.java | 15 +- .../rel/metadata/RelMdMinRowCount.java | 17 +- .../calcite/rel/metadata/RelMdRowCount.java | 12 +- .../calcite/rel/metadata/RelMdUtil.java | 15 +- .../rel/rel2sql/RelToSqlConverter.java | 12 +- .../calcite/rel/rules/MeasureRules.java | 4 +- .../calcite/rel/rules/PruneEmptyRules.java | 6 +- .../rel/rules/SortJoinTransposeRule.java | 4 +- .../rel/rules/SortRemoveRedundantRule.java | 3 + .../rel/rules/SortUnionTransposeRule.java | 9 +- .../java/org/apache/calcite/rex/RexUtil.java | 80 +++++ .../calcite/runtime/CalciteResource.java | 9 + .../org/apache/calcite/sql/SqlDialect.java | 34 +- .../calcite/sql/dialect/SqliteSqlDialect.java | 2 +- .../sql/validate/SqlValidatorImpl.java | 29 ++ .../calcite/sql2rel/RelDecorrelator.java | 29 +- .../sql2rel/TopDownGeneralDecorrelator.java | 35 +- .../org/apache/calcite/tools/RelBuilder.java | 52 ++- .../runtime/CalciteResource.properties | 3 + .../adapter/enumerable/EnumUtilsTest.java | 11 + .../rel/rel2sql/RelToSqlConverterTest.java | 68 ++++ .../apache/calcite/rex/RexProgramTest.java | 30 ++ .../org/apache/calcite/test/JdbcTest.java | 338 ++++++++++++++++++ .../apache/calcite/test/RelBuilderTest.java | 105 ++++++ .../apache/calcite/test/RelMetadataTest.java | 33 +- .../apache/calcite/test/RelOptRulesTest.java | 81 ++++- .../calcite/test/SqlToRelConverterTest.java | 9 + .../apache/calcite/test/SqlValidatorTest.java | 16 + .../enumerable/EnumerableMergeUnionTest.java | 30 ++ .../apache/calcite/test/RelOptRulesTest.xml | 197 +++++++++- .../calcite/test/SqlToRelConverterTest.xml | 12 + core/src/test/resources/sql/fetch.iq | 183 ++++++++++ .../org/apache/calcite/test/ServerTest.java | 37 ++ site/_docs/reference.md | 9 +- .../calcite/sql/parser/SqlParserTest.java | 19 + 41 files changed, 1619 insertions(+), 93 deletions(-) create mode 100644 core/src/test/resources/sql/fetch.iq diff --git a/core/src/main/codegen/templates/Parser.jj b/core/src/main/codegen/templates/Parser.jj index 7246e6084211..ce69124c4b5c 100644 --- a/core/src/main/codegen/templates/Parser.jj +++ b/core/src/main/codegen/templates/Parser.jj @@ -709,7 +709,7 @@ SqlNode ExprOrJoinOrOrderedQuery(ExprContext exprContext) : * * 
        *    [ OFFSET start { ROW | ROWS } ]
- *    [ FETCH { FIRST | NEXT } [ count ] { ROW | ROWS } ONLY ]
      + * [ FETCH { FIRST | NEXT } [ count | (expression) ] { ROW | ROWS } ONLY ] *
      */ SqlNode OrderedQueryOrExpr(ExprContext exprContext) : @@ -796,10 +796,26 @@ void FetchClause(SqlNode[] offsetFetch) : { // SQL:2008-style syntax. "OFFSET ... FETCH ...". // If you specify both LIMIT and FETCH, FETCH wins. - ( | ) offsetFetch[1] = UnsignedNumericLiteralOrParam() + ( | ) offsetFetch[1] = FetchCount() ( | ) } +/** + * Parses the row count of a FETCH clause. Expressions must be parenthesized. + */ +SqlNode FetchCount() : +{ + final SqlNode e; +} +{ + ( + e = UnsignedNumericLiteralOrParam() + | + e = Expression(ExprContext.ACCEPT_NON_QUERY) + ) + { return e; } +} + /** * Parses a LIMIT clause in an ORDER BY expression. */ diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java index d3da43466da7..40a824536b15 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java @@ -116,7 +116,7 @@ private EnumUtils() {} /** Converts a FETCH or OFFSET runtime value to {@link BigDecimal}. * *

      The value must be numeric and non-negative. */ - public static BigDecimal numberToBigDecimal(Object value, String kind) { + public static BigDecimal numberToBigDecimal(@Nullable Object value, String kind) { return numberToBigDecimal(value, kind, FetchOffsetRoundingPolicy.NONE); } @@ -124,8 +124,11 @@ public static BigDecimal numberToBigDecimal(Object value, String kind) { * *

      The value must be numeric and non-negative. The result is adjusted by * the configured rounding policy. */ - public static BigDecimal numberToBigDecimal(Object value, String kind, + public static BigDecimal numberToBigDecimal(@Nullable Object value, String kind, FetchOffsetRoundingPolicy roundingPolicy) { + if (value == null) { + throw new IllegalArgumentException(kind + " expression evaluated to NULL"); + } if (!(value instanceof Number)) { throw new IllegalArgumentException(kind + " must be a number"); } diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimit.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimit.java index 02fd54bdad86..de1f94d562d5 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimit.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimit.java @@ -106,13 +106,15 @@ public static EnumerableLimit create(final RelNode input, @Nullable RexNode offs v = builder.append("offset", Expressions.call(BuiltInMethod.SKIP_BIG_DECIMAL.method, v, - getExpression(offset, "OFFSET", roundingPolicyExp))); + getExpression(offset, "OFFSET", implementor, builder, + roundingPolicyExp, false))); } if (fetch != null) { v = builder.append("fetch", Expressions.call(BuiltInMethod.TAKE_BIG_DECIMAL.method, v, - getExpression(fetch, "FETCH", roundingPolicyExp))); + getExpression(fetch, "FETCH", implementor, builder, + roundingPolicyExp, true))); } builder.add(Expressions.return_(null, v)); @@ -120,7 +122,8 @@ public static EnumerableLimit create(final RelNode input, @Nullable RexNode offs } static Expression getExpression(RexNode rexNode, String kind, - Expression roundingPolicy) { + EnumerableRelImplementor implementor, BlockBuilder builder, + Expression roundingPolicy, boolean translateExpression) { final Expression value; if (rexNode instanceof RexDynamicParam) { final RexDynamicParam param = (RexDynamicParam) rexNode; @@ -128,8 +131,17 @@ static Expression getExpression(RexNode rexNode, String kind, Expressions.call(DataContext.ROOT, BuiltInMethod.DATA_CONTEXT_GET.method, Expressions.constant("?" + param.getIndex())); - } else { + } else if (rexNode instanceof RexLiteral) { value = Expressions.constant(RexLiteral.bigDecimalValue(rexNode)); + } else { + if (!translateExpression) { + throw new IllegalArgumentException(kind + " must be a literal or dynamic parameter"); + } + + value = + RexToLixTranslator.forAggregation(implementor.getTypeFactory(), + builder, null, implementor.getConformance()) + .translate(rexNode); } return Expressions.call( BuiltInMethod.NUMBER_TO_BIG_DECIMAL_LIMIT.method, diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimitSort.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimitSort.java index 325fe687ba4d..97d9fd8169b7 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimitSort.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimitSort.java @@ -104,14 +104,16 @@ public static EnumerableLimitSort create( if (this.fetch == null) { fetchVal = Expressions.constant(BigDecimal.valueOf(Integer.MAX_VALUE)); } else { - fetchVal = getExpression(this.fetch, "FETCH", roundingPolicyExp); + fetchVal = + getExpression(this.fetch, "FETCH", implementor, builder, roundingPolicyExp, true); } final Expression offsetVal; if (this.offset == null) { offsetVal = Expressions.constant(BigDecimal.ZERO); } else { - offsetVal = getExpression(this.offset, "OFFSET", roundingPolicyExp); + offsetVal = + getExpression(this.offset, "OFFSET", implementor, builder, roundingPolicyExp, false); } builder.add( diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeUnionRule.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeUnionRule.java index 7d47e639b78e..57f864794aa9 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeUnionRule.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeUnionRule.java @@ -29,6 +29,7 @@ import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexUtil; import org.apache.calcite.tools.RelBuilder; import org.apache.calcite.util.ImmutableBitSet; @@ -88,9 +89,13 @@ public EnumerableMergeUnionRule(Config config) { // Push down sort limit, if possible. RexNode inputFetch = null; if (sort.fetch != null) { - if (sort.offset == null) { + final boolean safeToReevaluate = + RexUtil.isDeterministic(sort.fetch); + if (sort.offset == null && safeToReevaluate) { inputFetch = sort.fetch; - } else if (sort.fetch instanceof RexLiteral && sort.offset instanceof RexLiteral) { + } else if (safeToReevaluate + && sort.fetch instanceof RexLiteral + && sort.offset instanceof RexLiteral) { inputFetch = call.builder().literal(RexLiteral.bigDecimalValue(sort.fetch) .add(RexLiteral.bigDecimalValue(sort.offset))); diff --git a/core/src/main/java/org/apache/calcite/interpreter/SortNode.java b/core/src/main/java/org/apache/calcite/interpreter/SortNode.java index 71d9f2b22e42..0f393a3e68d8 100644 --- a/core/src/main/java/org/apache/calcite/interpreter/SortNode.java +++ b/core/src/main/java/org/apache/calcite/interpreter/SortNode.java @@ -16,14 +16,22 @@ */ package org.apache.calcite.interpreter; +import org.apache.calcite.adapter.enumerable.EnumUtils; +import org.apache.calcite.adapter.enumerable.EnumerableRelImplementor; +import org.apache.calcite.adapter.enumerable.FetchOffsetRoundingPolicy; import org.apache.calcite.rel.RelFieldCollation; import org.apache.calcite.rel.core.Sort; import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; import org.apache.calcite.util.Util; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Ordering; +import org.checkerframework.checker.nullness.qual.Nullable; + +import java.math.BigDecimal; +import java.math.RoundingMode; import java.util.ArrayList; import java.util.Comparator; import java.util.List; @@ -35,37 +43,57 @@ * {@link org.apache.calcite.rel.core.Sort}. */ public class SortNode extends AbstractSingleNode { + private final @Nullable Scalar offsetScalar; + private final @Nullable Context offsetContext; + private final @Nullable Scalar fetchScalar; + private final @Nullable Context fetchContext; + private final FetchOffsetRoundingPolicy fetchOffsetRoundingPolicy; + public SortNode(Compiler compiler, Sort rel) { super(compiler, rel); - } - - private static int getValueAsInt(RexNode node) { - return requireNonNull(((RexLiteral) node).getValueAs(Integer.class), - () -> "getValueAs(Integer.class) for " + node); + if (rel.offset != null && !(rel.offset instanceof RexLiteral)) { + this.offsetScalar = compiler.compile(ImmutableList.of(rel.offset), null); + this.offsetContext = compiler.createContext(); + } else { + this.offsetScalar = null; + this.offsetContext = null; + } + if (rel.fetch != null && !(rel.fetch instanceof RexLiteral)) { + this.fetchScalar = compiler.compile(ImmutableList.of(rel.fetch), null); + this.fetchContext = compiler.createContext(); + } else { + this.fetchScalar = null; + this.fetchContext = null; + } + final Object roundingPolicy = compiler.getDataContext() + .get(EnumerableRelImplementor.FETCH_OFFSET_ROUNDING_POLICY); + this.fetchOffsetRoundingPolicy = + roundingPolicy instanceof FetchOffsetRoundingPolicy + ? (FetchOffsetRoundingPolicy) roundingPolicy + : FetchOffsetRoundingPolicy.NONE; } @Override public void run() throws InterruptedException { - final int offset = - rel.offset == null - ? 0 - : getValueAsInt(rel.offset); - final int fetch = - rel.fetch == null - ? -1 - : getValueAsInt(rel.fetch); + final BigDecimal offset = getOffset(); + final @Nullable BigDecimal fetch = getFetch(); // In pure limit mode. No sort required. Row row; loop: if (rel.getCollation().getFieldCollations().isEmpty()) { - for (int i = 0; i < offset; i++) { + BigDecimal skipped = BigDecimal.ZERO; + while (skipped.compareTo(offset) < 0) { row = source.receive(); if (row == null) { break loop; } + skipped = skipped.add(BigDecimal.ONE); } - if (fetch >= 0) { - for (int i = 0; i < fetch && (row = source.receive()) != null; i++) { + if (fetch != null) { + BigDecimal fetched = BigDecimal.ZERO; + while (fetched.compareTo(fetch) < 0 + && (row = source.receive()) != null) { sink.send(row); + fetched = fetched.add(BigDecimal.ONE); } } else { while ((row = source.receive()) != null) { @@ -79,10 +107,15 @@ private static int getValueAsInt(RexNode node) { list.add(row); } list.sort(comparator()); - final int end = fetch < 0 || offset + fetch > list.size() + final int start = offset.compareTo(BigDecimal.valueOf(list.size())) >= 0 + ? list.size() + : rowCount(offset); + final int available = list.size() - start; + final int end = fetch == null + || fetch.compareTo(BigDecimal.valueOf(available)) >= 0 ? list.size() - : offset + fetch; - for (int i = offset; i < end; i++) { + : start + rowCount(fetch); + for (int i = start; i < end; i++) { sink.send(list.get(i)); } } @@ -116,4 +149,35 @@ private static Comparator comparator(RelFieldCollation fieldCollation) { }; } } + + private @Nullable BigDecimal getFetch() { + if (rel.fetch == null) { + return null; + } + return getValue(rel.fetch, fetchScalar, fetchContext, "FETCH"); + } + + private BigDecimal getOffset() { + if (rel.offset == null) { + return BigDecimal.ZERO; + } + return getValue(rel.offset, offsetScalar, offsetContext, "OFFSET"); + } + + private BigDecimal getValue(RexNode node, @Nullable Scalar scalar, + @Nullable Context context, String kind) { + final @Nullable Object value; + if (node instanceof RexLiteral) { + value = RexLiteral.bigDecimalValue(node); + } else { + value = + requireNonNull(scalar, () -> kind + " scalar") + .execute(requireNonNull(context, () -> kind + " context")); + } + return EnumUtils.numberToBigDecimal(value, kind, fetchOffsetRoundingPolicy); + } + + private static int rowCount(BigDecimal value) { + return value.setScale(0, RoundingMode.CEILING).intValueExact(); + } } diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdMaxRowCount.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdMaxRowCount.java index e728c22e1ede..869f1ad50e78 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdMaxRowCount.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdMaxRowCount.java @@ -117,10 +117,12 @@ public Double getMaxRowCount(Sort rel, RelMetadataQuery mq) { rowCount = Double.POSITIVE_INFINITY; } - final double offset = literalValueApproximatedByDouble(rel.offset, 0D); + final double offset = + literalValueApproximatedByDouble(rel.offset, 0D); rowCount = Math.max(rowCount - offset, 0D); - final double limit = literalValueApproximatedByDouble(rel.fetch, rowCount); + final double limit = + literalValueApproximatedByDouble(rel.fetch, rowCount); return limit < rowCount ? limit : rowCount; } @@ -130,10 +132,12 @@ public Double getMaxRowCount(EnumerableLimit rel, RelMetadataQuery mq) { rowCount = Double.POSITIVE_INFINITY; } - final double offset = literalValueApproximatedByDouble(rel.offset, 0D); + final double offset = + literalValueApproximatedByDouble(rel.offset, 0D); rowCount = Math.max(rowCount - offset, 0D); - final double limit = literalValueApproximatedByDouble(rel.fetch, rowCount); + final double limit = + literalValueApproximatedByDouble(rel.fetch, rowCount); return limit < rowCount ? limit : rowCount; } @@ -214,7 +218,8 @@ public Double getMaxRowCount(RelSubset rel, RelMetadataQuery mq) { if (node instanceof Sort) { Sort sort = (Sort) node; if (sort.fetch instanceof RexLiteral) { - return literalValueApproximatedByDouble(sort.fetch, Double.POSITIVE_INFINITY); + return literalValueApproximatedByDouble(sort.fetch, + Double.POSITIVE_INFINITY); } } } diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdMinRowCount.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdMinRowCount.java index 869d34333547..2cb710f39808 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdMinRowCount.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdMinRowCount.java @@ -116,10 +116,13 @@ public Double getMinRowCount(Sort rel, RelMetadataQuery mq) { rowCount = 0D; } - final double offset = literalValueApproximatedByDouble(rel.offset, 0D); + final double offset = + literalValueApproximatedByDouble(rel.offset, 0D); rowCount = Math.max(rowCount - offset, 0D); - final double limit = literalValueApproximatedByDouble(rel.fetch, rowCount); + final double limit = + literalValueApproximatedByDouble(rel.fetch, + rel.fetch == null ? rowCount : 0D); return limit < rowCount ? limit : rowCount; } @@ -129,10 +132,13 @@ public Double getMinRowCount(EnumerableLimit rel, RelMetadataQuery mq) { rowCount = 0D; } - final double offset = literalValueApproximatedByDouble(rel.offset, 0D); + final double offset = + literalValueApproximatedByDouble(rel.offset, 0D); rowCount = Math.max(rowCount - offset, 0D); - final double limit = literalValueApproximatedByDouble(rel.fetch, rowCount); + final double limit = + literalValueApproximatedByDouble(rel.fetch, + rel.fetch == null ? rowCount : 0D); return limit < rowCount ? limit : rowCount; } @@ -174,7 +180,8 @@ public Double getMinRowCount(RelSubset rel, RelMetadataQuery mq) { if (node instanceof Sort) { Sort sort = (Sort) node; if (sort.fetch instanceof RexLiteral) { - return literalValueApproximatedByDouble(sort.fetch, Double.POSITIVE_INFINITY); + return literalValueApproximatedByDouble(sort.fetch, + Double.POSITIVE_INFINITY); } } } diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdRowCount.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdRowCount.java index e83f4c1da9f4..3e7824e1aac6 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdRowCount.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdRowCount.java @@ -165,10 +165,12 @@ public Double getRowCount(Calc rel, RelMetadataQuery mq) { return null; } - final double offset = literalValueApproximatedByDouble(rel.offset, 0D); + final double offset = + literalValueApproximatedByDouble(rel.offset, 0D); rowCount = Math.max(rowCount - offset, 0D); - final double limit = literalValueApproximatedByDouble(rel.fetch, rowCount); + final double limit = + literalValueApproximatedByDouble(rel.fetch, rowCount); return limit < rowCount ? limit : rowCount; } @@ -178,10 +180,12 @@ public Double getRowCount(Calc rel, RelMetadataQuery mq) { return null; } - final double offset = literalValueApproximatedByDouble(rel.offset, 0D); + final double offset = + literalValueApproximatedByDouble(rel.offset, 0D); rowCount = Math.max(rowCount - offset, 0D); - final double limit = literalValueApproximatedByDouble(rel.fetch, rowCount); + final double limit = + literalValueApproximatedByDouble(rel.fetch, rowCount); return limit < rowCount ? limit : rowCount; } diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdUtil.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdUtil.java index 1f6502243626..5b096289382e 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdUtil.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdUtil.java @@ -25,6 +25,7 @@ import org.apache.calcite.rel.core.JoinRelType; import org.apache.calcite.rel.core.Minus; import org.apache.calcite.rel.core.Project; +import org.apache.calcite.rel.core.Sort; import org.apache.calcite.rel.core.Union; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexCall; @@ -56,6 +57,7 @@ import java.util.ArrayList; import java.util.LinkedHashSet; import java.util.List; +import java.util.Objects; import java.util.Set; import static com.google.common.base.Preconditions.checkArgument; @@ -483,6 +485,9 @@ public static double literalValueApproximatedByDouble(@Nullable RexNode node, throw new IllegalArgumentException( "literal value " + number + " cannot be converted to BigDecimal"); } + if (decimal.signum() < 0) { + return defaultValue; + } if (decimal.abs().compareTo(BigDecimal.valueOf(Double.MAX_VALUE)) > 0) { throw new IllegalArgumentException( "literal value " + decimal + " exceeds double range"); @@ -1043,8 +1048,16 @@ private static boolean alreadySmaller(RelMetadataQuery mq, RelNode input, if (fetch == null) { return true; } + final RelNode strippedInput = input.stripped(); + if (strippedInput instanceof Sort) { + final Sort sort = (Sort) strippedInput; + if (Objects.equals(offset, sort.offset) + && Objects.equals(fetch, sort.fetch)) { + return true; + } + } final Double rowCount = mq.getMaxRowCount(input); - if (rowCount == null || offset instanceof RexDynamicParam || fetch instanceof RexDynamicParam) { + if (rowCount == null || offset instanceof RexDynamicParam || !(fetch instanceof RexLiteral)) { // Cannot be determined return false; } diff --git a/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java b/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java index 8871f24d3e86..718e6b1cb965 100644 --- a/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java +++ b/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java @@ -59,6 +59,7 @@ import org.apache.calcite.rex.RexLocalRef; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexProgram; +import org.apache.calcite.rex.RexUtil; import org.apache.calcite.sql.JoinConditionType; import org.apache.calcite.sql.JoinType; import org.apache.calcite.sql.SqlAsofJoin; @@ -1227,7 +1228,7 @@ public Result visit(Sort e) { sqlSelect.setOffset(offset); } if (e.fetch != null) { - SqlNode fetch = builder.context.toSql(null, e.fetch); + SqlNode fetch = toSqlFetch(e, builder.context); sqlSelect.setFetch(fetch); } return result(sqlSelect, ImmutableList.of(Clause.ORDER_BY), e, null); @@ -1285,13 +1286,20 @@ public Result visit(Sort e) { * The builder must have been created with OFFSET and FETCH clauses. */ void offsetFetch(Sort e, Builder builder) { if (e.fetch != null) { - builder.setFetch(builder.context.toSql(null, e.fetch)); + builder.setFetch(toSqlFetch(e, builder.context)); } if (e.offset != null) { builder.setOffset(builder.context.toSql(null, e.offset)); } } + private static SqlNode toSqlFetch(Sort sort, Context context) { + final RexNode fetch = requireNonNull(sort.fetch, "fetch"); + final @Nullable RexLiteral reduced = + RexUtil.reduceFetchToLiteral(sort.getCluster(), fetch); + return context.toSql(null, reduced == null ? fetch : reduced); + } + public boolean hasTrickyRollup(Sort e, Aggregate aggregate) { return !dialect.supportsAggregateFunction(SqlKind.ROLLUP) && dialect.supportsGroupByWithRollup() diff --git a/core/src/main/java/org/apache/calcite/rel/rules/MeasureRules.java b/core/src/main/java/org/apache/calcite/rel/rules/MeasureRules.java index 037a4d605459..f69810a14d42 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/MeasureRules.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/MeasureRules.java @@ -30,7 +30,6 @@ import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexCorrelVariable; import org.apache.calcite.rex.RexInputRef; -import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexShuttle; import org.apache.calcite.rex.RexUtil; @@ -508,8 +507,7 @@ protected ProjectSortMeasureRule(ProjectSortMeasureRuleConfig config) { relBuilder.push(sort.getInput()) .projectPlus(map.keySet()) - .sortLimit(sort.offset == null ? 0 : RexLiteral.numberValue(sort.offset), - sort.fetch == null ? -1 : RexLiteral.numberValue(sort.fetch), + .sortLimit(sort.offset, sort.fetch, sort.getSortExps()) .project(newProjects); call.transformTo(relBuilder.build()); diff --git a/core/src/main/java/org/apache/calcite/rel/rules/PruneEmptyRules.java b/core/src/main/java/org/apache/calcite/rel/rules/PruneEmptyRules.java index 02b0bd8af1b2..95331c69a414 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/PruneEmptyRules.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/PruneEmptyRules.java @@ -42,7 +42,6 @@ import org.apache.calcite.rel.logical.LogicalValues; import org.apache.calcite.rel.metadata.RelMdUtil; import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.rex.RexDynamicParam; import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; import org.apache.calcite.tools.RelBuilder; @@ -536,9 +535,8 @@ public interface SortFetchZeroRuleConfig extends PruneEmptyRule.Config { return new RemoveEmptySingleRule(this) { @Override public boolean matches(final RelOptRuleCall call) { Sort sort = call.rel(0); - return sort.fetch != null - && !(sort.fetch instanceof RexDynamicParam) - && RexLiteral.bigDecimalValue(sort.fetch).equals(BigDecimal.ZERO); + return sort.fetch instanceof RexLiteral + && BigDecimal.ZERO.equals(RexLiteral.bigDecimalValue(sort.fetch)); } }; } diff --git a/core/src/main/java/org/apache/calcite/rel/rules/SortJoinTransposeRule.java b/core/src/main/java/org/apache/calcite/rel/rules/SortJoinTransposeRule.java index 4310d6d65576..df967e56aad6 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/SortJoinTransposeRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/SortJoinTransposeRule.java @@ -105,9 +105,9 @@ public SortJoinTransposeRule(Class sortClass, final Sort sort = call.rel(0); final Join join = call.rel(1); - // Do nothing if SORT contains dynamic parameters in offset or fetch + // The pushed fetch is calculated from literal offset and fetch values. if (sort.offset instanceof RexDynamicParam - || sort.fetch instanceof RexDynamicParam) { + || sort.fetch != null && !(sort.fetch instanceof RexLiteral)) { return false; } diff --git a/core/src/main/java/org/apache/calcite/rel/rules/SortRemoveRedundantRule.java b/core/src/main/java/org/apache/calcite/rel/rules/SortRemoveRedundantRule.java index 9bcf026fc656..08563cdcdb86 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/SortRemoveRedundantRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/SortRemoveRedundantRule.java @@ -133,6 +133,9 @@ protected SortRemoveRedundantRule(final SortRemoveRedundantRule.Config config) { private static Optional getRowCountThreshold(Sort sort) { if (RelOptUtil.isLimit(sort)) { assert sort.fetch != null; + if (!(sort.fetch instanceof RexLiteral)) { + return Optional.empty(); + } final BigDecimal fetch = RexLiteral.bigDecimalValue(sort.fetch); // We don't need to deal with fetch is 0. diff --git a/core/src/main/java/org/apache/calcite/rel/rules/SortUnionTransposeRule.java b/core/src/main/java/org/apache/calcite/rel/rules/SortUnionTransposeRule.java index 416825ee926d..93b6af657c43 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/SortUnionTransposeRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/SortUnionTransposeRule.java @@ -23,7 +23,7 @@ import org.apache.calcite.rel.core.Union; import org.apache.calcite.rel.metadata.RelMdUtil; import org.apache.calcite.rel.metadata.RelMetadataQuery; -import org.apache.calcite.rex.RexDynamicParam; +import org.apache.calcite.rex.RexUtil; import org.apache.calcite.tools.RelBuilderFactory; import org.immutables.value.Value; @@ -67,13 +67,14 @@ public SortUnionTransposeRule( @Override public boolean matches(RelOptRuleCall call) { final Sort sort = call.rel(0); final Union union = call.rel(1); - // We only apply this rule if Union.all is true, Sort.offset is null and Sort.fetch is not - // a dynamic param. + // Re-evaluating a non-deterministic FETCH in every branch can produce a + // different limit from the top Sort. // There is a flag indicating if this rule should be applied when // Sort.fetch is null. return union.all && sort.offset == null - && !(sort.fetch instanceof RexDynamicParam) + && (sort.fetch == null + || RexUtil.isDeterministic(sort.fetch)) && (config.matchNullFetch() || sort.fetch != null); } diff --git a/core/src/main/java/org/apache/calcite/rex/RexUtil.java b/core/src/main/java/org/apache/calcite/rex/RexUtil.java index 3604e98dfd5b..b592093a5aef 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexUtil.java +++ b/core/src/main/java/org/apache/calcite/rex/RexUtil.java @@ -19,6 +19,7 @@ import org.apache.calcite.DataContexts; import org.apache.calcite.linq4j.function.Predicate1; import org.apache.calcite.plan.PlanTooComplexError; +import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.plan.RelOptPredicateList; import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.rel.RelCollation; @@ -48,6 +49,7 @@ import org.apache.calcite.util.ControlFlowException; import org.apache.calcite.util.ImmutableBitSet; import org.apache.calcite.util.Litmus; +import org.apache.calcite.util.NumberUtil; import org.apache.calcite.util.Pair; import org.apache.calcite.util.RangeSets; import org.apache.calcite.util.Sarg; @@ -63,9 +65,11 @@ import org.apiguardian.api.API; import org.checkerframework.checker.nullness.qual.Nullable; +import java.math.BigDecimal; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -840,6 +844,82 @@ public static boolean isDeterministic(RexNode e) { } } + /** Returns whether an expression contains a dynamic function. */ + public static boolean containsDynamicFunction(RexNode e) { + try { + e.accept( + new RexVisitorImpl(true) { + @Override public Void visitCall(RexCall call) { + if (call.getOperator().isDynamicFunction()) { + throw Util.FoundOne.NULL; + } + return super.visitCall(call); + } + }); + return false; + } catch (Util.FoundOne ex) { + Util.swallow(ex, null); + return true; + } + } + + /** Returns whether an expression contains a dynamic parameter. */ + public static boolean containsDynamicParam(RexNode e) { + try { + e.accept( + new RexVisitorImpl(true) { + @Override public Void visitDynamicParam(RexDynamicParam dynamicParam) { + throw Util.FoundOne.NULL; + } + }); + return false; + } catch (Util.FoundOne ex) { + Util.swallow(ex, null); + return true; + } + } + + /** Converts a FETCH expression result to its validated canonical representation. */ + public static BigDecimal validateFetchValue(@Nullable Number value) { + if (value == null) { + throw new IllegalArgumentException("FETCH expression evaluated to NULL"); + } + final BigDecimal decimal = NumberUtil.toBigDecimal(value); + if (decimal.signum() < 0) { + throw new IllegalArgumentException("FETCH value " + value + + " is out of range; expected a non-negative value"); + } + return decimal; + } + + /** Reduces a constant FETCH expression to a validated literal. */ + public static @Nullable RexLiteral reduceFetchToLiteral( + RelOptCluster cluster, RexNode fetch) { + final RexLiteral literal; + if (fetch instanceof RexLiteral) { + literal = (RexLiteral) fetch; + } else { + if (!isConstant(fetch) + || !isDeterministic(fetch) + || containsDynamicFunction(fetch) + || containsDynamicParam(fetch)) { + return null; + } + final RexExecutor executor = + Util.first(cluster.getPlanner().getExecutor(), EXECUTOR); + final List reducedValues = new ArrayList<>(1); + executor.reduce(cluster.getRexBuilder(), + Collections.singletonList(fetch), reducedValues); + final RexNode reduced = reducedValues.get(0); + if (!(reduced instanceof RexLiteral)) { + return null; + } + literal = (RexLiteral) reduced; + } + validateFetchValue(literal.getValueAs(Number.class)); + return literal; + } + public static List retainDeterministic(List list) { List conjunctions = new ArrayList<>(); for (RexNode x : list) { diff --git a/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java b/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java index c5047574a3d0..452c2bf84a7e 100644 --- a/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java +++ b/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java @@ -164,6 +164,15 @@ ExInstWithCause validatorContext(int a0, int a1, @BaseMessage("Values passed to {0} operator must have compatible types") ExInst incompatibleValueType(String a0); + @BaseMessage("FETCH expression must have a numeric type; actual type is ''{0}''") + ExInst fetchExpressionMustBeNumeric(String type); + + @BaseMessage("FETCH expression cannot reference table column ''{0}''") + ExInst fetchExpressionCannotReferenceColumn(String column); + + @BaseMessage("FETCH expression evaluated to NULL") + ExInst fetchExpressionEvaluatedToNull(); + @BaseMessage("Values in expression list must have compatible types") ExInst incompatibleTypesInList(); diff --git a/core/src/main/java/org/apache/calcite/sql/SqlDialect.java b/core/src/main/java/org/apache/calcite/sql/SqlDialect.java index e659d1d17eb2..164f212c6c7a 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlDialect.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlDialect.java @@ -1088,7 +1088,18 @@ protected static void unparseFetchUsingAnsi(SqlWriter writer, @Nullable SqlNode writer.startList(SqlWriter.FrameTypeEnum.FETCH); writer.keyword("FETCH"); writer.keyword("NEXT"); - fetch.unparse(writer, -1, -1); + if (fetch instanceof SqlLiteral + || fetch instanceof SqlDynamicParam) { + fetch.unparse(writer, -1, -1); + } else { + final SqlWriter.Frame expressionFrame = writer.startList("(", ")"); + if (fetch instanceof SqlCall) { + writer.getDialect().unparseCall(writer, (SqlCall) fetch, 0, 0); + } else { + fetch.unparse(writer, 0, 0); + } + writer.endList(expressionFrame); + } writer.keyword("ROWS"); writer.keyword("ONLY"); writer.endList(fetchFrame); @@ -1098,13 +1109,32 @@ protected static void unparseFetchUsingAnsi(SqlWriter writer, @Nullable SqlNode /** Unparses offset/fetch using "LIMIT fetch OFFSET offset" syntax. */ protected static void unparseFetchUsingLimit(SqlWriter writer, @Nullable SqlNode offset, @Nullable SqlNode fetch) { + unparseFetchUsingLimit(writer, offset, fetch, false); + } + + /** Unparses offset/fetch using "LIMIT fetch OFFSET offset" syntax, + * optionally allowing a scalar expression as fetch. */ + protected static void unparseFetchUsingLimit(SqlWriter writer, @Nullable SqlNode offset, + @Nullable SqlNode fetch, boolean allowExpression) { checkArgument(fetch != null || offset != null); - unparseLimit(writer, fetch); + unparseLimit(writer, fetch, allowExpression); unparseOffset(writer, offset); } protected static void unparseLimit(SqlWriter writer, @Nullable SqlNode fetch) { + unparseLimit(writer, fetch, false); + } + + private static void unparseLimit(SqlWriter writer, @Nullable SqlNode fetch, + boolean allowExpression) { if (fetch != null) { + if (!allowExpression + && !(fetch instanceof SqlLiteral) + && !(fetch instanceof SqlDynamicParam)) { + throw new IllegalArgumentException( + "LIMIT dialect does not support FETCH expressions that cannot " + + "be reduced to a literal"); + } writer.newlineAndIndent(); final SqlWriter.Frame fetchFrame = writer.startList(SqlWriter.FrameTypeEnum.FETCH); diff --git a/core/src/main/java/org/apache/calcite/sql/dialect/SqliteSqlDialect.java b/core/src/main/java/org/apache/calcite/sql/dialect/SqliteSqlDialect.java index 82376ae576ab..f31276413600 100644 --- a/core/src/main/java/org/apache/calcite/sql/dialect/SqliteSqlDialect.java +++ b/core/src/main/java/org/apache/calcite/sql/dialect/SqliteSqlDialect.java @@ -90,7 +90,7 @@ public SqliteSqlDialect(SqlDialect.Context context) { @Override public void unparseOffsetFetch(SqlWriter writer, @Nullable SqlNode offset, @Nullable SqlNode fetch) { - unparseFetchUsingLimit(writer, offset, fetch); + unparseFetchUsingLimit(writer, offset, fetch, true); } @Override public void unparseCall(SqlWriter writer, SqlCall call, diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java index 7021696b679c..0ea01a351451 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java @@ -1771,6 +1771,34 @@ private void handleOffsetFetch(@Nullable SqlNode offset, @Nullable SqlNode fetch } } + private void validateFetchExpression(@Nullable SqlNode fetch) { + if (fetch == null || fetch instanceof SqlDynamicParam) { + return; + } + if (SqlUtil.isNullLiteral(fetch, true)) { + throw newValidationError(fetch, + RESOURCE.fetchExpressionEvaluatedToNull()); + } + validateNoAggs(aggOrOverFinder, fetch, "FETCH"); + fetch.accept(new SqlBasicVisitor() { + @Override public Void visit(SqlIdentifier id) { + if (makeNullaryCall(id) != null) { + return null; + } + throw newValidationError(id, + RESOURCE.fetchExpressionCannotReferenceColumn(id.toString())); + } + }); + final SqlValidatorScope scope = getEmptyScope(); + inferUnknownTypes(typeFactory.createSqlType(SqlTypeName.DECIMAL), scope, fetch); + validateExpr(fetch, scope); + final RelDataType type = getValidatedNodeType(fetch); + if (!SqlTypeUtil.isNumeric(type)) { + throw newValidationError(fetch, + RESOURCE.fetchExpressionMustBeNumeric(type.getFullTypeString())); + } + } + /** * Performs expression rewrites which are always used unconditionally. These * rewrites massage the expression tree into a standard form so that the @@ -4499,6 +4527,7 @@ protected void validateSelect( validateWindowClause(select); validateQualifyClause(select); handleOffsetFetch(select.getOffset(), select.getFetch()); + validateFetchExpression(select.getFetch()); // Validate the SELECT clause late, because a select item might // depend on the GROUP BY list, or the window function might reference diff --git a/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java b/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java index 300ff959bc74..4e4104ad4876 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java @@ -575,6 +575,10 @@ protected RexNode removeCorrelationExpr( // Its output does not change the input ordering, so there's no // need to call propagateExpr. + if (isCorVarDefined && !canDecorrelateOffsetFetch(rel)) { + return null; + } + final RelNode oldInput = rel.getInput(); final Frame frame = getInvoke(oldInput, isCorVarDefined, rel, true); if (frame == null) { @@ -1137,8 +1141,31 @@ private static void shiftMapping(Map mapping, int startIndex, return register(sort, result, mapOldToNewOutputs, corDefOutputs); } + static boolean canDecorrelateOffsetFetch(Sort sort) { + final @Nullable RexLiteral fetch = sort.fetch == null + ? null + : RexUtil.reduceFetchToLiteral(sort.getCluster(), sort.fetch); + return isNonNegativeIntegralLiteral(sort.offset) + && (sort.fetch == null + || fetch != null && isNonNegativeIntegralLiteral(fetch)); + } + + private static boolean isNonNegativeIntegralLiteral(@Nullable RexNode node) { + if (node == null) { + return true; + } + if (!(node instanceof RexLiteral)) { + return false; + } + final @Nullable BigDecimal value = + ((RexLiteral) node).getValueAs(BigDecimal.class); + return value != null + && value.signum() >= 0 + && value.stripTrailingZeros().scale() <= 0; + } + protected @Nullable Frame decorrelateSortAsAggregate(Sort sort, final Frame frame) { - if (sort.offset != null || sort.fetch == null) { + if (sort.offset != null || !(sort.fetch instanceof RexLiteral)) { return null; } diff --git a/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java b/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java index 291eb619d0ed..4139cf4b2b99 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java @@ -233,12 +233,14 @@ public static RelNode decorrelateQuery(RelNode rel, RelBuilder builder) { RelNode preparedRel = prePlanner.findBestExp(); // start decorrelating - TopDownGeneralDecorrelator decorrelator = createEmptyDecorrelator(builder); RelNode decorrelateNode = rel; - try { - decorrelateNode = decorrelator.correlateElimination(preparedRel, true); - } catch (UnsupportedOperationException e) { - // if the correlation exists in an unsupported operator, retain the original plan. + if (canDecorrelateOffsetFetch(preparedRel, false)) { + TopDownGeneralDecorrelator decorrelator = createEmptyDecorrelator(builder); + try { + decorrelateNode = decorrelator.correlateElimination(preparedRel, true); + } catch (UnsupportedOperationException e) { + // if the correlation exists in an unsupported operator, retain the original plan. + } } HepProgram postProgram = HepProgram.builder() @@ -255,6 +257,29 @@ public static RelNode decorrelateQuery(RelNode rel, RelBuilder builder) { return postPlanner.findBestExp(); } + /** Returns whether correlated Sorts in a tree have OFFSET and FETCH values + * that can be decorrelated without changing their row-count semantics. */ + private static boolean canDecorrelateOffsetFetch(RelNode rel, + boolean isCorVarDefined) { + if (isCorVarDefined && rel instanceof Sort + && !RelDecorrelator.canDecorrelateOffsetFetch((Sort) rel)) { + return false; + } + if (rel instanceof Correlate) { + final Correlate correlate = (Correlate) rel; + if (!canDecorrelateOffsetFetch(correlate.getLeft(), isCorVarDefined)) { + return false; + } + return canDecorrelateOffsetFetch(correlate.getRight(), true); + } + for (RelNode input : rel.getInputs()) { + if (!canDecorrelateOffsetFetch(input, isCorVarDefined)) { + return false; + } + } + return true; + } + /** * Eliminates Correlate. * diff --git a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java index 36d56a9f0488..2309102ff826 100644 --- a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java +++ b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java @@ -86,6 +86,7 @@ import org.apache.calcite.rex.RexSubQuery; import org.apache.calcite.rex.RexUnknownAs; import org.apache.calcite.rex.RexUtil; +import org.apache.calcite.rex.RexVisitorImpl; import org.apache.calcite.rex.RexWindowBound; import org.apache.calcite.rex.RexWindowBounds; import org.apache.calcite.rex.RexWindowExclusion; @@ -108,6 +109,7 @@ import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.sql.type.SqlTypeUtil; import org.apache.calcite.sql.type.TableFunctionReturnTypeInference; import org.apache.calcite.sql.validate.SqlValidatorUtil; import org.apache.calcite.sql2rel.SqlToRelConverter; @@ -3801,8 +3803,7 @@ public RelBuilder sortLimit(Number offset, Number fetch, * * @param offsetNode RexLiteral means number of rows to skip is deterministic, * RexDynamicParam means number of rows to skip is dynamic. - * @param fetchNode RexLiteral means maximum number of rows to fetch is deterministic, - * RexDynamicParam mean maximum number is dynamic. + * @param fetchNode Maximum number of rows to fetch * @param nodes Sort expressions */ public RelBuilder sortLimit(@Nullable RexNode offsetNode, @Nullable RexNode fetchNode, @@ -3812,12 +3813,17 @@ public RelBuilder sortLimit(@Nullable RexNode offsetNode, @Nullable RexNode fetc throw new IllegalArgumentException("OFFSET node must be RexLiteral or RexDynamicParam"); } } - if (fetchNode != null) { - if (!(fetchNode instanceof RexLiteral || fetchNode instanceof RexDynamicParam)) { - throw new IllegalArgumentException("FETCH node must be RexLiteral or RexDynamicParam"); - } + if (fetchNode != null && !isValidFetchExpression(fetchNode)) { + throw new IllegalArgumentException( + "FETCH node must not reference input fields or contain aggregate functions, " + + "window functions, or subqueries"); + } + if (fetchNode != null + && !SqlTypeUtil.isNumeric(fetchNode.getType())) { + throw new IllegalArgumentException( + "FETCH node must have a numeric type; actual type is " + + fetchNode.getType().getFullTypeString()); } - final Registrar registrar = new Registrar(fields(), ImmutableList.of()); final List fieldCollations = registrar.registerFieldCollations(nodes); @@ -3884,6 +3890,38 @@ public RelBuilder sortLimit(@Nullable RexNode offsetNode, @Nullable RexNode fetc return this; } + private static boolean isValidFetchExpression(RexNode node) { + return Boolean.TRUE.equals(node.accept(new FetchExpressionVisitor())); + } + + /** Visitor that validates FETCH expressions. */ + private static class FetchExpressionVisitor + extends RexVisitorImpl<@Nullable Boolean> { + FetchExpressionVisitor() { + super(false); + } + + @Override public Boolean visitLiteral(RexLiteral literal) { + return true; + } + + @Override public Boolean visitDynamicParam(RexDynamicParam dynamicParam) { + return true; + } + + @Override public Boolean visitCall(RexCall call) { + if (call.getOperator().isAggregator()) { + return false; + } + for (RexNode operand : call.getOperands()) { + if (!Boolean.TRUE.equals(operand.accept(this))) { + return false; + } + } + return true; + } + } + private static RelFieldCollation collation(RexNode node, RelFieldCollation.Direction direction, RelFieldCollation.@Nullable NullDirection nullDirection, diff --git a/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties b/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties index a90099d7cb91..703536de88ec 100644 --- a/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties +++ b/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties @@ -61,6 +61,9 @@ ValidatorContext=From line {0,number,#}, column {1,number,#} to line {2,number,# CannotCastValue=Cast function cannot convert value of type {0} to type {1} UnknownDatatypeName=Unknown datatype name ''{0}'' IncompatibleValueType=Values passed to {0} operator must have compatible types +FetchExpressionMustBeNumeric=FETCH expression must have a numeric type; actual type is ''{0}'' +FetchExpressionCannotReferenceColumn=FETCH expression cannot reference table column ''{0}'' +FetchExpressionEvaluatedToNull=FETCH expression evaluated to NULL IncompatibleTypesInList=Values in expression list must have compatible types IncompatibleCharset=Cannot apply operation ''{0}'' to strings with different charsets ''{1}'' and ''{2}'' InvalidOrderByPos=ORDER BY is only allowed on top-level SELECT diff --git a/core/src/test/java/org/apache/calcite/adapter/enumerable/EnumUtilsTest.java b/core/src/test/java/org/apache/calcite/adapter/enumerable/EnumUtilsTest.java index 40b825939df9..70370d7bb1ab 100644 --- a/core/src/test/java/org/apache/calcite/adapter/enumerable/EnumUtilsTest.java +++ b/core/src/test/java/org/apache/calcite/adapter/enumerable/EnumUtilsTest.java @@ -34,6 +34,7 @@ import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; /** * Tests for {@link EnumUtils}. @@ -186,6 +187,16 @@ public final class EnumUtilsTest { is(BigDecimal.valueOf(2))); } + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testNumberToBigDecimalRejectsNull() { + final IllegalArgumentException e = + assertThrows(IllegalArgumentException.class, + () -> EnumUtils.numberToBigDecimal(null, "FETCH")); + assertThat(e.getMessage(), is("FETCH expression evaluated to NULL")); + } + @Test void testMethodCallExpression() { // test for Object.class method parameter type final ConstantExpression arg0 = Expressions.constant(1, int.class); diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java index 772c409692be..01f157893623 100644 --- a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java +++ b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java @@ -4950,6 +4950,74 @@ private SqlDialect nonOrdinalDialect() { .withSybase().ok(expectedSybase); } + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testFetchExpressionWithLimitDialect() { + final String query = "select \"product_id\"\n" + + "from \"product\"\n" + + "fetch next (1 + 2) rows only"; + final String expected = "SELECT `product_id`\n" + + "FROM `foodmart`.`product`\n" + + "LIMIT 3"; + sql(query).withMysql().ok(expected); + } + + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testNegativeFetchExpressionIsRejectedBeforeSqlGeneration() { + final String query = "select \"product_id\"\n" + + "from \"product\"\n" + + "fetch next (0 - 1) rows only"; + final String error = + "FETCH value -1 is out of range; expected a non-negative value"; + sql(query).throws_(error); + sql(query).withMysql().throws_(error); + sql(query).withSQLite().throws_(error); + } + + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testParameterizedFetchExpressionWithLimitDialect() { + final String query = "select \"product_id\"\n" + + "from \"product\"\n" + + "fetch next (? + 1) rows only"; + sql(query).withMysql().throws_( + "LIMIT dialect does not support FETCH expressions that cannot " + + "be reduced to a literal"); + } + + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testParameterizedFetchExpressionWithSQLite() { + final String query = "select \"product_id\"\n" + + "from \"product\"\n" + + "fetch next (? + 1) rows only"; + final String expected = "SELECT \"product_id\"\n" + + "FROM \"foodmart\".\"product\"\n" + + "LIMIT ? + 1"; + sql(query).withSQLite().ok(expected); + } + + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testDynamicFetchExpressionIsNotReduced() { + final String query = "select \"product_id\"\n" + + "from \"product\"\n" + + "fetch next (extract(day from current_date)) rows only"; + final String expected = "SELECT \"product_id\"\n" + + "FROM \"foodmart\".\"product\"\n" + + "FETCH NEXT (EXTRACT(DAY FROM CURRENT_DATE)) ROWS ONLY"; + sql(query).ok(expected); + sql(query).withMysql().throws_( + "LIMIT dialect does not support FETCH expressions that cannot " + + "be reduced to a literal"); + } + @Test void testSelectQueryComplex() { String query = "select count(*), \"units_per_case\" from \"product\" where \"cases_per_pallet\" > 100 " diff --git a/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java b/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java index b22218b95560..5f8b8edfb1db 100644 --- a/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java +++ b/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java @@ -83,6 +83,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.fail; import static java.util.Objects.requireNonNull; @@ -3631,6 +3632,35 @@ private void assertTypeAndToString( hasSize(0)); } + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testContainsDynamicParam() { + final RelDataType intType = typeFactory.createSqlType(SqlTypeName.INTEGER); + final RexNode literal = rexBuilder.makeExactLiteral(BigDecimal.ONE, intType); + final RexNode dynamicParam = rexBuilder.makeDynamicParam(intType, 0); + final RexNode expression = + rexBuilder.makeCall(SqlStdOperatorTable.PLUS, literal, dynamicParam); + + assertThat(RexUtil.containsDynamicParam(literal), is(false)); + assertThat(RexUtil.containsDynamicParam(dynamicParam), is(true)); + assertThat(RexUtil.containsDynamicParam(expression), is(true)); + } + + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testValidateFetchValueAllowsFractionalBigDecimal() { + assertThat(RexUtil.validateFetchValue(new BigDecimal("1.5")), + is(new BigDecimal("1.5"))); + + final IllegalArgumentException e = + assertThrows(IllegalArgumentException.class, + () -> RexUtil.validateFetchValue(new BigDecimal("-1.5"))); + assertThat(e.getMessage(), + containsString("FETCH value -1.5 is out of range")); + } + @Test void testConstantMap() { final RelDataType intType = typeFactory.createSqlType(SqlTypeName.INTEGER); final RelDataType bigintType = typeFactory.createSqlType(SqlTypeName.BIGINT); diff --git a/core/src/test/java/org/apache/calcite/test/JdbcTest.java b/core/src/test/java/org/apache/calcite/test/JdbcTest.java index 83ce5b1d78a6..f213b12dcb7e 100644 --- a/core/src/test/java/org/apache/calcite/test/JdbcTest.java +++ b/core/src/test/java/org/apache/calcite/test/JdbcTest.java @@ -3581,6 +3581,181 @@ public void checkOrderBy(final boolean desc, + "store_id=4; grocery_sqft=16844\n"); } + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testFetchExpression() { + CalciteAssert.that() + .query("select * from (values (1), (2), (3), (4)) as t(x)\n" + + "fetch next (1 + abs(-2)) rows only") + .returns("X=1\n" + + "X=2\n" + + "X=3\n"); + } + + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testBindableFetchExpression() { + try (Hook.Closeable ignored = Hook.ENABLE_BINDABLE.addThread(Hook.propertyJ(true))) { + final CalciteAssert.AssertThat with = CalciteAssert.that(); + with + .query("select * from (values (1), (2), (3), (4)) as t(x)\n" + + "fetch next (rand_integer(1) + 2) rows only") + .explainContains("BindableSort(fetch=[+(RAND_INTEGER(1), 2)])") + .returns("X=1\n" + + "X=2\n"); + with.query("select * from (values (1), (2), (3), (4)) as t(x)\n" + + "fetch next (cast(9223372036854775808 as decimal(20, 0))) rows only") + .returns("X=1\nX=2\nX=3\nX=4\n"); + with.query("select * from (values (1), (2), (3), (4)) as t(x)\n" + + "order by x fetch next ? rows only") + .explainContains("BindableSort(sort0=[$0], dir0=[ASC], fetch=[?0])") + .consumesPreparedStatement(p -> + p.setBigDecimal(1, new BigDecimal("1.5"))) + .returns("X=1\n" + + "X=2\n"); + } + } + + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testFetchExpressionFunctionArguments() { + final CalciteAssert.AssertThat with = CalciteAssert.that(); + final String values = "select * from (values (1), (2), (3)) as t(x)\n"; + with.query(values + "fetch next (abs(2)) rows only") + .returns("X=1\n" + + "X=2\n"); + with.query(values + "fetch next (abs(-2)) rows only") + .returns("X=1\n" + + "X=2\n"); + } + + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testFetchExpressionInvalidValue() { + final CalciteAssert.AssertThat with = CalciteAssert.that(); + final String values = "select * from (values (1), (2), (3)) as t(x)\n"; + with.query(values + "fetch next (0 - 1) rows only") + .throws_("FETCH must not be negative"); + with.query(values + "fetch next (-1) rows only") + .throws_("FETCH must not be negative"); + with.query(values + + "fetch next (cast(null as integer)) rows only") + .throws_("FETCH expression evaluated to NULL"); + } + + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testCorrelatedFetchExpressionInvalidValue() { + final String sqlPrefix = "select d.\"name\", e.\"name\"\n" + + "from \"hr\".\"depts\" d,\n" + + "lateral (select \"name\" from \"hr\".\"emps\"\n" + + " where \"deptno\" = d.\"deptno\"\n"; + for (String fetch : new String[] {"(0 - 1)", "(-1)"}) { + for (boolean topDown : new boolean[] {false, true}) { + CalciteAssert.hr() + .with(CalciteConnectionProperty.TOPDOWN_GENERAL_DECORRELATION_ENABLED, topDown) + .query(sqlPrefix + " fetch next " + fetch + " rows only) e") + .throws_("FETCH value -1 is out of range"); + } + } + } + + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testCorrelatedFractionalOffsetFetch() { + final String sqlPrefix = "select d.\"name\" as dname, e.\"name\" as ename\n" + + "from \"hr\".\"depts\" d,\n" + + "lateral (select \"empid\", \"name\" from \"hr\".\"emps\"\n" + + " where \"deptno\" = d.\"deptno\"\n" + + " order by \"empid\" "; + final String sqlSuffix = ") e\norder by e.\"empid\""; + for (boolean topDown : new boolean[] {false, true}) { + final CalciteAssert.AssertThat with = CalciteAssert.hr() + .with(CalciteConnectionProperty.TOPDOWN_GENERAL_DECORRELATION_ENABLED, topDown); + with.query(sqlPrefix + "fetch next (0.5 + 1) rows only" + sqlSuffix) + .returns("DNAME=Sales; ENAME=Bill\n" + + "DNAME=Sales; ENAME=Theodore\n"); + with.query(sqlPrefix + "offset 1.5 rows fetch next 1 row only" + sqlSuffix) + .returns("DNAME=Sales; ENAME=Sebastian\n"); + } + } + + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testCorrelatedPreparedFractionalOffset() throws Exception { + final String sql = "select d.\"name\" as dname, e.\"name\" as ename\n" + + "from \"hr\".\"depts\" d,\n" + + "lateral (select \"empid\", \"name\" from \"hr\".\"emps\"\n" + + " where \"deptno\" = d.\"deptno\"\n" + + " order by \"empid\" offset ? rows fetch next 1 row only) e\n" + + "order by e.\"empid\""; + for (boolean topDown : new boolean[] {false, true}) { + CalciteAssert.hr() + .with(CalciteConnectionProperty.TOPDOWN_GENERAL_DECORRELATION_ENABLED, topDown) + .doWithConnection(connection -> { + checkPreparedBigDecimalParameter(connection, sql, + new BigDecimal("1.5"), + "DNAME=Sales; ENAME=Sebastian\n"); + }); + } + } + + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testCorrelatedPreparedFetchExpression() throws Exception { + for (String fetch : new String[] {"?", "(? + 0)"}) { + final String sql = "select d.\"name\" as dname, e.\"name\" as ename\n" + + "from \"hr\".\"depts\" d,\n" + + "lateral (select \"empid\", \"name\" from \"hr\".\"emps\"\n" + + " where \"deptno\" = d.\"deptno\"\n" + + " order by \"empid\" fetch next " + fetch + " rows only) e\n" + + "order by e.\"empid\""; + for (boolean topDown : new boolean[] {false, true}) { + CalciteAssert.hr() + .with(CalciteConnectionProperty.TOPDOWN_GENERAL_DECORRELATION_ENABLED, topDown) + .doWithConnection(connection -> { + checkPreparedFetchRepeated(connection, sql, + new int[] {1, 3}, + new String[] { + "DNAME=Sales; ENAME=Bill\n", + "DNAME=Sales; ENAME=Bill\n" + + "DNAME=Sales; ENAME=Theodore\n" + + "DNAME=Sales; ENAME=Sebastian\n" + }); + checkPreparedParameterFails(connection, sql, -1, + "FETCH must not be negative"); + checkPreparedParameterNullFails(connection, sql, + "FETCH expression evaluated to NULL"); + }); + } + } + } + + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testFetchExpressionBeyondLong() { + final CalciteAssert.AssertThat with = CalciteAssert.that(); + final String values = "select * from (values (1), (2), (3), (4)) as t(x)\n"; + final String expected = "X=1\nX=2\nX=3\nX=4\n"; + with.query(values + "fetch next 9223372036854775808 rows only") + .returns(expected); + with.query(values + "fetch next " + + "(cast(9223372036854775808 as decimal(20, 0)) + 1) rows only") + .returns(expected); + with.query(values + "order by x fetch next " + + "(cast(9223372036854775808 as decimal(20, 0)) + 1) rows only") + .returns(expected); + } + /** Tests ORDER BY ... OFFSET ... FETCH. */ @Test void testOrderByOffsetFetch() { CalciteAssert.that() @@ -6058,6 +6233,169 @@ private CalciteAssert.AssertQuery withEmpDept(String sql) { "name=Theodore"); } + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testPreparedFetchExpression() throws Exception { + CalciteAssert.that() + .doWithConnection(connection -> { + final String values = + "select * from (values (1), (2), (3), (4)) as t(x)\n"; + checkPreparedFetch(connection, values + "fetch next (?) rows only", + 2, "X=1\nX=2\n"); + checkPreparedFetch(connection, values + "fetch next (? + 1) rows only", + 2, "X=1\nX=2\nX=3\n"); + checkPreparedFetch(connection, + values + "fetch next (abs(cast(? as integer))) rows only", + 2, "X=1\nX=2\n"); + checkPreparedFetch(connection, + values + "fetch next (abs(cast(? as integer))) rows only", + -2, "X=1\nX=2\n"); + checkPreparedFetchRepeated(connection, + values + "fetch next (?) rows only", + new int[] {1, 3}, + new String[] {"X=1\n", "X=1\nX=2\nX=3\n"}); + checkPreparedFetchRepeated(connection, + values + "fetch next (? + 1) rows only", + new int[] {0, 2, 3}, + new String[] {"X=1\n", "X=1\nX=2\nX=3\n", + "X=1\nX=2\nX=3\nX=4\n"}); + checkPreparedFetch(connection, + values + "fetch next (? + abs(2)) rows only", + 1, "X=1\nX=2\nX=3\n"); + checkPreparedBigDecimalParameter(connection, + values + "fetch next (cast(? as decimal(20, 0))) rows only", + new BigDecimal("9223372036854775808"), + "X=1\nX=2\nX=3\nX=4\n"); + + checkPreparedParameterFails(connection, + values + "fetch next (?) rows only", -1, + "FETCH must not be negative"); + checkPreparedParameterFails(connection, + values + "fetch next (? + 1) rows only", -2, + "FETCH must not be negative"); + }); + } + + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testBindablePreparedFetchExpression() throws Exception { + try (Hook.Closeable ignored = Hook.ENABLE_BINDABLE.addThread(Hook.propertyJ(true))) { + CalciteAssert.that() + .doWithConnection(connection -> { + final String values = + "select * from (values (1), (2), (3), (4)) as t(x)\n"; + checkPreparedFetch(connection, + values + "fetch next (? + 1) rows only", + 2, "X=1\nX=2\nX=3\n"); + checkPreparedFetchRepeated(connection, + values + "fetch next (? + 1) rows only", + new int[] {0, 2, 3}, + new String[] {"X=1\n", "X=1\nX=2\nX=3\n", + "X=1\nX=2\nX=3\nX=4\n"}); + checkPreparedBigDecimalParameter(connection, + values + "fetch next (cast(? as decimal(20, 0))) rows only", + new BigDecimal("9223372036854775808"), + "X=1\nX=2\nX=3\nX=4\n"); + }); + } + } + + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testBindablePreparedOffset() throws Exception { + try (Hook.Closeable ignored = Hook.ENABLE_BINDABLE.addThread(Hook.propertyJ(true))) { + CalciteAssert.that() + .doWithConnection(connection -> { + final String values = + "select * from (values (1), (2), (3), (4)) as t(x)\n"; + final String offset = values + "offset ? rows"; + checkPreparedBigDecimalParameter(connection, offset, + new BigDecimal("1.5"), + "X=3\nX=4\n"); + checkPreparedBigDecimalParameter(connection, offset, + BigDecimal.valueOf(Integer.MAX_VALUE).add(BigDecimal.ONE), ""); + + final String sortedOffset = values + "order by x desc offset ? rows"; + checkPreparedBigDecimalParameter(connection, sortedOffset, + new BigDecimal("1.5"), + "X=2\nX=1\n"); + checkPreparedParameterFails(connection, offset, -1, + "OFFSET must not be negative"); + checkPreparedParameterNullFails(connection, offset, + "OFFSET expression evaluated to NULL"); + }); + } + } + + private static void checkPreparedFetch(Connection connection, String sql, + int value, String expected) { + try (PreparedStatement p = connection.prepareStatement(sql)) { + p.setInt(1, value); + try (ResultSet r = p.executeQuery()) { + assertThat(CalciteAssert.toString(r), is(expected)); + } + } catch (SQLException e) { + throw TestUtil.rethrow(e); + } + } + + private static void checkPreparedBigDecimalParameter(Connection connection, String sql, + BigDecimal value, String expected) { + try (PreparedStatement p = connection.prepareStatement(sql)) { + p.setBigDecimal(1, value); + try (ResultSet r = p.executeQuery()) { + assertThat(CalciteAssert.toString(r), is(expected)); + } + } catch (SQLException e) { + throw TestUtil.rethrow(e); + } + } + + private static void checkPreparedFetchRepeated(Connection connection, String sql, + int[] values, String[] expected) { + try (PreparedStatement p = connection.prepareStatement(sql)) { + for (int i = 0; i < values.length; i++) { + p.setInt(1, values[i]); + try (ResultSet r = p.executeQuery()) { + assertThat(CalciteAssert.toString(r), is(expected[i])); + } + } + } catch (SQLException e) { + throw TestUtil.rethrow(e); + } + } + + private static void checkPreparedParameterFails(Connection connection, String sql, + long value, String expectedMessage) { + try (PreparedStatement p = connection.prepareStatement(sql)) { + if (value >= Integer.MIN_VALUE && value <= Integer.MAX_VALUE) { + p.setInt(1, (int) value); + } else { + p.setLong(1, value); + } + final SQLException e = + assertThrows(SQLException.class, p::executeQuery); + assertThat(e.getMessage(), containsString(expectedMessage)); + } catch (SQLException e) { + throw TestUtil.rethrow(e); + } + } + + private static void checkPreparedParameterNullFails(Connection connection, String sql, + String expectedMessage) { + try (PreparedStatement p = connection.prepareStatement(sql)) { + p.setNull(1, Types.INTEGER); + final SQLException e = + assertThrows(SQLException.class, p::executeQuery); + assertThat(e.getMessage(), containsString(expectedMessage)); + } catch (SQLException e) { + throw TestUtil.rethrow(e); + } + } + private void checkPreparedOffsetFetch(final int offset, final int fetch, final Matcher matcher) throws Exception { CalciteAssert.hr() diff --git a/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java b/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java index 493df7c30b03..7ad9a4d733fc 100644 --- a/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java @@ -52,7 +52,10 @@ import org.apache.calcite.rex.RexCorrelVariable; import org.apache.calcite.rex.RexFieldCollation; import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexLambdaRef; import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexNodeAndFieldIndex; +import org.apache.calcite.rex.RexSubQuery; import org.apache.calcite.rex.RexWindowBounds; import org.apache.calcite.runtime.CalciteException; import org.apache.calcite.schema.SchemaPlus; @@ -5647,6 +5650,108 @@ private static RelNode buildCorrelateWithJoin(JoinRelType type, RelBuilder build assertThat(mq.getMaxRowCount(planAfter), is(Double.POSITIVE_INFINITY)); } + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testFetchExpressionCannotReferenceInputField() { + final RelBuilder builder = RelBuilder.create(config().build()); + builder.scan("DEPT"); + final RexNode field = builder.field("DEPTNO"); + + assertThrows(IllegalArgumentException.class, + () -> builder.sortLimit(null, field, ImmutableList.of())); + assertThrows(IllegalArgumentException.class, + () -> builder.sortLimit(null, + builder.call(SqlStdOperatorTable.PLUS, builder.literal(1), field), + ImmutableList.of())); + assertThrows(IllegalArgumentException.class, + () -> builder.sortLimit(null, + new RexNodeAndFieldIndex(0, 0, "DEPTNO", field.getType()), + ImmutableList.of())); + } + + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testFetchExpressionMustHaveNumericType() { + final RelBuilder builder = RelBuilder.create(config().build()); + builder.scan("DEPT"); + + assertThrows(IllegalArgumentException.class, + () -> builder.sortLimit(null, builder.literal("x"), ImmutableList.of())); + builder.sortLimit(null, builder.literal(new BigDecimal("1.5")), + ImmutableList.of()); + } + + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testFetchExpressionAllowsScalarCallAndDynamicParameter() { + final RelBuilder builder = RelBuilder.create(config().build()); + final RelDataType intType = + builder.getTypeFactory().createSqlType(SqlTypeName.INTEGER); + builder.scan("DEPT") + .sortLimit(null, + builder.call(SqlStdOperatorTable.PLUS, + builder.getRexBuilder().makeDynamicParam(intType, 0), + builder.literal(1)), + ImmutableList.of()); + + assertThat( + builder.build(), hasTree("LogicalSort(fetch=[+(?0, 1)])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n")); + } + + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testFetchExpressionCannotContainAggregateWindowOrSubQuery() { + final RelBuilder builder = RelBuilder.create(config().build()); + final RelDataType intType = + builder.getTypeFactory().createSqlType(SqlTypeName.INTEGER); + builder.scan("DEPT"); + final RexNode aggregate = + builder.call(SqlStdOperatorTable.SUM, builder.literal(1)); + assertThrows(IllegalArgumentException.class, + () -> builder.sortLimit(null, aggregate, ImmutableList.of())); + + final RexNode over = + builder.getRexBuilder().makeOver(intType, + SqlStdOperatorTable.ROW_NUMBER, ImmutableList.of(), + ImmutableList.of(), ImmutableList.of(), + RexWindowBounds.UNBOUNDED_PRECEDING, + RexWindowBounds.UNBOUNDED_FOLLOWING, + true, true, false, false, false); + assertThrows(IllegalArgumentException.class, + () -> builder.sortLimit(null, over, ImmutableList.of())); + + final RelBuilder subQueryBuilder = RelBuilder.create(config().build()); + final RexNode subQuery = + RexSubQuery.scalar(subQueryBuilder.values(new String[] {"N"}, 1).build()); + assertThrows(IllegalArgumentException.class, + () -> builder.sortLimit(null, subQuery, ImmutableList.of())); + } + + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testFetchExpressionCannotContainLambda() { + final RelBuilder builder = RelBuilder.create(config().build()); + final RelDataType intType = + builder.getTypeFactory().createSqlType(SqlTypeName.INTEGER); + builder.scan("DEPT"); + final RexLambdaRef lambdaRef = new RexLambdaRef(0, "x", intType); + final RexNode lambda = + builder.getRexBuilder().makeLambdaCall( + builder.call(SqlStdOperatorTable.PLUS, lambdaRef, builder.literal(1)), + ImmutableList.of(lambdaRef)); + + assertThrows(IllegalArgumentException.class, + () -> builder.sortLimit(null, lambda, ImmutableList.of())); + assertThrows(IllegalArgumentException.class, + () -> builder.sortLimit(null, lambdaRef, ImmutableList.of())); + } + @Test void testAdoptConventionEnumerable() { final RelBuilder builder = RelBuilder.create(config().build()); RelNode root = builder diff --git a/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java b/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java index 9c3c30e3448e..460e066051fe 100644 --- a/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java @@ -1467,7 +1467,7 @@ void testColumnOriginsUnion() { @Test void testRowCountSortLimitBeyondLong() { final BigDecimal fetch = BigDecimal.valueOf(Long.MAX_VALUE).add(BigDecimal.ONE); final double fetchDouble = fetch.doubleValue(); - final String sql = "select * from emp order by ename limit " + fetchDouble; + final String sql = "select * from emp order by ename limit " + fetch.toPlainString(); final RelMetadataFixture fixture = sql(sql); fixture.assertThatRowCount(is(EMP_SIZE), is(0D), is(fetchDouble)); } @@ -1496,6 +1496,37 @@ void testColumnOriginsUnion() { fixture.assertThatRowCount(is(1d), is(0D), is(0d)); } + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testMinRowCountFetchExpression() { + final String sql = "select * from (values (1), (2)) as t(x)\n" + + "fetch next (2 - 2) rows only"; + final RelMetadataFixture fixture = sql(sql); + fixture.assertThatRowCount(is(2D), is(0D), is(2D)); + + fixture + .withCluster(cluster -> { + final RelOptPlanner planner = new VolcanoPlanner(); + planner.addRule(EnumerableRules.ENUMERABLE_VALUES_RULE); + planner.addRule(EnumerableRules.ENUMERABLE_PROJECT_RULE); + planner.addRule(EnumerableRules.ENUMERABLE_LIMIT_RULE); + planner.addRelTraitDef(ConventionTraitDef.INSTANCE); + return RelOptCluster.create(planner, cluster.getRexBuilder()); + }) + .withRelTransform(rel -> { + final RelOptPlanner planner = rel.getCluster().getPlanner(); + planner.setRoot(rel); + final RelTraitSet requiredOutputTraits = + rel.getCluster().traitSet().replace(EnumerableConvention.INSTANCE); + final RelNode root = planner.changeTraits(rel, requiredOutputTraits); + planner.setRoot(root); + return planner.findBestExp(); + }) + .assertThatRel(is(instanceOf(EnumerableLimit.class))) + .assertThatRowCount(is(2D), is(0D), is(2D)); + } + @Test void testRowCountSortLimitOffset() { final String sql = "select * from emp order by ename limit 10 offset 5"; /* 14 - 5 */ diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index 4ba2f4aa9623..5c687519daa0 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -1738,6 +1738,34 @@ private void checkJoinProjectTransposeDoesNotMatch(JoinRelType type) { .check(); } + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testSortUnionTransposeWithNonDeterministicFetch() { + final String sql = "select a.name from dept a\n" + + "union all\n" + + "select b.name from dept b\n" + + "order by name fetch next (rand_integer(10)) rows only"; + sql(sql) + .withPreRule(CoreRules.PROJECT_SET_OP_TRANSPOSE) + .withRule(CoreRules.SORT_UNION_TRANSPOSE) + .checkUnchanged(); + } + + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testSortUnionTransposePushesParameterizedFetchExpression() { + final String sql = "select a.name from dept a\n" + + "union all\n" + + "select b.name from dept b\n" + + "order by name fetch next (? + 1) rows only"; + sql(sql) + .withPreRule(CoreRules.PROJECT_SET_OP_TRANSPOSE) + .withRule(CoreRules.SORT_UNION_TRANSPOSE) + .check(); + } + @Test void testSortRemovalAllKeysConstant() { final String sql = "select count(*) as c\n" + "from sales.emp\n" @@ -5997,10 +6025,9 @@ private void checkEmptyJoin(RelOptFixture f) { } /** Test case for - * [CALCITE-6647] - * SortUnionTransposeRule should not push SORT past a UNION when SORT's fetch is DynamicParam - . */ - @Test void testSortWithDynamicParam() { + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testSortWithDynamicParamPushesOnce() { HepProgramBuilder builder = new HepProgramBuilder(); builder.addRuleClass(SortProjectTransposeRule.class); builder.addRuleClass(SortUnionTransposeRule.class); @@ -9730,6 +9757,19 @@ private void checkSemiJoinRuleOnAntiJoin(RelOptRule rule) { .check(); } + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testDecorrelateProjectWithFetchExpression() { + final String query = "SELECT name, " + + "(SELECT sal FROM emp where dept.deptno = emp.deptno order by sal " + + "fetch next (1 + 0) rows only) " + + "FROM dept"; + sql(query).withRule(CoreRules.PROJECT_SUB_QUERY_TO_CORRELATE) + .withLateDecorrelate(true) + .check(); + } + /** Test case for [CALCITE-7289] * Select NULL subquery throwing exception. */ @Test void testNullSelect() { @@ -12218,6 +12258,39 @@ private static RelNode applyAggregateRemoveLiteralAggRule(RelNode rel) { .check(); } + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testNondeterministicFetchPreventsDecorrelation() { + checkNondeterministicFetchPreventsDecorrelation(false); + } + + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testNondeterministicFetchPreventsTopDownDecorrelation() { + checkNondeterministicFetchPreventsDecorrelation(true); + } + + private void checkNondeterministicFetchPreventsDecorrelation(boolean enableTopDown) { + final String sql = "select t.deptno, e.ename\n" + + "from (select distinct deptno from emp) t,\n" + + "lateral (select ename from emp\n" + + " where emp.deptno = t.deptno\n" + + " order by sal\n" + + " fetch next (rand_integer(2) + 1) rows only) e"; + + final RelOptFixture fixture = sql(sql) + .withRule() // empty program + .withLateDecorrelate(true) + .withTopDownGeneralDecorrelate(enableTopDown); + if (enableTopDown) { + fixture.check(); + } else { + fixture.checkUnchanged(); + } + } + @Test void testTopDownGeneralDecorrelateForFilterSome() { final String sql = "select empno from emp where " + "empno > SOME(select empno from emp_b where emp.ename = emp_b.ename)"; diff --git a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java index 6b3255e653c7..6ce401502c93 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java @@ -1263,6 +1263,15 @@ public static void checkActualAndReferenceFiles() { sql(sql).ok(); } + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testFetchWithExpression() { + final String sql = + "select empno from emp fetch next (1 + abs(-2)) rows only"; + sql(sql).ok(); + } + /** Test case for * [CALCITE-439] * SqlValidatorUtil.uniquify() may not terminate under some conditions. */ diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index 018c27e898f1..2e472a968c63 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -10747,6 +10747,22 @@ void testGroupExpressionEquivalenceParams() { .rewritesTo(expected); } + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testFetchExpressionType() { + sql("select name from dept fetch next (^upper('x')^) rows only") + .fails("FETCH expression must have a numeric type; " + + "actual type is 'CHAR\\(1\\) NOT NULL'"); + sql("select name from dept fetch next (^'x'^) rows only") + .fails("FETCH expression must have a numeric type; " + + "actual type is 'CHAR\\(1\\) NOT NULL'"); + sql("select name from dept fetch next 1.5 rows only").ok(); + sql("select name from dept " + + "fetch next (^row_number() over ()^) rows only") + .fails("Windowed aggregate expression is illegal in FETCH clause"); + } + @Test void testRewriteWithOffsetWithoutOrderBy() { final String sql = "select name from dept offset 2"; final String expected = "SELECT `NAME`\n" diff --git a/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableMergeUnionTest.java b/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableMergeUnionTest.java index 68bb56cf366d..44055f707462 100644 --- a/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableMergeUnionTest.java +++ b/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableMergeUnionTest.java @@ -78,6 +78,36 @@ class EnumerableMergeUnionTest { "empid=45; name=Pascal"); } + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void mergeUnionDoesNotPushNonDeterministicFetch() { + tester(false, + new HrSchemaBig(), + "select * from (select empid, name from emps " + + "union all select empid, name from emps) " + + "order by empid fetch next (rand_integer(10)) rows only") + .explainContains("EnumerableLimitSort(sort0=[$0], dir0=[ASC], " + + "fetch=[RAND_INTEGER(10)])\n" + + " EnumerableMergeUnion(all=[true])\n" + + " EnumerableSort(sort0=[$0], dir0=[ASC])\n"); + } + + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void mergeUnionPushesParameterizedFetchExpression() { + tester(false, + new HrSchemaBig(), + "select * from (select empid, name from emps " + + "union all select empid, name from emps) " + + "order by empid fetch next (? + 1) rows only") + .explainContains("EnumerableLimit(fetch=[+(?0, 1)])\n" + + " EnumerableMergeUnion(all=[true])\n" + + " EnumerableLimitSort(sort0=[$0], dir0=[ASC], " + + "fetch=[+(?0, 1)])\n"); + } + @Test void mergeUnionAllOrderByName() { tester(false, new HrSchemaBig(), diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index ab26ca8524bc..32f423c3e17b 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -2950,6 +2950,46 @@ LogicalProject(NAME=[$1]) LogicalFilter(condition=[<=($3, 1)]) LogicalProject(SAL=[$5], EXPR$1=[EXTRACT(FLAG(YEAR), $4)], DEPTNO=[$7], rn=[ROW_NUMBER() OVER (PARTITION BY $7 ORDER BY EXTRACT(FLAG(YEAR), $4) NULLS LAST, $5 DESC NULLS FIRST)]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +]]> + + + + + + + + + + + + + + @@ -11305,6 +11345,95 @@ LogicalProject(USER=[USER]) LogicalAggregate(group=[{0}], EXPR$1=[SUM($1)]) LogicalProject(NAME=[$1], DEPTNO=[$0]) LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) +]]> + + + + + + + + + + + + + + + + + + + + + + + + + @@ -19923,7 +20052,55 @@ LogicalSort(sort0=[$0], dir0=[ASC], fetch=[0]) ]]> - + + + + + + + + + + + + + + + + + + + + diff --git a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml index 217b2bbc03b0..6e98c11baa86 100644 --- a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml @@ -2601,6 +2601,18 @@ LogicalSort(fetch=[5]) LogicalSort(fetch=[?0]) LogicalProject(EMPNO=[$0]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +]]> + + + + + + + + diff --git a/core/src/test/resources/sql/fetch.iq b/core/src/test/resources/sql/fetch.iq new file mode 100644 index 000000000000..8f4b0dd53d58 --- /dev/null +++ b/core/src/test/resources/sql/fetch.iq @@ -0,0 +1,183 @@ +# fetch.iq +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +!use post +!set outputformat mysql + +# FETCH accepts a parenthesized arithmetic expression. +select * +from (values (1), (2), (3), (4)) as t(x) +fetch next (1 + abs(-2)) rows only; ++---+ +| X | ++---+ +| 1 | +| 2 | +| 3 | ++---+ +(3 rows) + +!ok + +# FETCH accepts a parenthesized scalar expression. +select * +from (values (1), (2), (3), (4)) as t(x) +fetch next (abs(2)) rows only; ++---+ +| X | ++---+ +| 1 | +| 2 | ++---+ +(2 rows) + +!ok + +# FETCH values are not restricted to the BIGINT range. +select * +from (values (1), (2), (3), (4)) as t(x) +fetch next (cast(9223372036854775808 as decimal(20, 0)) + 1) rows only; ++---+ +| X | ++---+ +| 1 | +| 2 | +| 3 | +| 4 | ++---+ +(4 rows) + +!ok + +# FETCH expression cannot be negative. +select * +from (values (1), (2), (3)) as t(x) +fetch next (0 - 1) rows only; +FETCH must not be negative +!error + +# FETCH expression cannot evaluate to NULL. +select * +from (values (1), (2), (3)) as t(x) +fetch next (cast(null as integer)) rows only; +FETCH expression evaluated to NULL +!error + +# FETCH expression may have a fractional numeric type. +select * +from (values (1), (2), (3)) as t(x) +fetch next (1.5) rows only; ++---+ +| X | ++---+ +| 1 | +| 2 | ++---+ +(2 rows) + +!ok + +# FETCH expression cannot reference input columns. +select * +from (values (1), (2), (3)) as t(x) +fetch next (x) rows only; +FETCH expression cannot reference table column 'X' +!error + +# Expressions without parentheses are not allowed in FETCH. +select * +from (values (1), (2), (3)) as t(x) +fetch next 1 + 2 rows only; +Encountered "+" +!error + +# FETCH expression works with a table source. +select deptno, dname +from dept +order by deptno +fetch next (1 + 1) rows only; ++--------+-------------+ +| DEPTNO | DNAME | ++--------+-------------+ +| 10 | Sales | +| 20 | Marketing | ++--------+-------------+ +(2 rows) + +!ok + +# FETCH expression works together with OFFSET on a table source. +select deptno, dname +from dept +order by deptno +offset 1 rows +fetch next (1 + 1) rows only; ++--------+-------------+ +| DEPTNO | DNAME | ++--------+-------------+ +| 20 | Marketing | +| 30 | Engineering | ++--------+-------------+ +(2 rows) + +!ok + +# FETCH expression may contain a scalar function on a table source. +select deptno +from dept +order by deptno +fetch next (abs(-3)) rows only; ++--------+ +| DEPTNO | ++--------+ +| 10 | +| 20 | +| 30 | ++--------+ +(3 rows) + +!ok + +# FETCH expression cannot reference columns of a table source. +select deptno, dname +from dept +order by deptno +fetch next (deptno) rows only; +FETCH expression cannot reference table column 'DEPTNO' +!error + +# FETCH expression cannot reference columns even inside a larger expression. +select deptno, dname +from dept +order by deptno +fetch next (deptno + 1) rows only; +FETCH expression cannot reference table column 'DEPTNO' +!error + +# FETCH expression may be zero on a table source. +select deptno +from dept +order by deptno +fetch next (2 - 2) rows only; ++--------+ +| DEPTNO | ++--------+ ++--------+ +(0 rows) + +!ok diff --git a/server/src/test/java/org/apache/calcite/test/ServerTest.java b/server/src/test/java/org/apache/calcite/test/ServerTest.java index 355d39de7d63..39d434f23ae9 100644 --- a/server/src/test/java/org/apache/calcite/test/ServerTest.java +++ b/server/src/test/java/org/apache/calcite/test/ServerTest.java @@ -43,6 +43,7 @@ import java.math.BigDecimal; import java.sql.Connection; import java.sql.DriverManager; +import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; @@ -452,6 +453,42 @@ static Connection connect() throws SQLException { } } + /** Test case for + * [CALCITE-7592] + * Add expression support for FETCH. */ + @Test void testFetchExpressionCannotReferenceInputColumn() throws Exception { + try (Connection c = connect(); + Statement s = c.createStatement()) { + s.execute("create table person (id int not null, name varchar(20))"); + try (PreparedStatement p = + c.prepareStatement("insert into person (id, name) values (?, ?)")) { + p.setInt(1, 1); + p.setString(2, "foo"); + assertThat(p.executeUpdate(), is(1)); + } + + SQLException e = + assertThrows( + SQLException.class, () -> s.executeQuery("select * from person " + + "fetch next id rows only")); + assertThat(e.getMessage(), containsString("Encountered \"id\"")); + + e = + assertThrows( + SQLException.class, () -> s.executeQuery("select * from person " + + "fetch next (id) rows only")); + assertThat(e.getMessage(), + containsString("FETCH expression cannot reference table column 'ID'")); + + e = + assertThrows( + SQLException.class, () -> s.executeQuery("select * from person " + + "fetch next (1 + id) rows only")); + assertThat(e.getMessage(), + containsString("FETCH expression cannot reference table column 'ID'")); + } + } + /** Test case for * [CALCITE-6022] * Support "CREATE TABLE ... LIKE" DDL in server module. */ diff --git a/site/_docs/reference.md b/site/_docs/reference.md index 904ab0461967..56fbf8a86ca4 100644 --- a/site/_docs/reference.md +++ b/site/_docs/reference.md @@ -427,8 +427,13 @@ in the order that they appear in the list; for example: "SELECT x, y FROM t ORDER BY x, y" An optional trailing ASC / DESC and NULLS FIRST / NULLS LAST applies to all keys. -In *query*, *count* and *start* may each be either an unsigned numeric literal -or a dynamic parameter whose value is numeric. +In *query*, *start* may be either an unsigned numeric literal or a dynamic +parameter whose value is numeric. The *count* in a LIMIT clause may be either +an unsigned numeric literal or a dynamic parameter whose value is numeric. The +*count* in a FETCH clause may be an unsigned numeric literal, a dynamic +parameter whose value is numeric, or a scalar expression enclosed in +parentheses. A FETCH *count* expression cannot reference columns from the query +input, and cannot contain aggregate functions, window functions, or sub-queries. Support for decimal or non-integer values is adapter-dependent. An aggregate query is a query that contains a GROUP BY or a HAVING diff --git a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java index 4058d716bfb9..394aecf26f2a 100644 --- a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java +++ b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java @@ -4104,12 +4104,31 @@ void checkPeriodPredicate(Checker checker) { + "FROM `FOO`\n" + "OFFSET ? ROWS\n" + "FETCH NEXT ? ROWS ONLY"); + // CALCITE-7592: Arithmetic and scalar expressions are allowed within parentheses. + sql("select a from foo fetch next (1 + abs(-2)) rows only") + .ok("SELECT `A`\n" + + "FROM `FOO`\n" + + "FETCH NEXT (1 + ABS(-2)) ROWS ONLY"); + // Expressions without parentheses are not allowed. + sql("select a from foo fetch next 1 ^+^ 2 rows only") + .fails("(?s).*Encountered \"\\+\" at .*"); + sql("select a from foo fetch next ? ^+^ abs(2) rows only") + .fails("(?s).*Encountered \"\\+\" at .*"); // missing ROWS after FETCH sql("select a from foo offset 1 fetch next 3 ^only^") .fails("(?s).*Encountered \"only\" at .*"); // FETCH before OFFSET is illegal sql("select a from foo fetch next 3 rows only ^offset^ 1") .fails("(?s).*Encountered \"offset\" at .*"); + // Subqueries are not allowed in FETCH + sql("select a from foo fetch next ^select^ 2 rows only") + .fails("(?s).*Encountered \"select\" at .*"); + sql("select a from foo fetch next (^select^ 2) rows only") + .fails("(?s).*Encountered \"select\" at .*"); + sql("select a from foo fetch next (^select^ ?) rows only") + .fails("(?s).*Encountered \"select\" at .*"); + sql("select a from foo fetch next (^select^ max(a) from foo) rows only") + .fails("(?s).*Encountered \"select\" at .*"); } /** From edda20015c7f97f4b885b28447413c24d4909c77 Mon Sep 17 00:00:00 2001 From: Ruben Quesada Lopez Date: Thu, 30 Jul 2026 11:41:06 +0100 Subject: [PATCH 427/562] Site Readme: add the old instructions to manually publish the site, in case the automatic process fails --- site/README.md | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/site/README.md b/site/README.md index 9cba819dcbe9..a5759b50fd93 100644 --- a/site/README.md +++ b/site/README.md @@ -72,3 +72,45 @@ We identify release publishing by checking new release tags. If you are the Rele you only need to push the new tag 'calcite-x.y.z' to [Calcite Github repo](https://github.com/apache/calcite), and the Github workflow will do all the rest. The rules and scripts are in `.github/workflows/publish-website-on-release.yml`. + +## Legacy method: manually pushing to site + +This should normally **not** be required any more, but in case the automatic publication fails, +these are the (legacy) steps to manually publish the site: +- Push the commit with the changes to the `main` branch of this repository. +- Cherry-pick the commit from the `main` branch to the `site` branch of this repository. +- Checkout the `site` branch and build the website using [docker-compose](#previewing-the-website-locally-using-docker). +- Commit the generated content to the `main` branch of the `calcite-site` repository following these steps: + +1. `cd site/target` +2. `git init` +3. `git remote add origin git@github.com:apache/calcite-site.git` +4. `git fetch` +5. `git reset origin/main --soft` + +If you have not regenerated the javadoc (you shouldn't unless you are publishing a new release) +and they are missing, restore them: + +6. `git reset -- javadocAggregate/` +7. `git checkout -- javadocAggregate/` + +Restore the avatica site + +8. `git reset -- avatica/` +9. `git checkout -- avatica/` + +Push the changes +10. `git add .` +11. Commit: `git commit -m "Your commit message goes here"` +12. Push the site: `git push origin main` + +Within a few minutes, gitpubsub should kick in and you'll be able to +see the results at +[calcite.apache.org](https://calcite.apache.org/). + +This process also publishes Avatica's web site. Avatica's web site has +separate source (under `avatica/site`) but configures Jekyll to +generate files to `site/target/avatica`, which becomes an +[avatica](https://calcite.apache.org/avatica) +sub-directory when deployed. See +[Avatica site README](https://github.com/apache/calcite-avatica/blob/main/site/README.md). From 3892cdaf53bdf121d61dd61fab9be9b0f2d31a7f Mon Sep 17 00:00:00 2001 From: xuzifu666 <1206332514@qq.com> Date: Mon, 27 Jul 2026 17:31:12 +0800 Subject: [PATCH 428/562] [CALCITE-7673] Add support for the LIKE operator to the MongoDB adapter --- .../calcite/adapter/mongodb/MongoFilter.java | 182 ++++++++++++++++++ .../adapter/mongodb/MongoAdapterTest.java | 164 ++++++++++++++++ 2 files changed, 346 insertions(+) diff --git a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoFilter.java b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoFilter.java index 00a26c5fe3cb..c3d12a84be0b 100644 --- a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoFilter.java +++ b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoFilter.java @@ -31,6 +31,7 @@ import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexUtil; +import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.util.JsonBuilder; import org.apache.calcite.util.Pair; @@ -222,6 +223,10 @@ private Void translateMatch2(RexNode node, List> orMapList, return translateUnary("$ne", (RexCall) node, multimap, eqMap); case IS_NULL: return translateUnary("$eq", (RexCall) node, multimap, eqMap); + case LIKE: + return translateLike((RexCall) node, multimap); + case NOT: + return translateNot((RexCall) node, orMapList); default: throw new AssertionError("cannot translate " + node); } @@ -302,5 +307,182 @@ private Void translateUnary(String op, RexCall call, translateBinary2(op, left, right, multimap, eqMap); return null; } + + /** Translates LIKE to {$regex: ...}. */ + private Void translateLike(RexCall call, + Multimap> multimap) { + final RexNode left = stripCast(call.operands.get(0)); + final RexNode right = call.operands.get(1); + + // LIKE must have a literal on the right side + if (right.getKind() != SqlKind.LITERAL) { + throw new AssertionError("cannot translate LIKE with non-literal pattern: " + call); + } + final RexLiteral patternLiteral = (RexLiteral) right; + final String sqlPattern = patternLiteral.getValue2().toString(); + + final @Nullable Character escapeChar = escapeChar(call); + final String finalRegex = sqlLikeToMongoRegex(sqlPattern, escapeChar); + + switch (left.getKind()) { + case INPUT_REF: + final RexInputRef left1 = (RexInputRef) left; + String name = fieldNames.get(left1.getIndex()); + multimap.put(name, Pair.of("$regex", rexBuilder.makeLiteral(finalRegex))); + return null; + case ITEM: + String itemName = MongoRules.isItem((RexCall) left); + if (itemName != null) { + multimap.put(itemName, Pair.of("$regex", rexBuilder.makeLiteral(finalRegex))); + return null; + } + // fall through + default: + throw new AssertionError("cannot translate LIKE " + call); + } + } + + /** Translates NOT to a MongoDB $nor expression. */ + private Void translateNot(RexCall call, List> orMapList) { + final RexNode operand = call.operands.get(0); + switch (operand.getKind()) { + case LIKE: + return translateNotLike((RexCall) operand, orMapList); + default: + throw new AssertionError("cannot translate NOT " + call); + } + } + + /** Translates NOT LIKE to {$nor: [{field: {$regex: ...}}]}. */ + private Void translateNotLike(RexCall call, List> orMapList) { + final RexNode left = stripCast(call.operands.get(0)); + final RexNode right = call.operands.get(1); + + if (right.getKind() != SqlKind.LITERAL) { + throw new AssertionError("cannot translate NOT LIKE with non-literal pattern: " + call); + } + final RexLiteral patternLiteral = (RexLiteral) right; + final String sqlPattern = patternLiteral.getValue2().toString(); + + final @Nullable Character escapeChar = escapeChar(call); + final String finalRegex = sqlLikeToMongoRegex(sqlPattern, escapeChar); + + final String name; + switch (left.getKind()) { + case INPUT_REF: + final RexInputRef left1 = (RexInputRef) left; + name = fieldNames.get(left1.getIndex()); + break; + case ITEM: + String itemName = MongoRules.isItem((RexCall) left); + if (itemName != null) { + name = itemName; + break; + } + // fall through + default: + throw new AssertionError("cannot translate NOT LIKE " + call); + } + + Map regexMap = builder.map(); + Map regexOp = builder.map(); + regexOp.put("$regex", finalRegex); + regexMap.put(name, regexOp); + List norList = builder.list(); + norList.add(regexMap); + Map norMap = builder.map(); + norMap.put("$nor", norList); + orMapList.add(norMap); + return null; + } + + /** Strips a leading CAST, if any. MongoDB is implicitly typed. */ + private static RexNode stripCast(RexNode node) { + if (node.getKind() == SqlKind.CAST) { + return ((RexCall) node).operands.get(0); + } + return node; + } + + /** Returns the escape character declared in a LIKE expression, or null. */ + private static @Nullable Character escapeChar(RexCall call) { + if (call.operands.size() != 3) { + return null; + } + final RexNode escapeNode = call.operands.get(2); + if (escapeNode.getKind() != SqlKind.LITERAL) { + throw new AssertionError("cannot translate LIKE with non-literal escape: " + call); + } + final String escape = ((RexLiteral) escapeNode).getValue2().toString(); + if (escape.length() != 1) { + throw new AssertionError("cannot translate LIKE with multi-character escape: " + call); + } + return escape.charAt(0); + } + + /** + * Converts SQL LIKE pattern to MongoDB regex pattern. + * + *

      SQL: {@code %} matches zero or more characters, {@code _} matches a single + * character. MongoDB: {@code .*} matches zero or more characters, {@code .} + * matches a single character. + * + *

      We add {@code ^} and {@code $} anchors so that the entire string matches + * the pattern, just as SQL LIKE does. + */ + private static String sqlLikeToMongoRegex(String sqlPattern, @Nullable Character escapeChar) { + final StringBuilder regex = new StringBuilder(sqlPattern.length() * 2); + regex.append("^"); + for (int i = 0; i < sqlPattern.length(); i++) { + char c = sqlPattern.charAt(i); + if (escapeChar != null && c == escapeChar) { + if (i == sqlPattern.length() - 1) { + throw new AssertionError("Invalid escape sequence at end of LIKE pattern: " + + sqlPattern); + } + final char nextChar = sqlPattern.charAt(i + 1); + if (nextChar == '%' || nextChar == '_' || nextChar == escapeChar) { + regex.append(escapeRegexChar(nextChar)); + i++; + } else { + throw new AssertionError("Invalid escape sequence in LIKE pattern: " + sqlPattern); + } + } else if (c == '%') { + regex.append(".*"); + } else if (c == '_') { + regex.append('.'); + } else { + regex.append(escapeRegexChar(c)); + } + } + regex.append("$"); + return regex.toString(); + } + + /** + * Escapes a character for use in a MongoDB regex if it's a special regex character. + */ + private static String escapeRegexChar(char c) { + // MongoDB regex special characters that need escaping + switch (c) { + case '\\': + case '^': + case '$': + case '.': + case '|': + case '?': + case '*': + case '+': + case '(': + case ')': + case '[': + case ']': + case '{': + case '}': + return "\\" + c; + default: + return String.valueOf(c); + } + } } } diff --git a/mongodb/src/test/java/org/apache/calcite/adapter/mongodb/MongoAdapterTest.java b/mongodb/src/test/java/org/apache/calcite/adapter/mongodb/MongoAdapterTest.java index c4fef104f8b5..aa5dc299ca57 100644 --- a/mongodb/src/test/java/org/apache/calcite/adapter/mongodb/MongoAdapterTest.java +++ b/mongodb/src/test/java/org/apache/calcite/adapter/mongodb/MongoAdapterTest.java @@ -1211,4 +1211,168 @@ private static Consumer mongoChecker(final String... expected) { "CITY_SUBSTRING=RA", "CITY_SUBSTRING=UT"); } + + /** Test case for + * [CALCITE-7673] + * MongoDB Adapter can not support LIKE operator. */ + @Test void testLikePrefix() { + // Test LIKE on state field - matches states starting with 'A' + assertModel(MODEL) + .query("select state, city from zips where state like 'A%' order by state") + .returnsUnordered( + "STATE=AK; CITY=ANCHORAGE", + "STATE=AK; CITY=FAIRBANKS", + "STATE=AK; CITY=JUNEAU", + "STATE=AL; CITY=CENTER POINT", + "STATE=AL; CITY=TUSCALOOSA", + "STATE=AL; CITY=SOUTHSIDE", + "STATE=AR; CITY=CONWAY", + "STATE=AR; CITY=GRAVEL RIDGE", + "STATE=AR; CITY=JONESBORO", + "STATE=AZ; CITY=MESA", + "STATE=AZ; CITY=PHOENIX", + "STATE=AZ; CITY=YUMA"); + } + + /** Test case for LIKE operator with suffix pattern (ends with). */ + @Test void testLikeSuffix() { + assertModel(MODEL) + .query("select state, city from zips where city like '%TON' order by state, city") + .limit(5) + .returnsOrdered( + "STATE=DC; CITY=WASHINGTON", + "STATE=KY; CITY=HATTON", + "STATE=MA; CITY=BROCKTON", + "STATE=ME; CITY=LEWISTON", + "STATE=MN; CITY=NEW BRIGHTON"); + } + + /** Test case for LIKE operator with contains pattern. */ + @Test void testLikeContains() { + assertModel(MODEL) + .query("select state, city from zips where city like '%ING%' order by state") + .limit(5) + .returnsOrdered( + "STATE=DC; CITY=WASHINGTON", + "STATE=MA; CITY=FRAMINGHAM", + "STATE=MO; CITY=JENNINGS", + "STATE=MT; CITY=BILLINGS", + "STATE=NC; CITY=LEXINGTON"); + } + + /** Test case for LIKE operator with single character wildcard. */ + @Test void testLikeSingleChar() { + // Pattern 'NEW ______' matches cities starting with 'NEW ' followed by exactly 6 characters + // NEW IBERIA: "NEW " + "IBERIA" (6 chars) = matches + // NEW ORLEANS: "NEW " + "ORLEANS" (7 chars) = does not match + // NEW YORK: "NEW " + "YORK" (4 chars) = does not match + assertModel(MODEL) + .query("select city, state from zips where city like 'NEW ______' order by city") + .returnsOrdered( + "CITY=NEW IBERIA; STATE=LA"); + } + + /** Test case for LIKE operator combined with other filters. */ + @Test void testLikeCombinedWithOtherFilters() { + assertModel(MODEL) + .query("select city, state from zips where city like 'L%' and state = 'CA' order by city") + .returnsOrdered( + "CITY=LOS ANGELES; STATE=CA"); + } + + /** Test case for LIKE operator verifying the generated MongoDB regex. */ + @Test void testLikeGeneratedRegex() { + assertModel(MODEL) + .query("select state from zips where city like 'A%'") + .queryContains( + mongoChecker( + "{$match: {city: {$regex: '^A.*$'}}}", + "{$project: {STATE: '$state'}}")) + .returnsUnordered( + "STATE=AK", + "STATE=IA", + "STATE=SC", + "STATE=SD", + "STATE=TX"); + } + + /** Test case for LIKE operator with escape character on underscore. */ + @Test void testLikeEscapeUnderscore() { + // Without escape, '_' matches a single character. + assertModel(MODEL) + .query("select city from zips where city like 'BROOKLY_'") + .returnsUnordered("CITY=BROOKLYN"); + // With escape, '_' is a literal character and does not match BROOKLYN. + assertModel(MODEL) + .query("select city from zips where city like 'BROOKLY\\_' ESCAPE '\\'") + .returnsUnordered(); + } + + /** Test case for LIKE operator with escape character on percent. */ + @Test void testLikeEscapePercent() { + // Without escape, '%' matches zero or more characters. + assertModel(MODEL) + .query("select city from zips where city like 'BROOKLYN%'") + .returnsUnordered("CITY=BROOKLYN"); + // With escape, '%' is a literal character and does not match BROOKLYN. + assertModel(MODEL) + .query("select city from zips where city like 'BROOKLYN\\%' ESCAPE '\\'") + .returnsUnordered(); + } + + /** Test case for LIKE operator without a default escape character. */ + @Test void testLikeNoDefaultEscape() { + // Without ESCAPE, '\' is an ordinary character; 'A\%' matches cities + // starting with 'A%' and should return nothing. + assertModel(MODEL) + .query("select city from zips where city like 'A\\%'") + .returnsUnordered(); + } + + /** Test case for LIKE operator escaping regex special characters. */ + @Test void testLikeRegexSpecialChar() { + // '.' is an ordinary SQL LIKE character and must be escaped in the + // generated MongoDB regex; otherwise it would match arbitrary characters. + assertModel(MODEL) + .query("select city from zips where city like 'A.B%'") + .returnsUnordered(); + } + + /** Test case for LIKE operator with a custom escape character. */ + @Test void testLikeCustomEscapeChar() { + // Use '!' as the escape character. Here '%' is a wildcard. + assertModel(MODEL) + .query("select city from zips where city like 'BROOKLYN%' ESCAPE '!'") + .returnsUnordered("CITY=BROOKLYN"); + // '!%' makes '%' a literal character, so it does not match BROOKLYN. + assertModel(MODEL) + .query("select city from zips where city like 'BROOKLYN!%' ESCAPE '!'") + .returnsUnordered(); + } + + /** Test case for LIKE operator verifying the generated regex with escapes. */ + @Test void testLikeGeneratedRegexWithEscape() { + assertModel(MODEL) + .query("select state from zips where city like 'A\\_B\\%C%' ESCAPE '\\'") + .queryContains( + mongoChecker( + "{$match: {city: {$regex: '^A_B%C.*$'}}}", + "{$project: {STATE: '$state'}}")) + .returnsUnordered(); + } + + /** Test case for NOT LIKE operator. */ + @Test void testNotLike() { + assertModel(MODEL) + .query("select city from zips where city not like 'A%'") + .returnsCount(144); + } + + /** Test case for LIKE operator on ITEM (_MAP) access. */ + @Test void testLikeItem() { + assertModel(MODEL) + .query("select cast(_MAP['city'] as varchar) from \"mongo_raw\".\"zips\" " + + "where _MAP['city'] like 'A%'") + .returnsCount(5); + } } From 106c03e799f2b29b9fa9f04c555a2a462347ed28 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Fri, 31 Jul 2026 13:43:29 -0700 Subject: [PATCH 429/562] [CALCITE-7682] SESSION table function without the optional key descriptor fails at runtime Signed-off-by: Mihai Budiu --- .../calcite/adapter/enumerable/EnumUtils.java | 12 +++++-- .../adapter/enumerable/RexImpTable.java | 16 ++++++--- core/src/test/resources/sql/stream.iq | 36 +++++++++++++++++++ 3 files changed, 57 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java index 40a824536b15..d1c395c22606 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java @@ -1100,6 +1100,9 @@ static Expression tumblingWindowSelector( * Creates enumerable implementation that applies sessionization to elements from the input * enumerator based on a specified key. Elements are windowed into sessions separated by * periods with no input for at least the duration specified by gap parameter. + * + *

      The key is optional: pass -1 for {@code indexOfKeyColumn} to place every + * element on a single session timeline. */ public static Enumerable<@Nullable Object[]> sessionize( Enumerator<@Nullable Object[]> inputEnumerator, @@ -1144,7 +1147,9 @@ private static class SessionizationEnumerator implements Enumerator<@Nullable Ob * * @param inputEnumerator the enumerator to provide an array of objects as input * @param indexOfWatermarkedColumn the index of timestamp column upon which a watermark is built - * @param indexOfKeyColumn the index of column that acts as grouping key + * @param indexOfKeyColumn the index of column that acts as grouping key, + * or -1 if there is no key and all rows belong to + * a single session timeline * @param gap gap parameter */ SessionizationEnumerator(Enumerator<@Nullable Object[]> inputEnumerator, @@ -1194,8 +1199,11 @@ private void initialize() { Map<@Nullable Object, SortedMultiMap, @Nullable Object[]>> sessionKeyMap = new HashMap<>(); for (@Nullable Object[] element : elements) { + // A key column index of -1 means that there is no key; every element + // then maps to the same (null) key, forming one session timeline. + Object key = indexOfKeyColumn < 0 ? null : element[indexOfKeyColumn]; SortedMultiMap, @Nullable Object[]> session = - sessionKeyMap.computeIfAbsent(element[indexOfKeyColumn], k -> new SortedMultiMap<>()); + sessionKeyMap.computeIfAbsent(key, k -> new SortedMultiMap<>()); Object watermark = requireNonNull(element[indexOfWatermarkedColumn], "element[indexOfWatermarkedColumn]"); diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java index 2ca2528d5c0f..7a5ea10f146f 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java @@ -5113,14 +5113,20 @@ private static Method getMethod(Type comparisonReturnType, SqlKind kind) { private static class SessionImplementor implements TableFunctionCallImplementor { @Override public Expression implement(RexToLixTranslator translator, Expression inputEnumerable, RexCall call, PhysType inputPhysType, PhysType outputPhysType) { - RexCall timestampDescriptor = (RexCall) call.getOperands().get(0); - RexCall keyDescriptor = (RexCall) call.getOperands().get(1); - Expression gapInterval = translator.translate(call.getOperands().get(2)); + final List operands = call.getOperands(); + RexCall timestampDescriptor = (RexCall) operands.get(0); + // The gap is always the last operand; the key descriptor between them is + // optional. Without a key every row belongs to a single session + // timeline, which a key column index of -1 denotes. + Expression gapInterval = translator.translate(Util.last(operands)); + final int keyColIndex = + operands.size() > 2 && operands.get(1).getKind() == SqlKind.DESCRIPTOR + ? ((RexInputRef) ((RexCall) operands.get(1)).getOperands().get(0)).getIndex() + : -1; Expression wmColIndexExpr = Expressions.constant(((RexInputRef) timestampDescriptor.getOperands().get(0)).getIndex()); - Expression keyColIndexExpr = - Expressions.constant(((RexInputRef) keyDescriptor.getOperands().get(0)).getIndex()); + Expression keyColIndexExpr = Expressions.constant(keyColIndex); return Expressions.call(BuiltInMethod.SESSIONIZATION.method, Expressions.list( diff --git a/core/src/test/resources/sql/stream.iq b/core/src/test/resources/sql/stream.iq index 394114376265..b9df0692cc03 100644 --- a/core/src/test/resources/sql/stream.iq +++ b/core/src/test/resources/sql/stream.iq @@ -291,3 +291,39 @@ SELECT * FROM TABLE(SESSION((SELECT * FROM ORDERS), DESCRIPTOR(ROWTIME), DESCRIP (5 rows) !ok + +# Test case for [CALCITE-7682] SESSION table function without the optional key +# descriptor fails at runtime. +SELECT * FROM TABLE(SESSION(TABLE ORDERS, DESCRIPTOR(ROWTIME), INTERVAL '1' HOUR)); ++---------------------+----+---------+-------+---------------------+---------------------+ +| ROWTIME | ID | PRODUCT | UNITS | window_start | window_end | ++---------------------+----+---------+-------+---------------------+---------------------+ +| 2015-02-15 10:15:00 | 1 | paint | 10 | 2015-02-15 10:15:00 | 2015-02-15 12:10:00 | +| 2015-02-15 10:24:15 | 2 | paper | 5 | 2015-02-15 10:15:00 | 2015-02-15 12:10:00 | +| 2015-02-15 10:24:45 | 3 | brush | 12 | 2015-02-15 10:15:00 | 2015-02-15 12:10:00 | +| 2015-02-15 10:58:00 | 4 | paint | 3 | 2015-02-15 10:15:00 | 2015-02-15 12:10:00 | +| 2015-02-15 11:10:00 | 5 | paint | 3 | 2015-02-15 10:15:00 | 2015-02-15 12:10:00 | ++---------------------+----+---------+-------+---------------------+---------------------+ +(5 rows) + +!ok + +# As above, but with named parameters, so the KEY parameter is omitted rather +# than simply absent. +SELECT * FROM TABLE( + SESSION( + DATA => TABLE ORDERS, + TIMECOL => DESCRIPTOR(ROWTIME), + SIZE => INTERVAL '1' HOUR)); ++---------------------+----+---------+-------+---------------------+---------------------+ +| ROWTIME | ID | PRODUCT | UNITS | window_start | window_end | ++---------------------+----+---------+-------+---------------------+---------------------+ +| 2015-02-15 10:15:00 | 1 | paint | 10 | 2015-02-15 10:15:00 | 2015-02-15 12:10:00 | +| 2015-02-15 10:24:15 | 2 | paper | 5 | 2015-02-15 10:15:00 | 2015-02-15 12:10:00 | +| 2015-02-15 10:24:45 | 3 | brush | 12 | 2015-02-15 10:15:00 | 2015-02-15 12:10:00 | +| 2015-02-15 10:58:00 | 4 | paint | 3 | 2015-02-15 10:15:00 | 2015-02-15 12:10:00 | +| 2015-02-15 11:10:00 | 5 | paint | 3 | 2015-02-15 10:15:00 | 2015-02-15 12:10:00 | ++---------------------+----+---------+-------+---------------------+---------------------+ +(5 rows) + +!ok From e195061d61473acec5b5790052520abcd2a8c162 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Sun, 2 Aug 2026 16:41:06 -0700 Subject: [PATCH 430/562] [CALCITE-7678] Runtime equality for ROW values produces incorrect results Signed-off-by: Mihai Budiu --- .../adapter/enumerable/PhysTypeImpl.java | 76 ++- .../adapter/enumerable/RexImpTable.java | 11 + .../org/apache/calcite/plan/RelOptUtil.java | 13 +- .../org/apache/calcite/rex/RexSimplify.java | 5 +- .../apache/calcite/runtime/SqlFunctions.java | 60 ++ .../sql2rel/TopDownGeneralDecorrelator.java | 16 +- .../apache/calcite/util/BuiltInMethod.java | 1 + core/src/test/resources/sql/row-equality.iq | 615 ++++++++++++++++++ .../calcite/linq4j/function/Functions.java | 143 +++- site/_docs/reference.md | 26 + 10 files changed, 957 insertions(+), 9 deletions(-) create mode 100644 core/src/test/resources/sql/row-equality.iq diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/PhysTypeImpl.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/PhysTypeImpl.java index ce7d016a6ca0..2efbb25d22bb 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/PhysTypeImpl.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/PhysTypeImpl.java @@ -585,7 +585,42 @@ private RelDataType toStruct(RelDataType type) { } @Override public @Nullable Expression comparer() { - return format.comparer(); + final Expression comparer = format.comparer(); + if (comparer != null) { + return comparer; + } + if (anyFieldContainsStruct(rowType)) { + // A row or key containing struct (ROW) values needs deep equality; + // the default equality of the runtime representations of struct values (Object[], List) + // compares nested Object[] values by reference. Here we implement the "not + // distinct" semantics used by GROUP BY, DISTINCT and set operations. + return Expressions.call(BuiltInMethod.DEEP_COMPARER.method); + } + return null; + } + + /** Returns whether any field of {@code rowType} contains a struct value: + * the field is itself a struct, or a collection or map whose elements + * contain one. */ + private static boolean anyFieldContainsStruct(RelDataType rowType) { + return rowType.getFieldList().stream() + .anyMatch(f -> containsStruct(f.getType())); + } + + private static boolean containsStruct(RelDataType type) { + if (type.isStruct()) { + return true; + } + final RelDataType componentType = type.getComponentType(); + if (componentType != null && containsStruct(componentType)) { + return true; + } + final RelDataType keyType = type.getKeyType(); + if (keyType != null && containsStruct(keyType)) { + return true; + } + final RelDataType valueType = type.getValueType(); + return valueType != null && containsStruct(valueType); } private List fieldReferences( @@ -764,9 +799,17 @@ private static Expression getListExpression(Expressions.FluentList l Expression exp = getListExpressionAllowSingleElement(list); for (int i = list.size() - 1; i >= 0; i--) { if (nullExclusionFlags.get(i)) { + final RelDataType fieldType = + rowType.getFieldList().get(fields.get(i)).getType(); + // Under the SQL = operator, a NULL never compares TRUE. A + // ROW containing a NULL field (at any nesting depth) cannot + // compare TRUE either: the pairwise comparison of its fields yields + // UNKNOWN or FALSE. In both these cases the result is null. + final Expression isNull = fieldType.isStruct() + ? structIsNullOrContainsNullExpression(list.get(i), fieldType) + : Expressions.equal(list.get(i), Expressions.constant(null)); exp = - Expressions.condition( - Expressions.equal(list.get(i), Expressions.constant(null)), + Expressions.condition(isNull, Expressions.constant(null), exp); } @@ -774,6 +817,33 @@ private static Expression getListExpression(Expressions.FluentList l return Expressions.lambda(Function1.class, exp, v1); } + /** Returns an expression that evaluates whether {@code e}, a value of + * ROW type {@code type}, is null or has a null field, descending into + * struct-typed fields (but not into collection-typed fields). */ + private static Expression structIsNullOrContainsNullExpression(Expression e, + RelDataType type) { + Expression result = Expressions.equal(e, Expressions.constant(null)); + for (Ord field : Ord.zip(type.getFieldList())) { + final RelDataType fieldType = field.e.getType(); + if (!fieldType.isStruct() && !fieldType.isNullable()) { + continue; + } + // structAccess handles both runtime representations of a struct value + // (Object[] and List); it is only evaluated when e is not null, + // thanks to the short-circuit OR. + final Expression access = + Expressions.call(BuiltInMethod.STRUCT_ACCESS.method, e, + Expressions.constant(field.i), + Expressions.constant(field.e.getName())); + result = + Expressions.orElse(result, + fieldType.isStruct() + ? structIsNullOrContainsNullExpression(access, fieldType) + : Expressions.equal(access, Expressions.constant(null))); + } + return result; + } + @Override public Expression fieldReference( Expression expression, int field) { return fieldReference(expression, field, null); diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java index 7a5ea10f146f..1ea24311e345 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java @@ -3404,6 +3404,17 @@ private static class BinaryImplementor extends AbstractRexCallImplementor { final Type type1 = argValueList.get(1).getType(); final SqlBinaryOperator op = (SqlBinaryOperator) call.getOperator(); final RelDataType relDataType0 = call.getOperands().get(0).getType(); + + // Comparing whole ROW values needs three-valued logic: a NULL field + // makes the result UNKNOWN, which a boolean-valued comparison of the + // row representation cannot express. The call type is nullable + // whenever any field is (see SqlTypeUtil.containsNullable). + if (EQUALS_OPERATORS.contains(op) && relDataType0.isStruct()) { + return Expressions.call(SqlFunctions.class, + op.getKind() == SqlKind.EQUALS ? "rowEq" : "rowNe", + argValueList); + } + final Expression fieldComparator = generateCollatorExpression(relDataType0.getCollation()); if (fieldComparator != null) { diff --git a/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java b/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java index b615c31d8533..97146b625aa3 100644 --- a/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java +++ b/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java @@ -2377,7 +2377,7 @@ public static boolean equalType(String desc0, RelNode rel0, String desc1, * Returns a translation of the IS DISTINCT FROM (or IS * NOT DISTINCT FROM) sql operator. * - * @param neg if false, returns a translation of IS NOT DISTINCT FROM + * @param neg if true, returns a translation of IS NOT DISTINCT FROM */ public static RexNode isDistinctFrom( RexBuilder rexBuilder, @@ -2402,14 +2402,21 @@ public static RexNode isDistinctFrom( rexBuilder.makeFieldAccess( y, yField.getIndex()); + // Recurse into a struct field rather than comparing it whole: a + // nested "=" is three-valued, and IS [NOT] DISTINCT FROM must reduce + // to two-valued logic over scalar leaves. RexNode newCall = - isDistinctFromInternal(rexBuilder, newX, newY, neg); + newX.getType().isStruct() + ? isDistinctFrom(rexBuilder, newX, newY, neg) + : isDistinctFromInternal(rexBuilder, newX, newY, neg); if (ret == null) { ret = newCall; } else { + // Two rows are not distinct only when every field pair is not + // distinct, but they are distinct as soon as one pair is. ret = rexBuilder.makeCall( - SqlStdOperatorTable.AND, + neg ? SqlStdOperatorTable.AND : SqlStdOperatorTable.OR, ret, newCall); } diff --git a/core/src/main/java/org/apache/calcite/rex/RexSimplify.java b/core/src/main/java/org/apache/calcite/rex/RexSimplify.java index b275c8b6d97b..386964963ceb 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexSimplify.java +++ b/core/src/main/java/org/apache/calcite/rex/RexSimplify.java @@ -673,7 +673,10 @@ private > RexNode simplifyComparison(RexCall e, // Simplify "x x" final RexNode o0 = operands.get(0); final RexNode o1 = operands.get(1); - if (o0.equals(o1) && RexUtil.isDeterministic(o0)) { + // "x = x" does not hold for a ROW with a nullable field, which evaluates to UNKNOWN + final boolean nullableStruct = + o0.getType().isStruct() && SqlTypeUtil.containsNullable(o0.getType()); + if (o0.equals(o1) && RexUtil.isDeterministic(o0) && !nullableStruct) { RexNode newExpr; switch (e.getKind()) { case EQUALS: diff --git a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java index 4b9b48041fad..15d526393b33 100644 --- a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java +++ b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java @@ -2229,6 +2229,66 @@ public static boolean eq(Object b0, Object b1) { return b0.equals(b1); } + /** SQL = operator applied to ROW values, with the standard's + * three-valued row comparison: FALSE as soon as one field pair is unequal, + * UNKNOWN (null) when a field pair involves a NULL and no pair is unequal, + * TRUE otherwise. + * + *

      A nested ROW, represented as {@code Object[]}, follows the same rule. + * A collection-valued field is compared as a whole, because e.g., ARRAY equality + * uses IS NOT DISTINCT FROM semantics and never yields UNKNOWN. */ + public static @Nullable Boolean rowEq(@Nullable Object b0, @Nullable Object b1) { + if (b0 == null || b1 == null) { + return null; + } + final List l0 = rowAsList(b0); + final List l1 = rowAsList(b1); + if (l0 == null || l1 == null) { + // Not a representation we can take apart; fall back to total equality. + return Functions.compareListItems(b0, b1) == 0; + } + if (l0.size() != l1.size()) { + return false; + } + boolean sawNull = false; + for (int i = 0; i < l0.size(); i++) { + final Object f0 = l0.get(i); + final Object f1 = l1.get(i); + if (f0 == null || f1 == null) { + sawNull = true; + } else if (f0 instanceof Object[] && f1 instanceof Object[]) { + final Boolean nested = rowEq(f0, f1); + if (nested == null) { + sawNull = true; + } else if (!nested) { + return false; + } + } else if (Functions.compareListItems(f0, f1) != 0) { + return false; + } + } + return sawNull ? null : true; + } + + /** SQL <> operator applied to ROW values; the + * three-valued negation of {@link #rowEq}. */ + public static @Nullable Boolean rowNe(@Nullable Object b0, @Nullable Object b1) { + final Boolean eq = rowEq(b0, b1); + return eq == null ? null : !eq; + } + + /** Views a ROW value as the list of its fields; returns null if the value is + * not one of the representations a ROW may have. */ + private static @Nullable List rowAsList(Object o) { + if (o instanceof Object[]) { + return Arrays.asList((Object[]) o); + } + if (o instanceof List) { + return (List) o; + } + return null; + } + /** SQL = operator applied to List values. */ public static boolean eq(List b0, List b1) { return eqNullable(b0, b1); diff --git a/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java b/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java index 4139cf4b2b99..ead86414f228 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java @@ -892,7 +892,7 @@ private boolean tryReplaceFreeVarsToInputRef( if (pair != null) { // equi-condition will filter NULL values, so need to add IS NOT NULL for input ref if (condition.isA(SqlKind.EQUALS)) { - newConditions.add(builder.isNotNull(pair.right)); + newConditions.add(isNotNullDeep(pair.right)); } corDefToInputIndex.put(pair.left, pair.right.getIndex()); continue; @@ -909,6 +909,20 @@ private boolean tryReplaceFreeVarsToInputRef( return replacedCorDef.size() == corDefs.size() && corDefs.containsAll(replacedCorDef); } + /** Returns a condition that holds when an expression {@code ref} that may have a ROW type + * contains no 'NULL' field at any depth. */ + private RexNode isNotNullDeep(RexNode ref) { + if (!ref.getType().isStruct()) { + return builder.isNotNull(ref); + } + final List conditions = new ArrayList<>(); + conditions.add(builder.isNotNull(ref)); + for (int i = 0; i < ref.getType().getFieldCount(); i++) { + conditions.add(isNotNullDeep(builder.getRexBuilder().makeFieldAccess(ref, i))); + } + return RexUtil.composeConjunction(builder.getRexBuilder(), conditions); + } + private @Nullable Pair getPairOfFreeVarAndInputRefInEqui(RexNode condition) { if (!condition.isA(SqlKind.EQUALS) && !condition.isA(SqlKind.IS_NOT_DISTINCT_FROM)) { return null; diff --git a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java index 629357b4e07e..6b4e52fdf968 100644 --- a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java +++ b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java @@ -321,6 +321,7 @@ public enum BuiltInMethod { NULLS_COMPARATOR2(Functions.class, "nullsComparator", boolean.class, boolean.class, Comparator.class), ARRAY_COMPARER(Functions.class, "arrayComparer"), + DEEP_COMPARER(Functions.class, "deepComparer"), FUNCTION0_APPLY(Function0.class, "apply"), FUNCTION1_APPLY(Function1.class, "apply", Object.class), ARRAYS_AS_LIST(Arrays.class, "asList", Object[].class), diff --git a/core/src/test/resources/sql/row-equality.iq b/core/src/test/resources/sql/row-equality.iq new file mode 100644 index 000000000000..a61e38704a22 --- /dev/null +++ b/core/src/test/resources/sql/row-equality.iq @@ -0,0 +1,615 @@ +# row-equality.iq - Tests for equality of ROW values at runtime +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +!use scott +!set outputformat mysql + +# Test cases for https://issues.apache.org/jira/browse/CALCITE-7678 +# [CALCITE-7678] Runtime equality for ROW values produces incorrect results +# +# Calcite has one ROW type, compared everywhere with the standard's row +# semantics: "(a, b)" and "ROW(a, b)" have the same representation, and a +# CREATE TYPE structured type uses the same RelRecordType as a ROW +# expression. +# +# Each query below carries a comment recording its status on PostgreSQL 14. +# Note that Postgres does NOT implement the standard SQL semantics for nested ROW comparisons. + +##################################################################### +# GROUP BY + +# GROUP BY a flat ROW value. +# Validated on PostgreSQL 14: same result. +SELECT r, COUNT(*) AS c +FROM (SELECT ROW(x, y) AS r + FROM (VALUES (1, 'a'), (1, 'a'), (2, 'b')) AS v(x, y)) AS t +GROUP BY r +ORDER BY c; ++--------+---+ +| R | C | ++--------+---+ +| {2, b} | 1 | +| {1, a} | 2 | ++--------+---+ +(2 rows) + +!ok + +# GROUP BY a ROW value with a NULL field: grouping uses not-distinct +# semantics, so the two ROW(2, NULL) values belong to the same group. +# Validated on PostgreSQL 14: same result. +SELECT r, COUNT(*) AS c +FROM (SELECT ROW(x, y) AS r + FROM (VALUES (1, 'a'), (2, NULL), (2, NULL)) AS v(x, y)) AS t +GROUP BY r +ORDER BY c; ++-----------+---+ +| R | C | ++-----------+---+ +| {1, a} | 1 | +| {2, null} | 2 | ++-----------+---+ +(2 rows) + +!ok + +# GROUP BY a nested ROW value. +# Validated on PostgreSQL 14: same result. +SELECT r, COUNT(*) AS c +FROM (SELECT ROW(ROW(x, y), z) AS r + FROM (VALUES (1, 'a', 10), (1, 'a', 10), (2, 'b', 20)) AS v(x, y, z)) AS t +GROUP BY r +ORDER BY c; ++--------------+---+ +| R | C | ++--------------+---+ +| {{2, b}, 20} | 1 | +| {{1, a}, 10} | 2 | ++--------------+---+ +(2 rows) + +!ok + +##################################################################### +# DISTINCT + +# SELECT DISTINCT over ROW values, including ones with a NULL field. +# Validated on PostgreSQL 14: same result. +SELECT DISTINCT r +FROM (SELECT ROW(x, y) AS r + FROM (VALUES (1, 'a'), (1, 'a'), (2, NULL), (2, NULL)) AS v(x, y)) AS t +ORDER BY r; ++-----------+ +| R | ++-----------+ +| {1, a} | +| {2, null} | ++-----------+ +(2 rows) + +!ok + +##################################################################### +# Set operations + +# UNION removes duplicate ROW values, including ones with a NULL field. +# Validated on PostgreSQL 14: same result. +SELECT ROW(x, y) AS r +FROM (VALUES (1, 'a'), (2, NULL)) AS v(x, y) +UNION +SELECT ROW(x, y) AS r +FROM (VALUES (2, NULL)) AS w(x, y) +ORDER BY r; ++-----------+ +| R | ++-----------+ +| {1, a} | +| {2, null} | ++-----------+ +(2 rows) + +!ok + +# INTERSECT over ROW values. +# Validated on PostgreSQL 14: same result. +SELECT ROW(x, y) AS r +FROM (VALUES (1, 'a'), (2, 'b')) AS v(x, y) +INTERSECT +SELECT ROW(x, y) AS r +FROM (VALUES (2, 'b'), (3, 'c')) AS w(x, y); ++--------+ +| R | ++--------+ +| {2, b} | ++--------+ +(1 row) + +!ok + +# EXCEPT over ROW values. +# Validated on PostgreSQL 14: same result. +SELECT ROW(x, y) AS r +FROM (VALUES (1, 'a'), (2, 'b')) AS v(x, y) +EXCEPT +SELECT ROW(x, y) AS r +FROM (VALUES (2, 'b'), (3, 'c')) AS w(x, y); ++--------+ +| R | ++--------+ +| {1, a} | ++--------+ +(1 row) + +!ok + +##################################################################### +# JOIN on ROW values, strict equality + +# A flat ROW value equals itself and nothing else. +# Validated on PostgreSQL 14: same result. +WITH t(x, r) AS ( + SELECT x, ROW(x, y) + FROM (VALUES (1, 'a'), (2, 'b')) AS v(x, y)) +SELECT t1.x AS x1, t2.x AS x2 +FROM t AS t1 +JOIN t AS t2 ON t1.r = t2.r +ORDER BY x1; ++----+----+ +| X1 | X2 | ++----+----+ +| 1 | 1 | +| 2 | 2 | ++----+----+ +(2 rows) + +!ok + +# A nested ROW value equals itself and nothing else. +# Validated on PostgreSQL 14: same result. +WITH t(x, r) AS ( + SELECT x, ROW(ROW(x, y), z) + FROM (VALUES (1, 'a', 10), (2, 'b', 20)) AS v(x, y, z)) +SELECT t1.x AS x1, t2.x AS x2 +FROM t AS t1 +JOIN t AS t2 ON t1.r = t2.r +ORDER BY x1; ++----+----+ +| X1 | X2 | ++----+----+ +| 1 | 1 | +| 2 | 2 | ++----+----+ +(2 rows) + +!ok + +# A NULL field makes the strict comparison UNKNOWN, so ROW(2, NULL) does +# not join with itself. Postgres returns 2 rows instead of one, because +# in Postgres (1, 'a') is not the same as ROW(1, 'a'). +WITH t(x, r) AS ( + SELECT x, ROW(x, y) + FROM (VALUES (1, 'a'), (2, NULL)) AS v(x, y)) +SELECT t1.x AS x1, t2.x AS x2 +FROM t AS t1 +JOIN t AS t2 ON t1.r = t2.r +ORDER BY x1; ++----+----+ +| X1 | X2 | ++----+----+ +| 1 | 1 | ++----+----+ +(1 row) + +!ok + +# Same problem as above when validated on Postgres. +WITH t(x, r) AS ( + SELECT x, ROW(ROW(x, y), z) + FROM (VALUES (1, 'a', 10), (2, NULL, 20)) AS v(x, y, z)) +SELECT t1.x AS x1, t2.x AS x2 +FROM t AS t1 +JOIN t AS t2 ON t1.r = t2.r +ORDER BY x1; ++----+----+ +| X1 | X2 | ++----+----+ +| 1 | 1 | ++----+----+ +(1 row) + +!ok + +##################################################################### +# JOIN on ROW values using IS NOT DISTINCT FROM + +# Under not-distinct semantics the ROW(2, NULL) pair matches. +# Validated on PostgreSQL 14: same result. +WITH t(x, r) AS ( + SELECT x, ROW(x, y) + FROM (VALUES (1, 'a'), (2, NULL)) AS v(x, y)) +SELECT t1.x AS x1, t2.x AS x2 +FROM t AS t1 +JOIN t AS t2 ON t1.r IS NOT DISTINCT FROM t2.r +ORDER BY x1; ++----+----+ +| X1 | X2 | ++----+----+ +| 1 | 1 | +| 2 | 2 | ++----+----+ +(2 rows) + +!ok + +# The same, for a NULL in the inner ROW. +# Validated on PostgreSQL 14: same result. +WITH t(x, r) AS ( + SELECT x, ROW(ROW(x, y), z) + FROM (VALUES (1, 'a', 10), (2, NULL, 20)) AS v(x, y, z)) +SELECT t1.x AS x1, t2.x AS x2 +FROM t AS t1 +JOIN t AS t2 ON t1.r IS NOT DISTINCT FROM t2.r +ORDER BY x1; ++----+----+ +| X1 | X2 | ++----+----+ +| 1 | 1 | +| 2 | 2 | ++----+----+ +(2 rows) + +!ok + +##################################################################### +# Comparison in predicate position + +# Row comparison is three-valued. Row 2 is UNKNOWN because the only +# difference is a NULL field pair; row 3 is FALSE because the first fields +# already differ, which outranks the NULL pair. IS DISTINCT FROM stays +# two-valued throughout. +# +# Postgres answers TRUE for row 2, because tuples are not ROW values in Postgres. +WITH t(id, a, b) AS ( + SELECT id, ROW(x, y), ROW(z, w) + FROM (VALUES (1, 1, 'a', 1, 'a'), + (2, 1, NULL, 1, NULL), + (3, 1, NULL, 2, NULL), + (4, 1, 'a', 2, 'b')) AS v(id, x, y, z, w)) +SELECT id, a = b AS eq, a <> b AS ne, a IS DISTINCT FROM b AS dist +FROM t +ORDER BY id; ++----+-------+-------+-------+ +| ID | EQ | NE | DIST | ++----+-------+-------+-------+ +| 1 | true | false | false | +| 2 | | | false | +| 3 | false | true | true | +| 4 | false | true | true | ++----+-------+-------+-------+ +(4 rows) + +!ok + +# A NULL nested inside an inner ROW makes the comparison UNKNOWN too. +# Postgres answers TRUE for row 2. +WITH t(id, a, b) AS ( + SELECT id, ROW(ROW(x, y), x), ROW(ROW(z, w), z) + FROM (VALUES (1, 1, 'a', 1, 'a'), + (2, 1, NULL, 1, NULL)) AS v(id, x, y, z, w)) +SELECT id, a = b AS eq +FROM t +ORDER BY id; ++----+------+ +| ID | EQ | ++----+------+ +| 1 | true | +| 2 | | ++----+------+ +(2 rows) + +!ok + +# Constant folding must reach the same answer as the runtime. +# In PostgreSQL 14: the deeply nested ROWs evaluate to TRUE there. +SELECT ROW(1, CAST(NULL AS VARCHAR)) + = ROW(1, CAST(NULL AS VARCHAR)) AS flat, + ROW(ROW(1, CAST(NULL AS VARCHAR)), 2) + = ROW(ROW(1, CAST(NULL AS VARCHAR)), 2) AS nested2, + ROW(ROW(ROW(1, CAST(NULL AS VARCHAR)), 2), 3) + = ROW(ROW(ROW(1, CAST(NULL AS VARCHAR)), 2), 3) AS nested3; ++------+---------+---------+ +| FLAT | NESTED2 | NESTED3 | ++------+---------+---------+ +| | | | ++------+---------+---------+ +(1 row) + +!ok + +# With no nullable field anywhere, "x = x" still folds to TRUE. +# Validated on PostgreSQL 14: same result. +SELECT ROW(ROW(1, CAST('a' AS VARCHAR)), 2) + = ROW(ROW(1, CAST('a' AS VARCHAR)), 2) AS all_non_null; ++--------------+ +| ALL_NON_NULL | ++--------------+ +| true | ++--------------+ +(1 row) + +!ok + +# Three levels of nesting, with the NULL at the innermost level. The +# UNKNOWN has to propagate all the way out for =, while IS [NOT] DISTINCT +# FROM stays two-valued at every level. +WITH t(id, a, b) AS ( + SELECT id, ROW(ROW(ROW(x, y), x), x), ROW(ROW(ROW(z, w), z), z) + FROM (VALUES (1, 1, 'a', 1, 'a'), + (2, 1, NULL, 1, NULL), + (3, 1, NULL, 2, NULL)) AS v(id, x, y, z, w)) +SELECT id, a = b AS eq, a IS DISTINCT FROM b AS dist, + a IS NOT DISTINCT FROM b AS ndf +FROM t +ORDER BY id; ++----+-------+-------+-------+ +| ID | EQ | DIST | NDF | ++----+-------+-------+-------+ +| 1 | true | false | true | +| 2 | | false | true | +| 3 | false | true | false | ++----+-------+-------+-------+ +(3 rows) + +!ok + +##################################################################### +# Nested ROW values produced by a query + +WITH t(id, r) AS ( + SELECT id, ROW(ROW(x, y), x) + FROM (VALUES (1, 1, 'a'), (2, 1, NULL)) AS v(id, x, y)) +SELECT t1.id, t1.r = t2.r AS eq, t1.r IS DISTINCT FROM t2.r AS dist +FROM t AS t1 +JOIN t AS t2 ON t1.id = t2.id +ORDER BY t1.id; ++----+------+-------+ +| ID | EQ | DIST | ++----+------+-------+ +| 1 | true | false | +| 2 | | false | ++----+------+-------+ +(2 rows) + +!ok + +# Grouped after a UNION ALL: the four rows collapse to two values. +# Validated on PostgreSQL 14: same result. +SELECT COUNT(*) AS n FROM ( + SELECT r FROM ( + SELECT ROW(ROW(x, y), x) AS r + FROM (VALUES (1, 'a'), (2, NULL)) AS v(x, y) + UNION ALL + SELECT ROW(ROW(x, y), x) AS r + FROM (VALUES (1, 'a'), (2, NULL)) AS w(x, y)) AS u + GROUP BY r) AS g; ++---+ +| N | ++---+ +| 2 | ++---+ +(1 row) + +!ok + +# The NULL field is produced by an aggregate rather than written literally. +# In PostgreSQL 14 row 1 is TRUE. +SELECT g, ROW(ROW(g, MIN(v)), g) = ROW(ROW(g, MIN(v)), g) AS eq +FROM (VALUES (1, CAST(NULL AS VARCHAR)), (2, 'x')) AS t(g, v) +GROUP BY g +ORDER BY g; ++---+------+ +| G | EQ | ++---+------+ +| 1 | | +| 2 | true | ++---+------+ +(2 rows) + +!ok + +# PostgreSQL 14 differs: TRUE for row 2. +WITH t(id, r) AS ( + SELECT id, ROW(ROW(x, y), x) + FROM (VALUES (1, 1, 'a'), (2, 1, NULL)) AS v(id, x, y)) +SELECT id, + EXISTS (SELECT 1 FROM t AS u WHERE u.r = t.r) AS ex, + EXISTS (SELECT 1 FROM t AS u WHERE u.r IS NOT DISTINCT FROM t.r) AS ex_ndf +FROM t +ORDER BY id; ++----+-------+--------+ +| ID | EX | EX_NDF | ++----+-------+--------+ +| 1 | true | true | +| 2 | false | true | ++----+-------+--------+ +(2 rows) + +!ok + +##################################################################### +# IN predicates +# +# IN and NOT IN are defined in terms of =, so over ROW values they inherits +# three-valued comparisons. + +# Both operands are row constructors. +# Validated on PostgreSQL 14: same result. +SELECT ROW(1, CAST(NULL AS VARCHAR)) + IN (ROW(1, CAST(NULL AS VARCHAR))) AS in_null, + ROW(1, CAST(NULL AS VARCHAR)) + NOT IN (ROW(1, CAST(NULL AS VARCHAR))) AS notin_null, + ROW(1, 'a') IN (ROW(1, 'a'), ROW(2, 'b')) AS in_true, + ROW(1, CAST(NULL AS VARCHAR)) IN (ROW(2, 'b')) AS in_false; ++---------+------------+---------+----------+ +| IN_NULL | NOTIN_NULL | IN_TRUE | IN_FALSE | ++---------+------------+---------+----------+ +| | | true | false | ++---------+------------+---------+----------+ +(1 row) + +!ok + +# Struct-typed columns rather than constructors. +# In PostgreSQL 14 row 2 is TRUE/FALSE. +WITH t(id, a, b) AS ( + SELECT id, ROW(x, y), ROW(z, w) + FROM (VALUES (1, 1, 'a', 1, 'a'), + (2, 1, NULL, 1, NULL), + (3, 1, NULL, 2, NULL)) AS v(id, x, y, z, w)) +SELECT id, a IN (b) AS in_b, a NOT IN (b) AS notin_b +FROM t +ORDER BY id; ++----+-------+---------+ +| ID | IN_B | NOTIN_B | ++----+-------+---------+ +| 1 | true | false | +| 2 | | | +| 3 | false | true | ++----+-------+---------+ +(3 rows) + +!ok + +# Row-valued IN against a sub-query. +# Validated on PostgreSQL 14: same result. +SELECT (1, CAST(NULL AS VARCHAR)) + IN (SELECT 1, CAST(NULL AS VARCHAR)) AS in_subq; ++---------+ +| IN_SUBQ | ++---------+ +| | ++---------+ +(1 row) + +!ok + +##################################################################### +# ROW values inside a collection +# +# A ROW nested in a collection is compared by a different rule +# (IS NOT DISTINCT FROM) than a +# top-level ROW. Array equality is total: it compares element-wise and +# never yields UNKNOWN, so the NULL fields of the nested ROW values are +# treated as equal. ROW(NULL) is not equal with ROW(NULL), but +# ARRAY[ROW(NULL)] is equal to ARRAY[ROW(NULL)]. +# +# Postgres agrees with Calcite on every query in this section, because +# both compare collections totally. + +# Validated on PostgreSQL 14: same result. +SELECT ROW(1, CAST(NULL AS VARCHAR)) + = ROW(1, CAST(NULL AS VARCHAR)) AS bare, + ARRAY[ROW(1, CAST(NULL AS VARCHAR))] + = ARRAY[ROW(1, CAST(NULL AS VARCHAR))] AS in_array; ++------+----------+ +| BARE | IN_ARRAY | ++------+----------+ +| | true | ++------+----------+ +(1 row) + +!ok + +# Validated on PostgreSQL 14: same result. +SELECT ARRAY[ROW(1, CAST(NULL AS VARCHAR))] + = ARRAY[ROW(2, CAST(NULL AS VARCHAR))] AS differ; ++--------+ +| DIFFER | ++--------+ +| false | ++--------+ +(1 row) + +!ok + +# Validated on PostgreSQL 14: same result. +WITH t(id, a, b) AS ( + SELECT id, ARRAY[ROW(x, y)], ARRAY[ROW(z, w)] + FROM (VALUES (1, 1, 'a', 1, 'a'), + (2, 1, NULL, 1, NULL), + (3, 1, NULL, 2, NULL)) AS v(id, x, y, z, w)) +SELECT id, a = b AS eq, a IS DISTINCT FROM b AS dist +FROM t +ORDER BY id; ++----+-------+-------+ +| ID | EQ | DIST | ++----+-------+-------+ +| 1 | true | false | +| 2 | true | false | +| 3 | false | true | ++----+-------+-------+ +(3 rows) + +!ok + +# Validated on PostgreSQL 14: same result. +SELECT ROW(1, ARRAY[ROW(1, CAST(NULL AS VARCHAR))]) + = ROW(1, ARRAY[ROW(1, CAST(NULL AS VARCHAR))]) AS row_of_array; ++--------------+ +| ROW_OF_ARRAY | ++--------------+ +| true | ++--------------+ +(1 row) + +!ok + +# Validated on PostgreSQL 14: same result. +SELECT a, COUNT(*) AS c +FROM (SELECT ARRAY[ROW(x, y)] AS a + FROM (VALUES (1, 'a'), (2, NULL), (2, NULL)) AS v(x, y)) AS t +GROUP BY a +ORDER BY c; ++-------------+---+ +| A | C | ++-------------+---+ +| [{1, a}] | 1 | +| [{2, null}] | 2 | ++-------------+---+ +(2 rows) + +!ok + +# A MAP holding ROW values also uses IS NOT DISTINCT FROM +# Postgres has no MAP type. +SELECT c +FROM (SELECT COUNT(*) AS c + FROM (SELECT MAP['k', ROW(x, y)] AS m + FROM (VALUES (1, 'a'), (2, NULL), (2, NULL)) AS v(x, y)) AS t + GROUP BY m) AS g +ORDER BY c; ++---+ +| C | ++---+ +| 1 | +| 2 | ++---+ +(2 rows) + +!ok + +# End row-equality.iq diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/function/Functions.java b/linq4j/src/main/java/org/apache/calcite/linq4j/function/Functions.java index 6c0a41297b3c..f61f65a6d229 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/function/Functions.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/function/Functions.java @@ -81,6 +81,9 @@ private Functions() {} private static final EqualityComparer<@Nullable Object[]> ARRAY_COMPARER = new ArrayEqualityComparer(); + private static final EqualityComparer<@Nullable Object> DEEP_COMPARER = + new DeepEqualityComparer(); + private static final Function1 CONSTANT_NULL_FUNCTION1 = (Function1) s -> null; @@ -488,6 +491,20 @@ public static EqualityComparer selectorComparer( return new SelectorEqualityComparer<>(selector); } + /** + * Returns an {@link EqualityComparer} that compares values deeply: + * {@code Object[]} arrays and {@link List}s are compared element-wise and + * recursively, and compare equal to each other when their elements are + * equal, regardless of container kind; primitive arrays are compared by + * content; {@code null} equals {@code null}. + * + *

      This implements the SQL "not distinct" semantics used by + * {@code GROUP BY}, {@code DISTINCT} and set operations. */ + @SuppressWarnings("unchecked") + public static EqualityComparer deepComparer() { + return (EqualityComparer) DEEP_COMPARER; + } + /** Array equality comparer. */ private static class ArrayEqualityComparer implements EqualityComparer<@Nullable Object[]> { @@ -500,6 +517,127 @@ private static class ArrayEqualityComparer } } + /** Deep equality comparer; see {@link #deepComparer()}. */ + private static class DeepEqualityComparer + implements EqualityComparer<@Nullable Object> { + @Override public boolean equal(@Nullable Object v1, @Nullable Object v2) { + return deepEquals(v1, v2); + } + + @Override public int hashCode(@Nullable Object t) { + return deepHashCode(t); + } + + private static boolean deepEquals(@Nullable Object v1, @Nullable Object v2) { + if (v1 == v2) { + return true; + } + if (v1 == null || v2 == null) { + return false; + } + // Normalize both to List: an ARRAY of ROW is a List of Object[], + // and each element is normalized in turn. + final @Nullable List list1 = asListOrNull(v1); + final @Nullable List list2 = asListOrNull(v2); + if (list1 != null && list2 != null) { + final int n = list1.size(); + if (n != list2.size()) { + return false; + } + for (int i = 0; i < n; i++) { + if (!deepEquals(list1.get(i), list2.get(i))) { + return false; + } + } + return true; + } + if (list1 != null || list2 != null) { + return false; + } + if (v1 instanceof Map && v2 instanceof Map) { + return mapDeepEquals((Map) v1, (Map) v2); + } + if (v1.getClass().isArray() && v2.getClass().isArray()) { + // Primitive arrays (e.g. byte[] for BINARY values). + return Arrays.deepEquals(new Object[] {v1}, new Object[] {v2}); + } + return v1.equals(v2); + } + + /** Compares two maps as unordered sets of entries, comparing keys and + * values deeply. + * + *

      Java {@link Map#equals} is already order-independent, but it looks a + * key up by that key's own hashCode and equals, which matches a struct key + * only by reference; hence the scan. */ + private static boolean mapDeepEquals(Map m1, Map m2) { + if (m1.size() != m2.size()) { + return false; + } + // Remove on match, so that keys that are deep-equal but distinct to + // Java, as two Object[] with the same contents are, pair up one to one. + final List> unmatched = new ArrayList<>(m2.entrySet()); + for (Map.Entry e1 : m1.entrySet()) { + boolean found = false; + for (int i = 0; i < unmatched.size(); i++) { + final Map.Entry e2 = unmatched.get(i); + if (deepEquals(e1.getKey(), e2.getKey()) + && deepEquals(e1.getValue(), e2.getValue())) { + unmatched.remove(i); + found = true; + break; + } + } + if (!found) { + return false; + } + } + return true; + } + + /** Computes a hash code that is equal for values that + * {@link #deepEquals} considers equal; in particular, an + * {@code Object[]} and a {@link List} with equal elements hash alike. */ + private static int deepHashCode(@Nullable Object o) { + if (o == null) { + return 0x789d; + } + final @Nullable List list = asListOrNull(o); + if (list != null) { + int h = 1; + for (Object element : list) { + h = 31 * h + deepHashCode(element); + } + return h; + } + if (o instanceof Map) { + // Sum of per-entry hashes, as Map.hashCode does, so that the hash + // ignores entry order just as mapDeepEquals does. + int h = 0; + for (Map.Entry e : ((Map) o).entrySet()) { + h += deepHashCode(e.getKey()) ^ deepHashCode(e.getValue()); + } + return h; + } + if (o.getClass().isArray()) { + return Arrays.deepHashCode(new Object[] {o}); + } + return o.hashCode(); + } + + /** Views {@code o} as a list if it is a {@code List} or an + * {@code Object[]}; returns null otherwise. */ + private static @Nullable List asListOrNull(Object o) { + if (o instanceof List) { + return (List) o; + } + if (o instanceof Object[]) { + return Arrays.asList((Object[]) o); + } + return null; + } + } + /** Identity equality comparer. */ private static class IdentityEqualityComparer implements EqualityComparer { @@ -636,7 +774,10 @@ private static BigDecimal toBigDecimal(Number number) { : new BigDecimal(number.doubleValue()); } - private static int compareListItems(@Nullable Object item0, @Nullable Object item1) { + /** Compares two values as elements of a list, array or row: nested + * collections and arrays are compared element-wise, numbers are compared by + * value regardless of their Java type, and nulls sort last. */ + public static int compareListItems(@Nullable Object item0, @Nullable Object item1) { if (item0 == item1) { return 0; } diff --git a/site/_docs/reference.md b/site/_docs/reference.md index 56fbf8a86ca4..4b198449ae08 100644 --- a/site/_docs/reference.md +++ b/site/_docs/reference.md @@ -1483,6 +1483,32 @@ comp: | <=> {% endhighlight %} +Note: + +* Comparing two `ROW` values with `=` or `<>` compares their fields pairwise, + using three-valued logic: the result is FALSE if some pair of fields is + unequal, UNKNOWN if some pair involves a null and no pair is unequal, and + TRUE otherwise. For example, `ROW(1, NULL) = ROW(1, NULL)` is UNKNOWN, but + `ROW(1, NULL) = ROW(2, NULL)` is FALSE. +* `IS DISTINCT FROM` and `IS NOT DISTINCT FROM` treat nulls as equal, so on + `ROW` values they always return TRUE or FALSE. Two rows are distinct if some + pair of their fields is distinct. +* `JOIN ON ROW(a, b) = ROW(c, d)` uses this definition of row equality. This is + equivalent to expanding equality for rows to their corresponding fields recursively: + `JOIN a = c AND b = d`. +* `IN` and `NOT IN` are defined in terms of `=`, and over `ROW` values they + inherit the same three-valued result. + `ROW(1, NULL) IN (ROW(1, NULL))` and the corresponding `NOT IN` expression + evaluate to UNKNOWN. The quantified comparisons + `SOME`, `ANY` and `ALL` currently do not accept `ROW` operands. +* Comparing two collection values (`ARRAY`, `MULTISET`, `MAP`) treats NULL + elements as equal, so the result is never UNKNOWN. A `ROW` nested in a + collection is therefore compared the way `IS NOT DISTINCT FROM` compares it, + and a NULL inside a collection does *not* make a comparison of the enclosing + `ROW` value UNKNOWN. +* `GROUP BY`, `DISTINCT` and the set operators (`UNION`, `INTERSECT`, `EXCEPT`) + compare values as `IS NOT DISTINCT FROM` does. + ### Logical operators | Operator syntax | Description From c07a980c994a866165fe622d476dc5b0bdab46c1 Mon Sep 17 00:00:00 2001 From: Dongsheng He Date: Tue, 28 Jul 2026 23:01:42 +0800 Subject: [PATCH 431/562] [CALCITE-7679] RelToSqlConverter generates GROUP BY literals for dialects that do not support them when the constant is hidden by nested Projects --- .../calcite/rel/rel2sql/SqlImplementor.java | 33 ++++++++++++ .../rel/rel2sql/RelToSqlConverterTest.java | 50 +++++++++++++++++++ 2 files changed, 83 insertions(+) diff --git a/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java b/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java index b65a797f14fe..c2804db09ee3 100644 --- a/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java +++ b/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java @@ -2261,8 +2261,41 @@ && hasSortByOrdinal(node)) { if (groupKeysContainOver(agg)) { return true; } + + if (!dialect.supportsGroupByLiteral() + && hasGroupByLiteral(agg)) { + return true; + } + } + + return false; + } + + /** + * Returns whether any grouping key of {@code aggregate} is represented by + * a literal expression in this result's {@code SELECT} list. + */ + private boolean hasGroupByLiteral( + @UnknownInitialization Result this, Aggregate aggregate) { + if (!(node instanceof SqlSelect)) { + return false; } + final SqlNodeList selectList = ((SqlSelect) node).getSelectList(); + if (selectList.equals(SqlNodeList.SINGLETON_STAR)) { + return false; + } + + for (int groupKey : aggregate.getGroupSet()) { + if (groupKey >= selectList.size()) { + return false; + } + final SqlNode expression = SqlUtil.stripAs(selectList.get(groupKey)); + // A literal wrapped in a CAST is also considered a literal. + if (SqlUtil.isLiteral(expression, true)) { + return true; + } + } return false; } diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java index 01f157893623..ad43cd05912c 100644 --- a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java +++ b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java @@ -370,6 +370,56 @@ private static String toSql(RelNode root, SqlDialect dialect, .withInformix().ok(expectedInformix); } + /** Test case for + * [CALCITE-7679] + * RelToSqlConverter generates GROUP BY literals for dialects that do not + * support them when the constant is hidden by nested Projects. */ + @Test void testGroupByLiteralWithNestedProjects() { + final String query = "SELECT \"id\"\n" + + "FROM (\n" + + " SELECT \"id\"\n" + + " FROM (\n" + + " SELECT NULL AS \"id\"\n" + + " FROM \"employee\"\n" + + " ) AS \"t1\"\n" + + ") AS \"t2\"\n" + + "GROUP BY \"id\""; + final String expectedPostgresql = "SELECT \"id\"\n" + + "FROM (SELECT NULL AS \"id\"\n" + + "FROM \"foodmart\".\"employee\") AS \"t0\"\n" + + "GROUP BY \"id\""; + sql(query) + // Disable RelBuilder's eager Project merging to retain the nested + // Projects that reproduce the constant GROUP BY conversion issue. + .withConfig(c -> c.withRelBuilderConfigTransform(b -> b.withBloat(-1))) + .withPostgresql().ok(expectedPostgresql); + } + + /** Test case for + * [CALCITE-7679] + * RelToSqlConverter generates GROUP BY literals for dialects that do not + * support them when the constant is hidden by nested Projects. */ + @Test void testGroupByLiteralWithReorderedNestedProjects() { + final String query = "SELECT \"id\", \"employee_id\"\n" + + "FROM (\n" + + " SELECT \"id\", \"employee_id\"\n" + + " FROM (\n" + + " SELECT \"employee_id\", NULL AS \"id\"\n" + + " FROM \"employee\"\n" + + " ) AS \"t1\"\n" + + ") AS \"t2\"\n" + + "GROUP BY \"id\", \"employee_id\""; + final String expectedPostgresql = "SELECT \"id\", \"employee_id\"\n" + + "FROM (SELECT NULL AS \"id\", \"employee_id\"\n" + + "FROM \"foodmart\".\"employee\") AS \"t0\"\n" + + "GROUP BY \"id\", \"employee_id\""; + sql(query) + // Disable RelBuilder's eager Project merging to retain the nested + // Projects that reproduce the constant GROUP BY conversion issue. + .withConfig(c -> c.withRelBuilderConfigTransform(b -> b.withBloat(-1))) + .withPostgresql().ok(expectedPostgresql); + } + /** Test case for [CALCITE-6910] * RelToSql does not handle ASOF joins. */ @Test void testAsofJoin() { From 5f7d448663f898c6d47348023bc965c9dcbbfee8 Mon Sep 17 00:00:00 2001 From: bvolpato Date: Wed, 5 Aug 2026 02:45:03 -0400 Subject: [PATCH 432/562] [CALCITE-7440] RelToSqlConverter throws NPE (variable $cor1 not found) for correlated projection after semi-join rewrites --- .../calcite/rel/rel2sql/SqlImplementor.java | 8 +++++ .../rel/rel2sql/RelToSqlConverterTest.java | 30 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java b/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java index c2804db09ee3..18177ecae30a 100644 --- a/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java +++ b/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java @@ -867,6 +867,7 @@ public SqlNode toSql(@Nullable RexProgram program, RexNode rex) { case ALL: if (rex instanceof RexSubQuery) { subQuery = (RexSubQuery) rex; + registerSubQueryCorrelations(subQuery); sqlSubQuery = implementor().visitRoot(subQuery.rel).asQueryOrValues(); final List operands = subQuery.operands; SqlNode op0; @@ -894,6 +895,7 @@ public SqlNode toSql(@Nullable RexProgram program, RexNode rex) { case UNIQUE: case SCALAR_QUERY: subQuery = (RexSubQuery) rex; + registerSubQueryCorrelations(subQuery); sqlSubQuery = implementor().visitRoot(subQuery.rel).asQueryOrValues(); return subQuery.getOperator().createCall(POS, sqlSubQuery); @@ -939,6 +941,12 @@ public SqlNode toSql(@Nullable RexProgram program, RexNode rex) { } } + private void registerSubQueryCorrelations(RexSubQuery subQuery) { + for (CorrelationId id : RelOptUtil.getVariablesUsed(subQuery.rel)) { + implementor().correlTableMap.putIfAbsent(id, this); + } + } + private SqlNode callToSql(@Nullable RexProgram program, RexCall call0, boolean not) { final RexCall call1 = reverseCall(call0); diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java index ad43cd05912c..ce06127a9f31 100644 --- a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java +++ b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java @@ -12711,6 +12711,36 @@ public Sql schema(CalciteAssert.SchemaSpec schemaSpec) { sql(sql).schema(CalciteAssert.SchemaSpec.JDBC_SCOTT).ok(expected); } + /** Test case for + * [CALCITE-7440] + * RelToSqlConverter throws NPE (variable $cor1 not found) for correlated + * projection after semi-join rewrites.. */ + @Test void testPostgresqlRoundTripCorrelatedProjectWithSemiJoinRules() { + final String query = "WITH product_keys AS (\n" + + " SELECT p.\"product_id\",\n" + + " (SELECT MAX(p3.\"product_id\")\n" + + " FROM \"foodmart\".\"product\" p3\n" + + " WHERE p3.\"product_id\" = p.\"product_id\") AS \"mx\"\n" + + " FROM \"foodmart\".\"product\" p\n" + + ")\n" + + "SELECT DISTINCT pk.\"product_id\"\n" + + "FROM product_keys pk\n" + + "LEFT JOIN \"foodmart\".\"product\" p2 USING (\"product_id\")\n" + + "WHERE pk.\"product_id\" IN (\n" + + " SELECT p4.\"product_id\"\n" + + " FROM \"foodmart\".\"product\" p4\n" + + ")"; + + final RuleSet rules = + RuleSets.ofList(CoreRules.FILTER_SUB_QUERY_TO_MARK_CORRELATE, + CoreRules.PROJECT_SUB_QUERY_TO_MARK_CORRELATE, + CoreRules.MARK_TO_SEMI_OR_ANTI_JOIN_RULE, + CoreRules.SEMI_JOIN_JOIN_TRANSPOSE); + + final String generated = sql(query).withPostgresql().optimize(rules, null).exec(); + sql(generated).withPostgresql().exec(); + } + @Test void testNotBetween() { Sql f = fixture().withConvertletTable(new SqlRexConvertletTable() { @Override public @Nullable SqlRexConvertlet get(SqlCall call) { From 6a3dc0b31676172e81f461d74838031655bc37c8 Mon Sep 17 00:00:00 2001 From: bvolpato Date: Wed, 5 Aug 2026 02:45:03 -0400 Subject: [PATCH 433/562] [CALCITE-7439] RelToSqlConverter emits ambiguous GROUP BY after LEFT JOIN USING with semi-join rewrite --- .../rel/rel2sql/RelToSqlConverter.java | 108 ++++++++++- .../rel/rel2sql/RelToSqlConverterTest.java | 182 ++++++++++++++++++ 2 files changed, 288 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java b/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java index 718e6b1cb965..2f96e8f73b20 100644 --- a/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java +++ b/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java @@ -895,12 +895,17 @@ private List generateGroupList(Builder builder, + aggregate.getGroupSet() + ", just possibly a different order"; final List groupKeys = new ArrayList<>(); + final SqlJoin fromJoin = + builder.select.getFrom() instanceof SqlJoin ? (SqlJoin) builder.select.getFrom() : null; for (int key : groupList) { - final SqlNode field = builder.context.field(key); + SqlNode field = builder.context.field(key); + field = maybeQualifyJoinKey(field, key, fromJoin, aggregate.getInput()); groupKeys.add(field); } for (int key : sortedGroupList) { - final SqlNode field = builder.context.field(key); + SqlNode field = + maybeQualifyJoinKey(builder.context.field(key), key, fromJoin, + aggregate.getInput()); addSelect(selectList, field, aggregate.getRowType()); } switch (aggregate.getGroupType()) { @@ -942,6 +947,105 @@ private List generateGroupList(Builder builder, } } + /** Qualifies a group key when its aggregate input renders as a SQL join + * whose columns would otherwise be ambiguous. */ + private SqlNode maybeQualifyJoinKey(SqlNode field, int key, + @Nullable SqlJoin fromJoin, RelNode input) { + if (fromJoin == null) { + return field; + } + final @Nullable SqlNode qualified = joinField(input, key, fromJoin); + return qualified != null ? qualified : field; + } + + /** Resolves a field through row-preserving nodes and nested joins to the SQL + * relation that supplies it. Set operations are handled as aliased derived + * tables. */ + private static @Nullable SqlNode joinField(RelNode input, int field, + SqlNode from) { + final @Nullable String alias = SqlValidatorUtil.alias(from); + if (alias != null) { + return new SqlIdentifier( + ImmutableList.of(alias, + input.getRowType().getFieldList().get(field).getName()), POS); + } + if (input instanceof Project) { + final Project project = (Project) input; + return joinExpression(project.getInput(), project.getProjects().get(field), + from); + } + if (input instanceof Filter) { + return joinField(((Filter) input).getInput(), field, from); + } + final RelNode left; + final RelNode right; + final JoinRelType joinType; + final boolean correlateInput; + if (input instanceof Join) { + final Join join = (Join) input; + left = join.getLeft(); + right = join.getRight(); + joinType = join.getJoinType(); + correlateInput = false; + } else if (input instanceof Correlate) { + final Correlate correlateRel = (Correlate) input; + left = correlateRel.getLeft(); + right = correlateRel.getRight(); + joinType = correlateRel.getJoinType(); + correlateInput = true; + } else { + return null; + } + if (!joinType.projectsRight()) { + if (correlateInput) { + return from instanceof SqlJoin + ? joinField(left, field, ((SqlJoin) from).getLeft()) + : null; + } + return joinField(left, field, from); + } + if (!(from instanceof SqlJoin)) { + return null; + } + final SqlJoin fromJoin = (SqlJoin) from; + final int leftFieldCount = left.getRowType().getFieldCount(); + final RelNode side; + final SqlNode sqlSide; + final int sideField; + if (field < leftFieldCount) { + side = left; + sqlSide = fromJoin.getLeft(); + sideField = field; + } else { + side = right; + sqlSide = fromJoin.getRight(); + sideField = field - leftFieldCount; + } + return joinField(side, sideField, sqlSide); + } + + /** Converts field references and merged join keys in a project expression + * to qualified SQL expressions. */ + private static @Nullable SqlNode joinExpression(RelNode input, + RexNode expression, SqlNode from) { + if (expression instanceof RexInputRef) { + return joinField(input, ((RexInputRef) expression).getIndex(), from); + } + if (expression.getKind() == SqlKind.COALESCE) { + final List operands = new ArrayList<>(); + for (RexNode operand : ((RexCall) expression).getOperands()) { + final @Nullable SqlNode sqlOperand = + joinExpression(input, operand, from); + if (sqlOperand == null) { + return null; + } + operands.add(sqlOperand); + } + return SqlStdOperatorTable.COALESCE.createCall(POS, operands); + } + return null; + } + private static SqlNode groupItem(List groupKeys, ImmutableBitSet groupSet, ImmutableBitSet wholeGroupSet) { final List nodes = groupSet.asList().stream() diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java index ce06127a9f31..83a4f87b288e 100644 --- a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java +++ b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java @@ -50,6 +50,7 @@ import org.apache.calcite.rel.type.RelDataTypeSystem; import org.apache.calcite.rel.type.RelDataTypeSystemImpl; import org.apache.calcite.rex.RexCorrelVariable; +import org.apache.calcite.rex.RexNode; import org.apache.calcite.runtime.FlatLists; import org.apache.calcite.runtime.Hook; import org.apache.calcite.schema.SchemaPlus; @@ -147,6 +148,52 @@ private Sql sql(String sql) { return fixture().withSql(sql); } + private void assertPostgresqlSqlValid(String sql) { + try { + final SchemaPlus rootSchema = Frameworks.createRootSchema(true); + final SchemaPlus defaultSchema = + CalciteAssert.addSchema(rootSchema, CalciteAssert.SchemaSpec.JDBC_FOODMART); + final Planner planner = + getPlanner(null, + PostgresqlSqlDialect.DEFAULT.configureParser(SqlParser.config()), + defaultSchema, + SqlToRelConverter.config().withTrimUnusedFields(false), + ImmutableSet.of(), + DatabaseProduct.POSTGRESQL.getDialect().getTypeSystem(), + StandardConvertletTable.INSTANCE); + final SqlNode parsed = planner.parse(sql); + planner.validate(parsed); + } catch (Exception e) { + throw TestUtil.rethrow(e); + } + } + + private static RuleSet semiJoinRules() { + return RuleSets.ofList(CoreRules.PROJECT_SUB_QUERY_TO_MARK_CORRELATE, + CoreRules.FILTER_SUB_QUERY_TO_MARK_CORRELATE, + CoreRules.MARK_TO_SEMI_OR_ANTI_JOIN_RULE, + CoreRules.PROJECT_TO_SEMI_JOIN); + } + + private String postgresqlDistinctJoinSql(String select, String joinType, + String condition) { + final String query = "WITH product_keys AS (\n" + + " SELECT p.\"product_id\",\n" + + " (SELECT MAX(p3.\"product_id\")\n" + + " FROM \"foodmart\".\"product\" p3\n" + + " WHERE p3.\"product_id\" = p.\"product_id\") AS \"mx\"\n" + + " FROM \"foodmart\".\"product\" p\n" + + ")\n" + + "SELECT DISTINCT " + select + "\n" + + "FROM product_keys pk\n" + + joinType + " JOIN \"foodmart\".\"product\" p2 " + condition + "\n" + + "WHERE pk.\"product_id\" IN (\n" + + " SELECT p4.\"product_id\"\n" + + " FROM \"foodmart\".\"product\" p4\n" + + ")"; + return sql(query).withPostgresql().optimize(semiJoinRules(), null).exec(); + } + /** Initiates a test case with a given {@link RelNode} supplier. */ private Sql relFn(Function relFn) { return fixture() @@ -12741,6 +12788,141 @@ public Sql schema(CalciteAssert.SchemaSpec schemaSpec) { sql(generated).withPostgresql().exec(); } + /** Test case for + * [CALCITE-7439] + * RelToSqlConverter emits ambiguous GROUP BY after LEFT JOIN USING with + * semi-join rewrite.. */ + @Test void testPostgresqlRoundTripDistinctLeftJoinInSubqueryWithSemiJoinRules() { + final String generated = + postgresqlDistinctJoinSql("\"product_id\"", "LEFT", "USING (\"product_id\")"); + assertPostgresqlSqlValid(generated); + } + + @Test void testDistinctRightJoinUsing() { + final String generated = + postgresqlDistinctJoinSql("\"product_id\"", "RIGHT", "USING (\"product_id\")"); + assertThat( + generated, isLinux("SELECT \"product1\".\"product_id\"\n" + + "FROM (SELECT \"$cor0\".\"product_id\", \"t1\".\"EXPR$0\" AS \"mx\"\n" + + "FROM \"foodmart\".\"product\" AS \"$cor0\",\n" + + "LATERAL (SELECT MAX(\"product_id\") AS \"EXPR$0\"\n" + + "FROM \"foodmart\".\"product\"\n" + + "WHERE \"product_id\" = \"$cor0\".\"product_id\") AS \"t1\") AS \"t2\"\n" + + "RIGHT JOIN \"foodmart\".\"product\" AS \"product1\"" + + " ON \"t2\".\"product_id\" = \"product1\".\"product_id\"\n" + + "WHERE EXISTS (SELECT 1\n" + + "FROM (SELECT \"product_id\"\n" + + "FROM \"foodmart\".\"product\") AS \"t3\"\n" + + "WHERE \"t2\".\"product_id\" = \"t3\".\"product_id\")\n" + + "GROUP BY \"product1\".\"product_id\"")); + assertPostgresqlSqlValid(generated); + } + + @Test void testDistinctFullJoinUsing() { + final String generated = + postgresqlDistinctJoinSql("\"product_id\"", "FULL", "USING (\"product_id\")"); + assertThat(generated, + isLinux("SELECT COALESCE(\"t2\".\"product_id\"," + + " \"product1\".\"product_id\") AS \"product_id\"\n" + + "FROM (SELECT \"$cor0\".\"product_id\", \"t1\".\"EXPR$0\" AS \"mx\"\n" + + "FROM \"foodmart\".\"product\" AS \"$cor0\",\n" + + "LATERAL (SELECT MAX(\"product_id\") AS \"EXPR$0\"\n" + + "FROM \"foodmart\".\"product\"\n" + + "WHERE \"product_id\" = \"$cor0\".\"product_id\") AS \"t1\") AS \"t2\"\n" + + "FULL JOIN \"foodmart\".\"product\" AS \"product1\"" + + " ON \"t2\".\"product_id\" = \"product1\".\"product_id\"\n" + + "WHERE EXISTS (SELECT 1\n" + + "FROM (SELECT \"product_id\"\n" + + "FROM \"foodmart\".\"product\") AS \"t3\"\n" + + "WHERE \"t2\".\"product_id\" = \"t3\".\"product_id\")\n" + + "GROUP BY COALESCE(\"t2\".\"product_id\"," + + " \"product1\".\"product_id\")")); + assertPostgresqlSqlValid(generated); + } + + @Test void testDistinctFullJoinOnKeepsSelectedSide() { + final String condition = "ON pk.\"product_id\" = p2.\"product_id\""; + final String generated = + postgresqlDistinctJoinSql("pk.\"product_id\"", "FULL", condition); + assertThat( + generated, isLinux("SELECT \"t2\".\"product_id\"\n" + + "FROM (SELECT \"$cor0\".\"product_id\", \"t1\".\"EXPR$0\" AS \"mx\"\n" + + "FROM \"foodmart\".\"product\" AS \"$cor0\",\n" + + "LATERAL (SELECT MAX(\"product_id\") AS \"EXPR$0\"\n" + + "FROM \"foodmart\".\"product\"\n" + + "WHERE \"product_id\" = \"$cor0\".\"product_id\") AS \"t1\") AS \"t2\"\n" + + "FULL JOIN \"foodmart\".\"product\" AS \"product1\"" + + " ON \"t2\".\"product_id\" = \"product1\".\"product_id\"\n" + + "WHERE EXISTS (SELECT 1\n" + + "FROM (SELECT \"product_id\"\n" + + "FROM \"foodmart\".\"product\") AS \"t3\"\n" + + "WHERE \"t2\".\"product_id\" = \"t3\".\"product_id\")\n" + + "GROUP BY \"t2\".\"product_id\"")); + assertPostgresqlSqlValid(generated); + } + + @Test void testDistinctOverSemiJoinAndCorrelate() { + final Function relFn = b -> { + final Holder v = Holder.empty(); + b.values(new String[]{"id"}, 1) + .variable(v::set); + b.values(new String[]{"id"}, 1) + .filter( + b.equals(b.field("id"), + b.getRexBuilder().makeFieldAccess(v.get(), 0))); + final RexNode correlateId = b.field(2, 0, 0); + b.correlate(JoinRelType.INNER, v.get().id, correlateId); + b.values(new String[]{"id"}, 1); + final RexNode leftId = b.field(2, 0, 0); + return b.join(JoinRelType.SEMI, + b.equals(leftId, b.field(2, 1, 0))) + .project(leftId) + .distinct() + .build(); + }; + final String generated = relFn(relFn).exec(); + assertThat( + generated, isLinux("SELECT \"$cor0\".\"id\"\n" + + "FROM (VALUES (1)) AS \"$cor0\" (\"id\"),\n" + + "LATERAL (SELECT *\n" + + "FROM (VALUES (1)) AS \"t0\" (\"id\")\n" + + "WHERE \"id\" = \"$cor0\".\"id\") AS \"t1\"\n" + + "WHERE EXISTS (SELECT 1\n" + + "FROM (VALUES (1)) AS \"t2\" (\"id\")\n" + + "WHERE \"$cor0\".\"id\" = \"t2\".\"id\")\n" + + "GROUP BY \"$cor0\".\"id\"")); + assertPostgresqlSqlValid(generated); + } + + @Test void testDistinctOverNestedJoin() { + final Function relFn = b -> { + b.scan("EMP"); + b.scan("EMP"); + b.join(JoinRelType.INNER, + b.equals(b.field(2, 0, "EMPNO"), b.field(2, 1, "EMPNO"))); + b.scan("EMP"); + b.join(JoinRelType.INNER, + b.equals(b.field(2, 0, 0), b.field(2, 1, "EMPNO"))); + b.scan("DEPT"); + final RexNode firstDeptNo = b.field(2, 0, 7); + return b.join(JoinRelType.SEMI, + b.equals(b.field(2, 0, 7), b.field(2, 1, "DEPTNO"))) + .project(firstDeptNo) + .distinct() + .build(); + }; + relFn(relFn).ok("SELECT \"EMP\".\"DEPTNO\"\n" + + "FROM \"scott\".\"EMP\"\n" + + "INNER JOIN \"scott\".\"EMP\" AS \"EMP0\"" + + " ON \"EMP\".\"EMPNO\" = \"EMP0\".\"EMPNO\"\n" + + "INNER JOIN \"scott\".\"EMP\" AS \"EMP1\"" + + " ON \"EMP\".\"EMPNO\" = \"EMP1\".\"EMPNO\"\n" + + "WHERE EXISTS (SELECT 1\n" + + "FROM \"scott\".\"DEPT\"\n" + + "WHERE \"EMP\".\"DEPTNO\" = \"DEPT\".\"DEPTNO\")\n" + + "GROUP BY \"EMP\".\"DEPTNO\""); + } + @Test void testNotBetween() { Sql f = fixture().withConvertletTable(new SqlRexConvertletTable() { @Override public @Nullable SqlRexConvertlet get(SqlCall call) { From 8b2c702a82bac914b9033bb6ed3bb1974d8e0711 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Tue, 4 Aug 2026 22:01:54 -0700 Subject: [PATCH 434/562] [CALCITE-6002] CONTAINS_SUBSTR does not unparse correctly Signed-off-by: Mihai Budiu --- .../apache/calcite/runtime/SqlFunctions.java | 2 +- .../sql/fun/SqlContainsSubstrFunction.java | 60 +++++++++++++++++++ .../calcite/sql/fun/SqlLibraryOperators.java | 6 +- .../calcite/test/SqlOperatorUnparseTest.java | 6 -- .../calcite/sql/parser/SqlParserTest.java | 10 ++++ .../org/apache/calcite/sql/test/SqlTests.java | 2 +- 6 files changed, 74 insertions(+), 12 deletions(-) create mode 100644 core/src/main/java/org/apache/calcite/sql/fun/SqlContainsSubstrFunction.java diff --git a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java index 15d526393b33..cf80e7fc0e7a 100644 --- a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java +++ b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java @@ -1122,7 +1122,7 @@ public static List split(ByteString s, ByteString delimiter) { return nullFlag ? null : false; } - /** SQL CONTAINS_SUBSTR(jsonString, substr, json_scope=>jsonScope) + /** SQL CONTAINS_SUBSTR(jsonString, substr, json_scope => jsonScope) * operator. */ public static boolean containsSubstr(String jsonString, String substr, String jsonScope) { diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlContainsSubstrFunction.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlContainsSubstrFunction.java new file mode 100644 index 000000000000..b3e2e50ac43a --- /dev/null +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlContainsSubstrFunction.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.sql.fun; + +import org.apache.calcite.sql.SqlCall; +import org.apache.calcite.sql.SqlFunction; +import org.apache.calcite.sql.SqlFunctionCategory; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlWriter; +import org.apache.calcite.sql.type.OperandTypes; +import org.apache.calcite.sql.type.ReturnTypes; + +/** + * Definition of the "CONTAINS_SUBSTR(expression, string [, json_scope => + * json_scope_value ])" function; returns whether string exists as a + * substring in expression. + */ +public class SqlContainsSubstrFunction extends SqlFunction { + public SqlContainsSubstrFunction() { + super("CONTAINS_SUBSTR", SqlKind.OTHER_FUNCTION, + ReturnTypes.BOOLEAN_NULLABLE, null, + OperandTypes.ANY_STRING_OPTIONAL_STRING, + SqlFunctionCategory.STRING); + } + + /** + * The parser only accepts the optional third operand with the named + * syntax "json_scope => json_scope_value", both in Calcite and in + * BigQuery, so {@code unparse} must emit "JSON_SCOPE =>" before it. + */ + @Override public void unparse(SqlWriter writer, SqlCall call, int leftPrec, + int rightPrec) { + final SqlWriter.Frame frame = writer.startFunCall(getName()); + writer.sep(","); + call.operand(0).unparse(writer, 0, 0); + writer.sep(","); + call.operand(1).unparse(writer, 0, 0); + if (call.operandCount() == 3) { + writer.sep(","); + writer.keyword("JSON_SCOPE"); + writer.keyword("=>"); + call.operand(2).unparse(writer, 0, 0); + } + writer.endFunCall(frame); + } +} diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java index 539d6b327fd0..089f20eec935 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java @@ -1069,14 +1069,12 @@ static RelDataType deriveTypeSplit(SqlOperatorBinding operatorBinding, ReturnTypes.INTEGER_NULLABLE, OperandTypes.DATE, SqlFunctionCategory.TIMEDATE); - /** "CONTAINS_SUBSTR(expression, string[, json_scope => json_scope_value ])" + /** "CONTAINS_SUBSTR(expression, string[, json_scope => json_scope_value ])" * function; returns whether string exists as substring in expression, with optional * json_scope argument. */ @LibraryOperator(libraries = {BIG_QUERY}) public static final SqlFunction CONTAINS_SUBSTR = - SqlBasicFunction.create("CONTAINS_SUBSTR", - ReturnTypes.BOOLEAN_NULLABLE, OperandTypes.ANY_STRING_OPTIONAL_STRING, - SqlFunctionCategory.STRING); + new SqlContainsSubstrFunction(); /** The "MONTHNAME(datetime)" function; returns the name of the month, * in the current locale, of a TIMESTAMP or DATE argument. */ diff --git a/core/src/test/java/org/apache/calcite/test/SqlOperatorUnparseTest.java b/core/src/test/java/org/apache/calcite/test/SqlOperatorUnparseTest.java index b382a6412ab5..b4e4e66769ad 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlOperatorUnparseTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlOperatorUnparseTest.java @@ -114,10 +114,4 @@ String rewrite(StringAndPos sap) throws SqlParseException { void testSafeOffsetOperator() { super.testSafeOffsetOperator(); } - - @Override @Disabled("https://issues.apache.org/jira/browse/CALCITE-6002 " - + "CONTAINS_SUBSTR does not unparse correctly") - void testContainsSubstrFunc() { - super.testContainsSubstrFunc(); - } } diff --git a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java index 394aecf26f2a..beae2eb7bfd9 100644 --- a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java +++ b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java @@ -7657,6 +7657,16 @@ private static Consumer> checkWarnings( .ok("CAST(`X` AS VARBINARY)"); } + /** Test case for + * [CALCITE-6002] + * CONTAINS_SUBSTR does not unparse correctly. */ + @Test void testContainsSubstr() { + expr("CONTAINS_SUBSTR(x, 'a')") + .ok("CONTAINS_SUBSTR(`X`, 'a')"); + expr("CONTAINS_SUBSTR(x, 'a', json_scope=>'JSON_KEYS')") + .ok("CONTAINS_SUBSTR(`X`, 'a', JSON_SCOPE => 'JSON_KEYS')"); + } + @Test void testTimestampAdd() { final String sql = "select * from t\n" + "where timestampadd(month, 5, hiredate) < curdate"; diff --git a/testkit/src/main/java/org/apache/calcite/sql/test/SqlTests.java b/testkit/src/main/java/org/apache/calcite/sql/test/SqlTests.java index 7831767acd55..2cfaf8fc872d 100644 --- a/testkit/src/main/java/org/apache/calcite/sql/test/SqlTests.java +++ b/testkit/src/main/java/org/apache/calcite/sql/test/SqlTests.java @@ -395,7 +395,7 @@ public static void checkEx(@Nullable Throwable ex, if (sap.pos == null) { throw new AssertionError("Actual error had a position, but expected " + "error did not. Add error position carets to sql:\n" - + sqlWithCarets); + + sqlWithCarets, actualException); } } From d2c7f265c1be91b045e94b6fb6ac2de113598df0 Mon Sep 17 00:00:00 2001 From: xuzifu666 <1206332514@qq.com> Date: Tue, 4 Aug 2026 21:23:44 +0800 Subject: [PATCH 435/562] Test cases for [CALCITE-5418] Nested Queries are not expanded properly --- .../apache/calcite/test/RelOptRulesTest.java | 29 ++++++++++ .../apache/calcite/test/RelOptRulesTest.xml | 57 +++++++++++++++++++ core/src/test/resources/sql/sub-query.iq | 37 ++++++++++++ 3 files changed, 123 insertions(+) diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index 5c687519daa0..bad13247c3ee 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -9459,6 +9459,35 @@ private void checkSemiJoinRuleOnAntiJoin(RelOptRule rule) { sql(sql).withSubQueryRules().check(); } + /** Test case for + * [CALCITE-5716] + * Two level nested correlated subquery translates to incorrect ON + * condition. + * + *

      A middle-level EXISTS holds two sibling inner EXISTS sub-queries that + * correlate to different levels: {@code ea.empno = e.empno} references the + * outer {@code emp} ($cor0), while {@code e2.deptno = d.deptno} references + * the middle {@code dept} ($cor2). The rewrite must keep those correlation + * variables at their originating levels rather than collapsing the inner + * Correlate onto the outer variable. */ + @Test void testExpandFilterNestedExistsWithTwoSiblingInnerExists() { + final String sql = "SELECT deptno\n" + + "FROM emp e\n" + + "WHERE EXISTS (\n" + + " SELECT *\n" + + " FROM dept d\n" + + " WHERE d.deptno = e.deptno\n" + + " AND EXISTS (\n" + + " SELECT *\n" + + " FROM emp_address ea\n" + + " WHERE ea.empno = e.empno)\n" + + " AND EXISTS (\n" + + " SELECT *\n" + + " FROM emp e2\n" + + " WHERE e2.deptno = d.deptno))"; + sql(sql).withSubQueryRules().check(); + } + @Test void testDecorrelateExists() { final String sql = "select * from sales.emp\n" + "where EXISTS (\n" diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index 32f423c3e17b..c623e63038b1 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -5220,6 +5220,63 @@ LogicalProject(EMPNO=[$0]) LogicalProject(DEPTNO=[$0], i=[true]) LogicalFilter(condition=[=($1, 'dept2')]) LogicalTableScan(table=[[CATALOG, SALES, DEPTNULLABLES]]) +]]> + + + + + + + + + + + diff --git a/core/src/test/resources/sql/sub-query.iq b/core/src/test/resources/sql/sub-query.iq index 847cdb98d949..d941e1b8c309 100644 --- a/core/src/test/resources/sql/sub-query.iq +++ b/core/src/test/resources/sql/sub-query.iq @@ -4904,6 +4904,43 @@ FROM dept; !ok +# [CALCITE-5716] Two level nested correlated subquery translates to incorrect ON condition. +# A middle-level EXISTS holds two sibling inner EXISTS that correlate to +# different levels: ea.mgr = e.mgr references the outer emp, while +# e2.deptno = d.deptno references the middle dept. Each correlation must stay +# at its originating level. +SELECT e.empno +FROM emp e +WHERE EXISTS ( + SELECT 1 + FROM dept d + WHERE d.deptno = e.deptno + AND EXISTS ( + SELECT 1 FROM emp ea WHERE ea.mgr = e.mgr) + AND EXISTS ( + SELECT 1 FROM emp e2 WHERE e2.deptno = d.deptno)) +ORDER BY e.empno; ++-------+ +| EMPNO | ++-------+ +| 7369 | +| 7499 | +| 7521 | +| 7566 | +| 7654 | +| 7698 | +| 7782 | +| 7788 | +| 7844 | +| 7876 | +| 7900 | +| 7902 | +| 7934 | ++-------+ +(13 rows) + +!ok + # [CALCITE-7034] IllegalArgumentException when correlate subQuery in on clause and use rightside columns SELECT e1.* FROM emp e1 From 49612760235ff98cff418065018830f3d09380d7 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Wed, 5 Aug 2026 09:59:43 -0700 Subject: [PATCH 436/562] Test cases for some very old bugs now fixed Signed-off-by: Mihai Budiu --- .../java/org/apache/calcite/util/Bug.java | 19 ------- .../calcite/test/SqlToRelConverterTest.java | 5 -- .../apache/calcite/test/SqlValidatorTest.java | 25 +++++++++ .../calcite/test/SqlToRelConverterTest.xml | 4 +- .../calcite/sql/parser/SqlParserTest.java | 51 ++++++++----------- .../apache/calcite/test/SqlOperatorTest.java | 44 ++-------------- 6 files changed, 51 insertions(+), 97 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/util/Bug.java b/core/src/main/java/org/apache/calcite/util/Bug.java index 8f23ce7b1cf8..cbdc1980a4a3 100644 --- a/core/src/main/java/org/apache/calcite/util/Bug.java +++ b/core/src/main/java/org/apache/calcite/util/Bug.java @@ -54,8 +54,6 @@ public abstract class Bug { public static final boolean DT239_FIXED = false; - public static final boolean DT785_FIXED = false; - /** * Whether issue * FRG-377: Regular character set identifiers defined in SQL:2008 spec like @@ -64,17 +62,6 @@ public abstract class Bug { */ public static final boolean FRG377_FIXED = false; - /** - * Whether dtbug1684 "CURRENT_DATE not implemented in fennel calc" is fixed. - */ - public static final boolean DT1684_FIXED = false; - - /** - * Whether issue FRG-73: - * miscellaneous bugs with nested comments is fixed. - */ - public static final boolean FRG73_FIXED = false; - /** * Whether issue FRG-78: * collation clause should be on expression instead of identifier is @@ -82,12 +69,6 @@ public abstract class Bug { */ public static final boolean FRG78_FIXED = false; - /** - * Whether issue - * FRG-189: FarragoAutoVmOperatorTest.testSelect fails is fixed. - */ - public static final boolean FRG189_FIXED = false; - /** * Whether issue * FRG-254: environment-dependent failure for diff --git a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java index 6ce401502c93..928c29850ac1 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java @@ -55,7 +55,6 @@ import org.apache.calcite.sql.validate.SqlValidatorUtil; import org.apache.calcite.sql.validate.implicit.TypeCoercionImpl; import org.apache.calcite.test.catalog.MockCatalogReaderExtended; -import org.apache.calcite.util.Bug; import org.apache.calcite.util.TestUtil; import org.apache.calcite.util.Util; @@ -3123,10 +3122,6 @@ void checkCorrelatedMapSubQuery(boolean expand) { } @Test void testInterval() { - // temporarily disabled per DTbug 1212 - if (!Bug.DT785_FIXED) { - return; - } final String sql = "values(cast(interval '1' hour as interval hour to second))"; sql(sql).ok(); diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index 2e472a968c63..4088b0556b3f 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -10506,6 +10506,31 @@ void testGroupExpressionEquivalenceParams() { + "on orders.productid = products_temporal.productid").ok(); } + /** Test cases for [FRG-189]. */ + @Test void testScalarSubQueryTypeInSelectList() { + sql("SELECT *, (SELECT * FROM (VALUES(1))) FROM (VALUES(2))") + .type("RecordType(INTEGER NOT NULL EXPR$0, INTEGER EXPR$1) NOT NULL"); + sql("SELECT *, (SELECT * FROM (VALUES(CAST(10 as BIGINT))))\n" + + "FROM (VALUES(CAST(10 as bigint)))") + .type("RecordType(BIGINT NOT NULL EXPR$0, BIGINT EXPR$1) NOT NULL"); + sql("SELECT *, (SELECT * FROM (VALUES(10.5))) FROM (VALUES(10.5))") + .type("RecordType(DECIMAL(3, 1) NOT NULL EXPR$0," + + " DECIMAL(3, 1) EXPR$1) NOT NULL"); + sql("SELECT *, (SELECT * FROM (VALUES('this is a char')))\n" + + "FROM (VALUES('this is a char too'))") + .type("RecordType(CHAR(18) NOT NULL EXPR$0, CHAR(14) EXPR$1) NOT NULL"); + sql("SELECT *, (SELECT * FROM (VALUES(true))) FROM (values(false))") + .type("RecordType(BOOLEAN NOT NULL EXPR$0, BOOLEAN EXPR$1) NOT NULL"); + sql("SELECT *, (SELECT * FROM (VALUES(cast('abcd' as varchar(10)))))\n" + + "FROM (VALUES(CAST('abcd' as varchar(10))))") + .type("RecordType(VARCHAR(10) NOT NULL EXPR$0," + + " VARCHAR(10) EXPR$1) NOT NULL"); + sql("SELECT *, (SELECT * FROM (VALUES(TIMESTAMP '2006-01-01 12:00:05')))\n" + + "FROM (VALUES(TIMESTAMP '2006-01-01 12:00:05'))") + .type("RecordType(TIMESTAMP(0) NOT NULL EXPR$0," + + " TIMESTAMP(0) EXPR$1) NOT NULL"); + } + @Test void testScalarSubQuery() { sql("SELECT ename,(select name from dept where deptno=1) FROM emp").ok(); sql("SELECT ename,^(select losal, hisal from salgrade where grade=1)^ FROM emp") diff --git a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml index 6e98c11baa86..2cb1a1093304 100644 --- a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml @@ -4007,9 +4007,7 @@ LogicalProject(EXPR$0=[1]) diff --git a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java index beae2eb7bfd9..3fbe5af9be0c 100644 --- a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java +++ b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java @@ -4388,10 +4388,8 @@ void checkPeriodPredicate(Checker checker) { .ok("VALUES (ROW(2))"); // end of multiline comment without start - if (Bug.FRG73_FIXED) { - sql("values (1 */ 2)") - .fails("xx"); - } + sql("values (1 ^*/^ 2)") + .fails("(?s)Encountered \"\\*/\" at .*"); // SQL:2003, 5.2, syntax rule #10 "Within a , // any immediately followed by an without any @@ -4399,34 +4397,27 @@ void checkPeriodPredicate(Checker checker) { // comment introducer> for a that is a ". - // comment inside a comment - // Spec is unclear what should happen, but currently it crashes the - // parser, and that's bad - if (Bug.FRG73_FIXED) { - sql("values (1 + /* comment /* inner comment */ */ 2)").ok("xx"); - } - - // single-line comment inside multiline comment is illegal - // - // SQL-2003, 5.2: "Note 63 - Conforming programs should not place - // within a because if such a - // contains the sequence of characters "*/" without - // a preceding "/*" in the same , it will prematurely - // terminate the containing . - if (Bug.FRG73_FIXED) { - final String sql = "values /* multiline contains -- singline */\n" - + " (1)"; - sql(sql).fails("xxx"); - } + // Calcite does not nest bracketed comments (SQL:2003, 5.2, syntax rule + // #10 requires nesting); the first "*/" ends the comment, so the + // second "*/" is a stray token. + sql("values (1 ^+^ /* comment /* inner comment */ */ 2)") + .fails("(?s)Encountered \"\\+ \\*/\" at .*"); + + // A single-line comment within a multiline comment is treated as + // comment text, per SQL-2003, 5.2: "Note 63 - Conforming programs + // should not place within a + // because if such a contains the sequence of + // characters "*/" without a preceding "/*" in the same , it will prematurely terminate the containing + // . + sql("values /* multiline contains -- singline */\n" + + " (1)") + .ok("VALUES (ROW(1))"); // non-terminated multi-line comment inside single-line comment - if (Bug.FRG73_FIXED) { - // Test should fail, and it does, but it should give "*/" as the - // erroneous token. - final String sql = "values ( -- rest of line /* a comment\n" - + " 1, ^*/^ 2)"; - sql(sql).fails("Encountered \"/\\*\" at"); - } + sql("values ( -- rest of line /* a comment\n" + + " 1, ^*/^ 2)") + .fails("(?s)Encountered \"\\*/\" at .*"); sql("values (1 + /* comment -- rest of line\n" + " rest of comment */ 2)") diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java index 387915bca6b5..f0ad6867faec 100644 --- a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java +++ b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java @@ -2479,40 +2479,6 @@ void testCastToBoolean(CastType castType, SqlOperatorFixture f) { final SqlOperatorFixture f = fixture(); f.check("select * from (values(1))", SqlTests.INTEGER_TYPE_CHECKER, 1); - // Check return type on scalar sub-query in select list. Note return - // type is always nullable even if sub-query select value is NOT NULL. - // Bug FRG-189 causes this test to fail only in SqlOperatorTest; not - // in subtypes. - if (Bug.FRG189_FIXED) { - f.checkType("SELECT *,\n" - + " (SELECT * FROM (VALUES(1)))\n" - + "FROM (VALUES(2))", - "RecordType(INTEGER NOT NULL EXPR$0, INTEGER EXPR$1) NOT NULL"); - f.checkType("SELECT *,\n" - + " (SELECT * FROM (VALUES(CAST(10 as BIGINT))))\n" - + "FROM (VALUES(CAST(10 as bigint)))", - "RecordType(BIGINT NOT NULL EXPR$0, BIGINT EXPR$1) NOT NULL"); - f.checkType("SELECT *,\n" - + " (SELECT * FROM (VALUES(10.5)))\n" - + "FROM (VALUES(10.5))", - "RecordType(DECIMAL(3, 1) NOT NULL EXPR$0, DECIMAL(3, 1) EXPR$1) NOT NULL"); - f.checkType("SELECT *,\n" - + " (SELECT * FROM (VALUES('this is a char')))\n" - + "FROM (VALUES('this is a char too'))", - "RecordType(CHAR(18) NOT NULL EXPR$0, CHAR(14) EXPR$1) NOT NULL"); - f.checkType("SELECT *,\n" - + " (SELECT * FROM (VALUES(true)))\n" - + "FROM (values(false))", - "RecordType(BOOLEAN NOT NULL EXPR$0, BOOLEAN EXPR$1) NOT NULL"); - f.checkType(" SELECT *,\n" - + " (SELECT * FROM (VALUES(cast('abcd' as varchar(10)))))\n" - + "FROM (VALUES(CAST('abcd' as varchar(10))))", - "RecordType(VARCHAR(10) NOT NULL EXPR$0, VARCHAR(10) EXPR$1) NOT NULL"); - f.checkType("SELECT *,\n" - + " (SELECT * FROM (VALUES(TIMESTAMP '2006-01-01 12:00:05')))\n" - + "FROM (VALUES(TIMESTAMP '2006-01-01 12:00:05'))", - "RecordType(TIMESTAMP(0) NOT NULL EXPR$0, TIMESTAMP(0) EXPR$1) NOT NULL"); - } } @Test void testLiteralChain() { @@ -3653,12 +3619,10 @@ static void checkOverlaps(OverlapChecker c) { + " time '01:23:44') hour to second / 2", "08:25:52", "TIME(0) NOT NULL"); - if (Bug.DT1684_FIXED) { - f.checkBoolean("(date '1969-04-29' +" - + " (CURRENT_DATE - " - + " date '1969-04-29') day / 2) is not null", - true); - } + f.checkBoolean("(date '1969-04-29' +" + + " (CURRENT_DATE - " + + " date '1969-04-29') day / 2) is not null", + true); f.checkScalar("(date '2023-12-01' - date '2022-12-01') year", "+1", "INTERVAL YEAR NOT NULL"); f.checkScalar("(date '2022-12-01' - date '2023-12-01') year", From 27d2a536bbf7540ab242e5b5354e8e289ec33706 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Tue, 4 Aug 2026 21:06:27 -0700 Subject: [PATCH 437/562] [CALCITE-6753] DeterministicCodeOptimizer may lift method calls out of try-catch blocks Signed-off-by: Mihai Budiu --- .../apache/calcite/runtime/SqlFunctions.java | 4 - .../linq4j/tree/ClassDeclarationFinder.java | 26 +++ .../linq4j/test/DeterministicTest.java | 183 ++++++++++++++++++ 3 files changed, 209 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java index cf80e7fc0e7a..e3ad7c502d40 100644 --- a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java +++ b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java @@ -5305,10 +5305,6 @@ public static int toInt(java.sql.Time v) { return v == null ? castNonNull(null) : toInt(v); } - // Method tagged as non-deterministic because it can throw. - // The DeterministicCodeOptimizer may otherwise try to lift it out of try-catch blocks. - // See https://issues.apache.org/jira/browse/CALCITE-6753 - @NonDeterministic public static int toInt(String s) { return parseInt(s.trim()); } diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ClassDeclarationFinder.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ClassDeclarationFinder.java index bb0d93b013d0..b95253477292 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ClassDeclarationFinder.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ClassDeclarationFinder.java @@ -34,6 +34,10 @@ public class ClassDeclarationFinder extends Shuttle { protected final @Nullable ClassDeclarationFinder parent; + /** Visits a subtree without changing it. Used for the subtrees + * which must not be optimized. */ + private static final Shuttle PASS_THROUGH = new Shuttle(); + /** * The list of new final static fields to be added to the current class. */ @@ -153,6 +157,28 @@ protected ClassDeclarationFinder(ClassDeclarationFinder parent) { return visitor; } + /** + * Skips optimization of the entire {@code try} statement. + * + *

      An expression must not be factored out of a {@code try} statement: + * the initializer of the resulting static field runs during class + * initialization, outside the reach of the {@code catch} and + * {@code finally} handlers. For example, factoring a method call out of + * {@code try { return f(x); } catch (Exception e) { return null; }} + * (the shape generated for a safe cast) would make the exception escape + * as an {@code ExceptionInInitializerError} instead of yielding + * {@code null}. See + * [CALCITE-6753] + * DeterministicCodeOptimizer may lift method calls out of try-catch + * blocks. + * + * @param tryStatement statement to leave unchanged + * @return pass-through visitor + */ + @Override public Shuttle preVisit(TryStatement tryStatement) { + return PASS_THROUGH; + } + @Override public Expression visit(NewExpression newExpression, List arguments, @Nullable List memberDeclarations) { if (parent == null) { diff --git a/linq4j/src/test/java/org/apache/calcite/linq4j/test/DeterministicTest.java b/linq4j/src/test/java/org/apache/calcite/linq4j/test/DeterministicTest.java index 1ec9de8a608c..55b50d61fde9 100644 --- a/linq4j/src/test/java/org/apache/calcite/linq4j/test/DeterministicTest.java +++ b/linq4j/src/test/java/org/apache/calcite/linq4j/test/DeterministicTest.java @@ -511,6 +511,189 @@ private boolean isConstant(Expression e) { + "}\n")); } + /** Test case for + * [CALCITE-6753] + * DeterministicCodeOptimizer may lift method calls out of try-catch + * blocks. A deterministic method call must stay inside the try + * statement; a static field initializer would run outside the reach of + * the catch handler. */ + @Test void testMethodCallWithinTryCatchNotFactored() { + assertThat( + optimize( + Expressions.new_( + Runnable.class, + Collections.emptyList(), + Expressions.methodDecl( + 0, + int.class, + "test", + Collections.emptyList(), + Expressions.block( + Expressions.tryCatch( + Expressions.return_(null, + Expressions.call( + getMethod(Integer.class, "valueOf", + int.class), + Expressions.constant(0))), + Expressions.catch_( + Expressions.parameter(Exception.class, "e"), + Expressions.return_(null, + Expressions.constant(-1)))))))), + equalTo("{\n" + + " return new Runnable(){\n" + + " int test() {\n" + + " try {\n" + + " return Integer.valueOf(0);\n" + + " } catch (Exception e) {\n" + + " return -1;\n" + + " }\n" + + " }\n" + + "\n" + + " };\n" + + "}\n")); + } + + /** Expressions in a catch block are not factored out either, and factoring + * resumes for statements that follow the try statement. */ + @Test void testFactoringResumesAfterTryCatch() { + assertThat( + optimize( + Expressions.new_( + Runnable.class, + Collections.emptyList(), + Expressions.methodDecl( + 0, + int.class, + "test", + Collections.emptyList(), + Expressions.block( + Expressions.tryCatch( + Expressions.statement( + Expressions.call( + getMethod(Integer.class, "valueOf", + int.class), + Expressions.constant(0))), + Expressions.catch_( + Expressions.parameter(Exception.class, "e"), + Expressions.statement( + Expressions.call( + getMethod(Integer.class, "valueOf", + int.class), + Expressions.constant(1))))), + Expressions.return_(null, + Expressions.add(ONE, TWO)))))), + equalTo("{\n" + + " return new Runnable(){\n" + + " int test() {\n" + + " try {\n" + + " Integer.valueOf(0);\n" + + " } catch (Exception e) {\n" + + " Integer.valueOf(1);\n" + + " }\n" + + " return $L4J$C$1_2;\n" + + " }\n" + + "\n" + + " static final int $L4J$C$1_2 = 1 + 2;\n" + + " };\n" + + "}\n")); + } + + /** A try statement with only a finally block is not optimized either; + * moving an expression to a static field initializer would bypass the + * finally handler. */ + @Test void testExpressionWithinTryFinallyNotFactored() { + assertThat( + optimize( + Expressions.new_( + Runnable.class, + Collections.emptyList(), + Expressions.methodDecl( + 0, + int.class, + "test", + Collections.emptyList(), + Expressions.block( + Expressions.tryFinally( + Expressions.return_(null, + Expressions.call( + getMethod(Integer.class, "valueOf", + int.class), + Expressions.constant(0))), + Expressions.statement( + Expressions.add(ONE, TWO))))))), + equalTo("{\n" + + " return new Runnable(){\n" + + " int test() {\n" + + " try {\n" + + " return Integer.valueOf(0);\n" + + " } finally {\n" + + " 1 + 2;\n" + + " }\n" + + " }\n" + + "\n" + + " };\n" + + "}\n")); + } + + /** The optimizer must not add static fields to a class declared within a + * try statement. + * + *

      Factoring {@code 1 + 2} out to a static field of the {@code Callable} + * would move the evaluation into that field's initializer. The initializer + * still runs inside the try, when {@code new} first instantiates the + * class, but the JVM wraps anything a static initializer throws in an + * {@code ExceptionInInitializerError} (an {@code Error}, per JLS 12.4.2), + * which {@code catch (Exception e)} does not match. */ + @Test void testNestedClassWithinTryCatchNotFactored() { + assertThat( + optimize( + Expressions.new_( + Runnable.class, + Collections.emptyList(), + Expressions.methodDecl( + 0, + int.class, + "test", + Collections.emptyList(), + Expressions.block( + Expressions.tryCatch( + Expressions.return_(null, + Expressions.call( + Expressions.new_( + Callable.class, + Collections.emptyList(), + Expressions.methodDecl( + 0, + Object.class, + "call", + Collections.emptyList(), + Blocks.toFunctionBlock( + Expressions.add(ONE, TWO)))), + "call", + Collections.emptyList())), + Expressions.catch_( + Expressions.parameter(Exception.class, "e"), + Expressions.return_(null, + Expressions.constant(-1)))))))), + equalTo("{\n" + + " return new Runnable(){\n" + + " int test() {\n" + + " try {\n" + + " return new java.util.concurrent.Callable(){\n" + + " Object call() {\n" + + " return 1 + 2;\n" + + " }\n" + + "\n" + + " }.call();\n" + + " } catch (Exception e) {\n" + + " return -1;\n" + + " }\n" + + " }\n" + + "\n" + + " };\n" + + "}\n")); + } + @Test void testDeterministicClassNonDeterministicMethod() { assertThat( optimize( From be4aa17c21b82ac54a0a32c88706b4fcad99c03b Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Fri, 31 Jul 2026 10:51:16 -0700 Subject: [PATCH 438/562] [CALCITE-7683] SessionizationEnumerator produces wrong results for SESSION table function Signed-off-by: Mihai Budiu --- .../calcite/adapter/enumerable/EnumUtils.java | 34 +++++++++----- .../calcite/runtime/SortedMultiMap.java | 4 ++ core/src/test/resources/sql/stream.iq | 47 +++++++++++++++++++ 3 files changed, 73 insertions(+), 12 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java index d1c395c22606..9cd7ba802bdd 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java @@ -55,7 +55,6 @@ import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexProgramBuilder; import org.apache.calcite.runtime.PairList; -import org.apache.calcite.runtime.SortedMultiMap; import org.apache.calcite.runtime.SqlFunctions; import org.apache.calcite.runtime.Utilities; import org.apache.calcite.sql.SqlCollation; @@ -87,7 +86,9 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.NavigableMap; import java.util.TimeZone; +import java.util.TreeMap; import java.util.function.Function; import static org.apache.calcite.config.CalciteSystemProperty.JOIN_SELECTOR_COMPACT_CODE_THRESHOLD; @@ -1163,15 +1164,22 @@ private static class SessionizationEnumerator implements Enumerator<@Nullable Ob } @Override public @Nullable Object[] current() { - if (!initialized) { - initialize(); - initialized = true; - } return list.removeFirst(); } @Override public boolean moveNext() { - return initialized ? !list.isEmpty() : inputEnumerator.moveNext(); + // Sessionization needs to see all the input, so the first call consumes + // it entirely. Initializing here rather than in current() lets this + // method report that there is nothing to return, which happens when + // every row was discarded for having a NULL timestamp. + if (!initialized) { + initialized = true; + if (!inputEnumerator.moveNext()) { + return false; + } + initialize(); + } + return !list.isEmpty(); } @Override public void reset() { @@ -1196,24 +1204,26 @@ private void initialize() { elements.add(inputEnumerator.current()); } - Map<@Nullable Object, SortedMultiMap, @Nullable Object[]>> sessionKeyMap = - new HashMap<>(); + // The windows of each key are kept sorted by start time; the merge + // below only compares a window with the one that precedes it. + Map<@Nullable Object, NavigableMap, List<@Nullable Object[]>>> + sessionKeyMap = new HashMap<>(); for (@Nullable Object[] element : elements) { // A key column index of -1 means that there is no key; every element // then maps to the same (null) key, forming one session timeline. Object key = indexOfKeyColumn < 0 ? null : element[indexOfKeyColumn]; - SortedMultiMap, @Nullable Object[]> session = - sessionKeyMap.computeIfAbsent(key, k -> new SortedMultiMap<>()); Object watermark = requireNonNull(element[indexOfWatermarkedColumn], "element[indexOfWatermarkedColumn]"); + NavigableMap, List<@Nullable Object[]>> session = + sessionKeyMap.computeIfAbsent(key, k -> new TreeMap<>()); Pair initWindow = computeInitWindow(SqlFunctions.toLong(watermark), gap); - session.putMulti(initWindow, element); + session.computeIfAbsent(initWindow, k -> new ArrayList<>()).add(element); } // merge per key session windows if there is any overlap between windows. - for (Map.Entry<@Nullable Object, SortedMultiMap, @Nullable Object[]>> + for (Map.Entry<@Nullable Object, NavigableMap, List<@Nullable Object[]>>> perKeyEntry : sessionKeyMap.entrySet()) { Map, List<@Nullable Object[]>> finalWindowElementsMap = new HashMap<>(); Pair currentWindow = null; diff --git a/core/src/main/java/org/apache/calcite/runtime/SortedMultiMap.java b/core/src/main/java/org/apache/calcite/runtime/SortedMultiMap.java index 7b2448c5c94d..958d3849f9d5 100644 --- a/core/src/main/java/org/apache/calcite/runtime/SortedMultiMap.java +++ b/core/src/main/java/org/apache/calcite/runtime/SortedMultiMap.java @@ -28,6 +28,10 @@ * Map that allows you to partition values into lists according to a common * key, and then convert those lists into an iterator of sorted arrays. * + *

      Only the values are sorted, by {@link #arrays(Comparator)}; the keys are + * not, because this map extends {@link HashMap}. Use a {@link java.util.TreeMap} + * if you need to visit the keys in order. + * * @param Key type * @param Value type */ diff --git a/core/src/test/resources/sql/stream.iq b/core/src/test/resources/sql/stream.iq index b9df0692cc03..4328a63ccae2 100644 --- a/core/src/test/resources/sql/stream.iq +++ b/core/src/test/resources/sql/stream.iq @@ -327,3 +327,50 @@ SELECT * FROM TABLE( (5 rows) !ok + +# Test case for [CALCITE-7683] SessionizationEnumerator produces wrong results +# for SESSION table function. +# Here 10:05 and 10:30 are 25 minutes apart and 10:40 and 11:30 are 50 minutes +# apart, both more than the 15 minute gap, so the rows form three sessions. +SELECT * FROM TABLE( + SESSION( + (SELECT * FROM (VALUES + (TIMESTAMP '2020-01-01 10:00:00', 'a'), + (TIMESTAMP '2020-01-01 10:05:00', 'a'), + (TIMESTAMP '2020-01-01 10:30:00', 'a'), + (TIMESTAMP '2020-01-01 10:40:00', 'a'), + (TIMESTAMP '2020-01-01 11:30:00', 'a')) AS T(TS, UID)), + DESCRIPTOR(TS), DESCRIPTOR(UID), INTERVAL '15' MINUTE)) +ORDER BY TS; ++---------------------+-----+---------------------+---------------------+ +| TS | UID | window_start | window_end | ++---------------------+-----+---------------------+---------------------+ +| 2020-01-01 10:00:00 | a | 2020-01-01 10:00:00 | 2020-01-01 10:20:00 | +| 2020-01-01 10:05:00 | a | 2020-01-01 10:00:00 | 2020-01-01 10:20:00 | +| 2020-01-01 10:30:00 | a | 2020-01-01 10:30:00 | 2020-01-01 10:55:00 | +| 2020-01-01 10:40:00 | a | 2020-01-01 10:30:00 | 2020-01-01 10:55:00 | +| 2020-01-01 11:30:00 | a | 2020-01-01 11:30:00 | 2020-01-01 11:45:00 | ++---------------------+-----+---------------------+---------------------+ +(5 rows) + +!ok + +# Test case for [CALCITE-7683] SessionizationEnumerator produces wrong results +# for SESSION table function. +SELECT * FROM TABLE( + SESSION( + (SELECT * FROM (VALUES + (TIMESTAMP '2020-01-01 10:00:00', 'a'), + (TIMESTAMP '2020-01-01 10:05:00', 'a'), + (TIMESTAMP '2020-01-01 10:30:00', 'a')) AS T(TS, UID)), + DESCRIPTOR(TS), DESCRIPTOR(UID), INTERVAL '15' MINUTE)); ++---------------------+-----+---------------------+---------------------+ +| TS | UID | window_start | window_end | ++---------------------+-----+---------------------+---------------------+ +| 2020-01-01 10:00:00 | a | 2020-01-01 10:00:00 | 2020-01-01 10:20:00 | +| 2020-01-01 10:05:00 | a | 2020-01-01 10:00:00 | 2020-01-01 10:20:00 | +| 2020-01-01 10:30:00 | a | 2020-01-01 10:30:00 | 2020-01-01 10:45:00 | ++---------------------+-----+---------------------+---------------------+ +(3 rows) + +!ok From de64c09986520f929608156332e97b0786c6606f Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Wed, 5 Aug 2026 15:38:27 -0700 Subject: [PATCH 439/562] [CALCITE-7692] FLOOR/CEIL of INTERVAL produces wrong results Signed-off-by: Mihai Budiu --- .../sql2rel/StandardConvertletTable.java | 66 +++++++++++-------- .../calcite/test/SqlToRelConverterTest.java | 31 +++++++++ .../calcite/test/SqlToRelConverterTest.xml | 12 ++++ core/src/test/resources/sql/operator.iq | 41 ++++++++++++ site/_docs/reference.md | 8 +-- .../apache/calcite/test/SqlOperatorTest.java | 19 ++++-- 6 files changed, 135 insertions(+), 42 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql2rel/StandardConvertletTable.java b/core/src/main/java/org/apache/calcite/sql2rel/StandardConvertletTable.java index 92e0739624fe..bff49f3924ea 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/StandardConvertletTable.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/StandardConvertletTable.java @@ -830,36 +830,44 @@ protected RexNode convertCast( protected RexNode convertFloorCeil(SqlRexContext cx, SqlCall call) { final boolean floor = call.getKind() == SqlKind.FLOOR; final SqlParserPos pos = call.getParserPosition(); - // Rewrite floor, ceil of interval - if (call.operandCount() == 1 - && call.operand(0) instanceof SqlIntervalLiteral) { - final SqlIntervalLiteral literal = call.operand(0); - SqlIntervalLiteral.IntervalValue interval = - literal.getValueAs(SqlIntervalLiteral.IntervalValue.class); - BigDecimal val = - interval.getIntervalQualifier().getStartUnit().multiplier; - RexNode rexInterval = cx.convertExpression(literal); - + // Rewrite floor, ceil of an interval as arithmetic that rounds to a + // multiple of the interval's leading unit. + if (call.operandCount() == 1) { final RexBuilder rexBuilder = cx.getRexBuilder(); - RexNode zero = rexBuilder.makeExactLiteral(BigDecimal.valueOf(0)); - RexNode cond = ge(pos, rexBuilder, rexInterval, zero); - - RexNode pad = - rexBuilder.makeExactLiteral(val.subtract(BigDecimal.ONE)); - RexNode cast = - rexBuilder.makeReinterpretCast(pos, rexInterval.getType(), pad, - rexBuilder.makeLiteral(false)); - RexNode sum = - floor ? minus(pos, rexBuilder, rexInterval, cast) - : plus(pos, rexBuilder, rexInterval, cast); - - RexNode kase = floor - ? case_(rexBuilder, rexInterval, cond, sum) - : case_(rexBuilder, sum, cond, rexInterval); - - RexNode factor = rexBuilder.makeExactLiteral(val); - RexNode div = divideInt(pos, rexBuilder, kase, factor); - return multiply(pos, rexBuilder, div, factor); + final RexNode rexInterval = cx.convertExpression(call.operand(0)); + final SqlIntervalQualifier qualifier = + rexInterval.getType().getIntervalQualifier(); + if (qualifier != null) { + if (qualifier.timeFrameName != null) { + throw new UnsupportedOperationException((floor ? "FLOOR" : "CEIL") + + " of an interval with custom time frame '" + + qualifier.timeFrameName + "' is not supported"); + } + if (!RexUtil.isDeterministic(rexInterval)) { + throw new UnsupportedOperationException((floor ? "FLOOR" : "CEIL") + + " of a non-deterministic interval expression is not" + + " supported"); + } + BigDecimal val = qualifier.getStartUnit().multiplier; + RexNode zero = rexBuilder.makeExactLiteral(BigDecimal.valueOf(0)); + RexNode cond = ge(pos, rexBuilder, rexInterval, zero); + + RexNode pad = + rexBuilder.makeIntervalLiteral(val.subtract(BigDecimal.ONE), + qualifier); + RexNode sum = + floor ? minus(pos, rexBuilder, rexInterval, pad) + : plus(pos, rexBuilder, rexInterval, pad); + + // CASE operands are (when, then, else) + RexNode kase = floor + ? case_(rexBuilder, cond, rexInterval, sum) + : case_(rexBuilder, cond, sum, rexInterval); + + RexNode factor = rexBuilder.makeExactLiteral(val); + RexNode div = divideInt(pos, rexBuilder, kase, factor); + return multiply(pos, rexBuilder, div, factor); + } } // normal floor, ceil function diff --git a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java index 928c29850ac1..2f4ba8919429 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java @@ -81,6 +81,7 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.hasSize; +import static org.junit.jupiter.api.Assertions.assertThrows; /** * Unit test for {@link org.apache.calcite.sql2rel.SqlToRelConverter}. @@ -6304,6 +6305,36 @@ void checkUserDefinedOrderByOver(NullCollation nullCollation) { assertThat(plan, containsString("FLOOR($4, FLAG(WEEK))")); } + /** Test case for + * [CALCITE-7692] + * FLOOR/CEIL of INTERVAL produces wrong results. + * + *

      FLOOR and CEIL of an interval expression, literal or not, are rewritten + * as arithmetic that rounds to a multiple of the interval's leading unit. */ + @Test void testFloorCeilOfInterval() { + final String sql = "select floor(x) as f, ceil(x) as c\n" + + "from (values (interval '3:4:5' hour to second)) as t(x)"; + sql(sql).ok(); + } + + /** Test case for + * [CALCITE-7692] + * FLOOR/CEIL of INTERVAL produces wrong results. + * + *

      The rewrite evaluates its operand more than once, which is unsound + * for a non-deterministic operand; conversion must fail rather than + * produce incorrect results. */ + @Test void testFloorOfNonDeterministicInterval() { + final String sql = "select floor(x * rand()) as f\n" + + "from (values (interval '3:4:5' hour to second)) as t(x)"; + final UnsupportedOperationException e = + assertThrows(UnsupportedOperationException.class, + () -> sql(sql).toRel()); + assertThat(e.getMessage(), + is("FLOOR of a non-deterministic interval expression is not" + + " supported")); + } + /** Test case of * [CALCITE-5406] * Support the SELECT DISTINCT ON statement for PostgreSQL dialect. */ diff --git a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml index 2cb1a1093304..a8da739aa946 100644 --- a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml @@ -2624,6 +2624,18 @@ LogicalSort(fetch=[+(1, ABS(-2))]) + + + + + + + + =($0, 0), $0, -($0, 3599999)), 3600000), 3600000)], C=[*(/INT(CASE(>=($0, 0), +($0, 3599999), $0), 3600000), 3600000)]) + LogicalValues(tuples=[[{ 11045000 }]]) ]]> diff --git a/core/src/test/resources/sql/operator.iq b/core/src/test/resources/sql/operator.iq index 41a470e5f934..45bee30f0c96 100644 --- a/core/src/test/resources/sql/operator.iq +++ b/core/src/test/resources/sql/operator.iq @@ -842,4 +842,45 @@ SELECT !ok +# [CALCITE-7692] FLOOR/CEIL of INTERVAL produces wrong results +# FLOOR and CEIL of an interval round to the interval's leading unit, +# whether or not the operand is a literal. +select floor(x) = interval '3' hour as f, + ceil(x) = interval '4' hour as c +from (values (interval '3:4:5' hour to second)) as t(x); ++------+------+ +| F | C | ++------+------+ +| true | true | ++------+------+ +(1 row) + +!ok + +select floor(interval '-6.3' second) = interval '-7' second as fneg, + ceil(interval '-6.3' second) = interval '-6' second as cneg, + floor(interval '5-1' year to month) = interval '5' year as fym, + ceil(interval '-5-1' year to month) = interval '-5' year as cym; ++------+------+------+------+ +| FNEG | CNEG | FYM | CYM | ++------+------+------+------+ +| true | true | true | true | ++------+------+------+------+ +(1 row) + +!ok + +# The operand's interval type may be computed rather than declared; here +# HOUR + MINUTE yields INTERVAL HOUR TO MINUTE, whose leading unit is HOUR. +select floor(interval '2' hour + interval '90' minute) = interval '3' hour as fa, + ceil(interval '2' hour + interval '90' minute) = interval '4' hour as ca; ++------+------+ +| FA | CA | ++------+------+ +| true | true | ++------+------+ +(1 row) + +!ok + # End operator.iq diff --git a/site/_docs/reference.md b/site/_docs/reference.md index 4b198449ae08..fda50d5bfc99 100644 --- a/site/_docs/reference.md +++ b/site/_docs/reference.md @@ -1608,6 +1608,8 @@ Not implemented: | EXTRACT(timeUnit FROM datetime) | Extracts and returns the value of a specified datetime field from a datetime value expression | FLOOR(datetime TO timeUnit) | Rounds *datetime* down to *timeUnit* | CEIL(datetime TO timeUnit) | Rounds *datetime* up to *timeUnit* +| FLOOR(interval) | Rounds *interval* down to a multiple of its leading time unit; for example, `FLOOR(INTERVAL '3:04:05' HOUR TO SECOND)` returns `INTERVAL '3:00:00' HOUR TO SECOND` +| CEIL(interval) | Rounds *interval* up to a multiple of its leading time unit | YEAR(date) | Equivalent to `EXTRACT(YEAR FROM date)`. Returns an integer. | QUARTER(date) | Equivalent to `EXTRACT(QUARTER FROM date)`. Returns an integer between 1 and 4. | MONTH(date) | Equivalent to `EXTRACT(MONTH FROM date)`. Returns an integer between 1 and 12. @@ -1628,12 +1630,6 @@ standard SQL. Calls with parentheses, such as `CURRENT_DATE()` are accepted in c Not implemented: -* CEIL(interval) -* FLOOR(interval) -* \+ interval -* \- interval -* interval + interval -* interval - interval * interval / interval ### System functions diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java index f0ad6867faec..89a5ad1dadf6 100644 --- a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java +++ b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java @@ -14085,11 +14085,11 @@ private static void checkArrayConcatAggFuncFails(SqlOperatorFixture t) { f.checkNull("ceiling(cast(null as double))"); } + /** Test case for + * [CALCITE-7692] + * FLOOR/CEIL of INTERVAL produces wrong results. */ @Test void testCeilFuncInterval() { final SqlOperatorFixture f = fixture(); - if (!f.brokenTestsEnabled()) { - return; - } f.checkScalar("ceil(interval '3:4:5' hour to second)", "+4:00:00.000000", "INTERVAL HOUR TO SECOND NOT NULL"); f.checkScalar("ceil(interval '-6.3' second)", @@ -14318,11 +14318,11 @@ private static void checkArrayConcatAggFuncFails(SqlOperatorFixture t) { "-4", "INTEGER NOT NULL"); } + /** Test case for + * [CALCITE-7692] + * FLOOR/CEIL of INTERVAL produces wrong results. */ @Test void testFloorFuncInterval() { final SqlOperatorFixture f = fixture(); - if (!f.brokenTestsEnabled()) { - return; - } f.checkScalar("floor(interval '3:4:5' hour to second)", "+3:00:00.000000", "INTERVAL HOUR TO SECOND NOT NULL"); @@ -14332,6 +14332,12 @@ private static void checkArrayConcatAggFuncFails(SqlOperatorFixture t) { "+5-00", "INTERVAL YEAR TO MONTH NOT NULL"); f.checkScalar("floor(interval '-5-1' year to month)", "-6-00", "INTERVAL YEAR TO MONTH NOT NULL"); + f.checkNull("floor(cast(null as interval year))"); + if (!f.brokenTestsEnabled()) { + return; + } + // FLOOR(interval TO time unit) is not implemented; the validator accepts + // only DATE, TIME and TIMESTAMP before TO. f.checkScalar("floor(interval '-6.3' second to second)", "-7.000000", "INTERVAL SECOND NOT NULL"); f.checkScalar("floor(interval '6-3' minute to second to minute)", @@ -14348,7 +14354,6 @@ private static void checkArrayConcatAggFuncFails(SqlOperatorFixture t) { "201", "INTERVAL YEAR TO MONTH NOT NULL"); f.checkScalar("floor(interval '1004-1' year to month to millennium)", "2001-00", "INTERVAL YEAR TO MONTH NOT NULL"); - f.checkNull("floor(cast(null as interval year))"); } @Test void testTimestampAdd() { From 2da3b3d38f3910f0dbb18fa9a28f677ea3903052 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Wed, 5 Aug 2026 21:15:25 -0700 Subject: [PATCH 440/562] [CALCITE-5998] The SAFE_OFFSET operator can cause an index out of bounds exception Signed-off-by: Mihai Budiu --- .../calcite/test/SqlOperatorUnparseTest.java | 11 ---------- .../apache/calcite/test/SqlOperatorTest.java | 22 +++++++++++++++++++ 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/core/src/test/java/org/apache/calcite/test/SqlOperatorUnparseTest.java b/core/src/test/java/org/apache/calcite/test/SqlOperatorUnparseTest.java index b4e4e66769ad..f7c5f2e9a60d 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlOperatorUnparseTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlOperatorUnparseTest.java @@ -24,8 +24,6 @@ import org.apache.calcite.sql.test.SqlOperatorFixture; import org.apache.calcite.sql.test.SqlTestFactory; -import org.junit.jupiter.api.Disabled; - import java.util.function.Consumer; import java.util.function.UnaryOperator; @@ -105,13 +103,4 @@ String rewrite(StringAndPos sap) throws SqlParseException { } } } - - // Every test that is Disabled below corresponds to a bug. - // These tests should just be deleted when the corresponding bugs are fixed. - - @Override @Disabled("https://issues.apache.org/jira/browse/CALCITE-5998 " - + "The SAFE_OFFSET operator can cause an index out of bounds exception") - void testSafeOffsetOperator() { - super.testSafeOffsetOperator(); - } } diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java index 89a5ad1dadf6..7db6ab298db7 100644 --- a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java +++ b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java @@ -13851,6 +13851,17 @@ private static void checkArrayConcatAggFuncFails(SqlOperatorFixture t) { f.checkScalar("ARRAY[2,4,6][SAFE_OFFSET(-1)]", isNullValue(), "INTEGER"); f.checkScalar("ARRAY[2,4,6][SAFE_OFFSET(5)]", isNullValue(), "INTEGER"); f.checkNull("ARRAY[2,4,6][SAFE_OFFSET(null)]"); + // Test case for [CALCITE-5998] The SAFE_OFFSET operator can cause + // an index out of bounds exception; the index is not constant + f.check("select ARRAY[p3,p2,p1][SAFE_OFFSET(p0)]\n" + + "from (values (-1, 6, 4, 2)) as t(p0, p1, p2, p3)", + "INTEGER", isNullValue()); + f.check("select ARRAY[p3,p2,p1][SAFE_OFFSET(p1)]\n" + + "from (values (-1, 6, 4, 2)) as t(p0, p1, p2, p3)", + "INTEGER", isNullValue()); + f.check("select ARRAY[p3,p2,p1][SAFE_OFFSET(p3)]\n" + + "from (values (-1, 6, 4, 2)) as t(p0, p1, p2, p3)", + "INTEGER", 6); f.checkFails("^map['foo', 3, 'bar', 7][safe_offset('bar')]^", "Cannot apply 'SAFE_OFFSET' to arguments of type 'SAFE_OFFSET\\(>, \\)'\\. Supported form\\(s\\): " @@ -13869,6 +13880,17 @@ private static void checkArrayConcatAggFuncFails(SqlOperatorFixture t) { f.checkScalar("ARRAY[2,4,6][SAFE_ORDINAL(-1)]", isNullValue(), "INTEGER"); f.checkScalar("ARRAY[2,4,6][SAFE_ORDINAL(5)]", isNullValue(), "INTEGER"); f.checkNull("ARRAY[2,4,6][SAFE_ORDINAL(null)]"); + // Test case for [CALCITE-5998] The SAFE_OFFSET operator can cause + // an index out of bounds exception; the index is not constant + f.check("select ARRAY[p3,p2,p1][SAFE_ORDINAL(p0)]\n" + + "from (values (-1, 6, 4, 2)) as t(p0, p1, p2, p3)", + "INTEGER", isNullValue()); + f.check("select ARRAY[p3,p2,p1][SAFE_ORDINAL(p1)]\n" + + "from (values (-1, 6, 4, 2)) as t(p0, p1, p2, p3)", + "INTEGER", isNullValue()); + f.check("select ARRAY[p3,p2,p1][SAFE_ORDINAL(p3)]\n" + + "from (values (-1, 6, 4, 2)) as t(p0, p1, p2, p3)", + "INTEGER", 4); f.checkFails("^map['foo', 3, 'bar', 7][safe_ordinal('bar')]^", "Cannot apply 'SAFE_ORDINAL' to arguments of type 'SAFE_ORDINAL\\(>, \\)'\\. Supported form\\(s\\): " From c08787620168b14e3a2b6b6c0ab5acef9beb7e8c Mon Sep 17 00:00:00 2001 From: Kirill Tkalenko Date: Sun, 26 Jul 2026 07:06:42 +0300 Subject: [PATCH 441/562] [CALCITE-7662] Add expression support for OFFSET --- core/src/main/codegen/templates/Parser.jj | 25 ++- .../adapter/enumerable/EnumerableLimit.java | 10 +- .../enumerable/EnumerableLimitSort.java | 8 +- .../enumerable/EnumerableMergeUnionRule.java | 20 +- .../rel/metadata/RelMdMinRowCount.java | 6 +- .../calcite/rel/metadata/RelMdUtil.java | 5 +- .../rel/rel2sql/RelToSqlConverter.java | 11 +- .../rel/rules/SortJoinTransposeRule.java | 5 +- .../rel/rules/SortUnionTransposeRule.java | 24 ++- .../java/org/apache/calcite/rex/RexUtil.java | 69 ++++++- .../calcite/runtime/CalciteResource.java | 17 +- .../org/apache/calcite/sql/SqlDialect.java | 54 +++-- .../calcite/sql/dialect/MysqlSqlDialect.java | 10 +- .../calcite/sql/dialect/SqliteSqlDialect.java | 9 +- .../sql/validate/SqlValidatorImpl.java | 39 ++-- .../calcite/sql2rel/RelDecorrelator.java | 6 +- .../org/apache/calcite/tools/RelBuilder.java | 48 ++--- .../runtime/CalciteResource.properties | 7 +- .../rel/rel2sql/RelToSqlConverterTest.java | 71 +++++++ .../apache/calcite/rex/RexProgramTest.java | 14 ++ .../org/apache/calcite/test/JdbcTest.java | 184 +++++++++++++++-- .../apache/calcite/test/RelBuilderTest.java | 78 ++++++++ .../apache/calcite/test/RelMetadataTest.java | 31 +++ .../apache/calcite/test/RelOptRulesTest.java | 61 ++++++ .../calcite/test/SqlToRelConverterTest.java | 9 + .../apache/calcite/test/SqlValidatorTest.java | 27 ++- .../enumerable/EnumerableMergeUnionTest.java | 62 +++++- .../apache/calcite/test/RelOptRulesTest.xml | 117 +++++++++++ .../calcite/test/SqlToRelConverterTest.xml | 12 ++ core/src/test/resources/sql/offset.iq | 186 ++++++++++++++++++ .../org/apache/calcite/test/ServerTest.java | 25 +++ site/_docs/reference.md | 18 +- .../calcite/sql/parser/SqlParserTest.java | 21 +- 33 files changed, 1140 insertions(+), 149 deletions(-) create mode 100644 core/src/test/resources/sql/offset.iq diff --git a/core/src/main/codegen/templates/Parser.jj b/core/src/main/codegen/templates/Parser.jj index ce69124c4b5c..185a85f50836 100644 --- a/core/src/main/codegen/templates/Parser.jj +++ b/core/src/main/codegen/templates/Parser.jj @@ -689,13 +689,13 @@ SqlNode ExprOrJoinOrOrderedQuery(ExprContext exprContext) : * * 

        *    [ LIMIT { count | ALL } ]
- *    [ OFFSET start ]
      + * [ OFFSET { start | expression } ] *
      * *

      Trino syntax for limit: * * 

      - *    [ OFFSET start ]
+ *    [ OFFSET { start | expression } ]
  *    [ LIMIT { count | ALL } ]
      *
      * @@ -708,7 +708,7 @@ SqlNode ExprOrJoinOrOrderedQuery(ExprContext exprContext) : *

      SQL:2008 syntax for limit: * * 

      - *    [ OFFSET start { ROW | ROWS } ]
+ *    [ OFFSET { start | expression } { ROW | ROWS } ]
  *    [ FETCH { FIRST | NEXT } [ count | (expression) ] { ROW | ROWS } ONLY ]
      *
      */ @@ -783,7 +783,7 @@ void OffsetClause(Span s, SqlNode[] offsetFetch) : // ROW or ROWS is required in SQL:2008 but we make it optional // because it is not present in Postgres-style syntax. { s.add(this); } - offsetFetch[0] = UnsignedNumericLiteralOrParam() + offsetFetch[0] = OffsetCount() [ | ] } @@ -800,6 +800,23 @@ void FetchClause(SqlNode[] offsetFetch) : ( | ) } +/** + * Parses the start value or expression of an OFFSET clause. + */ +SqlNode OffsetCount() : +{ + final SqlNode e; +} +{ + // Unlike FETCH expressions, OFFSET expressions do not require parentheses + // and may start with a numeric literal or dynamic parameter. Therefore, a + // separate UnsignedNumericLiteralOrParam alternative would consume only + // the prefix of expressions such as "OFFSET 1 + 2" or "OFFSET ? + 1". + // Expression also covers standalone start values. + e = Expression(ExprContext.ACCEPT_NON_QUERY) + { return e; } +} + /** * Parses the row count of a FETCH clause. Expressions must be parenthesized. */ diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimit.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimit.java index de1f94d562d5..96a04be1b862 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimit.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimit.java @@ -107,14 +107,14 @@ public static EnumerableLimit create(final RelNode input, @Nullable RexNode offs builder.append("offset", Expressions.call(BuiltInMethod.SKIP_BIG_DECIMAL.method, v, getExpression(offset, "OFFSET", implementor, builder, - roundingPolicyExp, false))); + roundingPolicyExp))); } if (fetch != null) { v = builder.append("fetch", Expressions.call(BuiltInMethod.TAKE_BIG_DECIMAL.method, v, getExpression(fetch, "FETCH", implementor, builder, - roundingPolicyExp, true))); + roundingPolicyExp))); } builder.add(Expressions.return_(null, v)); @@ -123,7 +123,7 @@ public static EnumerableLimit create(final RelNode input, @Nullable RexNode offs static Expression getExpression(RexNode rexNode, String kind, EnumerableRelImplementor implementor, BlockBuilder builder, - Expression roundingPolicy, boolean translateExpression) { + Expression roundingPolicy) { final Expression value; if (rexNode instanceof RexDynamicParam) { final RexDynamicParam param = (RexDynamicParam) rexNode; @@ -134,10 +134,6 @@ static Expression getExpression(RexNode rexNode, String kind, } else if (rexNode instanceof RexLiteral) { value = Expressions.constant(RexLiteral.bigDecimalValue(rexNode)); } else { - if (!translateExpression) { - throw new IllegalArgumentException(kind + " must be a literal or dynamic parameter"); - } - value = RexToLixTranslator.forAggregation(implementor.getTypeFactory(), builder, null, implementor.getConformance()) diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimitSort.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimitSort.java index 97d9fd8169b7..68759436d0fd 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimitSort.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimitSort.java @@ -105,7 +105,7 @@ public static EnumerableLimitSort create( fetchVal = Expressions.constant(BigDecimal.valueOf(Integer.MAX_VALUE)); } else { fetchVal = - getExpression(this.fetch, "FETCH", implementor, builder, roundingPolicyExp, true); + getExpression(this.fetch, "FETCH", implementor, builder, roundingPolicyExp); } final Expression offsetVal; @@ -113,7 +113,7 @@ public static EnumerableLimitSort create( offsetVal = Expressions.constant(BigDecimal.ZERO); } else { offsetVal = - getExpression(this.offset, "OFFSET", implementor, builder, roundingPolicyExp, false); + getExpression(this.offset, "OFFSET", implementor, builder, roundingPolicyExp); } builder.add( @@ -125,10 +125,10 @@ public static EnumerableLimitSort create( builder.appendIfNotNull("comparator", pair.right)) .appendIfNotNull( builder.appendIfNotNull("offset", - Expressions.constant(offsetVal))) + offsetVal)) .appendIfNotNull( builder.appendIfNotNull("fetch", - Expressions.constant(fetchVal)))))); + fetchVal))))); return implementor.result(physType, builder.toBlock()); } } diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeUnionRule.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeUnionRule.java index 57f864794aa9..3f2bdc5fe1f9 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeUnionRule.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeUnionRule.java @@ -27,7 +27,6 @@ import org.apache.calcite.rel.logical.LogicalUnion; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeField; -import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexUtil; import org.apache.calcite.tools.RelBuilder; @@ -90,15 +89,16 @@ public EnumerableMergeUnionRule(Config config) { RexNode inputFetch = null; if (sort.fetch != null) { final boolean safeToReevaluate = - RexUtil.isDeterministic(sort.fetch); - if (sort.offset == null && safeToReevaluate) { - inputFetch = sort.fetch; - } else if (safeToReevaluate - && sort.fetch instanceof RexLiteral - && sort.offset instanceof RexLiteral) { - inputFetch = - call.builder().literal(RexLiteral.bigDecimalValue(sort.fetch) - .add(RexLiteral.bigDecimalValue(sort.offset))); + RexUtil.isDeterministic(sort.fetch) + && (sort.offset == null || RexUtil.isDeterministic(sort.offset)); + if (safeToReevaluate) { + if (sort.offset == null) { + inputFetch = sort.fetch; + } else { + inputFetch = + RexUtil.makeOffsetFetchSum( + sort.getCluster().getRexBuilder(), sort.offset, sort.fetch); + } } } diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdMinRowCount.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdMinRowCount.java index 2cb710f39808..298e77f2d8e4 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdMinRowCount.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdMinRowCount.java @@ -117,7 +117,8 @@ public Double getMinRowCount(Sort rel, RelMetadataQuery mq) { } final double offset = - literalValueApproximatedByDouble(rel.offset, 0D); + literalValueApproximatedByDouble(rel.offset, + rel.offset == null ? 0D : rowCount); rowCount = Math.max(rowCount - offset, 0D); final double limit = @@ -133,7 +134,8 @@ public Double getMinRowCount(EnumerableLimit rel, RelMetadataQuery mq) { } final double offset = - literalValueApproximatedByDouble(rel.offset, 0D); + literalValueApproximatedByDouble(rel.offset, + rel.offset == null ? 0D : rowCount); rowCount = Math.max(rowCount - offset, 0D); final double limit = diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdUtil.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdUtil.java index 5b096289382e..ab7dab016b4b 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdUtil.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdUtil.java @@ -29,7 +29,6 @@ import org.apache.calcite.rel.core.Union; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexCall; -import org.apache.calcite.rex.RexDynamicParam; import org.apache.calcite.rex.RexInputRef; import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexLocalRef; @@ -1057,7 +1056,9 @@ private static boolean alreadySmaller(RelMetadataQuery mq, RelNode input, } } final Double rowCount = mq.getMaxRowCount(input); - if (rowCount == null || offset instanceof RexDynamicParam || !(fetch instanceof RexLiteral)) { + if (rowCount == null + || (offset != null && !(offset instanceof RexLiteral)) + || !(fetch instanceof RexLiteral)) { // Cannot be determined return false; } diff --git a/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java b/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java index 2f96e8f73b20..a5b8858a3a09 100644 --- a/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java +++ b/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java @@ -1328,7 +1328,7 @@ public Result visit(Sort e) { SqlNodeList sortExps = exprList(builder.context, e.getSortExps()); sqlSelect.setOrderBy(sortExps); if (e.offset != null) { - SqlNode offset = builder.context.toSql(null, e.offset); + SqlNode offset = toSqlOffset(e, builder.context); sqlSelect.setOffset(offset); } if (e.fetch != null) { @@ -1393,10 +1393,17 @@ void offsetFetch(Sort e, Builder builder) { builder.setFetch(toSqlFetch(e, builder.context)); } if (e.offset != null) { - builder.setOffset(builder.context.toSql(null, e.offset)); + builder.setOffset(toSqlOffset(e, builder.context)); } } + private static SqlNode toSqlOffset(Sort sort, Context context) { + final RexNode offset = requireNonNull(sort.offset, "offset"); + final @Nullable RexLiteral reduced = + RexUtil.reduceOffsetToLiteral(sort.getCluster(), offset); + return context.toSql(null, reduced == null ? offset : reduced); + } + private static SqlNode toSqlFetch(Sort sort, Context context) { final RexNode fetch = requireNonNull(sort.fetch, "fetch"); final @Nullable RexLiteral reduced = diff --git a/core/src/main/java/org/apache/calcite/rel/rules/SortJoinTransposeRule.java b/core/src/main/java/org/apache/calcite/rel/rules/SortJoinTransposeRule.java index df967e56aad6..dfb2d8401d7c 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/SortJoinTransposeRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/SortJoinTransposeRule.java @@ -30,7 +30,6 @@ import org.apache.calcite.rel.metadata.RelMdUtil; import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.rex.RexBuilder; -import org.apache.calcite.rex.RexDynamicParam; import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; import org.apache.calcite.tools.RelBuilderFactory; @@ -106,8 +105,8 @@ public SortJoinTransposeRule(Class sortClass, final Join join = call.rel(1); // The pushed fetch is calculated from literal offset and fetch values. - if (sort.offset instanceof RexDynamicParam - || sort.fetch != null && !(sort.fetch instanceof RexLiteral)) { + if ((sort.offset != null && !(sort.offset instanceof RexLiteral)) + || (sort.fetch != null && !(sort.fetch instanceof RexLiteral))) { return false; } diff --git a/core/src/main/java/org/apache/calcite/rel/rules/SortUnionTransposeRule.java b/core/src/main/java/org/apache/calcite/rel/rules/SortUnionTransposeRule.java index 93b6af657c43..21ec54ec60fb 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/SortUnionTransposeRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/SortUnionTransposeRule.java @@ -23,6 +23,7 @@ import org.apache.calcite.rel.core.Union; import org.apache.calcite.rel.metadata.RelMdUtil; import org.apache.calcite.rel.metadata.RelMetadataQuery; +import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexUtil; import org.apache.calcite.tools.RelBuilderFactory; @@ -67,12 +68,14 @@ public SortUnionTransposeRule( @Override public boolean matches(RelOptRuleCall call) { final Sort sort = call.rel(0); final Union union = call.rel(1); - // Re-evaluating a non-deterministic FETCH in every branch can produce a - // different limit from the top Sort. + // Re-evaluating a non-deterministic OFFSET or FETCH in every branch can + // produce a different limit from the top Sort. // There is a flag indicating if this rule should be applied when // Sort.fetch is null. return union.all - && sort.offset == null + && (sort.offset == null + || sort.fetch != null + && RexUtil.isDeterministic(sort.offset)) && (sort.fetch == null || RexUtil.isDeterministic(sort.fetch)) && (config.matchNullFetch() || sort.fetch != null); @@ -81,6 +84,17 @@ public SortUnionTransposeRule( @Override public void onMatch(RelOptRuleCall call) { final Sort sort = call.rel(0); final Union union = call.rel(1); + // OFFSET cannot be pushed into each input independently. However, only + // the first OFFSET + FETCH rows of an input can contribute to the final + // result, so use that value as the input FETCH and retain the original + // OFFSET and FETCH in the top Sort. + final RexNode inputFetch; + if (sort.fetch == null || sort.offset == null) { + inputFetch = sort.fetch; + } else { + inputFetch = + RexUtil.makeOffsetFetchSum(sort.getCluster().getRexBuilder(), sort.offset, sort.fetch); + } List inputs = new ArrayList<>(); // Thus we use 'ret' as a flag to identify if we have finished pushing the // sort past a union. @@ -88,11 +102,11 @@ public SortUnionTransposeRule( final RelMetadataQuery mq = call.getMetadataQuery(); for (RelNode input : union.getInputs()) { if (!RelMdUtil.checkInputForCollationAndLimit(mq, input, - sort.getCollation(), sort.offset, sort.fetch)) { + sort.getCollation(), null, inputFetch)) { ret = false; Sort branchSort = sort.copy(sort.getTraitSet(), input, - sort.getCollation(), sort.offset, sort.fetch); + sort.getCollation(), null, inputFetch); inputs.add(branchSort); } else { inputs.add(input); diff --git a/core/src/main/java/org/apache/calcite/rex/RexUtil.java b/core/src/main/java/org/apache/calcite/rex/RexUtil.java index b592093a5aef..b7c30b7608f9 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexUtil.java +++ b/core/src/main/java/org/apache/calcite/rex/RexUtil.java @@ -66,6 +66,7 @@ import org.checkerframework.checker.nullness.qual.Nullable; import java.math.BigDecimal; +import java.math.RoundingMode; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -881,12 +882,49 @@ public static boolean containsDynamicParam(RexNode e) { /** Converts a FETCH expression result to its validated canonical representation. */ public static BigDecimal validateFetchValue(@Nullable Number value) { + return validateOffsetFetchValue(value, "FETCH"); + } + + /** Creates the FETCH needed when OFFSET and FETCH are pushed into an input. + * + *

      Enumerable execution rounds OFFSET and FETCH independently to whole + * row counts. Therefore, the input must fetch + * {@code CEIL(offset) + CEIL(fetch)} rows rather than + * {@code CEIL(offset + fetch)} rows. The latter can be one row smaller when + * both values have a fractional part. */ + public static RexNode makeOffsetFetchSum(RexBuilder rexBuilder, + RexNode offset, RexNode fetch) { + if (offset instanceof RexLiteral && fetch instanceof RexLiteral) { + return rexBuilder.makeExactLiteral( + RexLiteral.bigDecimalValue(offset).setScale(0, RoundingMode.CEILING) + .add(RexLiteral.bigDecimalValue(fetch) + .setScale(0, RoundingMode.CEILING))); + } + return rexBuilder.makeCall(SqlStdOperatorTable.PLUS, + ceil(rexBuilder, offset), ceil(rexBuilder, fetch)); + } + + private static RexNode ceil(RexBuilder rexBuilder, RexNode node) { + if (node instanceof RexLiteral) { + return rexBuilder.makeExactLiteral( + RexLiteral.bigDecimalValue(node).setScale(0, RoundingMode.CEILING)); + } + return rexBuilder.makeCall(SqlStdOperatorTable.CEIL, node); + } + + /** Converts an OFFSET expression result to its validated canonical representation. */ + public static BigDecimal validateOffsetValue(@Nullable Number value) { + return validateOffsetFetchValue(value, "OFFSET"); + } + + private static BigDecimal validateOffsetFetchValue(@Nullable Number value, + String kind) { if (value == null) { - throw new IllegalArgumentException("FETCH expression evaluated to NULL"); + throw new IllegalArgumentException(kind + " expression evaluated to NULL"); } final BigDecimal decimal = NumberUtil.toBigDecimal(value); if (decimal.signum() < 0) { - throw new IllegalArgumentException("FETCH value " + value + throw new IllegalArgumentException(kind + " value " + value + " is out of range; expected a non-negative value"); } return decimal; @@ -895,28 +933,39 @@ public static BigDecimal validateFetchValue(@Nullable Number value) { /** Reduces a constant FETCH expression to a validated literal. */ public static @Nullable RexLiteral reduceFetchToLiteral( RelOptCluster cluster, RexNode fetch) { + return reduceOffsetFetchToLiteral(cluster, fetch, "FETCH"); + } + + /** Reduces a constant OFFSET expression to a validated literal. */ + public static @Nullable RexLiteral reduceOffsetToLiteral( + RelOptCluster cluster, RexNode offset) { + return reduceOffsetFetchToLiteral(cluster, offset, "OFFSET"); + } + + private static @Nullable RexLiteral reduceOffsetFetchToLiteral( + RelOptCluster cluster, RexNode node, String kind) { final RexLiteral literal; - if (fetch instanceof RexLiteral) { - literal = (RexLiteral) fetch; + if (node instanceof RexLiteral) { + literal = (RexLiteral) node; } else { - if (!isConstant(fetch) - || !isDeterministic(fetch) - || containsDynamicFunction(fetch) - || containsDynamicParam(fetch)) { + if (!isConstant(node) + || !isDeterministic(node) + || containsDynamicFunction(node) + || containsDynamicParam(node)) { return null; } final RexExecutor executor = Util.first(cluster.getPlanner().getExecutor(), EXECUTOR); final List reducedValues = new ArrayList<>(1); executor.reduce(cluster.getRexBuilder(), - Collections.singletonList(fetch), reducedValues); + Collections.singletonList(node), reducedValues); final RexNode reduced = reducedValues.get(0); if (!(reduced instanceof RexLiteral)) { return null; } literal = (RexLiteral) reduced; } - validateFetchValue(literal.getValueAs(Number.class)); + validateOffsetFetchValue(literal.getValueAs(Number.class), kind); return literal; } diff --git a/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java b/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java index 452c2bf84a7e..2e5056da1d7b 100644 --- a/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java +++ b/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java @@ -164,14 +164,19 @@ ExInstWithCause validatorContext(int a0, int a1, @BaseMessage("Values passed to {0} operator must have compatible types") ExInst incompatibleValueType(String a0); - @BaseMessage("FETCH expression must have a numeric type; actual type is ''{0}''") - ExInst fetchExpressionMustBeNumeric(String type); + @BaseMessage("{0} expression must have a numeric type; actual type is ''{1}''") + ExInst offsetFetchExpressionMustBeNumeric(String kind, + String type); - @BaseMessage("FETCH expression cannot reference table column ''{0}''") - ExInst fetchExpressionCannotReferenceColumn(String column); + @BaseMessage("{0} expression cannot reference table column ''{1}''") + ExInst offsetFetchExpressionCannotReferenceColumn( + String kind, String column); - @BaseMessage("FETCH expression evaluated to NULL") - ExInst fetchExpressionEvaluatedToNull(); + @BaseMessage("{0} expression evaluated to NULL") + ExInst offsetFetchExpressionEvaluatedToNull(String kind); + + @BaseMessage("{0} must not be negative") + ExInst offsetFetchValueMustNotBeNegative(String kind); @BaseMessage("Values in expression list must have compatible types") ExInst incompatibleTypesInList(); diff --git a/core/src/main/java/org/apache/calcite/sql/SqlDialect.java b/core/src/main/java/org/apache/calcite/sql/SqlDialect.java index 164f212c6c7a..eef1c386d594 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlDialect.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlDialect.java @@ -1078,7 +1078,7 @@ protected static void unparseFetchUsingAnsi(SqlWriter writer, @Nullable SqlNode final SqlWriter.Frame offsetFrame = writer.startList(SqlWriter.FrameTypeEnum.OFFSET); writer.keyword("OFFSET"); - offset.unparse(writer, -1, -1); + unparseOffsetExpression(writer, offset); writer.keyword("ROWS"); writer.endList(offsetFrame); } @@ -1088,24 +1088,36 @@ protected static void unparseFetchUsingAnsi(SqlWriter writer, @Nullable SqlNode writer.startList(SqlWriter.FrameTypeEnum.FETCH); writer.keyword("FETCH"); writer.keyword("NEXT"); - if (fetch instanceof SqlLiteral - || fetch instanceof SqlDynamicParam) { - fetch.unparse(writer, -1, -1); - } else { - final SqlWriter.Frame expressionFrame = writer.startList("(", ")"); - if (fetch instanceof SqlCall) { - writer.getDialect().unparseCall(writer, (SqlCall) fetch, 0, 0); - } else { - fetch.unparse(writer, 0, 0); - } - writer.endList(expressionFrame); - } + unparseFetchExpression(writer, fetch); writer.keyword("ROWS"); writer.keyword("ONLY"); writer.endList(fetchFrame); } } + private static void unparseOffsetExpression(SqlWriter writer, SqlNode offset) { + unparseExpression(writer, offset); + } + + private static void unparseFetchExpression(SqlWriter writer, SqlNode fetch) { + if (fetch instanceof SqlLiteral + || fetch instanceof SqlDynamicParam) { + fetch.unparse(writer, -1, -1); + return; + } + final SqlWriter.Frame expressionFrame = writer.startList("(", ")"); + unparseExpression(writer, fetch); + writer.endList(expressionFrame); + } + + private static void unparseExpression(SqlWriter writer, SqlNode node) { + if (node instanceof SqlCall) { + writer.getDialect().unparseCall(writer, (SqlCall) node, 0, 0); + } else { + node.unparse(writer, 0, 0); + } + } + /** Unparses offset/fetch using "LIMIT fetch OFFSET offset" syntax. */ protected static void unparseFetchUsingLimit(SqlWriter writer, @Nullable SqlNode offset, @Nullable SqlNode fetch) { @@ -1113,12 +1125,12 @@ protected static void unparseFetchUsingLimit(SqlWriter writer, @Nullable SqlNode } /** Unparses offset/fetch using "LIMIT fetch OFFSET offset" syntax, - * optionally allowing a scalar expression as fetch. */ + * optionally allowing scalar expressions as fetch and offset. */ protected static void unparseFetchUsingLimit(SqlWriter writer, @Nullable SqlNode offset, @Nullable SqlNode fetch, boolean allowExpression) { checkArgument(fetch != null || offset != null); unparseLimit(writer, fetch, allowExpression); - unparseOffset(writer, offset); + unparseOffset(writer, offset, allowExpression); } protected static void unparseLimit(SqlWriter writer, @Nullable SqlNode fetch) { @@ -1145,7 +1157,19 @@ private static void unparseLimit(SqlWriter writer, @Nullable SqlNode fetch, } protected static void unparseOffset(SqlWriter writer, @Nullable SqlNode offset) { + unparseOffset(writer, offset, false); + } + + private static void unparseOffset(SqlWriter writer, @Nullable SqlNode offset, + boolean allowExpression) { if (offset != null) { + if (!allowExpression + && !(offset instanceof SqlLiteral) + && !(offset instanceof SqlDynamicParam)) { + throw new IllegalArgumentException( + "LIMIT dialect does not support OFFSET expressions that cannot " + + "be reduced to a literal"); + } writer.newlineAndIndent(); final SqlWriter.Frame offsetFrame = writer.startList(SqlWriter.FrameTypeEnum.OFFSET); diff --git a/core/src/main/java/org/apache/calcite/sql/dialect/MysqlSqlDialect.java b/core/src/main/java/org/apache/calcite/sql/dialect/MysqlSqlDialect.java index 03d4d2c50454..1f114a155331 100644 --- a/core/src/main/java/org/apache/calcite/sql/dialect/MysqlSqlDialect.java +++ b/core/src/main/java/org/apache/calcite/sql/dialect/MysqlSqlDialect.java @@ -147,7 +147,15 @@ public MysqlSqlDialect(Context context) { @Override public void unparseOffsetFetch(SqlWriter writer, @Nullable SqlNode offset, @Nullable SqlNode fetch) { - unparseFetchUsingLimit(writer, offset, fetch); + if (offset != null && fetch == null) { + // MySQL has no OFFSET-only syntax. Its documented unlimited-row form + // uses the maximum unsigned BIGINT value as LIMIT. + final SqlNode unlimited = + SqlLiteral.createExactNumeric("18446744073709551615", SqlParserPos.ZERO); + unparseFetchUsingLimit(writer, offset, unlimited); + } else { + unparseFetchUsingLimit(writer, offset, fetch); + } } @Override public @Nullable SqlNode emulateNullDirection(SqlNode node, diff --git a/core/src/main/java/org/apache/calcite/sql/dialect/SqliteSqlDialect.java b/core/src/main/java/org/apache/calcite/sql/dialect/SqliteSqlDialect.java index f31276413600..833ac5c10d96 100644 --- a/core/src/main/java/org/apache/calcite/sql/dialect/SqliteSqlDialect.java +++ b/core/src/main/java/org/apache/calcite/sql/dialect/SqliteSqlDialect.java @@ -90,7 +90,14 @@ public SqliteSqlDialect(SqlDialect.Context context) { @Override public void unparseOffsetFetch(SqlWriter writer, @Nullable SqlNode offset, @Nullable SqlNode fetch) { - unparseFetchUsingLimit(writer, offset, fetch, true); + if (offset != null && fetch == null) { + // SQLite has no OFFSET-only syntax. LIMIT -1 means no upper bound. + final SqlNode unlimited = + SqlLiteral.createExactNumeric("-1", SqlParserPos.ZERO); + unparseFetchUsingLimit(writer, offset, unlimited, true); + } else { + unparseFetchUsingLimit(writer, offset, fetch, true); + } } @Override public void unparseCall(SqlWriter writer, SqlCall call, diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java index 0ea01a351451..8a5d3dfe9c39 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java @@ -70,6 +70,7 @@ import org.apache.calcite.sql.SqlMerge; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlNodeList; +import org.apache.calcite.sql.SqlNumericLiteral; import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.SqlOperatorTable; import org.apache.calcite.sql.SqlOrderBy; @@ -1771,31 +1772,40 @@ private void handleOffsetFetch(@Nullable SqlNode offset, @Nullable SqlNode fetch } } - private void validateFetchExpression(@Nullable SqlNode fetch) { - if (fetch == null || fetch instanceof SqlDynamicParam) { + private void validateOffsetFetchExpression(@Nullable SqlNode node, + String kind) { + if (node == null || node instanceof SqlDynamicParam) { return; } - if (SqlUtil.isNullLiteral(fetch, true)) { - throw newValidationError(fetch, - RESOURCE.fetchExpressionEvaluatedToNull()); + if (SqlUtil.isNullLiteral(node, true)) { + throw newValidationError(node, + RESOURCE.offsetFetchExpressionEvaluatedToNull(kind)); + } + if (node instanceof SqlNumericLiteral + && requireNonNull(((SqlNumericLiteral) node).bigDecimalValue()) + .signum() < 0) { + throw newValidationError(node, + RESOURCE.offsetFetchValueMustNotBeNegative(kind)); } - validateNoAggs(aggOrOverFinder, fetch, "FETCH"); - fetch.accept(new SqlBasicVisitor() { + validateNoAggs(aggOrOverFinder, node, kind); + node.accept(new SqlBasicVisitor() { @Override public Void visit(SqlIdentifier id) { if (makeNullaryCall(id) != null) { return null; } throw newValidationError(id, - RESOURCE.fetchExpressionCannotReferenceColumn(id.toString())); + RESOURCE.offsetFetchExpressionCannotReferenceColumn(kind, + id.toString())); } }); final SqlValidatorScope scope = getEmptyScope(); - inferUnknownTypes(typeFactory.createSqlType(SqlTypeName.DECIMAL), scope, fetch); - validateExpr(fetch, scope); - final RelDataType type = getValidatedNodeType(fetch); + inferUnknownTypes(typeFactory.createSqlType(SqlTypeName.DECIMAL), scope, node); + validateExpr(node, scope); + final RelDataType type = getValidatedNodeType(node); if (!SqlTypeUtil.isNumeric(type)) { - throw newValidationError(fetch, - RESOURCE.fetchExpressionMustBeNumeric(type.getFullTypeString())); + throw newValidationError(node, + RESOURCE.offsetFetchExpressionMustBeNumeric(kind, + type.getFullTypeString())); } } @@ -4527,7 +4537,8 @@ protected void validateSelect( validateWindowClause(select); validateQualifyClause(select); handleOffsetFetch(select.getOffset(), select.getFetch()); - validateFetchExpression(select.getFetch()); + validateOffsetFetchExpression(select.getOffset(), "OFFSET"); + validateOffsetFetchExpression(select.getFetch(), "FETCH"); // Validate the SELECT clause late, because a select item might // depend on the GROUP BY list, or the window function might reference diff --git a/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java b/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java index 4e4104ad4876..daabe37b89d5 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java @@ -1142,10 +1142,14 @@ private static void shiftMapping(Map mapping, int startIndex, } static boolean canDecorrelateOffsetFetch(Sort sort) { + final @Nullable RexLiteral offset = sort.offset == null + ? null + : RexUtil.reduceOffsetToLiteral(sort.getCluster(), sort.offset); final @Nullable RexLiteral fetch = sort.fetch == null ? null : RexUtil.reduceFetchToLiteral(sort.getCluster(), sort.fetch); - return isNonNegativeIntegralLiteral(sort.offset) + return (sort.offset == null + || offset != null && isNonNegativeIntegralLiteral(offset)) && (sort.fetch == null || fetch != null && isNonNegativeIntegralLiteral(fetch)); } diff --git a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java index 2309102ff826..92c5a4141583 100644 --- a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java +++ b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java @@ -3801,29 +3801,14 @@ public RelBuilder sortLimit(Number offset, Number fetch, /** Creates a {@link Sort} by a list of expressions, with limitNode and offsetNode. * - * @param offsetNode RexLiteral means number of rows to skip is deterministic, - * RexDynamicParam means number of rows to skip is dynamic. + * @param offsetNode Number of rows to skip * @param fetchNode Maximum number of rows to fetch * @param nodes Sort expressions */ public RelBuilder sortLimit(@Nullable RexNode offsetNode, @Nullable RexNode fetchNode, Iterable nodes) { - if (offsetNode != null) { - if (!(offsetNode instanceof RexLiteral || offsetNode instanceof RexDynamicParam)) { - throw new IllegalArgumentException("OFFSET node must be RexLiteral or RexDynamicParam"); - } - } - if (fetchNode != null && !isValidFetchExpression(fetchNode)) { - throw new IllegalArgumentException( - "FETCH node must not reference input fields or contain aggregate functions, " - + "window functions, or subqueries"); - } - if (fetchNode != null - && !SqlTypeUtil.isNumeric(fetchNode.getType())) { - throw new IllegalArgumentException( - "FETCH node must have a numeric type; actual type is " - + fetchNode.getType().getFullTypeString()); - } + validateOffsetFetchExpression(offsetNode, "OFFSET"); + validateOffsetFetchExpression(fetchNode, "FETCH"); final Registrar registrar = new Registrar(fields(), ImmutableList.of()); final List fieldCollations = registrar.registerFieldCollations(nodes); @@ -3890,14 +3875,31 @@ public RelBuilder sortLimit(@Nullable RexNode offsetNode, @Nullable RexNode fetc return this; } - private static boolean isValidFetchExpression(RexNode node) { - return Boolean.TRUE.equals(node.accept(new FetchExpressionVisitor())); + private static void validateOffsetFetchExpression(@Nullable RexNode node, + String kind) { + if (node == null) { + return; + } + if (!isValidOffsetFetchExpression(node)) { + throw new IllegalArgumentException( + kind + " node must not reference input fields or contain aggregate functions, " + + "window functions, or subqueries"); + } + if (!SqlTypeUtil.isNumeric(node.getType())) { + throw new IllegalArgumentException( + kind + " node must have a numeric type; actual type is " + + node.getType().getFullTypeString()); + } + } + + private static boolean isValidOffsetFetchExpression(RexNode node) { + return Boolean.TRUE.equals(node.accept(new OffsetFetchExpressionVisitor())); } - /** Visitor that validates FETCH expressions. */ - private static class FetchExpressionVisitor + /** Visitor that validates OFFSET and FETCH expressions. */ + private static class OffsetFetchExpressionVisitor extends RexVisitorImpl<@Nullable Boolean> { - FetchExpressionVisitor() { + OffsetFetchExpressionVisitor() { super(false); } diff --git a/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties b/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties index 703536de88ec..636e117c7e0e 100644 --- a/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties +++ b/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties @@ -61,9 +61,10 @@ ValidatorContext=From line {0,number,#}, column {1,number,#} to line {2,number,# CannotCastValue=Cast function cannot convert value of type {0} to type {1} UnknownDatatypeName=Unknown datatype name ''{0}'' IncompatibleValueType=Values passed to {0} operator must have compatible types -FetchExpressionMustBeNumeric=FETCH expression must have a numeric type; actual type is ''{0}'' -FetchExpressionCannotReferenceColumn=FETCH expression cannot reference table column ''{0}'' -FetchExpressionEvaluatedToNull=FETCH expression evaluated to NULL +OffsetFetchExpressionMustBeNumeric={0} expression must have a numeric type; actual type is ''{1}'' +OffsetFetchExpressionCannotReferenceColumn={0} expression cannot reference table column ''{1}'' +OffsetFetchExpressionEvaluatedToNull={0} expression evaluated to NULL +OffsetFetchValueMustNotBeNegative={0} must not be negative IncompatibleTypesInList=Values in expression list must have compatible types IncompatibleCharset=Cannot apply operation ''{0}'' to strings with different charsets ''{1}'' and ''{2}'' InvalidOrderByPos=ORDER BY is only allowed on top-level SELECT diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java index 83a4f87b288e..9877bd5779ea 100644 --- a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java +++ b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java @@ -5060,6 +5060,20 @@ private SqlDialect nonOrdinalDialect() { sql(query).withMysql().ok(expected); } + /** Test case for + * [CALCITE-7662] + * Add expression support for OFFSET. */ + @Test void testOffsetExpressionWithLimitDialect() { + final String query = "select \"product_id\"\n" + + "from \"product\"\n" + + "offset 1 + 2 rows"; + final String expected = "SELECT `product_id`\n" + + "FROM `foodmart`.`product`\n" + + "LIMIT 18446744073709551615\n" + + "OFFSET 3"; + sql(query).withMysql().ok(expected); + } + /** Test case for * [CALCITE-7592] * Add expression support for FETCH. */ @@ -5074,6 +5088,20 @@ private SqlDialect nonOrdinalDialect() { sql(query).withSQLite().throws_(error); } + /** Test case for + * [CALCITE-7662] + * Add expression support for OFFSET. */ + @Test void testNegativeOffsetExpressionIsRejectedBeforeSqlGeneration() { + final String query = "select \"product_id\"\n" + + "from \"product\"\n" + + "offset 0 - 1 rows"; + final String error = + "OFFSET value -1 is out of range; expected a non-negative value"; + sql(query).throws_(error); + sql(query).withMysql().throws_(error); + sql(query).withSQLite().throws_(error); + } + /** Test case for * [CALCITE-7592] * Add expression support for FETCH. */ @@ -5086,6 +5114,18 @@ private SqlDialect nonOrdinalDialect() { + "be reduced to a literal"); } + /** Test case for + * [CALCITE-7662] + * Add expression support for OFFSET. */ + @Test void testParameterizedOffsetExpressionWithLimitDialect() { + final String query = "select \"product_id\"\n" + + "from \"product\"\n" + + "offset ? + 1 rows"; + sql(query).withMysql().throws_( + "LIMIT dialect does not support OFFSET expressions that cannot " + + "be reduced to a literal"); + } + /** Test case for * [CALCITE-7592] * Add expression support for FETCH. */ @@ -5099,6 +5139,37 @@ private SqlDialect nonOrdinalDialect() { sql(query).withSQLite().ok(expected); } + /** Test case for + * [CALCITE-7662] + * Add expression support for OFFSET. */ + @Test void testParameterizedOffsetExpressionWithSQLite() { + final String query = "select \"product_id\"\n" + + "from \"product\"\n" + + "offset ? + 1 rows"; + final String expected = "SELECT \"product_id\"\n" + + "FROM \"foodmart\".\"product\"\n" + + "LIMIT -1\n" + + "OFFSET ? + 1"; + sql(query).withSQLite().ok(expected); + } + + /** Test case for + * [CALCITE-7662] + * Add expression support for OFFSET. */ + @Test void testDynamicOffsetExpressionIsNotReduced() { + final String query = "select \"product_id\"\n" + + "from \"product\"\n" + + "offset extract(day from current_date) rows"; + final String expected = "SELECT \"product_id\"\n" + + "FROM \"foodmart\".\"product\"\n" + + "OFFSET EXTRACT(DAY FROM CURRENT_DATE) ROWS"; + sql(query).ok(expected); + sql(query).withPostgresql().ok(expected); + sql(query).withMysql().throws_( + "LIMIT dialect does not support OFFSET expressions that cannot " + + "be reduced to a literal"); + } + /** Test case for * [CALCITE-7592] * Add expression support for FETCH. */ diff --git a/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java b/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java index 5f8b8edfb1db..332f1865a731 100644 --- a/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java +++ b/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java @@ -3661,6 +3661,20 @@ private void assertTypeAndToString( containsString("FETCH value -1.5 is out of range")); } + /** Test case for + * [CALCITE-7662] + * Add expression support for OFFSET. */ + @Test void testValidateOffsetValueAllowsFractionalBigDecimal() { + assertThat(RexUtil.validateOffsetValue(new BigDecimal("1.5")), + is(new BigDecimal("1.5"))); + + final IllegalArgumentException e = + assertThrows(IllegalArgumentException.class, + () -> RexUtil.validateOffsetValue(new BigDecimal("-1.5"))); + assertThat(e.getMessage(), + containsString("OFFSET value -1.5 is out of range")); + } + @Test void testConstantMap() { final RelDataType intType = typeFactory.createSqlType(SqlTypeName.INTEGER); final RelDataType bigintType = typeFactory.createSqlType(SqlTypeName.BIGINT); diff --git a/core/src/test/java/org/apache/calcite/test/JdbcTest.java b/core/src/test/java/org/apache/calcite/test/JdbcTest.java index f213b12dcb7e..d235fa327c77 100644 --- a/core/src/test/java/org/apache/calcite/test/JdbcTest.java +++ b/core/src/test/java/org/apache/calcite/test/JdbcTest.java @@ -3593,6 +3593,20 @@ public void checkOrderBy(final boolean desc, + "X=3\n"); } + /** Test case for + * [CALCITE-7662] + * Add expression support for OFFSET. */ + @Test void testOffsetExpression() { + final CalciteAssert.AssertThat with = CalciteAssert.that(); + final String values = "select * from (values (1), (2), (3), (4)) as t(x)\n"; + with.query(values + "offset 1 + abs(-1) rows") + .returns("X=3\n" + + "X=4\n"); + with.query(values + "order by x desc offset 1 + abs(-1) rows") + .returns("X=2\n" + + "X=1\n"); + } + /** Test case for * [CALCITE-7592] * Add expression support for FETCH. */ @@ -3618,6 +3632,20 @@ public void checkOrderBy(final boolean desc, } } + /** Test case for + * [CALCITE-7662] + * Add expression support for OFFSET. */ + @Test void testBindableOffsetExpression() { + try (Hook.Closeable ignored = Hook.ENABLE_BINDABLE.addThread(Hook.propertyJ(true))) { + CalciteAssert.that() + .query("select * from (values (1), (2), (3), (4)) as t(x)\n" + + "offset rand_integer(1) + 2 rows") + .explainContains("BindableSort(offset=[+(RAND_INTEGER(1), 2)])") + .returns("X=3\n" + + "X=4\n"); + } + } + /** Test case for * [CALCITE-7592] * Add expression support for FETCH. */ @@ -3632,6 +3660,18 @@ public void checkOrderBy(final boolean desc, + "X=2\n"); } + /** Test case for + * [CALCITE-7662] + * Add expression support for OFFSET. */ + @Test void testOffsetExpressionFunctionArguments() { + final CalciteAssert.AssertThat with = CalciteAssert.that(); + final String values = "select * from (values (1), (2), (3)) as t(x)\n"; + with.query(values + "offset abs(2) rows") + .returns("X=3\n"); + with.query(values + "offset abs(-2) rows") + .returns("X=3\n"); + } + /** Test case for * [CALCITE-7592] * Add expression support for FETCH. */ @@ -3647,6 +3687,20 @@ public void checkOrderBy(final boolean desc, .throws_("FETCH expression evaluated to NULL"); } + /** Test case for + * [CALCITE-7662] + * Add expression support for OFFSET. */ + @Test void testOffsetExpressionInvalidValue() { + final CalciteAssert.AssertThat with = CalciteAssert.that(); + final String values = "select * from (values (1), (2), (3)) as t(x)\n"; + with.query(values + "offset 0 - 1 rows") + .throws_("OFFSET must not be negative"); + with.query(values + "offset -1 rows") + .throws_("OFFSET must not be negative"); + with.query(values + "offset cast(null as integer) rows") + .throws_("OFFSET expression evaluated to NULL"); + } + /** Test case for * [CALCITE-7592] * Add expression support for FETCH. */ @@ -3655,13 +3709,13 @@ public void checkOrderBy(final boolean desc, + "from \"hr\".\"depts\" d,\n" + "lateral (select \"name\" from \"hr\".\"emps\"\n" + " where \"deptno\" = d.\"deptno\"\n"; - for (String fetch : new String[] {"(0 - 1)", "(-1)"}) { - for (boolean topDown : new boolean[] {false, true}) { - CalciteAssert.hr() - .with(CalciteConnectionProperty.TOPDOWN_GENERAL_DECORRELATION_ENABLED, topDown) - .query(sqlPrefix + " fetch next " + fetch + " rows only) e") - .throws_("FETCH value -1 is out of range"); - } + for (boolean topDown : new boolean[] {false, true}) { + final CalciteAssert.AssertThat with = CalciteAssert.hr() + .with(CalciteConnectionProperty.TOPDOWN_GENERAL_DECORRELATION_ENABLED, topDown); + with.query(sqlPrefix + " fetch next (0 - 1) rows only) e") + .throws_("FETCH value -1 is out of range"); + with.query(sqlPrefix + " fetch next (-1) rows only) e") + .throws_("FETCH must not be negative"); } } @@ -3681,6 +3735,9 @@ public void checkOrderBy(final boolean desc, with.query(sqlPrefix + "fetch next (0.5 + 1) rows only" + sqlSuffix) .returns("DNAME=Sales; ENAME=Bill\n" + "DNAME=Sales; ENAME=Theodore\n"); + with.query(sqlPrefix + "offset 0.5 + 1 rows fetch next 1 row only" + + sqlSuffix) + .returns("DNAME=Sales; ENAME=Sebastian\n"); with.query(sqlPrefix + "offset 1.5 rows fetch next 1 row only" + sqlSuffix) .returns("DNAME=Sales; ENAME=Sebastian\n"); } @@ -3690,20 +3747,24 @@ public void checkOrderBy(final boolean desc, * [CALCITE-7592] * Add expression support for FETCH. */ @Test void testCorrelatedPreparedFractionalOffset() throws Exception { - final String sql = "select d.\"name\" as dname, e.\"name\" as ename\n" - + "from \"hr\".\"depts\" d,\n" - + "lateral (select \"empid\", \"name\" from \"hr\".\"emps\"\n" - + " where \"deptno\" = d.\"deptno\"\n" - + " order by \"empid\" offset ? rows fetch next 1 row only) e\n" - + "order by e.\"empid\""; - for (boolean topDown : new boolean[] {false, true}) { - CalciteAssert.hr() - .with(CalciteConnectionProperty.TOPDOWN_GENERAL_DECORRELATION_ENABLED, topDown) - .doWithConnection(connection -> { - checkPreparedBigDecimalParameter(connection, sql, - new BigDecimal("1.5"), - "DNAME=Sales; ENAME=Sebastian\n"); - }); + for (String offset + : new String[] {"?", "cast(? as decimal(2, 1)) + 0"}) { + final String sql = "select d.\"name\" as dname, e.\"name\" as ename\n" + + "from \"hr\".\"depts\" d,\n" + + "lateral (select \"empid\", \"name\" from \"hr\".\"emps\"\n" + + " where \"deptno\" = d.\"deptno\"\n" + + " order by \"empid\" offset " + offset + + " rows fetch next 1 row only) e\n" + + "order by e.\"empid\""; + for (boolean topDown : new boolean[] {false, true}) { + CalciteAssert.hr() + .with(CalciteConnectionProperty.TOPDOWN_GENERAL_DECORRELATION_ENABLED, topDown) + .doWithConnection(connection -> { + checkPreparedBigDecimalParameter(connection, sql, + new BigDecimal("1.5"), + "DNAME=Sales; ENAME=Sebastian\n"); + }); + } } } @@ -3739,6 +3800,37 @@ public void checkOrderBy(final boolean desc, } } + /** Test case for + * [CALCITE-7662] + * Add expression support for OFFSET. */ + @Test void testCorrelatedPreparedOffsetExpression() throws Exception { + for (String offset : new String[] {"?", "? + 0"}) { + final String sql = "select d.\"name\" as dname, e.\"name\" as ename\n" + + "from \"hr\".\"depts\" d,\n" + + "lateral (select \"empid\", \"name\" from \"hr\".\"emps\"\n" + + " where \"deptno\" = d.\"deptno\"\n" + + " order by \"empid\" offset " + offset + " rows) e\n" + + "order by e.\"empid\""; + for (boolean topDown : new boolean[] {false, true}) { + CalciteAssert.hr() + .with(CalciteConnectionProperty.TOPDOWN_GENERAL_DECORRELATION_ENABLED, topDown) + .doWithConnection(connection -> { + checkPreparedFetchRepeated(connection, sql, + new int[] {1, 2}, + new String[] { + "DNAME=Sales; ENAME=Theodore\n" + + "DNAME=Sales; ENAME=Sebastian\n", + "DNAME=Sales; ENAME=Sebastian\n" + }); + checkPreparedParameterFails(connection, sql, -1, + "OFFSET must not be negative"); + checkPreparedParameterNullFails(connection, sql, + "OFFSET expression evaluated to NULL"); + }); + } + } + } + /** Test case for * [CALCITE-7592] * Add expression support for FETCH. */ @@ -3756,6 +3848,22 @@ public void checkOrderBy(final boolean desc, .returns(expected); } + /** Test case for + * [CALCITE-7662] + * Add expression support for OFFSET. */ + @Test void testOffsetExpressionBeyondLong() { + final CalciteAssert.AssertThat with = CalciteAssert.that(); + final String values = "select * from (values (1), (2), (3), (4)) as t(x)\n"; + with.query(values + "offset 9223372036854775808 rows") + .returns(""); + with.query(values + "offset " + + "cast(9223372036854775808 as decimal(20, 0)) + 1 rows") + .returns(""); + with.query(values + "order by x offset " + + "cast(9223372036854775808 as decimal(20, 0)) + 1 rows") + .returns(""); + } + /** Tests ORDER BY ... OFFSET ... FETCH. */ @Test void testOrderByOffsetFetch() { CalciteAssert.that() @@ -6277,6 +6385,37 @@ private CalciteAssert.AssertQuery withEmpDept(String sql) { }); } + /** Test case for + * [CALCITE-7662] + * Add expression support for OFFSET. */ + @Test void testPreparedOffsetExpression() throws Exception { + CalciteAssert.that() + .doWithConnection(connection -> { + final String values = + "select * from (values (1), (2), (3), (4)) as t(x)\n"; + checkPreparedFetch(connection, values + "offset ? + 1 rows", + 1, "X=3\nX=4\n"); + checkPreparedFetch(connection, + values + "order by x desc offset ? + 1 rows", + 1, "X=2\nX=1\n"); + checkPreparedBigDecimalParameter(connection, + values + "offset cast(? as decimal(2, 1)) + 0 rows", + new BigDecimal("1.5"), "X=3\nX=4\n"); + checkPreparedFetch(connection, + values + "offset abs(cast(? as integer)) rows", + -2, "X=3\nX=4\n"); + checkPreparedBigDecimalParameter(connection, + values + "offset cast(? as decimal(20, 0)) rows", + new BigDecimal("9223372036854775808"), ""); + checkPreparedParameterFails(connection, + values + "offset ? + 1 rows", -2, + "OFFSET must not be negative"); + checkPreparedParameterNullFails(connection, + values + "offset ? + 1 rows", + "OFFSET expression evaluated to NULL"); + }); + } + /** Test case for * [CALCITE-7592] * Add expression support for FETCH. */ @@ -6311,6 +6450,9 @@ private CalciteAssert.AssertQuery withEmpDept(String sql) { .doWithConnection(connection -> { final String values = "select * from (values (1), (2), (3), (4)) as t(x)\n"; + checkPreparedFetch(connection, + values + "offset ? + 1 rows", + 1, "X=3\nX=4\n"); final String offset = values + "offset ? rows"; checkPreparedBigDecimalParameter(connection, offset, new BigDecimal("1.5"), diff --git a/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java b/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java index 7ad9a4d733fc..cce1aea3095b 100644 --- a/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java @@ -5670,6 +5670,22 @@ private static RelNode buildCorrelateWithJoin(JoinRelType type, RelBuilder build ImmutableList.of())); } + /** Test case for + * [CALCITE-7662] + * Add expression support for OFFSET. */ + @Test void testOffsetExpressionCannotReferenceInputField() { + final RelBuilder builder = RelBuilder.create(config().build()); + builder.scan("DEPT"); + final RexNode field = builder.field("DEPTNO"); + + assertThrows(IllegalArgumentException.class, + () -> builder.sortLimit(field, null, ImmutableList.of())); + assertThrows(IllegalArgumentException.class, + () -> builder.sortLimit( + builder.call(SqlStdOperatorTable.PLUS, builder.literal(1), field), + null, ImmutableList.of())); + } + /** Test case for * [CALCITE-7592] * Add expression support for FETCH. */ @@ -5683,6 +5699,19 @@ private static RelNode buildCorrelateWithJoin(JoinRelType type, RelBuilder build ImmutableList.of()); } + /** Test case for + * [CALCITE-7662] + * Add expression support for OFFSET. */ + @Test void testOffsetExpressionMustHaveNumericType() { + final RelBuilder builder = RelBuilder.create(config().build()); + builder.scan("DEPT"); + + assertThrows(IllegalArgumentException.class, + () -> builder.sortLimit(builder.literal("x"), null, ImmutableList.of())); + builder.sortLimit(builder.literal(new BigDecimal("1.5")), null, + ImmutableList.of()); + } + /** Test case for * [CALCITE-7592] * Add expression support for FETCH. */ @@ -5702,6 +5731,25 @@ private static RelNode buildCorrelateWithJoin(JoinRelType type, RelBuilder build + " LogicalTableScan(table=[[scott, DEPT]])\n")); } + /** Test case for + * [CALCITE-7662] + * Add expression support for OFFSET. */ + @Test void testOffsetExpressionAllowsScalarCallAndDynamicParameter() { + final RelBuilder builder = RelBuilder.create(config().build()); + final RelDataType intType = + builder.getTypeFactory().createSqlType(SqlTypeName.INTEGER); + builder.scan("DEPT") + .sortLimit( + builder.call(SqlStdOperatorTable.PLUS, + builder.getRexBuilder().makeDynamicParam(intType, 0), + builder.literal(1)), + null, ImmutableList.of()); + + assertThat( + builder.build(), hasTree("LogicalSort(offset=[+(?0, 1)])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n")); + } + /** Test case for * [CALCITE-7592] * Add expression support for FETCH. */ @@ -5732,6 +5780,36 @@ private static RelNode buildCorrelateWithJoin(JoinRelType type, RelBuilder build () -> builder.sortLimit(null, subQuery, ImmutableList.of())); } + /** Test case for + * [CALCITE-7662] + * Add expression support for OFFSET. */ + @Test void testOffsetExpressionCannotContainAggregateWindowOrSubQuery() { + final RelBuilder builder = RelBuilder.create(config().build()); + final RelDataType intType = + builder.getTypeFactory().createSqlType(SqlTypeName.INTEGER); + builder.scan("DEPT"); + final RexNode aggregate = + builder.call(SqlStdOperatorTable.SUM, builder.literal(1)); + assertThrows(IllegalArgumentException.class, + () -> builder.sortLimit(aggregate, null, ImmutableList.of())); + + final RexNode over = + builder.getRexBuilder().makeOver(intType, + SqlStdOperatorTable.ROW_NUMBER, ImmutableList.of(), + ImmutableList.of(), ImmutableList.of(), + RexWindowBounds.UNBOUNDED_PRECEDING, + RexWindowBounds.UNBOUNDED_FOLLOWING, + true, true, false, false, false); + assertThrows(IllegalArgumentException.class, + () -> builder.sortLimit(over, null, ImmutableList.of())); + + final RelBuilder subQueryBuilder = RelBuilder.create(config().build()); + final RexNode subQuery = + RexSubQuery.scalar(subQueryBuilder.values(new String[] {"N"}, 1).build()); + assertThrows(IllegalArgumentException.class, + () -> builder.sortLimit(subQuery, null, ImmutableList.of())); + } + /** Test case for * [CALCITE-7592] * Add expression support for FETCH. */ diff --git a/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java b/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java index 460e066051fe..a40927c37a04 100644 --- a/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java @@ -1527,6 +1527,37 @@ void testColumnOriginsUnion() { .assertThatRowCount(is(2D), is(0D), is(2D)); } + /** Test case for + * [CALCITE-7662] + * Add expression support for OFFSET. */ + @Test void testMinRowCountOffsetExpression() { + final String sql = "select * from (values (1), (2)) as t(x)\n" + + "offset 2 - 2 rows"; + final RelMetadataFixture fixture = sql(sql); + fixture.assertThatRowCount(is(2D), is(0D), is(2D)); + + fixture + .withCluster(cluster -> { + final RelOptPlanner planner = new VolcanoPlanner(); + planner.addRule(EnumerableRules.ENUMERABLE_VALUES_RULE); + planner.addRule(EnumerableRules.ENUMERABLE_PROJECT_RULE); + planner.addRule(EnumerableRules.ENUMERABLE_LIMIT_RULE); + planner.addRelTraitDef(ConventionTraitDef.INSTANCE); + return RelOptCluster.create(planner, cluster.getRexBuilder()); + }) + .withRelTransform(rel -> { + final RelOptPlanner planner = rel.getCluster().getPlanner(); + planner.setRoot(rel); + final RelTraitSet requiredOutputTraits = + rel.getCluster().traitSet().replace(EnumerableConvention.INSTANCE); + final RelNode root = planner.changeTraits(rel, requiredOutputTraits); + planner.setRoot(root); + return planner.findBestExp(); + }) + .assertThatRel(is(instanceOf(EnumerableLimit.class))) + .assertThatRowCount(is(2D), is(0D), is(2D)); + } + @Test void testRowCountSortLimitOffset() { final String sql = "select * from emp order by ename limit 10 offset 5"; /* 14 - 5 */ diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index bad13247c3ee..aafd9429d286 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -1766,6 +1766,48 @@ private void checkJoinProjectTransposeDoesNotMatch(JoinRelType type) { .check(); } + /** Test case for + * [CALCITE-7662] + * Add expression support for OFFSET. */ + @Test void testSortUnionTransposePushesLiteralOffset() { + final String sql = "select a.name from dept a\n" + + "union all\n" + + "select b.name from dept b\n" + + "order by name offset 2 rows fetch next 3 rows only"; + sql(sql) + .withPreRule(CoreRules.PROJECT_SET_OP_TRANSPOSE) + .withRule(CoreRules.SORT_UNION_TRANSPOSE) + .check(); + } + + /** Test case for + * [CALCITE-7662] + * Add expression support for OFFSET. */ + @Test void testSortUnionTransposePushesParameterizedOffsetExpression() { + final String sql = "select a.name from dept a\n" + + "union all\n" + + "select b.name from dept b\n" + + "order by name offset ? + 1 rows fetch next 2 rows only"; + sql(sql) + .withPreRule(CoreRules.PROJECT_SET_OP_TRANSPOSE) + .withRule(CoreRules.SORT_UNION_TRANSPOSE) + .check(); + } + + /** Test case for + * [CALCITE-7662] + * Add expression support for OFFSET. */ + @Test void testSortUnionTransposeWithNonDeterministicOffset() { + final String sql = "select a.name from dept a\n" + + "union all\n" + + "select b.name from dept b\n" + + "order by name offset rand_integer(10) rows fetch next 2 rows only"; + sql(sql) + .withPreRule(CoreRules.PROJECT_SET_OP_TRANSPOSE) + .withRule(CoreRules.SORT_UNION_TRANSPOSE) + .checkUnchanged(); + } + @Test void testSortRemovalAllKeysConstant() { final String sql = "select count(*) as c\n" + "from sales.emp\n" @@ -12551,6 +12593,25 @@ private void checkNondeterministicFetchPreventsDecorrelation(boolean enableTopDo .check(); } + /** Test case for + * [CALCITE-7662] + * Add expression support for OFFSET. */ + @Test void testTopDownGeneralDecorrelateForSubqueryWithOffsetExpression() { + final String sql = "select empno from emp where " + + "sal > SOME(select sal from emp_b where emp.deptno = emp_b.deptno " + + "order by emp_b.sal offset 1 + 1 rows " + + "fetch next (1 + 1) rows only)"; + + sql(sql) + .withRule( + CoreRules.FILTER_SUB_QUERY_TO_MARK_CORRELATE, + CoreRules.PROJECT_MERGE, + CoreRules.PROJECT_REMOVE) + .withLateDecorrelate(true) + .withTopDownGeneralDecorrelate(true) + .check(); + } + @Test void testTopDownGeneralDecorrelateForSubqueryWithCube() { final String sql = "select empno from emp where " + "sal < SOME(select avg(sal) from emp_b where emp.job = emp_b.job group by cube(deptno))"; diff --git a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java index 2f4ba8919429..a3cae00b9404 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java @@ -1253,6 +1253,15 @@ public static void checkActualAndReferenceFiles() { sql(sql).ok(); } + /** Test case for + * [CALCITE-7662] + * Add expression support for OFFSET. */ + @Test void testOffsetWithExpression() { + final String sql = + "select empno from emp offset 1 + abs(-2) rows"; + sql(sql).ok(); + } + @Test void testFetch() { final String sql = "select empno from emp fetch next 5 rows only"; sql(sql).ok(); diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index 4088b0556b3f..eaa4b17a4bc0 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -10788,6 +10788,25 @@ void testGroupExpressionEquivalenceParams() { .fails("Windowed aggregate expression is illegal in FETCH clause"); } + /** Test case for + * [CALCITE-7662] + * Add expression support for OFFSET. */ + @Test void testOffsetExpressionType() { + sql("select name from dept offset ^upper('x')^ rows") + .fails("OFFSET expression must have a numeric type; " + + "actual type is 'CHAR\\(1\\) NOT NULL'"); + sql("select name from dept offset ^'x'^ rows") + .fails("OFFSET expression must have a numeric type; " + + "actual type is 'CHAR\\(1\\) NOT NULL'"); + sql("select name from dept offset 1.5 rows").ok(); + sql("select name from dept offset ^deptno^ rows") + .fails("OFFSET expression cannot reference table column 'DEPTNO'"); + sql("select name from dept offset ^cast(null as integer)^ rows") + .fails("OFFSET expression evaluated to NULL"); + sql("select name from dept offset ^row_number() over ()^ rows") + .fails("Windowed aggregate expression is illegal in OFFSET clause"); + } + @Test void testRewriteWithOffsetWithoutOrderBy() { final String sql = "select name from dept offset 2"; final String expected = "SELECT `NAME`\n" @@ -10804,14 +10823,14 @@ void testGroupExpressionEquivalenceParams() { @Test void testNegativeFetchOffsetLimit() { sql("select name from dept limit ^-^1") .fails("(?s).*Encountered \"-\".*"); - sql("select name from dept offset ^-^1") - .fails("(?s).*Encountered \"-\".*"); + sql("select name from dept offset ^-1^") + .fails("OFFSET must not be negative"); sql("select name from dept fetch next ^-^1 rows only") .fails("(?s).*Encountered \"-\".*"); sql("select name from dept order by name limit ^-^1") .fails("(?s).*Encountered \"-\".*"); - sql("select name from dept order by name offset ^-^1") - .fails("(?s).*Encountered \"-\".*"); + sql("select name from dept order by name offset ^-1^") + .fails("OFFSET must not be negative"); sql("select name from dept order by name fetch next ^-^1 rows only") .fails("(?s).*Encountered \"-\".*"); } diff --git a/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableMergeUnionTest.java b/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableMergeUnionTest.java index 44055f707462..5ca85c077758 100644 --- a/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableMergeUnionTest.java +++ b/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableMergeUnionTest.java @@ -27,6 +27,7 @@ import org.junit.jupiter.api.Test; +import java.math.BigDecimal; import java.util.function.Consumer; /** @@ -105,7 +106,66 @@ class EnumerableMergeUnionTest { .explainContains("EnumerableLimit(fetch=[+(?0, 1)])\n" + " EnumerableMergeUnion(all=[true])\n" + " EnumerableLimitSort(sort0=[$0], dir0=[ASC], " - + "fetch=[+(?0, 1)])\n"); + + "fetch=[+(?0, 1)])\n") + .consumesPreparedStatement(p -> p.setInt(1, 1)) + .returnsOrdered( + "empid=1; name=Bill", + "empid=1; name=Bill"); + } + + /** Test case for + * [CALCITE-7662] + * Add expression support for OFFSET. */ + @Test void mergeUnionPushesParameterizedOffsetExpression() { + tester(false, + new HrSchemaBig(), + "select * from (select empid, name from emps " + + "union all select empid, name from emps) " + + "order by empid offset ? + 1 rows fetch next 2 rows only") + .explainContains("EnumerableLimit(offset=[+(?0, 1)], fetch=[2])\n" + + " EnumerableMergeUnion(all=[true])\n" + + " EnumerableLimitSort(sort0=[$0], dir0=[ASC], " + + "fetch=[+(CEIL(+(?0, 1)), 2)])\n") + .consumesPreparedStatement(p -> p.setInt(1, 1)) + .returnsOrdered( + "empid=2; name=Eric", + "empid=2; name=Eric"); + } + + /** Test case for + * [CALCITE-7662] + * Add expression support for OFFSET. */ + @Test void mergeUnionRoundsOffsetAndFetchSeparatelyWhenPushingLimit() { + tester(false, + new HrSchemaBig(), + "select * from (select empid from emps where empid <= 3 " + + "union all select empid from emps where empid >= 40) " + + "order by empid offset ? rows fetch next ? rows only") + .explainContains("EnumerableLimit(offset=[?0], fetch=[?1])\n" + + " EnumerableMergeUnion(all=[true])\n" + + " EnumerableLimitSort(sort0=[$0], dir0=[ASC], " + + "fetch=[+(CEIL(?0), CEIL(?1))])\n") + .consumesPreparedStatement(p -> { + p.setBigDecimal(1, new BigDecimal("0.5")); + p.setBigDecimal(2, new BigDecimal("0.5")); + }) + .returnsOrdered("empid=2"); + } + + /** Test case for + * [CALCITE-7662] + * Add expression support for OFFSET. */ + @Test void mergeUnionDoesNotPushNonDeterministicOffset() { + tester(false, + new HrSchemaBig(), + "select * from (select empid, name from emps " + + "union all select empid, name from emps) " + + "order by empid offset rand_integer(10) rows " + + "fetch next 2 rows only") + .explainContains("EnumerableLimitSort(sort0=[$0], dir0=[ASC], " + + "offset=[RAND_INTEGER(10)], fetch=[2])\n" + + " EnumerableMergeUnion(all=[true])\n" + + " EnumerableSort(sort0=[$0], dir0=[ASC])\n"); } @Test void mergeUnionAllOrderByName() { diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index c623e63038b1..ba303c062bd3 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -20106,6 +20106,36 @@ LogicalSort(sort0=[$0], dir0=[ASC], fetch=[0]) LogicalSort(sort0=[$0], dir0=[ASC], fetch=[0]) LogicalProject(NAME=[$1]) LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) +]]> + + + + + + + + + + + @@ -20136,6 +20166,36 @@ LogicalSort(sort0=[$0], dir0=[ASC], fetch=[+(?0, 1)]) LogicalSort(sort0=[$0], dir0=[ASC], fetch=[+(?0, 1)]) LogicalProject(NAME=[$1]) LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) +]]> + + + + + + + + + + + @@ -20154,6 +20214,24 @@ LogicalSort(sort0=[$0], dir0=[ASC], fetch=[RAND_INTEGER(10)]) LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) LogicalProject(NAME=[$1]) LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) +]]> + + + + + + + + @@ -21169,6 +21247,45 @@ LogicalProject(EMPNO=[$0]) LogicalTableScan(table=[[CATALOG, SALES, EMPNULLABLES]]) LogicalProject(EMPNO=[$0], DEPTNO=[$7]) LogicalTableScan(table=[[CATALOG, SALES, EMP_B]]) +]]> + + + + + SOME(select sal from emp_b where emp.deptno = emp_b.deptno order by emp_b.sal offset 1 + 1 rows fetch next (1 + 1) rows only)]]> + + + SOME($5, { +LogicalSort(sort0=[$0], dir0=[ASC], offset=[+(1, 1)], fetch=[+(1, 1)]) + LogicalProject(SAL=[$5]) + LogicalFilter(condition=[=($cor0.DEPTNO, $7)]) + LogicalTableScan(table=[[CATALOG, SALES, EMP_B]]) +})], variablesSet=[[$cor0]]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +]]> + + + ($5, $9)]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) + LogicalSort(sort0=[$0], dir0=[ASC], offset=[+(1, 1)], fetch=[+(1, 1)]) + LogicalProject(SAL=[$5]) + LogicalFilter(condition=[=($cor0.DEPTNO, $7)]) + LogicalTableScan(table=[[CATALOG, SALES, EMP_B]]) +]]> + + + ($5, $9), IS NOT DISTINCT FROM($7, $10))], joinType=[semi]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) + LogicalFilter(condition=[AND(>($2, +(1, 1)), <=($2, +(+(1, 1), +(1, 1))))]) + LogicalProject(SAL=[$5], DEPTNO=[$7], $f2=[ROW_NUMBER() OVER (PARTITION BY $7 ORDER BY $5 NULLS LAST)]) + LogicalTableScan(table=[[CATALOG, SALES, EMP_B]]) ]]> diff --git a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml index a8da739aa946..ea4e81d04448 100644 --- a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml @@ -6451,6 +6451,18 @@ LogicalSort(offset=[?0], fetch=[?1]) LogicalSort(offset=[?0]) LogicalProject(EMPNO=[$0]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +]]> + + + + + + + + diff --git a/core/src/test/resources/sql/offset.iq b/core/src/test/resources/sql/offset.iq new file mode 100644 index 000000000000..3f4769132184 --- /dev/null +++ b/core/src/test/resources/sql/offset.iq @@ -0,0 +1,186 @@ +# offset.iq +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +!use post +!set outputformat mysql + +# OFFSET accepts an arithmetic expression without parentheses. +select * +from (values (1), (2), (3), (4)) as t(x) +offset 1 + abs(-1) rows; ++---+ +| X | ++---+ +| 3 | +| 4 | ++---+ +(2 rows) + +!ok + +# OFFSET also accepts a parenthesized scalar expression. +select * +from (values (1), (2), (3), (4)) as t(x) +offset (abs(2)) rows; ++---+ +| X | ++---+ +| 3 | +| 4 | ++---+ +(2 rows) + +!ok + +# OFFSET values are not restricted to the BIGINT range. +select * +from (values (1), (2), (3), (4)) as t(x) +offset cast(9223372036854775808 as decimal(20, 0)) + 1 rows; ++---+ +| X | ++---+ ++---+ +(0 rows) + +!ok + +# OFFSET expression cannot be negative. +select * +from (values (1), (2), (3)) as t(x) +offset 0 - 1 rows; +OFFSET must not be negative +!error + +# OFFSET expression cannot evaluate to NULL. +select * +from (values (1), (2), (3)) as t(x) +offset cast(null as integer) rows; +OFFSET expression evaluated to NULL +!error + +# OFFSET expression may have a fractional numeric type. +select * +from (values (1), (2), (3)) as t(x) +offset 1.5 rows; ++---+ +| X | ++---+ +| 3 | ++---+ +(1 row) + +!ok + +# OFFSET expression cannot reference input columns. +select * +from (values (1), (2), (3)) as t(x) +offset x rows; +OFFSET expression cannot reference table column 'X' +!error + +# Parentheses are not required around an OFFSET expression. +select * +from (values (1), (2), (3)) as t(x) +offset 1 + 1 rows; ++---+ +| X | ++---+ +| 3 | ++---+ +(1 row) + +!ok + +# OFFSET expression works with a table source. +select deptno, dname +from dept +order by deptno +offset 1 + 1 rows; ++--------+-------------+ +| DEPTNO | DNAME | ++--------+-------------+ +| 30 | Engineering | +| 40 | Empty | ++--------+-------------+ +(2 rows) + +!ok + +# OFFSET expression works together with FETCH on a table source. +select deptno, dname +from dept +order by deptno +offset 1 + 1 rows +fetch next (1 + 1) rows only; ++--------+-------------+ +| DEPTNO | DNAME | ++--------+-------------+ +| 30 | Engineering | +| 40 | Empty | ++--------+-------------+ +(2 rows) + +!ok + +# OFFSET expression may contain a scalar function on a table source. +select deptno +from dept +order by deptno +offset abs(-2) rows; ++--------+ +| DEPTNO | ++--------+ +| 30 | +| 40 | ++--------+ +(2 rows) + +!ok + +# OFFSET expression cannot reference columns of a table source. +select deptno, dname +from dept +order by deptno +offset deptno rows; +OFFSET expression cannot reference table column 'DEPTNO' +!error + +# OFFSET expression cannot reference columns even inside a larger expression. +select deptno, dname +from dept +order by deptno +offset deptno + 1 rows; +OFFSET expression cannot reference table column 'DEPTNO' +!error + +# OFFSET expression may be zero on a table source. +select deptno +from dept +order by deptno +offset 2 - 2 rows; ++--------+ +| DEPTNO | ++--------+ +| 10 | +| 20 | +| 30 | +| 40 | ++--------+ +(4 rows) + +!ok diff --git a/server/src/test/java/org/apache/calcite/test/ServerTest.java b/server/src/test/java/org/apache/calcite/test/ServerTest.java index 39d434f23ae9..0be7fb16efa1 100644 --- a/server/src/test/java/org/apache/calcite/test/ServerTest.java +++ b/server/src/test/java/org/apache/calcite/test/ServerTest.java @@ -489,6 +489,31 @@ static Connection connect() throws SQLException { } } + /** Test case for + * [CALCITE-7662] + * Add expression support for OFFSET. */ + @Test void testOffsetExpressionCannotReferenceInputColumn() throws Exception { + try (Connection c = connect(); + Statement s = c.createStatement()) { + s.execute("create table person (id int not null, name varchar(20))"); + try (PreparedStatement p = + c.prepareStatement("insert into person (id, name) values (?, ?)")) { + p.setInt(1, 1); + p.setString(2, "foo"); + assertThat(p.executeUpdate(), is(1)); + } + + for (String offset : new String[] {"id", "(id)", "1 + id"}) { + final SQLException e = + assertThrows( + SQLException.class, () -> s.executeQuery("select * from person " + + "offset " + offset + " rows")); + assertThat(e.getMessage(), + containsString("OFFSET expression cannot reference table column 'ID'")); + } + } + } + /** Test case for * [CALCITE-6022] * Support "CREATE TABLE ... LIKE" DDL in server module. */ diff --git a/site/_docs/reference.md b/site/_docs/reference.md index fda50d5bfc99..befae3b62461 100644 --- a/site/_docs/reference.md +++ b/site/_docs/reference.md @@ -193,8 +193,8 @@ query: } [ ORDER BY { ALL [ ASC | DESC ] [ NULLS FIRST | NULLS LAST ] | orderItem [, orderItem]* } ] [ LIMIT [ start, ] { count | ALL } ] - [ OFFSET start { ROW | ROWS } ] - [ FETCH { FIRST | NEXT } [ count ] { ROW | ROWS } ONLY ] + [ OFFSET { start | expression } { ROW | ROWS } ] + [ FETCH { FIRST | NEXT } [ count | '(' expression ')' ] { ROW | ROWS } ONLY ] withItem: name @@ -215,7 +215,7 @@ select: [ WINDOW windowName AS windowSpec [, windowName AS windowSpec ]* ] [ QUALIFY booleanExpression ] [ ORDER BY orderItem [, orderItem ]* ] - [ LIMIT expression [ OFFSET expression ] ] + [ LIMIT expression [ OFFSET { start | expression } ] ] The optional, non-standard `BY` clause groups and orders the query by the specified expressions, and automatically adds them to the SELECT list @@ -429,11 +429,13 @@ An optional trailing ASC / DESC and NULLS FIRST / NULLS LAST applies to all keys In *query*, *start* may be either an unsigned numeric literal or a dynamic parameter whose value is numeric. The *count* in a LIMIT clause may be either -an unsigned numeric literal or a dynamic parameter whose value is numeric. The -*count* in a FETCH clause may be an unsigned numeric literal, a dynamic -parameter whose value is numeric, or a scalar expression enclosed in -parentheses. A FETCH *count* expression cannot reference columns from the query -input, and cannot contain aggregate functions, window functions, or sub-queries. +an unsigned numeric literal or a dynamic parameter whose value is numeric. An +OFFSET clause may contain an unsigned numeric literal, a dynamic parameter whose +value is numeric, or a scalar expression with optional parentheses. The *count* +in a FETCH clause may also be a scalar expression, but it must be enclosed in +parentheses. An OFFSET expression or FETCH *count* expression cannot reference +columns from the query input, and cannot contain aggregate functions, window +functions, or sub-queries. Support for decimal or non-integer values is adapter-dependent. An aggregate query is a query that contains a GROUP BY or a HAVING diff --git a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java index 3fbe5af9be0c..5e4271e4c8fc 100644 --- a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java +++ b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java @@ -4104,7 +4104,16 @@ void checkPeriodPredicate(Checker checker) { + "FROM `FOO`\n" + "OFFSET ? ROWS\n" + "FETCH NEXT ? ROWS ONLY"); - // CALCITE-7592: Arithmetic and scalar expressions are allowed within parentheses. + // Arithmetic and scalar expressions are allowed within parentheses. + sql("select a from foo offset 1 + abs(-2) rows") + .ok("SELECT `A`\n" + + "FROM `FOO`\n" + + "OFFSET 1 + ABS(-2) ROWS"); + // Parentheses remain optional in OFFSET. + sql("select a from foo offset (1 + abs(-2)) rows") + .ok("SELECT `A`\n" + + "FROM `FOO`\n" + + "OFFSET 1 + ABS(-2) ROWS"); sql("select a from foo fetch next (1 + abs(-2)) rows only") .ok("SELECT `A`\n" + "FROM `FOO`\n" @@ -4120,7 +4129,9 @@ void checkPeriodPredicate(Checker checker) { // FETCH before OFFSET is illegal sql("select a from foo fetch next 3 rows only ^offset^ 1") .fails("(?s).*Encountered \"offset\" at .*"); - // Subqueries are not allowed in FETCH + // Subqueries are not allowed in OFFSET or FETCH + sql("select a from foo offset ^(^select 2) rows") + .fails("Query expression encountered in illegal context"); sql("select a from foo fetch next ^select^ 2 rows only") .fails("(?s).*Encountered \"select\" at .*"); sql("select a from foo fetch next (^select^ 2) rows only") @@ -4137,6 +4148,12 @@ void checkPeriodPredicate(Checker checker) { * SQL:2008. */ @Test void testLimit() { + sql("select a from foo order by b, c limit 2 offset 1 + abs(-2)") + .ok("SELECT `A`\n" + + "FROM `FOO`\n" + + "ORDER BY `B`, `C`\n" + + "OFFSET 1 + ABS(-2) ROWS\n" + + "FETCH NEXT 2 ROWS ONLY"); sql("select a from foo order by b, c limit 2 offset 1") .ok("SELECT `A`\n" + "FROM `FOO`\n" From 8ed20e148f0f2af38b1cb1dd05e4b94d7a7fa1f9 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Thu, 6 Aug 2026 14:28:06 -0700 Subject: [PATCH 442/562] [CALCITE-7696] COVAR result type is derived only from first argument type Signed-off-by: Mihai Budiu --- .../rules/AggregateReduceFunctionsRule.java | 23 +++++++++++++++++-- .../rel/type/RelDataTypeSystemImpl.java | 9 +++++++- core/src/test/resources/sql/agg.iq | 23 +++++++++++++++++++ .../apache/calcite/test/SqlOperatorTest.java | 13 +++++++++++ 4 files changed, 65 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rel/rules/AggregateReduceFunctionsRule.java b/core/src/main/java/org/apache/calcite/rel/rules/AggregateReduceFunctionsRule.java index ac40cc9101a4..498d6e70470f 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/AggregateReduceFunctionsRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/AggregateReduceFunctionsRule.java @@ -802,7 +802,9 @@ private static RexNode reduceRegrSzz( aggCallMapping, rexBuilder, yIndex, argXAndYNotNullFilterOrdinal); final RexNode sumXSumY = - rexBuilder.makeCall(pos, SqlStdOperatorTable.MULTIPLY, sumX, sumY); + widenNumerator(pos, rexBuilder, + rexBuilder.makeCall(pos, SqlStdOperatorTable.MULTIPLY, sumX, sumY), + oldCallType); final RexNode countArg = getRegrCountRexNode(oldAggRel, oldCall, newCalls, aggCallMapping, @@ -874,7 +876,9 @@ private static RexNode reduceCovariance( aggCallMapping, rexBuilder, argYOrdinal, argXAndYNotNullFilterOrdinal); final RexNode sumXSumY = - rexBuilder.makeCall(pos, SqlStdOperatorTable.MULTIPLY, sumX, sumY); + widenNumerator(pos, rexBuilder, + rexBuilder.makeCall(pos, SqlStdOperatorTable.MULTIPLY, sumX, sumY), + oldCallType); final RexNode countArg = getRegrCountRexNode(oldAggRel, oldCall, newCalls, aggCallMapping, ImmutableIntList.of(argXOrdinal, argYOrdinal), @@ -888,6 +892,21 @@ private static RexNode reduceCovariance( return rexBuilder.makeCast(pos, oldCall.getType(), result); } + /** Widens {@code numerator} for a division whose result has + * {@code callType}. For REGR_SYY(int, double) both sums are over the int + * argument, and dividing them unwidened is an integer division that + * truncates. Never narrows: the numerator may exceed the call type's range + * before the division scales it down. */ + private static RexNode widenNumerator(SqlParserPos pos, RexBuilder rexBuilder, + RexNode numerator, RelDataType callType) { + final RelDataTypeFactory typeFactory = rexBuilder.getTypeFactory(); + final RelDataType divideType = + requireNonNull( + typeFactory.leastRestrictive( + ImmutableList.of(numerator.getType(), callType))); + return rexBuilder.ensureType(pos, divideType, numerator, true); + } + private static RexNode divide(SqlParserPos pos, boolean biased, RexBuilder rexBuilder, RexNode diff, RexNode countArg) { final RexNode denominator; diff --git a/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeSystemImpl.java b/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeSystemImpl.java index 50d66f5e5f9c..9bd698d8303a 100644 --- a/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeSystemImpl.java +++ b/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeSystemImpl.java @@ -20,6 +20,8 @@ import org.apache.calcite.sql.type.SqlTypeFamily; import org.apache.calcite.sql.type.SqlTypeName; +import com.google.common.collect.ImmutableList; + import org.checkerframework.checker.nullness.qual.Nullable; import java.math.RoundingMode; @@ -27,6 +29,8 @@ import static org.apache.calcite.sql.type.SqlTypeName.DEFAULT_INTERVAL_FRACTIONAL_SECOND_PRECISION; import static org.apache.calcite.sql.type.SqlTypeName.MIN_INTERVAL_START_PRECISION; +import static java.util.Objects.requireNonNull; + /** Default implementation of * {@link org.apache.calcite.rel.type.RelDataTypeSystem}, * providing parameters from the SQL standard. @@ -369,7 +373,10 @@ && getDefaultPrecision(typeName) != RelDataType.PRECISION_NOT_SPECIFIED) { @Override public RelDataType deriveCovarType(RelDataTypeFactory typeFactory, RelDataType arg0Type, RelDataType arg1Type) { - return arg0Type; + RelDataType type = + typeFactory.leastRestrictive(ImmutableList.of(arg0Type, arg1Type)); + return requireNonNull(type, () -> + "no least restrictive type for " + arg0Type + " and " + arg1Type); } @Override public RelDataType deriveFractionalRankType(RelDataTypeFactory typeFactory) { diff --git a/core/src/test/resources/sql/agg.iq b/core/src/test/resources/sql/agg.iq index 9355ffedae36..cbd50747b3a0 100644 --- a/core/src/test/resources/sql/agg.iq +++ b/core/src/test/resources/sql/agg.iq @@ -2987,6 +2987,29 @@ from "scott".emp; !ok +# [CALCITE-7696] COVAR result type is derived only from first argument type +# The result type is the least restrictive of the two argument types; +# COVAR_POP(INT, DOUBLE) computes on DOUBLE, not INT +select + covar_pop(x, y) as cp, + covar_samp(x, y) as cs, + regr_sxx(x, y) as sxx, + regr_syy(x, y) as syy +from (values (1, 0.5e0), (2, 1.0e0)) as t(x, y); +CP DOUBLE(15) +CS DOUBLE(15) +SXX DOUBLE(15) +SYY DOUBLE(15) +!type ++-------+------+-------+-----+ +| CP | CS | SXX | SYY | ++-------+------+-------+-----+ +| 0.125 | 0.25 | 0.125 | 0.5 | ++-------+------+-------+-----+ +(1 row) + +!ok + # [CALCITE-1776, CALCITE-2402] REGR_COUNT with group by SELECT SAL, regr_count(COMM, SAL) as "REGR_COUNT(COMM, SAL)", regr_count(EMPNO, SAL) as "REGR_COUNT(EMPNO, SAL)" diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java index 7db6ab298db7..7283e346e099 100644 --- a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java +++ b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java @@ -16159,6 +16159,16 @@ void testTimestampDiff(boolean coercionEnabled) { f.checkType("covar_pop(CAST(NULL AS INTEGER),CAST(NULL AS INTEGER))", "INTEGER"); f.checkAggType("covar_pop(1.5, 2.5)", "DECIMAL(2, 1) NOT NULL"); + // [CALCITE-7696] Result type is the least restrictive of the two + // argument types, not the type of the first argument + f.checkAggType("covar_pop(1, cast(2 as double))", "DOUBLE NOT NULL"); + f.checkAggType("covar_pop(1.5, cast(2 as double))", "DOUBLE NOT NULL"); + f.checkAggType("covar_pop(1.5, 2)", "DECIMAL(11, 1) NOT NULL"); + // A nullable argument makes the result nullable + f.checkAggType("covar_pop(1.5, cast(null as double))", "DOUBLE"); + f.checkAggType("covar_pop(cast(null as integer), 2.5)", "DECIMAL(11, 1)"); + f.checkAggType("covar_pop(cast(null as integer), cast(null as double))", + "DOUBLE"); if (!f.brokenTestsEnabled()) { return; } @@ -16183,6 +16193,7 @@ void testTimestampDiff(boolean coercionEnabled) { f.checkType("covar_samp(CAST(NULL AS INTEGER),CAST(NULL AS INTEGER))", "INTEGER"); f.checkAggType("covar_samp(1.5, 2.5)", "DECIMAL(2, 1)"); + f.checkAggType("covar_samp(1, cast(2 as double))", "DOUBLE"); // with zero values f.checkAgg("covar_samp(x, x)", new String[]{}, isNullValue()); } @@ -16204,6 +16215,7 @@ void testTimestampDiff(boolean coercionEnabled) { f.checkType("regr_sxx(CAST(NULL AS INTEGER), CAST(NULL AS INTEGER))", "INTEGER"); f.checkAggType("regr_sxx(1.5, 2.5)", "DECIMAL(2, 1) NOT NULL"); + f.checkAggType("regr_sxx(1, cast(2 as double))", "DOUBLE NOT NULL"); if (!f.brokenTestsEnabled()) { return; } @@ -16228,6 +16240,7 @@ void testTimestampDiff(boolean coercionEnabled) { f.checkType("regr_syy(CAST(NULL AS INTEGER), CAST(NULL AS INTEGER))", "INTEGER"); f.checkAggType("regr_syy(1.5, 2.5)", "DECIMAL(2, 1) NOT NULL"); + f.checkAggType("regr_syy(1, cast(2 as double))", "DOUBLE NOT NULL"); if (!f.brokenTestsEnabled()) { return; } From 0f91e628e30d585452d159ad7d987852a556f326 Mon Sep 17 00:00:00 2001 From: xuzifu666 <1206332514@qq.com> Date: Thu, 6 Aug 2026 15:30:29 +0800 Subject: [PATCH 443/562] Test case for [CALCITE-4792] SqlToRel should populate corralateId for join with corralated query in ON condition --- .../calcite/test/SqlToRelConverterTest.java | 11 ++++++++++ .../calcite/test/SqlToRelConverterTest.xml | 18 +++++++++++++++ core/src/test/resources/sql/sub-query.iq | 22 +++++++++++++++++++ 3 files changed, 51 insertions(+) diff --git a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java index a3cae00b9404..90ed8a6333be 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java @@ -377,6 +377,17 @@ public static void checkActualAndReferenceFiles() { sql(sql).withExpand(false).ok(); } + /** Test case for + * [CALCITE-4792] + * SqlToRel should populate corralateId for join with corralated query in ON condition. + */ + @Test void testJoinOnCorrelatedSubQuery() { + final String sql = "select * from emp inner join dept\n" + + "on emp.deptno = dept.deptno\n" + + "and exists (select 1 from emp e2 where e2.deptno = dept.deptno)"; + sql(sql).withExpand(false).ok(); + } + @Test void testJoinUsing() { sql("SELECT * FROM emp JOIN dept USING (deptno)").ok(); } diff --git a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml index ea4e81d04448..e916c0813190 100644 --- a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml @@ -4345,6 +4345,24 @@ LogicalValues(tuples=[[{ 1 }, { 2 }, { 3 }, { 4 }, { 5 }, { 6 }, { 7 }, { 8 }, { }))], joinType=[left]) LogicalValues(tuples=[[{ 1, 'a' }, { 2, 'b' }]]) LogicalValues(tuples=[[{ 1, 'a' }, { 2, 'b' }]]) +]]> + + + + + + + + diff --git a/core/src/test/resources/sql/sub-query.iq b/core/src/test/resources/sql/sub-query.iq index d941e1b8c309..e1cf4060814d 100644 --- a/core/src/test/resources/sql/sub-query.iq +++ b/core/src/test/resources/sql/sub-query.iq @@ -10110,5 +10110,27 @@ WHERE E1.SAL > (SELECT B1.COMM FROM BONUS B1 WHERE E1.ENAME = B1.ENAME LIMIT 2); +--------+ (0 rows) +!ok + +# [CALCITE-4792] SqlToRel should populate corralateId for join with corralated query in ON condition. +SELECT emp.ename, dept.dname +FROM emp INNER JOIN dept +ON emp.deptno = dept.deptno +AND EXISTS (SELECT 1 FROM emp e2 WHERE e2.deptno = dept.deptno AND e2.sal > 2900) +ORDER BY emp.ename; ++--------+------------+ +| ENAME | DNAME | ++--------+------------+ +| ADAMS | RESEARCH | +| CLARK | ACCOUNTING | +| FORD | RESEARCH | +| JONES | RESEARCH | +| KING | ACCOUNTING | +| MILLER | ACCOUNTING | +| SCOTT | RESEARCH | +| SMITH | RESEARCH | ++--------+------------+ +(8 rows) + !ok # End sub-query.iq From 7f1f02c3cd8276112d820613539876013311ac50 Mon Sep 17 00:00:00 2001 From: xuzifu666 <1206332514@qq.com> Date: Thu, 6 Aug 2026 14:53:24 +0800 Subject: [PATCH 444/562] Test case for [CALCITE-5420] SqlToRel should populate the correlate id of a Project for queries with aggregates --- core/src/test/resources/sql/sub-query.iq | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/core/src/test/resources/sql/sub-query.iq b/core/src/test/resources/sql/sub-query.iq index e1cf4060814d..d6364652ccd0 100644 --- a/core/src/test/resources/sql/sub-query.iq +++ b/core/src/test/resources/sql/sub-query.iq @@ -1206,6 +1206,24 @@ EnumerableCalc(expr#0..3=[{inputs}], ENAME=[$t1], DEEP2SAL=[$t3]) !plan !} +# [CALCITE-5420] SqlToRel should populate the correlate id of a Project for queries with aggregates +SELECT deptno, + SUM((SELECT char_length(dname) FROM "scott".dept + WHERE dept.deptno = emp.deptno)) AS s +FROM "scott".emp +GROUP BY deptno +ORDER BY deptno; ++--------+----+ +| DEPTNO | S | ++--------+----+ +| 10 | 30 | +| 20 | 40 | +| 30 | 30 | ++--------+----+ +(3 rows) + +!ok + # [CALCITE-1494] Inefficient plan for correlated sub-queries # Plan must have only one scan each of emp and dept. select sal From 4ff42e062eabead2234816bdedfa7b282bff3f3b Mon Sep 17 00:00:00 2001 From: Kirill Tkalenko Date: Fri, 7 Aug 2026 18:23:14 +0300 Subject: [PATCH 445/562] [CALCITE-7688] Support scalar subqueries in table function arguments --- .../apache/calcite/rel/rules/CoreRules.java | 10 ++ .../calcite/rel/rules/SubQueryRemoveRule.java | 87 +++++++++++ .../java/org/apache/calcite/rex/RexUtil.java | 21 ++- .../org/apache/calcite/tools/Programs.java | 2 + .../org/apache/calcite/rex/RexUtilTest.java | 46 ++++++ .../calcite/test/TableFunctionTest.java | 144 ++++++++++++++++++ .../java/org/apache/calcite/util/Smalls.java | 65 ++++++++ 7 files changed, 374 insertions(+), 1 deletion(-) create mode 100644 core/src/test/java/org/apache/calcite/rex/RexUtilTest.java diff --git a/core/src/main/java/org/apache/calcite/rel/rules/CoreRules.java b/core/src/main/java/org/apache/calcite/rel/rules/CoreRules.java index 104e34bfaebe..11708c5facf9 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/CoreRules.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/CoreRules.java @@ -518,6 +518,16 @@ private CoreRules() {} public static final SubQueryRemoveRule JOIN_SUB_QUERY_TO_CORRELATE = SubQueryRemoveRule.Config.JOIN.toRule(); + /** Rule that converts scalar sub-queries from table function arguments into + * {@link Correlate} instances. + * + * @see #PROJECT_SUB_QUERY_TO_CORRELATE + * @see #FILTER_SUB_QUERY_TO_CORRELATE + * @see #JOIN_SUB_QUERY_TO_CORRELATE */ + public static final SubQueryRemoveRule + TABLE_FUNCTION_SCAN_SCALAR_QUERY_TO_CORRELATE = + SubQueryRemoveRule.Config.TABLE_FUNCTION_SCAN_SCALAR_QUERY.toRule(); + /** Rule that converts sub-queries from filter expressions into * {@link Correlate} instances. It will rewrite SOME/EXISTS/IN to a LEFT MARK type Correlate. */ public static final SubQueryRemoveRule FILTER_SUB_QUERY_TO_MARK_CORRELATE = diff --git a/core/src/main/java/org/apache/calcite/rel/rules/SubQueryRemoveRule.java b/core/src/main/java/org/apache/calcite/rel/rules/SubQueryRemoveRule.java index f7a6ce552fe5..c14e654a0119 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/SubQueryRemoveRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/SubQueryRemoveRule.java @@ -28,6 +28,8 @@ import org.apache.calcite.rel.core.Join; import org.apache.calcite.rel.core.JoinRelType; import org.apache.calcite.rel.core.Project; +import org.apache.calcite.rel.core.TableFunctionScan; +import org.apache.calcite.rel.logical.LogicalCorrelate; import org.apache.calcite.rel.metadata.RelMdUtil; import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.rex.LogicVisitor; @@ -77,6 +79,7 @@ * @see CoreRules#FILTER_SUB_QUERY_TO_CORRELATE * @see CoreRules#PROJECT_SUB_QUERY_TO_CORRELATE * @see CoreRules#JOIN_SUB_QUERY_TO_CORRELATE + * @see CoreRules#TABLE_FUNCTION_SCAN_SCALAR_QUERY_TO_CORRELATE */ @Value.Enclosing public class SubQueryRemoveRule @@ -1017,6 +1020,78 @@ private static void matchFilter(SubQueryRemoveRule rule, call.transformTo(builder.build()); } + /** + * Rewrites one scalar sub-query in a table-function call into a + * {@link Correlate}. + * + *

      For example, converts: + * + * 

      {@code
+   * LogicalTableFunctionScan(invocation=[F(SCALAR_QUERY(SUB_QUERY_REL), 20)])
+   * }
      + * + *

      into: + * + * 

      {@code
+   * LogicalProject(TABLE_FUNCTION_FIELDS)
+   *   LogicalCorrelate(correlation=[$cor0], joinType=[inner],
+   *       requiredColumns=[{0}])
+   *     LogicalAggregate(group=[{}], scalarValue=[SINGLE_VALUE($0)])
+   *       SUB_QUERY_REL
+   *     LogicalTableFunctionScan(invocation=[F($cor0.scalarValue, 20)])
+   * }
      + * + *

      The aggregate implements scalar-query cardinality: zero rows produce + * {@code null}, and more than one row raises an error. The correlate makes + * the resulting value available to the table-function invocation, and the + * project removes the helper scalar field from the output. The rule rewrites + * one scalar sub-query per invocation; subsequent invocations rewrite any + * remaining scalar sub-queries. + */ + private static void matchTableFunctionScan(SubQueryRemoveRule rule, + RelOptRuleCall call) { + final TableFunctionScan scan = call.rel(0); + final RexSubQuery e = + requireNonNull( + RexUtil.SubQueryFinder.find(scan.getCall(), SqlKind.SCALAR_QUERY)); + + final RelBuilder builder = call.builder(); + builder.push(e.rel); + builder.aggregate(builder.groupKey(), + builder.aggregateCall(SqlStdOperatorTable.SINGLE_VALUE, + builder.field(0))); + final RelNode scalarValue = builder.build(); + + final CorrelationId correlationId = + scan.getCluster().createCorrel(); + final RexCorrelVariable correlationVariable = + (RexCorrelVariable) scan.getCluster().getRexBuilder() + .makeCorrel(scalarValue.getRowType(), correlationId); + final RexNode target = + scan.getCluster().getRexBuilder() + .makeFieldAccess(correlationVariable, 0); + final RexNode newCall = + scan.getCall().accept(new ReplaceSubQueryShuttle(e, target)); + final TableFunctionScan newScan = + (TableFunctionScan) scan.copy(scan.getTraitSet(), scan.getInputs(), + newCall, scan.getElementType(), scan.getRowType(), + scan.getColumnMappings()) + .withHints(scan.getHints()); + + final RelNode correlate = + LogicalCorrelate.create(scalarValue, newScan, ImmutableList.of(), + correlationId, ImmutableBitSet.of(0), JoinRelType.INNER); + builder.push(correlate); + final int scalarFieldCount = + scalarValue.getRowType().getFieldCount(); + builder.project( + IntStream.range(0, scan.getRowType().getFieldCount()) + .mapToObj(i -> builder.field(scalarFieldCount + i)) + .collect(Collectors.toList()), + scan.getRowType().getFieldNames()); + call.transformTo(builder.build()); + } + private static void matchJoin(SubQueryRemoveRule rule, RelOptRuleCall call) { final Join join = call.rel(0); final RelBuilder builder = call.builder(); @@ -1372,6 +1447,18 @@ public interface Config extends RelRule.Config { .anyInputs()) .withDescription("SubQueryRemoveRule:Join"); + Config TABLE_FUNCTION_SCAN_SCALAR_QUERY = + ImmutableSubQueryRemoveRule.Config.builder() + .withMatchHandler(SubQueryRemoveRule::matchTableFunctionScan) + .build() + .withOperandSupplier(b -> + b.operand(TableFunctionScan.class) + .predicate(scan -> + RexUtil.SubQueryFinder.find(scan.getCall(), + SqlKind.SCALAR_QUERY) != null) + .anyInputs()) + .withDescription("SubQueryRemoveRule:TableFunctionScanScalarQuery"); + Config PROJECT_ENABLE_MARK_JOIN = ImmutableSubQueryRemoveRule.Config.builder() .withMatchHandler(SubQueryRemoveRule::matchProjectEnableMarkJoin) .build() diff --git a/core/src/main/java/org/apache/calcite/rex/RexUtil.java b/core/src/main/java/org/apache/calcite/rex/RexUtil.java index b7c30b7608f9..2faa2633b004 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexUtil.java +++ b/core/src/main/java/org/apache/calcite/rex/RexUtil.java @@ -3365,6 +3365,7 @@ public static List collect(Project project) { * applied to an expression that contains a {@link RexSubQuery}. */ public static class SubQueryFinder extends RexVisitorImpl { public static final SubQueryFinder INSTANCE = new SubQueryFinder(); + private final @Nullable SqlKind kind; @SuppressWarnings("Guava") @Deprecated // to be removed before 2.0 @@ -3382,7 +3383,12 @@ public static class SubQueryFinder extends RexVisitorImpl { SubQueryFinder::containsSubQuery; private SubQueryFinder() { + this(null); + } + + private SubQueryFinder(@Nullable SqlKind kind) { super(true); + this.kind = kind; } /** Returns whether a {@link Project} contains a sub-query. */ @@ -3418,7 +3424,10 @@ public static boolean containsSubQuery(Join join) { } @Override public Void visitSubQuery(RexSubQuery subQuery) { - throw new Util.FoundOne(subQuery); + if (kind == null || subQuery.getKind() == kind) { + throw new Util.FoundOne(subQuery); + } + return super.visitSubQuery(subQuery); } public static @Nullable RexSubQuery find(Iterable nodes) { @@ -3440,6 +3449,16 @@ public static boolean containsSubQuery(Join join) { return (RexSubQuery) e.getNode(); } } + + /** Returns the first sub-query of the given kind, or {@code null}. */ + public static @Nullable RexSubQuery find(RexNode node, SqlKind kind) { + try { + node.accept(new SubQueryFinder(kind)); + return null; + } catch (Util.FoundOne e) { + return (RexSubQuery) e.getNode(); + } + } } /** Deep expressions simplifier. diff --git a/core/src/main/java/org/apache/calcite/tools/Programs.java b/core/src/main/java/org/apache/calcite/tools/Programs.java index 83f6834d2850..1307e70100cb 100644 --- a/core/src/main/java/org/apache/calcite/tools/Programs.java +++ b/core/src/main/java/org/apache/calcite/tools/Programs.java @@ -260,6 +260,7 @@ public static Program subQuery(RelMetadataProvider metadataProvider) { ImmutableList.of(CoreRules.FILTER_SUB_QUERY_TO_CORRELATE, CoreRules.PROJECT_SUB_QUERY_TO_CORRELATE, CoreRules.JOIN_SUB_QUERY_TO_CORRELATE, + CoreRules.TABLE_FUNCTION_SCAN_SCALAR_QUERY_TO_CORRELATE, CoreRules.PROJECT_OVER_SUM_TO_SUM0_RULE)); final Program oldProgram = of(builder.build(), true, metadataProvider); @@ -268,6 +269,7 @@ public static Program subQuery(RelMetadataProvider metadataProvider) { ImmutableList.of(CoreRules.FILTER_SUB_QUERY_TO_MARK_CORRELATE, CoreRules.PROJECT_SUB_QUERY_TO_MARK_CORRELATE, CoreRules.JOIN_SUB_QUERY_TO_CORRELATE, + CoreRules.TABLE_FUNCTION_SCAN_SCALAR_QUERY_TO_CORRELATE, CoreRules.PROJECT_OVER_SUM_TO_SUM0_RULE)); final Program newProgram = of(newBuilder.build(), true, metadataProvider); diff --git a/core/src/test/java/org/apache/calcite/rex/RexUtilTest.java b/core/src/test/java/org/apache/calcite/rex/RexUtilTest.java new file mode 100644 index 000000000000..229886f6d015 --- /dev/null +++ b/core/src/test/java/org/apache/calcite/rex/RexUtilTest.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.rex; + +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.tools.Frameworks; +import org.apache.calcite.tools.RelBuilder; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; + +/** Tests for {@link RexUtil}. */ +class RexUtilTest { + @Test void testSubQueryFinderByKind() { + final RelBuilder builder = + RelBuilder.create(Frameworks.newConfigBuilder().build()); + final RelNode rel = builder.values(new String[] {"i"}, 1).build(); + final RexSubQuery arrayQuery = RexSubQuery.array(rel); + final RexSubQuery scalarQuery = RexSubQuery.scalar(rel); + final RexNode expression = rel.getCluster().getRexBuilder() + .makeCall(SqlStdOperatorTable.ROW, arrayQuery, scalarQuery); + + assertSame(arrayQuery, RexUtil.SubQueryFinder.find(expression)); + assertSame(scalarQuery, + RexUtil.SubQueryFinder.find(expression, SqlKind.SCALAR_QUERY)); + assertNull(RexUtil.SubQueryFinder.find(expression, SqlKind.EXISTS)); + } +} diff --git a/core/src/test/java/org/apache/calcite/test/TableFunctionTest.java b/core/src/test/java/org/apache/calcite/test/TableFunctionTest.java index e34845c1e650..71aed9c3b94a 100644 --- a/core/src/test/java/org/apache/calcite/test/TableFunctionTest.java +++ b/core/src/test/java/org/apache/calcite/test/TableFunctionTest.java @@ -59,6 +59,9 @@ private CalciteAssert.AssertThat with() { final String m = Smalls.MULTIPLICATION_TABLE_METHOD.getName(); final String m2 = Smalls.FIBONACCI_TABLE_METHOD.getName(); final String m3 = Smalls.FIBONACCI_LIMIT_TABLE_METHOD.getName(); + final String m4 = Smalls.SCALAR_QUERY_ARGUMENTS_TABLE_METHOD.getName(); + final String m5 = + Smalls.SCALAR_QUERY_ARGUMENTS_TABLE_WITHOUT_COLUMN_METHOD.getName(); return CalciteAssert.model("{\n" + " version: '1.0',\n" + " schemas: [\n" @@ -77,6 +80,14 @@ private CalciteAssert.AssertThat with() { + " name: 'fibonacci2',\n" + " className: '" + c + "',\n" + " methodName: '" + m3 + "'\n" + + " }, {\n" + + " name: 'scalar_query_arguments',\n" + + " className: '" + c + "',\n" + + " methodName: '" + m4 + "'\n" + + " }, {\n" + + " name: 'scalar_query_arguments_without_column',\n" + + " className: '" + c + "',\n" + + " methodName: '" + m5 + "'\n" + " }\n" + " ]\n" + " }\n" @@ -400,6 +411,139 @@ private Connection getConnectionWithMultiplyFunction() throws SQLException { "row_name=row 2; c1=103; c2=106"); } + @Test void testTableFunctionWithScalarQueryLiteralAndColumnArguments() { + final String sql = "select d.n as outer_value,\n" + + " f.\"scalar_value\" as scalar_value,\n" + + " f.\"literal_value\" as literal_value,\n" + + " f.\"column_value\" as column_value\n" + + "from (values (100), (200)) as d(n)\n" + + "cross join lateral table(\"s\".\"scalar_query_arguments\"(\n" + + " (select 10), 20, d.n)) as f"; + with().query(sql) + .returnsUnordered( + "OUTER_VALUE=100; SCALAR_VALUE=10; LITERAL_VALUE=20; COLUMN_VALUE=100", + "OUTER_VALUE=200; SCALAR_VALUE=10; LITERAL_VALUE=20; COLUMN_VALUE=200"); + } + + @Test void testTableFunctionWithScalarQueriesAndColumnArguments() { + final String sql = "select d.n as outer_value,\n" + + " f.\"scalar_value\" as scalar_value,\n" + + " f.\"literal_value\" as literal_value,\n" + + " f.\"column_value\" as column_value\n" + + "from (values (100), (200)) as d(n)\n" + + "cross join lateral table(\"s\".\"scalar_query_arguments\"(\n" + + " (select 10), (select 20), d.n)) as f"; + with().query(sql) + .returnsUnordered( + "OUTER_VALUE=100; SCALAR_VALUE=10; LITERAL_VALUE=20; COLUMN_VALUE=100", + "OUTER_VALUE=200; SCALAR_VALUE=10; LITERAL_VALUE=20; COLUMN_VALUE=200"); + } + + @Test void testTableFunctionWithScalarQueryAndLiteralArguments() { + final String sql = "select f.\"scalar_value\" as scalar_value,\n" + + " f.\"literal_value\" as literal_value\n" + + "from table(\"s\".\"scalar_query_arguments_without_column\"(\n" + + " (select 10), 20)) as f"; + with().query(sql) + .returnsUnordered("SCALAR_VALUE=10; LITERAL_VALUE=20"); + } + + @Test void testTableFunctionWithEmptyScalarQuery() { + final String sql = "select f.\"scalar_value\" as scalar_value,\n" + + " f.\"literal_value\" as literal_value\n" + + "from table(\"s\".\"scalar_query_arguments_without_column\"(\n" + + " (select v from (values (10)) as q(v) where v < 0), 20)) as f"; + with().query(sql) + .returnsUnordered("SCALAR_VALUE=null; LITERAL_VALUE=20"); + } + + @Test void testTableFunctionWithMultiRowScalarQuery() { + final String sql = "select *\n" + + "from table(\"s\".\"scalar_query_arguments_without_column\"(\n" + + " (select v from (values (10), (20)) as q(v)), 20))"; + with().query(sql) + .throws_("more than one value in agg SINGLE_VALUE"); + } + + @Test void testTableFunctionWithCorrelatedScalarQueryAndLiteralArguments() { + final String sql = "select d.n as outer_value,\n" + + " f.\"scalar_value\" as scalar_value,\n" + + " f.\"literal_value\" as literal_value\n" + + "from (values (100), (200)) as d(n)\n" + + "cross join lateral table(\"s\".\"scalar_query_arguments_without_column\"(\n" + + " (select d.n + 1), 20)) as f"; + with().query(sql) + .returnsUnordered( + "OUTER_VALUE=100; SCALAR_VALUE=101; LITERAL_VALUE=20", + "OUTER_VALUE=200; SCALAR_VALUE=201; LITERAL_VALUE=20"); + } + + @Test void + testTableFunctionWithCorrelatedScalarQueryLiteralAndColumnArguments() { + final String sql = "select d.n as outer_value,\n" + + " f.\"scalar_value\" as scalar_value,\n" + + " f.\"literal_value\" as literal_value,\n" + + " f.\"column_value\" as column_value\n" + + "from (values (100), (200)) as d(n)\n" + + "cross join lateral table(\"s\".\"scalar_query_arguments\"(\n" + + " (select d.n + 1), 20, d.n)) as f"; + with().query(sql) + .returnsUnordered( + "OUTER_VALUE=100; SCALAR_VALUE=101; LITERAL_VALUE=20; COLUMN_VALUE=100", + "OUTER_VALUE=200; SCALAR_VALUE=201; LITERAL_VALUE=20; COLUMN_VALUE=200"); + } + + @Test void testTableFunctionWithScalarQueryExpressionLiteralAndColumnArguments() { + final String sql = "select d.n as outer_value,\n" + + " f.\"scalar_value\" as scalar_value,\n" + + " f.\"literal_value\" as literal_value,\n" + + " f.\"column_value\" as column_value\n" + + "from (values (100), (200)) as d(n)\n" + + "cross join lateral table(\"s\".\"scalar_query_arguments\"(\n" + + " (select 4) + (select 6), 20, d.n)) as f"; + with().query(sql) + .returnsUnordered( + "OUTER_VALUE=100; SCALAR_VALUE=10; LITERAL_VALUE=20; COLUMN_VALUE=100", + "OUTER_VALUE=200; SCALAR_VALUE=10; LITERAL_VALUE=20; COLUMN_VALUE=200"); + } + + @Test void testTableFunctionWithScalarQueryExpressionAndLiteralArguments() { + final String sql = "select f.\"scalar_value\" as scalar_value,\n" + + " f.\"literal_value\" as literal_value\n" + + "from table(\"s\".\"scalar_query_arguments_without_column\"(\n" + + " (select 4) + (select 6), 20)) as f"; + with().query(sql) + .returnsUnordered("SCALAR_VALUE=10; LITERAL_VALUE=20"); + } + + @Test void testTableFunctionWithRowScalarQueryLiteralAndColumnArguments() { + final String sql = "select d.n as outer_value,\n" + + " f.\"row_value_0\" as row_value_0,\n" + + " f.\"row_value_1\" as row_value_1,\n" + + " f.\"literal_value\" as literal_value,\n" + + " f.\"column_value\" as column_value\n" + + "from (values (100), (200)) as d(n)\n" + + "cross join lateral table(\"s\".\"scalar_query_arguments\"(\n" + + " (select row(1, 2)), 20, d.n)) as f"; + with().query(sql) + .returnsUnordered( + "OUTER_VALUE=100; ROW_VALUE_0=1; ROW_VALUE_1=2; " + + "LITERAL_VALUE=20; COLUMN_VALUE=100", + "OUTER_VALUE=200; ROW_VALUE_0=1; ROW_VALUE_1=2; " + + "LITERAL_VALUE=20; COLUMN_VALUE=200"); + } + + @Test void testTableFunctionWithRowScalarQueryAndLiteralArguments() { + final String sql = "select f.\"row_value_0\" as row_value_0,\n" + + " f.\"row_value_1\" as row_value_1,\n" + + " f.\"literal_value\" as literal_value\n" + + "from table(\"s\".\"scalar_query_arguments_without_column\"(\n" + + " (select row(1, 2)), 20)) as f"; + with().query(sql) + .returnsUnordered( + "ROW_VALUE_0=1; ROW_VALUE_1=2; LITERAL_VALUE=20"); + } + /** Tests a query with a table function in the FROM clause, * attempting to reference a column from the table function in the WHERE * clause but getting the case wrong. diff --git a/testkit/src/main/java/org/apache/calcite/util/Smalls.java b/testkit/src/main/java/org/apache/calcite/util/Smalls.java index 6f33f00669da..fe4def986ad9 100644 --- a/testkit/src/main/java/org/apache/calcite/util/Smalls.java +++ b/testkit/src/main/java/org/apache/calcite/util/Smalls.java @@ -111,6 +111,12 @@ public class Smalls { public static final Method MULTIPLICATION_TABLE_METHOD = Types.lookupMethod(Smalls.class, "multiplicationTable", int.class, int.class, Integer.class); + public static final Method SCALAR_QUERY_ARGUMENTS_TABLE_METHOD = + Types.lookupMethod(Smalls.class, "scalarQueryArgumentsTable", + Object.class, Integer.class, Integer.class); + public static final Method SCALAR_QUERY_ARGUMENTS_TABLE_WITHOUT_COLUMN_METHOD = + Types.lookupMethod(Smalls.class, + "scalarQueryArgumentsTableWithoutColumn", Object.class, int.class); public static final Method FIBONACCI_TABLE_METHOD = Types.lookupMethod(Smalls.class, "fibonacciTable"); public static final Method FIBONACCI_LIMIT_100_TABLE_METHOD = @@ -283,6 +289,65 @@ public static QueryableTable multiplicationTable(final int ncol, }; } + /** A one-row table containing the arguments passed to the function. */ + public static QueryableTable scalarQueryArgumentsTable( + final @Nullable Object scalarValue, + final @Nullable Integer literalValue, + final @Nullable Integer columnValue) { + final @Nullable Integer normalizedScalarValue; + final @Nullable Integer rowValue0; + final @Nullable Integer rowValue1; + if (scalarValue == null) { + normalizedScalarValue = null; + rowValue0 = null; + rowValue1 = null; + } else if (scalarValue instanceof Number) { + normalizedScalarValue = ((Number) scalarValue).intValue(); + rowValue0 = null; + rowValue1 = null; + } else if (scalarValue instanceof Object[]) { + final Object[] row = (Object[]) scalarValue; + if (row.length != 2 + || !(row[0] instanceof Number) + || !(row[1] instanceof Number)) { + throw new IllegalArgumentException("expected ROW with two numbers"); + } + normalizedScalarValue = null; + rowValue0 = ((Number) row[0]).intValue(); + rowValue1 = ((Number) row[1]).intValue(); + } else { + throw new IllegalArgumentException("expected a number or ROW"); + } + return new AbstractQueryableTable(Object[].class) { + @Override public RelDataType getRowType(RelDataTypeFactory typeFactory) { + return typeFactory.builder() + .add("scalar_value", typeFactory.createJavaType(Integer.class)) + .add("row_value_0", typeFactory.createJavaType(Integer.class)) + .add("row_value_1", typeFactory.createJavaType(Integer.class)) + .add("literal_value", typeFactory.createJavaType(Integer.class)) + .add("column_value", typeFactory.createJavaType(Integer.class)) + .build(); + } + + @Override public Queryable asQueryable( + QueryProvider queryProvider, SchemaPlus schema, String tableName) { + return Linq4j.asEnumerable( + Collections.singletonList( + new Object[] { + normalizedScalarValue, rowValue0, rowValue1, + literalValue, columnValue + })) + .asQueryable(); + } + }; + } + + /** A two-argument version of {@link #scalarQueryArgumentsTable}. */ + public static QueryableTable scalarQueryArgumentsTableWithoutColumn( + final @Nullable Object scalarValue, final int literalValue) { + return scalarQueryArgumentsTable(scalarValue, literalValue, null); + } + /** A function that generates the Fibonacci sequence. * *

      Interesting because it has one column and no arguments, From 9d935f72b83f9f9c684e4d15aecd2be18462a7f5 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Thu, 6 Aug 2026 14:07:57 -0700 Subject: [PATCH 446/562] [CALCITE-7695] COVAR and REGR aggregate results should be nullable if either argument is nullable Signed-off-by: Mihai Budiu --- .../apache/calcite/sql/type/ReturnTypes.java | 8 +++-- .../apache/calcite/test/SqlOperatorTest.java | 30 +++++++++---------- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql/type/ReturnTypes.java b/core/src/main/java/org/apache/calcite/sql/type/ReturnTypes.java index 5e5ab2207cda..f64881cf43a0 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/ReturnTypes.java +++ b/core/src/main/java/org/apache/calcite/sql/type/ReturnTypes.java @@ -1596,12 +1596,16 @@ private static RelDataType multivalentStringWithSepSumPrecision( final RelDataType relDataType = typeFactory.getTypeSystem().deriveCovarType(typeFactory, opBinding.getOperandType(0), opBinding.getOperandType(1)); + // These functions ignore rows where either argument is NULL, so a + // non-empty group can still aggregate zero rows and return NULL; + // COVAR_SAMP also returns NULL for a group with a single row. if (opBinding.hasEmptyGroup() || opBinding.hasFilter() + || opBinding.getOperandType(0).isNullable() + || opBinding.getOperandType(1).isNullable() || opBinding.getOperator().kind == SqlKind.COVAR_SAMP) { return typeFactory.createTypeWithNullability(relDataType, true); - } else { - return relDataType; } + return relDataType; }; public static final SqlReturnTypeInference PERCENTILE_DISC_CONT = diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java index 7283e346e099..93b2f90aaefd 100644 --- a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java +++ b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java @@ -16169,11 +16169,13 @@ void testTimestampDiff(boolean coercionEnabled) { f.checkAggType("covar_pop(cast(null as integer), 2.5)", "DECIMAL(11, 1)"); f.checkAggType("covar_pop(cast(null as integer), cast(null as double))", "DOUBLE"); - if (!f.brokenTestsEnabled()) { - return; - } - // with zero values - f.checkAgg("covar_pop(x)", new String[]{}, isNullValue()); + // [CALCITE-7695] COVAR and REGR aggregate results should be nullable if + // either argument is nullable + f.checkAggType("covar_pop(1.5, cast(null as double))", "DOUBLE"); + // Nullable without GROUP BY even for non-nullable arguments, since the + // input may be empty + f.checkColumnType("select covar_pop(1.5, 2.5) from (values (1))", + "DECIMAL(2, 1)"); } @Test void testCovarSampFunc() { @@ -16216,11 +16218,9 @@ void testTimestampDiff(boolean coercionEnabled) { "INTEGER"); f.checkAggType("regr_sxx(1.5, 2.5)", "DECIMAL(2, 1) NOT NULL"); f.checkAggType("regr_sxx(1, cast(2 as double))", "DOUBLE NOT NULL"); - if (!f.brokenTestsEnabled()) { - return; - } - // with zero values - f.checkAgg("regr_sxx(x)", new String[]{}, isNullValue()); + f.checkAggType("regr_sxx(1.5, cast(null as double))", "DOUBLE"); + f.checkColumnType("select regr_sxx(1.5, 2.5) from (values (1))", + "DECIMAL(2, 1)"); } @Test void testRegrSyyFunc() { @@ -16241,11 +16241,11 @@ void testTimestampDiff(boolean coercionEnabled) { "INTEGER"); f.checkAggType("regr_syy(1.5, 2.5)", "DECIMAL(2, 1) NOT NULL"); f.checkAggType("regr_syy(1, cast(2 as double))", "DOUBLE NOT NULL"); - if (!f.brokenTestsEnabled()) { - return; - } - // with zero values - f.checkAgg("regr_syy(x)", new String[]{}, isNullValue()); + // [CALCITE-7695] COVAR and REGR aggregate results should be nullable if + // either argument is nullable + f.checkAggType("regr_syy(1.5, cast(null as double))", "DOUBLE"); + f.checkColumnType("select regr_syy(1.5, 2.5) from (values (1))", + "DECIMAL(2, 1)"); } @Test void testStddevPopFunc() { From 42ff47d34c58367633c022b4cf0ab649633fc9c4 Mon Sep 17 00:00:00 2001 From: Sergey Nuyanzin Date: Sun, 5 Jul 2026 10:22:12 +0200 Subject: [PATCH 447/562] [CALCITE-7644] Window `ORDER BY` expression is unparsed as positional ordinal --- .../calcite/rel/rel2sql/SqlImplementor.java | 24 ++++++++++++++--- .../rel/rel2sql/RelToSqlConverterTest.java | 26 +++++++++++++++++++ 2 files changed, 47 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java b/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java index 18177ecae30a..e5b1d0d0c66b 100644 --- a/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java +++ b/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java @@ -1118,7 +1118,8 @@ public List toSql(Window.Group group, ImmutableList constan partitionKeys.add(this.field(partition)); } for (RelFieldCollation collation : group.orderKeys.getFieldCollations()) { - this.addOrderItem(orderByKeys, collation); + // A window ORDER BY has no ordinal notion; resolve via field(). + this.addOrderItem(orderByKeys, collation, false); } SqlLiteral isRows = SqlLiteral.createBoolean(group.isRows, POS); SqlNode lowerBound = null; @@ -1331,6 +1332,15 @@ public List fieldList() { } void addOrderItem(List orderByList, RelFieldCollation field) { + addOrderItem(orderByList, field, true); + } + + /** Adds an ORDER BY item, optionally allowing an expression key to be + * emitted as a positional ordinal. Ordinals are valid only in a + * statement-level ORDER BY; callers such as a window's ORDER BY must pass + * {@code allowsOrdinal = false}. */ + void addOrderItem(List orderByList, RelFieldCollation field, + boolean allowsOrdinal) { if (field.nullDirection != RelFieldCollation.NullDirection.UNSPECIFIED) { final boolean first = field.nullDirection == RelFieldCollation.NullDirection.FIRST; @@ -1344,7 +1354,7 @@ void addOrderItem(List orderByList, RelFieldCollation field) { RelFieldCollation.NullDirection.UNSPECIFIED); } } - orderByList.add(toSql(field)); + orderByList.add(toSql(field, allowsOrdinal)); } /** Converts a RexFieldCollation to an ORDER BY item. */ @@ -1467,7 +1477,15 @@ private SqlCall withOrder(SqlCall call, RelCollation collation) { /** Converts a collation to an ORDER BY item. */ public SqlNode toSql(RelFieldCollation collation) { - SqlNode node = orderField(collation.getFieldIndex()); + return toSql(collation, true); + } + + /** Converts a collation to an ORDER BY item; see + * {@link #addOrderItem(List, RelFieldCollation, boolean)}. */ + public SqlNode toSql(RelFieldCollation collation, boolean allowsOrdinal) { + SqlNode node = allowsOrdinal + ? orderField(collation.getFieldIndex()) + : field(collation.getFieldIndex()); switch (collation.getDirection()) { case DESCENDING: case STRICTLY_DESCENDING: diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java index 9877bd5779ea..e9cede1a4d52 100644 --- a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java +++ b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java @@ -6150,6 +6150,32 @@ private void checkLiteral2(String expression, String expected) { sql(query8).optimize(rules, hepPlanner).ok(expected8); } + /** Test case for + * [CALCITE-7644] + * Window's ORDER BY expression is unparsed as positional ordinal. */ + @Test void testWindowOrderByExpression() { + // A window ORDER BY expression must not be unparsed as a positional ordinal. + final String query = "SELECT \"employee_id\", \"salary\", \"hire_date\", " + + "SUM(\"salary\") OVER (" + + "PARTITION BY \"employee_id\" " + + "ORDER BY CASE WHEN \"salary\" > 1000 THEN 1 ELSE 0 END " + + "RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS \"sum_val\"\n" + + "FROM \"employee\""; + final String expected = "SELECT \"employee_id\", \"salary\", \"hire_date\", " + + "SUM(\"salary\") OVER (PARTITION BY \"employee_id\" " + + "ORDER BY CASE WHEN CAST(\"salary\" AS DECIMAL(14, 4)) > 1000.0000 " + + "THEN 1 ELSE 0 END " + + "RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS \"sum_val\"\n" + + "FROM \"foodmart\".\"employee\""; + + final HepProgramBuilder builder = new HepProgramBuilder(); + builder.addRuleClass(ProjectToWindowRule.class); + final HepPlanner hepPlanner = new HepPlanner(builder.build()); + final RuleSet rules = + RuleSets.ofList(CoreRules.PROJECT_TO_LOGICAL_PROJECT_AND_WINDOW); + sql(query).optimize(rules, hepPlanner).ok(expected); + } + /** Test case for * [CALCITE-6475] * RelToSql converter fails when the IN-list contains NULL From bc9efcbe34f5d1d3bec46f2b9df9495f881859fd Mon Sep 17 00:00:00 2001 From: Sergey Nuyanzin Date: Sun, 2 Aug 2026 23:10:05 +0200 Subject: [PATCH 448/562] [CALCITE-7686] Named argument should not be resolved as a column --- .../sql/validate/SqlValidatorImpl.java | 6 +- .../apache/calcite/sql2rel/AggConverter.java | 14 ++++- .../java/org/apache/calcite/test/UdfTest.java | 63 +++++++++++++++++++ .../java/org/apache/calcite/util/Smalls.java | 25 ++++++++ 4 files changed, 105 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java index 8a5d3dfe9c39..604c03d2d191 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java @@ -4701,9 +4701,11 @@ private void checkRollUp(@Nullable SqlNode grandParent, @Nullable SqlNode parent checkRollUp(grandParent, parent, stripDot, scope, contextClause); } else if (stripDot.getKind() == SqlKind.CONVERT || stripDot.getKind() == SqlKind.TRANSLATE - || stripDot.getKind() == SqlKind.CONVERT_ORACLE) { + || stripDot.getKind() == SqlKind.CONVERT_ORACLE + || stripDot.getKind() == SqlKind.ARGUMENT_ASSIGNMENT) { // only need to check operand[0] for - // CONVERT, TRANSLATE or CONVERT_ORACLE + // 1. CONVERT, TRANSLATE or CONVERT_ORACLE + // 2. for a named argument "value => name"; operand[1] is the parameter name, not a column SqlNode child = ((SqlCall) stripDot).getOperandList().get(0); checkRollUp(parent, current, child, scope, contextClause); } else if (stripDot.getKind() == SqlKind.LAMBDA) { diff --git a/core/src/main/java/org/apache/calcite/sql2rel/AggConverter.java b/core/src/main/java/org/apache/calcite/sql2rel/AggConverter.java index 65915b69e197..0193b17c3e00 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/AggConverter.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/AggConverter.java @@ -29,6 +29,7 @@ import org.apache.calcite.runtime.PairList; import org.apache.calcite.sql.SqlAggFunction; import org.apache.calcite.sql.SqlCall; +import org.apache.calcite.sql.SqlCallBinding; import org.apache.calcite.sql.SqlDataTypeSpec; import org.apache.calcite.sql.SqlDynamicParam; import org.apache.calcite.sql.SqlIdentifier; @@ -457,7 +458,18 @@ private void translateAgg(SqlCall call, @Nullable SqlNode filter, try { // switch out of agg mode bb.agg = null; - for (SqlNode operand : call.getOperandList()) { + // Permute named arguments ("name => value") into formal parameter order + // and strip the ARGUMENT_ASSIGNMENT wrappers, so that the parameter name + // identifiers are not converted as column references. + final boolean hasNamedArgument = + call.getOperandList().stream() + .anyMatch(node -> node.getKind() == SqlKind.ARGUMENT_ASSIGNMENT); + final List aggOperands = + hasNamedArgument + ? new SqlCallBinding(bb.getValidator(), bb.scope, call) + .permutedCall().getOperandList() + : call.getOperandList(); + for (SqlNode operand : aggOperands) { // special case for COUNT(*): delete the * if (operand instanceof SqlIdentifier) { diff --git a/core/src/test/java/org/apache/calcite/test/UdfTest.java b/core/src/test/java/org/apache/calcite/test/UdfTest.java index 63492af2bcfc..282cc0ae21b1 100644 --- a/core/src/test/java/org/apache/calcite/test/UdfTest.java +++ b/core/src/test/java/org/apache/calcite/test/UdfTest.java @@ -531,6 +531,26 @@ private CalciteAssert.AssertThat withUdf() { .returns("EXPR$0=0\n"); } + /** Test case for + * [CALCITE-7686] + * Named argument should not be resolved as a column. + * + *

      The parameter name of a {@code name => value} argument must not be + * resolved as a column reference by the roll-up check. */ + @Test void testUdfArgumentNameInSelectFromTable() { + final CalciteAssert.AssertThat with = withUdf(); + // Named-arg call as a SELECT item over a table scope (unlike VALUES, this + // routes through checkRollUpInSelectList). The parameter names "s" and "n" + // must not be resolved as columns of the FROM source. + with.query("select \"adhoc\".my_left(\"s\" => 'hello', \"n\" => 3) as c\n" + + "from \"adhoc\".\"EMPLOYEES\"") + .returnsCount(4); + // reverse order + with.query("select \"adhoc\".my_left(\"n\" => 3, \"s\" => 'hello') as c\n" + + "from \"adhoc\".\"EMPLOYEES\"") + .returnsCount(4); + } + /** Tests calling a user-defined function some of whose parameters are * optional. */ @Test void testUdfArgumentOptional() { @@ -740,6 +760,49 @@ private CalciteAssert.AssertThat withUdf() { .returns("P=560\n"); } + /** + * Test case for [CALCITE-7686] + * Named argument should not be resolved as a column. + * + *

      Tests calling a user-defined aggregate function by named arguments over a + * table scope. The parameter names must not be resolved as columns during + * sql-to-rel conversion. */ + @Test void testUserDefinedAggregateFunctionWithNamedArguments() { + final String empDept = JdbcTest.EmpDeptTableFactory.class.getName(); + final String namedSum = Smalls.MyNamedSumFunction.class.getName(); + final CalciteAssert.AssertThat with = CalciteAssert.model("{\n" + + " version: '1.0',\n" + + " schemas: [\n" + + " {\n" + + " name: 'adhoc',\n" + + " tables: [\n" + + " {\n" + + " name: 'EMPLOYEES',\n" + + " type: 'custom',\n" + + " factory: '" + empDept + "',\n" + + " operand: {'foo': true, 'bar': 345}\n" + + " }\n" + + " ],\n" + + " functions: [\n" + + " {\n" + + " name: 'MY_NAMED_SUM',\n" + + " className: '" + namedSum + "'\n" + + " }\n" + + " ]\n" + + " }\n" + + " ]\n" + + "}") + .withDefaultSchema("adhoc"); + // named arguments in physical order + with.query("select \"adhoc\".my_named_sum(\"v1\" => \"commission\", \"v2\" => 250) as p\n" + + "from \"adhoc\".EMPLOYEES\n") + .returns("P=1500\n"); + // named arguments in reverse order + with.query("select \"adhoc\".my_named_sum(\"v2\" => 250, \"v1\" => \"commission\") as p\n" + + "from \"adhoc\".EMPLOYEES\n") + .returns("P=1500\n"); + } + /** Test for * {@link org.apache.calcite.runtime.CalciteResource#firstParameterOfAdd(String)}. */ @Test void testUserDefinedAggregateFunction3() { diff --git a/testkit/src/main/java/org/apache/calcite/util/Smalls.java b/testkit/src/main/java/org/apache/calcite/util/Smalls.java index fe4def986ad9..0184a0f1681a 100644 --- a/testkit/src/main/java/org/apache/calcite/util/Smalls.java +++ b/testkit/src/main/java/org/apache/calcite/util/Smalls.java @@ -974,6 +974,31 @@ public static long result(long accumulator) { } } + /** Example of a UDAF with two named parameters (via {@code @Parameter}), + * so it can be called using named-argument notation. It sums {@code v1} + * for rows where {@code v1 > v2}. */ + public static class MyNamedSumFunction { + public MyNamedSumFunction() { + } + public int init() { + return 0; + } + public int add(int accumulator, + @Parameter(name = "v1") int v1, + @Parameter(name = "v2") int v2) { + if (v1 > v2) { + return accumulator + v1; + } + return accumulator; + } + public int merge(int accumulator0, int accumulator1) { + return accumulator0 + accumulator1; + } + public int result(int accumulator) { + return accumulator; + } + } + /** Example of a user-defined aggregate function (UDAF) with two parameters. * The constructor has an initialization parameter. */ public static class MyTwoParamsSumFunctionFilter1 { From 3e8545917e91a534db971f35b176d6eadf9caac8 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Thu, 6 Aug 2026 13:34:23 -0700 Subject: [PATCH 449/562] [CALCITE-6059] Optimizer does not correctly handle special floating point value -0.0E0 Signed-off-by: Mihai Budiu --- .../calcite/rel/rel2sql/SqlImplementor.java | 6 +- .../apache/calcite/runtime/SqlFunctions.java | 16 ++-- .../calcite/sql/parser/SqlParserUtil.java | 10 ++- .../rel/rel2sql/RelToSqlConverterTest.java | 5 +- .../apache/calcite/test/SqlFunctionsTest.java | 8 ++ .../apache/calcite/test/SqlOperatorTest.java | 78 +++++++++++++++++++ 6 files changed, 111 insertions(+), 12 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java b/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java index e5b1d0d0c66b..fb003caa9509 100644 --- a/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java +++ b/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java @@ -1653,8 +1653,10 @@ public static SqlNode toSql(RexLiteral literal) { case EXACT_NUMERIC: { if (SqlTypeName.APPROX_TYPES.contains(typeName)) { final Double d = castNonNull(literal.getValueAs(Double.class)); - // BigDecimal cannot represent IEEE 754 special values (NaN, ±Infinity). - if (!Double.isFinite(d)) { + // BigDecimal cannot represent IEEE 754 special values + // (NaN, ±Infinity) or negative zero. + if (!Double.isFinite(d) + || (d == 0 && Double.doubleToRawLongBits(d) != 0L)) { final SqlNode strLiteral = SqlLiteral.createCharString(d.toString(), POS); final SqlDataTypeSpec typeSpec = diff --git a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java index e3ad7c502d40..32fa29fbd026 100644 --- a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java +++ b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java @@ -5101,7 +5101,9 @@ public static double lesser(double b0, double b1) { /** CAST(FLOAT AS VARCHAR). */ public static String toString(float x) { if (x == 0) { - return "0E0"; + // The comparison 'x == 0' does not distinguish -0.0 from 0.0, + // but the bit pattern does + return Float.floatToRawIntBits(x) != 0 ? "-0E0" : "0E0"; } return Float.toString(x); } @@ -5109,7 +5111,9 @@ public static String toString(float x) { /** CAST(DOUBLE AS VARCHAR). */ public static String toString(double x) { if (x == 0) { - return "0E0"; + // The comparison 'x == 0' does not distinguish -0.0 from 0.0, + // but the bit pattern does + return Double.doubleToRawLongBits(x) != 0L ? "-0E0" : "0E0"; } return Double.toString(x); } @@ -5159,12 +5163,12 @@ public static boolean toBoolean(Number number) { return decimal.compareTo(BigDecimal.ZERO) != 0; } if (number instanceof Double) { - Double d = (Double) number; - return !d.equals(Double.valueOf(0)); + // Compare primitives: IEEE 754 treats -0.0 as equal to 0.0, + // whereas Double.equals does not + return ((Double) number).doubleValue() != 0d; } if (number instanceof Float) { - Float f = (Float) number; - return !f.equals(Float.valueOf(0)); + return ((Float) number).floatValue() != 0f; } return !number.equals(0); } diff --git a/core/src/main/java/org/apache/calcite/sql/parser/SqlParserUtil.java b/core/src/main/java/org/apache/calcite/sql/parser/SqlParserUtil.java index c48bb846fb8b..4b1ede7e570e 100644 --- a/core/src/main/java/org/apache/calcite/sql/parser/SqlParserUtil.java +++ b/core/src/main/java/org/apache/calcite/sql/parser/SqlParserUtil.java @@ -983,8 +983,14 @@ private static SqlNode convert(PrecedenceClimbingParser.Token token) { SqlNode firstItem = list.get(0); if (item.op == SqlStdOperatorTable.UNARY_MINUS && firstItem instanceof SqlNumericLiteral) { - return SqlLiteral.createNegative((SqlNumericLiteral) firstItem, - item.pos.plusAll(list)); + final SqlNumericLiteral num = (SqlNumericLiteral) firstItem; + // Do not fold "-0.0E0" into a literal: BigDecimal, which backs + // SqlNumericLiteral, cannot represent IEEE 754 negative zero. + // Keeping the unary minus call preserves the sign at runtime. + if (num.isExact() + || ((BigDecimal) requireNonNull(num.getValue())).signum() != 0) { + return SqlLiteral.createNegative(num, item.pos.plusAll(list)); + } } if (item.op == SqlStdOperatorTable.UNARY_PLUS && firstItem instanceof SqlNumericLiteral) { diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java index e9cede1a4d52..2a500bcce7d5 100644 --- a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java +++ b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java @@ -13070,10 +13070,11 @@ generated, isLinux("SELECT \"$cor0\".\"id\"\n" .ok("SELECT *\n" + "FROM (VALUES (CAST('-Infinity' AS DOUBLE))) AS \"t\" (\"EXPR$0\")"); - // Test Negative Zero + // Test Negative Zero: must round-trip through a CAST, because a + // SqlNumericLiteral cannot represent it sql("select cast('-0.0' as DOUBLE)") .ok("SELECT *\n" - + "FROM (VALUES (0E0)) AS \"t\" (\"EXPR$0\")"); + + "FROM (VALUES (CAST('-0.0' AS DOUBLE))) AS \"t\" (\"EXPR$0\")"); // Test Subnormal values sql("select cast('1e-310' as DOUBLE)") diff --git a/core/src/test/java/org/apache/calcite/test/SqlFunctionsTest.java b/core/src/test/java/org/apache/calcite/test/SqlFunctionsTest.java index e3827c12530e..ac040bd48c4b 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlFunctionsTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlFunctionsTest.java @@ -152,6 +152,7 @@ static List list() { @Test void testToString() { assertThat(SqlFunctions.toString(0f), is("0E0")); + assertThat(SqlFunctions.toString(-0f), is("-0E0")); assertThat(SqlFunctions.toString(1f), is("1.0")); assertThat(SqlFunctions.toString(1.5f), is("1.5")); assertThat(SqlFunctions.toString(-1.5f), is("-1.5")); @@ -159,8 +160,12 @@ static List list() { assertThat(SqlFunctions.toString(-0.0625f), is("-0.0625")); assertThat(SqlFunctions.toString(0.0625f), is("0.0625")); assertThat(SqlFunctions.toString(-5e-12f), is("-5.0E-12")); + assertThat(SqlFunctions.toString(Float.NaN), is("NaN")); + assertThat(SqlFunctions.toString(Float.POSITIVE_INFINITY), is("Infinity")); + assertThat(SqlFunctions.toString(Float.NEGATIVE_INFINITY), is("-Infinity")); assertThat(SqlFunctions.toString(0d), is("0E0")); + assertThat(SqlFunctions.toString(-0d), is("-0E0")); assertThat(SqlFunctions.toString(1d), is("1.0")); assertThat(SqlFunctions.toString(1.5d), is("1.5")); assertThat(SqlFunctions.toString(-1.5d), is("-1.5")); @@ -168,6 +173,9 @@ static List list() { assertThat(SqlFunctions.toString(-0.0625d), is("-0.0625")); assertThat(SqlFunctions.toString(0.0625d), is("0.0625")); assertThat(SqlFunctions.toString(-5e-12d), is("-5.0E-12")); + assertThat(SqlFunctions.toString(Double.NaN), is("NaN")); + assertThat(SqlFunctions.toString(Double.POSITIVE_INFINITY), is("Infinity")); + assertThat(SqlFunctions.toString(Double.NEGATIVE_INFINITY), is("-Infinity")); assertThat(SqlFunctions.toString(new BigDecimal("0")), is("0")); assertThat(SqlFunctions.toString(new BigDecimal("1")), is("1")); diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java index 93b2f90aaefd..1400f90a06ed 100644 --- a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java +++ b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java @@ -405,6 +405,84 @@ protected SqlOperatorFixture fixture() { } } + @Test void testFPSpecialValues() { + SqlOperatorFixture f = fixture(); + f.checkScalarApprox("CAST('Infinity' AS REAL)", + "REAL NOT NULL", "Infinity"); + f.checkScalarApprox("CAST('Infinity' AS DOUBLE)", + "DOUBLE NOT NULL", "Infinity"); + f.checkScalarApprox("CAST('Infinity' AS FLOAT)", + "FLOAT NOT NULL", "Infinity"); + f.checkScalarApprox("CAST('-Infinity' AS REAL)", + "REAL NOT NULL", "-Infinity"); + f.checkScalarApprox("CAST('-Infinity' AS DOUBLE)", + "DOUBLE NOT NULL", "-Infinity"); + f.checkScalarApprox("CAST('-Infinity' AS FLOAT)", + "FLOAT NOT NULL", "-Infinity"); + // Note: IEEE 754 specifies that there are several types of NaN: quiet and signaling. + // There is however only one way to write them. + // But when compared for equality they may not match. + f.checkScalarApprox("CAST('NaN' AS REAL)", + "REAL NOT NULL", "NaN"); + f.checkScalarApprox("CAST('NaN' AS DOUBLE)", + "DOUBLE NOT NULL", "NaN"); + f.checkScalarApprox("CAST('NaN' AS FLOAT)", + "FLOAT NOT NULL", "NaN"); + // [CALCITE-6059] Optimizer does not correctly handle + // special floating point value -0.0E0 + // The matcher is(-0.0d) checks the value bit-exactly: + // Double.equals distinguishes -0.0 from 0.0. + f.checkScalarApprox("CAST('-0E0' AS REAL)", + "REAL NOT NULL", is(-0.0d)); + f.checkScalarApprox("CAST('-0E0' AS DOUBLE)", + "DOUBLE NOT NULL", is(-0.0d)); + f.checkScalarApprox("CAST('-0E0' AS FLOAT)", + "FLOAT NOT NULL", is(-0.0d)); + f.checkScalarApprox("CAST('0E0' AS REAL)", + "REAL NOT NULL", is(0.0d)); + f.checkScalarApprox("CAST('0E0' AS DOUBLE)", + "DOUBLE NOT NULL", is(0.0d)); + f.checkScalarApprox("CAST('0E0' AS FLOAT)", + "FLOAT NOT NULL", is(0.0d)); + // Casting an approximate numeric to VARCHAR uses E notation, + // and must preserve the sign of a negative zero + f.checkString("CAST(CAST('-0E0' AS REAL) AS VARCHAR)", + "-0E0", "VARCHAR NOT NULL"); + f.checkString("CAST(CAST('-0E0' AS DOUBLE) AS VARCHAR)", + "-0E0", "VARCHAR NOT NULL"); + f.checkString("CAST(CAST('0E0' AS REAL) AS VARCHAR)", + "0E0", "VARCHAR NOT NULL"); + f.checkString("CAST(CAST('0E0' AS DOUBLE) AS VARCHAR)", + "0E0", "VARCHAR NOT NULL"); + // A nullable value is boxed at runtime; formatting must not depend + // on nullability. RAND() prevents constant folding, so the CAST + // executes at runtime on the boxed value. + f.checkString("CAST(CASE WHEN RAND() >= 0 THEN 0.0E0 ELSE NULL END" + + " AS VARCHAR)", + "0E0", "VARCHAR"); + // An array element is a boxed Double in the generated code + f.checkString("CAST(ARRAY[-0.0E0][1] AS VARCHAR)", + "-0E0", "VARCHAR"); + // 1/-0.0 = -Infinity: proves that the sign of the zero + // survives arithmetic at runtime. + f.checkScalarApprox("1E0 / CAST('-0E0' AS REAL)", + "DOUBLE NOT NULL", "-Infinity"); + f.checkScalarApprox("1E0 / CAST('-0E0' AS DOUBLE)", + "DOUBLE NOT NULL", "-Infinity"); + f.checkScalarApprox("1E0 / CAST('0E0' AS DOUBLE)", + "DOUBLE NOT NULL", "Infinity"); + // A negative zero written as a literal, rather than computed by a CAST + f.checkScalarApprox("1E0 / -0.0E0", + "DOUBLE NOT NULL", "-Infinity"); + f.checkString("CAST(-0.0E0 AS VARCHAR)", + "-0E0", "VARCHAR NOT NULL"); + // In comparisons -0.0 is equal to 0.0, so casting either to + // BOOLEAN yields FALSE + f.checkBoolean("CAST(CAST('-0E0' AS DOUBLE) AS BOOLEAN)", false); + f.checkBoolean("CAST(CAST('-0E0' AS REAL) AS BOOLEAN)", false); + f.checkBoolean("CAST(-0.0E0 AS BOOLEAN)", false); + } + @Test void testBetween() { final SqlOperatorFixture f = fixture(); f.setFor(SqlStdOperatorTable.BETWEEN, VmName.EXPAND); From 304252d4b65683a012f478348c21cf4260925ecc Mon Sep 17 00:00:00 2001 From: liuzhengri <1289206629@qq.com> Date: Fri, 31 Jul 2026 22:57:15 +0800 Subject: [PATCH 450/562] [CALCITE-7554] NlsString.compareTo inconsistent with equals/hashCode compareTo() only compared the decoded string, ignoring charset, collation, and byte representation. This broke the compareTo/equals contract: two NlsStrings with same text but different charset would compare as equal, causing TreeSet to silently collapse them. Fix: after string comparison, compare charsetName, collation (via toString()), and bytesValue. Uses Comparator.nullsFirst for null-safe comparison. --- .../org/apache/calcite/util/NlsString.java | 29 +++++++++- .../org/apache/calcite/util/UtilTest.java | 53 +++++++++++++++++++ 2 files changed, 80 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/util/NlsString.java b/core/src/main/java/org/apache/calcite/util/NlsString.java index b98984bf537a..b00041186476 100644 --- a/core/src/main/java/org/apache/calcite/util/NlsString.java +++ b/core/src/main/java/org/apache/calcite/util/NlsString.java @@ -38,6 +38,7 @@ import java.nio.charset.CharsetDecoder; import java.nio.charset.IllegalCharsetNameException; import java.nio.charset.UnsupportedCharsetException; +import java.util.Comparator; import java.util.List; import java.util.Locale; import java.util.Objects; @@ -185,10 +186,34 @@ private NlsString(@Nullable String stringValue, @Nullable ByteString bytesValue, } @Override public int compareTo(NlsString other) { + int cmp; if (collation != null && collation.getCollator() != null) { - return collation.getCollator().compare(getValue(), other.getValue()); + cmp = collation.getCollator().compare(getValue(), other.getValue()); + } else { + cmp = getValue().compareTo(other.getValue()); + } + if (cmp != 0) { + return cmp; + } + cmp = + Objects.compare( + charsetName, other.charsetName, + Comparator.nullsFirst(String::compareTo)); + if (cmp != 0) { + return cmp; + } + cmp = + Objects.compare( + collation, other.collation, + Comparator.nullsFirst( + Comparator.comparing(Object::toString))); + if (cmp != 0) { + return cmp; } - return getValue().compareTo(other.getValue()); + // Ensures compareTo==0 <-> equals==true + return Objects.compare( + bytesValue, other.bytesValue, + Comparator.nullsFirst(Comparator.naturalOrder())); } @Pure diff --git a/core/src/test/java/org/apache/calcite/util/UtilTest.java b/core/src/test/java/org/apache/calcite/util/UtilTest.java index 7a6df9a5a07f..c7cf33a66c0a 100644 --- a/core/src/test/java/org/apache/calcite/util/UtilTest.java +++ b/core/src/test/java/org/apache/calcite/util/UtilTest.java @@ -3002,6 +3002,59 @@ private void checkNameMultimap(String s, NameMultimap map) { assertThat(s2, hasToString(s.toString())); } + /** Tests that {@link NlsString#compareTo} is consistent with + * {@link NlsString#equals}: {@code x.compareTo(y) == 0} iff + * {@code x.equals(y)} for values that differ in charset or collation. */ + @Test void testNlsStringCompareToConsistency() { + // ("hello","LATIN1",null) vs ("hello","UTF-8",null) -> equals=false, compareTo!=0 + final NlsString latin1 = new NlsString("hello", "LATIN1", null); + final NlsString utf8 = new NlsString("hello", "UTF-8", null); + assertThat(latin1.equals(utf8), is(false)); + assertThat(latin1.compareTo(utf8), not(equalTo(0))); + + // ("hello","UTF-8",null) vs ("hello","UTF-8",IMPLICIT) -> equals=false, compareTo!=0 + final NlsString noColl = new NlsString("hello", "UTF-8", null); + final NlsString withColl = + new NlsString("hello", "UTF-8", SqlCollation.IMPLICIT); + assertThat(noColl.equals(withColl), is(false)); + assertThat(noColl.compareTo(withColl), not(equalTo(0))); + + // ("hello","UTF-8",IMPLICIT) vs ("hello","UTF-8",IMPLICIT) -> equals=true, compareTo==0 + final NlsString a = new NlsString("hello", "UTF-8", SqlCollation.IMPLICIT); + final NlsString b = new NlsString("hello", "UTF-8", SqlCollation.IMPLICIT); + assertThat(a.equals(b), is(true)); + assertThat(a.compareTo(b), is(0)); + + // ("hello",null,null) vs ("hello",null,null) -> equals=true, compareTo==0 + final NlsString n1 = new NlsString("hello", null, null); + final NlsString n2 = new NlsString("hello", null, null); + assertThat(n1.equals(n2), is(true)); + assertThat(n1.compareTo(n2), is(0)); + + // ("hello",null,null) vs ("hello","UTF-8",null) -> equals=false, compareTo!=0 + final NlsString n3 = new NlsString("hello", null, null); + final NlsString n4 = new NlsString("hello", "UTF-8", null); + assertThat(n3.equals(n4), is(false)); + assertThat(n3.compareTo(n4), not(equalTo(0))); + } + + @Test void testNlsStringTreeSetRetainsDistinctValues() { + // 4 values, same string "hello", different charset: TreeSet must keep all 4 + // Before fix: compareTo ignored charset, TreeSet collapsed them into 1 + final NlsString s1 = new NlsString("hello", "LATIN1", null); + final NlsString s2 = new NlsString("hello", "UTF-8", null); + final NlsString s3 = new NlsString("hello", "UTF-16", null); + final NlsString s4 = new NlsString("hello", null, null); + + final SortedSet set = new TreeSet<>(Arrays.asList(s1, s2, s3, s4)); + assertThat(set, hasSize(4)); + + // Add exact duplicate of s1: set size unchanged + final NlsString s1dup = new NlsString("hello", "LATIN1", null); + set.add(s1dup); + assertThat(set, hasSize(4)); + } + @Test void testCollationEncoding() { SqlCollation collation = new SqlCollation( From a0bb0bddc85eb7ac02f86a80b112e9f3881ff176 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Sat, 8 Aug 2026 18:22:42 -0700 Subject: [PATCH 451/562] [CALCITE-7063] Result type inferred for CONCAT_FUNCTION is incorrect for BINARY arguments Signed-off-by: Mihai Budiu --- .../adapter/enumerable/RexImpTable.java | 39 +++++++++++++++++-- .../apache/calcite/runtime/SqlFunctions.java | 37 ++++++++++++++++++ .../calcite/sql/fun/SqlLibraryOperators.java | 7 +++- .../apache/calcite/sql/type/ReturnTypes.java | 25 +++++++++++- .../apache/calcite/util/BuiltInMethod.java | 4 ++ .../apache/calcite/test/SqlFunctionsTest.java | 18 ++++++++- .../apache/calcite/test/SqlValidatorTest.java | 11 ++++++ site/_docs/reference.md | 6 +-- .../apache/calcite/test/SqlOperatorTest.java | 27 +++++++++++++ 9 files changed, 162 insertions(+), 12 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java index 1ea24311e345..12663a2bba73 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java @@ -762,10 +762,14 @@ void populate1() { defineMethod(BITNOT, BuiltInMethod.BIT_NOT.method, NullPolicy.STRICT); define(CONCAT, new ConcatImplementor()); - defineMethod(CONCAT_FUNCTION, BuiltInMethod.MULTI_STRING_CONCAT.method, - NullPolicy.STRICT); - defineMethod(CONCAT_FUNCTION_WITH_NULL, - BuiltInMethod.MULTI_STRING_CONCAT_WITH_NULL.method, NullPolicy.NONE); + define(CONCAT_FUNCTION, + new ConcatFunctionImplementor(BuiltInMethod.MULTI_STRING_CONCAT.method, + BuiltInMethod.MULTI_BYTE_STRING_CONCAT.method, NullPolicy.STRICT)); + define(CONCAT_FUNCTION_WITH_NULL, + new ConcatFunctionImplementor( + BuiltInMethod.MULTI_STRING_CONCAT_WITH_NULL.method, + BuiltInMethod.MULTI_BYTE_STRING_CONCAT_WITH_NULL.method, + NullPolicy.NONE)); defineMethod(CONCAT2, BuiltInMethod.STRING_CONCAT_WITH_NULL.method, NullPolicy.ALL); defineMethod(CONCAT_WS, @@ -3910,6 +3914,33 @@ private static class ConcatImplementor extends AbstractRexCallImplementor { } } + /** Implementor for the multivalent CONCAT functions. + * Dispatches to the binary-string runtime method when the result type is + * binary; the character and binary methods cannot share a signature because + * varargs calls require the exact array component type. */ + private static class ConcatFunctionImplementor + extends AbstractRexCallImplementor { + private final MethodImplementor stringImplementor; + private final MethodImplementor byteStringImplementor; + + ConcatFunctionImplementor(Method stringMethod, Method byteStringMethod, + NullPolicy nullPolicy) { + super("concat", nullPolicy, false); + stringImplementor = new MethodImplementor(stringMethod, nullPolicy, false); + byteStringImplementor = + new MethodImplementor(byteStringMethod, nullPolicy, false); + } + + @Override Expression implementSafe(RexToLixTranslator translator, + RexCall call, List argValueList) { + final MethodImplementor implementor = + SqlTypeName.BINARY_TYPES.contains(call.type.getSqlTypeName()) + ? byteStringImplementor + : stringImplementor; + return implementor.implementSafe(translator, call, argValueList); + } + } + /** Implementor for a value-constructor. */ private static class ValueConstructorImplementor extends AbstractRexCallImplementor { diff --git a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java index 32fa29fbd026..d2bd353052cc 100644 --- a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java +++ b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java @@ -1763,11 +1763,35 @@ public static ByteString concat(ByteString s0, ByteString s1) { return s0.concat(s1); } + /** Concatenates two binary strings. + * Returns null only when both b0 and b1 are null, + * otherwise null is treated as empty binary string. */ + public static @Nullable ByteString concatWithNull(@Nullable ByteString b0, + @Nullable ByteString b1) { + if (b0 == null) { + return b1; + } else if (b1 == null) { + return b0; + } else { + return b0.concat(b1); + } + } + /** SQL {@code CONCAT(arg0, arg1, arg2, ...)} function. */ public static String concatMulti(String... args) { return String.join("", args); } + /** SQL {@code CONCAT(arg0, arg1, arg2, ...)} function, + * applied to binary strings. */ + public static ByteString concatMulti(ByteString... args) { + ByteString result = ByteString.EMPTY; + for (ByteString arg : args) { + result = result.concat(arg); + } + return result; + } + /** SQL {@code CONCAT(arg0, ...)} function which can accept null * but never return null. Always treats null as empty string. */ public static String concatMultiWithNull(String... args) { @@ -1778,6 +1802,19 @@ public static String concatMultiWithNull(String... args) { return sb.toString(); } + /** SQL {@code CONCAT(arg0, ...)} function applied to binary strings. + * Accepts null arguments, treating each as an empty binary string, + * and never returns null. */ + public static ByteString concatMultiWithNull(ByteString... args) { + ByteString result = ByteString.EMPTY; + for (ByteString arg : args) { + if (arg != null) { + result = result.concat(arg); + } + } + return result; + } + /** SQL {@code CONCAT_WS(sep, arg1, arg2, ...)} function; * treats null arguments as empty strings. */ public static String concatMultiWithSeparator(String... args) { diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java index 089f20eec935..6b20e97997ee 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java @@ -1215,13 +1215,16 @@ static RelDataType deriveTypeSplit(SqlOperatorBinding operatorBinding, * "CONCAT(null, null, null)" returns "". * *

      It differs from {@link #CONCAT_FUNCTION} when processing - * null values. */ + * null values. + * + *

      Requires character operands because MSSQL and PostgreSQL convert + * every argument to a character string. */ @LibraryOperator(libraries = {MSSQL, POSTGRESQL}, exceptLibraries = {REDSHIFT}) public static final SqlFunction CONCAT_FUNCTION_WITH_NULL = SqlBasicFunction.create("CONCAT", ReturnTypes.MULTIVALENT_STRING_SUM_PRECISION_NOT_NULLABLE, OperandTypes.repeat(SqlOperandCountRanges.from(1), - OperandTypes.STRING), + OperandTypes.CHARACTER), SqlFunctionCategory.STRING) .withOperandTypeInference(InferTypes.RETURN_TYPE) .withKind(SqlKind.CONCAT_WITH_NULL); diff --git a/core/src/main/java/org/apache/calcite/sql/type/ReturnTypes.java b/core/src/main/java/org/apache/calcite/sql/type/ReturnTypes.java index f64881cf43a0..2ad00299f8e2 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/ReturnTypes.java +++ b/core/src/main/java/org/apache/calcite/sql/type/ReturnTypes.java @@ -1300,16 +1300,37 @@ private static RelDataType deriveNullable( * *

      concat(cast('a' as varchar(65535)), cast('b' as varchar(2)), cast('c' as varchar(2))) * returns varchar. + * + *

      The result is VARBINARY if all operands are of the BINARY family, + * and VARCHAR otherwise; operands of NULL or ANY type do not influence + * this choice. Mixing operands of the CHARACTER and BINARY families is + * an error. + * + *

      concat(x'0a', x'0b') returns varbinary(2). */ public static final SqlReturnTypeInference MULTIVALENT_STRING_SUM_PRECISION = opBinding -> { boolean hasPrecisionNotSpecifiedOperand = false; boolean precisionOverflow = false; + boolean hasBinaryOperand = false; + boolean hasCharacterOperand = false; int typePrecision; long amount = 0; List operandTypes = opBinding.collectOperandTypes(); final RelDataTypeFactory typeFactory = opBinding.getTypeFactory(); final RelDataTypeSystem typeSystem = typeFactory.getTypeSystem(); + for (RelDataType operandType : operandTypes) { + if (operandType.getFamily() == SqlTypeFamily.BINARY) { + hasBinaryOperand = true; + } else if (SqlTypeUtil.inCharFamily(operandType)) { + hasCharacterOperand = true; + } + } + if (hasBinaryOperand && hasCharacterOperand) { + throw opBinding.newError(RESOURCE.needSameTypeParameter()); + } + final SqlTypeName typeName = + hasBinaryOperand ? SqlTypeName.VARBINARY : SqlTypeName.VARCHAR; for (RelDataType operandType : operandTypes) { int operandPrecision = operandType.getPrecision(); amount = (long) operandPrecision + amount; @@ -1317,7 +1338,7 @@ private static RelDataType deriveNullable( hasPrecisionNotSpecifiedOperand = true; break; } - if (amount > typeSystem.getMaxPrecision(SqlTypeName.VARCHAR)) { + if (amount > typeSystem.getMaxPrecision(typeName)) { precisionOverflow = true; break; } @@ -1329,7 +1350,7 @@ private static RelDataType deriveNullable( } return opBinding.getTypeFactory() - .createSqlType(SqlTypeName.VARCHAR, typePrecision); + .createSqlType(typeName, typePrecision); }; /** diff --git a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java index 6b4e52fdf968..8e1e19054850 100644 --- a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java +++ b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java @@ -572,6 +572,10 @@ public enum BuiltInMethod { MULTI_STRING_CONCAT(SqlFunctions.class, "concatMulti", String[].class), MULTI_STRING_CONCAT_WITH_NULL(SqlFunctions.class, "concatMultiWithNull", String[].class), + MULTI_BYTE_STRING_CONCAT(SqlFunctions.class, "concatMulti", + ByteString[].class), + MULTI_BYTE_STRING_CONCAT_WITH_NULL(SqlFunctions.class, "concatMultiWithNull", + ByteString[].class), MULTI_STRING_CONCAT_WITH_SEPARATOR(SqlFunctions.class, "concatMultiWithSeparator", String[].class), MULTI_TYPE_STRING_ARRAY_CONCAT_WITH_SEPARATOR(SqlFunctions.class, diff --git a/core/src/test/java/org/apache/calcite/test/SqlFunctionsTest.java b/core/src/test/java/org/apache/calcite/test/SqlFunctionsTest.java index ac040bd48c4b..508008deb0ef 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlFunctionsTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlFunctionsTest.java @@ -258,8 +258,13 @@ static List list() { // it is treated like empty string, if both values are null, returns null. // As the following tests show. assertThat(concatWithNull("a", null), is("a")); - assertThat(concatWithNull(null, null), is(nullValue())); + assertThat(concatWithNull((String) null, null), is(nullValue())); assertThat(concatWithNull(null, "b"), is("b")); + // Binary strings + assertThat(concatWithNull(b("0a"), b("0b")), is(b("0a0b"))); + assertThat(concatWithNull(b("0a"), null), is(b("0a"))); + assertThat(concatWithNull(null, b("0b")), is(b("0b"))); + assertThat(concatWithNull((ByteString) null, null), is(nullValue())); } @Test void testConcatMulti() { @@ -270,6 +275,9 @@ static List list() { assertThat(concatMulti((String) null), is("null")); assertThat(concatMulti((String) null, null), is("nullnull")); assertThat(concatMulti("a", null, "b"), is("anullb")); + // Binary strings + assertThat(concatMulti(b("0a"), b("0b"), b("0c")), is(b("0a0b0c"))); + assertThat(concatMulti(new ByteString[0]), is(ByteString.EMPTY)); } @Test void testConcatMultiWithNull() { @@ -279,6 +287,14 @@ static List list() { assertThat(concatMultiWithNull((String) null, ""), is("")); assertThat(concatMultiWithNull((String) null, null, null), is("")); assertThat(concatMultiWithNull("a", null, "b"), is("ab")); + // Binary strings; null is treated as an empty binary string + assertThat(concatMultiWithNull(b("0a"), null, b("0c")), is(b("0a0c"))); + assertThat(concatMultiWithNull((ByteString) null, null), is(ByteString.EMPTY)); + } + + /** Creates a {@link ByteString} from a hex string. */ + private static ByteString b(String hex) { + return ByteString.of(hex, 16); } @Test void testConcatMultiWithSeparator() { diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index eaa4b17a4bc0..6c10478dbd54 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -883,6 +883,17 @@ static SqlOperatorTable operatorTableFor(SqlLibrary library) { s.withExpr("concat('a', 'b')").ok(); s.withExpr("concat(x'12', x'34')").ok(); s.withExpr("concat(_UTF16'a', _UTF16'b', _UTF16'c')").ok(); + // Test case for [CALCITE-7063] + // Result type inferred for CONCAT_FUNCTION is incorrect for BINARY arguments + // The PostgreSQL variant produces a character result; binary arguments + // are implicitly cast to character strings + s.withExpr("concat(x'12', x'34')").columnType("VARCHAR NOT NULL"); + s.withExpr("concat('a', x'12')").columnType("VARCHAR NOT NULL"); + s.withExpr("concat(x'12', x'34')") + .withWhole(true) + .withTypeCoercion(false) + .fails("(?s)Cannot apply 'CONCAT' to arguments of type " + + "'CONCAT\\(, \\)'\\. .*"); s.withExpr("concat('aabbcc', 'ab', '+-')") .columnType("VARCHAR(10) NOT NULL"); s.withExpr("concat('aabbcc', CAST(NULL AS VARCHAR(20)), '+-')") diff --git a/site/_docs/reference.md b/site/_docs/reference.md index befae3b62461..08938ec364f9 100644 --- a/site/_docs/reference.md +++ b/site/_docs/reference.md @@ -3042,9 +3042,9 @@ In the following: | b o p r | CHR(integer) | Returns the character whose UTF-8 code is *integer* | b | CODE_POINTS_TO_BYTES(integers) | Converts *integers*, an array of integers between 0 and 255 inclusive, into bytes; throws error if any element is out of range | b | CODE_POINTS_TO_STRING(integers) | Converts *integers*, an array of integers between 0 and 0xD7FF or between 0xE000 and 0x10FFFF inclusive, into string; throws error if any element is out of range -| o r | CONCAT(string, string) | Concatenates two strings, returns null only when both string arguments are null, otherwise treats null as empty string -| b m | CONCAT(string [, string ]*) | Concatenates one or more strings, returns null if any of the arguments is null -| p q | CONCAT(string [, string ]*) | Concatenates one or more strings, null is treated as empty string +| o r | CONCAT(string, string) | Concatenates two character or binary strings, returns null only when both string arguments are null, otherwise treats null as empty string +| b m | CONCAT(string [, string ]*) | Concatenates one or more character or binary strings, returns null if any of the arguments is null; mixing character and binary arguments is not allowed +| p q | CONCAT(string [, string ]*) | Concatenates one or more character strings, null is treated as empty string; binary arguments are implicitly cast to character strings | m | CONCAT_WS(separator, str1 [, string ]*) | Concatenates one or more strings, returns null only when separator is null, otherwise treats null arguments as empty strings | p | CONCAT_WS(separator, any [, any ]*) | Concatenates all but the first argument, returns null only when separator is null, otherwise treats null arguments as empty strings | q | CONCAT_WS(separator, str1, str2 [, string ]*) | Concatenates two or more strings, requires at least 3 arguments (up to 254), treats null arguments as empty strings diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java index 1400f90a06ed..cdae823649bd 100644 --- a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java +++ b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java @@ -2685,6 +2685,14 @@ private static void checkConcatFunc(SqlOperatorFixture f) { f.checkString("concat('', '', 'a')", "a", "VARCHAR(1) NOT NULL"); f.checkString("concat('', '', '')", "", "VARCHAR(0) NOT NULL"); f.checkFails("^concat()^", INVALID_ARGUMENTS_NUMBER, false); + // Test case for [CALCITE-7063] + // Result type inferred for CONCAT_FUNCTION is incorrect for BINARY arguments + f.checkString("concat(x'0a', x'0b', x'0c')", "0a0b0c", "VARBINARY(3) NOT NULL"); + f.checkString("concat(cast(x'0a' as varbinary), x'0b')", "0a0b", + "VARBINARY NOT NULL"); + f.checkNull("concat(x'0a', cast(null as varbinary))"); + f.checkFails("^concat('a', x'0a')^", "Parameters must be of the same type", + false); } /** Test case for @@ -2711,6 +2719,14 @@ private static void checkConcatFuncWithNull(SqlOperatorFixture f) { f.checkString("concat(null, null, null)", "", "VARCHAR NOT NULL"); f.checkString("concat('', null, '')", "", "VARCHAR NOT NULL"); f.checkFails("^concat()^", INVALID_ARGUMENTS_NUMBER, false); + // Test case for [CALCITE-7063] + // Result type inferred for CONCAT_FUNCTION is incorrect for BINARY arguments + // MSSQL and PostgreSQL CONCAT convert every argument to a character + // string, so binary arguments are implicitly cast to character + f.checkString("concat(x'0a', x'0b', x'0c')", "0a0b0c", "VARCHAR NOT NULL"); + f.checkString("concat(x'0a', cast(null as varbinary), x'0c')", "0a0c", + "VARCHAR NOT NULL"); + f.checkString("concat('a', x'0a')", "a0a", "VARCHAR NOT NULL"); } private static void checkConcat2Func(SqlOperatorFixture f) { @@ -2732,6 +2748,17 @@ private static void checkConcat2Func(SqlOperatorFixture f) { f.checkNull("concat(null, null)"); f.checkFails("^concat('a', 'b', 'c')^", INVALID_ARGUMENTS_NUMBER, false); f.checkFails("^concat('a')^", INVALID_ARGUMENTS_NUMBER, false); + // Test case for [CALCITE-7063] + // Result type inferred for CONCAT_FUNCTION is incorrect for BINARY arguments + f.checkString("concat(x'0a', x'0b')", "0a0b", "VARBINARY(2) NOT NULL"); + f.checkString("concat(x'0a', cast(null as varbinary))", "0a", + "VARBINARY NOT NULL"); + f.checkNull("concat(cast(null as varbinary), cast(null as varbinary))"); + f.checkFails("^concat('a', x'0a')^", + "Cannot apply 'CONCAT' to arguments of type " + + "'CONCAT\\(, \\)'\\. Supported " + + "form\\(s\\): 'CONCAT\\(, \\)'", + false); } /** Test case for From ca9e2e5cdaac48b4d536712447e1bc53f889278f Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Sat, 8 Aug 2026 17:04:54 -0700 Subject: [PATCH 452/562] [CALCITE-6743] Type inference for ARRAY_INSERT function produces an inconsistent result Signed-off-by: Mihai Budiu --- .../calcite/sql/fun/SqlLibraryOperators.java | 9 +-- .../sql/validate/SqlValidatorUtil.java | 20 +++++- .../apache/calcite/test/SqlValidatorTest.java | 64 +++++++++++++++++++ 3 files changed, 86 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java index 6b20e97997ee..6e328261c65c 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java @@ -1593,12 +1593,13 @@ private static RelDataType arrayInsertReturnType(SqlOperatorBinding opBinding) { // The spec says that "ARRAY_INSERT may pad the array with NULL values if the // position is large", it implies that in the result the element type is always nullable. type = opBinding.getTypeFactory().createTypeWithNullability(type, true); - // make explicit CAST for array elements and inserted element to the biggest type - // if array component type is not equal to the inserted element type - if (!componentType.equalsSansFieldNamesAndNullability(elementType2)) { - // For array_insert, 0 is the array arg and 2 is the inserted element + // make explicit CAST for array elements and inserted element to the biggest type. + // For array_insert, 0 is the array arg and 2 is the inserted element + if (!elementType2.equalsSansFieldNamesAndNullability(type)) { SqlValidatorUtil. adjustTypeForArrayFunctions(type, opBinding, 2); + } + if (!componentType.equalsSansFieldNamesAndNullability(type)) { SqlValidatorUtil. adjustTypeForArrayFunctions(type, opBinding, 0); } diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorUtil.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorUtil.java index 3b67e5bbc328..84495c0a6680 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorUtil.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorUtil.java @@ -1439,6 +1439,7 @@ public static void adjustTypeForArrayFunctions( RelDataType targetType, SqlOperatorBinding opBinding, int... indexes) { if (opBinding instanceof SqlCallBinding) { requireNonNull(targetType, "array function target type"); + final SqlValidator validator = ((SqlCallBinding) opBinding).getValidator(); SqlCall call = ((SqlCallBinding) opBinding).getCall(); List operands = call.getOperandList(); for (int idx : indexes) { @@ -1448,9 +1449,19 @@ public static void adjustTypeForArrayFunctions( // such as spark array, the SqlKind is other function. // however, the name is same for those different array forms. && "ARRAY".equals(((SqlBasicCall) operand).getOperator().getName())) { - call.setOperand(idx, castArrayElementTo(operand, targetType)); + call.setOperand(idx, castArrayElementTo(validator, operand, targetType)); + // The rewrite changes the element types of the array constructor, + // so the type the validator has recorded for it must change too + RelDataType priorType = validator.getValidatedNodeTypeIfKnown(operand); + if (priorType != null) { + validator.setValidatedNodeType(operand, + SqlTypeUtil.createArrayType(opBinding.getTypeFactory(), + targetType, priorType.isNullable())); + } } else { - call.setOperand(idx, castTo(operand, targetType)); + SqlNode cast = castTo(operand, targetType); + call.setOperand(idx, cast); + validator.setValidatedNodeType(cast, targetType); } } } @@ -1536,11 +1547,13 @@ private static SqlNode castTo(SqlNode node, RelDataType type) { * Each element of original 'node' is cast to the desired 'type', preserving the * nullability of the 'type'. * + * @param validator Validator used, to record the new types * @param node the {@link SqlNode} the sqlnode representing an array * @param type the target {@link RelDataType} the target type * @return a new {@link SqlNode} representing the CAST operation */ - private static SqlNode castArrayElementTo(SqlNode node, RelDataType type) { + private static SqlNode castArrayElementTo(SqlValidator validator, + SqlNode node, RelDataType type) { int i = 0; for (SqlNode operand : ((SqlBasicCall) node).getOperandList()) { SqlNode castedOperand = @@ -1548,6 +1561,7 @@ private static SqlNode castArrayElementTo(SqlNode node, RelDataType type) { operand, SqlTypeUtil.convertTypeToSpec(type).withNullable(type.isNullable())); ((SqlBasicCall) node).setOperand(i++, castedOperand); + validator.setValidatedNodeType(castedOperand, type); } return node; } diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index 6c10478dbd54..892d68f11789 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -73,6 +73,7 @@ import org.apache.calcite.testlib.annotations.LocaleEnUs; import org.apache.calcite.util.Bug; import org.apache.calcite.util.ImmutableBitSet; +import org.apache.calcite.util.Util; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -107,6 +108,7 @@ import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.notNullValue; import static org.hamcrest.CoreMatchers.nullValue; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.hasToString; @@ -14433,6 +14435,68 @@ private void checkCustomColumnResolving(String table) { assertThat(cast.getParserPosition().getLineNum(), is(1)); } + /** Test case for + * [CALCITE-6743] Type inference for ARRAY_INSERT function produces an + * inconsistent result. */ + @Test void testArrayFunctionAdjustedOperandTypes() throws SqlParseException { + // The array constructor is rewritten to ARRAY(CAST(1 AS DOUBLE), ...); + // its registered type must be DOUBLE ARRAY, not INTEGER ARRAY + final SqlCall call = + checkArrayOperandType("select array_insert(array(1, 2, 3), 3, cast(4 as double))", + "DOUBLE ARRAY NOT NULL", "DOUBLE", "DOUBLE NOT NULL"); + // The inserted element already has the target type; it must keep the + // user's cast, without a redundant second cast around it + assertThat(Util.last(call.getOperandList()), + hasToString("CAST(4 AS DOUBLE)")); + checkArrayOperandType( + "select array_append(array(1, 2, 3), cast(4 as double))", + "DOUBLE NOT NULL ARRAY NOT NULL", "DOUBLE NOT NULL", + "DOUBLE NOT NULL"); + checkArrayOperandType( + "select array_prepend(array(1, 2, 3), cast(4 as double))", + "DOUBLE NOT NULL ARRAY NOT NULL", "DOUBLE NOT NULL", + "DOUBLE NOT NULL"); + // Only the inserted element is cast (to the array component type); + // the array constructor keeps its original type + checkArrayOperandType( + "select array_append(array(1, 2, 3), cast(4 as tinyint))", + "INTEGER NOT NULL ARRAY NOT NULL", "INTEGER NOT NULL", + "INTEGER NOT NULL"); + } + + /** Validates a query whose select list is a single call to an array + * function with an array constructor as its first argument; checks the + * validated types of the array argument, of the elements of the + * constructor, and of the inserted element after the function's type + * inference has adjusted them. Returns the validated call. */ + private SqlCall checkArrayOperandType(String sql, String expectedArrayType, + String expectedElementType, String expectedInsertedType) + throws SqlParseException { + final SqlParser parser = SqlParser.create(sql, SqlParser.config()); + final SqlNode node = parser.parseQuery(); + final SqlValidator validator = fixture() + .withOperatorTable(operatorTableFor(SqlLibrary.SPARK)) + .factory.createValidator(); + final SqlSelect select = (SqlSelect) validator.validate(node); + final SqlCall call = (SqlCall) select.getSelectList().get(0); + final SqlNode array = call.getOperandList().get(0); + final RelDataType arrayType = validator.getValidatedNodeType(array); + assertThat(arrayType.getFullTypeString(), is(expectedArrayType)); + final RelDataType componentType = arrayType.getComponentType(); + assertThat(componentType, notNullValue()); + for (SqlNode element : ((SqlCall) array).getOperandList()) { + final RelDataType elementType = validator.getValidatedNodeType(element); + assertThat(elementType.getFullTypeString(), is(expectedElementType)); + assertThat(elementType, is(componentType)); + } + // The last operand is the inserted element; its type may differ from + // the array component type only in nullability + final SqlNode inserted = Util.last(call.getOperandList()); + assertThat(validator.getValidatedNodeType(inserted).getFullTypeString(), + is(expectedInsertedType)); + return call; + } + @Test void testValidateParameterizedExpression() throws SqlParseException { final SqlParser.Config config = SqlParser.config(); final SqlValidator validator = fixture().factory.createValidator(); From 1f73594e9d965a9aafed2fd2bd1c0fe4293594ea Mon Sep 17 00:00:00 2001 From: xuzifu666 <1206332514@qq.com> Date: Thu, 6 Aug 2026 16:22:24 +0800 Subject: [PATCH 453/562] Test cases for [CALCITE-5413] Nested Subqueries with correlated variables are not decorrelated correctly --- .../apache/calcite/test/RelOptRulesTest.java | 18 ++++++++ .../apache/calcite/test/RelOptRulesTest.xml | 44 +++++++++++++++++++ core/src/test/resources/sql/new-decorr.iq | 35 +++++++++++++++ 3 files changed, 97 insertions(+) diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index aafd9429d286..0cfb76b9d5de 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -9530,6 +9530,24 @@ private void checkSemiJoinRuleOnAntiJoin(RelOptRule rule) { sql(sql).withSubQueryRules().check(); } + /** Test case for + * [CALCITE-5413] + * Nested Subqueries with correlated variables are not decorrelated correctly. + */ + @Test void testExpandFilterNestedExistsCorrelatingTwoLevels() { + final String sql = "SELECT deptno\n" + + "FROM emp e\n" + + "WHERE EXISTS (\n" + + " SELECT *\n" + + " FROM dept d\n" + + " WHERE EXISTS(\n" + + " SELECT *\n" + + " FROM emp_address ea\n" + + " WHERE d.deptno = e.deptno\n" + + " AND ea.empno = e.empno))"; + sql(sql).withSubQueryRules().check(); + } + @Test void testDecorrelateExists() { final String sql = "select * from sales.emp\n" + "where EXISTS (\n" diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index ba303c062bd3..0b544e3f1c63 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -5220,6 +5220,50 @@ LogicalProject(EMPNO=[$0]) LogicalProject(DEPTNO=[$0], i=[true]) LogicalFilter(condition=[=($1, 'dept2')]) LogicalTableScan(table=[[CATALOG, SALES, DEPTNULLABLES]]) +]]> + + + + + + + + + + + diff --git a/core/src/test/resources/sql/new-decorr.iq b/core/src/test/resources/sql/new-decorr.iq index 3aef6e527608..a1caa699d63c 100644 --- a/core/src/test/resources/sql/new-decorr.iq +++ b/core/src/test/resources/sql/new-decorr.iq @@ -496,4 +496,39 @@ SELECT empno, (SELECT row_number() OVER (PARTITION BY dname ORDER BY emp.sal) FR !ok !} + +# [CALCITE-5413] Nested Subqueries with correlated variables are not decorrelated correctly +# The innermost subquery correlates to both the outermost (e) and the middle (d) query levels. +SELECT e.empno +FROM emp e +WHERE EXISTS ( + SELECT * + FROM dept d + WHERE EXISTS ( + SELECT * + FROM emp ea + WHERE d.deptno = e.deptno + AND ea.empno = e.empno)) +ORDER BY e.empno; ++-------+ +| EMPNO | ++-------+ +| 7369 | +| 7499 | +| 7521 | +| 7566 | +| 7654 | +| 7698 | +| 7782 | +| 7788 | +| 7839 | +| 7844 | +| 7876 | +| 7900 | +| 7902 | +| 7934 | ++-------+ +(14 rows) + +!ok # End new-decorr.iq From 1f3625b4c9ab1546d8c5e4db9d1d2b728cc27679 Mon Sep 17 00:00:00 2001 From: xuzifu666 <1206332514@qq.com> Date: Thu, 6 Aug 2026 14:25:03 +0800 Subject: [PATCH 454/562] [CALCITE-7693] Move MongoDB LIKE-to-regex conversion into runtime.Like for consistency --- .../java/org/apache/calcite/runtime/Like.java | 66 +++++++++++++++ .../org/apache/calcite/runtime/LikeTest.java | 50 ++++++++++++ .../calcite/adapter/mongodb/MongoFilter.java | 80 ++----------------- 3 files changed, 124 insertions(+), 72 deletions(-) create mode 100644 core/src/test/java/org/apache/calcite/runtime/LikeTest.java diff --git a/core/src/main/java/org/apache/calcite/runtime/Like.java b/core/src/main/java/org/apache/calcite/runtime/Like.java index ac074afa3da7..376e7ca58db3 100644 --- a/core/src/main/java/org/apache/calcite/runtime/Like.java +++ b/core/src/main/java/org/apache/calcite/runtime/Like.java @@ -110,6 +110,72 @@ static String sqlToRegexLike( return javaPattern.toString(); } + /** + * Translates a SQL LIKE pattern to an anchored regular expression, with an + * optional escape string. + * + *

      Similar to {@link #sqlToRegexLike}, except that the result is anchored + * with {@code ^} and {@code $} so that the entire value must match, as SQL + * LIKE requires. The translation is not specific to any dialect; it is used, + * for example, by the MongoDB adapter. + */ + public static String sqlToRegexAnchored( + String sqlPattern, + @Nullable CharSequence escapeStr) { + final char escapeChar; + if (escapeStr != null) { + if (escapeStr.length() != 1) { + throw invalidEscapeCharacter(escapeStr.toString()); + } + escapeChar = escapeStr.charAt(0); + } else { + escapeChar = 0; + } + return sqlToRegexAnchored(sqlPattern, escapeChar); + } + + /** + * Translates a SQL LIKE pattern to an anchored regular expression. + */ + public static String sqlToRegexAnchored( + String sqlPattern, + char escapeChar) { + final int len = sqlPattern.length(); + final StringBuilder javaPattern = new StringBuilder(len + len); + javaPattern.append('^'); + for (int i = 0; i < len; i++) { + char c = sqlPattern.charAt(i); + if (c == escapeChar) { + if (i == (sqlPattern.length() - 1)) { + throw invalidEscapeSequence(sqlPattern, i); + } + char nextChar = sqlPattern.charAt(i + 1); + if ((nextChar == '_') + || (nextChar == '%') + || (nextChar == escapeChar)) { + if (JAVA_REGEX_SPECIALS.indexOf(nextChar) >= 0) { + javaPattern.append('\\'); + } + javaPattern.append(nextChar); + i++; + } else { + throw invalidEscapeSequence(sqlPattern, i); + } + } else if (c == '_') { + javaPattern.append('.'); + } else if (c == '%') { + javaPattern.append(".*"); + } else { + if (JAVA_REGEX_SPECIALS.indexOf(c) >= 0) { + javaPattern.append('\\'); + } + javaPattern.append(c); + } + } + javaPattern.append('$'); + return javaPattern.toString(); + } + private static RuntimeException invalidEscapeCharacter(String s) { return new RuntimeException( "Invalid escape character '" + s + "'"); diff --git a/core/src/test/java/org/apache/calcite/runtime/LikeTest.java b/core/src/test/java/org/apache/calcite/runtime/LikeTest.java new file mode 100644 index 000000000000..549ddc3571ac --- /dev/null +++ b/core/src/test/java/org/apache/calcite/runtime/LikeTest.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.runtime; + +import org.junit.jupiter.api.Test; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; + +/** Unit tests for {@link Like}. */ +class LikeTest { + + /** Test case for + * [CALCITE-7693] + * Move MongoDB LIKE-to-regex conversion into runtime.Like for consistency. */ + @Test void testSqlToRegexAnchored() { + assertThat(Like.sqlToRegexAnchored("", null), is("^$")); + assertThat(Like.sqlToRegexAnchored("abc", null), is("^abc$")); + assertThat(Like.sqlToRegexAnchored("A%", null), is("^A.*$")); + assertThat(Like.sqlToRegexAnchored("A_", null), is("^A.$")); + assertThat(Like.sqlToRegexAnchored("%abc%", null), is("^.*abc.*$")); + // '.' is an ordinary SQL LIKE character; it must be escaped so that it is + // literal in the generated regex. + assertThat(Like.sqlToRegexAnchored("A.B%", null), is("^A\\.B.*$")); + } + + @Test void testSqlToRegexAnchoredWithEscape() { + // '\' escapes the wildcards, making them literal. + assertThat(Like.sqlToRegexAnchored("A\\_B\\%C%", "\\"), is("^A_B%C.*$")); + assertThat(Like.sqlToRegexAnchored("BROOKLYN\\%", "\\"), is("^BROOKLYN%$")); + // A custom escape character. + assertThat(Like.sqlToRegexAnchored("BROOKLYN!%", "!"), is("^BROOKLYN%$")); + // The escape character followed by itself is a literal escape character. + assertThat(Like.sqlToRegexAnchored("A\\\\B", "\\"), is("^A\\\\B$")); + } +} diff --git a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoFilter.java b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoFilter.java index c3d12a84be0b..93f96d79b079 100644 --- a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoFilter.java +++ b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoFilter.java @@ -31,6 +31,7 @@ import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexUtil; +import org.apache.calcite.runtime.Like; import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.util.JsonBuilder; @@ -321,8 +322,8 @@ private Void translateLike(RexCall call, final RexLiteral patternLiteral = (RexLiteral) right; final String sqlPattern = patternLiteral.getValue2().toString(); - final @Nullable Character escapeChar = escapeChar(call); - final String finalRegex = sqlLikeToMongoRegex(sqlPattern, escapeChar); + final @Nullable String escapeStr = escapeStr(call); + final String finalRegex = Like.sqlToRegexAnchored(sqlPattern, escapeStr); switch (left.getKind()) { case INPUT_REF: @@ -364,8 +365,8 @@ private Void translateNotLike(RexCall call, List> orMapList) final RexLiteral patternLiteral = (RexLiteral) right; final String sqlPattern = patternLiteral.getValue2().toString(); - final @Nullable Character escapeChar = escapeChar(call); - final String finalRegex = sqlLikeToMongoRegex(sqlPattern, escapeChar); + final @Nullable String escapeStr = escapeStr(call); + final String finalRegex = Like.sqlToRegexAnchored(sqlPattern, escapeStr); final String name; switch (left.getKind()) { @@ -404,8 +405,8 @@ private static RexNode stripCast(RexNode node) { return node; } - /** Returns the escape character declared in a LIKE expression, or null. */ - private static @Nullable Character escapeChar(RexCall call) { + /** Returns the escape string declared in a LIKE expression, or null. */ + private static @Nullable String escapeStr(RexCall call) { if (call.operands.size() != 3) { return null; } @@ -417,72 +418,7 @@ private static RexNode stripCast(RexNode node) { if (escape.length() != 1) { throw new AssertionError("cannot translate LIKE with multi-character escape: " + call); } - return escape.charAt(0); - } - - /** - * Converts SQL LIKE pattern to MongoDB regex pattern. - * - *

      SQL: {@code %} matches zero or more characters, {@code _} matches a single - * character. MongoDB: {@code .*} matches zero or more characters, {@code .} - * matches a single character. - * - *

      We add {@code ^} and {@code $} anchors so that the entire string matches - * the pattern, just as SQL LIKE does. - */ - private static String sqlLikeToMongoRegex(String sqlPattern, @Nullable Character escapeChar) { - final StringBuilder regex = new StringBuilder(sqlPattern.length() * 2); - regex.append("^"); - for (int i = 0; i < sqlPattern.length(); i++) { - char c = sqlPattern.charAt(i); - if (escapeChar != null && c == escapeChar) { - if (i == sqlPattern.length() - 1) { - throw new AssertionError("Invalid escape sequence at end of LIKE pattern: " - + sqlPattern); - } - final char nextChar = sqlPattern.charAt(i + 1); - if (nextChar == '%' || nextChar == '_' || nextChar == escapeChar) { - regex.append(escapeRegexChar(nextChar)); - i++; - } else { - throw new AssertionError("Invalid escape sequence in LIKE pattern: " + sqlPattern); - } - } else if (c == '%') { - regex.append(".*"); - } else if (c == '_') { - regex.append('.'); - } else { - regex.append(escapeRegexChar(c)); - } - } - regex.append("$"); - return regex.toString(); - } - - /** - * Escapes a character for use in a MongoDB regex if it's a special regex character. - */ - private static String escapeRegexChar(char c) { - // MongoDB regex special characters that need escaping - switch (c) { - case '\\': - case '^': - case '$': - case '.': - case '|': - case '?': - case '*': - case '+': - case '(': - case ')': - case '[': - case ']': - case '{': - case '}': - return "\\" + c; - default: - return String.valueOf(c); - } + return escape; } } } From eac4d851da5688d520e8ee1ae229fbfc5b19a97a Mon Sep 17 00:00:00 2001 From: Ruben Quesada Lopez Date: Fri, 31 Jul 2026 16:27:34 +0200 Subject: [PATCH 455/562] [CALCITE-5584] Publish website action only triggers for the last commit in the chain --- .../publish-non-release-website-updates.yml | 23 ++++++++++++++----- 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/.github/workflows/publish-non-release-website-updates.yml b/.github/workflows/publish-non-release-website-updates.yml index c72a64233ffd..0dde26d7644b 100644 --- a/.github/workflows/publish-non-release-website-updates.yml +++ b/.github/workflows/publish-non-release-website-updates.yml @@ -35,15 +35,26 @@ jobs: with: fetch-depth: 0 ref: site - - name: Cherry pick the commit to site + - name: Cherry pick all commits in push to site + env: + COMMITS: ${{ toJson(github.event.commits) }} run: | git config user.email ${{ github.actor }}@users.noreply.github.com git config user.name ${{ github.actor }} - git cherry-pick --strategy=recursive -X theirs $GITHUB_SHA - if [ $? -neq 0 ]; then - git status | sed -n 's/deleted by us://p' | xargs git add - git cherry-pick --continue --no-edit - fi + + # Extract each commit SHA from the push payload and iterate chronologically + echo "$COMMITS" | jq -r '.[].id' | while read -r commit_sha; do + echo "----------------------------------------" + echo "Cherry-picking commit: $commit_sha" + echo "----------------------------------------" + + if ! git cherry-pick --strategy=recursive -X theirs "$commit_sha"; then + echo "Handling cherry-pick conflict..." + git status | sed -n 's/deleted by us://p' | xargs -r git add + git cherry-pick --continue --no-edit + fi + done + git push origin site publish-website: From bfd490490cf9febbe1fde8c239265d416d437fb8 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Wed, 5 Aug 2026 19:01:23 -0700 Subject: [PATCH 456/562] [CALCITE-7451] REINTERPRET should not be used in logical plans Signed-off-by: Mihai Budiu --- .../adapter/enumerable/RexImpTable.java | 15 ------- .../enumerable/RexToLixTranslator.java | 18 ++++++++ .../java/org/apache/calcite/plan/Strong.java | 2 +- .../apache/calcite/rel/rules/CoreRules.java | 5 ++- .../calcite/rel/rules/ReduceDecimalsRule.java | 11 +++++ .../org/apache/calcite/rex/RexBuilder.java | 41 +++++++++++++------ .../java/org/apache/calcite/rex/RexUtil.java | 20 +++++++++ .../java/org/apache/calcite/sql/SqlKind.java | 5 +++ .../calcite/sql/fun/SqlStdOperatorTable.java | 7 ++++ .../calcite/sql2rel/SqlToRelConverter.java | 27 ++---------- .../apache/calcite/test/RelOptRulesTest.java | 1 + .../calcite/test/SqlToRelConverterTest.java | 34 +++++++++++++++ .../calcite/test/SqlToRelConverterTest.xml | 28 +++++++++++++ core/src/test/resources/sql/operator.iq | 14 +++++++ 14 files changed, 175 insertions(+), 53 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java index 12663a2bba73..36fd19f67bb5 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java @@ -513,7 +513,6 @@ import static org.apache.calcite.sql.fun.SqlStdOperatorTable.RAND_INTEGER; import static org.apache.calcite.sql.fun.SqlStdOperatorTable.RANK; import static org.apache.calcite.sql.fun.SqlStdOperatorTable.REGR_COUNT; -import static org.apache.calcite.sql.fun.SqlStdOperatorTable.REINTERPRET; import static org.apache.calcite.sql.fun.SqlStdOperatorTable.REPLACE; import static org.apache.calcite.sql.fun.SqlStdOperatorTable.RIGHTSHIFT; import static org.apache.calcite.sql.fun.SqlStdOperatorTable.ROUND; @@ -1174,7 +1173,6 @@ void populate2() { define(SAFE_CAST, new CastImplementor()); define(TRY_CAST, new CastImplementor()); - define(REINTERPRET, new ReinterpretImplementor()); define(CONVERT, new ConvertImplementor()); define(TRANSLATE, new TranslateImplementor()); @@ -3800,19 +3798,6 @@ private static RelDataType nullifyType(JavaTypeFactory typeFactory, } } - /** Implementor for the {@code REINTERPRET} internal SQL operator. */ - private static class ReinterpretImplementor extends AbstractRexCallImplementor { - ReinterpretImplementor() { - super("reinterpret", NullPolicy.STRICT, false); - } - - @Override Expression implementSafe(final RexToLixTranslator translator, - final RexCall call, final List argValueList) { - assert call.getOperands().size() == 1; - return argValueList.get(0); - } - } - /** Implementor for sort_array. */ private static class SortArrayImplementor extends AbstractRexCallImplementor { SortArrayImplementor() { diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java index 9bad3dc3f5ff..cf9cbde5e378 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java @@ -720,6 +720,20 @@ private Expression getConvertExpression( case INTEGER: case TINYINT: case SMALLINT: { + if (sourceType.getFamily() == SqlTypeFamily.INTERVAL_DAY_TIME + || sourceType.getFamily() == SqlTypeFamily.INTERVAL_YEAR_MONTH) { + // An interval is represented by its count of base units (milliseconds + // or months); the cast yields the count of the interval's end unit, + // truncated towards zero. + final BigDecimal multiplier = + sourceType.getSqlTypeName().getEndUnit().multiplier; + final Expression ticks = EnumUtils.convert(operand, long.class); + final Expression scaled = multiplier.equals(BigDecimal.ONE) + ? ticks + : Expressions.divide(ticks, + Expressions.constant(multiplier.longValueExact())); + return EnumUtils.convert(scaled, typeFactory.getJavaClass(targetType)); + } if (SqlTypeName.NUMERIC_TYPES.contains(sourceType.getSqlTypeName())) { Type javaClass = typeFactory.getJavaClass(targetType); Primitive primitive = Primitive.of(javaClass); @@ -1393,6 +1407,10 @@ private static Expression scaleValue( // multiplyDivide cannot handle DECIMALs, but for DECIMAL // target types the result is already scaled. && targetType.getSqlTypeName() != SqlTypeName.DECIMAL + // Integer targets divide before narrowing, in getConvertExpression; + // dividing here, after the narrowing, would overflow for tick counts + // wider than the target type. + && !SqlTypeName.INT_TYPES.contains(targetType.getSqlTypeName()) && (sourceFamily == SqlTypeFamily.INTERVAL_YEAR_MONTH || sourceFamily == SqlTypeFamily.INTERVAL_DAY_TIME)) { // Scale to the given field. diff --git a/core/src/main/java/org/apache/calcite/plan/Strong.java b/core/src/main/java/org/apache/calcite/plan/Strong.java index ee349b837c48..b6aa9678ed8d 100644 --- a/core/src/main/java/org/apache/calcite/plan/Strong.java +++ b/core/src/main/java/org/apache/calcite/plan/Strong.java @@ -358,7 +358,7 @@ private static Map createPolicyMap() { map.put(SqlKind.DIVIDE, Policy.ANY); map.put(SqlKind.CAST, Policy.ANY); - map.put(SqlKind.REINTERPRET, Policy.ANY); + map.put(SqlKind.REINTERPRET, Policy.ANY); // deprecated, kept until removed map.put(SqlKind.TRIM, Policy.ANY); map.put(SqlKind.LTRIM, Policy.ANY); map.put(SqlKind.RTRIM, Policy.ANY); diff --git a/core/src/main/java/org/apache/calcite/rel/rules/CoreRules.java b/core/src/main/java/org/apache/calcite/rel/rules/CoreRules.java index 11708c5facf9..98cbeaae2194 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/CoreRules.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/CoreRules.java @@ -221,7 +221,10 @@ private CoreRules() {} /** Rule that reduces operations on the DECIMAL type, such as casts or * arithmetic, into operations involving more primitive types such as BIGINT - * and DOUBLE. */ + * and DOUBLE. + * + * @deprecated See {@link ReduceDecimalsRule}. */ + @Deprecated // to be removed before 2.0 public static final ReduceDecimalsRule CALC_REDUCE_DECIMALS = ReduceDecimalsRule.Config.DEFAULT.toRule(); diff --git a/core/src/main/java/org/apache/calcite/rel/rules/ReduceDecimalsRule.java b/core/src/main/java/org/apache/calcite/rel/rules/ReduceDecimalsRule.java index 06bf0f67eecd..8918bf55f9e8 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/ReduceDecimalsRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/ReduceDecimalsRule.java @@ -73,8 +73,19 @@ * would like to push down decimal operations to an external database. * * @see CoreRules#CALC_REDUCE_DECIMALS + * + * @deprecated The rule rewrites decimal values as their unscaled BIGINT + * representation, connected by REINTERPRET operators. This assumes a physical + * representation of DECIMAL values that only an adapter or calling convention + * knows, so the rewritten plan is no longer a logical plan. This rule is opt-in + * for engines that represent DECIMAL values as scaled integers. The + * REINTERPRET operator is deprecated. */ +@Deprecated // to be removed before 2.0 @Value.Enclosing +// Immutables copies this suppression into the generated class, which +// references this deprecated class +@SuppressWarnings("deprecation") public class ReduceDecimalsRule extends RelRule implements TransformationRule { diff --git a/core/src/main/java/org/apache/calcite/rex/RexBuilder.java b/core/src/main/java/org/apache/calcite/rex/RexBuilder.java index 95030b5b9053..667de88227f5 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexBuilder.java +++ b/core/src/main/java/org/apache/calcite/rex/RexBuilder.java @@ -874,9 +874,6 @@ public RexNode makeCast( } return literal2; } - } else if (SqlTypeUtil.isExactNumeric(type) - && SqlTypeUtil.isInterval(exp.getType())) { - return makeCastIntervalToExact(pos, type, exp); } else if (sqlType == SqlTypeName.BOOLEAN && SqlTypeUtil.isExactNumeric(exp.getType())) { return makeCastExactToBoolean(type, exp); @@ -1125,16 +1122,6 @@ private RexNode makeCastBooleanToExact(RelDataType toType, RexNode exp) { casted, makeNullLiteral(toType))); } - private RexNode makeCastIntervalToExact(SqlParserPos pos, RelDataType toType, RexNode exp) { - final TimeUnit endUnit = exp.getType().getSqlTypeName().getEndUnit(); - final TimeUnit baseUnit = baseUnit(exp.getType().getSqlTypeName()); - final BigDecimal multiplier = baseUnit.multiplier; - final BigDecimal divider = endUnit.multiplier; - RexNode value = - multiplyDivide(pos, decodeIntervalOrDecimal(pos, exp), multiplier, divider); - return ensureType(pos, toType, value, false); - } - public RexNode multiplyDivide(RexNode e, BigDecimal multiplier, BigDecimal divider) { return multiplyDivide(SqlParserPos.ZERO, e, multiplier, divider); @@ -1177,7 +1164,10 @@ public RexNode multiplyDivide(SqlParserPos pos, RexNode e, BigDecimal multiplier * arithmetic, but is often required for rounding and * explicit casts. * @return the integer reinterpreted as an opaque decimal type + * + * @deprecated The REINTERPRET operator is deprecated */ + @Deprecated // to be removed before 2.0 public RexNode encodeIntervalOrDecimal( RexNode value, RelDataType type, @@ -1185,6 +1175,11 @@ public RexNode encodeIntervalOrDecimal( return encodeIntervalOrDecimal(SqlParserPos.ZERO, value, type, checkOverflow); } + /** Encodes an interval or decimal, with an explicit parser position. + * + * @deprecated The REINTERPRET operator is deprecated + */ + @Deprecated // to be removed before 2.0 public RexNode encodeIntervalOrDecimal( SqlParserPos pos, RexNode value, @@ -1201,11 +1196,19 @@ public RexNode encodeIntervalOrDecimal( * * @param node the interval or decimal value as an opaque type * @return an integer representation of the decimal value + * + * @deprecated The REINTERPRET operator is deprecated */ + @Deprecated // to be removed before 2.0 public RexNode decodeIntervalOrDecimal(RexNode node) { return decodeIntervalOrDecimal(SqlParserPos.ZERO, node); } + /** Decodes an interval or decimal, with an explicit parser position. + * + * @deprecated The REINTERPRET operator is deprecated + */ + @Deprecated // to be removed before 2.0 public RexNode decodeIntervalOrDecimal(SqlParserPos pos, RexNode node) { assert SqlTypeUtil.isDecimal(node.getType()) || SqlTypeUtil.isInterval(node.getType()); @@ -1289,7 +1292,13 @@ public RexNode makeAbstractCast( * @param exp expression to be casted * @param checkOverflow whether an overflow check is required * @return a RexCall with two operands and a special return type + * + * @deprecated The REINTERPRET operator is deprecated; its semantics depend + * on the physical representation of values, which only an adapter or + * calling convention knows. Use {@link #makeCast(RelDataType, RexNode)} + * instead */ + @Deprecated // to be removed before 2.0 public RexNode makeReinterpretCast( RelDataType type, RexNode exp, @@ -1305,7 +1314,13 @@ public RexNode makeReinterpretCast( * @param exp expression to be cast * @param checkOverflow whether an overflow check is required * @return a RexCall with two operands and a special return type + * + * @deprecated The REINTERPRET operator is deprecated; its semantics depend + * on the physical representation of values, which only an adapter or + * calling convention knows. Use + * {@link #makeCast(SqlParserPos, RelDataType, RexNode)} instead */ + @Deprecated // to be removed before 2.0 public RexNode makeReinterpretCast( SqlParserPos pos, RelDataType type, diff --git a/core/src/main/java/org/apache/calcite/rex/RexUtil.java b/core/src/main/java/org/apache/calcite/rex/RexUtil.java index 2faa2633b004..99045f0a3cb6 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexUtil.java +++ b/core/src/main/java/org/apache/calcite/rex/RexUtil.java @@ -1067,7 +1067,12 @@ public static boolean containsFieldAccess(RexNode node) { * @param expr expression possibly in need of expansion * @param recurse whether to check nested calls * @return whether the expression requires expansion + * + * @deprecated Used only by + * {@link org.apache.calcite.rel.rules.ReduceDecimalsRule}, which is + * deprecated */ + @Deprecated // to be removed before 2.0 public static boolean requiresDecimalExpansion( RexNode expr, boolean recurse) { @@ -1118,7 +1123,12 @@ public static boolean requiresDecimalExpansion( /** * Determines whether any operand of a set requires decimal expansion. + * + * @deprecated Used only by + * {@link org.apache.calcite.rel.rules.ReduceDecimalsRule}, which is + * deprecated */ + @Deprecated // to be removed before 2.0 public static boolean requiresDecimalExpansion( List operands, boolean recurse) { @@ -1136,7 +1146,12 @@ public static boolean requiresDecimalExpansion( /** * Returns whether a {@link RexProgram} contains expressions which require * decimal expansion. + * + * @deprecated Used only by + * {@link org.apache.calcite.rel.rules.ReduceDecimalsRule}, which is + * deprecated */ + @Deprecated // to be removed before 2.0 public static boolean requiresDecimalExpansion( RexProgram program, boolean recurse) { @@ -1149,6 +1164,11 @@ public static boolean requiresDecimalExpansion( return false; } + /** Returns whether a REINTERPRET call performs an overflow check. + * + * @deprecated The REINTERPRET operator is deprecated + */ + @Deprecated // to be removed before 2.0 public static boolean canReinterpretOverflow(RexCall call) { assert call.isA(SqlKind.REINTERPRET) : "call is not a reinterpret"; return call.operands.size() > 1; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlKind.java b/core/src/main/java/org/apache/calcite/sql/SqlKind.java index a70c71df7614..92bbfe4c5997 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlKind.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlKind.java @@ -1005,6 +1005,11 @@ public enum SqlKind { /** * The internal REINTERPRET operator (meaning a reinterpret cast). * An internal operator that does not appear in SQL syntax. + * + *

      Do not use. The + * {@link org.apache.calcite.sql.fun.SqlStdOperatorTable#REINTERPRET} + * operator is deprecated and will be removed, together with this value; + * use {@link #CAST} instead. */ REINTERPRET, diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java index 20a5e690ffe6..be9f23abe672 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java @@ -1786,7 +1786,14 @@ public class SqlStdOperatorTable extends ReflectiveSqlOperatorTable { * it accepts one operand and stores the target type as the return type. It * performs an overflow check if it has any second operand, whether * true or not. + * + * @deprecated The semantics of REINTERPRET depend on the physical + * representation of values, which only an adapter or calling convention + * knows; a logical plan must not contain this operator. + * The enumerable convention does not implement it. + * Use {@link #CAST} instead. */ + @Deprecated // to be removed before 2.0 public static final SqlSpecialOperator REINTERPRET = new SqlSpecialOperator("Reinterpret", SqlKind.REINTERPRET) { @Override public SqlOperandCountRange getOperandCountRange() { diff --git a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java index 3ca7ae44f854..5c524eee17b4 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java @@ -6572,23 +6572,12 @@ private class HistogramShuttle extends RexShuttle { if (histogramOp != null) { final RelDataType histogramType = computeHistogramType(type); - // For DECIMAL, since it's already represented as a bigint we - // want to do a reinterpretCast instead of a cast to avoid - // losing any precision. - boolean reinterpretCast = - type.getSqlTypeName() == SqlTypeName.DECIMAL; - // Replace original expression with CAST of not one // of the supported types if (histogramType != type) { exprs = new ArrayList<>(exprs); - exprs.set( - 0, - reinterpretCast - ? rexBuilder.makeReinterpretCast( - call.getParserPosition(), histogramType, exprs.get(0), - rexBuilder.makeLiteral(false)) - : rexBuilder.makeCast(call.getParserPosition(), histogramType, exprs.get(0))); + exprs.set(0, + rexBuilder.makeCast(call.getParserPosition(), histogramType, exprs.get(0))); } RexNode over = @@ -6615,16 +6604,8 @@ private class HistogramShuttle extends RexShuttle { // If needed, post Cast result back to original // type. if (histogramType != type) { - if (reinterpretCast) { - histogramCall = - rexBuilder.makeReinterpretCast(call.getParserPosition(), - type, - histogramCall, - rexBuilder.makeLiteral(false)); - } else { - histogramCall = - rexBuilder.makeCast(call.getParserPosition(), type, histogramCall); - } + histogramCall = + rexBuilder.makeCast(call.getParserPosition(), type, histogramCall); } return histogramCall; diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index 0cfb76b9d5de..ca5cc42385f5 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -10812,6 +10812,7 @@ public interface Config extends RelRule.Config { * Test case for * [CALCITE-3319] * AssertionError for ReduceDecimalsRule. */ + @SuppressWarnings("deprecation") // tests the deprecated ReduceDecimalsRule @Test void testReduceDecimal() { final String sql = "select ename from emp where sal > cast (100.0 as decimal(4, 1))"; sql(sql) diff --git a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java index 90ed8a6333be..05093c2d3c2e 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java @@ -6355,6 +6355,40 @@ void checkUserDefinedOrderByOver(NullCollation nullCollation) { + " supported")); } + /** Test case for + * [CALCITE-7451] + * REINTERPRET should not be used in logical plans. + * + *

      Casting an interval to an exact numeric type, which the TIMESTAMPDIFF + * family of functions relies on, remains a CAST call in the logical plan; + * it used to be rewritten in terms of the deprecated REINTERPRET + * operator. */ + @Test void testCastIntervalToNumericNoReinterpret() { + final String sql = "select cast(x as integer) as i,\n" + + " cast(x as decimal(6, 1)) as d,\n" + + " timestampdiff(minute, ts, ts) as m\n" + + "from (values (interval '90' minute,\n" + + " timestamp '2020-01-01 00:00:00')) as t(x, ts)"; + final String plan = RelOptUtil.toString(sql(sql).toRel()); + assertThat(plan, not(containsString("Reinterpret"))); + sql(sql).ok(); + } + + /** Test case for + * [CALCITE-7451] + * REINTERPRET should not be used in logical plans. + * + *

      The deprecated REINTERPRET operator must not appear in a logical plan; + * FLOOR and CEIL of an interval literal used to produce one. */ + @Test void testFloorCeilOfIntervalLiteral() { + final String sql = "select floor(interval '3:4:5' hour to second) as f,\n" + + " ceil(interval '3:4:5' hour to second) as c\n" + + "from emp"; + final String plan = RelOptUtil.toString(sql(sql).toRel()); + assertThat(plan, not(containsString("Reinterpret"))); + sql(sql).ok(); + } + /** Test case of * [CALCITE-5406] * Support the SELECT DISTINCT ON statement for PostgreSQL dialect. */ diff --git a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml index e916c0813190..bafc21fc6261 100644 --- a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml @@ -802,6 +802,21 @@ GROUP BY GROUPING SETS ( LogicalAggregate(group=[{0, 1}], groups=[[{0, 1}, {0}]]) LogicalProject(EMPNO=[$0], EXPR$1=[CASE(SEARCH($1, Sarg['Eric':VARCHAR(20), 'Fred':VARCHAR(20)]:VARCHAR(20)), 'Manager', 'Other ')]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +]]> + + + + + + + + @@ -2636,6 +2651,19 @@ from (values (interval '3:4:5' hour to second)) as t(x)]]> =($0, 0), $0, -($0, 3599999)), 3600000), 3600000)], C=[*(/INT(CASE(>=($0, 0), +($0, 3599999), $0), 3600000), 3600000)]) LogicalValues(tuples=[[{ 11045000 }]]) +]]> + + + + + + + + =(11045000:INTERVAL HOUR TO SECOND, 0), 11045000:INTERVAL HOUR TO SECOND, -(11045000:INTERVAL HOUR TO SECOND, 3599999:INTERVAL HOUR TO SECOND)), 3600000), 3600000)], C=[*(/INT(CASE(>=(11045000:INTERVAL HOUR TO SECOND, 0), +(11045000:INTERVAL HOUR TO SECOND, 3599999:INTERVAL HOUR TO SECOND), 11045000:INTERVAL HOUR TO SECOND), 3600000), 3600000)]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) ]]> diff --git a/core/src/test/resources/sql/operator.iq b/core/src/test/resources/sql/operator.iq index 45bee30f0c96..5b0a082595d2 100644 --- a/core/src/test/resources/sql/operator.iq +++ b/core/src/test/resources/sql/operator.iq @@ -883,4 +883,18 @@ select floor(interval '2' hour + interval '90' minute) = interval '3' hour as fa !ok +# [CALCITE-7451] REINTERPRET should not be used in logical plans +# CAST of a non-literal interval to an exact numeric type. A DECIMAL target +# preserves the fractional part; an integer target truncates towards zero. +select cast(x as decimal(2,1)) as d, cast(x as integer) as i +from (values (interval '1.29' second(1,2))) as t(x); ++-----+---+ +| D | I | ++-----+---+ +| 1.2 | 1 | ++-----+---+ +(1 row) + +!ok + # End operator.iq From 8d85cb13bf0cf82a08a5db7aadcaa57c50b7485a Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Mon, 10 Aug 2026 16:21:57 -0700 Subject: [PATCH 457/562] [CALCITE-7705] LISTAGG result type is never nullable Signed-off-by: Mihai Budiu --- .../calcite/sql/fun/SqlLibraryOperators.java | 4 ++-- .../calcite/sql/fun/SqlStdOperatorTable.java | 3 ++- core/src/test/resources/sql/agg.iq | 24 +++++++++++++++++++ .../apache/calcite/test/SqlOperatorTest.java | 15 ++++++++++++ 4 files changed, 43 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java index 6e328261c65c..aef50bbc63b8 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java @@ -846,7 +846,7 @@ static RelDataType deriveTypeSplit(SqlOperatorBinding operatorBinding, @LibraryOperator(libraries = {BIG_QUERY, POSTGRESQL}, exceptLibraries = {REDSHIFT}) public static final SqlAggFunction STRING_AGG = SqlBasicAggFunction - .create(SqlKind.STRING_AGG, ReturnTypes.ARG0_NULLABLE, + .create(SqlKind.STRING_AGG, ReturnTypes.ARG0_NULLABLE_IF_EMPTY, OperandTypes.STRING.or(OperandTypes.STRING_STRING)) .withFunctionType(SqlFunctionCategory.SYSTEM) .withSyntax(SqlSyntax.ORDERED_FUNCTION); @@ -862,7 +862,7 @@ static RelDataType deriveTypeSplit(SqlOperatorBinding operatorBinding, SqlBasicAggFunction .create(SqlKind.GROUP_CONCAT, ReturnTypes.andThen(ReturnTypes::stripOrderBy, - ReturnTypes.ARG0_NULLABLE), + ReturnTypes.ARG0_NULLABLE_IF_EMPTY), OperandTypes.STRING.or(OperandTypes.STRING_STRING)) .withFunctionType(SqlFunctionCategory.SYSTEM) .withAllowsNullTreatment(false) diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java index be9f23abe672..ef7240d35e21 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java @@ -2540,7 +2540,8 @@ public class SqlStdOperatorTable extends ReflectiveSqlOperatorTable { * The LISTAGG operator. String aggregator function. */ public static final SqlAggFunction LISTAGG = - new SqlListaggAggFunction(SqlKind.LISTAGG, ReturnTypes.ARG0_NULLABLE); + new SqlListaggAggFunction(SqlKind.LISTAGG, + ReturnTypes.ARG0_NULLABLE_IF_EMPTY); /** * The FUSION operator. Multiset aggregator function. diff --git a/core/src/test/resources/sql/agg.iq b/core/src/test/resources/sql/agg.iq index cbd50747b3a0..d6557e34f1ec 100644 --- a/core/src/test/resources/sql/agg.iq +++ b/core/src/test/resources/sql/agg.iq @@ -3380,6 +3380,30 @@ select listagg(ename) as combined_name from emp; !ok +# [CALCITE-7705] LISTAGG result type is never nullable +# Empty input yields NULL even though the argument is NOT NULL. +select listagg(v, ',') as r from (values ('a')) as t(v) where false; ++---+ +| R | ++---+ +| | ++---+ +(1 row) + +!ok + +# The IS NULL test must not be simplified away based on the result type. +select r is null as n from ( + select listagg(v, ',') as r from (values ('a')) as t(v) where false); ++------+ +| N | ++------+ +| true | ++------+ +(1 row) + +!ok + select listagg(ename) within group(order by gender, ename) as combined_name from emp; +-------------------------------------------------------+ | COMBINED_NAME | diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java index cdae823649bd..c6be7efedfec 100644 --- a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java +++ b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java @@ -12935,6 +12935,14 @@ private static void checkDecodeFunc(SqlOperatorFixture f) { false); f.checkAggType("listagg('test')", "CHAR(4) NOT NULL"); f.checkAggType("listagg('test', ', ')", "CHAR(4) NOT NULL"); + // Test case for [CALCITE-7705] + // LISTAGG result type is never nullable + // Nullable without GROUP BY even for a non-nullable argument, since the + // input may be empty + f.checkColumnType("select listagg('test') from (values (1))", "CHAR(4)"); + // A FILTER clause may exclude all rows, so the result is nullable + f.checkColumnType("select listagg('test') filter (where x > 1) " + + "from (values (1)) as t(x) group by x", "CHAR(4)"); final String[] values1 = {"'hello'", "CAST(null AS CHAR)", "'world'", "'!'"}; f.checkAgg("listagg(x)", values1, isSingle("hello,world,! ")); final String[] values2 = {"0", "1", "2", "3"}; @@ -12950,6 +12958,10 @@ private static void checkDecodeFunc(SqlOperatorFixture f) { private static void checkStringAggFunc(SqlOperatorFixture f) { final String[] values = {"'x'", "null", "'yz'"}; + // Test case for [CALCITE-7705] + // LISTAGG result type is never nullable + f.checkColumnType("select string_agg('x', ',') from (values (1))", + "CHAR(1)"); f.checkAgg("string_agg(x)", values, isSingle("x ,yz")); f.checkAgg("string_agg(x,':')", values, isSingle("x :yz")); f.checkAgg("string_agg(x,':' order by x)", values, isSingle("x :yz")); @@ -13008,6 +13020,9 @@ private static void checkStringAggFuncFails(SqlOperatorFixture f) { private static void checkGroupConcatFunc(SqlOperatorFixture f) { final String[] values = {"'x'", "null", "'yz'"}; + // Test case for [CALCITE-7705] + // LISTAGG result type is never nullable + f.checkColumnType("select group_concat('x') from (values (1))", "CHAR(1)"); f.checkAgg("group_concat(x)", values, isSingle("x ,yz")); f.checkAgg("group_concat(x,':')", values, isSingle("x :yz")); f.checkAgg("group_concat(x,':' order by x)", values, isSingle("x :yz")); From 179b23ad95114de17f25d83217abe51de8b4eb9d Mon Sep 17 00:00:00 2001 From: zzwqqq Date: Tue, 11 Aug 2026 10:22:33 +0800 Subject: [PATCH 458/562] [CALCITE-7702] JoinAggregateTransposeRule produces a non-equivalent plan when the aggregate with empty input and empty group set --- .../rel/rules/JoinAggregateTransposeRule.java | 5 ++ .../test/JoinAggregateTransposeRuleTest.java | 33 +++++++++ .../test/JoinAggregateTransposeRuleTest.xml | 70 +++++++++++++++++++ .../test/resources/sql/join-agg-transpose.iq | 14 ++++ 4 files changed, 122 insertions(+) diff --git a/core/src/main/java/org/apache/calcite/rel/rules/JoinAggregateTransposeRule.java b/core/src/main/java/org/apache/calcite/rel/rules/JoinAggregateTransposeRule.java index 99cf46141d7c..39393e879a34 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/JoinAggregateTransposeRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/JoinAggregateTransposeRule.java @@ -106,6 +106,11 @@ protected JoinAggregateTransposeRule(Config config) { // in case we decide to extend this rule for lazy aggregation. && isAggregateSupported(left, true) && groupOutput.contains(info.leftSet()) + // An aggregate returns one row on empty input if its group set is empty. + // Pull-up adds group keys and may lose that row. Require the input to be + // known non-empty. + && (!left.getGroupSet().isEmpty() + || Boolean.FALSE.equals(mq.isEmpty(left.getInput()))) // The right side must be unique on its join keys (no row duplication) && Boolean.TRUE.equals(mq.areColumnsUnique(right, info.rightSet())); } diff --git a/core/src/test/java/org/apache/calcite/test/JoinAggregateTransposeRuleTest.java b/core/src/test/java/org/apache/calcite/test/JoinAggregateTransposeRuleTest.java index 97f784522a61..fd1a9ad56898 100644 --- a/core/src/test/java/org/apache/calcite/test/JoinAggregateTransposeRuleTest.java +++ b/core/src/test/java/org/apache/calcite/test/JoinAggregateTransposeRuleTest.java @@ -161,6 +161,39 @@ private static RelOptFixture sql(String sql) { sql(sql).withRule(CoreRules.JOIN_AGGREGATE_TRANSPOSE).checkUnchanged(); } + /** + * Tests that the rule does not pull an aggregate with an empty group set when + * its input is empty. + */ + @Test void testNoPullAggregateWithEmptyGroupSetOnEmptyInput() { + final String sql = "select g.emp_count, d.deptno\n" + + "from (select count(*) as emp_count from emp where false) g\n" + + "join (select deptno from dept where deptno = 10) d on true"; + sql(sql).withRule(CoreRules.JOIN_AGGREGATE_TRANSPOSE).checkUnchanged(); + } + + /** + * Tests that the rule does not pull an aggregate with an empty group set when + * its input may be empty. + */ + @Test void testNoPullAggregateWithEmptyGroupSetOnPotentiallyEmptyInput() { + final String sql = "select g.emp_count, d.deptno\n" + + "from (select count(*) as emp_count from emp) g\n" + + "join (select deptno from dept where deptno = 10) d on true"; + sql(sql).withRule(CoreRules.JOIN_AGGREGATE_TRANSPOSE).checkUnchanged(); + } + + /** + * Tests that the rule pulls an aggregate with an empty group set when its + * input is known to be non-empty. + */ + @Test void testPullAggregateWithEmptyGroupSetOnNonEmptyInput() { + final String sql = "select g.emp_count, d.deptno\n" + + "from (select count(*) as emp_count\n" + + " from (values (1)) as v(n)) g\n" + + "join (select deptno from dept where deptno = 10) d on true"; + sql(sql).withRule(CoreRules.JOIN_AGGREGATE_TRANSPOSE).check(); + } @AfterAll static void checkActualAndReferenceFiles() { fixture().diffRepos.checkActualAndReferenceFiles(); diff --git a/core/src/test/resources/org/apache/calcite/test/JoinAggregateTransposeRuleTest.xml b/core/src/test/resources/org/apache/calcite/test/JoinAggregateTransposeRuleTest.xml index 8c7c7c59640e..26aa8d629f2e 100644 --- a/core/src/test/resources/org/apache/calcite/test/JoinAggregateTransposeRuleTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/JoinAggregateTransposeRuleTest.xml @@ -16,6 +16,45 @@ ~ limitations under the License. --> + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/core/src/test/resources/sql/join-agg-transpose.iq b/core/src/test/resources/sql/join-agg-transpose.iq index 3cccb32da2a9..756b13b23332 100644 --- a/core/src/test/resources/sql/join-agg-transpose.iq +++ b/core/src/test/resources/sql/join-agg-transpose.iq @@ -75,4 +75,18 @@ EnumerableCalc(expr#0..3=[{inputs}], DEPTNO=[$t0], TOTAL_SAL=[$t3], DNAME=[$t2]) EnumerableTableScan(table=[[scott, DEPT]]) !plan +# Tests that the rule preserves a row from an aggregate with an empty group set + +select g.emp_count, d.deptno +from (select count(*) as emp_count from emp where false) g +join (select deptno from dept where deptno = 10) d on true; ++-----------+--------+ +| EMP_COUNT | DEPTNO | ++-----------+--------+ +| 0 | 10 | ++-----------+--------+ +(1 row) + +!ok + # End join-agg-transpose.iq From a1285d7490687df1405e9111a1ec7faaa85e8513 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Tue, 4 Aug 2026 16:39:17 -0700 Subject: [PATCH 459/562] [CALCITE-7689] MAP equality compares maps by insertion order Signed-off-by: Mihai Budiu --- core/src/test/resources/sql/blank.iq | 10 +- core/src/test/resources/sql/map-equality.iq | 256 ++++++++++++++++++ .../calcite/linq4j/function/Functions.java | 20 +- .../calcite/linq4j/function/FunctionTest.java | 52 ++++ site/_docs/reference.md | 6 + 5 files changed, 333 insertions(+), 11 deletions(-) create mode 100644 core/src/test/resources/sql/map-equality.iq diff --git a/core/src/test/resources/sql/blank.iq b/core/src/test/resources/sql/blank.iq index a84952e7f933..9a1062876f08 100644 --- a/core/src/test/resources/sql/blank.iq +++ b/core/src/test/resources/sql/blank.iq @@ -224,11 +224,11 @@ select min(m) as min_m, min(r) as min_r from complex_t; -+-----------------+----------------------------------------------------------------------------------------------+-----------------------+-----------------+------------------------------------------------------------------------------------------+------------------------+ -| MAX_A | MAX_M | MAX_R | MIN_A | MIN_M | MIN_R | -+-----------------+----------------------------------------------------------------------------------------------+-----------------------+-----------------+------------------------------------------------------------------------------------------+------------------------+ -| [100, 200, 300] | {physics =96.2, chemistry =91.8, biology =89.5, computer_science=98.7} | {Charlie Chen, 35, c} | [1, 2, 3, 4, 5] | {leadership =88.9, teamwork =94.2, communication =91.5, problem_solving=97.8} | {Alice Johnson, 30, a} | -+-----------------+----------------------------------------------------------------------------------------------+-----------------------+-----------------+------------------------------------------------------------------------------------------+------------------------+ ++-----------------+--------------------------------------------+-----------------------+-----------------+----------------------------------------------------------------------------------------------+------------------------+ +| MAX_A | MAX_M | MAX_R | MIN_A | MIN_M | MIN_R | ++-----------------+--------------------------------------------+-----------------------+-----------------+----------------------------------------------------------------------------------------------+------------------------+ +| [100, 200, 300] | {math =95.5, science=88.0, english=92.3} | {Charlie Chen, 35, c} | [1, 2, 3, 4, 5] | {physics =96.2, chemistry =91.8, biology =89.5, computer_science=98.7} | {Alice Johnson, 30, a} | ++-----------------+--------------------------------------------+-----------------------+-----------------+----------------------------------------------------------------------------------------------+------------------------+ (1 row) !ok diff --git a/core/src/test/resources/sql/map-equality.iq b/core/src/test/resources/sql/map-equality.iq new file mode 100644 index 000000000000..eae2a1bf9292 --- /dev/null +++ b/core/src/test/resources/sql/map-equality.iq @@ -0,0 +1,256 @@ +# map-equality.iq - Tests for comparison of MAP values +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +!use scott +!set outputformat mysql + +# [CALCITE-7689] MAP equality compares maps by insertion order +# Two maps are equal exactly when they contain the same keys and map them to equal values. + +SELECT MAP[1, 2, 3, 4] = MAP[1, 2, 3, 4] AS eq; ++------+ +| EQ | ++------+ +| true | ++------+ +(1 row) + +!ok + +# Same contents in a different insertion order: equal +SELECT MAP[1, 2, 3, 4] = MAP[3, 4, 1, 2] AS eq; ++------+ +| EQ | ++------+ +| true | ++------+ +(1 row) + +!ok + +SELECT MAP['a', 1, 'b', 2] = MAP['b', 2, 'a', 1] AS eq; ++------+ +| EQ | ++------+ +| true | ++------+ +(1 row) + +!ok + +SELECT MAP[1, 2, 3, 4] <> MAP[3, 4, 1, 2] AS ne; ++-------+ +| NE | ++-------+ +| false | ++-------+ +(1 row) + +!ok + +# Different value for key 3: not equal +SELECT MAP[1, 2, 3, 4] = MAP[1, 2, 3, 5] AS eq; ++-------+ +| EQ | ++-------+ +| false | ++-------+ +(1 row) + +!ok + +# Different keys: not equal +SELECT MAP[1, 2, 3, 4] = MAP[1, 2, 4, 4] AS eq; ++-------+ +| EQ | ++-------+ +| false | ++-------+ +(1 row) + +!ok + +# Different sizes: not equal +SELECT MAP[1, 2, 3, 4] = MAP[1, 2] AS eq; ++-------+ +| EQ | ++-------+ +| false | ++-------+ +(1 row) + +!ok + +# IS NOT DISTINCT FROM agrees with = +SELECT MAP[1, 2, 3, 4] IS NOT DISTINCT FROM MAP[3, 4, 1, 2] AS x; ++------+ +| X | ++------+ +| true | ++------+ +(1 row) + +!ok + +SELECT MAP[1, 2, 3, 4] IS DISTINCT FROM MAP[3, 4, 1, 2] AS x; ++-------+ +| X | ++-------+ +| false | ++-------+ +(1 row) + +!ok + +# IN is defined in terms of = +SELECT MAP[3, 4, 1, 2] IN (MAP[1, 2, 3, 4], MAP[5, 6, 7, 8]) AS x; ++------+ +| X | ++------+ +| true | ++------+ +(1 row) + +!ok + +# NULL values inside a map follow IS NOT DISTINCT FROM semantics, as for +# arrays: two NULL values are considered equal +SELECT MAP[1, NULL, 3, 4] = MAP[3, 4, 1, NULL] AS eq; ++------+ +| EQ | ++------+ +| true | ++------+ +(1 row) + +!ok + +SELECT MAP[1, NULL, 3, 4] = MAP[3, 4, 1, 2] AS eq; ++-------+ +| EQ | ++-------+ +| false | ++-------+ +(1 row) + +!ok + +# A map nested inside an array is also compared by contents +SELECT ARRAY[MAP[1, 2, 3, 4]] = ARRAY[MAP[3, 4, 1, 2]] AS eq; ++------+ +| EQ | ++------+ +| true | ++------+ +(1 row) + +!ok + +# Maps whose values are arrays +SELECT MAP[1, ARRAY[1, 2], 3, ARRAY[3]] = MAP[3, ARRAY[3], 1, ARRAY[1, 2]] AS eq; ++------+ +| EQ | ++------+ +| true | ++------+ +(1 row) + +!ok + +# Maps whose values are maps +SELECT MAP[1, MAP[1, 2, 3, 4], 2, MAP[5, 6]] = MAP[2, MAP[5, 6], 1, MAP[3, 4, 1, 2]] AS eq; ++------+ +| EQ | ++------+ +| true | ++------+ +(1 row) + +!ok + +# Maps whose values are rows +SELECT MAP['a', ROW(1, 2), 'b', ROW(3, 4)] = MAP['b', ROW(3, 4), 'a', ROW(1, 2)] AS eq; ++------+ +| EQ | ++------+ +| true | ++------+ +(1 row) + +!ok + +# = in a WHERE clause +SELECT COUNT(*) AS c FROM (VALUES 1) WHERE MAP[1, 2, 3, 4] = MAP[3, 4, 1, 2]; ++---+ +| C | ++---+ +| 1 | ++---+ +(1 row) + +!ok + +# = in a CASE condition +SELECT CASE WHEN MAP[1, 2, 3, 4] = MAP[3, 4, 1, 2] THEN 'same' ELSE 'different' END AS x; ++-----------+ +| X | ++-----------+ +| same | ++-----------+ +(1 row) + +!ok + +# DISTINCT reaches the same verdict as =: one map, not two +SELECT COUNT(*) AS c +FROM (SELECT DISTINCT m FROM (VALUES MAP[1, 2, 3, 4], MAP[3, 4, 1, 2]) AS t(m)); ++---+ +| C | ++---+ +| 1 | ++---+ +(1 row) + +!ok + +# GROUP BY reaches the same verdict as =: one group of two rows +SELECT COUNT(*) AS c +FROM (VALUES MAP[1, 2, 3, 4], MAP[3, 4, 1, 2]) AS t(m) +GROUP BY m; ++---+ +| C | ++---+ +| 2 | ++---+ +(1 row) + +!ok + +# Joining on a map key matches reordered maps +SELECT COUNT(*) AS c +FROM (VALUES MAP[1, 2, 3, 4]) AS l(m) +JOIN (VALUES MAP[3, 4, 1, 2]) AS r(m) ON l.m = r.m; ++---+ +| C | ++---+ +| 1 | ++---+ +(1 row) + +!ok + +# End map-equality.iq diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/function/Functions.java b/linq4j/src/main/java/org/apache/calcite/linq4j/function/Functions.java index f61f65a6d229..89ee4699e1e4 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/function/Functions.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/function/Functions.java @@ -735,17 +735,14 @@ public static int compareLists(List b0, List b1) { /** * Compares two maps. * - *

      Since maps in Calcite are implemented using {@link java.util.LinkedHashMap}, - * which guarantees insertion order, this method follows DuckDB's behavior by - * comparing entries in order. For each entry, it first compares the key and - * then the value. + *

      Entries are compared in a canonical order, sorted by key and then by value. */ public static int compareMaps(Map b0, Map b1) { if (b0 == b1) { return 0; } - final Iterator> i0 = b0.entrySet().iterator(); - final Iterator> i1 = b1.entrySet().iterator(); + final Iterator> i0 = sortedEntries(b0).iterator(); + final Iterator> i1 = sortedEntries(b1).iterator(); while (i0.hasNext() && i1.hasNext()) { Map.Entry e0 = i0.next(); Map.Entry e1 = i1.next(); @@ -767,6 +764,17 @@ public static int compareMaps(Map b0, Map b1) { return 0; } + /** Returns the entries of a map in a canonical order that does not depend + * on the map's iteration order: sorted by key, ties broken by value. */ + private static List> sortedEntries(Map map) { + final List> entries = new ArrayList<>(map.entrySet()); + entries.sort((e0, e1) -> { + final int c = compareListItems(e0.getKey(), e1.getKey()); + return c != 0 ? c : compareListItems(e0.getValue(), e1.getValue()); + }); + return entries; + } + private static BigDecimal toBigDecimal(Number number) { return number instanceof BigDecimal ? (BigDecimal) number : number instanceof BigInteger ? new BigDecimal((BigInteger) number) diff --git a/linq4j/src/test/java/org/apache/calcite/linq4j/function/FunctionTest.java b/linq4j/src/test/java/org/apache/calcite/linq4j/function/FunctionTest.java index 8fba58935048..8c6f758b1b64 100644 --- a/linq4j/src/test/java/org/apache/calcite/linq4j/function/FunctionTest.java +++ b/linq4j/src/test/java/org/apache/calcite/linq4j/function/FunctionTest.java @@ -20,9 +20,12 @@ import java.util.Arrays; import java.util.Collections; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.function.IntFunction; +import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.hasToString; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -80,6 +83,55 @@ class FunctionTest { Functions.all(empty, Functions.truePredicate1())); } + /** Unit test for {@link Functions#compareMaps}. Maps are unordered, so the + * comparison must not depend on insertion order, and must return 0 exactly + * when the maps have equal contents; see + * [CALCITE-7689] + * MAP equality compares maps by insertion order. */ + @Test void testCompareMaps() { + // Equal contents, same and different insertion order + assertThat(Functions.compareMaps(map(1, 2, 3, 4), map(1, 2, 3, 4)), is(0)); + assertThat(Functions.compareMaps(map(1, 2, 3, 4), map(3, 4, 1, 2)), is(0)); + assertThat(Functions.compareMaps(map(3, 4, 1, 2), map(1, 2, 3, 4)), is(0)); + + // Different value for one key: unequal, antisymmetric + final Map a = map(1, 2, 3, 4); + final Map c = map(1, 2, 3, 5); + assertTrue(Functions.compareMaps(a, c) < 0); + assertTrue(Functions.compareMaps(c, a) > 0); + + // Different keys and different sizes + assertTrue(Functions.compareMaps(map(1, 2), map(2, 2)) != 0); + assertTrue(Functions.compareMaps(map(1, 2), map(1, 2, 3, 4)) < 0); + assertTrue(Functions.compareMaps(map(1, 2, 3, 4), map(1, 2)) > 0); + + // The order must be transitive: b equals a, so b and a must compare to c + // with the same sign + final Map b = map(3, 4, 1, 2); + assertTrue(Functions.compareMaps(b, c) < 0); + + // Null values compare equal to each other and follow the same rule + assertThat(Functions.compareMaps(map(1, null, 3, 4), map(3, 4, 1, null)), + is(0)); + assertTrue(Functions.compareMaps(map(1, null, 3, 4), map(1, 2, 3, 4)) != 0); + + // Nested maps as values are also compared by contents + assertThat( + Functions.compareMaps(map("k", map(1, 2, 3, 4)), + map("k", map(3, 4, 1, 2))), + is(0)); + } + + /** Creates a {@link LinkedHashMap} whose iteration order is the order of + * the given alternating keys and values. */ + private static Map map(Object... kv) { + final Map result = new LinkedHashMap<>(); + for (int i = 0; i < kv.length; i += 2) { + result.put(kv[i], kv[i + 1]); + } + return result; + } + /** Unit test for {@link Functions#generate}. */ @Test void testGenerate() { final IntFunction xx = diff --git a/site/_docs/reference.md b/site/_docs/reference.md index 08938ec364f9..71c067e27b77 100644 --- a/site/_docs/reference.md +++ b/site/_docs/reference.md @@ -1508,6 +1508,12 @@ Note: collection is therefore compared the way `IS NOT DISTINCT FROM` compares it, and a NULL inside a collection does *not* make a comparison of the enclosing `ROW` value UNKNOWN. +* A `MAP` value is an unordered mapping of keys to values: two maps are equal + exactly when they contain the same keys and map each key to equal values, + regardless of the order in which the entries were written. Keys and values + are compared using `<=>`, so a NULL value equals a NULL value, and the + result is never UNKNOWN. For example, `MAP[1, 2, 3, 4] = MAP[3, 4, 1, 2]` + is TRUE, and so is `MAP[1, NULL] = MAP[1, NULL]`. * `GROUP BY`, `DISTINCT` and the set operators (`UNION`, `INTERSECT`, `EXCEPT`) compare values as `IS NOT DISTINCT FROM` does. From 0adddf21a9ed2ef645682feb0ccebe0c4b11c245 Mon Sep 17 00:00:00 2001 From: Tisya Bhatia Date: Mon, 3 Aug 2026 11:56:27 -0500 Subject: [PATCH 460/562] [CALCITE-7647] Support SELECT * in GROUP BY ALL and ORDER BY ALL --- .../calcite/runtime/CalciteResource.java | 6 ---- .../sql/validate/SqlValidatorImpl.java | 32 ++++++++++++++++--- .../runtime/CalciteResource.properties | 2 -- .../apache/calcite/test/SqlValidatorTest.java | 28 ++++++++++++---- core/src/test/resources/sql/agg.iq | 31 ++++++++++++++++++ core/src/test/resources/sql/sort.iq | 15 +++++++++ site/_docs/reference.md | 6 ++++ 7 files changed, 102 insertions(+), 18 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java b/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java index 2e5056da1d7b..cc27cb5c2005 100644 --- a/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java +++ b/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java @@ -406,9 +406,6 @@ ExInst naturalOrUsingColumnNotCompatible(String a0, @BaseMessage("Windowed aggregate expression is illegal in {0} clause") ExInst windowedAggregateIllegalInClause(String a0); - @BaseMessage("GROUP BY ALL requires an explicit SELECT list; ''*'' is not supported") - ExInst groupByAllRequiresExplicitSelectList(); - @BaseMessage("Aggregate expressions cannot be nested") ExInst nestedAggIllegal(); @@ -800,9 +797,6 @@ ExInst illegalArgumentForTableFunctionCall(String a0, @BaseMessage("Streaming ORDER BY must start with monotonic expression") ExInst streamMustOrderByMonotonic(); - @BaseMessage("ORDER BY ALL requires an explicit SELECT list; ''*'' is not supported") - ExInst orderByAllRequiresExplicitSelectList(); - @BaseMessage("Set operator cannot combine streaming and non-streaming inputs") ExInst streamSetOpInconsistentInputs(); diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java index 604c03d2d191..c2d12ffd186d 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java @@ -5328,6 +5328,24 @@ protected void validateOrderList(SqlSelect select) { } } + /** Expands a single "*" or "t.*" select item into its underlying columns, + * for a GROUP BY ALL / ORDER BY ALL rewrite. + * + *

      Calls the private {@code expandStar} core directly (not the public + * {@code expandStar(SqlNodeList, SqlSelect, boolean)} wrapper), with fresh + * collections: the wrapper would derive types over every select item and + * mark the expanded list, poisoning {@link AggregatingSelectScope}'s + * memoized grouping set with the not-yet-rewritten placeholder. The fresh + * {@code items}/{@code fields} must stay paired for NATURAL/USING index + * alignment. */ + private List expandStarForAllRewrite(SqlSelect select, SqlNode starItem) { + final SelectScope scope = (SelectScope) getWhereScope(select); + final List items = new ArrayList<>(); + expandStar(items, catalogReader.nameMatcher().createSet(), PairList.of(), + false, scope, starItem, false); + return items; + } + protected void rewriteOrderByAll(SqlSelect select) { final SqlNodeList orderList = select.getOrderList(); if (orderList == null || orderList.size() != 1) { @@ -5360,8 +5378,10 @@ protected void rewriteOrderByAll(SqlSelect select) { for (SqlNode selectItem : select.getSelectList()) { final SqlNode expr = SqlUtil.stripAs(selectItem); if (expr instanceof SqlIdentifier && ((SqlIdentifier) expr).isStar()) { - throw newValidationError(expr, - RESOURCE.orderByAllRequiresExplicitSelectList()); + for (SqlNode column : expandStarForAllRewrite(select, expr)) { + keys.add(applyOrderByAllDirection(column, desc, nulls, pos)); + } + continue; } keys.add(applyOrderByAllDirection(expr, desc, nulls, pos)); } @@ -5561,8 +5581,12 @@ private void rewriteGroupByAll(SqlSelect select) { } final SqlNode expr = SqlUtil.stripAs(selectItem); if (expr instanceof SqlIdentifier && ((SqlIdentifier) expr).isStar()) { - throw newValidationError(expr, - RESOURCE.groupByAllRequiresExplicitSelectList()); + for (SqlNode column : expandStarForAllRewrite(select, expr)) { + if (aggOrOverFinder.findAgg(column) == null) { + keys.add(column); + } + } + continue; } if (aggOrOverFinder.findAgg(expr) == null) { keys.add(expr); diff --git a/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties b/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties index 636e117c7e0e..c90ac9c4ed35 100644 --- a/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties +++ b/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties @@ -136,7 +136,6 @@ GroupingInWrongClause={0} operator may only occur in SELECT, HAVING or ORDER BY NotSelectDistinctExpr=Expression ''{0}'' is not in the select clause AggregateIllegalInClause=Aggregate expression is illegal in {0} clause WindowedAggregateIllegalInClause=Windowed aggregate expression is illegal in {0} clause -GroupByAllRequiresExplicitSelectList=GROUP BY ALL requires an explicit SELECT list; ''*'' is not supported NestedAggIllegal=Aggregate expressions cannot be nested MeasureIllegal=Measure expressions can only occur within AGGREGATE function MeasureMustBeInAggregateQuery=Measure expressions can only occur within a GROUP BY query @@ -261,7 +260,6 @@ CannotConvertToStream=Cannot convert table ''{0}'' to stream CannotConvertToRelation=Cannot convert stream ''{0}'' to relation StreamMustGroupByMonotonic=Streaming aggregation requires at least one monotonic expression in GROUP BY clause StreamMustOrderByMonotonic=Streaming ORDER BY must start with monotonic expression -OrderByAllRequiresExplicitSelectList=ORDER BY ALL requires an explicit SELECT list; ''*'' is not supported StreamSetOpInconsistentInputs=Set operator cannot combine streaming and non-streaming inputs CannotStreamValues=Cannot stream VALUES CyclicDefinition=Cannot resolve ''{0}''; it references view ''{1}'', whose definition is cyclic diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index 892d68f11789..38f883743ecf 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -7366,9 +7366,17 @@ public boolean isBangEqualAllowed() { sql("select deptno, sal from emp order by all").ok(); // direction applies to every expanded key sql("select deptno, sal from emp order by all desc").ok(); - // SELECT * can't be expanded here - sql("select ^*^ from emp order by all") - .fails("(?s).*ORDER BY ALL requires an explicit SELECT list.*"); + // SELECT * validates with ORDER BY ALL; what the star expands to as sort + // keys is asserted explicitly by rewritesTo below. + sql("select * from emp order by all").ok(); + sql("select * from emp order by all desc").ok(); + // Multiple qualified stars validate together (no cross-expansion error). + sql("select emp.*, dept.* from emp, dept order by all desc").ok(); + // Show the expanded sort keys. + sql("select * from dept order by all") + .rewritesTo("SELECT *\n" + + "FROM `DEPT`\n" + + "ORDER BY `DEPT`.`DEPTNO`, `DEPT`.`NAME`"); // Aliases that shadow other column names must not confuse expansion sql("select empno as deptno, deptno as empno from emp order by all").ok(); // verify "x" still resolves and the two features coexist @@ -7822,9 +7830,17 @@ public boolean isBangEqualAllowed() { // only aggregates -> global aggregation (one group), still valid sql("select count(*) from emp group by all").ok(); - // SELECT * cannot be expanded at group-validation time -> clear error - sql("select ^*^ from emp group by all") - .fails("(?s).*GROUP BY ALL requires an explicit SELECT list.*"); + // SELECT * validates with GROUP BY ALL; what the star expands to as + // grouping keys is asserted explicitly by rewritesTo below. + sql("select * from emp group by all").ok(); + sql("select *, count(*) from emp group by all").ok(); + sql("select * from emp natural join dept group by all").ok(); + // Show the expanded grouping keys: GROUP BY ALL is replaced by the + // star's underlying columns (behavior validated against DuckDB). + sql("select * from dept group by all") + .rewritesTo("SELECT *\n" + + "FROM `DEPT`\n" + + "GROUP BY `DEPT`.`DEPTNO`, `DEPT`.`NAME`"); // contains-an-aggregate sql("select deptno, substring(job, 1), count(*) + 1 as c, 'x' as x\n" diff --git a/core/src/test/resources/sql/agg.iq b/core/src/test/resources/sql/agg.iq index d6557e34f1ec..bc24ba60145f 100644 --- a/core/src/test/resources/sql/agg.iq +++ b/core/src/test/resources/sql/agg.iq @@ -4688,4 +4688,35 @@ FROM emp; !use scott +# [CALCITE-7647] Support SELECT * in GROUP BY ALL and ORDER BY ALL. +# GROUP BY ALL expands SELECT * to every underlying column; +# the star columns become grouping keys and the aggregate is excluded. +select *, count(*) as c from (values (1, 'a'), (1, 'a'), (2, 'b')) as t(x, y) +group by all +order by x; ++---+---+---+ +| X | Y | C | ++---+---+---+ +| 1 | a | 2 | +| 2 | b | 1 | ++---+---+---+ +(2 rows) + +!ok + +# [CALCITE-7647] GROUP BY ALL deduplicates a column that SELECT * and an +# explicit reference both contribute to the grouping keys. +select *, x from (values (1, 'a'), (1, 'a'), (2, 'b')) as t(x, y) +group by all +order by y; ++---+---+---+ +| X | Y | X | ++---+---+---+ +| 1 | a | 1 | +| 2 | b | 2 | ++---+---+---+ +(2 rows) + +!ok + # End agg.iq diff --git a/core/src/test/resources/sql/sort.iq b/core/src/test/resources/sql/sort.iq index 0e8d84cff947..a5320474257a 100644 --- a/core/src/test/resources/sql/sort.iq +++ b/core/src/test/resources/sql/sort.iq @@ -568,4 +568,19 @@ order by all; !ok +# [CALCITE-7647] Support SELECT * in GROUP BY ALL and ORDER BY ALL. +# ORDER BY ALL expands SELECT * to every underlying column. +select * from (values (2, 'b'), (1, 'a'), (1, 'c')) as t(x, y) +order by all; ++---+---+ +| X | Y | ++---+---+ +| 1 | a | +| 1 | c | +| 2 | b | ++---+---+ +(3 rows) + +!ok + # End sort.iq diff --git a/site/_docs/reference.md b/site/_docs/reference.md index 71c067e27b77..59ddce4af544 100644 --- a/site/_docs/reference.md +++ b/site/_docs/reference.md @@ -426,6 +426,9 @@ in the order that they appear in the list; for example: "SELECT x, y FROM t ORDER BY ALL" is equivalent to "SELECT x, y FROM t ORDER BY x, y" An optional trailing ASC / DESC and NULLS FIRST / NULLS LAST applies to all keys. +A `*` in the SELECT clause is expanded to its underlying columns, each of which +becomes a sort key; for example, "SELECT * FROM t ORDER BY ALL" sorts by every +column of `t`. In *query*, *start* may be either an unsigned numeric literal or a dynamic parameter whose value is numeric. The *count* in a LIMIT clause may be either @@ -468,6 +471,9 @@ GROUP BY ALL on its own groups by every expression in the SELECT clause that is not an aggregate function; for example, "SELECT deptno, SUM(sal) FROM emp GROUP BY ALL" is equivalent to "SELECT deptno, SUM(sal) FROM emp GROUP BY deptno". +A `*` in the SELECT clause is expanded to its underlying columns, each of which +becomes a grouping key; for example, +"SELECT *, COUNT(*) FROM emp GROUP BY ALL" groups by every column of `emp`. *selectWithoutFrom* is equivalent to VALUES, but is not standard SQL and is only allowed in certain From 139aecdb09472514f69f06ea474d443c0dc06f98 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Tue, 4 Aug 2026 15:44:49 -0700 Subject: [PATCH 461/562] [CALCITE-7670] Uncollect should support LEFT JOIN UNNEST Signed-off-by: Mihai Budiu --- .../enumerable/EnumerableUncollect.java | 30 +- .../enumerable/EnumerableUncollectRule.java | 3 +- .../calcite/interpreter/UncollectNode.java | 15 + .../apache/calcite/rel/core/Uncollect.java | 47 ++- .../rel/logical/ToLogicalConverter.java | 2 +- .../calcite/rel/mutable/MutableRels.java | 4 +- .../calcite/rel/mutable/MutableUncollect.java | 32 +- .../apache/calcite/rel/rules/CoreRules.java | 6 + .../rules/CorrelateUncollectOuterRule.java | 105 ++++++ .../rel/rules/UnnestDecorrelateRule.java | 16 +- .../apache/calcite/runtime/SqlFunctions.java | 131 +++++++- .../calcite/sql2rel/SqlToRelConverter.java | 4 +- .../org/apache/calcite/tools/RelBuilder.java | 30 +- .../apache/calcite/util/BuiltInMethod.java | 3 +- .../calcite/sql2rel/RelFieldTrimmerTest.java | 2 +- .../apache/calcite/test/SqlFunctionsTest.java | 124 ++++++- core/src/test/resources/sql/unnest.iq | 316 ++++++++++++++++++ 17 files changed, 822 insertions(+), 48 deletions(-) create mode 100644 core/src/main/java/org/apache/calcite/rel/rules/CorrelateUncollectOuterRule.java diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableUncollect.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableUncollect.java index 8f193a4acede..143cabad999a 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableUncollect.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableUncollect.java @@ -51,16 +51,20 @@ public EnumerableUncollect(RelOptCluster cluster, RelTraitSet traitSet, *

      Use {@link #create} unless you know what you're doing. */ public EnumerableUncollect(RelOptCluster cluster, RelTraitSet traitSet, RelNode child, boolean withOrdinality) { - this(cluster, traitSet, child, withOrdinality, true); + this(cluster, traitSet, child, withOrdinality, true, false); } /** Creates an EnumerableUncollect. * - *

      Use {@link #create} unless you know what you're doing. */ + *

      Use {@link #create} unless you know what you're doing. + * + * @param isOuter If true, an empty or NULL collection yields one row of + * NULLs (LEFT JOIN); if false, it yields no rows (INNER) */ public EnumerableUncollect(RelOptCluster cluster, RelTraitSet traitSet, - RelNode child, boolean withOrdinality, boolean expandStructFields) { + RelNode child, boolean withOrdinality, boolean expandStructFields, + boolean isOuter) { super(cluster, traitSet, child, withOrdinality, Collections.emptyList(), - expandStructFields); + expandStructFields, isOuter); assert getConvention() instanceof EnumerableConvention; assert getConvention() == child.getConvention(); } @@ -90,18 +94,20 @@ public static EnumerableUncollect create(RelTraitSet traitSet, RelNode input, * @param expandStructFields If true, a collection whose element type is a struct * produces one output column per struct field; if false, * a single column typed as the whole element + * @param isOuter If true, an empty or NULL collection yields one row of + * NULLs (LEFT JOIN); if false, it yields no rows (INNER) */ public static EnumerableUncollect create(RelTraitSet traitSet, RelNode input, - boolean withOrdinality, boolean expandStructFields) { + boolean withOrdinality, boolean expandStructFields, boolean isOuter) { final RelOptCluster cluster = input.getCluster(); return new EnumerableUncollect(cluster, traitSet, input, withOrdinality, - expandStructFields); + expandStructFields, isOuter); } @Override public EnumerableUncollect copy(RelTraitSet traitSet, RelNode newInput) { return new EnumerableUncollect(getCluster(), traitSet, newInput, - withOrdinality, expandStructFields); + withOrdinality, expandStructFields, isOuter); } @Override public Result implement(EnumerableRelImplementor implementor, Prefer pref) { @@ -136,8 +142,11 @@ public static EnumerableUncollect create(RelTraitSet traitSet, RelNode input, && !withOrdinality) { // Solves CALCITE-4063: if we are processing a single field, which is a struct with a // single item inside, and no ordinality; the result must be a scalar, hence use a - // special lambda that does not return lists, but the (single) items within those lists - lambdaForStructWithSingleItem = Expressions.call(BuiltInMethod.FLAT_LIST.method); + // special lambda that does not return lists, but the (single) items within those + // lists. The outer variant returns one NULL scalar for an empty or NULL collection. + lambdaForStructWithSingleItem = + Expressions.call(isOuter ? BuiltInMethod.FLAT_LIST_OUTER.method + : BuiltInMethod.FLAT_LIST.method); } else { fieldCounts.add(elementType.getFieldCount()); inputTypes.add(FlatProductInputType.LIST); @@ -161,7 +170,8 @@ public static EnumerableUncollect create(RelTraitSet traitSet, RelNode input, Expressions.constant(Ints.toArray(fieldCounts)), Expressions.constant(withOrdinality), Expressions.constant( - inputTypes.toArray(new FlatProductInputType[0]))); + inputTypes.toArray(new FlatProductInputType[0])), + Expressions.constant(isOuter)); builder.add( Expressions.return_(null, Expressions.call(child_, diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableUncollectRule.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableUncollectRule.java index 95a9237c2222..906404c3ab94 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableUncollectRule.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableUncollectRule.java @@ -49,6 +49,7 @@ protected EnumerableUncollectRule(Config config) { convert(input, input.getTraitSet().replace(EnumerableConvention.INSTANCE)); return EnumerableUncollect.create(traitSet, newInput, - uncollect.withOrdinality, uncollect.expandStructFields); + uncollect.withOrdinality, uncollect.expandStructFields, + uncollect.isOuter); } } diff --git a/core/src/main/java/org/apache/calcite/interpreter/UncollectNode.java b/core/src/main/java/org/apache/calcite/interpreter/UncollectNode.java index 01f4d578f463..725f577c4a79 100644 --- a/core/src/main/java/org/apache/calcite/interpreter/UncollectNode.java +++ b/core/src/main/java/org/apache/calcite/interpreter/UncollectNode.java @@ -33,15 +33,26 @@ public UncollectNode(Compiler compiler, Uncollect uncollect) { } @Override public void run() throws InterruptedException { + // Under isOuter an empty or NULL collection still produces one row, with + // every column NULL. + final int width = rel.getRowType().getFieldCount(); Row row = null; while ((row = source.receive()) != null) { for (Object value : row.getValues()) { if (value == null) { + if (rel.isOuter) { + sink.send(Row.of(new Object[width])); + continue; + } throw new NullPointerException("NULL value for unnest."); } int i = 1; if (value instanceof List) { List list = (List) value; + if (list.isEmpty() && rel.isOuter) { + sink.send(Row.of(new Object[width])); + continue; + } for (Object o : list) { if (rel.withOrdinality) { sink.send(Row.of(o, i++)); @@ -51,6 +62,10 @@ public UncollectNode(Compiler compiler, Uncollect uncollect) { } } else if (value instanceof Map) { Map map = (Map) value; + if (map.isEmpty() && rel.isOuter) { + sink.send(Row.of(new Object[width])); + continue; + } for (Object key : map.keySet()) { if (rel.withOrdinality) { sink.send(Row.of(key, map.get(key), i++)); diff --git a/core/src/main/java/org/apache/calcite/rel/core/Uncollect.java b/core/src/main/java/org/apache/calcite/rel/core/Uncollect.java index e607509ddc18..039a17efbfcf 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/Uncollect.java +++ b/core/src/main/java/org/apache/calcite/rel/core/Uncollect.java @@ -56,10 +56,20 @@ * output column per struct field; if {@code false} it produces a single * column typed as the whole element (Trino semantics). Maps always expand * into a key and a value column, regardless of this flag. + * + *

      {@code isOuter} controls what happens to an empty or {@code NULL} + * collection: if {@code true} (LEFT JOIN semantics) one row is emitted with + * every element column set to {@code NULL}; if {@code false} (INNER + * semantics) no row is emitted. Every element column is therefore nullable + * when {@code isOuter}. */ public class Uncollect extends SingleRel { public final boolean withOrdinality; + /** If true, an empty or NULL collection yields a single row whose element + * columns are all NULL, rather than no rows at all. */ + public final boolean isOuter; + /** If true, a collection whose element type is a struct expands into one * output column per struct field; if false, it produces a single column * typed as the whole element. */ @@ -90,7 +100,8 @@ public Uncollect(RelOptCluster cluster, RelTraitSet traitSet, RelNode input, // Non-empty item aliases historically implied that struct elements are not // expanded (Presto dialect), so this constructor derives // {@code expandStructFields} from their absence. - this(cluster, traitSet, input, withOrdinality, itemAliases, itemAliases.isEmpty()); + this(cluster, traitSet, input, withOrdinality, itemAliases, itemAliases.isEmpty(), + false); } /** Creates an Uncollect. @@ -101,14 +112,18 @@ public Uncollect(RelOptCluster cluster, RelTraitSet traitSet, RelNode input, * @param expandStructFields If true, a collection whose element type is a struct * produces one output column per struct field; if false, * a single column typed as the whole element + * @param isOuter If true, an empty or NULL collection yields one row of + * NULLs (LEFT JOIN); if false, it yields no rows (INNER) */ @SuppressWarnings("method.invocation.invalid") public Uncollect(RelOptCluster cluster, RelTraitSet traitSet, RelNode input, - boolean withOrdinality, List itemAliases, boolean expandStructFields) { + boolean withOrdinality, List itemAliases, boolean expandStructFields, + boolean isOuter) { super(cluster, traitSet, input); this.withOrdinality = withOrdinality; this.itemAliases = ImmutableList.copyOf(itemAliases); this.expandStructFields = expandStructFields; + this.isOuter = isOuter; requireNonNull(deriveRowType(), "invalid child rowType"); } @@ -118,7 +133,8 @@ public Uncollect(RelOptCluster cluster, RelTraitSet traitSet, RelNode input, public Uncollect(RelInput input) { this(input.getCluster(), input.getTraitSet(), input.getInput(), input.getBoolean("withOrdinality", false), Collections.emptyList(), - input.getBoolean("expandStructFields", true)); + input.getBoolean("expandStructFields", true), + input.getBoolean("isOuter", false)); } /** @@ -151,16 +167,19 @@ public static Uncollect create( * @param expandStructFields If true, a collection whose element type is a struct * produces one output column per struct field; if false, * a single column typed as the whole element + * @param isOuter If true, an empty or NULL collection yields one row of + * NULLs (LEFT JOIN); if false, it yields no rows (INNER) */ public static Uncollect create( RelTraitSet traitSet, RelNode input, boolean withOrdinality, List itemAliases, - boolean expandStructFields) { + boolean expandStructFields, + boolean isOuter) { final RelOptCluster cluster = input.getCluster(); return new Uncollect(cluster, traitSet, input, withOrdinality, itemAliases, - expandStructFields); + expandStructFields, isOuter); } //~ Methods ---------------------------------------------------------------- @@ -172,7 +191,8 @@ public static Uncollect create( @Override public RelWriter explainTerms(RelWriter pw) { return super.explainTerms(pw) .itemIf("withOrdinality", withOrdinality, withOrdinality) - .itemIf("expandStructFields", expandStructFields, !expandStructFields); + .itemIf("expandStructFields", expandStructFields, !expandStructFields) + .itemIf("isOuter", isOuter, isOuter); } @Override public final RelNode copy(RelTraitSet traitSet, @@ -183,7 +203,7 @@ public static Uncollect create( public RelNode copy(RelTraitSet traitSet, RelNode input) { assert traitSet.containsIfApplicable(Convention.NONE); return new Uncollect(getCluster(), traitSet, input, withOrdinality, itemAliases, - expandStructFields); + expandStructFields, isOuter); } /** @@ -287,7 +307,18 @@ public static RelDataType deriveUncollectRowType(RelNode rel, builder.add(SqlUnnestOperator.ORDINALITY_COLUMN_NAME, SqlTypeName.INTEGER); } - return builder.build(); + final RelDataType rowType = builder.build(); + if (!isOuter) { + return rowType; + } + // Under isOuter an empty or NULL collection yields a row of NULLs, so + // every output column is nullable, including the ordinality column. + final RelDataTypeFactory.Builder outerBuilder = typeFactory.builder(); + for (RelDataTypeField field : rowType.getFieldList()) { + outerBuilder.add(field.getName(), + typeFactory.createTypeWithNullability(field.getType(), true)); + } + return outerBuilder.build(); } /** Gets the aliases for the unnest items. */ diff --git a/core/src/main/java/org/apache/calcite/rel/logical/ToLogicalConverter.java b/core/src/main/java/org/apache/calcite/rel/logical/ToLogicalConverter.java index 4ff564f1fd54..b15f06b6ea58 100644 --- a/core/src/main/java/org/apache/calcite/rel/logical/ToLogicalConverter.java +++ b/core/src/main/java/org/apache/calcite/rel/logical/ToLogicalConverter.java @@ -190,7 +190,7 @@ public ToLogicalConverter(RelBuilder relBuilder) { final RelNode input = visit(uncollect.getInput()); return Uncollect.create(input.getTraitSet(), input, uncollect.withOrdinality, uncollect.getItemAliases(), - uncollect.expandStructFields); + uncollect.expandStructFields, uncollect.isOuter); } throw new AssertionError("Need to implement logical converter for " diff --git a/core/src/main/java/org/apache/calcite/rel/mutable/MutableRels.java b/core/src/main/java/org/apache/calcite/rel/mutable/MutableRels.java index 176be5cfec66..092d45c8fe9a 100644 --- a/core/src/main/java/org/apache/calcite/rel/mutable/MutableRels.java +++ b/core/src/main/java/org/apache/calcite/rel/mutable/MutableRels.java @@ -257,7 +257,7 @@ public static RelNode fromMutable(MutableRel node, RelBuilder relBuilder) { final MutableUncollect uncollect = (MutableUncollect) node; final RelNode child = fromMutable(uncollect.getInput(), relBuilder); return Uncollect.create(child.getTraitSet(), child, uncollect.withOrdinality, - Collections.emptyList(), uncollect.expandStructFields); + Collections.emptyList(), uncollect.expandStructFields, uncollect.isOuter); } case WINDOW: { final MutableWindow window = (MutableWindow) node; @@ -379,7 +379,7 @@ public static MutableRel toMutable(RelNode rel) { final Uncollect uncollect = (Uncollect) rel; final MutableRel input = toMutable(uncollect.getInput()); return MutableUncollect.of(uncollect.getRowType(), input, - uncollect.withOrdinality, uncollect.expandStructFields); + uncollect.withOrdinality, uncollect.expandStructFields, uncollect.isOuter); } if (rel instanceof Window) { final Window window = (Window) rel; diff --git a/core/src/main/java/org/apache/calcite/rel/mutable/MutableUncollect.java b/core/src/main/java/org/apache/calcite/rel/mutable/MutableUncollect.java index bae3854f6948..0dc09b2a001e 100644 --- a/core/src/main/java/org/apache/calcite/rel/mutable/MutableUncollect.java +++ b/core/src/main/java/org/apache/calcite/rel/mutable/MutableUncollect.java @@ -26,12 +26,15 @@ public class MutableUncollect extends MutableSingleRel { public final boolean withOrdinality; public final boolean expandStructFields; + public final boolean isOuter; private MutableUncollect(RelDataType rowType, - MutableRel input, boolean withOrdinality, boolean expandStructFields) { + MutableRel input, boolean withOrdinality, boolean expandStructFields, + boolean isOuter) { super(MutableRelType.UNCOLLECT, rowType, input); this.withOrdinality = withOrdinality; this.expandStructFields = expandStructFields; + this.isOuter = isOuter; } /** @@ -44,7 +47,7 @@ private MutableUncollect(RelDataType rowType, */ public static MutableUncollect of(RelDataType rowType, MutableRel input, boolean withOrdinality) { - return of(rowType, input, withOrdinality, true); + return of(rowType, input, withOrdinality, true, false); } /** @@ -59,10 +62,25 @@ public static MutableUncollect of(RelDataType rowType, * struct field; if false, a single column * typed as the whole element */ + /** + * Creates a MutableUncollect. + * + * @param rowType Row type + * @param input Input relational expression + * @param withOrdinality Whether the output contains an extra + * {@code ORDINALITY} column + * @param expandStructFields If true, a collection whose element type + * is a struct produces one output column per + * struct field; if false, a single column + * typed as the whole element + * @param isOuter If true, an empty or NULL collection yields one + * row of NULLs; if false, it yields no rows + */ public static MutableUncollect of(RelDataType rowType, - MutableRel input, boolean withOrdinality, boolean expandStructFields) { + MutableRel input, boolean withOrdinality, boolean expandStructFields, + boolean isOuter) { return new MutableUncollect(rowType, input, withOrdinality, - expandStructFields); + expandStructFields, isOuter); } @Override public boolean equals(@Nullable Object obj) { @@ -70,21 +88,23 @@ public static MutableUncollect of(RelDataType rowType, || obj instanceof MutableUncollect && withOrdinality == ((MutableUncollect) obj).withOrdinality && expandStructFields == ((MutableUncollect) obj).expandStructFields + && isOuter == ((MutableUncollect) obj).isOuter && input.equals(((MutableUncollect) obj).input); } @Override public int hashCode() { - return Objects.hash(input, withOrdinality, expandStructFields); + return Objects.hash(input, withOrdinality, expandStructFields, isOuter); } @Override public StringBuilder digest(StringBuilder buf) { return buf.append("Uncollect(withOrdinality: ").append(withOrdinality) .append(", expandStructFields: ").append(expandStructFields) + .append(", isOuter: ").append(isOuter) .append(")"); } @Override public MutableRel clone() { return MutableUncollect.of(rowType, input.clone(), withOrdinality, - expandStructFields); + expandStructFields, isOuter); } } diff --git a/core/src/main/java/org/apache/calcite/rel/rules/CoreRules.java b/core/src/main/java/org/apache/calcite/rel/rules/CoreRules.java index 98cbeaae2194..f827843cc64f 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/CoreRules.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/CoreRules.java @@ -1008,6 +1008,12 @@ private CoreRules() {} public static final AggregateRemoveLiteralAggRule AGGREGATE_REMOVE_LITERAL_AGG = AggregateRemoveLiteralAggRule.Config.DEFAULT.toRule(); + /** Rule that moves the outer join semantics of a {@link Correlate} over an + * {@link Uncollect} onto the {@code Uncollect}, leaving an inner + * {@code Correlate} that {@link #UNNEST_DECORRELATE} may then remove. */ + public static final CorrelateUncollectOuterRule CORRELATE_UNCOLLECT_OUTER = + CorrelateUncollectOuterRule.Config.DEFAULT.toRule(); + /** Rule that converts a {@link Correlate} after an {@link Uncollect} into a simple * Uncollect, if possible. */ public static final RelOptRule UNNEST_DECORRELATE = diff --git a/core/src/main/java/org/apache/calcite/rel/rules/CorrelateUncollectOuterRule.java b/core/src/main/java/org/apache/calcite/rel/rules/CorrelateUncollectOuterRule.java new file mode 100644 index 000000000000..6cbf781c5f21 --- /dev/null +++ b/core/src/main/java/org/apache/calcite/rel/rules/CorrelateUncollectOuterRule.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.rel.rules; + +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelRule; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.Correlate; +import org.apache.calcite.rel.core.JoinRelType; +import org.apache.calcite.rel.core.Project; +import org.apache.calcite.rel.core.Uncollect; +import org.apache.calcite.rel.logical.LogicalValues; + +import org.immutables.value.Value; + +/** + * Rule that moves the outer join semantics of a {@link Correlate} over an + * {@link Uncollect} onto the {@code Uncollect} itself. + * + *

      Input plan: + * 

      + * Correlate(cor=[$cor0], joinType=[left])
+ *   left (any RelNode)
+ *   Uncollect(isOuter=[any_boolean])
+ *     Project($cor0.f, ...)
+ *       LogicalValues(tuples=[[{ 0 }]])
+ *
      + * + *

      Converted to: + * 

      + * Correlate(cor=[$cor0], joinType=[inner])
+ *   left
+ *   Uncollect(isOuter=[true])
+ *     Project($cor0.f, ...)
+ *       LogicalValues(tuples=[[{ 0 }]])
+ *
      + * + * @see CoreRules#CORRELATE_UNCOLLECT_OUTER + */ +@Value.Enclosing +public class CorrelateUncollectOuterRule + extends RelRule + implements TransformationRule { + + protected CorrelateUncollectOuterRule(Config config) { + super(config); + } + + @Override public boolean matches(RelOptRuleCall call) { + final Correlate correlate = call.rel(0); + if (correlate.getJoinType() != JoinRelType.LEFT) { + return false; + } + // Expect "LogicalValues { 0 }" + final LogicalValues values = call.rel(4); + return values.getTuples().size() == 1; + } + + @Override public void onMatch(RelOptRuleCall call) { + final Correlate correlate = call.rel(0); + final Uncollect uncollect = call.rel(2); + + // Note: this is correct even if uncollect(isOuter=[true]) already + final Uncollect outerUncollect = + Uncollect.create(uncollect.getTraitSet(), uncollect.getInput(), + uncollect.withOrdinality, uncollect.getItemAliases(), + uncollect.expandStructFields, true); + final RelNode newCorrelate = + correlate.copy(correlate.getTraitSet(), correlate.getLeft(), + outerUncollect, correlate.getCorrelationId(), + correlate.getRequiredColumns(), JoinRelType.INNER); + call.transformTo(newCorrelate); + } + + /** Rule configuration. */ + @Value.Immutable + public interface Config extends RelRule.Config { + Config DEFAULT = ImmutableCorrelateUncollectOuterRule.Config.of() + .withOperandSupplier(b0 -> + b0.operand(Correlate.class).inputs( + b1 -> b1.operand(RelNode.class).anyInputs(), + b2 -> b2.operand(Uncollect.class) + .oneInput(b3 -> b3.operand(Project.class) + .oneInput(b4 -> b4.operand(LogicalValues.class) + .anyInputs())))); + + @Override default CorrelateUncollectOuterRule toRule() { + return new CorrelateUncollectOuterRule(this); + } + } +} diff --git a/core/src/main/java/org/apache/calcite/rel/rules/UnnestDecorrelateRule.java b/core/src/main/java/org/apache/calcite/rel/rules/UnnestDecorrelateRule.java index 71005b52fd45..50dbc5e8f521 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/UnnestDecorrelateRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/UnnestDecorrelateRule.java @@ -22,6 +22,7 @@ import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Correlate; import org.apache.calcite.rel.core.CorrelationId; +import org.apache.calcite.rel.core.JoinRelType; import org.apache.calcite.rel.core.Project; import org.apache.calcite.rel.core.Uncollect; import org.apache.calcite.rel.logical.LogicalValues; @@ -44,6 +45,8 @@ /** Convert representations of a projected Unnest that use LogicalCorrelate into * simple Unnest representations. * + * @see CorrelateUncollectOuterRule + * *

      Original plan: * LogicalProject // only uses rightmost columns of correlate, outerProject * LogicalCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{...}]) @@ -97,6 +100,11 @@ private boolean extractFieldReferences( @Override public void onMatch(RelOptRuleCall call) { Project outerProject = call.rel(0); Correlate cor = call.rel(1); + if (cor.getJoinType() != JoinRelType.INNER) { + // Removing the correlate is only sound for INNER. + // A LEFT correlate must first be converted by CorrelateUncollectOuterRule. + return; + } CorrelationId corId = cor.getCorrelationId(); RelNode left = call.rel(2); @@ -116,6 +124,11 @@ private boolean extractFieldReferences( Uncollect uncollect = call.rel(uncollectIndex); Project project = call.rel(uncollectIndex + 1); + // Expect "LogicalValues { 0 }" + LogicalValues values = call.rel(uncollectIndex + 2); + if (values.getTuples().size() != 1) { + return; + } List projects = project.getProjects(); if (projects.size() != 1) { @@ -143,7 +156,8 @@ private boolean extractFieldReferences( } } builder.project(requireNonNull(field, "field")) - .uncollect(uncollect.getItemAliases(), uncollect.withOrdinality); + .uncollect(uncollect.getItemAliases(), uncollect.withOrdinality, + uncollect.expandStructFields, uncollect.isOuter); if (innerProject != null) { builder.project(innerProject.getProjects()); } diff --git a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java index d2bd353052cc..b797264a8bb8 100644 --- a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java +++ b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java @@ -216,6 +216,24 @@ public class SqlFunctions { a0 -> a0 == null ? Linq4j.emptyEnumerable() : Linq4j.asEnumerable(a0).<@Nullable Object>select(SqlFunctions::structValue); + /** Single NULL element, the outer-mode result for an empty or NULL + * collection. */ + private static final List<@Nullable Object> SINGLE_NULL = + Collections.singletonList(null); + + /** Like {@link #LIST_AS_ENUMERABLE}, but for outer join mode: an empty or NULL + * collection yields one NULL element rather than no elements. */ + private static final Function1, Enumerable<@Nullable Object>> + OUTER_LIST_AS_ENUMERABLE = + a0 -> a0 == null || a0.isEmpty() ? Linq4j.asEnumerable(SINGLE_NULL) + : Linq4j.asEnumerable(a0); + + /** Like {@link #STRUCT_LIST_AS_ENUMERABLE}, but for outer join mode. */ + private static final Function1, Enumerable<@Nullable Object>> + OUTER_STRUCT_LIST_AS_ENUMERABLE = + a0 -> a0 == null || a0.isEmpty() ? Linq4j.asEnumerable(SINGLE_NULL) + : Linq4j.asEnumerable(a0).<@Nullable Object>select(SqlFunctions::structValue); + /** Converts one element of a collection of structs to its Object[] struct * value. Elements arrive as List or as Object[]; null elements stay null. */ @SuppressWarnings("rawtypes") @@ -7683,8 +7701,22 @@ public static String arrayToString(List list, String delimiter, @Nullable String * Function that, given a certain List containing single-item structs (i.e. arrays / lists with * a single item), builds an Enumerable that returns those single items inside the structs. */ - public static Function1, Enumerable> flatList() { - return inputList -> Linq4j.asEnumerable(inputList).select(v -> structAccess(v, 0, null)); + public static Function1, Enumerable<@Nullable Object>> flatList() { + // A NULL collection unnests to no rows, like an empty one. + return inputList -> inputList == null ? Linq4j.emptyEnumerable() + : Linq4j.asEnumerable(inputList) + .<@Nullable Object>select(v -> structAccess(v, 0, null)); + } + + /** + * Variant of {@link #flatList} for outer mode: an empty or {@code NULL} + * collection yields one {@code NULL} element rather than no elements. + */ + public static Function1, Enumerable<@Nullable Object>> flatListOuter() { + return inputList -> inputList == null || inputList.isEmpty() + ? Linq4j.asEnumerable(SINGLE_NULL) + : Linq4j.asEnumerable(inputList) + .<@Nullable Object>select(v -> structAccess(v, 0, null)); } /** @@ -7695,28 +7727,34 @@ public static Function1, Enumerable> flatList() { *

      This is the standard semantics for SQL {@code UNNEST(a, b, ...)}: the * i-th output row pairs element {@code a[i]} with element {@code b[i]}. * Shorter collections are padded with {@code NULL}. + * + *

      When {@code outer}, a row whose collections are all empty or + * {@code NULL} still produces one output row, with every element column set + * to {@code NULL}. This is the LEFT JOIN semantics of {@code Uncollect}. */ public static Function1>> flatZip( final int[] fieldCounts, final boolean withOrdinality, - final FlatProductInputType[] inputTypes) { + final FlatProductInputType[] inputTypes, final boolean outer) { if (fieldCounts.length == 1) { if (!withOrdinality && inputTypes[0] == FlatProductInputType.SCALAR) { // Simple unnest without ordinality //noinspection unchecked - return (Function1) LIST_AS_ENUMERABLE; + return outer ? (Function1) OUTER_LIST_AS_ENUMERABLE + : (Function1) LIST_AS_ENUMERABLE; } else if (!withOrdinality && inputTypes[0] == FlatProductInputType.STRUCT) { // A single collection of structs kept whole, without ordinality: the // output row type has a single (ROW-typed) column, so PhysTypeImpl // optimizes the row format down to SCALAR, under which rows are bare // struct values rather than singleton lists. //noinspection unchecked - return (Function1) STRUCT_LIST_AS_ENUMERABLE; + return (Function1) (outer ? OUTER_STRUCT_LIST_AS_ENUMERABLE + : STRUCT_LIST_AS_ENUMERABLE); } else { // unnest with ordinality for a single column - return row -> z2(new Object[] { row }, fieldCounts, withOrdinality, inputTypes); + return row -> z2(new Object[] { row }, fieldCounts, withOrdinality, inputTypes, outer); } } - return lists -> z2((Object[]) lists, fieldCounts, withOrdinality, inputTypes); + return lists -> z2((Object[]) lists, fieldCounts, withOrdinality, inputTypes, outer); } /** @@ -7729,11 +7767,13 @@ public static Function1>> flatZip( * of scalars or of structs kept whole) * @param withOrdinality whether to append a 1-based ordinality column * @param inputTypes type of elements in each collection (SCALAR, LIST, STRUCT, or MAP) + * @param outer whether to emit one all-NULL row when every collection is + * empty or NULL, rather than no rows */ @SuppressWarnings("rawtypes") private static Enumerable> z2( Object[] lists, int[] fieldCounts, boolean withOrdinality, - FlatProductInputType[] inputTypes) { + FlatProductInputType[] inputTypes, boolean outer) { final List>> enumerators = new ArrayList<>(); final int[] widths = new int[lists.length]; int totalFieldCount = 0; @@ -7741,6 +7781,15 @@ private static Enumerable> z2( final int fieldCount = fieldCounts[i]; final FlatProductInputType inputType = inputTypes[i]; final Object inputObject = lists[i]; + if (inputObject == null) { + // A NULL collection contributes no elements, like an empty one. Under + // outer mode the wrapper below turns "no elements at all" into the + // single NULL-padded row. + enumerators.add(Linq4j.emptyEnumerator()); + widths[i] = fieldCount < 0 ? 1 : fieldCount; + totalFieldCount += widths[i]; + continue; + } switch (inputType) { case SCALAR: @SuppressWarnings("unchecked") List list = @@ -7782,13 +7831,77 @@ private static Enumerable> z2( ++totalFieldCount; } final int fieldCount = totalFieldCount; + if (!outer) { + return new AbstractEnumerable>() { + @Override public Enumerator> enumerator() { + return new ZipPaddedEnumerator(enumerators, widths, fieldCount, withOrdinality); + } + }; + } + @SuppressWarnings("unchecked") final FlatLists.ComparableList nullRow = + (FlatLists.ComparableList) FlatLists.of(Collections.nCopies(fieldCount, null)); return new AbstractEnumerable>() { @Override public Enumerator> enumerator() { - return new ZipPaddedEnumerator(enumerators, widths, fieldCount, withOrdinality); + return new DefaultIfEmptyEnumerator( + new ZipPaddedEnumerator(enumerators, widths, fieldCount, withOrdinality), + nullRow); } }; } + /** Enumerator that yields the rows of another enumerator, or a single + * default row if that enumerator yields none. + * + *

      This is the {@code defaultIfEmpty} operation of LINQ. It implements + * the outer (LEFT JOIN) semantics of {@code Uncollect}, where the default + * row is all NULL. */ + @SuppressWarnings("rawtypes") + private static class DefaultIfEmptyEnumerator + implements Enumerator> { + private final Enumerator> inner; + private final FlatLists.ComparableList defaultRow; + /** Whether {@link #inner} has yielded at least one row. */ + private boolean innerMoved; + /** Whether {@link #defaultRow} is the row currently being returned. */ + private boolean onDefaultRow; + + DefaultIfEmptyEnumerator( + Enumerator> inner, + FlatLists.ComparableList defaultRow) { + this.inner = inner; + this.defaultRow = defaultRow; + } + + @Override public boolean moveNext() { + if (onDefaultRow) { + return false; + } + if (inner.moveNext()) { + innerMoved = true; + return true; + } + if (innerMoved) { + return false; + } + onDefaultRow = true; + return true; + } + + @Override public FlatLists.ComparableList current() { + return onDefaultRow ? defaultRow : inner.current(); + } + + @Override public void reset() { + inner.reset(); + innerMoved = false; + onDefaultRow = false; + } + + @Override public void close() { + inner.close(); + } + } + public static Object[] array(Object... args) { return args; } diff --git a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java index 5c524eee17b4..357441d316f2 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java @@ -2899,7 +2899,7 @@ private void convertUnnest(Blackboard bb, SqlCall call, @Nullable List f uncollect = relBuilder .push(child) .project(exprs) - .uncollect(itemAliases, operator.withOrdinality) + .uncollect(itemAliases, operator.withOrdinality, itemAliases.isEmpty(), false) .let(r -> fieldNames == null ? r : r.rename(fieldNames)) .build(); } else { @@ -2908,7 +2908,7 @@ private void convertUnnest(Blackboard bb, SqlCall call, @Nullable List f uncollect = relBuilder .push(child) .project(exprs) - .uncollect(Collections.emptyList(), operator.withOrdinality) + .uncollect(Collections.emptyList(), operator.withOrdinality, true, false) .let(r -> fieldNames == null ? r : r.rename(fieldNames)) .build(); } diff --git a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java index 92c5a4141583..7cc7437c6223 100644 --- a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java +++ b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java @@ -2383,8 +2383,34 @@ public RelBuilder projectNamed(Iterable nodes, * @param itemAliases Operand item aliases, never null * @param withOrdinality If {@code withOrdinality}, the output contains an extra * {@code ORDINALITY} column + * + * @deprecated Use + * {@link #uncollect(List, boolean, boolean, boolean)}, which controls every + * flag explicitly. This overload derives {@code expandStructFields} from the + * item aliases, which cannot express a collection of structs that is kept + * whole without aliases, and it cannot create an outer {@code Uncollect}. */ + @Deprecated // to be removed before 2.0 public RelBuilder uncollect(List itemAliases, boolean withOrdinality) { + return uncollect(itemAliases, withOrdinality, + requireNonNull(itemAliases, "itemAliases").isEmpty(), false); + } + + /** + * Creates an {@link Uncollect} with given item aliases, with explicit control + * over every flag. + * + * @param itemAliases Operand item aliases, never null + * @param withOrdinality If {@code withOrdinality}, the output contains an extra + * {@code ORDINALITY} column + * @param expandStructFields If true, a collection whose element type is a struct + * produces one output column per struct field; if false, a single column typed + * as the whole element + * @param isOuter If {@code isOuter}, an empty or NULL collection yields one row + * of NULLs (LEFT JOIN); otherwise it yields no rows (INNER) + */ + public RelBuilder uncollect(List itemAliases, boolean withOrdinality, + boolean expandStructFields, boolean isOuter) { Frame frame = stack.pop(); stack.push( new Frame( @@ -2393,7 +2419,9 @@ public RelBuilder uncollect(List itemAliases, boolean withOrdinality) { cluster.traitSetOf(Convention.NONE), frame.rel, withOrdinality, - requireNonNull(itemAliases, "itemAliases")))); + requireNonNull(itemAliases, "itemAliases"), + expandStructFields, + isOuter))); return this; } diff --git a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java index 8e1e19054850..bbce7d8de444 100644 --- a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java +++ b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java @@ -334,8 +334,9 @@ public enum BuiltInMethod { PAIR_LIST_COPY_OF(PairList.Helper.class, "copyOf", Object.class, Object.class, Object[].class), FLAT_ZIP(SqlFunctions.class, "flatZip", int[].class, boolean.class, - FlatProductInputType[].class), + FlatProductInputType[].class, boolean.class), FLAT_LIST(SqlFunctions.class, "flatList"), + FLAT_LIST_OUTER(SqlFunctions.class, "flatListOuter"), LIST_N(FlatLists.class, "copyOf", Comparable[].class), LIST1(FlatLists.class, "ofSingle", Object.class), LIST2(FlatLists.class, "of", Object.class, Object.class), diff --git a/core/src/test/java/org/apache/calcite/sql2rel/RelFieldTrimmerTest.java b/core/src/test/java/org/apache/calcite/sql2rel/RelFieldTrimmerTest.java index 86623ccb0aa8..535dc07b1602 100644 --- a/core/src/test/java/org/apache/calcite/sql2rel/RelFieldTrimmerTest.java +++ b/core/src/test/java/org/apache/calcite/sql2rel/RelFieldTrimmerTest.java @@ -597,7 +597,7 @@ public static Frameworks.ConfigBuilder config() { .project( builder.call(SqlStdOperatorTable.ARRAY_VALUE_CONSTRUCTOR, builder.field(v.get(), "DEPTNO"), builder.field(v.get(), "DEPTNO"))) - .uncollect(Collections.emptyList(), false) + .uncollect(Collections.emptyList(), false, true, false) .correlate(JoinRelType.LEFT, v.get().id, builder.field(2, 0, "DEPTNO")) .aggregate(builder.groupKey("ENAME"), builder.max(builder.field("EMPNO"))) .build(); diff --git a/core/src/test/java/org/apache/calcite/test/SqlFunctionsTest.java b/core/src/test/java/org/apache/calcite/test/SqlFunctionsTest.java index 508008deb0ef..08d684f0316d 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlFunctionsTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlFunctionsTest.java @@ -2175,7 +2175,7 @@ private static List> zipScalars( new SqlFunctions.FlatProductInputType[n]; Arrays.fill(types, SCALAR); final Function1>> fn = - SqlFunctions.flatZip(fieldCounts, withOrdinality, types); + SqlFunctions.flatZip(fieldCounts, withOrdinality, types, false); final Object arg = n == 1 ? inputs[0] : inputs; final List> rows = new ArrayList<>(); for (FlatLists.ComparableList row : fn.apply(arg)) { @@ -2249,7 +2249,7 @@ private static List> zipScalars( @SuppressWarnings({"rawtypes", "unchecked"}) final Function1>> fn = SqlFunctions.flatZip(new int[]{2, 2}, false, - new SqlFunctions.FlatProductInputType[]{LIST, LIST}); + new SqlFunctions.FlatProductInputType[]{LIST, LIST}, false); final List> col1 = Arrays.asList(FlatLists.of(1, 2), FlatLists.of(3, 4)); final List> col2 = @@ -2280,7 +2280,7 @@ private static List> rowArray() { SqlFunctions.flatZip( new int[]{-1, -1}, // one output column per collection false, // no ordinality - new SqlFunctions.FlatProductInputType[]{STRUCT, SCALAR}); + new SqlFunctions.FlatProductInputType[]{STRUCT, SCALAR}, false); final List> rows = new ArrayList<>(); for (FlatLists.ComparableList row @@ -2304,7 +2304,7 @@ private static List> rowArray() { SqlFunctions.flatZip( new int[]{2, -1}, // two columns from the struct, one scalar column false, // no ordinality - new SqlFunctions.FlatProductInputType[]{LIST, SCALAR}); + new SqlFunctions.FlatProductInputType[]{LIST, SCALAR}, false); final List> rows = new ArrayList<>(); for (FlatLists.ComparableList row @@ -2329,7 +2329,7 @@ private static List> rowArray() { final Function1>> fn = SqlFunctions.flatZip( new int[]{-1}, false, - new SqlFunctions.FlatProductInputType[]{STRUCT}); + new SqlFunctions.FlatProductInputType[]{STRUCT}, false); final List rows = new ArrayList<>(); for (Object row : (Enumerable) fn.apply(rowArray())) { @@ -2344,4 +2344,118 @@ private static List> rowArray() { // UNNEST of a null array yields no rows. assertThat(((Enumerable) fn.apply(null)).any(), is(false)); } + + // Tests for the outer mode of flatZip (Uncollect.isOuter): an empty or + // NULL collection produces one all-NULL output row instead of none. + + @SuppressWarnings({"rawtypes", "unchecked"}) + @Test void testFlatZipOuterScalar() { + // Models SELECT u.x FROM t LEFT JOIN UNNEST(t.arr) AS u(x) ON TRUE + final Function1>> fn = + SqlFunctions.flatZip(new int[]{-1}, false, + new SqlFunctions.FlatProductInputType[]{SCALAR}, true); + + // arr = [1, 2] + assertThat(((Enumerable) fn.apply(Arrays.asList(1, 2))).toList(), + is(Arrays.asList(1, 2))); + // arr = [] + assertThat(((Enumerable) fn.apply(Collections.emptyList())).toList(), + is(Collections.singletonList(null))); + // arr = NULL + assertThat(((Enumerable) fn.apply(null)).toList(), + is(Collections.singletonList(null))); + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + @Test void testFlatZipOuterWholeStruct() { + // Models, under PRESTO conformance (struct elements kept whole), + // SELECT u.s FROM t LEFT JOIN UNNEST(t.arr) AS u(s) ON TRUE. + final Function1>> fn = + SqlFunctions.flatZip(new int[]{-1}, false, + new SqlFunctions.FlatProductInputType[]{STRUCT}, true); + + // arr = [ROW(1, 'x'), ROW(2, 'y')] + final List rows = ((Enumerable) fn.apply(rowArray())).toList(); + assertThat(rows, hasSize(2)); + assertArrayEquals(new Object[]{1, "x"}, (Object[]) rows.get(0)); + // arr = [] + assertThat(((Enumerable) fn.apply(Collections.emptyList())).toList(), + is(Collections.singletonList(null))); + // arr = NULL + assertThat(((Enumerable) fn.apply(null)).toList(), + is(Collections.singletonList(null))); + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + @Test void testFlatZipOuterWithOrdinality() { + // Models SELECT u.x, u.o + // FROM t LEFT JOIN UNNEST(t.arr) WITH ORDINALITY AS u(x, o) ON TRUE + final Function1>> fn = + SqlFunctions.flatZip(new int[]{-1}, true, + new SqlFunctions.FlatProductInputType[]{SCALAR}, true); + + // arr = [7, 8] + final List> rows = new ArrayList<>(); + for (FlatLists.ComparableList row + : fn.apply(Arrays.asList(7, 8))) { + rows.add(new ArrayList<>(row)); + } + assertThat(rows, + is(Arrays.asList(Arrays.asList(7, 1), Arrays.asList(8, 2)))); + + // arr = []; the padded row has a NULL ordinal + rows.clear(); + for (FlatLists.ComparableList row + : fn.apply(Collections.emptyList())) { + rows.add(new ArrayList<>(row)); + } + assertThat(rows, is(Collections.singletonList(Arrays.asList(null, null)))); + + // arr = NULL + rows.clear(); + for (FlatLists.ComparableList row : fn.apply(null)) { + rows.add(new ArrayList<>(row)); + } + assertThat(rows, is(Collections.singletonList(Arrays.asList(null, null)))); + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + @Test void testFlatZipOuterMultipleCollections() { + // Models SELECT u.x, u.y FROM t LEFT JOIN UNNEST(t.a, t.b) AS u(x, y) ON TRUE + final Function1>> fn = + SqlFunctions.flatZip(new int[]{-1, -1}, false, + new SqlFunctions.FlatProductInputType[]{SCALAR, SCALAR}, true); + + // (a, b) = ([], [7]): zip pads a + final List> rows = new ArrayList<>(); + for (FlatLists.ComparableList row + : fn.apply(new Object[]{Collections.emptyList(), Arrays.asList(7)})) { + rows.add(new ArrayList<>(row)); + } + assertThat(rows, is(Collections.singletonList(Arrays.asList(null, 7)))); + + // (a, b) = ([], NULL): one all-NULL row + rows.clear(); + for (FlatLists.ComparableList row + : fn.apply(new Object[]{Collections.emptyList(), null})) { + rows.add(new ArrayList<>(row)); + } + assertThat(rows, is(Collections.singletonList(Arrays.asList(null, null)))); + } + + @Test void testFlatListOuter() { + // Models SELECT u.x FROM t LEFT JOIN UNNEST(t.arr) AS u(x) ON TRUE + // for arr ROW(a INTEGER) ARRAY + final Function1, Enumerable> fn = + (Function1) SqlFunctions.flatListOuter(); + // arr = [ROW(1), ROW(2)] + assertThat(fn.apply(Arrays.asList(FlatLists.of(1), FlatLists.of(2))).toList(), + is(Arrays.asList(1, 2))); + // arr = [] + assertThat(fn.apply(Collections.emptyList()).toList(), + is(Collections.singletonList(null))); + // arr = NULL + assertThat(fn.apply(null).toList(), + is(Collections.singletonList(null))); + } } diff --git a/core/src/test/resources/sql/unnest.iq b/core/src/test/resources/sql/unnest.iq index 054234fcbed3..c4650d77d35d 100644 --- a/core/src/test/resources/sql/unnest.iq +++ b/core/src/test/resources/sql/unnest.iq @@ -815,4 +815,320 @@ SELECT * FROM UNNEST(ARRAY[ !ok +# Tests for [CALCITE-7670] Uncollect should support LEFT JOIN UNNEST +!use scott + +# LEFT JOIN UNNEST +# Validated on PostgreSQL 14: same result. +WITH t(id, arr) AS ( + SELECT 1, ARRAY[10, 20] FROM (VALUES (0)) AS z(k) + UNION ALL + SELECT 2, ARRAY(SELECT 1 FROM (VALUES (0)) AS z(k) WHERE FALSE) + FROM (VALUES (0)) AS z(k) + UNION ALL + SELECT 3, CAST(NULL AS INTEGER ARRAY) FROM (VALUES (0)) AS z(k)) +SELECT t.id, u.x +FROM t LEFT JOIN UNNEST(t.arr) AS u(x) ON TRUE +ORDER BY t.id, u.x; ++----+----+ +| ID | X | ++----+----+ +| 1 | 10 | +| 1 | 20 | +| 2 | | +| 3 | | ++----+----+ +(4 rows) + +!ok + +# An inner UNNEST still drops the rows. +# Validated on PostgreSQL 14: same result. +WITH t(id, arr) AS ( + SELECT 1, ARRAY[10, 20] FROM (VALUES (0)) AS z(k) + UNION ALL + SELECT 3, CAST(NULL AS INTEGER ARRAY) FROM (VALUES (0)) AS z(k)) +SELECT t.id, u.x +FROM t, UNNEST(t.arr) AS u(x) +ORDER BY t.id, u.x; ++----+----+ +| ID | X | ++----+----+ +| 1 | 10 | +| 1 | 20 | ++----+----+ +(2 rows) + +!ok + +# WITH ORDINALITY: the NULL row from OUTER JOIN has no ordinal. +# Validated on PostgreSQL 14: same result. +WITH t(id, arr) AS ( + SELECT 1, ARRAY[10, 20] FROM (VALUES (0)) AS z(k) + UNION ALL + SELECT 3, CAST(NULL AS INTEGER ARRAY) FROM (VALUES (0)) AS z(k)) +SELECT t.id, u.x, u.o +FROM t LEFT JOIN UNNEST(t.arr) WITH ORDINALITY AS u(x, o) ON TRUE +ORDER BY t.id, u.x; ++----+----+---+ +| ID | X | O | ++----+----+---+ +| 1 | 10 | 1 | +| 1 | 20 | 2 | +| 3 | | | ++----+----+---+ +(3 rows) + +!ok + +# A filtering ON condition: the left row survives even when the condition +# rejects every element of a non-empty collection. The padding must come +# from the join, not from the collection being empty. +# Validated on PostgreSQL 14: same result. +WITH t(id, arr) AS ( + SELECT 1, ARRAY[10] FROM (VALUES (0)) AS z(k) + UNION ALL + SELECT 2, ARRAY[20, 30] FROM (VALUES (0)) AS z(k)) +SELECT t.id, u.x +FROM t LEFT JOIN UNNEST(t.arr) AS u(x) ON u.x > 15 +ORDER BY t.id, u.x; ++----+----+ +| ID | X | ++----+----+ +| 1 | | +| 2 | 20 | +| 2 | 30 | ++----+----+ +(3 rows) + +!ok + +# A NATURAL LEFT JOIN's derived condition compares common columns, so it can +# reject rows. Here t.x = 1 matches no element, and the left row survives. +# The condition becomes a Filter between the Correlate and the Uncollect, +# which is what stops CorrelateUncollectOuterRule from matching this shape. +# Validated on PostgreSQL 14: same result. +SELECT * +FROM (VALUES (1)) AS t(x) +NATURAL LEFT JOIN UNNEST(ARRAY[2, 3]) AS u(x); ++---+ +| X | ++---+ +| 1 | ++---+ +(1 row) + +!ok + +# Multi-collection UNNEST and LEFT JOIN +# Validated on PostgreSQL 14: same result. +WITH t(id, a, b) AS ( + SELECT 1, ARRAY[10, 20], ARRAY['p'] FROM (VALUES (0)) AS z(k) + UNION ALL + SELECT 3, CAST(NULL AS INTEGER ARRAY), CAST(NULL AS VARCHAR ARRAY) + FROM (VALUES (0)) AS z(k)) +SELECT t.id, u.x, u.y +FROM t LEFT JOIN UNNEST(t.a, t.b) AS u(x, y) ON TRUE +ORDER BY t.id, u.x; ++----+----+---+ +| ID | X | Y | ++----+----+---+ +| 1 | 10 | p | +| 1 | 20 | | +| 3 | | | ++----+----+---+ +(3 rows) + +!ok + +# MAP collection and LEFT JOIN +# Postgres does not support MAP values, so this is not validated on Postgres. +WITH t(id, m) AS ( + SELECT 1, MAP['a', 10, 'b', 20] FROM (VALUES (0)) AS z(k) + UNION ALL + SELECT 3, CAST(NULL AS MAP) FROM (VALUES (0)) AS z(k)) +SELECT t.id, u.k, u.v +FROM t LEFT JOIN UNNEST(t.m) AS u(k, v) ON TRUE +ORDER BY t.id, u.k; ++----+---+----+ +| ID | K | V | ++----+---+----+ +| 1 | a | 10 | +| 1 | b | 20 | +| 3 | | | ++----+---+----+ +(3 rows) + +!ok + +# An array of single-field ROW values expands to a single scalar column. +# Validated on PostgreSQL 14 using a named composite type (Postgres cannot +# type an anonymous ROW array): same result. +WITH t(id, arr) AS ( + SELECT 1, ARRAY[ROW(10), ROW(20)] FROM (VALUES (0)) AS z(k) + UNION ALL + SELECT 3, CAST(NULL AS ROW(a INTEGER) ARRAY) FROM (VALUES (0)) AS z(k)) +SELECT t.id, u.x +FROM t LEFT JOIN UNNEST(t.arr) AS u(x) ON TRUE +ORDER BY t.id, u.x; ++----+----+ +| ID | X | ++----+----+ +| 1 | 10 | +| 1 | 20 | +| 3 | | ++----+----+ +(3 rows) + +!ok + +# With LEFT JOIN every element column and the ordinality column must be nullable +WITH t(id, arr) AS ( + SELECT 1, ARRAY[10, 20] FROM (VALUES (0)) AS z(k)) +SELECT t.id, u.x, u.o +FROM t LEFT JOIN UNNEST(t.arr) WITH ORDINALITY AS u(x, o) ON TRUE; +ID INTEGER(10) NOT NULL +X INTEGER(10) +O INTEGER(10) +!type + +!use hr + +# Struct elements, expanded into one column per field (standard semantics): +# dept 30 has no employees. +# validated on PostgreSQL 14 with an equivalent composite type. +SELECT d."deptno", e."empid", e."name" +FROM "hr"."depts" AS d +LEFT JOIN UNNEST(d."employees") AS e ON TRUE +ORDER BY d."deptno", e."empid"; ++--------+-------+-----------+ +| deptno | empid | name | ++--------+-------+-----------+ +| 10 | 100 | Bill | +| 10 | 150 | Sebastian | +| 30 | | | +| 40 | 200 | Eric | ++--------+-------+-----------+ +(4 rows) + +!ok + +!use hr-presto + +# Struct elements kept whole (Trino semantics, PRESTO conformance). +SELECT d."deptno", e."emp" +FROM "hr"."depts" AS d +LEFT JOIN UNNEST(d."employees") AS e("emp") ON TRUE +ORDER BY d."deptno"; ++--------+------------------------------------+ +| deptno | emp | ++--------+------------------------------------+ +| 10 | {100, 10, Bill, 10000.0, 1000} | +| 10 | {150, 10, Sebastian, 7000.0, null} | +| 30 | | +| 40 | {200, 20, Eric, 8000.0, 500} | ++--------+------------------------------------+ +(4 rows) + +!ok + +# The same query with CorrelateUncollectOuterRule +!set hep-rules " ++CoreRules.CORRELATE_UNCOLLECT_OUTER" +SELECT d."deptno", e."emp" +FROM "hr"."depts" AS d +LEFT JOIN UNNEST(d."employees") AS e("emp") ON TRUE +ORDER BY d."deptno"; ++--------+------------------------------------+ +| deptno | emp | ++--------+------------------------------------+ +| 10 | {100, 10, Bill, 10000.0, 1000} | +| 10 | {150, 10, Sebastian, 7000.0, null} | +| 30 | | +| 40 | {200, 20, Eric, 8000.0, 500} | ++--------+------------------------------------+ +(4 rows) + +!ok + +!use scott + +# CorrelateUncollectOuterRule converts the LEFT correlate over an Uncollect +# into an INNER correlate over an outer Uncollect; UnnestDecorrelateRule +# then eliminates the correlate entirely, padding included. +!set hep-rules " ++CoreRules.CORRELATE_UNCOLLECT_OUTER ++CoreRules.UNNEST_DECORRELATE" + +# Validated on PostgreSQL 14: same result. +WITH t(id, arr) AS ( + SELECT 1, ARRAY[10, 20] FROM (VALUES (0)) AS z(k) + UNION ALL + SELECT 3, CAST(NULL AS INTEGER ARRAY) FROM (VALUES (0)) AS z(k)) +SELECT u.x +FROM t LEFT JOIN UNNEST(t.arr) AS u(x) ON TRUE +ORDER BY u.x; ++----+ +| X | ++----+ +| 10 | +| 20 | +| | ++----+ +(3 rows) + +!ok + +# The correlate is gone +EnumerableSort(sort0=[$0], dir0=[ASC]) + EnumerableUncollect(isOuter=[true]) + EnumerableUnion(all=[true]) + EnumerableCalc(expr#0=[{inputs}], expr#1=[10], expr#2=[20], expr#3=[ARRAY($t1, $t2)], EXPR$1=[$t3]) + EnumerableValues(tuples=[[{ 0 }]]) + EnumerableCalc(expr#0..1=[{inputs}], EXPR$1=[$t1]) + EnumerableValues(tuples=[[{ 3, null }]]) +!plan + +# The same elimination for an array of single-field ROW values. +WITH t(id, arr) AS ( + SELECT 1, ARRAY[ROW(10), ROW(20)] FROM (VALUES (0)) AS z(k) + UNION ALL + SELECT 3, CAST(NULL AS ROW(a INTEGER) ARRAY) FROM (VALUES (0)) AS z(k)) +SELECT u.x +FROM t LEFT JOIN UNNEST(t.arr) AS u(x) ON TRUE +ORDER BY u.x; ++----+ +| X | ++----+ +| 10 | +| 20 | +| | ++----+ +(3 rows) + +!ok + +# UnnestDecorrelateRule alone must not fire on a LEFT correlate +!set hep-rules " ++CoreRules.UNNEST_DECORRELATE" + +# Validated on PostgreSQL 14: same result. +WITH t(id, arr) AS ( + SELECT 1, ARRAY[10, 20] FROM (VALUES (0)) AS z(k) + UNION ALL + SELECT 3, CAST(NULL AS INTEGER ARRAY) FROM (VALUES (0)) AS z(k)) +SELECT u.x +FROM t LEFT JOIN UNNEST(t.arr) AS u(x) ON TRUE +ORDER BY u.x; ++----+ +| X | ++----+ +| 10 | +| 20 | +| | ++----+ +(3 rows) + +!ok + # End unnest.iq From 8cf1597dcead01ae0c9a9d5eeec6454fd5e51d53 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Mon, 10 Aug 2026 16:05:18 -0700 Subject: [PATCH 462/562] [CALCITE-7706] ARG_MIN ignores nullability of second argument Signed-off-by: Mihai Budiu --- .../calcite/sql/fun/SqlStdOperatorTable.java | 9 ++++-- core/src/test/resources/sql/agg.iq | 30 +++++++++++++++++++ .../apache/calcite/test/SqlOperatorTest.java | 15 ++++++++++ 3 files changed, 52 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java index ef7240d35e21..d50bc7299e67 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java @@ -68,6 +68,7 @@ import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.apache.calcite.sql.type.SqlTypeFamily; import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.sql.type.SqlTypeTransforms; import org.apache.calcite.sql.util.ReflectiveSqlOperatorTable; import org.apache.calcite.sql.validate.SqlConformance; import org.apache.calcite.sql.validate.SqlConformanceEnum; @@ -1125,7 +1126,9 @@ public class SqlStdOperatorTable extends ReflectiveSqlOperatorTable { */ public static final SqlBasicAggFunction ARG_MAX = SqlBasicAggFunction.create("ARG_MAX", SqlKind.ARG_MAX, - ReturnTypes.ARG0_NULLABLE_IF_EMPTY, OperandTypes.ANY_COMPARABLE) + ReturnTypes.ARG0_NULLABLE_IF_EMPTY + .andThen(SqlTypeTransforms.TO_NULLABLE), + OperandTypes.ANY_COMPARABLE) .withGroupOrder(Optionality.FORBIDDEN) .withFunctionType(SqlFunctionCategory.SYSTEM); @@ -1134,7 +1137,9 @@ public class SqlStdOperatorTable extends ReflectiveSqlOperatorTable { */ public static final SqlBasicAggFunction ARG_MIN = SqlBasicAggFunction.create("ARG_MIN", SqlKind.ARG_MIN, - ReturnTypes.ARG0_NULLABLE_IF_EMPTY, OperandTypes.ANY_COMPARABLE) + ReturnTypes.ARG0_NULLABLE_IF_EMPTY + .andThen(SqlTypeTransforms.TO_NULLABLE), + OperandTypes.ANY_COMPARABLE) .withGroupOrder(Optionality.FORBIDDEN) .withFunctionType(SqlFunctionCategory.SYSTEM); diff --git a/core/src/test/resources/sql/agg.iq b/core/src/test/resources/sql/agg.iq index bc24ba60145f..41f2436a25b7 100644 --- a/core/src/test/resources/sql/agg.iq +++ b/core/src/test/resources/sql/agg.iq @@ -3866,6 +3866,36 @@ group by deptno; !ok +# [CALCITE-7706] ARG_MIN ignores nullability of second argument +# Rows whose comparator is NULL are skipped, so a group where every +# comparator value is NULL yields NULL even though the value argument +# is NOT NULL. ARG_MAX behaves the same. +select g, arg_min(v, c) as mi, arg_max(v, c) as ma +from (values (1, 10, cast(null as integer))) as t(g, v, c) +group by g; ++---+----+----+ +| G | MI | MA | ++---+----+----+ +| 1 | | | ++---+----+----+ +(1 row) + +!ok + +# The IS NULL test must not be simplified away based on the result type. +select mi is null as n1, ma is null as n2 from ( + select g, arg_min(v, c) as mi, arg_max(v, c) as ma + from (values (1, 10, cast(null as integer))) as t(g, v, c) + group by g); ++------+------+ +| N1 | N2 | ++------+------+ +| true | true | ++------+------+ +(1 row) + +!ok + # ARG_MIN, ARG_MAX applied to an integer. select arg_min(deptno, empno) as mi, arg_max(deptno, empno) as ma, diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java index c6be7efedfec..9473246f62e9 100644 --- a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java +++ b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java @@ -18138,11 +18138,26 @@ void checkBitOr(SqlOperatorFixture f0, FunctionAlias functionAlias) { final Consumer consumer = f -> { f.checkAgg("arg_min(mod(x, 3), x)", xValues, isSingle("2")); f.checkAgg("arg_max(mod(x, 3), x)", xValues, isSingle("1")); + f.checkAggType("arg_min(1, 2)", "INTEGER NOT NULL"); + f.checkAggType("arg_max(1, 2)", "INTEGER NOT NULL"); + // Test cases for [CALCITE-7706] + // ARG_MIN ignores nullability of second argument + f.checkAggType("arg_min(1, cast(null as integer))", "INTEGER"); + f.checkAggType("arg_max(1, cast(null as integer))", "INTEGER"); + f.checkAggType("arg_min(cast(null as integer), 2)", "INTEGER"); + f.checkAggType("arg_max(cast(null as integer), 2)", "INTEGER"); + // Nullable without GROUP BY even for non-nullable arguments, since the + // input may be empty + f.checkColumnType("select arg_min(1, 2) from (values (1))", "INTEGER"); }; final Consumer consumer2 = f -> { f.checkAgg("min_by(mod(x, 3), x)", xValues, isSingle("2")); f.checkAgg("max_by(mod(x, 3), x)", xValues, isSingle("1")); + // Test cases for [CALCITE-7706] + // ARG_MIN ignores nullability of second argument + f.checkAggType("min_by(1, cast(null as integer))", "INTEGER"); + f.checkAggType("max_by(1, cast(null as integer))", "INTEGER"); }; consumer.accept(f0); From c85bf15653720bb29e0cba085b9182dd1506c520 Mon Sep 17 00:00:00 2001 From: Sergey Nuyanzin Date: Sun, 9 Aug 2026 00:09:30 +0200 Subject: [PATCH 463/562] [CALCITE-7698] RelDecorrelator throws AssertionError when decorrelating a correlated sub-query with outer join --- .../calcite/sql2rel/RelDecorrelator.java | 1 + .../calcite/sql2rel/RelDecorrelatorTest.java | 57 +++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java b/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java index daabe37b89d5..fbf949ac189a 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java @@ -4153,6 +4153,7 @@ private Frame supplyMissingCorVars(RelNode oldInput, Frame frame, } } + corDefOutputs.putAll(frame.corDefOutputs); return createFrameWithValueGenerator(oldInput, frame, miss, corDefOutputs); } diff --git a/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java b/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java index a15d741dc947..80ef7f588ade 100644 --- a/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java +++ b/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java @@ -1737,6 +1737,63 @@ public static Frameworks.ConfigBuilder config() { assertThat(after, hasTree(planAfter)); } + /** Test case for [CALCITE-7698]; + * RelDecorrelator throws AssertionError when decorrelating + * a correlated sub-query with outer join. */ + @Test void testExistsWithCorrelatedOnWhereLeftJoin() { + final String sql = "WITH l(a, b, c) AS (VALUES (1, 1, 1), (2, 2, 2), (3, 3, 3)),\n" + + " r(d, e, f) AS (VALUES (1, 1, 1), (2, 2, 2), (3, 3, 3)),\n" + + " t(i, j, k) AS (VALUES (1, 1, 1), (2, 2, 2), (3, 3, 3))\n" + + "SELECT * FROM l WHERE EXISTS (\n" + + " SELECT * FROM (SELECT f FROM r WHERE r.d = l.a AND r.e > 10) r1\n" + + " LEFT JOIN (SELECT i, k FROM t WHERE t.j = l.b AND i < 50) t1 ON r1.f = t1.k)"; + final String planAfter = "" + + "LogicalProject(A=[$0], B=[$1], C=[$2])\n" + + " LogicalJoin(condition=[AND(=($0, $3), =($1, $4))], joinType=[inner])\n" + + " LogicalValues(tuples=[[{ 1, 1, 1 }, { 2, 2, 2 }, { 3, 3, 3 }]])\n" + + " LogicalProject(EXPR$0=[$1], EXPR$1=[$2], $f2=[true])\n" + + " LogicalJoin(condition=[AND(=($0, $4), IS NOT DISTINCT FROM($2, $5))], joinType=[left])\n" + + " LogicalJoin(condition=[true], joinType=[inner])\n" + + " LogicalProject(F=[$2], EXPR$0=[$0])\n" + + " LogicalFilter(condition=[>($1, 10)])\n" + + " LogicalValues(tuples=[[{ 1, 1, 1 }, { 2, 2, 2 }, { 3, 3, 3 }]])\n" + + " LogicalProject(EXPR$1=[$1])\n" + + " LogicalValues(tuples=[[{ 1, 1, 1 }, { 2, 2, 2 }, { 3, 3, 3 }]])\n" + + " LogicalProject(I=[$0], K=[$2], EXPR$1=[$1])\n" + + " LogicalFilter(condition=[<($0, 50)])\n" + + " LogicalValues(tuples=[[{ 1, 1, 1 }, { 2, 2, 2 }, { 3, 3, 3 }]])\n"; + assertThat(decorrelateSql(sql), hasTree(planAfter)); + } + + private RelNode decorrelateSql(String sql) { + final FrameworkConfig frameworkConfig = config().build(); + final RelBuilder builder = RelBuilder.create(frameworkConfig); + final RelOptCluster cluster = builder.getCluster(); + final Planner planner = Frameworks.getPlanner(frameworkConfig); + final RelNode originalRel; + try { + final SqlNode parse = planner.parse(sql); + final SqlNode validate = planner.validate(parse); + originalRel = planner.rel(validate).rel; + } catch (Exception e) { + throw TestUtil.rethrow(e); + } + final HepProgram hepProgram = HepProgram.builder() + .addRuleCollection( + ImmutableList.of( + CoreRules.FILTER_SUB_QUERY_TO_CORRELATE, + CoreRules.PROJECT_SUB_QUERY_TO_CORRELATE, + CoreRules.JOIN_SUB_QUERY_TO_CORRELATE)) + .build(); + final Program program = + Programs.of(hepProgram, true, requireNonNull(cluster.getMetadataProvider())); + final RelNode before = + program.run(cluster.getPlanner(), originalRel, cluster.traitSet(), + Collections.emptyList(), Collections.emptyList()); + return RelDecorrelator.decorrelateQuery(before, builder, + RuleSets.ofList(Collections.emptyList()), RuleSets.ofList(Collections.emptyList())); + } + /** * Test case for [CALCITE-7661] * RelDecorrelator loses shared correlation constraint across inner join inputs. From 3828a4b8ebae3141eeb225fc18e7d66b599b995c Mon Sep 17 00:00:00 2001 From: microbluey Date: Tue, 11 Aug 2026 13:49:41 +0800 Subject: [PATCH 464/562] [CALCITE-7487] ProjectJoinTransposeRule throws ArrayIndexOutOfBoundsException in PushProjector when a Join input has a zero-column row type PushProjector.locateAllRefs contains a workaround, originally added for Fennel, that arbitrarily projects the first column of a Join or SetOp input when nothing else is projected from it. The workaround assumes the input has a first column to fall back on. That assumption does not hold for a zero-column input: a Values with an empty row type and a single empty tuple returns one row with zero columns and is the identity for cross join. It arises when an Aggregate with GROUP BY () has its output pruned to zero columns. In that case the workaround sets a bit that points past the input's fields, and createProjectRefsAndExprs later uses that bit to index into an empty field list, throwing ArrayIndexOutOfBoundsException. Guard both the left and the right workaround on the corresponding input having at least one field, so that the workaround is skipped rather than producing an out-of-range reference. Each guard tests the field count of the input it protects: nFields for the left, nFieldsRight for the right. Both directions crash, so add a regression test for each. --- .../calcite/rel/rules/PushProjector.java | 15 ++++-- .../apache/calcite/test/RelOptRulesTest.java | 50 +++++++++++++++++++ .../apache/calcite/test/RelOptRulesTest.xml | 38 ++++++++++++++ 3 files changed, 100 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rel/rules/PushProjector.java b/core/src/main/java/org/apache/calcite/rel/rules/PushProjector.java index 20310a95c9e6..caa9b4d4dedf 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/PushProjector.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/PushProjector.java @@ -457,13 +457,22 @@ public boolean locateAllRefs() { || (childRel instanceof SetOp)) { // if nothing is projected from the children, arbitrarily project // the first columns; this is necessary since Fennel doesn't - // handle 0-column projections - if (nProject == 0 && childPreserveExprs.isEmpty()) { + // handle 0-column projections. + // + // An input may legitimately have a zero-column row type: for + // example, a Values with an empty row type and a single empty + // tuple, which returns one row with zero columns and is the + // identity for cross join. There is no first column to fall back + // on in that case, so skip the workaround rather than set a bit + // that points past the input's fields; createProjectRefsAndExprs + // would use it to index into an empty field list. + if (nProject == 0 && childPreserveExprs.isEmpty() && nFields > 0) { projRefs.set(0); nProject = 1; } if (childRel instanceof Join) { - if (nRightProject == 0 && rightPreserveExprs.isEmpty()) { + if (nRightProject == 0 && rightPreserveExprs.isEmpty() + && nFieldsRight > 0) { projRefs.set(nFields); nRightProject = 1; } diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index ca5cc42385f5..06ee61c26c17 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -1559,6 +1559,56 @@ private void checkSemiOrAntiJoinProjectTranspose(JoinRelType type) { checkJoinProjectTransposeDoesNotMatch(JoinRelType.LEFT_MARK); } + /** Test case for + * [CALCITE-7487] + * ProjectJoinTransposeRule throws ArrayIndexOutOfBoundsException in + * PushProjector when a Join input has a zero-column row type. */ + @Test void testProjectJoinTransposeWithZeroColumnRightInput() { + relFn(b -> zeroColumnJoinInputRelFn(b, false)) + .withRule(CoreRules.PROJECT_JOIN_TRANSPOSE).check(); + } + + /** Test case for + * [CALCITE-7487] + * ProjectJoinTransposeRule throws ArrayIndexOutOfBoundsException in + * PushProjector when a Join input has a zero-column row type. */ + @Test void testProjectJoinTransposeWithZeroColumnLeftInput() { + relFn(b -> zeroColumnJoinInputRelFn(b, true)) + .withRule(CoreRules.PROJECT_JOIN_TRANSPOSE).check(); + } + + /** Builds {@code Project(CAST(col1))} over a cross join in which one input is + * DEE -- a {@link org.apache.calcite.rel.core.Values} with an empty row type, + * the identity for cross join. The project must be non-identity, otherwise + * {@link RelBuilder} collapses it away and the rule never fires. */ + private static RelNode zeroColumnJoinInputRelFn(RelBuilder b, + boolean deeOnLeft) { + final RelDataTypeFactory typeFactory = b.getTypeFactory(); + final RelDataType bigintType = + typeFactory.createSqlType(SqlTypeName.BIGINT); + final RelNode nonEmpty = b + .values( + ImmutableList.of( + ImmutableList.of( + (RexLiteral) b.getRexBuilder().makeZeroLiteral(bigintType))), + typeFactory.builder().add("col1", bigintType).build()) + .build(); + final RelNode dee = b + .values(ImmutableList.of(ImmutableList.of()), + typeFactory.builder().build()) + .build(); + final RelDataType varcharType = + typeFactory.createSqlType(SqlTypeName.VARCHAR); + return b + .push(deeOnLeft ? dee : nonEmpty) + .push(deeOnLeft ? nonEmpty : dee) + .join(JoinRelType.INNER, b.literal(true)) + // DEE contributes no fields, so the sole column is at index 0 + // whichever side it is on. + .project(b.getRexBuilder().makeCast(varcharType, b.field(0))) + .build(); + } + /** A SEMI, ANTI or LEFT_MARK join does not project its right input, so * {@link JoinProjectTransposeRule} must not pull projects above it. */ private void checkJoinProjectTransposeDoesNotMatch(JoinRelType type) { diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index 0b544e3f1c63..49681f45a3fc 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -12480,6 +12480,44 @@ LogicalProject(DEPTNO=[$0]) LogicalAggregate(group=[{}], DUMMY=[COUNT()]) LogicalProject(EMPNO=[$0]) LogicalTableScan(table=[[scott, EMP]]) +]]> + + + + + + + + + + + + + + + + From c54ad37be9fc8656356f0d55e730a63ebb93e1fc Mon Sep 17 00:00:00 2001 From: xuzifu666 <1206332514@qq.com> Date: Fri, 7 Aug 2026 21:41:29 +0800 Subject: [PATCH 465/562] [CALCITE-7697] Simplify window PARTITION BY and ORDER BY keys in RelBuilder --- .../org/apache/calcite/tools/RelBuilder.java | 63 ++++++++++- .../rel/rel2sql/RelToSqlConverterTest.java | 5 +- .../apache/calcite/test/RelBuilderTest.java | 102 ++++++++++++++++++ .../apache/calcite/test/RelOptRulesTest.xml | 12 +-- core/src/test/resources/sql/sub-query.iq | 4 +- 5 files changed, 175 insertions(+), 11 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java index 7cc7437c6223..97d2e3080dfe 100644 --- a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java +++ b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java @@ -5142,12 +5142,73 @@ private OverCall orderBy_(ImmutableList sortKeys) { } }; final RelDataType type = op.inferReturnType(bind); + final ImmutableList newPartitionKeys = + simplifyPartitionKeys(partitionKeys); + final ImmutableList newSortKeys = + simplifySortKeys(newPartitionKeys, sortKeys); final RexNode over = getRexBuilder() - .makeOver(pos, type, op, operands, partitionKeys, sortKeys, + .makeOver(pos, type, op, operands, newPartitionKeys, newSortKeys, lowerBound, upperBound, exclude, rows, allowPartial, nullWhenCountZero, distinct, ignoreNulls); return aliasMaybe(over, alias); } + + /** Removes constant keys from a window's {@code PARTITION BY}. A constant + * partition key places every row in the same partition, so it does not + * partition the data and can be dropped. */ + private ImmutableList simplifyPartitionKeys( + List partitionKeys) { + final ImmutableList.Builder newKeys = ImmutableList.builder(); + for (RexNode key : partitionKeys) { + if (!RexUtil.isConstant(key)) { + newKeys.add(key); + } + } + return newKeys.build(); + } + + /** Removes redundant keys from a window's {@code ORDER BY}. A sort key is + * redundant if it is constant, or if it is functionally determined by the + * partition keys and earlier sort keys (those columns are fixed within a + * partition, so the key cannot affect the ordering). For example, with + * {@code PARTITION BY x, y ORDER BY x + y, z} the key {@code x + y} only + * references fixed columns and is dropped, leaving {@code ORDER BY z}. */ + private ImmutableList simplifySortKeys( + List partitionKeys, List sortKeys) { + // A RANGE frame with a value offset (e.g. RANGE BETWEEN 5 PRECEDING) + // derives its bounds from the sort key values, so its keys must be kept. + if (!rows + && (lowerBound.getOffset() != null || upperBound.getOffset() != null)) { + return ImmutableList.copyOf(sortKeys); + } + // Columns whose value is fixed within a partition: partition keys plus + // columns pinned by an earlier single-column sort keys. + ImmutableBitSet fixedColumns = ImmutableBitSet.of(); + for (RexNode key : partitionKeys) { + if (key instanceof RexInputRef) { + fixedColumns = fixedColumns.set(((RexInputRef) key).getIndex()); + } + } + final ImmutableList.Builder newSortKeys = + ImmutableList.builder(); + for (RexFieldCollation collation : sortKeys) { + final RexNode key = collation.left; + if (RexUtil.isConstant(key)) { + continue; + } + final ImmutableBitSet keyColumns = RelOptUtil.InputFinder.bits(key); + if (!keyColumns.isEmpty() + && RexUtil.isDeterministic(key) + && fixedColumns.contains(keyColumns)) { + continue; + } + newSortKeys.add(collation); + if (key instanceof RexInputRef) { + fixedColumns = fixedColumns.set(((RexInputRef) key).getIndex()); + } + } + return newSortKeys.build(); + } } /** Collects the extra expressions needed for {@link #aggregate}. diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java index 2a500bcce7d5..779b320574a1 100644 --- a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java +++ b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java @@ -2858,8 +2858,9 @@ private SqlDialect nonOrdinalDialect() { @Test void testNoNeedRewriteOrderByConstantsForOver() { final String query = "select row_number() over " + "(order by 1 nulls last) from \"employee\""; - // Default dialect keep numeric constant keys in the over of order-by. - sql(query).ok("SELECT ROW_NUMBER() OVER (ORDER BY 1)\n" + // A constant ORDER BY key places every row in the same peer group, so it + // is removed when the window is built, leaving an empty OVER clause. + sql(query).ok("SELECT ROW_NUMBER() OVER ()\n" + "FROM \"foodmart\".\"employee\""); } diff --git a/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java b/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java index cce1aea3095b..ebe8990c1ddc 100644 --- a/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java @@ -1186,6 +1186,108 @@ private RexNode caseCall(RelBuilder b, RexNode ref, RexNode... nodes) { assertThat(f.apply(createBuilder()), hasTree(expected)); } + /** Tests that RelBuilder removes a constant key from a window's + * {@code PARTITION BY}, since a constant partition key places every row in + * the same partition. */ + @Test void testProjectOverConstantPartitionKey() { + final Function f = b -> b.scan("EMP") + .project(b.field("DEPTNO"), + b.aggregateCall(SqlStdOperatorTable.ROW_NUMBER) + .over() + .partitionBy(b.literal(1)) + .orderBy(b.field("EMPNO")) + .rowsUnbounded() + .as("x")) + .build(); + final String expected = "" + + "LogicalProject(DEPTNO=[$7], x=[ROW_NUMBER() OVER (ORDER BY $0)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + assertThat(f.apply(createBuilder()), hasTree(expected)); + } + + /** Tests that RelBuilder keeps non-constant partition keys and drops only the + * constant one. */ + @Test void testProjectOverPartialConstantPartitionKey() { + final Function f = b -> b.scan("EMP") + .project(b.field("DEPTNO"), + b.aggregateCall(SqlStdOperatorTable.SUM, b.field("SAL")) + .over() + .partitionBy(b.field("DEPTNO"), b.literal(1)) + .orderBy(b.field("EMPNO")) + .rowsUnbounded() + .as("x")) + .build(); + final String expected = "" + + "LogicalProject(DEPTNO=[$7], " + + "x=[SUM($5) OVER (PARTITION BY $7 ORDER BY $0 RANGE BETWEEN " + + "UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + assertThat(f.apply(createBuilder()), hasTree(expected)); + } + + /** Tests that RelBuilder removes a constant key from a window's + * {@code ORDER BY}. */ + @Test void testProjectOverConstantSortKey() { + final Function f = b -> b.scan("EMP") + .project(b.field("DEPTNO"), + b.aggregateCall(SqlStdOperatorTable.ROW_NUMBER) + .over() + .partitionBy() + .orderBy(b.literal(1), b.field("EMPNO")) + .rowsUnbounded() + .as("x")) + .build(); + final String expected = "" + + "LogicalProject(DEPTNO=[$7], x=[ROW_NUMBER() OVER (ORDER BY $0)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + assertThat(f.apply(createBuilder()), hasTree(expected)); + } + + /** Tests that RelBuilder removes a sort key that is functionally determined + * by the partition keys: with {@code PARTITION BY DEPTNO, SAL ORDER BY + * DEPTNO + SAL, EMPNO} the key {@code DEPTNO + SAL} references only fixed + * columns and is dropped, leaving {@code ORDER BY EMPNO}. */ + @Test void testProjectOverFunctionallyDependentSortKey() { + final Function f = b -> b.scan("EMP") + .project(b.field("DEPTNO"), + b.aggregateCall(SqlStdOperatorTable.ROW_NUMBER) + .over() + .partitionBy(b.field("DEPTNO"), b.field("SAL")) + .orderBy( + b.call(SqlStdOperatorTable.PLUS, b.field("DEPTNO"), + b.field("SAL")), + b.field("EMPNO")) + .rowsUnbounded() + .as("x")) + .build(); + final String expected = "" + + "LogicalProject(DEPTNO=[$7], " + + "x=[ROW_NUMBER() OVER (PARTITION BY $7, $5 ORDER BY $0)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + assertThat(f.apply(createBuilder()), hasTree(expected)); + } + + /** Tests that RelBuilder keeps a sort key that would otherwise be dropped + * (here {@code DEPTNO}, which equals the partition key) when the frame is a + * RANGE with a value offset, because such a frame derives its bounds from the + * sort key values. */ + @Test void testProjectOverRangeOffsetKeepsSortKey() { + final Function f = b -> b.scan("EMP") + .project(b.field("DEPTNO"), + b.aggregateCall(SqlStdOperatorTable.SUM, b.field("SAL")) + .over() + .partitionBy(b.field("DEPTNO")) + .orderBy(b.field("DEPTNO")) + .rangeBetween(b.preceding(b.literal(5)), b.currentRow()) + .as("x")) + .build(); + final String expected = "" + + "LogicalProject(DEPTNO=[$7], " + + "x=[SUM($5) OVER (PARTITION BY $7 ORDER BY $7 RANGE 5 PRECEDING)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + assertThat(f.apply(createBuilder()), hasTree(expected)); + } + @Test void testRename() { final RelBuilder builder = RelBuilder.create(config().build()); diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index 49681f45a3fc..96ca66bbc72f 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -17412,7 +17412,7 @@ from ( @@ -17437,7 +17437,7 @@ from ( @@ -17445,7 +17445,7 @@ LogicalProject(COL1=[SUM(100) OVER (ORDER BY $7, $0 RANGE BETWEEN CURRENT ROW AN diff --git a/core/src/test/resources/sql/sub-query.iq b/core/src/test/resources/sql/sub-query.iq index d6364652ccd0..5d1bbb738675 100644 --- a/core/src/test/resources/sql/sub-query.iq +++ b/core/src/test/resources/sql/sub-query.iq @@ -8640,7 +8640,7 @@ EnumerableCalc(expr#0..1=[{inputs}], T1B=[$t1]) EnumerableValues(tuples=[[{ 'val1a', 6 }, { 'val1b', 8 }, { 'val1a', 16 }, { 'val1a', 16 }, { 'val1c', 8 }, { 'val1d', null }, { 'val1d', null }, { 'val1e', 10 }, { 'val1e', 10 }, { 'val1d', 10 }, { 'val1a', 6 }, { 'val1e', 10 }]]) EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableAggregate(group=[{0}], EXPR$0=[MAX($4)]) - EnumerableWindow(window#0=[window(partition {0, 1, 3} order by [3] aggs [RANK()])]) + EnumerableWindow(window#0=[window(partition {0, 1, 3} aggs [RANK()])]) EnumerableMergeJoin(condition=[=($2, $3)], joinType=[inner]) EnumerableSort(sort0=[$2], dir0=[ASC]) EnumerableValues(tuples=[[{ 'val2a', 6, 12 }, { 'val1b', 10, 12 }, { 'val1b', 8, 16 }, { 'val1c', 12, 16 }, { 'val1b', null, 16 }, { 'val2e', 8, null }, { 'val1f', 19, null }, { 'val1b', 10, 12 }, { 'val1b', 8, 16 }, { 'val1c', 12, 16 }, { 'val1e', 8, null }, { 'val1f', 19, null }, { 'val1b', null, 16 }]]) @@ -8681,7 +8681,7 @@ EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableNestedLoopJoin(condition=[>(CAST($0):BIGINT, $1)], joinType=[inner]) EnumerableValues(tuples=[[{ 6 }, { 8 }, { 16 }, { 16 }, { 8 }, { null }, { null }, { 10 }, { 10 }, { 10 }, { 6 }, { 10 }]]) EnumerableAggregate(group=[{}], EXPR$0=[MAX($3)]) - EnumerableWindow(window#0=[window(partition {1, 2} order by [1] aggs [RANK()])]) + EnumerableWindow(window#0=[window(partition {1, 2} aggs [RANK()])]) EnumerableAggregate(group=[{0, 1}], T3D=[MAX($2)]) EnumerableValues(tuples=[[{ 6, 12, 110 }, { 6, 12, 10 }, { 10, 12, 219 }, { 10, 12, 19 }, { 8, 16, 319 }, { 8, 16, 19 }, { 17, 16, 519 }, { 17, 16, 19 }, { null, 16, 419 }, { null, 16, 19 }, { 8, null, 719 }, { 8, null, 19 }]]) !plan From fc6e97f3db1f4a6a0efa742a1090c929147f8793 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Mon, 10 Aug 2026 11:48:20 -0700 Subject: [PATCH 466/562] [CALCITE-7699] ARRAY_INSERT fails in validation when first argument is not an array constructor Signed-off-by: Mihai Budiu --- .../calcite/sql/fun/SqlLibraryOperators.java | 4 +- .../sql/validate/SqlValidatorUtil.java | 39 ++++++++++++++- .../apache/calcite/test/SqlOperatorTest.java | 50 +++++++++++++++++++ 3 files changed, 89 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java index aef50bbc63b8..174f562a805e 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java @@ -1471,7 +1471,7 @@ private static RelDataType arrayAppendPrependReturnType(SqlOperatorBinding opBin adjustTypeForArrayFunctions(type, opBinding, 1); } else { SqlValidatorUtil. - adjustTypeForArrayFunctions(type, opBinding, 0); + adjustArrayTypeForArrayFunctions(type, opBinding, 0); } } @@ -1601,7 +1601,7 @@ private static RelDataType arrayInsertReturnType(SqlOperatorBinding opBinding) { } if (!componentType.equalsSansFieldNamesAndNullability(type)) { SqlValidatorUtil. - adjustTypeForArrayFunctions(type, opBinding, 0); + adjustArrayTypeForArrayFunctions(type, opBinding, 0); } boolean nullable = arrayType.isNullable() || elementType1.isNullable(); return SqlTypeUtil.createArrayType(opBinding.getTypeFactory(), type, nullable); diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorUtil.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorUtil.java index 84495c0a6680..ca1b2f927680 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorUtil.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorUtil.java @@ -1428,6 +1428,9 @@ public static void adjustTypeForArrayConstructor( * if targetType is double, this method would ensure that the elements of the * first array and the second operand are cast to double. * + *

      Use {@link #adjustArrayTypeForArrayFunctions} for an operand that is + * itself an array whose element type must become {@code targetType}. + * * @param targetType The target {@link RelDataType} to which the operands should be cast. * @param opBinding The {@link SqlOperatorBinding} context, which provides access to the * {@link SqlCall} and its operands. @@ -1437,6 +1440,30 @@ public static void adjustTypeForArrayConstructor( */ public static void adjustTypeForArrayFunctions( RelDataType targetType, SqlOperatorBinding opBinding, int... indexes) { + adjustTypeForArrayFunctions(targetType, opBinding, false, indexes); + } + + /** + * Same as {@link #adjustTypeForArrayFunctions}, for operands that are + * arrays whose element type must become {@code targetType}. + * + *

      The two methods differ for an operand that is not a call to the ARRAY + * constructor (e.g., a CAST expression): this method casts such an operand to an array of + * {@code targetType} rather than to {@code targetType} itself. + * + * @param targetType The array element type to which the operands' elements + * should be cast. + * @param arrayOperands The indexes of the array operands within the {@link SqlCall} + * that need to be adjusted to the target type. + */ + public static void adjustArrayTypeForArrayFunctions( + RelDataType targetType, SqlOperatorBinding opBinding, int... arrayOperands) { + adjustTypeForArrayFunctions(targetType, opBinding, true, arrayOperands); + } + + private static void adjustTypeForArrayFunctions( + RelDataType targetType, SqlOperatorBinding opBinding, boolean arrayOperands, + int... indexes) { if (opBinding instanceof SqlCallBinding) { requireNonNull(targetType, "array function target type"); final SqlValidator validator = ((SqlCallBinding) opBinding).getValidator(); @@ -1459,9 +1486,17 @@ public static void adjustTypeForArrayFunctions( targetType, priorType.isNullable())); } } else { - SqlNode cast = castTo(operand, targetType); + RelDataType castType = targetType; + if (arrayOperands) { + // An array operand that is not an ARRAY constructor call must be + // cast to an array of the target type, not to the target type + castType = + SqlTypeUtil.createArrayType(opBinding.getTypeFactory(), + targetType, opBinding.getOperandType(idx).isNullable()); + } + SqlNode cast = castTo(operand, castType); call.setOperand(idx, cast); - validator.setValidatedNodeType(cast, targetType); + validator.setValidatedNodeType(cast, castType); } } } diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java index 9473246f62e9..95381426f936 100644 --- a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java +++ b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java @@ -8343,6 +8343,22 @@ void checkRegexpExtract(SqlOperatorFixture f0, FunctionAlias functionAlias) { f.checkFails("^array_append(array[1, 2], true)^", "INTEGER is not comparable to BOOLEAN", false); + // Test cases for [CALCITE-7699] + // ARRAY_INSERT fails in validation when first argument is not an array + // constructor + f.checkScalar("array_append(cast(array[1, 2, 3] as integer array), " + + "cast(4 as double))", + "[1.0, 2.0, 3.0, 4.0]", "DOUBLE NOT NULL ARRAY NOT NULL"); + f.checkScalar("array_append(array_distinct(array[1, 2, 3]), " + + "cast(4 as double))", + "[1.0, 2.0, 3.0, 4.0]", "DOUBLE NOT NULL ARRAY NOT NULL"); + // Array of arrays as a non-constructor operand + f.checkScalar("array_append(" + + "cast(array[array[1, 2]] as integer array array), " + + "array[cast(3 as double)])", + "[[1.0, 2.0], [3.0]]", + "DOUBLE NOT NULL ARRAY NOT NULL ARRAY NOT NULL"); + // element cast to the biggest type f.checkScalar("array_append(array(cast(1 as tinyint)), 2)", "[1, 2]", "INTEGER NOT NULL ARRAY NOT NULL"); @@ -8682,6 +8698,16 @@ void checkRegexpExtract(SqlOperatorFixture f0, FunctionAlias functionAlias) { f.checkFails("^array_prepend(array[1, 2], true)^", "INTEGER is not comparable to BOOLEAN", false); + // Test case for [CALCITE-7699] + // ARRAY_INSERT fails in validation when first argument is not an array + // constructor + f.checkScalar("array_prepend(cast(array[1, 2, 3] as integer array), " + + "cast(4 as double))", + "[4.0, 1.0, 2.0, 3.0]", "DOUBLE NOT NULL ARRAY NOT NULL"); + f.checkScalar("array_prepend(array_distinct(array[1, 2, 3]), " + + "cast(4 as double))", + "[4.0, 1.0, 2.0, 3.0]", "DOUBLE NOT NULL ARRAY NOT NULL"); + // element cast to the biggest type f.checkScalar("array_prepend(array(1), cast(3 as float))", "[3.0, 1.0]", "FLOAT NOT NULL ARRAY NOT NULL"); @@ -9015,6 +9041,30 @@ void checkArrayReverseFunc(SqlOperatorFixture f0, SqlFunction function, + "An index shall be either < 0 or > 0 \\(the first element has index 1\\) " + "and not exceeds the allowed limit.", true); + // Test case for [CALCITE-7699] + // ARRAY_INSERT fails in validation when first argument is not an array + // constructor + f1.checkScalar("array_insert(cast(array[1, 2, 3] as integer array), 3, " + + "cast(4 as double))", + "[1.0, 2.0, 4.0, 3.0]", "DOUBLE ARRAY NOT NULL"); + f1.checkScalar("array_insert(array_distinct(array[1, 2, 3]), 3, " + + "cast(4 as double))", + "[1.0, 2.0, 4.0, 3.0]", "DOUBLE ARRAY NOT NULL"); + f1.checkScalar("array_insert(cast(array[1, 2] as integer array), 2, 2.5)", + "[1.0, 2.5, 2.0]", "DECIMAL(11, 1) ARRAY NOT NULL"); + f1.checkNull("array_insert(cast(null as integer array), 3, " + + "cast(4 as double))"); + f1.checkType("array_insert(cast(null as integer array), 3, " + + "cast(4 as double))", "DOUBLE ARRAY"); + // Array of arrays as a non-constructor operand + f1.checkScalar("array_insert(" + + "cast(array[array[1, 2]] as integer array array), 1, array[3])", + "[[3], [1, 2]]", "INTEGER NOT NULL ARRAY ARRAY NOT NULL"); + f1.checkScalar("array_insert(" + + "cast(array[array[1, 2]] as integer array array), 1, " + + "array[cast(3 as double)])", + "[[3.0], [1.0, 2.0]]", "DOUBLE NOT NULL ARRAY ARRAY NOT NULL"); + f1.checkScalar("array_insert(array[1, 2, 3], 3, 4)", "[1, 2, 4, 3]", "INTEGER ARRAY NOT NULL"); f1.checkScalar("array_insert(array[1, 2, 3], 3, cast(null as integer))", From d810d430b562338166b9a43b053d7a8964421c56 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Mon, 10 Aug 2026 15:21:22 -0700 Subject: [PATCH 467/562] [CALCITE-7704] ARRAY_INSERT crashes in code generation with array-of-arrays argument Signed-off-by: Mihai Budiu --- .../calcite/sql/validate/SqlValidatorUtil.java | 8 ++++++-- .../org/apache/calcite/test/SqlOperatorTest.java | 14 ++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorUtil.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorUtil.java index ca1b2f927680..b4465ab4345a 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorUtil.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorUtil.java @@ -1476,14 +1476,18 @@ private static void adjustTypeForArrayFunctions( // such as spark array, the SqlKind is other function. // however, the name is same for those different array forms. && "ARRAY".equals(((SqlBasicCall) operand).getOperator().getName())) { - call.setOperand(idx, castArrayElementTo(validator, operand, targetType)); + RelDataType elementType = + arrayOperands ? targetType + : requireNonNull(targetType.getComponentType(), + () -> "componentType of " + targetType); + call.setOperand(idx, castArrayElementTo(validator, operand, elementType)); // The rewrite changes the element types of the array constructor, // so the type the validator has recorded for it must change too RelDataType priorType = validator.getValidatedNodeTypeIfKnown(operand); if (priorType != null) { validator.setValidatedNodeType(operand, SqlTypeUtil.createArrayType(opBinding.getTypeFactory(), - targetType, priorType.isNullable())); + elementType, priorType.isNullable())); } } else { RelDataType castType = targetType; diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java index 95381426f936..6082c27638df 100644 --- a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java +++ b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java @@ -8358,6 +8358,10 @@ void checkRegexpExtract(SqlOperatorFixture f0, FunctionAlias functionAlias) { + "array[cast(3 as double)])", "[[1.0, 2.0], [3.0]]", "DOUBLE NOT NULL ARRAY NOT NULL ARRAY NOT NULL"); + // Test case for [CALCITE-7704] + // ARRAY_INSERT crashes in code generation with array-of-arrays argument + f.checkScalar("array_append(array[array[cast(1 as double)]], array[2])", + "[[1.0], [2.0]]", "DOUBLE NOT NULL ARRAY NOT NULL ARRAY NOT NULL"); // element cast to the biggest type f.checkScalar("array_append(array(cast(1 as tinyint)), 2)", "[1, 2]", @@ -8707,6 +8711,10 @@ void checkRegexpExtract(SqlOperatorFixture f0, FunctionAlias functionAlias) { f.checkScalar("array_prepend(array_distinct(array[1, 2, 3]), " + "cast(4 as double))", "[4.0, 1.0, 2.0, 3.0]", "DOUBLE NOT NULL ARRAY NOT NULL"); + // Test case for [CALCITE-7704] + // ARRAY_INSERT crashes in code generation with array-of-arrays argument + f.checkScalar("array_prepend(array[array[cast(1 as double)]], array[2])", + "[[2.0], [1.0]]", "DOUBLE NOT NULL ARRAY NOT NULL ARRAY NOT NULL"); // element cast to the biggest type f.checkScalar("array_prepend(array(1), cast(3 as float))", "[3.0, 1.0]", @@ -9064,6 +9072,12 @@ void checkArrayReverseFunc(SqlOperatorFixture f0, SqlFunction function, + "cast(array[array[1, 2]] as integer array array), 1, " + "array[cast(3 as double)])", "[[3.0], [1.0, 2.0]]", "DOUBLE NOT NULL ARRAY ARRAY NOT NULL"); + // Test case for [CALCITE-7704] + // ARRAY_INSERT crashes in code generation with array-of-arrays argument + f1.checkScalar("array_insert(array[array[cast(1 as double)]], 1, array[2])", + "[[2.0], [1.0]]", "DOUBLE NOT NULL ARRAY ARRAY NOT NULL"); + f1.checkScalar("array_insert(array[array[1]], 1, array[2.5])", + "[[2.5], [1.0]]", "DECIMAL(11, 1) NOT NULL ARRAY ARRAY NOT NULL"); f1.checkScalar("array_insert(array[1, 2, 3], 3, 4)", "[1, 2, 4, 3]", "INTEGER ARRAY NOT NULL"); From 1164a231b446c7491508a8c0e6ef890de722b800 Mon Sep 17 00:00:00 2001 From: Stamatis Zampetakis Date: Mon, 22 Jun 2026 17:59:47 +0200 Subject: [PATCH 468/562] [CALCITE-7617] Improve type safety of RelJson API using generics --- .../calcite/rel/externalize/RelJson.java | 20 ++++++------ .../rel/externalize/RelJsonReader.java | 4 +-- .../calcite/InvalidStaticInitializer.java | 32 +++++++++++++++++++ .../calcite/plan/RelOptPlanReaderTest.java | 26 +++++++++++---- 4 files changed, 64 insertions(+), 18 deletions(-) create mode 100644 core/src/test/java/org/apache/calcite/InvalidStaticInitializer.java diff --git a/core/src/main/java/org/apache/calcite/rel/externalize/RelJson.java b/core/src/main/java/org/apache/calcite/rel/externalize/RelJson.java index 2409ac6090e2..51c178db063d 100644 --- a/core/src/main/java/org/apache/calcite/rel/externalize/RelJson.java +++ b/core/src/main/java/org/apache/calcite/rel/externalize/RelJson.java @@ -120,7 +120,7 @@ public class RelJson { ImmutableList.of(NlsString.class, BigDecimal.class, ByteString.class, Boolean.class, TimestampString.class, DateString.class, TimeString.class); - private final Map constructorMap = new HashMap<>(); + private final Map> constructorMap = new HashMap<>(); private final @Nullable JsonBuilder jsonBuilder; private final InputTranslator inputTranslator; private final SqlOperatorTable operatorTable; @@ -209,9 +209,9 @@ private static > T enumVal(Class clazz, Map public RelNode create(Map map) { String type = get(map, "type"); - Constructor constructor = getConstructor(type); + Constructor constructor = getConstructor(type); try { - return (RelNode) constructor.newInstance(map); + return constructor.newInstance(map); } catch (InstantiationException | ClassCastException | InvocationTargetException | IllegalAccessException e) { throw new RuntimeException( @@ -219,12 +219,11 @@ public RelNode create(Map map) { } } - public Constructor getConstructor(String type) { - Constructor constructor = constructorMap.get(type); + public Constructor getConstructor(String type) { + Constructor constructor = constructorMap.get(type); if (constructor == null) { - Class clazz = typeNameToClass(type); + Class clazz = typeNameToClass(type); try { - //noinspection unchecked constructor = clazz.getConstructor(RelInput.class); } catch (NoSuchMethodException e) { throw new RuntimeException("class does not have required constructor, " @@ -239,18 +238,19 @@ public Constructor getConstructor(String type) { * Converts a type name to a class. E.g. {@code getClass("LogicalProject")} * returns {@link org.apache.calcite.rel.logical.LogicalProject}.class. */ - public Class typeNameToClass(String type) { + public Class typeNameToClass(String type) { if (!type.contains(".")) { for (String package_ : PACKAGES) { try { - return Class.forName(package_ + type); + return Class.forName(package_ + type, false, RelJson.class.getClassLoader()) + .asSubclass(RelNode.class); } catch (ClassNotFoundException e) { // ignore } } } try { - return Class.forName(type); + return Class.forName(type, false, RelJson.class.getClassLoader()).asSubclass(RelNode.class); } catch (ClassNotFoundException e) { throw new RuntimeException("unknown type " + type); } diff --git a/core/src/main/java/org/apache/calcite/rel/externalize/RelJsonReader.java b/core/src/main/java/org/apache/calcite/rel/externalize/RelJsonReader.java index 61a86d9b5fb4..2cdd785fe835 100644 --- a/core/src/main/java/org/apache/calcite/rel/externalize/RelJsonReader.java +++ b/core/src/main/java/org/apache/calcite/rel/externalize/RelJsonReader.java @@ -131,7 +131,7 @@ private void readRels(List> jsonRels) { private void readRel(final Map jsonRel) { String id = (String) requireNonNull(jsonRel.get("id"), "jsonRel.id"); String type = (String) requireNonNull(jsonRel.get("relOp"), "jsonRel.relOp"); - Constructor constructor = relJson.getConstructor(type); + Constructor constructor = relJson.getConstructor(type); RelInput input = new RelInput() { @Override public RelOptCluster getCluster() { return cluster; @@ -309,7 +309,7 @@ public ImmutableList getTuple(List jsonTuple) { } }; try { - final RelNode rel = (RelNode) constructor.newInstance(input); + final RelNode rel = constructor.newInstance(input); relMap.put(id, rel); lastRel = rel; } catch (InstantiationException | IllegalAccessException e) { diff --git a/core/src/test/java/org/apache/calcite/InvalidStaticInitializer.java b/core/src/test/java/org/apache/calcite/InvalidStaticInitializer.java new file mode 100644 index 000000000000..448c766a885c --- /dev/null +++ b/core/src/test/java/org/apache/calcite/InvalidStaticInitializer.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite; + +/** + * An invalid class that always fail if initialized. The class may be sub-classed to cover + * test cases where a static initializer is not allowed to be triggered. All the classes in the + * hierarchy are called by reflection thus appear as unused. + */ +@SuppressWarnings("unused") +public class InvalidStaticInitializer { + static { + throwError(); + } + private static void throwError() { + throw new AssertionError("Static initializer must not be triggered"); + } +} diff --git a/core/src/test/java/org/apache/calcite/plan/RelOptPlanReaderTest.java b/core/src/test/java/org/apache/calcite/plan/RelOptPlanReaderTest.java index c274e3d17786..cb38e64106aa 100644 --- a/core/src/test/java/org/apache/calcite/plan/RelOptPlanReaderTest.java +++ b/core/src/test/java/org/apache/calcite/plan/RelOptPlanReaderTest.java @@ -27,6 +27,7 @@ import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.sameInstance; import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.fail; /** @@ -40,22 +41,22 @@ class RelOptPlanReaderTest { assertThat(relJson.classToTypeName(LogicalProject.class), is("LogicalProject")); assertThat(relJson.typeNameToClass("LogicalProject"), - sameInstance((Class) LogicalProject.class)); + sameInstance(LogicalProject.class)); // in org.apache.calcite.adapter.jdbc.JdbcRules outer class assertThat(relJson.classToTypeName(JdbcRules.JdbcProject.class), is("JdbcProject")); assertThat(relJson.typeNameToClass("JdbcProject"), - equalTo((Class) JdbcRules.JdbcProject.class)); + equalTo(JdbcRules.JdbcProject.class)); try { - Class clazz = relJson.typeNameToClass("NonExistentRel"); + Class clazz = relJson.typeNameToClass("NonExistentRel"); fail("expected exception, got " + clazz); } catch (RuntimeException e) { assertThat(e.getMessage(), is("unknown type NonExistentRel")); } try { - Class clazz = + Class clazz = relJson.typeNameToClass("org.apache.calcite.rel.NonExistentRel"); fail("expected exception, got " + clazz); } catch (RuntimeException e) { @@ -67,11 +68,11 @@ class RelOptPlanReaderTest { assertThat(relJson.classToTypeName(MyRel.class), is("org.apache.calcite.plan.RelOptPlanReaderTest$MyRel")); assertThat(relJson.typeNameToClass(MyRel.class.getName()), - equalTo((Class) MyRel.class)); + equalTo(MyRel.class)); // Using canonical name (with '$'), not found try { - Class clazz = + Class clazz = relJson.typeNameToClass(MyRel.class.getCanonicalName()); fail("expected exception, got " + clazz); } catch (RuntimeException e) { @@ -81,6 +82,19 @@ class RelOptPlanReaderTest { } } + /** + * Tests loading of a class not implementing the {@code RelNode} interface + * throws an informative {@code ClassCastException}. Additionally, the test + * ensures that the type conversion does not trigger class initialization. + */ + @Test void testTypeNameToClassWithNoRelNodeClass() { + RelJson relJson = RelJson.create(); + ClassCastException e = + assertThrows(ClassCastException.class, + () -> relJson.typeNameToClass("org.apache.calcite.InvalidStaticInitializer")); + assertThat(e.getMessage(), is("class org.apache.calcite.InvalidStaticInitializer")); + } + /** Dummy relational expression. */ static class MyRel extends AbstractRelNode { MyRel(RelOptCluster cluster, RelTraitSet traitSet) { From e5c1aa1f0f511db55857d6e23a8d38ef2a7f2bcf Mon Sep 17 00:00:00 2001 From: Stamatis Zampetakis Date: Wed, 24 Jun 2026 19:53:28 +0200 Subject: [PATCH 469/562] [CALCITE-7713] Add allowlist option in ClassNameFilter --- .github/workflows/main.yml | 8 ++ build.gradle.kts | 1 + .../calcite/config/CalciteSystemProperty.java | 19 ++++ .../apache/calcite/model/ClassNameFilter.java | 89 ++++++++----------- .../calcite/model/ClassNameFilterTest.java | 89 +++++++++++++++++++ .../calcite/model/ModelHandlerTest.java | 25 ++---- site/_docs/history.md | 5 ++ sqlsh | 3 +- sqlsh.bat | 4 +- 9 files changed, 170 insertions(+), 73 deletions(-) create mode 100644 core/src/test/java/org/apache/calcite/model/ClassNameFilterTest.java diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 165f2752ab25..df77ec38eefb 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -71,6 +71,8 @@ jobs: remote-build-cache-proxy-enabled: false arguments: --scan --no-parallel --no-daemon build javadoc - name: 'sqlline and sqllsh' + env: + JAVA_OPTS: "-Dcalcite.model.classes.allowed=org.apache." shell: cmd run: | call sqlline.bat -e '!quit' @@ -101,6 +103,8 @@ jobs: remote-build-cache-proxy-enabled: false arguments: --scan --no-parallel --no-daemon build - name: 'sqlline and sqllsh' + env: + JAVA_OPTS: "-Dcalcite.model.classes.allowed=org.apache." shell: cmd run: | call sqlline.bat -e '!quit' @@ -131,6 +135,8 @@ jobs: remote-build-cache-proxy-enabled: false arguments: --scan --no-parallel --no-daemon build - name: 'sqlline and sqllsh' + env: + JAVA_OPTS: "-Dcalcite.model.classes.allowed=org.apache." shell: cmd run: | call sqlline.bat -e '!quit' @@ -324,6 +330,8 @@ jobs: remote-build-cache-proxy-enabled: false arguments: --scan --no-parallel --no-daemon build javadoc - name: 'sqlline and sqllsh' + env: + JAVA_OPTS: "-Dcalcite.model.classes.allowed=org.apache." run: | ./sqlline -e '!quit' echo diff --git a/build.gradle.kts b/build.gradle.kts index 44e00c7fa39a..4a4561672034 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -915,6 +915,7 @@ allprojects { passProperty("user.timezone", "UTC") passProperty("calcite.avatica.version", props.string("calcite.avatica.version")) passProperty("gradle.rootDir", rootDir.toString()) + systemProperty("calcite.model.classes.allowed", "org.,java.lang.") val props = System.getProperties() for (e in props.propertyNames() as `java.util`.Enumeration) { if (e.startsWith("calcite.") || e.startsWith("avatica.")) { diff --git a/core/src/main/java/org/apache/calcite/config/CalciteSystemProperty.java b/core/src/main/java/org/apache/calcite/config/CalciteSystemProperty.java index b0efb1a05d4d..39ba5c2f3281 100644 --- a/core/src/main/java/org/apache/calcite/config/CalciteSystemProperty.java +++ b/core/src/main/java/org/apache/calcite/config/CalciteSystemProperty.java @@ -455,6 +455,25 @@ public final class CalciteSystemProperty { public static final CalciteSystemProperty JOIN_SELECTOR_COMPACT_CODE_THRESHOLD = intProperty("calcite.join.selector.compact.code.threshold", 100); + /** + * Comma-separated allowlist of class-name patterns that may be loaded + * by reflection from a Calcite model (user-defined functions, custom + * schemas/tables, JDBC drivers, dialect factories, lattice statistic + * providers). + * + *

      By default, the allowlist is empty and class loading is fully disabled. + * When non-empty, a class name must match the allowlist in addition to + * clearing the denylist. + * + *

      Pattern syntax: a pattern ending in {@code "."} matches any class + * in that package or its sub-packages; otherwise the pattern matches a + * class name exactly. + * + * @see org.apache.calcite.model.ModelHandler + */ + public static final CalciteSystemProperty MODEL_CLASSES_ALLOWED = + stringProperty("calcite.model.classes.allowed", ""); + /** * Comma-separated patterns to add to the built-in denylist of class * names that may not be loaded by reflection from a Calcite model diff --git a/core/src/main/java/org/apache/calcite/model/ClassNameFilter.java b/core/src/main/java/org/apache/calcite/model/ClassNameFilter.java index 5a0a948cdcf4..0c37c2c5cbea 100644 --- a/core/src/main/java/org/apache/calcite/model/ClassNameFilter.java +++ b/core/src/main/java/org/apache/calcite/model/ClassNameFilter.java @@ -20,34 +20,25 @@ import com.google.common.collect.ImmutableList; +import org.apiguardian.api.API; import org.checkerframework.checker.nullness.qual.Nullable; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -import java.util.function.Predicate; /** * Filters class names that may be loaded by reflection from a Calcite * model: user-defined functions, custom schemas, custom tables, JDBC * drivers, dialect factories, and lattice statistic providers. * - *

      {@link #standard()} returns the filter applied by - * {@link ModelHandler}: the built-in {@link #DEFAULT_DENYLIST} together - * with any patterns from - * {@link CalciteSystemProperty#MODEL_CLASSES_DENIED} (which - * extends the denylist). - * - *

      The denylist is a comma-separated pattern string. A pattern ending - * in {@code "."} matches any class in that package or its sub-packages; + *

      The behavior of the filter is determined by the allowlist and the denylist. + * Both lists are comma separated patterns determining a package or class name. + * A pattern ending in {@code "."} matches any class in that package or its sub-packages; * otherwise the pattern matches a class name exactly. Whitespace around * commas is ignored. - * - *

      The denylist is not a sandbox. Any string passed to a - * {@code className}, {@code factory}, {@code jdbcDriver}, - * {@code sqlDialectFactory}, or {@code statisticProvider} field is - * classpath-equivalent; only accept models from trusted sources. */ -class ClassNameFilter implements Predicate { +@API(since = "1.43.0", status = API.Status.EXPERIMENTAL) +public final class ClassNameFilter { /** Built-in denylist: class-name patterns known to enable RCE when * registered as UDFs, schema/table factories, JDBC drivers, dialect * factories, or lattice statistic providers. */ @@ -75,74 +66,68 @@ class ClassNameFilter implements Predicate { + "jdk.internal."; /** Cache shared by all factory calls; filters are immutable and small, - * so identical denylist inputs need only be parsed once. */ + * so identical (denylist, allowlist) inputs need only be parsed once. */ private static final ConcurrentMap CACHE = new ConcurrentHashMap<>(); /** The standard filter, built once from the built-in denylist plus - * the {@link CalciteSystemProperty#MODEL_CLASSES_DENIED} extension. - * Initialized via {@link #of} so it shares the same cache. */ + * the {@link CalciteSystemProperty} pair. Initialized via {@link #of} + * so it shares the same cache. */ private static final ClassNameFilter STANDARD = of( append(DEFAULT_DENYLIST, - CalciteSystemProperty.MODEL_CLASSES_DENIED.value())); + CalciteSystemProperty.MODEL_CLASSES_DENIED.value()), + CalciteSystemProperty.MODEL_CLASSES_ALLOWED.value()); private final ImmutableList denylist; + private final ImmutableList allowlist; - private ClassNameFilter(String denylist) { + private ClassNameFilter(String denylist, String allowlist) { this.denylist = parse(denylist); + this.allowlist = parse(allowlist); } - /** Returns the standard filter used by {@link ModelHandler}: the + /** Returns the standard filter used by {@link ModelHandler}. The * built-in {@link #DEFAULT_DENYLIST} (extended by - * {@link CalciteSystemProperty#MODEL_CLASSES_DENIED}). */ + * {@link CalciteSystemProperty#MODEL_CLASSES_DENIED}) plus the + * allowlist from + * {@link CalciteSystemProperty#MODEL_CLASSES_ALLOWED}. */ static ClassNameFilter standard() { return STANDARD; } - /** Returns a filter parsed from a comma-separated denylist pattern - * string; may be empty. Filters are cached, so repeated calls with - * the same argument return the same instance. */ - static ClassNameFilter of(String denylist) { - return CACHE.computeIfAbsent(denylist, ClassNameFilter::new); + /** Returns a filter parsed from comma-separated {@code denylist} and + * {@code allowlist} pattern strings; either may be empty. Filters are + * cached, so repeated calls with the same arguments return the same + * instance. */ + public static ClassNameFilter of(String denylist, String allowlist) { + // NUL is forbidden in JVM class names, so concatenating with NUL is + // an injection-proof cache key. + String key = denylist + '\0' + allowlist; + return CACHE.computeIfAbsent(key, + k -> new ClassNameFilter(denylist, allowlist)); } - /** Returns whether {@code classRef} is allowed (not on the denylist). - * A null reference is allowed. - * - *

      {@code classRef} may be a plain class name or the - * {@code "ClassName#STATIC_FIELD"} form accepted by - * {@link org.apache.calcite.avatica.AvaticaUtils#instantiatePlugin}; - * the field portion is stripped before matching. */ - @Override public boolean test(@Nullable String classRef) { + /** Throws {@link SecurityException} if {@code classRef} is not allowed + * by this filter. A null reference is a no-op. */ + void check(@Nullable String classRef) { if (classRef == null) { - return true; + return; } String className = stripFieldRef(classRef); for (String pattern : denylist) { if (matches(pattern, className)) { - return false; + throw new SecurityException( + "Class '" + className + "' rejected by the denylist (pattern '" + pattern + "')."); } } - return true; - } - /** Throws {@link SecurityException} if {@code classRef} is on the - * denylist. A null reference is a no-op. */ - void check(@Nullable String classRef) { - if (classRef == null) { - return; - } - String className = stripFieldRef(classRef); - for (String pattern : denylist) { + for (String pattern : allowlist) { if (matches(pattern, className)) { - throw new SecurityException("Class '" + className - + "' is rejected by the Calcite class-name filter " - + "(matches denylist pattern '" + pattern + "'). " - + "If this load is unintended, adjust the model; the " - + "denylist cannot be loosened at runtime."); + return; } } + throw new SecurityException("Class '" + className + "' rejected by the allowlist."); } private static String stripFieldRef(String classRef) { diff --git a/core/src/test/java/org/apache/calcite/model/ClassNameFilterTest.java b/core/src/test/java/org/apache/calcite/model/ClassNameFilterTest.java new file mode 100644 index 000000000000..be459d961112 --- /dev/null +++ b/core/src/test/java/org/apache/calcite/model/ClassNameFilterTest.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.model; + +import org.junit.jupiter.api.Test; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Unit tests for {@link ClassNameFilter}. + */ +public class ClassNameFilterTest { + @Test void testDefaultProdValuesForDenyAllowList() { + // This represents the default prod configuration of the project + // where the system properties both default to empty and + // basically every class is rejected mainly due to the empty allowlist + ClassNameFilter cf = ClassNameFilter.of("", ""); + assertThrows(SecurityException.class, () -> cf.check("java.lang.String")); + assertThrows(SecurityException.class, + () -> cf.check("org.apache.calcite.adapter.jdbc.JdbcSchema$Factory")); + } + + @Test void testAllowListWithSinglePackagePattern() { + ClassNameFilter cf = ClassNameFilter.of("", "org."); + assertThrows(SecurityException.class, () -> cf.check("java.lang.String")); + assertDoesNotThrow(() -> cf.check("org.apache.calcite.adapter.jdbc.JdbcSchema$Factory")); + } + + @Test void testAllowListWithSinglePackageNoPattern() { + ClassNameFilter cf = ClassNameFilter.of("", "org"); + assertThrows(SecurityException.class, () -> cf.check("java.lang.String")); + assertThrows(SecurityException.class, + () -> cf.check("org.apache.calcite.adapter.jdbc.JdbcSchema$Factory")); + } + + @Test void testAllowListWithMultiplePackagePatterns() { + ClassNameFilter cf = ClassNameFilter.of("", "org.,java."); + assertDoesNotThrow(() -> cf.check("java.lang.String")); + assertDoesNotThrow(() -> cf.check("org.apache.calcite.adapter.jdbc.JdbcSchema$Factory")); + assertThrows(SecurityException.class, () -> cf.check("com.sun.media.sound.Toolkit")); + } + + @Test void testAllowListWithSingleClass() { + ClassNameFilter cf = ClassNameFilter.of("", "java.lang.String"); + assertDoesNotThrow(() -> cf.check("java.lang.String")); + assertThrows(SecurityException.class, + () -> cf.check("org.apache.calcite.adapter.jdbc.JdbcSchema$Factory")); + assertThrows(SecurityException.class, () -> cf.check("java.lang.Math")); + } + + @Test void testAllowListWithMultipleClasses() { + ClassNameFilter cf = ClassNameFilter.of("", "java.lang.String,java.lang.Math"); + assertDoesNotThrow(() -> cf.check("java.lang.String")); + assertThrows(SecurityException.class, () -> cf.check("java.lang.StringBuffer")); + assertThrows(SecurityException.class, + () -> cf.check("org.apache.calcite.adapter.jdbc.JdbcSchema$Factory")); + assertDoesNotThrow(() -> cf.check("java.lang.Math")); + } + + @Test void testStandardFilter() { + // Note that test specific system properties are in effect + ClassNameFilter cf = ClassNameFilter.standard(); + assertDoesNotThrow(() -> cf.check("java.lang.String")); + assertDoesNotThrow(() -> cf.check("org.apache.calcite.adapter.jdbc.JdbcSchema$Factory")); + SecurityException x1 = + assertThrows(SecurityException.class, () -> cf.check("com.sun.media.sound.Toolkit")); + assertThat(x1.getMessage(), containsString("rejected by the allowlist")); + SecurityException x2 = + assertThrows(SecurityException.class, () -> cf.check("javax.naming.InitialContext")); + assertThat(x2.getMessage(), containsString("rejected by the denylist")); + } +} diff --git a/core/src/test/java/org/apache/calcite/model/ModelHandlerTest.java b/core/src/test/java/org/apache/calcite/model/ModelHandlerTest.java index 4cebda1454d4..438ec79c3c66 100644 --- a/core/src/test/java/org/apache/calcite/model/ModelHandlerTest.java +++ b/core/src/test/java/org/apache/calcite/model/ModelHandlerTest.java @@ -31,7 +31,6 @@ import java.sql.DriverManager; import java.util.Properties; import java.util.Set; -import java.util.function.Predicate; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.is; @@ -39,6 +38,7 @@ import static org.hamcrest.CoreMatchers.notNullValue; import static org.hamcrest.CoreMatchers.sameInstance; import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertThrows; import static java.util.Objects.requireNonNull; @@ -82,7 +82,7 @@ public class ModelHandlerTest { SchemaPlus root = CalciteSchema.createRootSchema(false, false).plus(); // java.lang.String is not in the standard denylist; the custom // filter denies the whole java.lang. package. - ClassNameFilter strict = ClassNameFilter.of("java.lang."); + ClassNameFilter strict = ClassNameFilter.of("java.lang.", "java."); String model = "inline:{" + " version: '1.0'," + " defaultSchema: 'X'," @@ -164,31 +164,20 @@ public class ModelHandlerTest { "org.apache.calcite.adapter.jdbc.JdbcSchema$Factory#INSTANCE"); } - @Test void testPredicateContract() { - // ClassNameFilter implements Predicate: true means "allowed". - Predicate filter = ClassNameFilter.standard(); - assertThat(filter.test(null), is(true)); - assertThat(filter.test("javax.naming.InitialContext"), is(false)); - assertThat(filter.test("java.lang.Runtime#getRuntime"), is(false)); - assertThat( - filter.test( - "org.apache.calcite.adapter.jdbc.JdbcSchema$Factory"), is(true)); - } - @Test void testFactoryMethodsCacheInstances() { // standard() returns a single cached instance. assertThat(ClassNameFilter.standard(), sameInstance(ClassNameFilter.standard())); // of() returns the same instance for equal inputs. - ClassNameFilter a = ClassNameFilter.of("com.evil."); - ClassNameFilter b = ClassNameFilter.of("com.evil."); + ClassNameFilter a = ClassNameFilter.of("com.evil.", "javax."); + ClassNameFilter b = ClassNameFilter.of("com.evil.", "javax."); assertThat(a, sameInstance(b)); // Different inputs produce different instances. - ClassNameFilter c = ClassNameFilter.of("com.evil.,com.example."); + ClassNameFilter c = ClassNameFilter.of("com.evil.,com.example.", "javax."); assertThat(a, not(sameInstance(c))); // The cached filter behaves as configured. - assertThat(a.test("com.evil.Payload"), is(false)); - assertThat(a.test("javax.naming.InitialContext"), is(true)); + assertThrows(SecurityException.class, () -> a.check("com.evil.Payload")); + assertDoesNotThrow(() -> a.check("javax.naming.InitialContext")); } @Test void testAppendCombinesPatternStrings() { diff --git a/site/_docs/history.md b/site/_docs/history.md index c72d6f78d938..9520e7d05f39 100644 --- a/site/_docs/history.md +++ b/site/_docs/history.md @@ -54,6 +54,11 @@ other software versions as specified in gradle.properties. filter evaluation now run in Java, and the `arrow-gandiva` dependency is no longer included in the Arrow module or BOM. +* [CALCITE-7713] +Class loading from model files has been disabled by default. Any attempt to load +classes from model files will lead to `SecurityException` unless an appropriate +pattern is set in `calcite.model.classes.allowed` system property. + #### New features {: #new-features-1-43-0} diff --git a/sqlsh b/sqlsh index 437060a249f6..18c9d305a39d 100755 --- a/sqlsh +++ b/sqlsh @@ -43,6 +43,7 @@ if [ "x$CACHE_SQLLINE_CLASSPATH" != "xY" ] || [ ! -f "$CP" ]; then fi VM_OPTS= -JAVA_OPTS="-Djavax.xml.parsers.DocumentBuilderFactory=com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl ${JAVA_OPTS}" +# Running sqlsh is explicitly for OS adapter, so we can set the allowed classes to avoid security exception when loading the model file. +JAVA_OPTS="-Djavax.xml.parsers.DocumentBuilderFactory=com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl -Dcalcite.model.classes.allowed=org.apache.calcite.adapter.os. ${JAVA_OPTS}" exec java $VM_OPTS -cp "${CP}" $JAVA_OPTS org.apache.calcite.adapter.os.SqlShell "$@" diff --git a/sqlsh.bat b/sqlsh.bat index 2ad45c54392b..ec4bed181684 100644 --- a/sqlsh.bat +++ b/sqlsh.bat @@ -31,7 +31,7 @@ if not defined CACHE_SQLLINE_CLASSPATH ( if exist "%CP%" del "%CP%" ) if not exist "%CP%" (call "%DIRNAME%\gradlew" --console plain -q :buildSqllineClasspath) - -set JAVA_OPTS=-Djavax.xml.parsers.DocumentBuilderFactory=com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl %JAVA_OPTS% +:: Running sqlsh is explicitly for OS adapter, so we can set the allowed classes to avoid security exception when loading the model file. +set JAVA_OPTS=-Djavax.xml.parsers.DocumentBuilderFactory=com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl -Dcalcite.model.classes.allowed=org.apache.calcite.adapter.os. %JAVA_OPTS% java -Xmx1g -cp "%CP%" %JAVA_OPTS% org.apache.calcite.adapter.os.SqlShell %* From e39c536e3d4857c2f3a2704e4da50218ee051de0 Mon Sep 17 00:00:00 2001 From: zzwqqq Date: Thu, 13 Aug 2026 10:24:57 +0800 Subject: [PATCH 470/562] [CALCITE-7711] Add a rule to convert LEFT or RIGHT OUTER JOIN with IS NULL to ANTI JOIN --- .../apache/calcite/rel/rules/CoreRules.java | 5 + .../rel/rules/OuterJoinToAntiJoinRule.java | 222 ++++++++++++++++ .../test/OuterJoinToAntiJoinRuleTest.java | 129 +++++++++ .../test/OuterJoinToAntiJoinRuleTest.xml | 247 ++++++++++++++++++ core/src/test/resources/sql/planner.iq | 43 +++ 5 files changed, 646 insertions(+) create mode 100644 core/src/main/java/org/apache/calcite/rel/rules/OuterJoinToAntiJoinRule.java create mode 100644 core/src/test/java/org/apache/calcite/test/OuterJoinToAntiJoinRuleTest.java create mode 100644 core/src/test/resources/org/apache/calcite/test/OuterJoinToAntiJoinRuleTest.xml diff --git a/core/src/main/java/org/apache/calcite/rel/rules/CoreRules.java b/core/src/main/java/org/apache/calcite/rel/rules/CoreRules.java index f827843cc64f..53fe7c71fd21 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/CoreRules.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/CoreRules.java @@ -753,6 +753,11 @@ private CoreRules() {} public static final SemiJoinRule.JoinToSemiJoinRule JOIN_TO_SEMI_JOIN = SemiJoinRule.JoinToSemiJoinRule.JoinToSemiJoinRuleConfig.DEFAULT.toRule(); + /** Rule that converts an outer join followed by {@code IS NULL} on its + * null-generating side to an anti join. */ + public static final OuterJoinToAntiJoinRule OUTER_JOIN_TO_ANTI_JOIN = + OuterJoinToAntiJoinRule.Config.DEFAULT.toRule(); + /** Rule that pushes a {@link Join} * past a non-distinct {@link Union} as its left input. */ public static final JoinUnionTransposeRule JOIN_LEFT_UNION_TRANSPOSE = diff --git a/core/src/main/java/org/apache/calcite/rel/rules/OuterJoinToAntiJoinRule.java b/core/src/main/java/org/apache/calcite/rel/rules/OuterJoinToAntiJoinRule.java new file mode 100644 index 000000000000..2b005633704d --- /dev/null +++ b/core/src/main/java/org/apache/calcite/rel/rules/OuterJoinToAntiJoinRule.java @@ -0,0 +1,222 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.rel.rules; + +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptUtil; +import org.apache.calcite.plan.RelRule; +import org.apache.calcite.plan.Strong; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.Filter; +import org.apache.calcite.rel.core.Join; +import org.apache.calcite.rel.core.JoinRelType; +import org.apache.calcite.rel.logical.LogicalFilter; +import org.apache.calcite.rel.logical.LogicalJoin; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexUtil; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.tools.RelBuilder; +import org.apache.calcite.util.ImmutableBitSet; + +import org.checkerframework.checker.nullness.qual.Nullable; +import org.immutables.value.Value; + +import java.util.ArrayList; +import java.util.List; + +/** + * Planner rule that converts an outer join followed by {@code IS NULL} + * on its null-generating side to an anti join. + * + *

      For example, the query + * + * 

      {@code
+ * SELECT e.empno, d.name
+ * FROM Emp AS e
+ * LEFT JOIN Dept AS d ON e.deptno = d.deptno
+ * WHERE d.deptno IS NULL AND e.empno > 10
+ * }
      + * + *

      has the following plan: + * + * 

      {@code
+ * LogicalProject(EMPNO=[$0], NAME=[$10])
+ *   LogicalFilter(condition=[AND(IS NULL($9), >($0, 10))])
+ *     LogicalJoin(condition=[=($7, $9)], joinType=[left])
+ *       LogicalTableScan(table=[[CATALOG, SALES, EMP]])
+ *       LogicalTableScan(table=[[CATALOG, SALES, DEPT]])
+ * }
      + * + *

      The rule converts it to: + * + * 

      {@code
+ * LogicalProject(EMPNO=[$0], NAME=[$10])
+ *   LogicalFilter(condition=[>($0, 10)])
+ *     LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3],
+ *         HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7],
+ *         SLACKER=[$8], DEPTNO0=[null:INTEGER], NAME=[null:VARCHAR(10)])
+ *       LogicalJoin(condition=[=($7, $9)], joinType=[anti])
+ *         LogicalTableScan(table=[[CATALOG, SALES, EMP]])
+ *         LogicalTableScan(table=[[CATALOG, SALES, DEPT]])
+ * }
      + * + *

      The {@code IS NULL} predicate must be a top-level conjunct over a field + * from the null-generating input. A field that is non-nullable in that input + * is safe. For a nullable field, the join condition must not be TRUE when its + * value is NULL. + */ +@Value.Enclosing +public class OuterJoinToAntiJoinRule + extends RelRule + implements TransformationRule { + + /** Creates an OuterJoinToAntiJoinRule. */ + protected OuterJoinToAntiJoinRule(Config config) { + super(config); + } + + @Override public void onMatch(RelOptRuleCall call) { + final Filter filter = call.rel(0); + final Join join = call.rel(1); + + // Field indexes below assume that the join has no system-field prefix. + if (!join.getSystemFieldList().isEmpty()) { + return; + } + // Rewriting may change the number and order of condition evaluations. + if (!RexUtil.isDeterministic(filter.getCondition()) + || !RexUtil.isDeterministic(join.getCondition())) { + return; + } + + final boolean leftJoin = join.getJoinType() == JoinRelType.LEFT; + // Correlated RIGHT joins are not supported because converting them requires + // swapping the inputs and remapping correlation references. + if (!leftJoin && !join.getVariablesSet().isEmpty()) { + return; + } + // Only top-level conjuncts can independently prove that a row is unmatched. + final List remainingConditions = + new ArrayList<>(RelOptUtil.conjunctions(filter.getCondition())); + final RexNode nullCondition = + findSafeNullCondition(remainingConditions, join, leftJoin); + if (nullCondition == null) { + return; + } + remainingConditions.remove(nullCondition); + + final RelNode newLeft = leftJoin ? join.getLeft() : join.getRight(); + final RelNode newRight = leftJoin ? join.getRight() : join.getLeft(); + final RexNode condition = leftJoin + ? join.getCondition() + : JoinCommuteRule.swapJoinCond(join.getCondition(), join, + join.getCluster().getRexBuilder()); + final RelBuilder builder = call.builder() + .push(newLeft) + .push(newRight) + .join(JoinRelType.ANTI, condition, join.getVariablesSet()) + .hints(join.getHints()); + + // An anti join projects only its left input. Its rows are unmatched, so every + // field of the null-generating input is NULL. Reinsert typed NULLs to restore + // the outer join's row type. + final int leftCount = join.getLeft().getRowType().getFieldCount(); + final int nullOffset = leftJoin ? leftCount : 0; + final List projects = new ArrayList<>(builder.fields()); + insertNulls(projects, join.getRowType(), nullOffset, + newRight.getRowType().getFieldCount(), builder); + + builder.project(projects, join.getRowType().getFieldNames()) + .filter(filter.getVariablesSet(), remainingConditions) + .convert(filter.getRowType(), false); + call.transformTo(builder.build()); + } + + /** Returns an {@code IS NULL} condition on a null-generating input field + * that is non-nullable, or for which the join condition cannot be TRUE when + * the field is NULL; returns null if there is no such condition. */ + private static @Nullable RexNode findSafeNullCondition( + List conditions, Join join, boolean leftJoin) { + final int leftCount = join.getLeft().getRowType().getFieldCount(); + for (RexNode condition : conditions) { + if (!(condition instanceof RexCall) + || !condition.isA(SqlKind.IS_NULL)) { + continue; + } + final RexNode operand = ((RexCall) condition).getOperands().get(0); + if (!(operand instanceof RexInputRef)) { + continue; + } + final int index = ((RexInputRef) operand).getIndex(); + final boolean inputOnLeft = index < leftCount; + if (inputOnLeft == leftJoin) { + continue; + } + final int inputIndex = inputOnLeft ? index : index - leftCount; + final RelNode input = inputOnLeft ? join.getLeft() : join.getRight(); + final RelDataType type = input.getRowType() + .getFieldList().get(inputIndex).getType(); + // If the input field is nullable, IS NULL may also be true for a matched + // row. It proves the row is unmatched only if the field is non-nullable, + // or, for a nullable field, the join condition cannot be TRUE when the + // field is NULL. + if (!type.isNullable() + || Strong.isNotTrue(join.getCondition(), ImmutableBitSet.of(index))) { + return condition; + } + } + return null; + } + + /** Inserts typed NULL expressions for fields in the original row type. */ + private static void insertNulls(List projects, RelDataType rowType, + int offset, int count, RelBuilder builder) { + final List nulls = new ArrayList<>(count); + for (int i = 0; i < count; i++) { + final RelDataType type = + rowType.getFieldList().get(offset + i).getType(); + nulls.add(builder.getRexBuilder().makeNullLiteral(type)); + } + projects.addAll(offset, nulls); + } + + /** Rule configuration. */ + @Value.Immutable + public interface Config extends RelRule.Config { + Config DEFAULT = ImmutableOuterJoinToAntiJoinRule.Config.of() + .withOperandFor(LogicalFilter.class, LogicalJoin.class); + + @Override default OuterJoinToAntiJoinRule toRule() { + return new OuterJoinToAntiJoinRule(this); + } + + /** Defines an operand tree for the given classes. */ + default Config withOperandFor(Class filterClass, + Class joinClass) { + return withOperandSupplier(b -> + b.operand(filterClass).oneInput(b2 -> + b2.operand(joinClass) + .predicate(join -> join.getJoinType() == JoinRelType.LEFT + || join.getJoinType() == JoinRelType.RIGHT) + .anyInputs())) + .as(Config.class); + } + } +} diff --git a/core/src/test/java/org/apache/calcite/test/OuterJoinToAntiJoinRuleTest.java b/core/src/test/java/org/apache/calcite/test/OuterJoinToAntiJoinRuleTest.java new file mode 100644 index 000000000000..64db0403861d --- /dev/null +++ b/core/src/test/java/org/apache/calcite/test/OuterJoinToAntiJoinRuleTest.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.test; + +import org.apache.calcite.rel.rules.CoreRules; +import org.apache.calcite.rel.rules.OuterJoinToAntiJoinRule; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Test; + +/** + * Tests for {@link OuterJoinToAntiJoinRule}. + * + *

      [CALCITE-7711] + * Add a rule to convert LEFT or RIGHT OUTER JOIN with IS NULL to ANTI JOIN. + */ +class OuterJoinToAntiJoinRuleTest { + + private static RelOptFixture fixture() { + return RelOptFixture.DEFAULT.withDiffRepos( + DiffRepository.lookup(OuterJoinToAntiJoinRuleTest.class)); + } + + private static RelOptFixture sql(String sql) { + return fixture().sql(sql) + .withRule(CoreRules.OUTER_JOIN_TO_ANTI_JOIN); + } + + @Test void testLeftJoin() { + final String sql = "select e.empno, d.name\n" + + "from emp e left join dept d on e.deptno = d.deptno\n" + + "where d.deptno is null and e.empno > 10"; + sql(sql).check(); + } + + @Test void testNullableJoinKey() { + final String sql = "select e.empno\n" + + "from emp e left join deptnullables d on e.deptno = d.deptno\n" + + "where d.deptno is null"; + sql(sql).check(); + } + + @Test void testRightJoin() { + final String sql = "select e.ename, d.name\n" + + "from emp e right join dept d on e.deptno = d.deptno\n" + + "where e.empno is null"; + sql(sql).check(); + } + + @Test void testCorrelatedLeftJoin() { + final String sql = "select e.empno\n" + + "from emp e left join dept d\n" + + "on e.deptno = d.deptno and exists (\n" + + " select 1 from dept d2 where d2.name = d.name)\n" + + "where d.deptno is null"; + sql(sql).check(); + } + + @Test void testCorrelatedRightJoin() { + final String sql = "select d.deptno\n" + + "from emp e right join dept d\n" + + "on e.deptno = d.deptno and exists (\n" + + " select 1 from dept d2 where d2.name = d.name)\n" + + "where e.empno is null"; + sql(sql).checkUnchanged(); + } + + @Test void testNullableNonJoinColumn() { + final String sql = "select e.empno\n" + + "from emp e left join deptnullables d on e.deptno = d.deptno\n" + + "where d.name is null"; + sql(sql).checkUnchanged(); + } + + @Test void testIsNullOnPreservedInput() { + final String sql = "select e.empno\n" + + "from emp e left join dept d on e.deptno = d.deptno\n" + + "where e.comm is null"; + sql(sql).checkUnchanged(); + } + + @Test void testNullSafeJoinCondition() { + final String sql = "select e.empno\n" + + "from empnullables e left join deptnullables d\n" + + "on e.deptno is not distinct from d.deptno\n" + + "where d.deptno is null"; + sql(sql).checkUnchanged(); + } + + @Test void testIsNullInDisjunction() { + final String sql = "select e.empno\n" + + "from emp e left join dept d on e.deptno = d.deptno\n" + + "where d.deptno is null or e.empno > 10"; + sql(sql).checkUnchanged(); + } + + @Test void testNonDeterministicFilter() { + final String sql = "select e.empno\n" + + "from emp e left join dept d on e.deptno = d.deptno\n" + + "where d.deptno is null and rand() > 0.5"; + sql(sql).checkUnchanged(); + } + + @Test void testNonDeterministicJoinCondition() { + final String sql = "select e.empno\n" + + "from emp e left join dept d\n" + + "on e.deptno = d.deptno and rand() > 0.5\n" + + "where d.deptno is null"; + sql(sql).checkUnchanged(); + } + + @AfterAll static void checkActualAndReferenceFiles() { + fixture().diffRepos.checkActualAndReferenceFiles(); + } +} diff --git a/core/src/test/resources/org/apache/calcite/test/OuterJoinToAntiJoinRuleTest.xml b/core/src/test/resources/org/apache/calcite/test/OuterJoinToAntiJoinRuleTest.xml new file mode 100644 index 000000000000..44be8d1fa03b --- /dev/null +++ b/core/src/test/resources/org/apache/calcite/test/OuterJoinToAntiJoinRuleTest.xml @@ -0,0 +1,247 @@ + + + + + + + + + + + + + + + + + + + + + + + + + 10]]> + + + ($0, 10))]) + LogicalJoin(condition=[=($7, $9)], joinType=[left]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) + LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) +]]> + + + + + + + + + + + + + 10]]> + + + ($0, 10))]) + LogicalJoin(condition=[=($7, $9)], joinType=[left]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) + LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) +]]> + + + ($0, 10)]) + LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7], SLACKER=[$8], DEPTNO0=[null:INTEGER], NAME=[null:VARCHAR(10)]) + LogicalJoin(condition=[=($7, $9)], joinType=[anti]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) + LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) +]]> + + + + + 0.5]]> + + + (RAND(), CAST(0.5:DECIMAL(2, 1)):DOUBLE NOT NULL))]) + LogicalJoin(condition=[=($7, $9)], joinType=[left]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) + LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) +]]> + + + + + 0.5 +where d.deptno is null]]> + + + (RAND(), 0.5E0))], joinType=[left]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) + LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) +]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/core/src/test/resources/sql/planner.iq b/core/src/test/resources/sql/planner.iq index 0adbe5785b1e..c6f3c72623e8 100644 --- a/core/src/test/resources/sql/planner.iq +++ b/core/src/test/resources/sql/planner.iq @@ -685,4 +685,47 @@ EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[>($t2, $t4)], expr#6=[> !ok !set planner-rules original +# [CALCITE-7711] Add a rule to convert LEFT or RIGHT OUTER JOIN with IS NULL to ANTI JOIN +!set planner-rules " ++CoreRules.OUTER_JOIN_TO_ANTI_JOIN" +select l.id as left_id, r.id as right_id +from (values (1), (2), (3)) as l(id) +left join (values (1), (3)) as r(id) on l.id = r.id +where r.id is null +order by l.id; ++---------+----------+ +| LEFT_ID | RIGHT_ID | ++---------+----------+ +| 2 | | ++---------+----------+ +(1 row) + +!ok +EnumerableCalc(expr#0=[{inputs}], expr#1=[null:INTEGER], proj#0..1=[{exprs}]) + EnumerableMergeJoin(condition=[=($0, $1)], joinType=[anti]) + EnumerableValues(tuples=[[{ 1 }, { 2 }, { 3 }]]) + EnumerableValues(tuples=[[{ 1 }, { 3 }]]) +!plan + +# RIGHT JOIN keeps the non-commutative join condition after swapping inputs. +select l.id as left_id, r.id as right_id +from (values (1), (3), (5)) as l(id) +right join (values (0), (4), (6)) as r(id) on l.id > r.id +where l.id is null +order by r.id; ++---------+----------+ +| LEFT_ID | RIGHT_ID | ++---------+----------+ +| | 6 | ++---------+----------+ +(1 row) + +!ok +EnumerableCalc(expr#0=[{inputs}], expr#1=[null:INTEGER], ID=[$t1], ID0=[$t0]) + EnumerableNestedLoopJoin(condition=[>($1, $0)], joinType=[anti]) + EnumerableValues(tuples=[[{ 0 }, { 4 }, { 6 }]]) + EnumerableValues(tuples=[[{ 1 }, { 3 }, { 5 }]]) +!plan +!set planner-rules original + # End planner.iq From 5dbfc795298052a27b1b938f56248805be300e79 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Fri, 14 Aug 2026 10:54:57 -0700 Subject: [PATCH 471/562] [CALCITE-7720] Scalar subquery with ROW ARRAY fails in code generation Signed-off-by: Mihai Budiu --- .../enumerable/RexToLixTranslator.java | 15 +++- core/src/test/resources/sql/struct.iq | 83 +++++++++++++++++++ 2 files changed, 96 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java index cf9cbde5e378..f102fd69d79e 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java @@ -1552,13 +1552,24 @@ private static Expression scaleValue( return result; } + /** Returns the Java type of a local variable that holds a value of a given type. + * + *

      For the reasoning behind this implementation + * + * @see org.apache.calcite.jdbc.JavaTypeFactoryImpl.SyntheticRecordType + * @see JavaTypeFactory#getJavaClass(RelDataType) */ + private Type javaVariableType(RelDataType type) { + final Type javaType = typeFactory.getJavaClass(type); + return javaType instanceof Class ? javaType : Object.class; + } + /** * Returns an {@code Expression} for null literal without losing its type * information. */ private ConstantExpression getTypedNullLiteral(RexLiteral literal) { assert literal.isNull(); - Type javaClass = typeFactory.getJavaClass(literal.getType()); + Type javaClass = javaVariableType(literal.getType()); switch (literal.getType().getSqlTypeName()) { case DATE: case TIME: @@ -1674,7 +1685,7 @@ private Result implementPrev(RexCall call) { * } */ private Result implementCaseWhen(RexCall call) { - final Type returnType = typeFactory.getJavaClass(call.getType()); + final Type returnType = javaVariableType(call.getType()); final ParameterExpression valueVariable = Expressions.parameter(returnType, list.newName("case_when_value")); diff --git a/core/src/test/resources/sql/struct.iq b/core/src/test/resources/sql/struct.iq index bf25d4cadb28..08c876205066 100644 --- a/core/src/test/resources/sql/struct.iq +++ b/core/src/test/resources/sql/struct.iq @@ -337,4 +337,87 @@ select row(emp.*, dept.*).deptno0 from emp join dept on emp.deptno = dept.deptno !ok +# [CALCITE-7720] Scalar subquery with ROW ARRAY fails in code generation. + +# A scalar sub-query that returns an element of an array of ROW. +select (select t.arr[1] from (values (0))) as v +from (select ARRAY[ROW(1, 2)] as arr) as t; ++--------+ +| V | ++--------+ +| {1, 2} | ++--------+ +(1 row) + +!ok + +# The same element access, without a sub-query +select t.arr[1] as v from (select ARRAY[ROW(1, 2)] as arr) as t; ++--------+ +| V | ++--------+ +| {1, 2} | ++--------+ +(1 row) + +!ok + +# A CASE whose result is a ROW +select case when x = 1 then ROW(1, 2) else ROW(3, 4) end as v +from (values (1)) as t(x); ++--------+ +| V | ++--------+ +| {1, 2} | ++--------+ +(1 row) + +!ok + +# A CASE whose result is an element of an array of ROW +select case when x = 1 then arr[1] else arr[2] end as v +from (values (1)) as t(x), (select ARRAY[ROW(1, 2), ROW(3, 4)] as arr) as u; ++--------+ +| V | ++--------+ +| {1, 2} | ++--------+ +(1 row) + +!ok + +# A field access that returns a ROW +select t.arr[1]."EXPR$0" as v from (select ARRAY[ROW(ROW(1, 2), 3)] as arr) as t; ++--------+ +| V | ++--------+ +| {1, 2} | ++--------+ +(1 row) + +!ok + +# A field access on a ROW column that returns a ROW +select t.r."EXPR$0" as v from (select ROW(ROW(1, 2), 3) as r) as t; ++--------+ +| V | ++--------+ +| {1, 2} | ++--------+ +(1 row) + +!ok + +# A CASE that returns a ROW obtained by a field access +select case when x = 1 then t.arr[1]."EXPR$0" else ROW(9, 9) end as v +from (values (1)) as t2(x), (select ARRAY[ROW(ROW(1, 2), 3)] as arr) as t; ++--------+ +| V | ++--------+ +| {1, 2} | ++--------+ +(1 row) + +!ok + # End struct.iq From 064d66cb6de591f9ae8d859ec3f4d0da0bcac931 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Thu, 13 Aug 2026 21:58:48 -0700 Subject: [PATCH 472/562] [CALCITE-7719] Field access on an element of a ROW array built by a sub-query raises IllegalArgumentException Signed-off-by: Mihai Budiu --- .../org/apache/calcite/rex/LogicVisitor.java | 28 +++++++++++++++++-- core/src/test/resources/sql/sub-query.iq | 13 +++++++++ 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rex/LogicVisitor.java b/core/src/main/java/org/apache/calcite/rex/LogicVisitor.java index 68a0d0aa9638..b18613b104aa 100644 --- a/core/src/main/java/org/apache/calcite/rex/LogicVisitor.java +++ b/core/src/main/java/org/apache/calcite/rex/LogicVisitor.java @@ -31,13 +31,20 @@ import static java.util.Objects.requireNonNull; /** - * Visitor pattern for traversing a tree of {@link RexNode} objects. + * Visitor that, given the {@link Logic} in force at the root of an + * expression, computes the Logic in force at every occurrence of a sought + * sub-expression {@code seek}. Results are collected in {@code logicCollection}. + * + *

      This value is meaningful only for expressions that evaluate to Boolean values. */ public class LogicVisitor extends RexUnaryBiVisitor<@Nullable Logic> { private final RexNode seek; private final Collection logicCollection; - /** Creates a LogicVisitor. */ + /** Creates a LogicVisitor. + * + * @param seek Expression whose occurrences to find + * @param logicCollection Receives the Logic in force for each occurrence of {@code seek} */ private LogicVisitor(RexNode seek, Collection logicCollection) { super(true); this.seek = seek; @@ -51,6 +58,14 @@ private LogicVisitor(RexNode seek, Collection logicCollection) { * answer) with the fewest possibilities (that is, we prefer one that * returns [true as true, false as false, unknown as false] over one that * distinguishes false from unknown). + * + *

      If {@code seek} occurs multiple times, the result is + * a single Logic that is safe for every one of them. If the occurrences + * are evaluated under different Logic values, the result is + * {@link Logic#TRUE_FALSE_UNKNOWN}, which is safe for any occurrence. + * + * @throws IllegalArgumentException if {@code seek} does not occur in + * {@code nodes} */ public static Logic find(Logic logic, List nodes, RexNode seek) { @@ -74,6 +89,9 @@ public static Logic find(Logic logic, List nodes, } } + /** Appends to {@code logicList}, for each occurrence of {@code seek} + * within {@code node} in depth-first order, the Logic in force at that + * occurrence. */ public static void collect(RexNode node, RexNode seek, Logic logic, List logicList) { node.accept(new LogicVisitor(seek, logicList), logic); @@ -137,6 +155,12 @@ public static void collect(RexNode node, RexNode seek, Logic logic, @Override public @Nullable Logic visitFieldAccess(RexFieldAccess fieldAccess, @Nullable Logic arg) { + // Not a Boolean value + Logic logic = requireNonNull(arg, "arg"); + if (logic == Logic.TRUE) { + logic = Logic.TRUE_FALSE_UNKNOWN; + } + super.visitFieldAccess(fieldAccess, logic); return end(fieldAccess, arg); } diff --git a/core/src/test/resources/sql/sub-query.iq b/core/src/test/resources/sql/sub-query.iq index 5d1bbb738675..08c5bcf6e4b1 100644 --- a/core/src/test/resources/sql/sub-query.iq +++ b/core/src/test/resources/sql/sub-query.iq @@ -10152,3 +10152,16 @@ ORDER BY emp.ename; !ok # End sub-query.iq + +# [CALCITE-7719] Field access on an element of a ROW array built by a +# sub-query raises IllegalArgumentException. +select t.a[1]."EXPR$0"."EXPR$1" as v +from (select array(select ROW(ROW(1, 2), 3) from (values (0))) as a) as t; ++---+ +| V | ++---+ +| 2 | ++---+ +(1 row) + +!ok From 08f201ab35c017cab99b0f682c29a90374d9d8cc Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Thu, 13 Aug 2026 16:36:14 -0700 Subject: [PATCH 473/562] [CALCITE-7717] Add a Collect.isValid method to check type invariants Signed-off-by: Mihai Budiu --- .../org/apache/calcite/rel/core/Collect.java | 28 +++++++ .../apache/calcite/rel/core/CollectTest.java | 80 +++++++++++++++++++ 2 files changed, 108 insertions(+) create mode 100644 core/src/test/java/org/apache/calcite/rel/core/CollectTest.java diff --git a/core/src/main/java/org/apache/calcite/rel/core/Collect.java b/core/src/main/java/org/apache/calcite/rel/core/Collect.java index 475cc966d3a1..d2f89eeec3ba 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/Collect.java +++ b/core/src/main/java/org/apache/calcite/rel/core/Collect.java @@ -29,9 +29,12 @@ import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.sql.type.SqlTypeUtil; +import org.apache.calcite.util.Litmus; import com.google.common.collect.Iterables; +import org.checkerframework.checker.nullness.qual.Nullable; + import java.util.List; import static java.util.Objects.requireNonNull; @@ -184,6 +187,31 @@ public RelNode copy(RelTraitSet traitSet, RelNode input) { return new Collect(getCluster(), traitSet, input, rowType()); } + @Override public boolean isValid(Litmus litmus, @Nullable Context context) { + final RelDataTypeFactory typeFactory = getCluster().getTypeFactory(); + final RelDataType inputRow = getInput().getRowType(); + if (getCollectionType() == SqlTypeName.MAP && inputRow.getFieldCount() != 2) { + return litmus.fail("MAP requires an input with exactly two fields;" + + " input row type is {}", inputRow); + } + final RelDataType derived = + deriveRowType(typeFactory, getCollectionType(), getFieldName(), inputRow); + if (rowType().equals(derived)) { + return super.isValid(litmus, context); + } + // A Collect created for a collection query constructor derives its element + // type from the input row type; see #create(RelNode, SqlKind, String). + final RelDataType derivedForQuery = + deriveRowType(typeFactory, getCollectionType(), getFieldName(), + SqlTypeUtil.deriveCollectionQueryComponentType(typeFactory, + getCollectionType(), inputRow)); + if (rowType().equals(derivedForQuery)) { + return super.isValid(litmus, context); + } + return litmus.fail("row type {} does not match the type {}" + + " derived from the input", rowType(), derived); + } + @Override public RelNode accept(RelShuttle shuttle) { return shuttle.visit(this); } diff --git a/core/src/test/java/org/apache/calcite/rel/core/CollectTest.java b/core/src/test/java/org/apache/calcite/rel/core/CollectTest.java new file mode 100644 index 000000000000..694eebf1f93c --- /dev/null +++ b/core/src/test/java/org/apache/calcite/rel/core/CollectTest.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.rel.core; + +import org.apache.calcite.plan.Convention; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.test.RelBuilderTest; +import org.apache.calcite.tools.RelBuilder; +import org.apache.calcite.util.Litmus; + +import org.junit.jupiter.api.Test; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; + +/** + * Test cases for [CALCITE-7717] + * Add a Collect.isValid method to check type invariants. + */ +class CollectTest { + @Test void testIsValid() { + final RelBuilder b = RelBuilder.create(RelBuilderTest.config().build()); + final RelNode oneColumn = b.values(new String[] {"i"}, 1, 2).build(); + final RelDataTypeFactory typeFactory = oneColumn.getCluster().getTypeFactory(); + + // Element type is the input row type. 'x' is the name of the result field + final Collect collect0 = Collect.create(oneColumn, SqlKind.ARRAY_QUERY_CONSTRUCTOR, "x"); + assertThat(collect0.isValid(Litmus.IGNORE, null), is(true)); + + // Element type is the type of the sole input column. + final Collect collect1 = + new Collect(oneColumn.getCluster(), + oneColumn.getCluster().traitSetOf(Convention.NONE), oneColumn, + Collect.deriveRowType(typeFactory, SqlTypeName.ARRAY, "x", + oneColumn.getRowType().getFieldList().get(0).getType())); + assertThat(collect1.isValid(Litmus.IGNORE, null), is(true)); + + // Array over two columns is invalid + final RelNode twoColumns = b.values(new String[] {"k", "v"}, 1, "a").build(); + final Collect mismatched = + new Collect(oneColumn.getCluster(), + oneColumn.getCluster().traitSetOf(Convention.NONE), oneColumn, + Collect.deriveRowType(typeFactory, SqlTypeName.ARRAY, "x", + twoColumns.getRowType())); + assertThat(mismatched.isValid(Litmus.IGNORE, null), is(false)); + + final RelDataType mapRowType = + Collect.deriveRowType(typeFactory, SqlTypeName.MAP, "x", twoColumns.getRowType()); + + // A MAP(subquery) over an input that does not have exactly two columns is invalid + final Collect mapOverOneColumn = + new Collect(oneColumn.getCluster(), + oneColumn.getCluster().traitSetOf(Convention.NONE), oneColumn, mapRowType); + assertThat(mapOverOneColumn.isValid(Litmus.IGNORE, null), is(false)); + + // The same MAP row type over the two-column input is valid. + final Collect mapOverTwoColumns = + new Collect(twoColumns.getCluster(), + twoColumns.getCluster().traitSetOf(Convention.NONE), twoColumns, mapRowType); + assertThat(mapOverTwoColumns.isValid(Litmus.IGNORE, null), is(true)); + } +} From 2e7bdede6f63028d1e96fca56a690e7fb481b7f0 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Fri, 17 Jul 2026 17:01:12 -0700 Subject: [PATCH 474/562] [CALCITE-5987] SqlImplementor loses type information for literals Signed-off-by: Mihai Budiu --- .../rel/rel2sql/RelToSqlConverter.java | 26 +- .../calcite/rel/rel2sql/SqlImplementor.java | 110 +++++++- .../RelToSqlConverterRoundTripTest.java | 43 +++ .../rel/rel2sql/RelToSqlConverterTest.java | 249 ++++++++++++++++-- 4 files changed, 401 insertions(+), 27 deletions(-) create mode 100644 core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterRoundTripTest.java diff --git a/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java b/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java index a5b8858a3a09..7806ab5e75ef 100644 --- a/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java +++ b/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java @@ -139,9 +139,16 @@ public class RelToSqlConverter extends SqlImplementor private final Deque stack = new ArrayDeque<>(); /** Creates a RelToSqlConverter. */ - @SuppressWarnings("argument.type.incompatible") public RelToSqlConverter(SqlDialect dialect) { - super(dialect); + this(dialect, false); + } + + /** Creates a RelToSqlConverter; if {@code preserveLiteralTypes}, literals + * whose type is not implied by their SQL text are wrapped in CASTs; + * see {@link SqlImplementor#toSql(RexProgram, RexLiteral, SqlDialect)}. */ + @SuppressWarnings("argument.type.incompatible") + public RelToSqlConverter(SqlDialect dialect, boolean preserveLiteralTypes) { + super(dialect, preserveLiteralTypes); dispatcher = ReflectUtil.createMethodDispatcher(Result.class, this, "visit", RelNode.class); @@ -1397,18 +1404,25 @@ void offsetFetch(Sort e, Builder builder) { } } - private static SqlNode toSqlOffset(Sort sort, Context context) { + private SqlNode toSqlOffset(Sort sort, Context context) { final RexNode offset = requireNonNull(sort.offset, "offset"); final @Nullable RexLiteral reduced = RexUtil.reduceOffsetToLiteral(sort.getCluster(), offset); - return context.toSql(null, reduced == null ? offset : reduced); + return offsetFetchToSql(context, reduced == null ? offset : reduced); } - private static SqlNode toSqlFetch(Sort sort, Context context) { + private SqlNode toSqlFetch(Sort sort, Context context) { final RexNode fetch = requireNonNull(sort.fetch, "fetch"); final @Nullable RexLiteral reduced = RexUtil.reduceFetchToLiteral(sort.getCluster(), fetch); - return context.toSql(null, reduced == null ? fetch : reduced); + return offsetFetchToSql(context, reduced == null ? fetch : reduced); + } + + /** Converts an OFFSET or FETCH expression; these can never have a CAST. */ + private SqlNode offsetFetchToSql(Context context, RexNode rex) { + return preserveLiteralTypes && rex instanceof RexLiteral + ? SqlImplementor.toSql(null, (RexLiteral) rex) + : context.toSql(null, rex); } public boolean hasTrickyRollup(Sort e, Aggregate aggregate) { diff --git a/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java b/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java index fb003caa9509..60b068438ab3 100644 --- a/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java +++ b/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java @@ -101,6 +101,7 @@ import org.apache.calcite.sql.type.SqlTypeFactoryImpl; import org.apache.calcite.sql.type.SqlTypeFamily; import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.sql.type.SqlTypeUtil; import org.apache.calcite.sql.util.SqlBasicVisitor; import org.apache.calcite.sql.util.SqlShuttle; import org.apache.calcite.sql.validate.SqlValidatorUtil; @@ -140,6 +141,7 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.UUID; import java.util.function.Function; @@ -173,6 +175,12 @@ public abstract class SqlImplementor { SqlLiteral.createExactNumeric("1", POS); public final SqlDialect dialect; + + /** If true, literals whose type is not implied by their SQL text are + * wrapped in CASTs that make the type explicit; + * see {@link #toSql(RexProgram, RexLiteral, SqlDialect)}. */ + public final boolean preserveLiteralTypes; + protected final Set aliasSet = new LinkedHashSet<>(); protected final Map correlTableMap = new HashMap<>(); @@ -187,7 +195,12 @@ public abstract class SqlImplementor { new RexBuilder(new SqlTypeFactoryImpl(RelDataTypeSystemImpl.DEFAULT)); protected SqlImplementor(SqlDialect dialect) { + this(dialect, false); + } + + protected SqlImplementor(SqlDialect dialect, boolean preserveLiteralTypes) { this.dialect = requireNonNull(dialect, "dialect"); + this.preserveLiteralTypes = preserveLiteralTypes; } /** Visits a relational expression that has no parent. */ @@ -694,6 +707,13 @@ protected Context(SqlDialect dialect, int fieldCount, boolean ignoreCast) { public abstract SqlNode field(int ordinal); + /** Returns whether literals whose type is not implied by their SQL text + * are wrapped in CASTs; + * see {@link SqlImplementor#toSql(RexProgram, RexLiteral, SqlDialect)}. */ + protected boolean preservesLiteralTypes() { + return false; + } + /** Creates a reference to a field to be used in an ORDER BY clause. * *

      By default, it returns the same result as {@link #field}. @@ -798,7 +818,9 @@ public SqlNode toSql(@Nullable RexProgram program, RexNode rex) { } case LITERAL: - return SqlImplementor.toSql(program, (RexLiteral) rex); + return preservesLiteralTypes() + ? SqlImplementor.toSql(program, (RexLiteral) rex, dialect) + : SqlImplementor.toSql(program, (RexLiteral) rex); case CASE: final RexCall caseCall = (RexCall) rex; @@ -1608,6 +1630,77 @@ public static SqlNode toSql(@Nullable RexProgram program, RexLiteral literal) { } } + /** Converts a {@link RexLiteral} in the context of a {@link RexProgram} + * to a {@link SqlNode}, preserving the literal's type. + * + *

      The SQL text of a literal does not always imply the literal's type: + * {@code 1} parses as INTEGER even if the literal's type is TINYINT, and + * {@code NULL} loses its type entirely. This method wraps such literals in + * a CAST that makes the type explicit; {@code dialect} supplies the SQL + * syntax of the CAST target type. */ + public static SqlNode toSql(@Nullable RexProgram program, RexLiteral literal, + SqlDialect dialect) { + switch (literal.getTypeName()) { + case ROW: + // Cast the fields rather than the ROW call, because few dialects can + // parse a cast to a ROW type. + //noinspection unchecked + final List list = castNonNull(literal.getValueAs(List.class)); + return SqlStdOperatorTable.ROW.createCall(POS, + list.stream().map(e -> toSql(program, e, dialect)) + .collect(toImmutableList())); + + case SYMBOL: + case SARG: + return toSql(program, literal); + + default: + final SqlNode node = toSql(program, literal); + // A result that is not a SqlLiteral is already a CAST; for example + // NaN becomes CAST('NaN' AS DOUBLE). + return node instanceof SqlLiteral + ? castIfTypeAmbiguous((SqlLiteral) node, literal.getType(), dialect) + : node; + } + } + + /** Wraps a literal in a CAST to {@code type} if the type that the + * validator would infer for the literal's SQL text differs from + * {@code type}. */ + private static SqlNode castIfTypeAmbiguous(SqlLiteral literal, RelDataType type, + SqlDialect dialect) { + switch (type.getSqlTypeName()) { + case NULL: + case ANY: + case UNKNOWN: + // No valid SQL syntax for casts to these types + return literal; + default: + break; + } + if (typeMatches(literal, type)) { + return literal; + } + final SqlNode castSpec = dialect.getCastSpec(type); + if (castSpec == null) { + return literal; + } + return SqlStdOperatorTable.CAST.createCall(POS, literal, castSpec); + } + + /** Returns whether the type that the validator infers for {@code literal}'s + * SQL text matches {@code type}. Ignores nullability, and collation. */ + private static boolean typeMatches(SqlLiteral literal, RelDataType type) { + final RelDataTypeFactory typeFactory = RexBuilder.DEFAULT.getTypeFactory(); + final RelDataType impliedType = literal.createSqlType(typeFactory); + if (SqlTypeUtil.isCharacter(impliedType) && SqlTypeUtil.isCharacter(type)) { + return impliedType.getSqlTypeName() == type.getSqlTypeName() + && impliedType.getPrecision() == type.getPrecision() + && Objects.equals(impliedType.getCharset(), type.getCharset()); + } + return SqlTypeUtil.equalSansNullability(typeFactory, impliedType, type); + } + /** Converts a {@link RexLiteral} to a {@link SqlLiteral}. */ public static SqlNode toSql(RexLiteral literal) { SqlTypeName typeName = literal.getTypeName(); @@ -1724,10 +1817,21 @@ protected Context getAliasContext(RexCorrelVariable variable) { * to use it. It is a good way to convert a {@link RexNode} to SQL text. */ public static class SimpleContext extends Context { private final IntFunction field; + private final boolean preserveLiteralTypes; public SimpleContext(SqlDialect dialect, IntFunction field) { + this(dialect, field, false); + } + + public SimpleContext(SqlDialect dialect, IntFunction field, + boolean preserveLiteralTypes) { super(dialect, 0, false); this.field = field; + this.preserveLiteralTypes = preserveLiteralTypes; + } + + @Override protected boolean preservesLiteralTypes() { + return preserveLiteralTypes; } @Override public SqlImplementor implementor() { @@ -1750,6 +1854,10 @@ protected abstract class BaseContext extends Context { return SqlImplementor.this.getAliasContext(variable); } + @Override protected boolean preservesLiteralTypes() { + return preserveLiteralTypes; + } + @Override public SqlImplementor implementor() { return SqlImplementor.this; } diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterRoundTripTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterRoundTripTest.java new file mode 100644 index 000000000000..eeef73b29471 --- /dev/null +++ b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterRoundTripTest.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.rel.rel2sql; + +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +/** + * Runs every test of {@link RelToSqlConverterTest} through the round trip + * SQL → Rel → SQL → Rel → SQL + * and checks that the second and third SQL are the same. + * + *

      Tests whose Calcite-dialect output cannot be parsed or validated are skipped. + */ +class RelToSqlConverterRoundTripTest extends RelToSqlConverterTest { + @Override Sql fixture() { + return super.fixture().withRoundTrip(); + } + + @Disabled("SUM(DISTINCT) OVER expands into a deeper CASE on every re-parse," + + " so the conversion never reaches a fixed point") + @Test @Override void testConvertWindowToSql() { + } + + @Disabled("UNION ALL gains a subquery alias only on the second" + + " re-parse, so the second and third SQL differ") + @Test @Override void testThreeQueryUnion() { + } +} diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java index 779b320574a1..8e2b6efb53b0 100644 --- a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java +++ b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java @@ -34,6 +34,7 @@ import org.apache.calcite.rel.hint.RelHint; import org.apache.calcite.rel.logical.LogicalAggregate; import org.apache.calcite.rel.logical.LogicalFilter; +import org.apache.calcite.rel.logical.LogicalSort; import org.apache.calcite.rel.rules.AggregateGroupingSetsToUnionRule; import org.apache.calcite.rel.rules.AggregateJoinTransposeRule; import org.apache.calcite.rel.rules.AggregateProjectMergeRule; @@ -49,7 +50,9 @@ import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rel.type.RelDataTypeSystem; import org.apache.calcite.rel.type.RelDataTypeSystemImpl; +import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexCorrelVariable; +import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; import org.apache.calcite.runtime.FlatLists; import org.apache.calcite.runtime.Hook; @@ -111,6 +114,7 @@ import org.checkerframework.checker.nullness.qual.Nullable; import org.junit.jupiter.api.Test; +import org.opentest4j.TestAbortedException; import java.math.BigDecimal; import java.util.Collection; @@ -137,7 +141,7 @@ */ class RelToSqlConverterTest { - private Sql fixture() { + Sql fixture() { return new Sql(CalciteAssert.SchemaSpec.JDBC_FOODMART, "?", CalciteSqlDialect.DEFAULT, SqlParser.Config.DEFAULT, ImmutableSet.of(), UnaryOperator.identity(), null, ImmutableList.of(), StandardConvertletTable.INSTANCE); @@ -288,6 +292,19 @@ private static String toSql(RelNode root, SqlDialect dialect, .getSql(); } + /** Converts a relational expression to SQL in a given dialect, wrapping + * literals whose type is not implied by their SQL text in CASTs. */ + private static String toSqlPreservingLiteralTypes(RelNode root, SqlDialect dialect) { + final RelToSqlConverter converter = new RelToSqlConverter(dialect, true); + final SqlNode sqlNode = converter.visitRoot(root).asStatement(); + return sqlNode.toSqlString(c -> + c.withDialect(dialect) + .withAlwaysUseParentheses(false) + .withSelectListItemsOnSeparateLines(false) + .withUpdateSetListNewline(false) + .withIndentation(0)).getSql(); + } + /** * Test for [CALCITE-5988] * SqlImplementor.toSql cannot emit VARBINARY literals. @@ -308,6 +325,143 @@ private static String toSql(RelNode root, SqlDialect dialect, sql(query).withMysql().ok(expected); } + /** Creates a relational expression that projects literals of many types. */ + private static RelNode projectOfLiterals() { + final RelBuilder b = relBuilder(); + final RelDataTypeFactory typeFactory = b.getTypeFactory(); + final RexBuilder rexBuilder = b.getRexBuilder(); + return b + .scan("EMP") + .project( + rexBuilder.makeLiteral(1, + typeFactory.createSqlType(SqlTypeName.TINYINT)), + rexBuilder.makeLiteral(1, + typeFactory.createSqlType(SqlTypeName.SMALLINT)), + rexBuilder.makeLiteral(1, + typeFactory.createSqlType(SqlTypeName.INTEGER)), + rexBuilder.makeLiteral(1, + typeFactory.createSqlType(SqlTypeName.BIGINT)), + rexBuilder.makeLiteral(new BigDecimal("1.50"), + typeFactory.createSqlType(SqlTypeName.DECIMAL, 10, 2)), + rexBuilder.makeLiteral(0.5, + typeFactory.createSqlType(SqlTypeName.REAL)), + rexBuilder.makeLiteral(0.5, + typeFactory.createSqlType(SqlTypeName.DOUBLE)), + // makeCast folds the cast into a literal with type VARCHAR(10) + rexBuilder.makeCast( + typeFactory.createSqlType(SqlTypeName.VARCHAR, 10), + rexBuilder.makeLiteral("abc")), + rexBuilder.makeLiteral("abc", + typeFactory.createSqlType(SqlTypeName.CHAR, 3)), + rexBuilder.makeNullLiteral( + typeFactory.createSqlType(SqlTypeName.INTEGER)), + b.literal(true)) + .build(); + } + + /** Test case for + * [CALCITE-5987] + * SqlImplementor loses type information for literals. */ + @Test void testPreserveLiteralTypes() { + final RelNode root = projectOfLiterals(); + final SqlDialect dialect = DatabaseProduct.CALCITE.getDialect(); + // Without the option, every literal except the NULL loses its type; + // visit(Project) casts NULL literals regardless of the option + final String expected = "SELECT 1 AS \"$f0\", 1 AS \"$f1\", 1 AS \"$f2\"," + + " 1 AS \"$f3\", 1.50 AS \"$f4\", 5E-1 AS \"$f5\", 5E-1 AS \"$f6\"," + + " 'abc' AS \"$f7\", 'abc' AS \"$f8\", CAST(NULL AS INTEGER) AS \"$f9\"," + + " TRUE AS \"$f10\"\n" + + "FROM \"scott\".\"EMP\""; + assertThat(toSql(root, dialect), isLinux(expected)); + // With the option, literals whose SQL text parses to a different type + // use a cast + final String expectedPreserved = "SELECT" + + " CAST(1 AS TINYINT) AS \"$f0\"," + + " CAST(1 AS SMALLINT) AS \"$f1\"," + + " 1 AS \"$f2\"," + + " CAST(1 AS BIGINT) AS \"$f3\"," + + " CAST(1.50 AS DECIMAL(10, 2)) AS \"$f4\"," + + " CAST(5E-1 AS REAL) AS \"$f5\"," + + " 5E-1 AS \"$f6\"," + + " CAST('abc' AS VARCHAR(10) CHARACTER SET \"ISO-8859-1\") AS \"$f7\"," + + " 'abc' AS \"$f8\"," + + " CAST(NULL AS INTEGER) AS \"$f9\"," + + " TRUE AS \"$f10\"\n" + + "FROM \"scott\".\"EMP\""; + assertThat(toSqlPreservingLiteralTypes(root, dialect), + isLinux(expectedPreserved)); + } + + /** As {@link #testPreserveLiteralTypes()}, but for literals in a VALUES + * clause. */ + @Test void testPreserveLiteralTypesValues() { + final RelBuilder b = relBuilder(); + final RelDataTypeFactory typeFactory = b.getTypeFactory(); + final RexBuilder rexBuilder = b.getRexBuilder(); + final RelDataType tinyint = typeFactory.createSqlType(SqlTypeName.TINYINT); + final RelDataType varchar5 = + typeFactory.createSqlType(SqlTypeName.VARCHAR, 5); + final RelDataType rowType = typeFactory.builder() + .add("a", tinyint) + .add("b", varchar5) + .build(); + final RelNode root = b + .values( + ImmutableList.of( + ImmutableList.of(rexBuilder.makeLiteral(1, tinyint), + (RexLiteral) rexBuilder.makeCast(varchar5, + rexBuilder.makeLiteral("x"))), + ImmutableList.of(rexBuilder.makeLiteral(2, tinyint), + (RexLiteral) rexBuilder.makeCast(varchar5, + rexBuilder.makeLiteral("y")))), + rowType) + .build(); + final SqlDialect dialect = DatabaseProduct.CALCITE.getDialect(); + final String expectedPreserved = "SELECT *\n" + + "FROM (VALUES" + + " (CAST(1 AS TINYINT)," + + " CAST('x' AS VARCHAR(5) CHARACTER SET \"ISO-8859-1\")),\n" + + "(CAST(2 AS TINYINT)," + + " CAST('y' AS VARCHAR(5) CHARACTER SET \"ISO-8859-1\")))" + + " AS \"t\" (\"a\", \"b\")"; + assertThat(toSqlPreservingLiteralTypes(root, dialect), + isLinux(expectedPreserved)); + } + + /** Parses a SQL query and converts it to a relational expression. */ + private static RelNode sqlToRel(String sql, SchemaPlus defaultSchema, + SqlParser.Config parserConfig, Set librarySet, + SqlToRelConverter.Config config, SqlDialect dialect, + SqlRexConvertletTable convertletTable) throws Exception { + final Planner planner = + getPlanner(null, parserConfig, defaultSchema, config, librarySet, + dialect.getTypeSystem(), convertletTable); + final SqlNode parse = planner.parse(sql); + final SqlNode validate = planner.validate(parse); + return planner.rel(validate).project(); + } + + /** As {@link #testPreserveLiteralTypes()}, but the type of a FETCH or + * OFFSET literal carries no information, so those literals are never + * cast, whatever their type. */ + @Test void testPreserveLiteralTypesFetchOffset() { + final RelBuilder b = relBuilder(); + final RelDataTypeFactory typeFactory = b.getTypeFactory(); + final RexBuilder rexBuilder = b.getRexBuilder(); + final RelDataType bigint = typeFactory.createSqlType(SqlTypeName.BIGINT); + final RelNode root = + LogicalSort.create(b.scan("EMP").build(), RelCollations.EMPTY, + rexBuilder.makeLiteral(2, bigint), + rexBuilder.makeLiteral(3, bigint)); + final SqlDialect dialect = DatabaseProduct.CALCITE.getDialect(); + final String expectedPreserved = "SELECT *\n" + + "FROM \"scott\".\"EMP\"\n" + + "OFFSET 2 ROWS\n" + + "FETCH NEXT 3 ROWS ONLY"; + assertThat(toSqlPreservingLiteralTypes(root, dialect), + isLinux(expectedPreserved)); + } + /** Test case for * [CALCITE-2152] * SQL parser unable to parse SQL with nested joins produced by RelToSqlConverter. */ @@ -12306,6 +12460,10 @@ static class Sql { private final SqlParser.Config parserConfig; private final UnaryOperator config; private final SqlRexConvertletTable convertletTable; + /** If true, {@link #exec()} additionally checks that Calcite-dialect + * output round-trips when literal types are preserved; + * see {@link #checkRoundTrip(RelNode, SchemaPlus)}. */ + private final boolean roundTrip; Sql(CalciteAssert.SchemaSpec schemaSpec, String sql, SqlDialect dialect, SqlParser.Config parserConfig, Set librarySet, @@ -12313,6 +12471,17 @@ static class Sql { @Nullable Function relFn, List> transforms, SqlRexConvertletTable convertletTable) { + this(schemaSpec, sql, dialect, parserConfig, librarySet, config, relFn, + transforms, convertletTable, false); + } + + Sql(CalciteAssert.SchemaSpec schemaSpec, String sql, SqlDialect dialect, + SqlParser.Config parserConfig, Set librarySet, + UnaryOperator config, + @Nullable Function relFn, + List> transforms, + SqlRexConvertletTable convertletTable, + boolean roundTrip) { this.schemaSpec = schemaSpec; this.sql = sql; this.dialect = dialect; @@ -12322,21 +12491,22 @@ static class Sql { this.parserConfig = parserConfig; this.config = config; this.convertletTable = convertletTable; + this.roundTrip = roundTrip; } Sql withSql(String sql) { return new Sql(schemaSpec, sql, dialect, parserConfig, librarySet, config, - relFn, transforms, convertletTable); + relFn, transforms, convertletTable, roundTrip); } Sql dialect(SqlDialect dialect) { return new Sql(schemaSpec, sql, dialect, parserConfig, librarySet, config, - relFn, transforms, convertletTable); + relFn, transforms, convertletTable, roundTrip); } Sql relFn(Function relFn) { return new Sql(schemaSpec, sql, dialect, parserConfig, librarySet, config, - relFn, transforms, convertletTable); + relFn, transforms, convertletTable, roundTrip); } Sql withCalcite() { @@ -12579,12 +12749,12 @@ Sql withOracleModifiedTypeSystem() { Sql parserConfig(SqlParser.Config parserConfig) { return new Sql(schemaSpec, sql, dialect, parserConfig, librarySet, config, - relFn, transforms, convertletTable); + relFn, transforms, convertletTable, roundTrip); } Sql withConfig(UnaryOperator config) { return new Sql(schemaSpec, sql, dialect, parserConfig, librarySet, config, - relFn, transforms, convertletTable); + relFn, transforms, convertletTable, roundTrip); } final Sql withLibrary(SqlLibrary library) { @@ -12593,7 +12763,7 @@ final Sql withLibrary(SqlLibrary library) { Sql withLibrarySet(Iterable librarySet) { return new Sql(schemaSpec, sql, dialect, parserConfig, - ImmutableSet.copyOf(librarySet), config, relFn, transforms, convertletTable); + ImmutableSet.copyOf(librarySet), config, relFn, transforms, convertletTable, roundTrip); } Sql optimize(final RuleSet ruleSet, @@ -12610,12 +12780,20 @@ Sql optimize(final RuleSet ruleSet, ImmutableList.of(), ImmutableList.of()); }); return new Sql(schemaSpec, sql, dialect, parserConfig, librarySet, config, - relFn, transforms, convertletTable); + relFn, transforms, convertletTable, roundTrip); } Sql withConvertletTable(SqlRexConvertletTable convertletTable) { return new Sql(schemaSpec, sql, dialect, parserConfig, librarySet, config, - relFn, transforms, convertletTable); + relFn, transforms, convertletTable, roundTrip); + } + + /** Returns a copy of this Sql whose {@link #exec()} also checks the + * round trip of Calcite-dialect output; + * see {@link #checkRoundTrip(RelNode, SchemaPlus)}. */ + Sql withRoundTrip() { + return new Sql(schemaSpec, sql, dialect, parserConfig, librarySet, + config, relFn, transforms, convertletTable, true); } Sql ok(String expectedQuery) { @@ -12647,28 +12825,59 @@ String exec() { final RelBuilder relBuilder = RelBuilder.create(frameworkConfig); rel = relFn.apply(relBuilder); } else { - final SqlToRelConverter.Config config = this.config.apply(SqlToRelConverter.config() - .withTrimUnusedFields(false)); - RelDataTypeSystem typeSystem = dialect.getTypeSystem(); - final Planner planner = - getPlanner(null, parserConfig, defaultSchema, config, librarySet, typeSystem, - convertletTable); - SqlNode parse = planner.parse(sql); - SqlNode validate = planner.validate(parse); - rel = planner.rel(validate).project(); + rel = + sqlToRel(sql, defaultSchema, parserConfig, librarySet, + sqlToRelConverterConfig(), dialect, convertletTable); } for (Function transform : transforms) { rel = transform.apply(rel); } - return toSql(rel, dialect); + final String result = toSql(rel, dialect); + if (roundTrip && dialect instanceof CalciteSqlDialect) { + checkRoundTrip(rel, defaultSchema); + } + return result; } catch (Exception e) { throw TestUtil.rethrow(e); } } + /** Checks the round trip Rel → SQL1 → Rel1 → SQL2 → + * Rel2 → SQL3, where every conversion preserves literal types: + * SQL3 must equal SQL2. */ + private void checkRoundTrip(RelNode rel, SchemaPlus defaultSchema) { + final String sql1 = toSqlPreservingLiteralTypes(rel, dialect); + final RelNode rel1 = parseBack(sql1, defaultSchema); + final String sql2 = toSqlPreservingLiteralTypes(rel1, dialect); + final RelNode rel2 = parseBack(sql2, defaultSchema); + final String sql3 = toSqlPreservingLiteralTypes(rel2, dialect); + assertThat(sql3, is(sql2)); + } + + /** Parses SQL generated for the Calcite dialect and converts it back to + * a relational expression. Aborts the test if the SQL cannot be parsed + * or validated. */ + private RelNode parseBack(String sql, SchemaPlus defaultSchema) { + try { + return sqlToRel(sql, defaultSchema, SqlParser.Config.DEFAULT, + librarySet, sqlToRelConverterConfig(), dialect, convertletTable); + } catch (Exception | AssertionError e) { + throw new TestAbortedException("cannot re-parse: " + sql, e); + } + } + + /** Materializes the SQL-to-rel configuration for this test: the default + * configuration, with field trimming disabled so that the rel keeps the + * shape that the test expects, transformed by the operator that the test + * supplied to {@link #withConfig}. */ + private SqlToRelConverter.Config sqlToRelConverterConfig() { + return this.config.apply(SqlToRelConverter.config() + .withTrimUnusedFields(false)); + } + public Sql schema(CalciteAssert.SchemaSpec schemaSpec) { return new Sql(schemaSpec, sql, dialect, parserConfig, librarySet, config, - relFn, transforms, convertletTable); + relFn, transforms, convertletTable, roundTrip); } } From 50044ea3059b0a5bee370423167ffb6070810162 Mon Sep 17 00:00:00 2001 From: Stamatis Zampetakis Date: Thu, 13 Aug 2026 16:02:13 +0300 Subject: [PATCH 475/562] [CALCITE-7712] Enforce Input annotation on ReflectiveSchema.Factory operands --- .../adapter/java/ReflectiveSchema.java | 23 +++++++- .../calcite/test/ReflectiveSchemaTest.java | 58 +++++++++++++++++++ site/_docs/history.md | 5 ++ 3 files changed, 84 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/adapter/java/ReflectiveSchema.java b/core/src/main/java/org/apache/calcite/adapter/java/ReflectiveSchema.java index 24dbe8a3d27c..07e29b2b2233 100644 --- a/core/src/main/java/org/apache/calcite/adapter/java/ReflectiveSchema.java +++ b/core/src/main/java/org/apache/calcite/adapter/java/ReflectiveSchema.java @@ -56,6 +56,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.Nullable; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; import java.lang.reflect.Array; import java.lang.reflect.Constructor; import java.lang.reflect.Field; @@ -293,6 +297,15 @@ private static class ReflectiveTable } } + /** + * Designates a type that can be used as input in the {@link Factory}. + */ + @Retention(RetentionPolicy.RUNTIME) + @Target(ElementType.TYPE) + public @interface Input { + + } + /** Factory that creates a schema by instantiating an object and looking at * its public fields. * @@ -321,22 +334,28 @@ private static class ReflectiveTable * Employee[] EMPS; * Department[] DEPTS; * } + * + *

      The class operand must be annotated as {@link Input} otherwise it cannot + * be used in this factory. */ public static class Factory implements SchemaFactory { @Override public Schema create(SchemaPlus parentSchema, String name, Map operand) { Class clazz; Object target; - final Object className = operand.get("class"); + final String className = (String) operand.get("class"); if (className != null) { try { - clazz = Class.forName((String) className); + clazz = Class.forName(className, false, Factory.class.getClassLoader()); } catch (ClassNotFoundException e) { throw new RuntimeException("Error loading class " + className, e); } } else { throw new RuntimeException("Operand 'class' is required"); } + if (!clazz.isAnnotationPresent(Input.class)) { + throw new IllegalArgumentException(clazz + " is not annotated with @Input"); + } final Object methodName = operand.get("staticMethod"); if (methodName != null) { try { diff --git a/core/src/test/java/org/apache/calcite/test/ReflectiveSchemaTest.java b/core/src/test/java/org/apache/calcite/test/ReflectiveSchemaTest.java index 3c86c0125539..e4979b95f3fa 100644 --- a/core/src/test/java/org/apache/calcite/test/ReflectiveSchemaTest.java +++ b/core/src/test/java/org/apache/calcite/test/ReflectiveSchemaTest.java @@ -20,6 +20,7 @@ import org.apache.calcite.avatica.util.DateTimeUtils; import org.apache.calcite.config.Lex; import org.apache.calcite.jdbc.CalciteConnection; +import org.apache.calcite.jdbc.CalciteSchema; import org.apache.calcite.jdbc.Driver; import org.apache.calcite.linq4j.Enumerable; import org.apache.calcite.linq4j.Linq4j; @@ -44,6 +45,7 @@ import org.apache.calcite.util.Util; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; @@ -61,6 +63,7 @@ import java.util.Arrays; import java.util.BitSet; import java.util.List; +import java.util.Map; import java.util.Properties; import static org.apache.calcite.test.Matchers.isListOf; @@ -72,6 +75,7 @@ import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.hasToString; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; @@ -1102,4 +1106,58 @@ public static class DateColumnSchema { assertNotNull(statistic); assertThat(statistic.getRowCount(), is(2D)); } + + /** Test for [CALCITE-7712] + * Enforce Input annotation on ReflectiveSchema.Factory operands. */ + @Test void testFactoryCreateWithAnnotatedClassOperand() { + ReflectiveSchema.Factory factory = new ReflectiveSchema.Factory(); + SchemaPlus root = CalciteSchema.createRootSchema(false).plus(); + Map operand = + ImmutableMap.of("class", "org.apache.calcite.test.ReflectiveSchemaTest$ValidClassOp"); + assertNotNull(factory.create(root, "ignore", operand)); + } + + /** Test for [CALCITE-7712] + * Enforce Input annotation on ReflectiveSchema.Factory operands. The test ensures + * invalid classes are rejected with an informative exception.*/ + @Test void testFactoryCreateWithInvalidClassOperandThrows() { + ReflectiveSchema.Factory factory = new ReflectiveSchema.Factory(); + SchemaPlus root = CalciteSchema.createRootSchema(false).plus(); + Map operand = + ImmutableMap.of("class", "org.apache.calcite.test.ReflectiveSchemaTest$InvalidClassOp"); + IllegalArgumentException e = + assertThrows(IllegalArgumentException.class, () -> factory.create(root, "ignore", operand)); + assertThat(e.getMessage(), + is("class org.apache.calcite.test.ReflectiveSchemaTest$InvalidClassOp is not annotated " + + "with @Input")); + } + + /** Test for [CALCITE-7712] + * Enforce Input annotation on ReflectiveSchema.Factory operands. The test ensures + * invalid classes are not initialized.*/ + @Test void testFactoryCreateWithInvalidClassOperandDoesNotTriggerInitializers() { + ReflectiveSchema.Factory factory = new ReflectiveSchema.Factory(); + SchemaPlus root = CalciteSchema.createRootSchema(false).plus(); + Map operand = + ImmutableMap.of("class", "org.apache.calcite.InvalidStaticInitializer"); + IllegalArgumentException e = + assertThrows(IllegalArgumentException.class, () -> factory.create(root, "ignore", operand)); + assertThat(e.getMessage(), + is("class org.apache.calcite.InvalidStaticInitializer is not annotated with @Input")); + } + + /** + * A valid class operand for {@link ReflectiveSchema.Factory} with proper annotation. + */ + @ReflectiveSchema.Input + public static class ValidClassOp { + public ValidClassOp() {} + } + + /** + * An invalid class operand for {@link ReflectiveSchema.Factory} due to missing annotation. + */ + public static class InvalidClassOp { + public InvalidClassOp() {} + } } diff --git a/site/_docs/history.md b/site/_docs/history.md index 9520e7d05f39..3aba1fd07588 100644 --- a/site/_docs/history.md +++ b/site/_docs/history.md @@ -54,6 +54,11 @@ other software versions as specified in gradle.properties. filter evaluation now run in Java, and the `arrow-gandiva` dependency is no longer included in the Arrow module or BOM. +* [CALCITE-7712] +`ReflectiveSchema.Factory` requires the class operand to explicitly use the new +`ReflectiveSchema.Input` annotation. If the annotation is not present the +creation will fail with `IllegalArgumentException: class X is not annotated @Input`. + * [CALCITE-7713] Class loading from model files has been disabled by default. Any attempt to load classes from model files will lead to `SecurityException` unless an appropriate From 29d08d76462bfcca19dca8165a616826324a961c Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Fri, 14 Aug 2026 10:50:33 -0700 Subject: [PATCH 476/562] [CALCITE-7721] CAST of a ROW ARRAY element raises NullPointerException Signed-off-by: Mihai Budiu --- .../enumerable/RexToLixTranslator.java | 6 ++++ core/src/test/resources/sql/struct.iq | 35 +++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java index f102fd69d79e..35373aca3e39 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java @@ -361,6 +361,12 @@ private Expression getRowConvertExpression( return Expressions.constant(null); } assert sourceType.getSqlTypeName() == SqlTypeName.ROW; + if (Types.getComponentType(operand.getType()) == null) { + // A ROW value is represented as an Object[] at runtime, but the + // expression that produces it may have type Object, for example an + // element of an array; the field access below requires an array. + operand = Expressions.convert_(operand, Object[].class); + } List targetTypes = targetType.getFieldList(); List sourceTypes = sourceType.getFieldList(); assert targetTypes.size() == sourceTypes.size(); diff --git a/core/src/test/resources/sql/struct.iq b/core/src/test/resources/sql/struct.iq index 08c876205066..4356030dc530 100644 --- a/core/src/test/resources/sql/struct.iq +++ b/core/src/test/resources/sql/struct.iq @@ -420,4 +420,39 @@ from (values (1)) as t2(x), (select ARRAY[ROW(ROW(1, 2), 3)] as arr) as t; !ok +# [CALCITE-7721] CAST of a ROW ARRAY element raises NullPointerException. +select cast(t.arr[1] as row(x integer, y integer)) as v +from (select ARRAY[ROW(1, 2)] as arr) as t; ++--------+ +| V | ++--------+ +| {1, 2} | ++--------+ +(1 row) + +!ok + +# The previous test using COALESCE, which casts both of its arguments to the +# least restrictive type +select coalesce(t.arr[1], ROW(9, 9)) as v from (select ARRAY[ROW(1, 2)] as arr) as t; ++--------+ +| V | ++--------+ +| {1, 2} | ++--------+ +(1 row) + +!ok + +# COALESCE over a ROW +select coalesce(t.r, ROW(9, 9)) as v from (select ROW(1, 2) as r) as t; ++--------+ +| V | ++--------+ +| {1, 2} | ++--------+ +(1 row) + +!ok + # End struct.iq From 434218c7eeb664d56386dc22296c433a45afe95a Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Wed, 12 Aug 2026 14:43:31 -0700 Subject: [PATCH 477/562] [CALCITE-7334] Compiling the generated Java code throws an exception when a scalar subquery is used in the SELECT list Signed-off-by: Mihai Budiu --- .../enumerable/EnumerableMergeJoin.java | 8 +++-- core/src/test/resources/sql/sub-query.iq | 35 +++++++++++++++++++ 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeJoin.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeJoin.java index 557362e6bc29..fe65699e0c12 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeJoin.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeJoin.java @@ -18,6 +18,7 @@ import org.apache.calcite.adapter.java.JavaTypeFactory; import org.apache.calcite.linq4j.EnumerableDefaults; +import org.apache.calcite.linq4j.function.Function1; import org.apache.calcite.linq4j.tree.BlockBuilder; import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.linq4j.tree.Expressions; @@ -530,9 +531,12 @@ public static EnumerableMergeJoin create(RelNode left, RelNode right, Expressions.list( leftExpression, rightExpression, - Expressions.lambda( + // Force Function1: a single BOOLEAN key would otherwise + // deduce Predicate1, which does not match the signature of + // EnumerableDefaults.mergeJoin + Expressions.lambda(Function1.class, leftKeyPhysType.record(leftExpressions), left_), - Expressions.lambda( + Expressions.lambda(Function1.class, rightKeyPhysType.record(rightExpressions), right_), predicate, EnumUtils.joinSelector(joinType, diff --git a/core/src/test/resources/sql/sub-query.iq b/core/src/test/resources/sql/sub-query.iq index 08c5bcf6e4b1..4bc3fae31ab5 100644 --- a/core/src/test/resources/sql/sub-query.iq +++ b/core/src/test/resources/sql/sub-query.iq @@ -10150,6 +10150,41 @@ ORDER BY emp.ename; +--------+------------+ (8 rows) +!ok + +!use scott + +# Test case for [CALCITE-7334] Compiling the generated Java code throws an exception when +# a scalar sub-query is used in the SELECT list. +# Result validated against PostgreSQL 14. +select + (SELECT + COUNT(*) + FROM + "scott".emp + WHERE + v.empno >= 2), 3 +from "scott".emp as v; ++--------+--------+ +| EXPR$0 | EXPR$1 | ++--------+--------+ +| 14 | 3 | +| 14 | 3 | +| 14 | 3 | +| 14 | 3 | +| 14 | 3 | +| 14 | 3 | +| 14 | 3 | +| 14 | 3 | +| 14 | 3 | +| 14 | 3 | +| 14 | 3 | +| 14 | 3 | +| 14 | 3 | +| 14 | 3 | ++--------+--------+ +(14 rows) + !ok # End sub-query.iq From 9a1fdf67ab0b533d2a8d291945784d3890fc1511 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Mon, 17 Aug 2026 12:02:11 -0700 Subject: [PATCH 478/562] [CALCITE-7723] Queries using Collect with ROW results throw AssertionFailure Signed-off-by: Mihai Budiu --- .../sql2rel/RelStructuredTypeFlattener.java | 8 ++--- .../calcite/test/SqlToRelConverterTest.java | 16 ++++++++++ .../calcite/test/SqlToRelConverterTest.xml | 30 +++++++++++++++++++ 3 files changed, 50 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql2rel/RelStructuredTypeFlattener.java b/core/src/main/java/org/apache/calcite/sql2rel/RelStructuredTypeFlattener.java index 6489edb05d70..3ab73813eb28 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/RelStructuredTypeFlattener.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/RelStructuredTypeFlattener.java @@ -229,9 +229,6 @@ private RelNode tryRestructure(RelNode root, RelNode flattened) { .projectNamed(structuringExps, resultFieldNames, true) .build(); restructured = RelOptUtil.copyRelHints(flattened, restructured); - // REVIEW jvs 23-Mar-2005: How do we make sure that this - // implementation stays in Java? Fennel can't handle - // structured types. return restructured; } else { return flattened; @@ -513,7 +510,10 @@ public void rewriteRel(LogicalCorrelate rel) { } public void rewriteRel(Collect rel) { - rewriteGeneric(rel); + // Flattening does not rewrite collection element types + final RelNode newInput = + tryRestructure(rel.getInput(), getNewForOldRel(rel.getInput())); + setNewForOldRel(rel, rel.copy(rel.getTraitSet(), newInput)); } public void rewriteRel(Uncollect rel) { diff --git a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java index 05093c2d3c2e..fb814f3e889d 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java @@ -4635,6 +4635,22 @@ void checkCorrelatedMapSubQuery(boolean expand) { sql(sql).ok(); } + /** Test case for + * [CALCITE-7723] + * Queries using Collect with ROW results throw AssertionFailure. */ + @Test void testArraySubqueryOfNestedRow() { + final String sql = "SELECT ARRAY(SELECT ROW(ROW(1, 2), 3) FROM (VALUES (0)))"; + sql(sql).ok(); + } + + /** Test case for + * [CALCITE-7723] + * Queries using Collect with ROW results throw AssertionFailure. */ + @Test void testMapSubqueryOfNestedRow() { + final String sql = "SELECT MAP(SELECT ROW(1, 2), 'x' FROM (VALUES (0)))"; + sql(sql).ok(); + } + @Test void testArraySubqueryOrderByProjectedField() { final String sql = "SELECT ARRAY(SELECT empno FROM emp ORDER BY empno)"; sql(sql).ok(); diff --git a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml index bafc21fc6261..38479e0da83d 100644 --- a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml @@ -634,6 +634,21 @@ LogicalProject(EXPR$0=[$1]) Collect(field=[EXPR$0]) LogicalProject(EMPNO=[$0]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +]]> + + + + + + + + @@ -5342,6 +5357,21 @@ LogicalProject(DEPTNO=[$7], NAME=[$10]) LogicalJoin(condition=[=($7, $9)], joinType=[left]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) +]]> + + + + + + + + From 5f51eda867e3a03ec687756fbaa1ef555c624e27 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Thu, 13 Aug 2026 17:22:33 -0700 Subject: [PATCH 479/562] [CALCITE-7718] Lambda capturing ROW field crashes at compilation with assertion failure Signed-off-by: Mihai Budiu --- .../org/apache/calcite/plan/RelOptUtil.java | 13 --------- .../calcite/test/SqlToRelConverterTest.java | 28 +++++++++++++++++++ .../calcite/test/SqlToRelConverterTest.xml | 24 ++++++++++++++++ core/src/test/resources/sql/lambda.iq | 26 +++++++++++++++++ 4 files changed, 78 insertions(+), 13 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java b/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java index 97146b625aa3..42e86b5c4be5 100644 --- a/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java +++ b/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java @@ -70,7 +70,6 @@ import org.apache.calcite.rex.RexExecutorImpl; import org.apache.calcite.rex.RexFieldAccess; import org.apache.calcite.rex.RexInputRef; -import org.apache.calcite.rex.RexLambda; import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexLocalRef; import org.apache.calcite.rex.RexNode; @@ -3371,12 +3370,6 @@ private static RexShuttle pushShuttle(final Project project) { @Override public RexNode visitInputRef(RexInputRef ref) { return project.getProjects().get(ref.getIndex()); } - - @Override public RexNode visitLambda(RexLambda lambda) { - // Lambda body references are at a different scope level. - // Do not remap indices inside lambda body against this project. - return lambda; - } }; } @@ -3400,12 +3393,6 @@ private static RexShuttle pushShuttle(final Calc calc) { @Override public RexNode visitInputRef(RexInputRef ref) { return projects.get(ref.getIndex()); } - - @Override public RexNode visitLambda(RexLambda lambda) { - // Lambda body references are at a different scope level. - // Do not remap indices inside lambda body against this calc. - return lambda; - } }; } diff --git a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java index fb814f3e889d..d56b7d54375b 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java @@ -158,6 +158,34 @@ public static void checkActualAndReferenceFiles() { } + /** Test case for + * [CALCITE-7718] + * Lambda capturing ROW field crashes at compilation with assertion + * failure. */ + @Test void testLambdaExpressionWithStructCaptureMerge() { + final String sql = "select \"EXISTS\"(array(1, 2), x -> x = t.r.\"EXPR$0\")\n" + + "from (select ROW(1, 2) as r) as t"; + fixture() + .withFactory(c -> + c.withOperatorTable(t -> SqlValidatorTest.operatorTableFor(SqlLibrary.SPARK))) + .withSql(sql) + .ok(); + } + + /** Test case for + * [CALCITE-7718] + * Lambda capturing ROW field crashes at compilation with assertion + * failure. */ + @Test void testLambdaExpressionWithStructCaptureMergeOverScan() { + final String sql = "select \"EXISTS\"(array(1, 2), x -> x = t.r.\"EXPR$0\")\n" + + "from (select ROW(deptno, sal) as r from emp) as t"; + fixture() + .withFactory(c -> + c.withOperatorTable(t -> SqlValidatorTest.operatorTableFor(SqlLibrary.SPARK))) + .withSql(sql) + .ok(); + } + /** Test case for * [CALCITE-3679] * Allow lambda expressions in SQL queries. */ diff --git a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml index 38479e0da83d..0d97150eb63d 100644 --- a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml @@ -5166,6 +5166,30 @@ LogicalProject(EXPR$0=[HIGHER_ORDER_FUNCTION($7, (X, DEPTNO) -> +(DEPTNO, 1))]) OR(=(N, 1), =(N, 3)))]) LogicalValues(tuples=[[{ 0 }]]) +]]> + + + + + x = t.r."EXPR$0") +from (select ROW(1, 2) as r) as t]]> + + + =(X, ROW(1, 2).EXPR$0))]) + LogicalValues(tuples=[[{ 0 }]]) +]]> + + + + + x = t.r."EXPR$0") +from (select ROW(deptno, sal) as r from emp) as t]]> + + + =(X, ROW($7, $5).EXPR$0))]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) ]]> diff --git a/core/src/test/resources/sql/lambda.iq b/core/src/test/resources/sql/lambda.iq index 82808ff19dad..accc60b4e8f4 100644 --- a/core/src/test/resources/sql/lambda.iq +++ b/core/src/test/resources/sql/lambda.iq @@ -145,3 +145,29 @@ select "EXISTS"(array(1, 2, 3), x -> "EXISTS"(array(1, 2, 3), y -> x + y = 4)) a (1 row) !ok + +# [CALCITE-7718] Lambda capturing ROW field crashes at compilation with +# assertion failure. +# Lambda captures a field of a struct value from the enclosing query. +select "EXISTS"(array(1, 2), x -> x = t.r."EXPR$0") from (select ROW(1, 2) as r) as t; ++--------+ +| EXPR$0 | ++--------+ +| true | ++--------+ +(1 row) + +!ok + +# Same, but the captured struct is built from table columns +# Jane is in dept 10 +select "EXISTS"(array(5, 10), x -> x = t.r."EXPR$0") +from (select ROW(deptno, 1) as r from emp where ename = 'Jane') as t; ++--------+ +| EXPR$0 | ++--------+ +| true | ++--------+ +(1 row) + +!ok From d9b8de6e6c321da9a1cc5be5903046b740845fe7 Mon Sep 17 00:00:00 2001 From: Sean Broeder Date: Wed, 19 Aug 2026 06:54:26 -0700 Subject: [PATCH 480/562] [CALCITE-7724] SqlUtil#lookupSubjectRoutines rejects a valid operator when getFunctionKind() remaps its kind and 2+ candidates share a name Now map both sides of the comparison through getFunctionKind() before comparing. --- .../java/org/apache/calcite/sql/SqlUtil.java | 5 +++- .../apache/calcite/test/SqlValidatorTest.java | 24 +++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/calcite/sql/SqlUtil.java b/core/src/main/java/org/apache/calcite/sql/SqlUtil.java index b5c91a1c7ad0..1dba9d573e33 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlUtil.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlUtil.java @@ -558,9 +558,12 @@ public static SqlLiteral concatenateLiterals(List lits) { private static Iterator filterOperatorRoutinesByKind( Iterator routines, final SqlKind sqlKind) { + // Mirror getFunctionKind() on both sides, or an operator whose kind maps to + // something else (e.g. POSITION -> OTHER_FUNCTION) can fail to match itself. + final SqlKind sqlFunctionKind = sqlKind.getFunctionKind(); return Iterators.filter(routines, operator -> requireNonNull(operator, "operator") - .getKind().getFunctionKind() == sqlKind); + .getKind().getFunctionKind() == sqlFunctionKind); } /** diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index 38f883743ecf..f0494539f0e5 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -1043,6 +1043,30 @@ void testDyadicCollateOperator() { .fails("Parameters must be of the same type"); } + /** Test case for + * [CALCITE-7724] SqlUtil#lookupSubjectRoutines rejects a valid operator when its + * SqlKind is remapped by SqlKind#getFunctionKind() and two operator-table entries + * resolve to it. + * + *

      The kind-based fourth pass in {@code filterOperatorRoutinesByKind} maps only + * the candidate's kind through {@code getFunctionKind()}, not the call's own kind - + * so an operator whose kind is remapped (e.g. {@link SqlKind#POSITION}) can fail to + * match itself once a second candidate for the same name exists. */ + @Test void testFunctionKindMismatchWithDuplicateOperatorTableEntry() { + // Chaining the operator table with itself ensures that each appears twice. + final SqlOperatorTable duplicated = + SqlOperatorTables.chain(SqlStdOperatorTable.instance(), SqlStdOperatorTable.instance()); + expr("position('mouse' in 'house')") + .withOperatorTable(duplicated) + .ok(); + expr("char_length('string')") + .withOperatorTable(duplicated) + .ok(); + expr("character_length('string')") + .withOperatorTable(duplicated) + .ok(); + } + @Test void testTrim() { expr("trim('mustache' FROM 'beard')").ok(); expr("trim(both 'mustache' FROM 'beard')").ok(); From 4d4b1119c83316120bff2cf630538817a97532c6 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Tue, 18 Aug 2026 20:19:45 -0700 Subject: [PATCH 481/562] [CALCITE-7727] Comparing UUID <> '' always returns FALSE Signed-off-by: Mihai Budiu --- .../calcite/jdbc/JavaTypeFactoryImpl.java | 3 + .../calcite/rel/externalize/RelJson.java | 3 +- .../apache/calcite/runtime/SqlFunctions.java | 66 ++++- .../calcite/sql/parser/SqlParserUtil.java | 3 +- .../implicit/AbstractTypeCoercion.java | 21 +- .../apache/calcite/util/BuiltInMethod.java | 2 +- .../apache/calcite/plan/RelWriterTest.java | 4 +- .../apache/calcite/test/TypeCoercionTest.java | 22 ++ core/src/test/resources/sql/misc.iq | 252 +++++++++++++++++- site/_docs/history.md | 18 ++ site/_docs/reference.md | 5 +- .../apache/calcite/test/CalciteAssert.java | 22 +- 12 files changed, 392 insertions(+), 29 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/jdbc/JavaTypeFactoryImpl.java b/core/src/main/java/org/apache/calcite/jdbc/JavaTypeFactoryImpl.java index 58882f6a1b2d..2114921971d4 100644 --- a/core/src/main/java/org/apache/calcite/jdbc/JavaTypeFactoryImpl.java +++ b/core/src/main/java/org/apache/calcite/jdbc/JavaTypeFactoryImpl.java @@ -53,6 +53,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.UUID; import java.util.stream.Collectors; import static org.apache.calcite.util.ReflectUtil.isStatic; @@ -226,6 +227,8 @@ private static Type fieldType(Field field) { return ByteString.class; case GEOMETRY: return Geometry.class; + case UUID: + return UUID.class; case SYMBOL: return Enum.class; case ANY: diff --git a/core/src/main/java/org/apache/calcite/rel/externalize/RelJson.java b/core/src/main/java/org/apache/calcite/rel/externalize/RelJson.java index 51c178db063d..8fa5abd149d8 100644 --- a/core/src/main/java/org/apache/calcite/rel/externalize/RelJson.java +++ b/core/src/main/java/org/apache/calcite/rel/externalize/RelJson.java @@ -53,6 +53,7 @@ import org.apache.calcite.rex.RexWindowBound; import org.apache.calcite.rex.RexWindowBounds; import org.apache.calcite.rex.RexWindowExclusion; +import org.apache.calcite.runtime.SqlFunctions; import org.apache.calcite.sql.SqlAggFunction; import org.apache.calcite.sql.SqlFunction; import org.apache.calcite.sql.SqlIdentifier; @@ -874,7 +875,7 @@ public RexNode toRex(RelOptCluster cluster, Object o) { } else if (sqlTypeName == SqlTypeName.BINARY || sqlTypeName == SqlTypeName.VARBINARY) { literal = ByteString.of((String) literal, 16); } else if (sqlTypeName == SqlTypeName.UUID) { - literal = UUID.fromString((String) literal); + literal = SqlFunctions.stringToUuid((String) literal); } return rexBuilder.makeLiteral(literal, type); } diff --git a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java index b797264a8bb8..d50d7279a538 100644 --- a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java +++ b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java @@ -43,6 +43,7 @@ import org.apache.calcite.sql.SqlIntervalQualifier; import org.apache.calcite.sql.SqlUtil; import org.apache.calcite.sql.fun.SqlLibraryOperators; +import org.apache.calcite.sql.parser.SqlParserUtil; import org.apache.calcite.util.NumberUtil; import org.apache.calcite.util.TimeWithTimeZoneString; import org.apache.calcite.util.TimestampWithTimeZoneString; @@ -351,9 +352,70 @@ public static String uuidToString(UUID uuid) { return uuid.toString(); } + /** Converts a string to a UUID: 32 hexadecimal digits. All of the following give + * the UUID {@code 123e4567-e89b-12d3-a456-426655440000}: + * + * 

      +   * 123e4567-e89b-12d3-a456-426655440000
+   * 123E4567-E89B-12D3-A456-426655440000
+   * 123e4567e89b12d3a456426655440000
+   * {123e4567-e89b-12d3-a456-426655440000}
+   * {123e4567e89b12d3a456426655440000}
+   * 123e-4567-e89b-12d3-a456-4266-5544-0000
+   * 123e4567-e89b12d3-a4564266-55440000
+   * 123e-4567e89b-12d3a456426655440000
+   *
      + * + *

      and each of the following is an error: + * + * 

      +   * 1-2-3-4-5                              a group is not four digits wide
+   * 123e456-7e89b-12d3-a456-426655440000   as above, though 36 characters long
+   * 123e4567--e89b-12d3-a456-426655440000  empty group
+   * -123e4567e89b12d3a456426655440000      leading hyphen
+   * 123e4567e89b12d3a456426655440000-      trailing hyphen
+   * {123e4567-e89b-12d3-a456-426655440000  unbalanced brace
+   * 123e4567-e89b-12d3-a456-42665544000    31 digits
+   *
      + * + *

      Blanks are never trimmed. + */ + public static UUID stringToUuid(String s) { + String body = s; + if (body.length() > 1 + && body.charAt(0) == '{' + && body.charAt(body.length() - 1) == '}') { + body = body.substring(1, body.length() - 1); + } + final StringBuilder digits = new StringBuilder(32); + for (int i = 0; i < body.length(); i++) { + final char c = body.charAt(i); + if (c == '-') { + // A hyphen separates groups, so it must follow a complete group of four + // digits and cannot be the last character + if (digits.length() == 0 + || digits.length() % 4 != 0 + || digits.length() == 32 + || body.charAt(i - 1) == '-') { + throw new IllegalArgumentException("Invalid UUID string: " + s); + } + } else if (SqlParserUtil.isHexDigit(c) && digits.length() < 32) { + digits.append(c); + } else { + throw new IllegalArgumentException("Invalid UUID string: " + s); + } + } + if (digits.length() != 32) { + throw new IllegalArgumentException("Invalid UUID string: " + s); + } + return new UUID( + Long.parseUnsignedLong(digits.substring(0, 16), 16), + Long.parseUnsignedLong(digits.substring(16), 16)); + } + public static UUID binaryToUuid(ByteString bytes) { - if (bytes.length() < 16) { - throw new IllegalArgumentException("Need at least 16 bytes for UUID"); + if (bytes.length() != 16) { + throw new IllegalArgumentException("Need exactly 16 bytes for UUID"); } ByteBuffer byteBuffer = ByteBuffer.wrap(bytes.getBytes()); long mostSignificantBits = byteBuffer.getLong(); diff --git a/core/src/main/java/org/apache/calcite/sql/parser/SqlParserUtil.java b/core/src/main/java/org/apache/calcite/sql/parser/SqlParserUtil.java index 4b1ede7e570e..c6eed1bce217 100644 --- a/core/src/main/java/org/apache/calcite/sql/parser/SqlParserUtil.java +++ b/core/src/main/java/org/apache/calcite/sql/parser/SqlParserUtil.java @@ -21,6 +21,7 @@ import org.apache.calcite.config.CalciteSystemProperty; import org.apache.calcite.rel.type.RelDataTypeSystem; import org.apache.calcite.runtime.CalciteContextException; +import org.apache.calcite.runtime.SqlFunctions; import org.apache.calcite.sql.SqlBinaryOperator; import org.apache.calcite.sql.SqlCall; import org.apache.calcite.sql.SqlDateLiteral; @@ -407,7 +408,7 @@ public static SqlTimestampLiteral parseTimestampWithLocalTimeZoneLiteral( } public static SqlUuidLiteral parseUuidLiteral(String s, SqlParserPos pos) { - UUID uuid = UUID.fromString(s); + UUID uuid = SqlFunctions.stringToUuid(s); return SqlLiteral.createUuid(uuid, pos); } diff --git a/core/src/main/java/org/apache/calcite/sql/validate/implicit/AbstractTypeCoercion.java b/core/src/main/java/org/apache/calcite/sql/validate/implicit/AbstractTypeCoercion.java index 32d39229b500..22f6e16c61f1 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/implicit/AbstractTypeCoercion.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/implicit/AbstractTypeCoercion.java @@ -291,13 +291,17 @@ protected boolean needToCast(SqlValidatorScope scope, SqlNode node, return false; } - // No casts to binary except from strings - if (SqlTypeUtil.isBinary(fromType) && !SqlTypeUtil.isString(toType)) { + // No casts from binary except to strings and UUID + if (SqlTypeUtil.isBinary(fromType) + && !SqlTypeUtil.isString(toType) + && toType.getSqlTypeName() != SqlTypeName.UUID) { return false; } - // No casts from binary except to strings - if (SqlTypeUtil.isBinary(toType) && !SqlTypeUtil.isString(fromType)) { + // No casts to binary except from strings and UUID + if (SqlTypeUtil.isBinary(toType) + && !SqlTypeUtil.isString(fromType) + && fromType.getSqlTypeName() != SqlTypeName.UUID) { return false; } @@ -525,14 +529,15 @@ private RelDataType getTightestCommonTypeOrThrow( return factory.leastRestrictive(ImmutableList.of(type1, type2)); } + // CHARACTER or BINARY < UUID -> UUID, similar to CHAR < INT -> INT if ((SqlTypeUtil.isCharacter(type1) || SqlTypeUtil.isBinary(type1)) - && type2.getSqlTypeName() == SqlTypeName.UUID) { - return factory.createTypeWithNullability(type1, anyNullable); + && typeName2 == SqlTypeName.UUID) { + return factory.createTypeWithNullability(type2, anyNullable); } if ((SqlTypeUtil.isCharacter(type2) || SqlTypeUtil.isBinary(type2)) - && type1.getSqlTypeName() == SqlTypeName.UUID) { - return factory.createTypeWithNullability(type2, anyNullable); + && typeName1 == SqlTypeName.UUID) { + return factory.createTypeWithNullability(type1, anyNullable); } // DATETIME < CHARACTER -> DATETIME diff --git a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java index bbce7d8de444..948b62f28816 100644 --- a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java +++ b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java @@ -546,7 +546,7 @@ public enum BuiltInMethod { IS_JSON_ARRAY(JsonFunctions.class, "isJsonArray", String.class), IS_JSON_SCALAR(JsonFunctions.class, "isJsonScalar", String.class), ST_GEOM_FROM_EWKT(SpatialTypeFunctions.class, "ST_GeomFromEWKT", String.class), - UUID_FROM_STRING(UUID.class, "fromString", String.class), + UUID_FROM_STRING(SqlFunctions.class, "stringToUuid", String.class), UUID_TO_STRING(SqlFunctions.class, "uuidToString", UUID.class), UUID_TO_BINARY(SqlFunctions.class, "uuidToBinary", UUID.class), INT_TO_BINARY(SqlFunctions.class, "intToBinary", Object.class, int.class, boolean.class), diff --git a/core/src/test/java/org/apache/calcite/plan/RelWriterTest.java b/core/src/test/java/org/apache/calcite/plan/RelWriterTest.java index 8fd02a8c50d3..a107afd17f0d 100644 --- a/core/src/test/java/org/apache/calcite/plan/RelWriterTest.java +++ b/core/src/test/java/org/apache/calcite/plan/RelWriterTest.java @@ -53,6 +53,7 @@ import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexProgramBuilder; import org.apache.calcite.rex.RexWindowBounds; +import org.apache.calcite.runtime.SqlFunctions; import org.apache.calcite.schema.SchemaPlus; import org.apache.calcite.sql.SqlExplainFormat; import org.apache.calcite.sql.SqlExplainLevel; @@ -99,7 +100,6 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.UUID; import java.util.function.Consumer; import java.util.function.Function; import java.util.stream.Stream; @@ -641,7 +641,7 @@ private static Fixture relFn(Function relFn) { .build(); return b.values(rowType, 0).project( b.getRexBuilder().makeUuidLiteral( - UUID.fromString("123e4567-e89b-12d3-a456-426655440000"))) + SqlFunctions.stringToUuid("123e4567-e89b-12d3-a456-426655440000"))) .build(); }; relFn(relFn) diff --git a/core/src/test/java/org/apache/calcite/test/TypeCoercionTest.java b/core/src/test/java/org/apache/calcite/test/TypeCoercionTest.java index e4f521d0bb2d..32db4091bfe6 100644 --- a/core/src/test/java/org/apache/calcite/test/TypeCoercionTest.java +++ b/core/src/test/java/org/apache/calcite/test/TypeCoercionTest.java @@ -395,6 +395,26 @@ private static ImmutableList combine( f.comparisonCommonType(f.charType, f.varcharType, f.varcharType); f.comparisonCommonType(f.intType, f.charType, f.intType); f.comparisonCommonType(f.doubleType, f.charType, f.doubleType); + // Test cases for [CALCITE-7727] Comparing UUID <> '' always returns FALSE. + final RelDataType char0Type = f.typeFactory.createSqlType(SqlTypeName.CHAR, 0); + final RelDataType char36Type = f.typeFactory.createSqlType(SqlTypeName.CHAR, 36); + final RelDataType char40Type = f.typeFactory.createSqlType(SqlTypeName.CHAR, 40); + final RelDataType binary16Type = + f.typeFactory.createSqlType(SqlTypeName.BINARY, 16); + final RelDataType binary20Type = + f.typeFactory.createSqlType(SqlTypeName.BINARY, 20); + f.comparisonCommonType(f.uuidType, char0Type, f.uuidType); + f.comparisonCommonType(f.uuidType, f.charType, f.uuidType); + f.comparisonCommonType(f.uuidType, char36Type, f.uuidType); + f.comparisonCommonType(f.uuidType, char40Type, f.uuidType); + f.comparisonCommonType(f.uuidType, f.varchar20Type, f.uuidType); + f.comparisonCommonType(f.uuidType, f.varcharType, f.uuidType); + f.comparisonCommonType(f.uuidType, f.binaryType, f.uuidType); + f.comparisonCommonType(f.uuidType, binary16Type, f.uuidType); + f.comparisonCommonType(f.uuidType, binary20Type, f.uuidType); + f.comparisonCommonType(f.uuidType, f.varbinaryType, f.uuidType); + f.comparisonCommonType(f.uuidType, f.uuidType, f.uuidType); + // TIMESTAMP f.comparisonCommonType(f.timestampType, f.timestampType, f.timestampType); f.comparisonCommonType(f.dateType, f.timestampType, f.timestampType); @@ -796,6 +816,7 @@ static class Fixture { final RelDataType nullableVarchar20Type; final RelDataType geometryType; final RelDataType nullableGeometryType; + final RelDataType uuidType; /** Creates a Fixture. */ public static Fixture create(SqlTestFactory testFactory) { @@ -846,6 +867,7 @@ protected Fixture(RelDataTypeFactory typeFactory, nullableVarchar20Type = this.typeFactory.createTypeWithNullability(varchar20Type, true); geometryType = this.typeFactory.createSqlType(SqlTypeName.GEOMETRY); nullableGeometryType = this.typeFactory.createTypeWithNullability(geometryType, true); + uuidType = this.typeFactory.createSqlType(SqlTypeName.UUID); // Initialize category types diff --git a/core/src/test/resources/sql/misc.iq b/core/src/test/resources/sql/misc.iq index f5457a79c7d8..e26ce01159f2 100644 --- a/core/src/test/resources/sql/misc.iq +++ b/core/src/test/resources/sql/misc.iq @@ -87,8 +87,48 @@ SELECT CAST('123e4567-e89b-12d3-a456-426655440000' AS UUID); !ok +# Hyphens are optional separators, so this denotes the same UUID. PostgreSQL +# accepts the same set of spellings. SELECT CAST('123e4567e89b12d3a456426655440000' AS UUID); -java.lang.IllegalArgumentException: Invalid UUID string: 123e4567e89b12d3a456426655440000 ++--------------------------------------+ +| EXPR$0 | ++--------------------------------------+ +| 123e4567-e89b-12d3-a456-426655440000 | ++--------------------------------------+ +(1 row) + +!ok + +SELECT CAST('{123e4567-e89b-12d3-a456-426655440000}' AS UUID); ++--------------------------------------+ +| EXPR$0 | ++--------------------------------------+ +| 123e4567-e89b-12d3-a456-426655440000 | ++--------------------------------------+ +(1 row) + +!ok + +SELECT CAST('123e-4567-e89b-12d3-a456-4266-5544-0000' AS UUID); ++--------------------------------------+ +| EXPR$0 | ++--------------------------------------+ +| 123e4567-e89b-12d3-a456-426655440000 | ++--------------------------------------+ +(1 row) + +!ok + +SELECT CAST('1-2-3-4-5' AS UUID); +java.lang.IllegalArgumentException: Invalid UUID string: 1-2-3-4-5 +!error + +SELECT CAST('123e456-7e89b-12d3-a456-426655440000' AS UUID); +java.lang.IllegalArgumentException: Invalid UUID string: 123e456-7e89b-12d3-a456-426655440000 +!error + +SELECT CAST('123e4567--e89b-12d3-a456-426655440000' AS UUID); +java.lang.IllegalArgumentException: Invalid UUID string: 123e4567--e89b-12d3-a456-426655440000 !error SELECT CAST(UUID '123e4567-e89b-12d3-a456-426655440000' AS VARCHAR); @@ -122,7 +162,7 @@ SELECT CAST(x'123e4567e89b12d3a456426655440000' AS UUID); !ok SELECT CAST(x'00' AS UUID); -java.lang.IllegalArgumentException: Need at least 16 bytes for UUID +java.lang.IllegalArgumentException: Need exactly 16 bytes for UUID !error SELECT UUID '123e4567-e89b-12d3-a456-426655440000' = '123e4567-e89b-12d3-a456-426655440000'; @@ -135,6 +175,214 @@ SELECT UUID '123e4567-e89b-12d3-a456-426655440000' = '123e4567-e89b-12d3-a456-42 !ok +# [CALCITE-7727] Comparing UUID <> '' always returns FALSE. +# Matches PostgreSQL +SELECT UUID '123e4567-e89b-12d3-a456-426655440000' <> '' AS C; +java.lang.IllegalArgumentException: Invalid UUID string: +!error + +# Matches PostgreSQL +SELECT UUID '123e4567-e89b-12d3-a456-426655440000' = '123e4567' AS C; +java.lang.IllegalArgumentException: Invalid UUID string: 123e4567 +!error + +# A trailing blank does not denote a UUID either +# Matches PostgreSQL +SELECT UUID '123e4567-e89b-12d3-a456-426655440000' + = '123e4567-e89b-12d3-a456-426655440000 ' AS C; +java.lang.IllegalArgumentException: Invalid UUID string: 123e4567-e89b-12d3-a456-426655440000 +!error + +# Matches PostgreSQL +SELECT CAST('' AS UUID) AS C; +java.lang.IllegalArgumentException: Invalid UUID string: +!error + +# Matches PostgreSQL +SELECT CAST(' ' AS UUID) AS C; +java.lang.IllegalArgumentException: Invalid UUID string: +!error + +# Blanks are not trimmed +# Matches PostgreSQL +SELECT CAST(' 123e4567-e89b-12d3-a456-426655440000' AS UUID) AS C; +java.lang.IllegalArgumentException: Invalid UUID string: 123e4567-e89b-12d3-a456-426655440000 +!error + +# The CHAR(40) string has extra spaces, so casting it to UUID fails +# Matches PostgreSQL, which rejects char(40) the same way +SELECT UUID '123e4567-e89b-12d3-a456-426655440000' + = CAST('123e4567-e89b-12d3-a456-426655440000' AS CHAR(40)) AS C; +java.lang.IllegalArgumentException: Invalid UUID string: 123e4567-e89b-12d3-a456-426655440000 +!error + +# CHAR(36) is exactly the width of the UUID, so there is no padding +SELECT UUID '123e4567-e89b-12d3-a456-426655440000' + = CAST('123e4567-e89b-12d3-a456-426655440000' AS CHAR(36)) AS C; ++------+ +| C | ++------+ +| true | ++------+ +(1 row) + +!ok + +# Matches PostgreSQL +SELECT UUID '123e4567-e89b-12d3-a456-426655440000' + = '123E4567-E89B-12D3-A456-426655440000' AS C; ++------+ +| C | ++------+ +| true | ++------+ +(1 row) + +!ok + +# IN uses the comparison common type +SELECT UUID '123e4567-e89b-12d3-a456-426655440000' + IN ('123e4567-e89b-12d3-a456-426655440000', + '123E4567-E89B-12D3-A456-426655440001') AS C; ++------+ +| C | ++------+ +| true | ++------+ +(1 row) + +!ok + +# explain +SELECT u <> '' FROM (VALUES (CAST(NULL AS UUID))) AS t(u); +SELECT "T"."U" <> CAST('' AS UUID) +FROM (VALUES ROW(CAST(NULL AS UUID))) AS "T" ("U") +!explain-validated-on Calcite + +SELECT u = f AS EQ_FULL, u = g AS EQ_UPPER, u = b AS EQ_BINARY +FROM (VALUES (CAST('123e4567-e89b-12d3-a456-426655440000' AS UUID), + '123e4567-e89b-12d3-a456-426655440000', + '123E4567-E89B-12D3-A456-426655440000', + x'123e4567e89b12d3a456426655440000'), + (CAST(NULL AS UUID), + '123e4567-e89b-12d3-a456-426655440000', + '123E4567-E89B-12D3-A456-426655440000', + x'123e4567e89b12d3a456426655440000')) + AS t(u, f, g, b); ++---------+----------+-----------+ +| EQ_FULL | EQ_UPPER | EQ_BINARY | ++---------+----------+-----------+ +| true | true | true | +| | | | ++---------+----------+-----------+ +(2 rows) + +!ok + +SELECT CAST(u AS VARCHAR) AS C +FROM (VALUES (CAST('123e4567-e89b-12d3-a456-426655440000' AS UUID)), + (CAST(NULL AS UUID))) AS t(u); ++--------------------------------------+ +| C | ++--------------------------------------+ +| 123e4567-e89b-12d3-a456-426655440000 | +| | ++--------------------------------------+ +(2 rows) + +!ok + +WITH t(u) AS (VALUES (CAST('123e4567-e89b-12d3-a456-426655440000' AS UUID)), + (CAST(NULL AS UUID))) +SELECT CAST(u AS VARBINARY) AS B, u = u AS SELF FROM t; ++----------------------------------+------+ +| B | SELF | ++----------------------------------+------+ +| 123e4567e89b12d3a456426655440000 | true | +| | | ++----------------------------------+------+ +(2 rows) + +!ok + +# UUID columns as grouping and sorting keys +WITH t(u) AS (VALUES (CAST('123e4567-e89b-12d3-a456-426655440000' AS UUID)), + (CAST('123e4567-e89b-12d3-a456-426655440001' AS UUID)), + (CAST('123e4567-e89b-12d3-a456-426655440000' AS UUID)), + (CAST(NULL AS UUID))) +SELECT u, COUNT(*) AS C FROM t GROUP BY u ORDER BY u; ++--------------------------------------+---+ +| U | C | ++--------------------------------------+---+ +| 123e4567-e89b-12d3-a456-426655440000 | 2 | +| 123e4567-e89b-12d3-a456-426655440001 | 1 | +| | 1 | ++--------------------------------------+---+ +(3 rows) + +!ok + +# UUID columns as join keys +WITH t(u) AS (VALUES (CAST('123e4567-e89b-12d3-a456-426655440000' AS UUID)), + (CAST('123e4567-e89b-12d3-a456-426655440001' AS UUID))) +SELECT t1.u FROM t AS t1 JOIN t AS t2 ON t1.u = t2.u ORDER BY 1; ++--------------------------------------+ +| U | ++--------------------------------------+ +| 123e4567-e89b-12d3-a456-426655440000 | +| 123e4567-e89b-12d3-a456-426655440001 | ++--------------------------------------+ +(2 rows) + +!ok + +# Binary compared to UUID +SELECT UUID '123e4567-e89b-12d3-a456-426655440000' = x'123e4567e89b12d3a456426655440000' AS C; ++------+ +| C | ++------+ +| true | ++------+ +(1 row) + +!ok + +SELECT x'123e4567e89b12d3a456426655440000' = UUID '123e4567-e89b-12d3-a456-426655440000' AS C; ++------+ +| C | ++------+ +| true | ++------+ +(1 row) + +!ok + +SELECT UUID '123e4567-e89b-12d3-a456-426655440000' = x'00' AS C; +java.lang.IllegalArgumentException: Need exactly 16 bytes for UUID +!error + +SELECT UUID '123e4567-e89b-12d3-a456-426655440000' + = x'123e4567e89b12d3a456426655440000ff' AS C; +java.lang.IllegalArgumentException: Need exactly 16 bytes for UUID +!error + +SELECT CAST(x'123e4567e89b12d3a456426655440000ff' AS UUID) AS C; +java.lang.IllegalArgumentException: Need exactly 16 bytes for UUID +!error + +# Hyphens are optional, so this string denotes the same UUID. +# Matches PostgreSQL +SELECT UUID '123e4567-e89b-12d3-a456-426655440000' + = '123e4567e89b12d3a456426655440000' AS C; ++------+ +| C | ++------+ +| true | ++------+ +(1 row) + +!ok + SELECT CAST(NULL AS UUID); +--------+ | EXPR$0 | diff --git a/site/_docs/history.md b/site/_docs/history.md index 3aba1fd07588..d1e0caae3a71 100644 --- a/site/_docs/history.md +++ b/site/_docs/history.md @@ -64,6 +64,24 @@ Class loading from model files has been disabled by default. Any attempt to load classes from model files will lead to `SecurityException` unless an appropriate pattern is set in `calcite.model.classes.allowed` system property. +* [CALCITE-7727] +Comparing a `UUID` with a character or binary value now converts that value to a +`UUID`, the same direction as comparing a string with a number or a datetime. +Previously the `UUID` was converted to the other operand's type. A value that does not +denote a `UUID` is now an error rather than a comparison that silently fails. + +* [CALCITE-7727] +Converting a string to a `UUID` now follows PostgreSQL: 32 hexadecimal digits of +either case, optionally enclosed in braces, optionally separated by a hyphen +after any complete group of four digits. Forms such as +`123e4567e89b12d3a456426655440000` and `{123e4567-e89b-12d3-a456-426655440000}` +are now accepted. Malformed strings are now rejected instead of being converted +to a different `UUID`; `java.util.UUID.fromString`, used previously, does not +check the width of each group, and turned `1-2-3-4-5` into +`00000001-0002-0003-0004-000000000005`. Converting a binary to a `UUID` now +requires exactly 16 bytes; a longer value used to be truncated. Blanks are not +trimmed. + #### New features {: #new-features-1-43-0} diff --git a/site/_docs/reference.md b/site/_docs/reference.md index 59ddce4af544..dd385a4c8741 100644 --- a/site/_docs/reference.md +++ b/site/_docs/reference.md @@ -1299,7 +1299,7 @@ name will have been converted to upper case also. | TIMESTAMP [ WITHOUT TIME ZONE ] | Date and time | Example: TIMESTAMP '1969-07-20 20:17:40' | TIMESTAMP WITH LOCAL TIME ZONE | Date and time with local time zone | Example: TIMESTAMP WITH LOCAL TIME ZONE '1969-07-20 20:17:40' | TIMESTAMP WITH TIME ZONE | Date and time with time zone | Example: TIMESTAMP WITH TIME ZONE '1969-07-20 20:17:40 America/Los Angeles' -| UUID | An 128-bit UUID | Example: UUID '123e4567-e89b-12d3-a456-426655440000' +| UUID | An 128-bit UUID | Example: UUID '123e4567-e89b-12d3-a456-426655440000'. A string converts to a `UUID` if it holds 32 hexadecimal digits of either case, optionally enclosed in braces, optionally separated by a hyphen after any complete group of four digits; a binary converts if it is exactly 16 bytes. Anything else is an error. Blanks are not trimmed. | INTERVAL timeUnit [ TO timeUnit ] | Date time interval | Examples: INTERVAL '1-5' YEAR TO MONTH, INTERVAL '45' DAY, INTERVAL '1 2:34:56.789' DAY TO SECOND | GEOMETRY | Geometry | Examples: ST_GeomFromText('POINT (30 10)') @@ -1825,7 +1825,8 @@ i: implicit cast / e: explicit cast / x: not allowed * Binary comparison (`=`, `<`, `<=`, `<>`, `>`, `>=`): if operands are `STRING` and `TIMESTAMP`, promote to `TIMESTAMP`; make `1 = true` and `0 = false` always evaluate to `TRUE`; - if there is numeric type operand, find common type for both operands. + if there is numeric type operand, find common type for both operands; + if operands are `UUID` and `CHARACTER` or `BINARY`, promote to `UUID`. * `IN` sub-query: compare type of LHS and RHS, and find the common type; if it is struct type, find wider type for every field; * `IN` expression list: compare every expression to find the common type; diff --git a/testkit/src/main/java/org/apache/calcite/test/CalciteAssert.java b/testkit/src/main/java/org/apache/calcite/test/CalciteAssert.java index cbc2f6c7999c..aa27c7b77a8c 100644 --- a/testkit/src/main/java/org/apache/calcite/test/CalciteAssert.java +++ b/testkit/src/main/java/org/apache/calcite/test/CalciteAssert.java @@ -155,6 +155,12 @@ public class CalciteAssert { private CalciteAssert() {} + // Define string constants before DB to prevent recursive + // static initializers + private static final String TEST_MYSQL_URL = "jdbc:mysql://localhost/foodmart"; + + private static final String TEST_MYSQL_DRIVER = "com.mysql.jdbc.Driver"; + /** * Which database to use for tests that require a JDBC data source. * @@ -163,10 +169,6 @@ private CalciteAssert() {} public static final DatabaseInstance DB = DatabaseInstance.valueOf(CalciteSystemProperty.TEST_DB.value()); - private static String testMysqlUrl = "jdbc:mysql://localhost/foodmart"; - - private static String testMysqlDriver = "com.mysql.jdbc.Driver"; - /** Implementation of {@link AssertThat} that does nothing. */ private static final AssertThat DISABLED = new AssertThat(ConnectionFactories.empty(), ImmutableList.of()) { @@ -2056,14 +2058,14 @@ public enum DatabaseInstance { + "/h2/target/foodmart;user=foodmart;password=foodmart", "foodmart", "foodmart", "org.h2.Driver", "foodmart"), null, null), MYSQL( - new ConnectionSpec(testMysqlUrl, "foodmart", - "foodmart", testMysqlDriver, "foodmart"), null, null), + new ConnectionSpec(TEST_MYSQL_URL, "foodmart", + "foodmart", TEST_MYSQL_DRIVER, "foodmart"), null, null), STARROCKS( - new ConnectionSpec(testMysqlUrl, "foodmart", - "foodmart", testMysqlDriver, "foodmart"), null, null), + new ConnectionSpec(TEST_MYSQL_URL, "foodmart", + "foodmart", TEST_MYSQL_DRIVER, "foodmart"), null, null), DORIS( - new ConnectionSpec(testMysqlUrl, "foodmart", - "foodmart", testMysqlDriver, "foodmart"), null, null), + new ConnectionSpec(TEST_MYSQL_URL, "foodmart", + "foodmart", TEST_MYSQL_DRIVER, "foodmart"), null, null), ORACLE( new ConnectionSpec("jdbc:oracle:thin:@localhost:1521:XE", "foodmart", "foodmart", "oracle.jdbc.OracleDriver", "FOODMART"), null, null), From 53c5244d5e2927cdda68ddba9c522d86e145865f Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Tue, 18 Aug 2026 22:45:10 -0700 Subject: [PATCH 482/562] [CALCITE-7728] Linq4j can simplify expressions without regards for 'safety' Signed-off-by: Mihai Budiu --- .../adapter/enumerable/RexImpTable.java | 10 +- .../calcite/linq4j/tree/BlockBuilder.java | 10 + .../calcite/linq4j/tree/Expressions.java | 244 ++++++++++++++++++ .../calcite/linq4j/tree/OptimizeShuttle.java | 46 +++- .../calcite/linq4j/test/BlockBuilderTest.java | 70 +++++ .../calcite/linq4j/test/ExpressionTest.java | 88 +++++++ .../calcite/linq4j/test/OptimizerTest.java | 80 ++++++ 7 files changed, 534 insertions(+), 14 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java index 36fd19f67bb5..ed7eac4cde9d 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java @@ -5100,14 +5100,18 @@ private static class QuantifyCollectionImplementor extends AbstractRexCallImplem final RexCall binaryImplementorRexCall = (RexCall) translator.builder.makeCall(call.getParserPosition(), binaryOperator, leftRex, translator.builder.makeDynamicParam(rightComponentType, 0)); + // The comparison is evaluated inside the lambda, and it reads the lambda + // parameter, so its statements must go into the lambda's block + final RexToLixTranslator lambdaTranslator = translator.setBlock(lambdaBuilder); final List binaryImplementorArgs = ImmutableList.of( new RexToLixTranslator.Result( - genIsNullStatement(translator, leftExpr), leftExpr), + genIsNullStatement(lambdaTranslator, leftExpr), leftExpr), new RexToLixTranslator.Result( - genIsNullStatement(translator, lambdaArg), lambdaArg)); + genIsNullStatement(lambdaTranslator, lambdaArg), lambdaArg)); final RexToLixTranslator.Result condition = - binaryImplementor.implement(translator, binaryImplementorRexCall, binaryImplementorArgs); + binaryImplementor.implement(lambdaTranslator, binaryImplementorRexCall, + binaryImplementorArgs); lambdaBuilder.add(Expressions.return_(null, condition.valueVariable)); final FunctionExpression predicate = Expressions.lambda(lambdaBuilder.toBlock(), lambdaArg); diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/BlockBuilder.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/BlockBuilder.java index febc454d2712..28aab938a104 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/BlockBuilder.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/BlockBuilder.java @@ -258,6 +258,8 @@ protected boolean isSimpleExpression(@Nullable Expression expr) { if (expr instanceof UnaryExpression) { UnaryExpression una = (UnaryExpression) expr; return una.getNodeType() == ExpressionType.Convert + // A cast may raise ClassCastException, or unbox a null + && !Expressions.mayThrow(una) && isSimpleExpression(una.expression); } return false; @@ -408,6 +410,14 @@ private boolean optimize(Shuttle optimizer, boolean performInline) { // anonymous classes. count = Integer.MAX_VALUE; } + if (count == 0 + && statement.initializer != null + && Expressions.mayThrow(statement.initializer)) { + // Never read, but computing the value may raise a runtime error that + // the program is expected to raise. Keep the declaration, and treat + // it like any other statement that cannot be inlined. + count = 100; + } Expression normalized = normalizeDeclaration(statement); expressionForReuse.remove(normalized); switch (count) { diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/Expressions.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/Expressions.java index ebebd2ca6921..5ba55093b651 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/Expressions.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/Expressions.java @@ -32,6 +32,7 @@ import java.lang.reflect.Field; import java.lang.reflect.Member; import java.lang.reflect.Method; +import java.lang.reflect.Modifier; import java.lang.reflect.Type; import java.math.BigDecimal; import java.math.BigInteger; @@ -515,6 +516,20 @@ public static boolean isConstantNull(Expression e) { && ((ConstantExpression) e).value == null; } + /** Returns whether evaluating a node may cause a runtime error, for example + * a division by zero or an arithmetic overflow. + * + *

      An optimization must not discard a node that may throw, even when its + * value is unused: the error is part of the meaning of the program. It is + * still free to discard a node that Java would not have evaluated anyway, + * such as the untaken branch of {@code true ? x : y}. + */ + public static boolean mayThrow(Node node) { + final MayThrowVisitor visitor = new MayThrowVisitor(); + node.accept(visitor); + return visitor.mayThrow; + } + /** * Creates a ConditionalExpression that represents a conditional * statement. @@ -3290,6 +3305,235 @@ public interface FluentList extends List { FluentList appendAll(T... ts); } + /** Visitor that detects whether a node may cause a runtime error. + * + *

      The analysis is conservative: a false positive is a missed + * simplification, but a false negative is a lost runtime error. + * + *

      These nodes may throw: + * + *

        + *
      • calling anything - a method, a constructor, or a function value - as + * the callee decides whether to throw, and calling on a null target raises + * {@link NullPointerException}; + *
      • reading an array element may raise + * {@link ArrayIndexOutOfBoundsException} or {@link NullPointerException}; + *
      • reading an instance field if the target is null; + *
      • creating an array may raise {@link NegativeArraySizeException}; + *
      • a cast that is not statically known to succeed may raise + * {@link ClassCastException}; + *
      • unboxing raises {@link NullPointerException} on a null box; + * may implied by an operator, e.g. {@code integer + 1}; + *
      • division and remainder (divide by zero), and checked + * arithmetic, which may overflow; + *
      • a {@code throw}, and a {@code try} whose body may throw. + *
      + * + *

      Everything else - reading a variable or a static field, comparing two + * references, {@code instanceof}, string concatenation, and Java arithmetic + * that wraps around - is assumed not to throw. An unrecognized node is unsafe. + * + * @see #mayThrow(Node) */ + private static class MayThrowVisitor extends VisitorImpl<@Nullable Void> { + boolean mayThrow = false; + + @Override public @Nullable Void visit(MethodCallExpression call) { + mayThrow = true; + return super.visit(call); + } + + @Override public @Nullable Void visit(InvocationExpression invocation) { + mayThrow = true; + return super.visit(invocation); + } + + @Override public @Nullable Void visit(DynamicExpression dynamic) { + mayThrow = true; + return super.visit(dynamic); + } + + @Override public @Nullable Void visit(NewExpression newExpression) { + mayThrow = true; + return super.visit(newExpression); + } + + @Override public @Nullable Void visit(NewArrayExpression newArray) { + mayThrow = true; + return super.visit(newArray); + } + + @Override public @Nullable Void visit(ListInitExpression listInit) { + mayThrow = true; + return super.visit(listInit); + } + + @Override public @Nullable Void visit(MemberInitExpression memberInit) { + mayThrow = true; + return super.visit(memberInit); + } + + @Override public @Nullable Void visit(IndexExpression indexExpression) { + mayThrow = true; + return super.visit(indexExpression); + } + + @Override public @Nullable Void visit(MemberExpression member) { + if (!Modifier.isStatic(member.field.getModifiers())) { + mayThrow = true; + } + return super.visit(member); + } + + @Override public @Nullable Void visit(ThrowStatement throwStatement) { + mayThrow = true; + return super.visit(throwStatement); + } + + @Override public @Nullable Void visit(TryStatement tryStatement) { + mayThrow = true; + return super.visit(tryStatement); + } + + @Override public @Nullable Void visit(BinaryExpression binary) { + final Type left = binary.expression0.getType(); + final Type right = binary.expression1.getType(); + switch (binary.getNodeType()) { + case Assign: + case Coalesce: + break; + case Equal: + case NotEqual: + // Comparing a primitive with a reference unboxes the reference; + // comparing two references compares them by identity. + if (Primitive.is(left) != Primitive.is(right)) { + mayThrow = true; + } + break; + case Add: + // "+" is concatenation, not addition, if either operand is a String + if (left == String.class || right == String.class) { + break; + } + // fall through + case AddAssign: + case And: + case AndAlso: + case AndAssign: + case ExclusiveOr: + case ExclusiveOrAssign: + case GreaterThan: + case GreaterThanOrEqual: + case LeftShift: + case LeftShiftAssign: + case LessThan: + case LessThanOrEqual: + case Multiply: + case MultiplyAssign: + case Or: + case OrAssign: + case OrElse: + case Power: + case PowerAssign: + case RightShift: + case RightShiftAssign: + case Subtract: + case SubtractAssign: + // The operator itself cannot fail, but it may unbox an operand + if (!Primitive.is(left) || !Primitive.is(right)) { + mayThrow = true; + } + break; + case Divide: + case DivideAssign: + case DivideChecked: + case Mod: + case Modulo: + case ModuloAssign: + // May divide by zero + mayThrow = true; + break; + case AddAssignChecked: + case AddChecked: + case MultiplyAssignChecked: + case MultiplyChecked: + case SubtractAssignChecked: + case SubtractChecked: + // May overflow + mayThrow = true; + break; + default: + // A node type that no one has classified yet + mayThrow = true; + break; + } + return super.visit(binary); + } + + @Override public @Nullable Void visit(UnaryExpression unary) { + final Type operand = unary.expression.getType(); + switch (unary.getNodeType()) { + case Quote: + case TypeAs: + break; + case Decrement: + case Increment: + case IsFalse: + case IsTrue: + case Negate: + case Not: + case OnesComplement: + case PostDecrementAssign: + case PostIncrementAssign: + case PreDecrementAssign: + case PreIncrementAssign: + case UnaryPlus: + // The operator itself cannot fail, but it unboxes its operand. + if (!Primitive.is(operand)) { + mayThrow = true; + } + break; + case Convert: + if (!castAlwaysSucceeds(operand, unary.getType())) { + mayThrow = true; + } + break; + case ConvertChecked: + case NegateChecked: + // May overflow + mayThrow = true; + break; + case Unbox: + // Unboxing a null raises NullPointerException + mayThrow = true; + break; + case ArrayLength: + // Reads a field of an array, which may be null + mayThrow = true; + break; + default: + // A node type that no one has classified yet + mayThrow = true; + break; + } + return super.visit(unary); + } + + /** Returns whether a cast from {@code from} to {@code to} is known to + * succeed. A cast whose source is a primitive cannot fail; + * neither can a widening reference conversion, such + * as {@code (Object) s}. Any other cast may raise + * {@link ClassCastException} or, when unboxing, + * {@link NullPointerException}. */ + private static boolean castAlwaysSucceeds(Type from, Type to) { + if (Primitive.is(from)) { + return true; + } + return from instanceof Class + && to instanceof Class + && ((Class) to).isAssignableFrom((Class) from); + } + } + /** Fluent array list. * * @param element type */ diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/OptimizeShuttle.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/OptimizeShuttle.java index b6ea7e5faace..a5f04cdc3541 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/OptimizeShuttle.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/OptimizeShuttle.java @@ -104,7 +104,8 @@ private static void addComplement(ExpressionType eq, ExpressionType ne) { ? expression1 : expression2; } - if (expression1.equals(expression2)) { + if (expression1.equals(expression2) + && !Expressions.mayThrow(expression0)) { // a ? b : b === b return expression1; } @@ -190,7 +191,10 @@ && eq(cmp.expression1, expression2)) { case Equal: case NotEqual: if (eq(expression0, expression1)) { - return binary.getNodeType() == Equal ? TRUE_EXPR : FALSE_EXPR; + // "a == a" discards the evaluation of "a", so it must not throw + if (!Expressions.mayThrow(expression0)) { + return binary.getNodeType() == Equal ? TRUE_EXPR : FALSE_EXPR; + } } else if (expression0 instanceof ConstantExpression && expression1 instanceof ConstantExpression) { ConstantExpression c0 = (ConstantExpression) expression0; @@ -225,11 +229,11 @@ && eq(cmp.expression1, expression2)) { // fall through case AndAlso: case OrElse: - result = visit0(binary, expression0, expression1); + result = visit0(binary, expression0, expression1, false); if (result != null) { return result; } - result = visit0(binary, expression1, expression0); + result = visit0(binary, expression1, expression0, true); if (result != null) { return result; } @@ -240,18 +244,30 @@ && eq(cmp.expression1, expression2)) { return super.visit(binary, expression0, expression1); } + /** Simplifies a binary expression whose {@code expression0} operand may be a + * constant. + * + *

      {@code evaluated} says whether Java evaluates {@code expression1} before + * the operator produces its result. It is false when {@code expression1} is + * the right operand of {@code &&} or {@code ||}, which short-circuits; + * discarding a short-circuited operand cannot lose a runtime error. */ private @Nullable Expression visit0( BinaryExpression binary, Expression expression0, - Expression expression1) { + Expression expression1, + boolean evaluated) { Boolean always; switch (binary.getNodeType()) { case AndAlso: always = always(expression0); if (always != null) { - return always - ? expression1 - : FALSE_EXPR; + if (always) { + return expression1; + } + // "x && false" still evaluates x + if (!evaluated || !Expressions.mayThrow(expression1)) { + return FALSE_EXPR; + } } break; case OrElse: @@ -259,12 +275,19 @@ && eq(cmp.expression1, expression2)) { if (always != null) { // true or x --> true // false or x --> x - return always - ? TRUE_EXPR - : expression1; + if (!always) { + return expression1; + } + // "x || true" still evaluates x + if (!evaluated || !Expressions.mayThrow(expression1)) { + return TRUE_EXPR; + } } break; case Equal: + // Not guarded by mayThrow: "x == null" for a primitive x does not + // compile, so this simplification is not optional. Evaluation of x is + // preserved by its declaration, which BlockBuilder keeps. if (isConstantNull(expression1) && isKnownNotNull(expression0)) { return FALSE_EXPR; @@ -277,6 +300,7 @@ && isKnownNotNull(expression0)) { } break; case NotEqual: + // See the comment on Equal above if (isConstantNull(expression1) && isKnownNotNull(expression0)) { return TRUE_EXPR; diff --git a/linq4j/src/test/java/org/apache/calcite/linq4j/test/BlockBuilderTest.java b/linq4j/src/test/java/org/apache/calcite/linq4j/test/BlockBuilderTest.java index 181a067f75aa..6950beae82b3 100644 --- a/linq4j/src/test/java/org/apache/calcite/linq4j/test/BlockBuilderTest.java +++ b/linq4j/src/test/java/org/apache/calcite/linq4j/test/BlockBuilderTest.java @@ -66,6 +66,76 @@ public void prepareBuilder() { + "}\n")); } + /** Unit test for + * [CALCITE-7728] + * Linq4j can simplify expressions without regards for 'safety'. + * + *

      A local variable that is never read is removed, unless computing its + * value may raise a runtime error that the program is expected to raise. */ + @Test void testUnusedDeclarationThatMayThrow() { + final ParameterExpression i = Expressions.parameter(int.class, "i"); + b.append("x", Expressions.divide(ONE, i)); + b.add(Expressions.return_(null, TWO)); + assertThat(b.toBlock(), + hasToString("{\n" + + " final int x = 1 / i;\n" + + " return 2;\n" + + "}\n")); + } + + /** Test case for + * [CALCITE-7728] + * Linq4j can simplify expressions without regards for 'safety'. + * + *

      Indexing an array may raise {@link ArrayIndexOutOfBoundsException} or + * {@link NullPointerException}. */ + @Test void testUnusedDeclarationThatIndexesArray() { + final ParameterExpression a = Expressions.parameter(int[].class, "a"); + final ParameterExpression i = Expressions.parameter(int.class, "i"); + b.append("x", Expressions.arrayIndex(a, i)); + b.add(Expressions.return_(null, TWO)); + assertThat(b.toBlock(), + hasToString("{\n" + + " final int x = a[i];\n" + + " return 2;\n" + + "}\n")); + } + + /** Test case for + * [CALCITE-7728] + * Linq4j can simplify expressions without regards for 'safety'. + */ + @Test void testUnusedDeclarationThatCasts() { + final ParameterExpression o = Expressions.parameter(Object.class, "o"); + b.append("x", Expressions.convert_(o, String.class)); + b.add(Expressions.return_(null, TWO)); + // Cast may throw, cannot be removed + assertThat(b.toBlock(), + hasToString("{\n" + + " final String x = (String) o;\n" + + " return 2;\n" + + "}\n")); + } + + /** Test case for + * [CALCITE-7728] + * Linq4j can simplify expressions without regards for 'safety'. + */ + @Test void testUnusedDeclarationThatWidens() { + final ParameterExpression str = Expressions.parameter(String.class, "str"); + b.append("x", Expressions.convert_(str, Object.class)); + // Cast to Object cannot throw, it can be removed + b.add(Expressions.return_(null, TWO)); + assertThat(b.toBlock(), hasToString("{\n return 2;\n}\n")); + } + + @Test void testUnusedDeclarationThatCannotThrow() { + final ParameterExpression i = Expressions.parameter(int.class, "i"); + b.append("x", Expressions.add(ONE, i)); + b.add(Expressions.return_(null, TWO)); + assertThat(b.toBlock(), hasToString("{\n return 2;\n}\n")); + } + @Test void testTestCustomOptimizer() { BlockBuilder b = new BlockBuilder() { @Override protected Shuttle createOptimizeShuttle() { diff --git a/linq4j/src/test/java/org/apache/calcite/linq4j/test/ExpressionTest.java b/linq4j/src/test/java/org/apache/calcite/linq4j/test/ExpressionTest.java index d962204531b2..c75c2a213998 100644 --- a/linq4j/src/test/java/org/apache/calcite/linq4j/test/ExpressionTest.java +++ b/linq4j/src/test/java/org/apache/calcite/linq4j/test/ExpressionTest.java @@ -1717,6 +1717,94 @@ public void checkBlockBuilder(boolean optimizing, String expected) { + ".add(\"1\").build()")); } + /** Test cases for + * [CALCITE-7728] + * Linq4j can simplify expressions without regards for 'safety'. + * + *

      Checks {@link Expressions#mayThrow} for all possible expressions */ + @Test void testMayThrow() { + final ParameterExpression i = Expressions.parameter(int.class, "i"); + final ParameterExpression j = Expressions.parameter(int.class, "j"); + final ParameterExpression o = Expressions.parameter(Object.class, "o"); + final ParameterExpression str = Expressions.parameter(String.class, "str"); + final ParameterExpression box = Expressions.parameter(Integer.class, "box"); + final ParameterExpression a = Expressions.parameter(int[].class, "a"); + final ParameterExpression all = Expressions.parameter(AllType.class, "all"); + + // Reading a variable or a constant + assertMayThrow(i, false); + assertMayThrow(ONE, false); + + // Java arithmetic wraps around; comparison, bit manipulation + assertMayThrow(Expressions.add(i, j), false); + assertMayThrow(Expressions.multiply(i, j), false); + assertMayThrow(Expressions.negate(i), false); + assertMayThrow(Expressions.lessThan(i, j), false); + assertMayThrow(Expressions.leftShift(i, j), false); + assertMayThrow( + Expressions.andAlso(Expressions.lessThan(i, j), + Expressions.equal(i, j)), false); + assertMayThrow(Expressions.typeIs(o, String.class), false); + assertMayThrow(Expressions.add(str, str), false); + assertMayThrow(Expressions.equal(str, o), false); + + // An operator unboxes its operands, and a null box raises + // NullPointerException + assertMayThrow(Expressions.add(box, i), true); + assertMayThrow(Expressions.negate(box), true); + assertMayThrow(Expressions.lessThan(box, i), true); + assertMayThrow(Expressions.equal(box, i), true); + + // Division may divide by zero; checked arithmetic may overflow + assertMayThrow(Expressions.divide(i, j), true); + assertMayThrow(Expressions.modulo(i, j), true); + assertMayThrow(Expressions.addChecked(i, j), true); + assertMayThrow(Expressions.negateChecked(i), true); + + // An operand that may throw infects the whole expression + assertMayThrow(Expressions.add(ONE, Expressions.divide(ONE, i)), true); + assertMayThrow( + Expressions.condition(Expressions.lessThan(i, j), + Expressions.divide(ONE, i), ONE), true); + + // Reading an array element, and the length of an array + assertMayThrow(Expressions.arrayIndex(a, i), true); + assertMayThrow(Expressions.field(a, "length"), true); + + // Reading an instance field; a static field has no target to be null + assertMayThrow(Expressions.field(all, "i"), true); + assertMayThrow(Expressions.field(null, Integer.class, "MAX_VALUE"), false); + + // Calling a method or a constructor, and creating an array + assertMayThrow(Expressions.call(o, "toString"), true); + assertMayThrow(Expressions.new_(Object.class), true); + assertMayThrow(Expressions.newArrayBounds(int.class, 1, i), true); + assertMayThrow(Expressions.newArrayInit(int.class, ONE, TWO), true); + + // A cast that cannot fail: a primitive conversion, boxing, or a widening + // reference conversion + assertMayThrow(Expressions.convert_(i, long.class), false); + assertMayThrow(Expressions.convert_(i, Integer.class), false); + assertMayThrow(Expressions.convert_(str, Object.class), false); + + // Some casts may raise ClassCastException, and unboxing may throw NPE + assertMayThrow(Expressions.convert_(o, String.class), true); + assertMayThrow(Expressions.convert_(box, int.class), true); + assertMayThrow(Expressions.unbox(box, int.class), true); + + // Throwing, and a block that contains a throw + assertMayThrow(Expressions.throw_(Expressions.new_(RuntimeException.class)), + true); + assertMayThrow( + Expressions.block( + Expressions.throw_(Expressions.new_(RuntimeException.class))), + true); + } + + private static void assertMayThrow(Node node, boolean mayThrow) { + assertThat(node.toString(), Expressions.mayThrow(node), is(mayThrow)); + } + /** An enum. */ enum MyEnum { X, diff --git a/linq4j/src/test/java/org/apache/calcite/linq4j/test/OptimizerTest.java b/linq4j/src/test/java/org/apache/calcite/linq4j/test/OptimizerTest.java index 62c89c73277b..8fb0191ec2af 100644 --- a/linq4j/src/test/java/org/apache/calcite/linq4j/test/OptimizerTest.java +++ b/linq4j/src/test/java/org/apache/calcite/linq4j/test/OptimizerTest.java @@ -902,4 +902,84 @@ class OptimizerTest { + " }\n" + "}\n")); } + + /** Unit test for + * [CALCITE-7728] + * Linq4j can simplify expressions without regards for 'safety'. + * + *

      An expression that may throw, such as "1 / i", must survive a + * simplification that would otherwise discard it. It may be discarded when + * Java would not have evaluated it anyway. */ + @Test void testDoNotDiscardExpressionThatMayThrow() { + final ParameterExpression i = Expressions.parameter(int.class, "i"); + final Expression divide = Expressions.equal(Expressions.divide(ONE, i), ONE); + final Expression safe = Expressions.equal(i, ONE); + + // "x && false" evaluates x + assertThat(optimize(Expressions.andAlso(divide, FALSE)), + is("{\n return 1 / i == 1 && false;\n}\n")); + assertThat(optimize(Expressions.andAlso(safe, FALSE)), + is("{\n return false;\n}\n")); + + // "false && x" does not evaluate x + assertThat(optimize(Expressions.andAlso(FALSE, divide)), + is("{\n return false;\n}\n")); + + // "x || true" evaluates x + assertThat(optimize(Expressions.orElse(divide, TRUE)), + is("{\n return 1 / i == 1 || true;\n}\n")); + assertThat(optimize(Expressions.orElse(safe, TRUE)), + is("{\n return true;\n}\n")); + + // "a ? b : b" evaluates a + assertThat(optimize(Expressions.condition(divide, ONE, ONE)), + is("{\n return 1 / i == 1 ? 1 : 1;\n}\n")); + assertThat(optimize(Expressions.condition(safe, ONE, ONE)), + is("{\n return 1;\n}\n")); + + // "a == a" evaluates a + assertThat( + optimize( + Expressions.equal(Expressions.divide(ONE, i), + Expressions.divide(ONE, i))), + is("{\n return 1 / i == 1 / i;\n}\n")); + assertThat(optimize(Expressions.equal(i, i)), + is("{\n return true;\n}\n")); + } + + /** Unit test for + * [CALCITE-7728] + * Linq4j can simplify expressions without regards for 'safety'. + * + *

      Indexing an array may raise {@link ArrayIndexOutOfBoundsException} or + * {@link NullPointerException}. */ + @Test void testDoNotDiscardArrayIndex() { + final ParameterExpression a = Expressions.parameter(int[].class, "a"); + final ParameterExpression i = Expressions.parameter(int.class, "i"); + final Expression index = + Expressions.equal(Expressions.arrayIndex(a, i), ONE); + + // "a[i] == 1 && false" evaluates "a[i] == 1" + assertThat(optimize(Expressions.andAlso(index, FALSE)), + is("{\n return a[i] == 1 && false;\n}\n")); + + // "false && a[i] == 1" does not evaluate "a[i] == 1" + assertThat(optimize(Expressions.andAlso(FALSE, index)), + is("{\n return false;\n}\n")); + + // "a[i] == 1 || true" evaluates "a[i] == 1" + assertThat(optimize(Expressions.orElse(index, TRUE)), + is("{\n return a[i] == 1 || true;\n}\n")); + + // "a[i] == 1 ? 1 : 1" evaluates "a[i] == 1" + assertThat(optimize(Expressions.condition(index, ONE, ONE)), + is("{\n return a[i] == 1 ? 1 : 1;\n}\n")); + + // "a[i] == a[i]" evaluates "a[i]" + assertThat( + optimize( + Expressions.equal(Expressions.arrayIndex(a, i), + Expressions.arrayIndex(a, i))), + is("{\n return a[i] == a[i];\n}\n")); + } } From 5c10db961046d830edbf69c571298bf38432baba Mon Sep 17 00:00:00 2001 From: xuzifu666 <1206332514@qq.com> Date: Sun, 9 Aug 2026 17:43:41 +0800 Subject: [PATCH 483/562] [CALCITE-7701] Support IGNORE NULLS for FIRST_VALUE/LAST_VALUE window functions in the enumerable convention --- .../adapter/enumerable/EnumerableWindow.java | 13 +- .../adapter/enumerable/RexImpTable.java | 100 +++++++++++ .../adapter/enumerable/WinAggContext.java | 5 + core/src/test/resources/sql/winagg.iq | 157 ++++++++++++++++++ 4 files changed, 274 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java index 1b60c561261e..e57d3dd9e8c3 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java @@ -229,7 +229,14 @@ private static void sampleOfTheGeneratedWindowedAggregate() { for (int aggIdx = 0; aggIdx < aggregateCalls.size(); aggIdx++) { AggregateCall call = aggregateCalls.get(aggIdx); if (call.ignoreNulls()) { - throw new UnsupportedOperationException("IGNORE NULLS not supported"); + switch (call.getAggregation().getKind()) { + case FIRST_VALUE: + case LAST_VALUE: + // IGNORE NULLS is implemented for these functions below. + break; + default: + throw new UnsupportedOperationException("IGNORE NULLS not supported"); + } } aggs.add(new AggImpState(aggIdx, call, true, implementorTable)); } @@ -821,6 +828,10 @@ private void declareAndResetState(final JavaTypeFactory typeFactory, @Override public RexWindowExclusion getExclude() { return exclusion; } + + @Override public boolean ignoreNulls() { + return agg.call.ignoreNulls(); + } }; String aggName = "a" + agg.aggIdx; if (CalciteSystemProperty.DEBUG.value()) { diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java index ed7eac4cde9d..ae40e33c1334 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java @@ -35,6 +35,7 @@ import org.apache.calcite.linq4j.tree.OptimizeShuttle; import org.apache.calcite.linq4j.tree.ParameterExpression; import org.apache.calcite.linq4j.tree.Primitive; +import org.apache.calcite.linq4j.tree.Types; import org.apache.calcite.linq4j.tree.UnsignedType; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; @@ -2491,12 +2492,111 @@ protected FirstLastValueImplementor(SeekType seekType) { AggResultContext result) { WinAggResultContext winResult = (WinAggResultContext) result; + final boolean ignoreNulls = + info instanceof WinAggContext && ((WinAggContext) info).ignoreNulls(); + if (ignoreNulls) { + return implementResultIgnoreNulls(info, winResult); + } + return Expressions.condition(winResult.hasRows(), winResult.rowTranslator( winResult.computeIndex(Expressions.constant(0), seekType)) .translate(winResult.rexArguments().get(0), info.returnType()), getDefaultValue(info.returnType())); } + + /** + * Implements FIRST_VALUE / LAST_VALUE with IGNORE NULLS by scanning the + * frame (forward for FIRST_VALUE, backward for LAST_VALUE) and returning + * the first non-null argument value, or null if all rows in the frame are + * null (or the frame is empty). + * + *

      Generated code (for FIRST_VALUE; LAST_VALUE scans backward): + * 

      {@code
+     *   BoxType res = null;
+     *   if (hasRows) {
+     *     for (int seekIdx = startIndex; seekIdx <= endIndex; seekIdx++) {
+     *       BoxType seekValue = rowTranslator.translate(arg, boxType);
+     *       if (seekValue != null) {
+     *         res = seekValue;
+     *         break;
+     *       }
+     *     }
+     *   }
+     *   return res;
+     * }
      + */ + private Expression implementResultIgnoreNulls(AggContext info, + WinAggResultContext winResult) { + final Type returnType = info.returnType(); + final RexNode arg = winResult.rexArguments().get(0); + + // Use a boxed type internally so that a NULL comparison is always valid, + // even when the (frame-guaranteed non-empty) return type is a primitive. + // The surrounding window implementation converts the result back to the + // declared return type. + final Type boxType = Types.box(returnType); + + final ParameterExpression res = + Expressions.parameter(0, boxType, + winResult.currentBlock().newName( + seekType == SeekType.START ? "first_value" : "last_value")); + // res = null + winResult.currentBlock().add(Expressions.declare(0, res, NULL_EXPR)); + + final ParameterExpression idx = + Expressions.parameter(int.class, + winResult.currentBlock().newName("seekIdx")); + + // startIndex() and endIndex() are the concrete row indices of the frame + // bounds for the current row, already resolved by EnumerableWindow. For + // unbounded windows they span the whole partition; for RANGE windows they + // span the peer group(s) included in the frame. + // Scan direction: FIRST_VALUE walks from start to end, LAST_VALUE walks + // from end back to start. + final boolean forward = seekType == SeekType.START; + final Expression from = + forward ? winResult.startIndex() : winResult.endIndex(); + final Expression to = + forward ? winResult.endIndex() : winResult.startIndex(); + final Expression condition = + forward + ? Expressions.lessThanOrEqual(idx, to) + : Expressions.greaterThanOrEqual(idx, to); + final Expression post = + forward + ? Expressions.postIncrementAssign(idx) + : Expressions.postDecrementAssign(idx); + + // Build the loop body: + // BoxType seekValue = rowTranslator.translate(arg, boxType); + // if (seekValue != null) { + // res = seekValue; + // break; + // } + final BlockBuilder loopBody = winResult.nestBlock(); + final Expression value = + winResult.rowTranslator(idx).translate(arg, boxType); + final ParameterExpression valueVar = + Expressions.parameter(0, boxType, loopBody.newName("seekValue")); + loopBody.add(Expressions.declare(0, valueVar, value)); + loopBody.add( + Expressions.ifThen( + Expressions.notEqual(valueVar, NULL_EXPR), + Expressions.block( + Expressions.statement(Expressions.assign(res, valueVar)), + Expressions.break_(null)))); + winResult.exitBlock(); + final BlockStatement loopBodyBlock = loopBody.toBlock(); + + // Wrap the scan in: if (hasRows) { for (...) { ... } } + winResult.currentBlock().add( + Expressions.ifThen(winResult.hasRows(), + Expressions.for_( + Expressions.declare(0, idx, from), + condition, post, loopBodyBlock))); + return res; + } } /** Implementor for the {@code FIRST_VALUE} windowed aggregate function. */ diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/WinAggContext.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/WinAggContext.java index 28fe9a96f53e..a867a813fbf7 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/WinAggContext.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/WinAggContext.java @@ -26,4 +26,9 @@ public interface WinAggContext extends AggContext { /** The exclude clause of the group of the window function. */ RexWindowExclusion getExclude(); + + /** Whether the window function ignores NULL values (IGNORE NULLS). */ + default boolean ignoreNulls() { + return false; + } } diff --git a/core/src/test/resources/sql/winagg.iq b/core/src/test/resources/sql/winagg.iq index 1f07d4351d8d..f8195f91aa83 100644 --- a/core/src/test/resources/sql/winagg.iq +++ b/core/src/test/resources/sql/winagg.iq @@ -1323,4 +1323,161 @@ java.sql.SQLException: Error while executing SQL "select first_value(sal) filter from emp": FILTER clause is not supported for window function FIRST_VALUE !error +# [CALCITE-7701] Support IGNORE NULLS for FIRST_VALUE/LAST_VALUE window functions in the enumerable convention +# Verified against Oracle +# FIRST_VALUE with IGNORE NULLS returns the first non-null value in the frame +# (or NULL if the frame is empty or all values are null). +select o, v, + first_value(v) ignore nulls over (order by o rows 2 preceding) as fv +from (values (1, 1), (2, cast(null as integer)), (3, 3), + (4, cast(null as integer)), (5, cast(null as integer))) as t(o, v); ++---+---+----+ +| O | V | FV | ++---+---+----+ +| 1 | 1 | 1 | +| 2 | | 1 | +| 3 | 3 | 1 | +| 4 | | 3 | +| 5 | | 3 | ++---+---+----+ +(5 rows) + +!ok + +# LAST_VALUE with IGNORE NULLS returns the last non-null value in the frame. +select o, v, + last_value(v) ignore nulls over (order by o rows 2 preceding) as lv +from (values (1, 1), (2, cast(null as integer)), (3, 3), + (4, cast(null as integer)), (5, cast(null as integer))) as t(o, v); ++---+---+----+ +| O | V | LV | ++---+---+----+ +| 1 | 1 | 1 | +| 2 | | 1 | +| 3 | 3 | 3 | +| 4 | | 3 | +| 5 | | 3 | ++---+---+----+ +(5 rows) + +!ok + +# IGNORE NULLS returns NULL when every row in the frame is null. +select o, v, + first_value(v) ignore nulls + over (order by o rows between 1 preceding and 1 preceding) as fv +from (values (1, cast(null as integer)), (2, cast(null as integer)), + (3, 5)) as t(o, v); ++---+---+----+ +| O | V | FV | ++---+---+----+ +| 1 | | | +| 2 | | | +| 3 | 5 | | ++---+---+----+ +(3 rows) + +!ok + +# RESPECT NULLS (the default) still returns the boundary value, including NULL. +select o, v, + first_value(v) respect nulls over (order by o rows 2 preceding) as fv, + last_value(v) over (order by o rows 2 preceding) as lv +from (values (1, 1), (2, cast(null as integer)), (3, 3)) as t(o, v); ++---+---+----+----+ +| O | V | FV | LV | ++---+---+----+----+ +| 1 | 1 | 1 | 1 | +| 2 | | 1 | | +| 3 | 3 | 1 | 3 | ++---+---+----+----+ +(3 rows) + +!ok + +# IGNORE NULLS works with an unbounded ROWS window. +select o, v, + first_value(v) ignore nulls + over (order by o rows between unbounded preceding and unbounded following) as fv, + last_value(v) ignore nulls + over (order by o rows between unbounded preceding and unbounded following) as lv +from (values (1, 1), (2, cast(null as integer)), (3, 3), + (4, cast(null as integer)), (5, cast(null as integer))) as t(o, v); ++---+---+----+----+ +| O | V | FV | LV | ++---+---+----+----+ +| 1 | 1 | 1 | 3 | +| 2 | | 1 | 3 | +| 3 | 3 | 1 | 3 | +| 4 | | 1 | 3 | +| 5 | | 1 | 3 | ++---+---+----+----+ +(5 rows) + +!ok + +# IGNORE NULLS works with the default RANGE frame (UNBOUNDED PRECEDING TO CURRENT ROW). +select o, v, + first_value(v) ignore nulls over (order by o) as fv, + last_value(v) ignore nulls over (order by o) as lv +from (values (1, 1), (2, cast(null as integer)), (3, 3), + (4, cast(null as integer)), (5, cast(null as integer))) as t(o, v); ++---+---+----+----+ +| O | V | FV | LV | ++---+---+----+----+ +| 1 | 1 | 1 | 1 | +| 2 | | 1 | 1 | +| 3 | 3 | 1 | 3 | +| 4 | | 1 | 3 | +| 5 | | 1 | 3 | ++---+---+----+----+ +(5 rows) + +!ok + +# IGNORE NULLS works with a symmetric RANGE window. +select o, v, + first_value(v) ignore nulls + over (order by o range between 1 preceding and 1 following) as fv, + last_value(v) ignore nulls + over (order by o range between 1 preceding and 1 following) as lv +from (values (1, 1), (2, cast(null as integer)), (3, 3), + (4, cast(null as integer)), (5, cast(null as integer))) as t(o, v); ++---+---+----+----+ +| O | V | FV | LV | ++---+---+----+----+ +| 1 | 1 | 1 | 1 | +| 2 | | 1 | 3 | +| 3 | 3 | 3 | 3 | +| 4 | | 3 | 3 | +| 5 | | | | ++---+---+----+----+ +(5 rows) + +!ok + +# IGNORE NULLS works with RANGE peers: the current-row peer group is included. +# The result rows are ordered to make the peer-group ordering deterministic +# and to match Oracle. +select o, v, + first_value(v) ignore nulls + over (order by o range between unbounded preceding and current row) as fv, + last_value(v) ignore nulls + over (order by o range between unbounded preceding and current row) as lv +from (values (1, 1), (2, cast(null as integer)), (2, 3), + (4, cast(null as integer)), (5, cast(null as integer))) as t(o, v) +order by o, v; ++---+---+----+----+ +| O | V | FV | LV | ++---+---+----+----+ +| 1 | 1 | 1 | 1 | +| 2 | 3 | 1 | 3 | +| 2 | | 1 | 3 | +| 4 | | 1 | 3 | +| 5 | | 1 | 3 | ++---+---+----+----+ +(5 rows) + +!ok + # End winagg.iq From f0454f78cff8661ef6c1288e9af630165d6e4edd Mon Sep 17 00:00:00 2001 From: Ruben Quesada Lopez Date: Tue, 18 Aug 2026 16:01:15 +0100 Subject: [PATCH 484/562] [CALCITE-7726] Improve identifier validation on MemberExpression and ParameterExpression --- .../enumerable/StrictAggImplementor.java | 4 +- .../calcite/jdbc/JavaTypeFactoryImpl.java | 12 +- .../jdbc/SyntheticRecordFieldNameTest.java | 109 +++++++++++++++++ .../test/WindowPartitionAliasTest.java | 80 +++++++++++++ .../calcite/linq4j/tree/MemberExpression.java | 5 + .../linq4j/tree/ParameterExpression.java | 7 +- .../org/apache/calcite/linq4j/tree/Types.java | 15 +++ .../linq4j/tree/IdentifierValidationTest.java | 113 ++++++++++++++++++ 8 files changed, 336 insertions(+), 9 deletions(-) create mode 100644 core/src/test/java/org/apache/calcite/jdbc/SyntheticRecordFieldNameTest.java create mode 100644 core/src/test/java/org/apache/calcite/test/WindowPartitionAliasTest.java create mode 100644 linq4j/src/test/java/org/apache/calcite/linq4j/tree/IdentifierValidationTest.java diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/StrictAggImplementor.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/StrictAggImplementor.java index 54569acc57c7..0457ffa61f4c 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/StrictAggImplementor.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/StrictAggImplementor.java @@ -163,9 +163,7 @@ protected abstract void implementNotNullAdd(AggContext info, return EnumUtils.convert( implementNotNullResult(info, result), info.returnType()); } - String tmpName = result.accumulator().isEmpty() - ? "ar" - : (result.accumulator().get(0) + "$Res"); + String tmpName = result.accumulator().isEmpty() ? "ar" : "acc$Res"; ParameterExpression res = Expressions.parameter(0, info.returnType(), result.currentBlock().newName(tmpName)); diff --git a/core/src/main/java/org/apache/calcite/jdbc/JavaTypeFactoryImpl.java b/core/src/main/java/org/apache/calcite/jdbc/JavaTypeFactoryImpl.java index 2114921971d4..32aed69e3d6e 100644 --- a/core/src/main/java/org/apache/calcite/jdbc/JavaTypeFactoryImpl.java +++ b/core/src/main/java/org/apache/calcite/jdbc/JavaTypeFactoryImpl.java @@ -367,7 +367,8 @@ private Type createSyntheticType(RelRecordType type) { "Record" + type.getFieldCount() + "_" + syntheticTypes.size(); final SyntheticRecordType syntheticType = new SyntheticRecordType(type, name); - for (final RelDataTypeField recordField : type.getFieldList()) { + for (final Ord ord : Ord.zip(type.getFieldList())) { + final RelDataTypeField recordField = ord.e; final Type fieldClass = getJavaClass(recordField.getType()); // A field whose type has no real Java class is stored as Object[] at // runtime, like all rows in enumerable convention. For example, the @@ -395,10 +396,17 @@ private Type createSyntheticType(RelRecordType type) { final Type javaClass = fieldClass instanceof Class ? fieldClass : Object[].class; + // Prefer the SQL field name to allow downstream reflection-based lookups + // (e.g. Avatica's Meta.CursorFactory.record(), which reads results + // out of the synthetic class by SQL column name); fall back to a + // positional name if the SQL name is not a legal Java identifier + final String rawName = recordField.getName(); + final String fieldName = + Types.isValidJavaIdentifier(rawName) ? rawName : "f" + ord.i; syntheticType.fields.add( new RecordFieldImpl( syntheticType, - recordField.getName(), + fieldName, javaClass, recordField.getType().isNullable() && !Primitive.is(javaClass), diff --git a/core/src/test/java/org/apache/calcite/jdbc/SyntheticRecordFieldNameTest.java b/core/src/test/java/org/apache/calcite/jdbc/SyntheticRecordFieldNameTest.java new file mode 100644 index 000000000000..3ae027e91943 --- /dev/null +++ b/core/src/test/java/org/apache/calcite/jdbc/SyntheticRecordFieldNameTest.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.jdbc; + +import org.apache.calcite.linq4j.tree.Types; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.sql.type.SqlTypeName; + +import com.google.common.collect.ImmutableList; + +import org.junit.jupiter.api.Test; + +import java.lang.reflect.Type; +import java.util.List; + +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.hasSize; + +/** + * Tests for the Java field names of the synthetic classes built by + * {@link JavaTypeFactoryImpl} from a {@code RelRecordType}. + * + *

      SQL quoted identifiers admit characters that are not legal in a + * Java identifier. When a SQL field name is a valid Java identifier it + * is reused as the field name of the synthetic class; otherwise the + * factory falls back to a positional name ({@code f0}, {@code f1}, ...). + * The original SQL names are preserved on the {@link RelDataType} + * regardless. + */ +public class SyntheticRecordFieldNameTest { + + @Test void testValidSqlNamesReusedAsJavaNames() { + final JavaTypeFactoryImpl typeFactory = new JavaTypeFactoryImpl(); + final RelDataType rowType = typeFactory.builder() + .add("empid", SqlTypeName.INTEGER) + .add("name", SqlTypeName.VARCHAR) + .add("deptno", SqlTypeName.INTEGER) + .build(); + final Type javaType = typeFactory.getJavaClass(rowType); + assertThat(javaType, instanceOf(Types.RecordType.class)); + final List fields = + ((Types.RecordType) javaType).getRecordFields(); + assertThat(fields, hasSize(3)); + assertThat(fields.get(0).getName(), is("empid")); + assertThat(fields.get(1).getName(), is("name")); + assertThat(fields.get(2).getName(), is("deptno")); + } + + @Test void testNonIdentifierSqlNamesFallBackToPositional() { + final JavaTypeFactoryImpl typeFactory = new JavaTypeFactoryImpl(); + final RelDataType rowType = typeFactory.builder() + .add("has space", SqlTypeName.INTEGER) + .add("a.b", SqlTypeName.VARCHAR) + .add("ok", SqlTypeName.INTEGER) + .build(); + final Type javaType = typeFactory.getJavaClass(rowType); + assertThat(javaType, instanceOf(Types.RecordType.class)); + final List fields = + ((Types.RecordType) javaType).getRecordFields(); + assertThat(fields, hasSize(3)); + // The first two SQL names are not legal Java identifiers, so they + // are replaced by positional names; the third one is fine and is + // reused verbatim + assertThat(fields.get(0).getName(), is("f0")); + assertThat(fields.get(1).getName(), is("f1")); + assertThat(fields.get(2).getName(), is("ok")); + for (Types.RecordField f : fields) { + assertThat("field names must be valid Java identifiers", + Types.isValidJavaIdentifier(f.getName()), is(true)); + } + // The SQL names survive on the relational rowtype, which is where + // column labels come from + final JavaTypeFactoryImpl.SyntheticRecordType syntheticType = + (JavaTypeFactoryImpl.SyntheticRecordType) javaType; + assertThat(syntheticType.relType, is(rowType)); + assertThat(rowType.getFieldNames().get(0), is("has space")); + assertThat(rowType.getFieldNames().get(1), is("a.b")); + assertThat(rowType.getFieldNames().get(2), is("ok")); + } + + @Test void testListOverloadStillUsesPositionalNames() { + // The createSyntheticType(List l) overload produces f0..fn + // (the caller supplies no names) + final JavaTypeFactoryImpl typeFactory = new JavaTypeFactoryImpl(); + final Type fromTypes = + typeFactory.createSyntheticType(ImmutableList.of(Integer.class, String.class)); + final List fields = + ((Types.RecordType) fromTypes).getRecordFields(); + assertThat(fields, hasSize(2)); + assertThat(fields.get(0).getName(), is("f0")); + assertThat(fields.get(1).getName(), is("f1")); + } +} diff --git a/core/src/test/java/org/apache/calcite/test/WindowPartitionAliasTest.java b/core/src/test/java/org/apache/calcite/test/WindowPartitionAliasTest.java new file mode 100644 index 000000000000..05c18555c887 --- /dev/null +++ b/core/src/test/java/org/apache/calcite/test/WindowPartitionAliasTest.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.test; + +import org.junit.jupiter.api.Test; + +/** + * Tests for column aliases used as {@code PARTITION BY} keys of a window + * aggregate. + * + *

      SQL allows quoted identifiers to contain almost any character; + * however, the enumerable convention materialises multi-column + * {@code PARTITION BY} keys as fields of a synthetic Java class emitted + * into generated source. Field names that are not valid Java identifiers + * are transparently renamed to positional placeholders + * ({@code f0}, {@code f1}, ...) inside the synthetic class, while the + * original SQL name is preserved at the rowtype level for outer + * references. + */ +class WindowPartitionAliasTest { + + /** A quoted alias that is a valid Java identifier plans and runs + * normally when used as one of several {@code PARTITION BY} keys. The + * projection wraps the aliased column in a computation so field- + * trimming does not fold the sub-query away. */ + @Test void testValidIdentifierAlias() { + final String sql = "select \"aliased_deptno\"," + + " count(*) over (" + + " partition by \"aliased_deptno\", \"empid\") as c" + + " from (" + + " select \"deptno\" + 1 as \"aliased_deptno\", \"empid\"" + + " from \"hr\".\"emps\")"; + CalciteAssert.hr() + .query(sql) + .runs(); + } + + /** A quoted alias containing a space (a legal SQL identifier character + * that is not a legal Java identifier character) still runs: the + * synthetic partition-key class carries a positional field name while + * the outer projection continues to see the SQL alias. */ + @Test void testAliasWithSpaceRuns() { + final String sql = "select \"has space\"," + + " count(*) over (" + + " partition by \"has space\", \"empid\") as c" + + " from (" + + " select \"deptno\" + 1 as \"has space\", \"empid\"" + + " from \"hr\".\"emps\")"; + CalciteAssert.hr() + .query(sql) + .runs(); + } + + /** Same shape as above, with a punctuation character. */ + @Test void testAliasWithPunctuationRuns() { + final String sql = "select \"a.b\"," + + " count(*) over (" + + " partition by \"a.b\", \"empid\") as c" + + " from (" + + " select \"deptno\" + 1 as \"a.b\", \"empid\"" + + " from \"hr\".\"emps\")"; + CalciteAssert.hr() + .query(sql) + .runs(); + } +} diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/MemberExpression.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/MemberExpression.java index 8c4d37d7dc9c..54d0d7361815 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/MemberExpression.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/MemberExpression.java @@ -22,6 +22,8 @@ import java.lang.reflect.Modifier; import java.util.Objects; +import static com.google.common.base.Preconditions.checkArgument; + import static java.util.Objects.requireNonNull; /** @@ -39,6 +41,9 @@ public MemberExpression(@Nullable Expression expression, PseudoField field) { super(ExpressionType.MemberAccess, field.getType()); this.expression = expression; this.field = requireNonNull(field, "field"); + checkArgument(Types.isValidJavaIdentifier(field.getName()), + "field name should be a valid java identifier: %s", + field.getName()); if (!Modifier.isStatic(field.getModifiers())) { requireNonNull(expression, "must specify expression if field is not static"); diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ParameterExpression.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ParameterExpression.java index 17e95171b684..9e3c05d982ae 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ParameterExpression.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ParameterExpression.java @@ -41,12 +41,11 @@ public ParameterExpression(Type type) { public ParameterExpression(int modifier, Type type, String name) { super(ExpressionType.Parameter, type); - checkArgument(Character.isJavaIdentifierStart(name.charAt(0)), - "parameter name should be valid java identifier: %s. " - + "The first character is invalid.", + checkArgument(Types.isValidJavaIdentifier(requireNonNull(name, "name")), + "parameter name should be a valid java identifier: %s", name); this.modifier = modifier; - this.name = requireNonNull(name, "name"); + this.name = name; } @Override public Expression accept(Shuttle shuttle) { diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/Types.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/Types.java index 09d371216e99..e3a2a8390ac9 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/Types.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/Types.java @@ -47,6 +47,21 @@ public abstract class Types { private Types() {} + /** + * Returns whether {@code name} is a syntactically valid Java identifier. + */ + public static boolean isValidJavaIdentifier(String name) { + if (name.isEmpty() || !Character.isJavaIdentifierStart(name.charAt(0))) { + return false; + } + for (int i = 1; i < name.length(); i++) { + if (!Character.isJavaIdentifierPart(name.charAt(i))) { + return false; + } + } + return true; + } + /** * Creates a type with generic parameters. */ diff --git a/linq4j/src/test/java/org/apache/calcite/linq4j/tree/IdentifierValidationTest.java b/linq4j/src/test/java/org/apache/calcite/linq4j/tree/IdentifierValidationTest.java new file mode 100644 index 000000000000..3040e00f0638 --- /dev/null +++ b/linq4j/src/test/java/org/apache/calcite/linq4j/tree/IdentifierValidationTest.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.linq4j.tree; + +import org.checkerframework.checker.nullness.qual.Nullable; +import org.junit.jupiter.api.Test; + +import java.lang.reflect.Modifier; +import java.lang.reflect.Type; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Tests for {@link Types#isValidJavaIdentifier} and for the constructor + * checks in the expression-tree nodes that emit an identifier into + * generated source: {@link ParameterExpression} (also used by + * {@link FieldDeclaration}) and {@link MemberExpression}. + */ +public class IdentifierValidationTest { + /** A representative bad name: starts with a legal identifier character + * but contains characters that {@code Character#isJavaIdentifierPart} + * rejects. Historically only the first character was checked. */ + private static final String BAD_NAME = "a b.c"; + + @Test void testIsValidJavaIdentifier() { + assertThat(Types.isValidJavaIdentifier("a"), is(true)); + assertThat(Types.isValidJavaIdentifier("f0"), is(true)); + assertThat(Types.isValidJavaIdentifier("_a$b9"), is(true)); + assertThat(Types.isValidJavaIdentifier(""), is(false)); + assertThat(Types.isValidJavaIdentifier("9a"), is(false)); + assertThat(Types.isValidJavaIdentifier("a b"), is(false)); + assertThat(Types.isValidJavaIdentifier("a.b"), is(false)); + assertThat(Types.isValidJavaIdentifier("a\nb"), is(false)); + assertThat(Types.isValidJavaIdentifier(BAD_NAME), is(false)); + // First-character-only validation would accept this; the full check must reject it + assertThat(Types.isValidJavaIdentifier("ok name"), is(false)); + } + + @Test void testParameterExpressionRejectsNonIdentifier() { + // Valid names are accepted + ParameterExpression p = + new ParameterExpression(0, int.class, "p0"); + assertThat(p.name, is("p0")); + assertThrows(IllegalArgumentException.class, () -> + new ParameterExpression(0, int.class, BAD_NAME)); + assertThrows(IllegalArgumentException.class, () -> + new ParameterExpression(0, int.class, "ok name")); + } + + @Test void testFieldDeclarationCoveredViaParameterExpression() { + // FieldDeclaration emits parameter.name as a field name; it takes a + // ParameterExpression, so the constructor check above covers it + assertThrows(IllegalArgumentException.class, () -> + new FieldDeclaration(Modifier.PUBLIC, + new ParameterExpression(0, int.class, BAD_NAME), null)); + } + + @Test void testMemberExpressionRejectsNonIdentifierFieldName() { + // MemberExpression takes a PseudoField and emits field.getName() + // directly; it does not go through ParameterExpression + assertThrows(IllegalArgumentException.class, () -> + new MemberExpression(null, new NamedStaticField(BAD_NAME))); + // Sane names still work + MemberExpression m = + new MemberExpression(null, new NamedStaticField("f0")); + assertThat(m.field.getName(), is("f0")); + } + + /** A synthetic static field with an arbitrary name. */ + private static class NamedStaticField implements PseudoField { + private final String name; + + NamedStaticField(String name) { + this.name = name; + } + + @Override public String getName() { + return name; + } + + @Override public Type getType() { + return int.class; + } + + @Override public int getModifiers() { + return Modifier.PUBLIC | Modifier.STATIC; + } + + @Override public @Nullable Object get(@Nullable Object o) { + return 0; + } + + @Override public Type getDeclaringClass() { + return Object.class; + } + } +} From 886b2d26e0224094ff0fe1343496ddb50aa6126b Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Wed, 19 Aug 2026 21:13:06 -0700 Subject: [PATCH 485/562] [CALCITE-7729] Linq4j BlockBuilder.optimize can optimize away expressions that throw Signed-off-by: Mihai Budiu --- .../adapter/enumerable/RexImpTable.java | 28 ++++- .../org/apache/calcite/test/JdbcTest.java | 3 +- .../calcite/test/ReflectiveSchemaTest.java | 4 +- .../calcite/linq4j/tree/BlockBuilder.java | 108 ++++++++++++++-- .../calcite/linq4j/test/BlockBuilderTest.java | 116 ++++++++++++++++++ .../calcite/linq4j/test/ExpressionTest.java | 4 +- .../calcite/linq4j/test/OptimizerTest.java | 6 +- 7 files changed, 247 insertions(+), 22 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java index ae40e33c1334..6ef0182b301c 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java @@ -2498,11 +2498,33 @@ protected FirstLastValueImplementor(SeekType seekType) { return implementResultIgnoreNulls(info, winResult); } - return Expressions.condition(winResult.hasRows(), + // Generates: + // T first_last_value; + // if () { + // + // first_last_value = ; + // } else { + // first_last_value = ; + // } + // + // Reading a row is only valid when the frame has rows: on an empty frame + // computeIndex returns -1. + final ParameterExpression res = + Expressions.parameter(0, info.returnType(), + result.currentBlock().newName("first_last_value")); + final BlockBuilder thenBlock = result.nestBlock(); + final Expression value = winResult.rowTranslator( winResult.computeIndex(Expressions.constant(0), seekType)) - .translate(winResult.rexArguments().get(0), info.returnType()), - getDefaultValue(info.returnType())); + .translate(winResult.rexArguments().get(0), info.returnType()); + thenBlock.add(Expressions.statement(Expressions.assign(res, value))); + result.exitBlock(); + result.currentBlock().add(Expressions.declare(0, res, null)); + result.currentBlock().add( + Expressions.ifThenElse(winResult.hasRows(), thenBlock.toBlock(), + Expressions.statement( + Expressions.assign(res, getDefaultValue(info.returnType()))))); + return res; } /** diff --git a/core/src/test/java/org/apache/calcite/test/JdbcTest.java b/core/src/test/java/org/apache/calcite/test/JdbcTest.java index d235fa327c77..6dc4a48dc99b 100644 --- a/core/src/test/java/org/apache/calcite/test/JdbcTest.java +++ b/core/src/test/java/org/apache/calcite/test/JdbcTest.java @@ -2772,6 +2772,7 @@ private void checkNullableTimestamp(CalciteAssert.Config config) { + " final org.apache.calcite.test.schemata.hr.Employee current" + " = (org.apache.calcite.test.schemata.hr.Employee) inputEnumerator.current();\n" + " final String input_value = current.name;\n" + + " final int input_value0 = current.deptno;\n" + " Integer case_when_value;\n" + " if ($L4J$C$org_apache_calcite_runtime_SqlFunctions_ne_) {\n" + " case_when_value = $L4J$C$Integer_valueOf_1_;\n" @@ -2780,7 +2781,7 @@ private void checkNullableTimestamp(CalciteAssert.Config config) { + " }\n" + " final Integer binary_call_value0 = " + "case_when_value == null ? null : " - + "Integer.valueOf(current.deptno + case_when_value.intValue());\n" + + "Integer.valueOf(input_value0 + case_when_value.intValue());\n" + " return input_value == null || binary_call_value0 == null" + " ? null" + " : org.apache.calcite.runtime.SqlFunctions.substring(input_value, " diff --git a/core/src/test/java/org/apache/calcite/test/ReflectiveSchemaTest.java b/core/src/test/java/org/apache/calcite/test/ReflectiveSchemaTest.java index e4979b95f3fa..cc0205c9878f 100644 --- a/core/src/test/java/org/apache/calcite/test/ReflectiveSchemaTest.java +++ b/core/src/test/java/org/apache/calcite/test/ReflectiveSchemaTest.java @@ -764,7 +764,9 @@ private void check(ResultSetMetaData metaData, String columnName, .planContains( "final Long input_value = current.wrapperLong;") .planContains( - "return input_value == null ? null : Long.valueOf(input_value.longValue() / current.primitiveLong);") + "final long input_value0 = current.primitiveLong;") + .planContains( + "return input_value == null ? null : Long.valueOf(input_value.longValue() / input_value0);") .returns("C=null\n"); } diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/BlockBuilder.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/BlockBuilder.java index 28aab938a104..5d811ea68c02 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/BlockBuilder.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/BlockBuilder.java @@ -360,7 +360,7 @@ private boolean optimize(Shuttle optimizer, boolean performInline) { for (Statement statement : statements) { if (statement instanceof DeclarationStatement && performInline) { DeclarationStatement decl = (DeclarationStatement) statement; - useCounter.map.put(decl.parameter, new Slot()); + useCounter.map.put(decl.parameter, new ParameterUse()); } // We are added only counters up to current statement. // It is fine to count usages as the latter declarations cannot be used @@ -378,7 +378,7 @@ private boolean optimize(Shuttle optimizer, boolean performInline) { for (Statement oldStatement : oldStatements) { if (oldStatement instanceof DeclarationStatement) { DeclarationStatement statement = (DeclarationStatement) oldStatement; - final Slot slot = useCounter.map.get(statement.parameter); + final ParameterUse slot = useCounter.map.get(statement.parameter); int count = slot == null ? Integer.MAX_VALUE - 10 : slot.count; if (count > 1 && isSimpleExpression(statement.initializer)) { // Inline simple final constants @@ -410,13 +410,15 @@ private boolean optimize(Shuttle optimizer, boolean performInline) { // anonymous classes. count = Integer.MAX_VALUE; } - if (count == 0 - && statement.initializer != null - && Expressions.mayThrow(statement.initializer)) { - // Never read, but computing the value may raise a runtime error that - // the program is expected to raise. Keep the declaration, and treat - // it like any other statement that cannot be inlined. - count = 100; + if (statement.initializer != null + && (count == 0 || slot != null && slot.conditional)) { + // The value is either never read, or read only conditionally. + // If it may raise a runtime error, keep the declaration where it is. + final Expression initializer = + subMap.isEmpty() ? statement.initializer : statement.initializer.accept(visitor); + if (Expressions.mayThrow(initializer)) { + count = 100; + } } Expression normalized = normalizeDeclaration(statement); expressionForReuse.remove(normalized); @@ -603,15 +605,92 @@ private static class InlineVariableVisitor extends SubstituteVariableVisitor { /** Use counter. */ private static class UseCounter extends VisitorImpl { - private final IdentityHashMap map = new IdentityHashMap<>(); + /** Map each parameter to information about how it is used. */ + private final IdentityHashMap map = new IdentityHashMap<>(); + /** Whether the node being visited is evaluated only if some other + * expression permits it, as "a" in the expression "c ? a : b". */ + private boolean inConditional = false; + + /** Visits a node that is evaluated only under a condition. */ + private void acceptConditionally(Node node) { + final boolean prev = inConditional; + inConditional = true; + node.accept(this); + inConditional = prev; + } + + @Override public Void visit(TernaryExpression ternary) { + if (ternary.getNodeType() != ExpressionType.Conditional) { + return super.visit(ternary); + } + ternary.expression0.accept(this); + acceptConditionally(ternary.expression1); + acceptConditionally(ternary.expression2); + return null; + } + + @Override public Void visit(BinaryExpression binary) { + switch (binary.getNodeType()) { + case AndAlso: + case OrElse: + // The right operand is evaluated only if the left one has not already decided the result + binary.expression0.accept(this); + acceptConditionally(binary.expression1); + return null; + default: + return super.visit(binary); + } + } + + @Override public Void visit(WhileStatement whileStatement) { + // The body may never run + whileStatement.condition.accept(this); + acceptConditionally(whileStatement.body); + return null; + } + + @Override public Void visit(ForStatement forStatement) { + // The body and the "post" expression may not run + Expressions.acceptNodes(forStatement.declarations, this); + if (forStatement.condition != null) { + forStatement.condition.accept(this); + } + if (forStatement.post != null) { + acceptConditionally(forStatement.post); + } + acceptConditionally(forStatement.body); + return null; + } + + @Override public Void visit(ForEachStatement forEachStatement) { + // The body may not run + forEachStatement.parameter.accept(this); + forEachStatement.iterable.accept(this); + acceptConditionally(forEachStatement.body); + return null; + } + + @Override public Void visit(ConditionalStatement conditionalStatement) { + // In "if (c0) s0 else if (c1) s1 ... else s", only "c0" is unconditional + final List list = conditionalStatement.expressionList; + for (int i = 0; i < list.size(); i++) { + if (i == 0) { + list.get(i).accept(this); + } else { + acceptConditionally(list.get(i)); + } + } + return null; + } @Override public Void visit(ParameterExpression parameter) { - final Slot slot = map.get(parameter); + final ParameterUse slot = map.get(parameter); if (slot != null) { // Count use of parameter, if it's registered. It's OK if // parameter is not registered. It might be beyond the control // of this block. slot.count++; + slot.conditional |= inConditional; } return super.visit(parameter); } @@ -626,9 +705,12 @@ private static class UseCounter extends VisitorImpl { } /** - * Holds the number of times a declaration was used. + * Holds information about the uses of one ParameterExpression within an expression. */ - private static class Slot { + private static class ParameterUse { + /** How many times the declaration is read. */ private int count; + /** Whether at least one read is evaluated only under a condition. */ + private boolean conditional; } } diff --git a/linq4j/src/test/java/org/apache/calcite/linq4j/test/BlockBuilderTest.java b/linq4j/src/test/java/org/apache/calcite/linq4j/test/BlockBuilderTest.java index 6950beae82b3..5cc772c0b545 100644 --- a/linq4j/src/test/java/org/apache/calcite/linq4j/test/BlockBuilderTest.java +++ b/linq4j/src/test/java/org/apache/calcite/linq4j/test/BlockBuilderTest.java @@ -83,6 +83,122 @@ public void prepareBuilder() { + "}\n")); } + /** Test case for + * [CALCITE-7729] + * Linq4j BlockBuilder.optimize can optimize away expressions that + * throw. */ + @Test void testDeclarationUsedOnlyInBranchThatFoldsAway() { + final ParameterExpression i = Expressions.parameter(int.class, "i"); + final Expression x = b.append("x", Expressions.divide(ONE, i)); + b.add( + Expressions.return_(null, + Expressions.condition(Expressions.constant(true), TWO, x))); + assertThat(b.toBlock(), + hasToString("{\n" + + " final int x = 1 / i;\n" + + " return 2;\n" + + "}\n")); + } + + /** Test case for + * [CALCITE-7729] + * Linq4j BlockBuilder.optimize can optimize away expressions that + * throw. */ + @Test void testDeclarationUsedInBranchThatSurvives() { + final ParameterExpression i = Expressions.parameter(int.class, "i"); + final ParameterExpression c = Expressions.parameter(boolean.class, "c"); + final Expression x = b.append("x", Expressions.divide(ONE, i)); + b.add(Expressions.return_(null, Expressions.condition(c, TWO, x))); + assertThat(b.toBlock(), + hasToString("{\n" + + " final int x = 1 / i;\n" + + " return c ? 2 : x;\n" + + "}\n")); + } + + /** Test case for + * [CALCITE-7729] + * Linq4j BlockBuilder.optimize can optimize away expressions that + * throw. + * + *

      Nested conditionals: "x" is read in a branch of an inner conditional, + * which is itself in a branch. */ + @Test void testNestedConditionalBranch() { + final ParameterExpression i = Expressions.parameter(int.class, "i"); + final ParameterExpression c = Expressions.parameter(boolean.class, "c"); + final ParameterExpression d = Expressions.parameter(boolean.class, "d"); + final Expression x = b.append("x", Expressions.divide(ONE, i)); + b.add( + Expressions.return_(null, + Expressions.condition(c, + Expressions.condition(d, TWO, x), ONE))); + assertThat(b.toBlock(), + hasToString("{\n" + + " final int x = 1 / i;\n" + + " return c ? (d ? 2 : x) : 1;\n" + + "}\n")); + } + + /** Test case for + * [CALCITE-7729] + * Linq4j BlockBuilder.optimize can optimize away expressions that + * throw. + * + *

      Nested conditionals: "x" is read in the condition of an inner + * conditional, so it is evaluated only if the outer condition holds. */ + @Test void testNestedConditionalCondition() { + final ParameterExpression i = Expressions.parameter(int.class, "i"); + final ParameterExpression c = Expressions.parameter(boolean.class, "c"); + final Expression x = b.append("x", Expressions.divide(ONE, i)); + b.add( + Expressions.return_(null, + Expressions.condition(c, + Expressions.condition( + Expressions.greaterThan(x, ONE), TWO, ONE), + ONE))); + assertThat(b.toBlock(), + hasToString("{\n" + + " final int x = 1 / i;\n" + + " return c ? (x > 1 ? 2 : 1) : 1;\n" + + "}\n")); + } + + /** Test case for + * [CALCITE-7729] + * Linq4j BlockBuilder.optimize can optimize away expressions that + * throw. */ + @Test void testDeclarationUsedOnlyInWhileBody() { + final ParameterExpression i = Expressions.parameter(int.class, "i"); + final ParameterExpression c = Expressions.parameter(boolean.class, "c"); + final ParameterExpression y = Expressions.parameter(int.class, "y"); + final Expression x = b.append("x", Expressions.divide(ONE, i)); + b.add(Expressions.declare(0, y, ONE)); + b.add( + Expressions.while_(c, + Expressions.statement(Expressions.assign(y, x)))); + assertThat(b.toBlock(), + hasToString("{\n" + + " final int x = 1 / i;\n" + + " int y = 1;\n" + + " while (c) {\n" + + " y = x;\n" + + " }\n" + + "}\n")); + } + + /** Test case for + * [CALCITE-7729] + * Linq4j BlockBuilder.optimize can optimize away expressions that + * throw. */ + @Test void testPureDeclarationIsInlinedIntoBranch() { + // Test with expression that does not throw + final ParameterExpression i = Expressions.parameter(int.class, "i"); + final ParameterExpression c = Expressions.parameter(boolean.class, "c"); + final Expression x = b.append("x", Expressions.add(ONE, i)); + b.add(Expressions.return_(null, Expressions.condition(c, TWO, x))); + assertThat(b.toBlock(), hasToString("{\n return c ? 2 : 1 + i;\n}\n")); + } + /** Test case for * [CALCITE-7728] * Linq4j can simplify expressions without regards for 'safety'. diff --git a/linq4j/src/test/java/org/apache/calcite/linq4j/test/ExpressionTest.java b/linq4j/src/test/java/org/apache/calcite/linq4j/test/ExpressionTest.java index c75c2a213998..3482e2b86797 100644 --- a/linq4j/src/test/java/org/apache/calcite/linq4j/test/ExpressionTest.java +++ b/linq4j/src/test/java/org/apache/calcite/linq4j/test/ExpressionTest.java @@ -1496,8 +1496,8 @@ public void checkBlockBuilder(boolean optimizing, String expected) { assertThat(Expressions.toString(builder.toBlock()), is("{\n" + " final Short v = (Short) ((Object[]) p)[4];\n" - + " return (Number) v == null ? null : (" - + "(Number) v).intValue() == 1997;\n" + + " final int v5 = ((Number) v).intValue();\n" + + " return (Number) v == null ? null : v5 == 1997;\n" + "}\n")); } diff --git a/linq4j/src/test/java/org/apache/calcite/linq4j/test/OptimizerTest.java b/linq4j/src/test/java/org/apache/calcite/linq4j/test/OptimizerTest.java index 8fb0191ec2af..ce1e429c6a7f 100644 --- a/linq4j/src/test/java/org/apache/calcite/linq4j/test/OptimizerTest.java +++ b/linq4j/src/test/java/org/apache/calcite/linq4j/test/OptimizerTest.java @@ -868,8 +868,9 @@ class OptimizerTest { x_)))), equalTo("{\n" + " long x = 0L;\n" + + " final long y = System.currentTimeMillis();\n" + " if (System.nanoTime() > 0L) {\n" - + " x = System.currentTimeMillis();\n" + + " x = y;\n" + " }\n" + " System.out.println(x);\n" + "}\n")); @@ -897,8 +898,9 @@ class OptimizerTest { Expressions.statement(Expressions.assign(x_, y_))))), equalTo("{\n" + " long x = 0L;\n" + + " final long y = System.currentTimeMillis();\n" + " if (System.currentTimeMillis() > 0L) {\n" - + " x = System.currentTimeMillis();\n" + + " x = y;\n" + " }\n" + "}\n")); } From 2dcdce603d694a8ef084107f3e1c1309561af1ff Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Wed, 19 Aug 2026 21:20:14 -0700 Subject: [PATCH 486/562] [CALCITE-7725] Review safety of checked arithmetic operators Signed-off-by: Mihai Budiu --- .../org/apache/calcite/prepare/Prepare.java | 6 +++++ .../org/apache/calcite/rex/RexAnalyzer.java | 7 ++++++ .../java/org/apache/calcite/rex/RexCall.java | 14 ++++++++--- .../org/apache/calcite/rex/RexSimplify.java | 20 ++++++++++----- .../calcite/sql2rel/SqlToRelConverter.java | 16 ++++++++++++ .../apache/calcite/rex/RexProgramTest.java | 24 ++++++++++++++++++ core/src/test/resources/sql/cast.iq | 25 +++++++++++++++++++ 7 files changed, 102 insertions(+), 10 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/prepare/Prepare.java b/core/src/main/java/org/apache/calcite/prepare/Prepare.java index 1f2e96688a25..eac4fa6dd347 100644 --- a/core/src/main/java/org/apache/calcite/prepare/Prepare.java +++ b/core/src/main/java/org/apache/calcite/prepare/Prepare.java @@ -260,6 +260,12 @@ public PreparedResult prepareSql( // Convert some operations to use checked arithmetic: // - all arithmetic operations on exact types if the conformance requires checked arithmetic // - all arithmetic that produces INTERVAL results, regardless of the conformance + // + // SqlToRelConverter already runs ConvertToChecked. This second conversion is needed for: + // - INTERVAL arithmetic under a conformance without checked + // arithmetic (SqlToRelConverter installs no converter at all) + // - expressions that SqlToRelConverter does not build through + // Blackboard#convertExpression ConvertToChecked checkedConv = new ConvertToChecked(root.rel.getCluster().getRexBuilder(), convertToChecked); RelNode rel = checkedConv.visit(root.rel); diff --git a/core/src/main/java/org/apache/calcite/rex/RexAnalyzer.java b/core/src/main/java/org/apache/calcite/rex/RexAnalyzer.java index a124168137d8..5ca25da26555 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexAnalyzer.java +++ b/core/src/main/java/org/apache/calcite/rex/RexAnalyzer.java @@ -19,6 +19,7 @@ import org.apache.calcite.linq4j.Linq4j; import org.apache.calcite.plan.RelOptPredicateList; import org.apache.calcite.rel.metadata.NullSentinel; +import org.apache.calcite.sql.SqlKind; import org.apache.calcite.util.NlsString; import org.apache.calcite.util.Pair; import org.apache.calcite.util.Util; @@ -136,6 +137,12 @@ private static class VariableCollector extends RexVisitorImpl { } @Override public Void visitCall(RexCall call) { + if (SqlKind.CHECKED_ARITHMETIC.contains(call.getKind())) { + // RexInterpreter computes with unbounded values, so it cannot tell + // whether checked arithmetic overflows + ++unsupportedCount; + return null; + } switch (call.getKind()) { case CAST: case M2V: diff --git a/core/src/main/java/org/apache/calcite/rex/RexCall.java b/core/src/main/java/org/apache/calcite/rex/RexCall.java index deac203607d8..27d93189c527 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexCall.java +++ b/core/src/main/java/org/apache/calcite/rex/RexCall.java @@ -222,12 +222,15 @@ private boolean digestWithType() { // Only boolean-valued calls can be always-true; e.g. CAST(TRUE AS INTEGER) // evaluates to 1 (INTEGER), not a boolean, even though its operand is // always true. + // An expression that may throw is never always-true: "1 / 0 IS NOT NULL" + // raises an error rather than returning TRUE. if (getType().getSqlTypeName() != SqlTypeName.BOOLEAN) { return false; } switch (getKind()) { case IS_NOT_NULL: - return !operands.get(0).getType().isNullable(); + return !operands.get(0).getType().isNullable() + && RexSimplify.isSafeExpression(operands.get(0)); case IS_NOT_TRUE: case IS_FALSE: case NOT: @@ -240,7 +243,8 @@ private boolean digestWithType() { final Sarg sarg = ((RexLiteral) operands.get(1)).getValueAs(Sarg.class); return requireNonNull(sarg, "sarg").isAll() && (sarg.nullAs == RexUnknownAs.TRUE - || !operands.get(0).getType().isNullable()); + || !operands.get(0).getType().isNullable()) + && RexSimplify.isSafeExpression(operands.get(0)); default: return false; } @@ -253,7 +257,8 @@ private boolean digestWithType() { } switch (getKind()) { case IS_NULL: - return !operands.get(0).getType().isNullable(); + return !operands.get(0).getType().isNullable() + && RexSimplify.isSafeExpression(operands.get(0)); case IS_NOT_TRUE: case IS_FALSE: case NOT: @@ -266,7 +271,8 @@ private boolean digestWithType() { final Sarg sarg = ((RexLiteral) operands.get(1)).getValueAs(Sarg.class); return requireNonNull(sarg, "sarg").isNone() && (sarg.nullAs == RexUnknownAs.FALSE - || !operands.get(0).getType().isNullable()); + || !operands.get(0).getType().isNullable()) + && RexSimplify.isSafeExpression(operands.get(0)); default: return false; } diff --git a/core/src/main/java/org/apache/calcite/rex/RexSimplify.java b/core/src/main/java/org/apache/calcite/rex/RexSimplify.java index 386964963ceb..fe9f7b5ca06e 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexSimplify.java +++ b/core/src/main/java/org/apache/calcite/rex/RexSimplify.java @@ -1034,8 +1034,11 @@ private RexNode simplifyNot(RexCall call, RexUnknownAs unknownAs) { private RexNode simplifyUnaryMinus(RexCall call, RexUnknownAs unknownAs) { final RexNode a = call.getOperands().get(0); - if (a.getKind() == SqlKind.MINUS_PREFIX) { - // -(-(x)) ==> x + if (call.getKind() == SqlKind.MINUS_PREFIX + && a.getKind() == SqlKind.MINUS_PREFIX) { + // -(-(x)) ==> x. + // Not valid for checked arithmetic, where negation of the minimum value + // of the type throws. return simplify(((RexCall) a).getOperands().get(0), unknownAs); } return simplifyGenericNode(call); @@ -1548,13 +1551,9 @@ enum SafeRexVisitor implements RexVisitor { safeOps.add(SqlKind.ARRAY_VALUE_CONSTRUCTOR); safeOps.add(SqlKind.PLUS_PREFIX); safeOps.add(SqlKind.MINUS_PREFIX); - safeOps.add(SqlKind.CHECKED_MINUS_PREFIX); safeOps.add(SqlKind.PLUS); safeOps.add(SqlKind.MINUS); safeOps.add(SqlKind.TIMES); - safeOps.add(SqlKind.CHECKED_PLUS); - safeOps.add(SqlKind.CHECKED_MINUS); - safeOps.add(SqlKind.CHECKED_TIMES); safeOps.add(SqlKind.IS_FALSE); safeOps.add(SqlKind.IS_NOT_FALSE); safeOps.add(SqlKind.IS_TRUE); @@ -1599,6 +1598,13 @@ enum SafeRexVisitor implements RexVisitor { SqlKind sqlKind = call.getKind(); SqlOperator sqlOperator = call.getOperator(); + if (SqlKind.CHECKED_ARITHMETIC.contains(sqlKind)) { + // Checked arithmetic throws on overflow, so it is only safe when the + // arithmetic is never performed, i.e. when an operand is NULL. + return RexVisitorImpl.visitArrayAnd(this, call.operands) + && call.operands.stream().anyMatch(o -> RexUtil.isNullLiteral(o, true)); + } + switch (sqlKind) { case DIVIDE: case MOD: @@ -1683,6 +1689,8 @@ enum SafeRexVisitor implements RexVisitor { * *

      Division is an unsafe operator; consider the following: * 

      case when a > 0 then 1 / a else null end
      + * + *

      Checked arithmetic is unsafe too, because it throws on overflow */ static boolean isSafeExpression(RexNode r) { return r.accept(SafeRexVisitor.INSTANCE); diff --git a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java index 357441d316f2..c486e8ccded7 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java @@ -291,6 +291,9 @@ public class SqlToRelConverter { private int explainParamCount; public final SqlToRelConverter.Config config; private final RelBuilder relBuilder; + /** Rewrites arithmetic into checked arithmetic; null if the conformance + * does not require checked arithmetic. */ + private final @Nullable RexShuttle checkedConverter; /** * Fields used in name resolution for correlated sub-queries. @@ -377,6 +380,13 @@ public SqlToRelConverter( config.getRelBuilderFactory().create(cluster, validator != null ? validator.getCatalogReader().unwrap(RelOptSchema.class) : null) .transform(config.getRelBuilderConfigTransform()); + // Simplification assumes that arithmetic never throws, which is wrong for + // checked arithmetic; so every expression is converted to checked + // arithmetic as soon as it is built, before anything can simplify it + this.checkedConverter = + validator != null && validator.config().conformance().checkedArithmetic() + ? new ConvertToChecked(rexBuilder, true).converter + : null; this.hintStrategies = config.getHintStrategyTable(); cluster.setHintStrategies(this.hintStrategies); @@ -5938,6 +5948,12 @@ ImmutableList retrieveCursors() { } @Override public RexNode convertExpression(SqlNode expr) { + final RexNode rex = convertExpression0(expr); + // Convert arithmetic to checked arithmetic if needed + return checkedConverter == null ? rex : rex.accept(checkedConverter); + } + + private RexNode convertExpression0(SqlNode expr) { // If we're in aggregation mode and this is an expression in the // GROUP BY clause, return a reference to the field. AggConverter agg = this.agg; diff --git a/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java b/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java index 332f1865a731..ca063c48559b 100644 --- a/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java +++ b/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java @@ -4788,6 +4788,30 @@ private SqlSpecialOperatorWithPolicy(String name, SqlKind kind, int prec, boolea checkSimplify(add(zero, sub(nullInt, nullInt)), "null:INTEGER"); } + /** Unit test for + * [CALCITE-7725] + * Review safety of checked arithmetic operators. */ + @Test void testSimplifyCheckedArithmetic() { + final RexNode a = vIntNotNull(1); + final RexNode b = vIntNotNull(2); + final RexNode checkedMul = + rexBuilder.makeCall(SqlStdOperatorTable.CHECKED_MULTIPLY, a, b); + + // Unchecked arithmetic wraps around, so it never throws + checkSimplify(isNotNull(mul(a, b)), "true"); + checkSimplify(add(mul(a, b), nullInt), "null:INTEGER"); + + // Checked arithmetic throws on overflow + checkSimplifyUnchanged(isNotNull(checkedMul)); + checkSimplifyUnchanged( + rexBuilder.makeCall(SqlStdOperatorTable.CHECKED_PLUS, checkedMul, nullInt)); + + // A checked operation with a NULL operand is never performed, hence safe + checkSimplify( + rexBuilder.makeCall(SqlStdOperatorTable.CHECKED_PLUS, a, nullInt), + "null:INTEGER"); + } + @Test void testSimplifyCastWithConstantReduction() { RexNode dateStr = literal("2020-10-30"); RelDataType nullableDateType = diff --git a/core/src/test/resources/sql/cast.iq b/core/src/test/resources/sql/cast.iq index ce7b13b8b9f1..832c99e4d2c8 100644 --- a/core/src/test/resources/sql/cast.iq +++ b/core/src/test/resources/sql/cast.iq @@ -65,6 +65,31 @@ select 2147483647 * 2147483647; Caused by: java.lang.ArithmeticException !error +# Test cases for [CALCITE-7725] Review safety of checked arithmetic operators +# https://issues.apache.org/jira/browse/CALCITE-7725 + +# A checked operation with a NULL operand is never performed, so the whole +# expression can still be simplified to NULL +select cast(null as integer) + empno as c from emp where empno = 7369; ++---+ +| C | ++---+ +| | ++---+ +(1 row) + +!ok + +# "empno * 100000000" overflows, so "IS NOT NULL" must not become TRUE +select empno from emp where empno = 7369 and empno * 100000000 is not null; +integer overflow +!error + +# and "x + NULL" must not become NULL without computing x +select empno * 100000000 + cast(null as integer) as c from emp where empno = 7369; +integer overflow +!error + !use scott # Cast a character literal to a timestamp; note: the plan does not contain CAST From 0f84c5c8c9172afcf22d39c893e6e3b1c5c6b1d4 Mon Sep 17 00:00:00 2001 From: xuzifu666 <1206332514@qq.com> Date: Tue, 18 Aug 2026 17:53:54 +0800 Subject: [PATCH 487/562] [CALCITE-6284] Invalid conversion triggers ClassCastException --- .../calcite/adapter/enumerable/EnumUtils.java | 9 ++++ .../apache/calcite/runtime/SqlFunctions.java | 16 ++++++- .../adapter/enumerable/EnumUtilsTest.java | 23 ++++++++++ .../org/apache/calcite/test/JdbcTest.java | 43 +++++++++++++++++++ 4 files changed, 90 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java index 9cd7ba802bdd..fd60a29ae750 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java @@ -539,6 +539,15 @@ public static Expression convert(Expression operand, Type fromType, } } } + if (toType == Number.class + && (fromType == Object.class || fromType == String.class)) { + // E.g. from "Object" to "Number". + // Generate "x == null ? null : SqlFunctions.toBigDecimal(x)". + return Expressions.condition( + Expressions.equal(operand, RexImpTable.NULL_EXPR), + RexImpTable.NULL_EXPR, + Expressions.call(SqlFunctions.class, "toBigDecimal", operand)); + } if (toPrimitive != null) { if (fromPrimitive != null) { // E.g. from "float" to "double" diff --git a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java index d50d7279a538..65a26e05d6c0 100644 --- a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java +++ b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java @@ -5584,7 +5584,18 @@ public static double toDouble(Object o) { } public static BigDecimal toBigDecimal(String s) { - return new BigDecimal(s.trim()); + if (s == null) { + throw new NumberFormatException( + "Cannot convert null string to BigDecimal"); + } + try { + return new BigDecimal(s.trim()); + } catch (NumberFormatException e) { + NumberFormatException ex = + new NumberFormatException("Invalid value for BigDecimal: \"" + s + "\""); + ex.initCause(e); + throw ex; + } } public static BigDecimal toBigDecimal(Number number) { @@ -5597,6 +5608,9 @@ public static BigDecimal toBigDecimal(Number number) { } public static BigDecimal toBigDecimal(Object o) { + if (o == null) { + throw new NumberFormatException("Cannot convert null to BigDecimal"); + } return o instanceof Number ? toBigDecimal((Number) o) : toBigDecimal(o.toString()); } diff --git a/core/src/test/java/org/apache/calcite/adapter/enumerable/EnumUtilsTest.java b/core/src/test/java/org/apache/calcite/adapter/enumerable/EnumUtilsTest.java index 70370d7bb1ab..d92f50acfae3 100644 --- a/core/src/test/java/org/apache/calcite/adapter/enumerable/EnumUtilsTest.java +++ b/core/src/test/java/org/apache/calcite/adapter/enumerable/EnumUtilsTest.java @@ -41,6 +41,29 @@ */ public final class EnumUtilsTest { + /** Test case for + * [CALCITE-6284] + * Invalid conversion triggers ClassCastException. */ + @Test void testObjectToNumberConvert() { + // Object x; + final ParameterExpression objectVariable = + Expressions.parameter(0, Object.class, "x"); + final Expression objectToNumber = + EnumUtils.convert(objectVariable, Number.class); + assertThat(Expressions.toString(objectToNumber), + is("x == null ? (java.math.BigDecimal) null" + + " : org.apache.calcite.runtime.SqlFunctions.toBigDecimal(x)")); + + // String s; + final ParameterExpression stringVariable = + Expressions.parameter(0, String.class, "s"); + final Expression stringToNumber = + EnumUtils.convert(stringVariable, Number.class); + assertThat(Expressions.toString(stringToNumber), + is("s == null ? (java.math.BigDecimal) null" + + " : org.apache.calcite.runtime.SqlFunctions.toBigDecimal(s)")); + } + @Test void testDateTypeToInnerTypeConvert() { // java.sql.Date x; final ParameterExpression date = diff --git a/core/src/test/java/org/apache/calcite/test/JdbcTest.java b/core/src/test/java/org/apache/calcite/test/JdbcTest.java index 6dc4a48dc99b..a75983c76a98 100644 --- a/core/src/test/java/org/apache/calcite/test/JdbcTest.java +++ b/core/src/test/java/org/apache/calcite/test/JdbcTest.java @@ -9696,6 +9696,49 @@ void checkCalciteSchemaGetSubSchemaMap(boolean cache) { } } + /** Test case for + * [CALCITE-6284] + * Invalid conversion triggers ClassCastException. */ + @Test void bindStringParameter() { + for (SqlTypeName tpe : SqlTypeName.INT_TYPES) { + final String sql = + "with cte as (select cast(100 as " + tpe.getName() + ") as empid)" + + "select * from cte where empid = ?"; + + CalciteAssert.hr() + .query(sql) + .consumesPreparedStatement(p -> { + p.setString(1, "100"); + }) + .returnsUnordered("EMPID=100"); + } + } + + @Test void bindInvalidStringParameter() { + for (SqlTypeName tpe : SqlTypeName.INT_TYPES) { + final String sql = + "with cte as (select cast(100 as " + tpe.getName() + ") as empid)" + + "select * from cte where empid = ?"; + + final SQLException e = + assertThrows(SQLException.class, + () -> CalciteAssert.hr() + .query(sql) + .consumesPreparedStatement(p -> { + p.setString(1, "abc"); + }) + .returnsUnordered("")); + // Should produce a meaningful error, not ClassCastException + final Throwable cause = e.getCause(); + assertThat("Expected NumberFormatException for tpe=" + tpe, + cause, instanceOf(NumberFormatException.class)); + assertThat("Error message should contain the invalid value", + cause.getMessage(), containsString("abc")); + assertThat("Original NumberFormatException should be preserved as cause", + cause.getCause(), instanceOf(NumberFormatException.class)); + } + } + @Test void bindShortParameter() { for (SqlTypeName tpe : SqlTypeName.INT_TYPES) { final String sql = From c5fe0ce7e149221bb7d07dcd707efacd0a2e54fd Mon Sep 17 00:00:00 2001 From: xuzifu666 <1206332514@qq.com> Date: Wed, 19 Aug 2026 19:08:42 +0800 Subject: [PATCH 488/562] [CALCITE-5168] Allow AS after parenthesized JOIN --- core/src/main/codegen/templates/Parser.jj | 4 +-- .../calcite/sql/validate/AliasNamespace.java | 16 ++++++++++- .../sql/validate/SqlValidatorImpl.java | 10 +++++-- .../apache/calcite/test/SqlValidatorTest.java | 17 +++++++----- core/src/test/resources/sql/join.iq | 17 ++++++++++++ .../calcite/sql/parser/SqlParserTest.java | 27 ++++++++++--------- 6 files changed, 67 insertions(+), 24 deletions(-) diff --git a/core/src/main/codegen/templates/Parser.jj b/core/src/main/codegen/templates/Parser.jj index 185a85f50836..16bf6b5e9509 100644 --- a/core/src/main/codegen/templates/Parser.jj +++ b/core/src/main/codegen/templates/Parser.jj @@ -2516,9 +2516,7 @@ SqlNode TableRef3(ExprContext exprContext, boolean lateral) : // Standard SQL (and Postgres) allow applying "AS alias" to a JOIN, // e.g. "FROM (a CROSS JOIN b) AS c". The new alias obscures the // internal aliases, and columns cannot be referenced if they are - // not unique. TODO: Support this behavior; see - // [CALCITE-5168] Allow AS after parenthesized JOIN - checkNotJoin(tableRef); + // not unique. if (columnAliasList == null) { tableRef = SqlStdOperatorTable.AS.createCall( Span.of(tableRef).end(this), tableRef, alias); diff --git a/core/src/main/java/org/apache/calcite/sql/validate/AliasNamespace.java b/core/src/main/java/org/apache/calcite/sql/validate/AliasNamespace.java index 04296d98172d..99f84154e77b 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/AliasNamespace.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/AliasNamespace.java @@ -20,6 +20,7 @@ import org.apache.calcite.rel.type.RelDataTypeFactoryImpl; import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.rel.type.SingleColumnAliasRelDataType; +import org.apache.calcite.rel.type.StructKind; import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlCall; import org.apache.calcite.sql.SqlIdentifier; @@ -82,7 +83,20 @@ protected AliasNamespace( final List operands = call.getOperandList(); final SqlValidatorNamespace childNs = validator.getNamespaceOrThrow(operands.get(0)); - final RelDataType rowType = childNs.getRowTypeSansSystemColumns(); + final RelDataType rowType0 = childNs.getRowTypeSansSystemColumns(); + final RelDataType rowType; + if (rowType0.isStruct()) { + rowType = rowType0; + } else { + // Joins produce RelCrossType, which is not a struct. Convert to a struct + // so that columns can be resolved via the alias. + rowType = validator.getTypeFactory().builder() + .kind(StructKind.FULLY_QUALIFIED) + .addAll( + Util.transform(rowType0.getFieldList(), + f -> Pair.of(f.getName(), f.getType()))) + .build(); + } final RelDataType aliasedType; if (operands.size() == 2) { final SqlNode node = operands.get(0); diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java index c2d12ffd186d..6642a1522b76 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java @@ -2864,11 +2864,17 @@ private SqlNode registerFrom( expr = call.operand(0); final boolean needAliasNamespace = call.operandCount() > 2 || expr.getKind() == SqlKind.VALUES || expr.getKind() == SqlKind.UNNEST - || expr.getKind() == SqlKind.COLLECTION_TABLE; + || expr.getKind() == SqlKind.COLLECTION_TABLE + || expr.getKind() == SqlKind.JOIN; + // For an aliased join, the join's children must not be visible outside + // the alias. Prevent JoinScope.addChild from propagating children to + // the using scope by using parentScope. + final SqlValidatorScope exprUsingScope = + expr.getKind() == SqlKind.JOIN ? parentScope : usingScope; newExpr = registerFrom( parentScope, - usingScope, + exprUsingScope, !needAliasNamespace, expr, enclosingNode, diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index f0494539f0e5..854a5ce8fff2 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -6649,12 +6649,17 @@ void testReturnsCorrectRowTypeOnCombinedJoin() { sql("select * from (emp join bonus using (job))\n" + "join dept using (deptno)").ok(); - // Cannot alias a JOIN (until - // [CALCITE-5168] Allow AS after parenthesized JOIN - // is fixed). - sql("select * from (emp ^join^ bonus using (job)) as x\n" - + "join dept using (deptno)") - .fails("Join expression encountered in illegal context"); + // [CALCITE-5168] Allow AS after parenthesized JOIN. + sql("select x.empno from (emp cross join dept) as x").ok(); + sql("select x.empno from (emp join bonus using (job)) as x").ok(); + sql("select x.a from ((select empno from emp) cross join " + + "(select deptno from dept)) as x (a, c)") + .ok(); + // Inner aliases are obscured by the new alias. + sql("select ^emp^.empno from (emp cross join dept) as x") + .fails("Table 'EMP' not found"); + sql("select ^bonus^.job from (emp join bonus using (job)) as x") + .fails("Table 'BONUS' not found"); sql("select * from (emp join bonus using (job))\n" + "join dept using (^dname^)") .fails("Column 'DNAME' not found in any table"); diff --git a/core/src/test/resources/sql/join.iq b/core/src/test/resources/sql/join.iq index 35363424e9a2..ca1ba1451cff 100644 --- a/core/src/test/resources/sql/join.iq +++ b/core/src/test/resources/sql/join.iq @@ -302,6 +302,23 @@ cross join (bonus as b !ok +# [CALCITE-5168] Allow AS after parenthesized JOIN +select d.dname, j.empno, j.ename +from dept as d +cross join (emp as e cross join (values (1)) as b(dummy)) as j +where j.empno = 7369; ++------------+-------+-------+ +| DNAME | EMPNO | ENAME | ++------------+-------+-------+ +| ACCOUNTING | 7369 | SMITH | +| RESEARCH | 7369 | SMITH | +| SALES | 7369 | SMITH | +| OPERATIONS | 7369 | SMITH | ++------------+-------+-------+ +(4 rows) + +!ok + # Join plus TABLE select e.ename, d.dname from emp as e diff --git a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java index 5e4271e4c8fc..32f52812ce93 100644 --- a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java +++ b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java @@ -7860,9 +7860,9 @@ private static Consumer> checkWarnings( // is syntactically and semantically valid; but // "select t.i from (t cross join u) as x" // is semantically invalid. - // TODO: Support this in Calcite. - sql("select * from (t cross ^join^ u) as x") - .fails("Join expression encountered in illegal context"); + sql("select * from (t cross join u) as x") + .ok("SELECT *\n" + + "FROM (`T` CROSS JOIN `U`) AS `X`"); sql("select *\n" + "from (t cross ^join^ u)\n" + " tablesample substitute('medium')") @@ -8026,21 +8026,24 @@ private static Consumer> checkWarnings( + "CROSS JOIN (TABLE `T2`)"; sql(sql3).ok(expected3); - // Adding an alias to the previous query makes it invalid - // (The error message and location could be improved) + final String expected4 = "SELECT *\n" + + "FROM ((SELECT *\n" + + "FROM `T`) CROSS JOIN (TABLE `T2`)) AS `X`"; final String sql4 = "SELECT *\n" + "FROM ((((((((((((SELECT * FROM t)))\n" - + " cross ^join^ ((table t2))))))))))) X"; + + " cross join ((table t2))))))))))) X"; final String sql5 = "SELECT *\n" + "FROM ((((((((((((SELECT * FROM t)))\n" - + " cross ^join^ ((table t2))))))))))) as X"; + + " cross join ((table t2))))))))))) as X"; final String sql6 = "SELECT *\n" + "FROM ((((((((((((SELECT * FROM t)))\n" - + " cross ^join^ ((table t2))))))))))) as X (a, b, c)"; - final String message = "Join expression encountered in illegal context"; - sql(sql4).fails(message); - sql(sql5).fails(message); - sql(sql6).fails(message); + + " cross join ((table t2))))))))))) as X (a, b, c)"; + sql(sql4).ok(expected4); + sql(sql5).ok(expected4); + final String expected6 = "SELECT *\n" + + "FROM ((SELECT *\n" + + "FROM `T`) CROSS JOIN (TABLE `T2`)) AS `X` (`A`, `B`, `C`)"; + sql(sql6).ok(expected6); } @Test void testProcedureCall() { From 270fc1bd3f2303158823b7d2c21cda5a87932d2e Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Fri, 21 Aug 2026 18:06:54 -0700 Subject: [PATCH 489/562] [CALCITE-7488] ProjectJoinTransposeRule produces row-type mismatch when pushing a compound expression containing a nullability-narrowing CAST through an outer Join Signed-off-by: Mihai Budiu --- .../calcite/rel/rules/PushProjector.java | 11 ++- .../apache/calcite/test/RelOptRulesTest.java | 63 +++++++++++++++ .../apache/calcite/test/RelOptRulesTest.xml | 80 +++++++++++++++++++ 3 files changed, 150 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rel/rules/PushProjector.java b/core/src/main/java/org/apache/calcite/rel/rules/PushProjector.java index caa9b4d4dedf..9dc5172896cb 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/PushProjector.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/PushProjector.java @@ -726,14 +726,17 @@ private static class InputSpecialOpFinder extends RexVisitorImpl { return null; } - private boolean isStrong(final ImmutableBitSet exprArgs, final RexNode call) { + private boolean canPush(final ImmutableBitSet exprArgs, final RexNode call) { // If the expressions do not use any of the inputs that require output to be null, // no need to check. Otherwise, check that the expression is null. // For example, in an "left outer join", we don't require that expressions // pushed down into the left input to be strong. On the other hand, // expressions pushed into the right input must be. In that case, // strongFields == right input fields. - return !strongFields.intersects(exprArgs) || strong.isNull(call); + if (!strongFields.intersects(exprArgs)) { + return true; + } + return strong.isNull(call) && call.getType().isNullable(); } private boolean preserve(RexNode call) { @@ -743,13 +746,13 @@ private boolean preserve(RexNode call) { // it only references expressions on the right final ImmutableBitSet exprArgs = RelOptUtil.InputFinder.bits(call); if (exprArgs.cardinality() > 0) { - if (leftFields.contains(exprArgs) && isStrong(exprArgs, call)) { + if (leftFields.contains(exprArgs) && canPush(exprArgs, call)) { if (!preserveLeft.contains(call)) { preserveLeft.add(call); } return true; } else if (requireNonNull(rightFields, "rightFields").contains(exprArgs) - && isStrong(exprArgs, call)) { + && canPush(exprArgs, call)) { requireNonNull(preserveRight, "preserveRight"); if (!preserveRight.contains(call)) { preserveRight.add(call); diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index 06ee61c26c17..76ff0a831335 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -1609,6 +1609,69 @@ private static RelNode zeroColumnJoinInputRelFn(RelBuilder b, .build(); } + /** Test case for + * [CALCITE-7488] + * ProjectJoinTransposeRule produces row-type mismatch when pushing a compound + * expression containing a nullability-narrowing CAST through an outer + * Join. */ + @Test void testProjectJoinTransposeNarrowingCastInCompoundExpr() { + relFn(b -> castInCaseRelFn(b, JoinRelType.LEFT, true)) + .withRule(CoreRules.PROJECT_JOIN_TRANSPOSE).check(); + } + + /** As {@link #testProjectJoinTransposeNarrowingCastInCompoundExpr()}, but the + * null-generating input of the join is the left one. */ + @Test void testProjectJoinTransposeNarrowingCastInCompoundExprRightJoin() { + relFn(b -> castInCaseRelFn(b, JoinRelType.RIGHT, true)) + .withRule(CoreRules.PROJECT_JOIN_TRANSPOSE).check(); + } + + /** As {@link #testProjectJoinTransposeNarrowingCastInCompoundExpr()}, but both + * inputs of the join are null-generating. */ + @Test void testProjectJoinTransposeNarrowingCastInCompoundExprFullJoin() { + relFn(b -> castInCaseRelFn(b, JoinRelType.FULL, true)) + .withRule(CoreRules.PROJECT_JOIN_TRANSPOSE).check(); + } + + /** Without the narrowing casts the expression has a nullable type, so pushing + * it into the null-generating input does not change its type, and the rule + * still pushes it. */ + @Test void testProjectJoinTransposeNullableCompoundExpr() { + relFn(b -> castInCaseRelFn(b, JoinRelType.LEFT, false)) + .withRule(CoreRules.PROJECT_JOIN_TRANSPOSE).check(); + } + + /** Builds {@code Project(CASE(DNAME IS NOT NULL, DNAME, LOC))} over an outer + * join of EMP and DEPT, with DEPT on the null-generating side. The CASE is + * null whenever DEPT's columns are null, so it is a candidate for being pushed + * into the DEPT input. */ + private static RelNode castInCaseRelFn(RelBuilder b, JoinRelType joinType, + boolean narrowing) { + final RexBuilder rb = b.getRexBuilder(); + if (joinType == JoinRelType.RIGHT) { + b.scan("DEPT").scan("EMP"); + } else { + b.scan("EMP").scan("DEPT"); + } + b.join(joinType, + b.equals(b.field(2, 0, "DEPTNO"), b.field(2, 1, "DEPTNO"))); + RexNode dname = b.field("DEPT", "DNAME"); + RexNode loc = b.field("DEPT", "LOC"); + if (narrowing) { + dname = rb.makeCast(notNullType(b, dname), dname, false, false); + loc = rb.makeCast(notNullType(b, loc), loc, false, false); + } + return b.project( + b.call(SqlStdOperatorTable.CASE, + b.call(SqlStdOperatorTable.IS_NOT_NULL, b.field("DEPT", "DNAME")), + dname, loc)) + .build(); + } + + private static RelDataType notNullType(RelBuilder b, RexNode e) { + return b.getTypeFactory().createTypeWithNullability(e.getType(), false); + } + /** A SEMI, ANTI or LEFT_MARK join does not project its right input, so * {@link JoinProjectTransposeRule} must not pull projects above it. */ private void checkJoinProjectTransposeDoesNotMatch(JoinRelType type) { diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index 96ca66bbc72f..33d72da4d3da 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -12436,6 +12436,86 @@ LogicalProject(EXPR$0=[$1], EXPR$1=[$3]) LogicalProject($f1=[$1], EXPR$0=[ITEM($0, 0)]) LogicalProject(C_NATIONKEY=[$0], $f1=[ITEM($0, 0)]) LogicalTableScan(table=[[CATALOG, SALES, CUSTOMER]]) +]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 7d95022b6de81f34cf1fa6d77846064d1aa24341 Mon Sep 17 00:00:00 2001 From: Ruben Quesada Lopez Date: Fri, 21 Aug 2026 13:25:05 +0100 Subject: [PATCH 490/562] [CALCITE-7667] Improve string-literal encoding in pushdown translators (follow-up) --- .../apache/calcite/adapter/pig/PigFilter.java | 9 ++++++--- .../pig/PigFilterLiteralEscapeTest.java | 20 +++++++++++++++---- .../apache/calcite/test/PigAdapterTest.java | 6 +++--- 3 files changed, 25 insertions(+), 10 deletions(-) diff --git a/pig/src/main/java/org/apache/calcite/adapter/pig/PigFilter.java b/pig/src/main/java/org/apache/calcite/adapter/pig/PigFilter.java index f36b0dd6f5d3..cc6a8a8c0f07 100644 --- a/pig/src/main/java/org/apache/calcite/adapter/pig/PigFilter.java +++ b/pig/src/main/java/org/apache/calcite/adapter/pig/PigFilter.java @@ -135,9 +135,12 @@ private static boolean containsOnlyConjunctions(RexNode condition) { * Converts a literal to a Pig Latin string literal. */ private static String getLiteralAsString(RexLiteral literal) { - // Pig Latin string literals use `''` to represent a single `'` inside - // a `'...'` literal, so double any embedded `'` before wrapping final String raw = RexLiteral.stringValue(literal); - return '\'' + (raw != null ? raw.replace("'", "''") : null) + '\''; + // Escape before wrapping + return '\'' + + (raw != null + ? raw.replace("\\", "\\\\").replace("'", "\\'") + : null) + + '\''; } } diff --git a/pig/src/test/java/org/apache/calcite/adapter/pig/PigFilterLiteralEscapeTest.java b/pig/src/test/java/org/apache/calcite/adapter/pig/PigFilterLiteralEscapeTest.java index f90bbb3cbd70..a2cbebc2b4b8 100644 --- a/pig/src/test/java/org/apache/calcite/adapter/pig/PigFilterLiteralEscapeTest.java +++ b/pig/src/test/java/org/apache/calcite/adapter/pig/PigFilterLiteralEscapeTest.java @@ -57,20 +57,32 @@ private static String call(RexLiteral literal) throws Throwable { assertThat(call(charLiteral("alice")), is("'alice'")); } + @Test void valueWithBackslash() throws Throwable { + assertThat(call(charLiteral("a\\b")), is("'a\\\\b'")); + } + + @Test void valueWithBackslashBeforeApostrophe() throws Throwable { + assertThat(call(charLiteral("a\\'b")), is("'a\\\\\\'b'")); + } + + @Test void valueWithTrailingBackslash() throws Throwable { + assertThat(call(charLiteral("a\\")), is("'a\\\\'")); + } + @Test void valueWithApostrophe() throws Throwable { - assertThat(call(charLiteral("O'Brien")), is("'O''Brien'")); + assertThat(call(charLiteral("O'Brien")), is("'O\\'Brien'")); } @Test void valueWithApostropheAtTheEnd() throws Throwable { - assertThat(call(charLiteral("a'")), is("'a'''")); + assertThat(call(charLiteral("a'")), is("'a\\''")); } @Test void valueWithApostropheAtTheStart() throws Throwable { - assertThat(call(charLiteral("'a")), is("'''a'")); + assertThat(call(charLiteral("'a")), is("'\\'a'")); } @Test void valueWithMultipleApostrophes() throws Throwable { - assertThat(call(charLiteral("a''b")), is("'a''''b'")); + assertThat(call(charLiteral("a''b")), is("'a\\'\\'b'")); } @Test void emptyValue() throws Throwable { diff --git a/pig/src/test/java/org/apache/calcite/test/PigAdapterTest.java b/pig/src/test/java/org/apache/calcite/test/PigAdapterTest.java index 62b247a756b8..18a61e468b27 100644 --- a/pig/src/test/java/org/apache/calcite/test/PigAdapterTest.java +++ b/pig/src/test/java/org/apache/calcite/test/PigAdapterTest.java @@ -58,9 +58,9 @@ class PigAdapterTest extends AbstractPigTest { } @Test void testFilterWithSingleQuote() { - // A string literal containing a single quote must be doubled per Pig Latin + // A string literal containing a single quote must be escaped per Pig Latin // string-literal rules so it does not break out of the '...' literal in - // the generated FILTER statement. + // the generated FILTER statement. Verified against pig. CalciteAssert.that() .with(MODEL) .query("select * from \"t\" where \"tc0\" = 'a''b'") @@ -69,7 +69,7 @@ class PigAdapterTest extends AbstractPigTest { pigScriptChecker("t = LOAD '" + getFullPathForTestDataFile("data.txt") + "' USING PigStorage() AS (tc0:chararray, tc1:chararray);\n" - + "t = FILTER t BY (tc0 == 'a''b');")); + + "t = FILTER t BY (tc0 == 'a\\'b');")); } @Test void testImplWithMultipleFilters() { From 8cec041b3807ef3e2a644dad7e6900f4fd3044d6 Mon Sep 17 00:00:00 2001 From: Ruben Quesada Lopez Date: Fri, 21 Aug 2026 17:12:48 +0100 Subject: [PATCH 491/562] [CALCITE-7731] Bound plain-notation expansion of DECIMAL literals to prevent parse-time OutOfMemoryError --- .../calcite/config/CalciteSystemProperty.java | 16 ++++++++++++++++ .../calcite/rel/rel2sql/SqlImplementor.java | 8 ++++++-- .../org/apache/calcite/rex/RexBuilder.java | 4 ++++ .../apache/calcite/sql/SqlNumericLiteral.java | 4 ++++ .../java/org/apache/calcite/sql/SqlUtil.java | 14 ++++++++++++++ .../calcite/sql/parser/SqlParserUtil.java | 11 +++++++---- .../calcite/sql/parser/SqlParserTest.java | 19 +++++++++++++++++++ 7 files changed, 70 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/config/CalciteSystemProperty.java b/core/src/main/java/org/apache/calcite/config/CalciteSystemProperty.java index 39ba5c2f3281..0d14fed45b84 100644 --- a/core/src/main/java/org/apache/calcite/config/CalciteSystemProperty.java +++ b/core/src/main/java/org/apache/calcite/config/CalciteSystemProperty.java @@ -492,6 +492,22 @@ public final class CalciteSystemProperty { public static final CalciteSystemProperty MODEL_CLASSES_DENIED = stringProperty("calcite.model.classes.denied", ""); + /** + * Maximum number of decimal digits that the plain-notation expansion of a {@code DECIMAL} + * literal may contain. + * + *

      {@link java.math.BigDecimal} accepts any {@code int} exponent, so without a bound + * a ~15-character literal such as {@code DECIMAL '1E2147483647'} would ask + * {@link java.math.BigDecimal#toPlainString()} to materialize one character per digit, + * i.e. a multi-gigabyte allocation that would end in {@link OutOfMemoryError} inside the parser. + * + *

      Default {@code 10000} is well beyond any dialect's realistic maximum {@code DECIMAL} + * precision while still bounding worst-case allocation to ~20 KB. Raise if a dialect + * legitimately needs more. + */ + public static final CalciteSystemProperty MAX_DECIMAL_LITERAL_PLAIN_DIGITS = + intProperty("calcite.parser.maxDecimalLiteralPlainDigits", 10_000, v -> v > 0); + private static CalciteSystemProperty booleanProperty(String key, boolean defaultValue) { // Note that "" -> true (convenient for command-lines flags like '-Dflag') diff --git a/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java b/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java index 60b068438ab3..a7a52579112c 100644 --- a/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java +++ b/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java @@ -1758,8 +1758,12 @@ public static SqlNode toSql(RexLiteral literal) { } return SqlLiteral.createApproxNumeric(d.toString(), POS); } else { - return SqlLiteral.createExactNumeric( - castNonNull(literal.getValueAs(BigDecimal.class)).toPlainString(), POS); + final BigDecimal bd = castNonNull(literal.getValueAs(BigDecimal.class)); + if (!SqlUtil.isBoundedDecimal(bd)) { + throw new IllegalStateException( + "DECIMAL literal exceeds the configured plain-notation bound: " + bd); + } + return SqlLiteral.createExactNumeric(bd.toPlainString(), POS); } } case APPROXIMATE_NUMERIC: diff --git a/core/src/main/java/org/apache/calcite/rex/RexBuilder.java b/core/src/main/java/org/apache/calcite/rex/RexBuilder.java index 667de88227f5..efcb80f20367 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexBuilder.java +++ b/core/src/main/java/org/apache/calcite/rex/RexBuilder.java @@ -1549,6 +1549,10 @@ protected RexLiteral makeLiteral( } else if (type.getScale() != RelDataType.SCALE_NOT_SPECIFIED) { o = ((BigDecimal) o).setScale(type.getScale(), typeFactory.getTypeSystem().roundingMode()); if (type.getScale() < 0) { + if (!SqlUtil.isBoundedDecimal((BigDecimal) o)) { + throw new IllegalArgumentException("Cannot convert " + o + " to " + type + + ": plain-notation expansion exceeds the configured bound"); + } o = new BigDecimal(((BigDecimal) o).toPlainString()); } } diff --git a/core/src/main/java/org/apache/calcite/sql/SqlNumericLiteral.java b/core/src/main/java/org/apache/calcite/sql/SqlNumericLiteral.java index 29572c136079..575388ed5621 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlNumericLiteral.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlNumericLiteral.java @@ -90,6 +90,10 @@ public boolean isExact() { @Override public String toValue() { final BigDecimal bd = getValueNonNull(); if (exact) { + if (!SqlUtil.isBoundedDecimal(bd)) { + throw new IllegalArgumentException("DECIMAL literal '" + bd + + "' exceeds the configured plain-notation bound"); + } return bd.toPlainString(); } return Util.toScientificNotation(bd); diff --git a/core/src/main/java/org/apache/calcite/sql/SqlUtil.java b/core/src/main/java/org/apache/calcite/sql/SqlUtil.java index 1dba9d573e33..e6bccfe51e3e 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlUtil.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlUtil.java @@ -17,6 +17,7 @@ package org.apache.calcite.sql; import org.apache.calcite.avatica.util.ByteString; +import org.apache.calcite.config.CalciteSystemProperty; import org.apache.calcite.linq4j.Ord; import org.apache.calcite.linq4j.function.Functions; import org.apache.calcite.rel.RelNode; @@ -58,6 +59,7 @@ import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.checker.nullness.qual.PolyNull; +import java.math.BigDecimal; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.nio.charset.UnsupportedCharsetException; @@ -962,6 +964,18 @@ public static String getAliasedSignature( return ret.toString(); } + /** + * Returns whether {@code value}'s plain-notation expansion fits within + * the configured bound + * ({@link CalciteSystemProperty#MAX_DECIMAL_LITERAL_PLAIN_DIGITS}). + * Callers that are about to feed {@code value} to {@code toPlainString} + * (or the equivalent) must gate on this method first. + */ + public static boolean isBoundedDecimal(BigDecimal value) { + final long limit = CalciteSystemProperty.MAX_DECIMAL_LITERAL_PLAIN_DIGITS.value(); + return (long) value.precision() + Math.abs((long) value.scale()) <= limit; + } + /** * Wraps an exception with context. */ diff --git a/core/src/main/java/org/apache/calcite/sql/parser/SqlParserUtil.java b/core/src/main/java/org/apache/calcite/sql/parser/SqlParserUtil.java index c6eed1bce217..3c0b6bc99b6d 100644 --- a/core/src/main/java/org/apache/calcite/sql/parser/SqlParserUtil.java +++ b/core/src/main/java/org/apache/calcite/sql/parser/SqlParserUtil.java @@ -338,15 +338,18 @@ public static SqlDateLiteral parseDateLiteral(String s, SqlParserPos pos) { } public static SqlNumericLiteral parseDecimalLiteral(String s, SqlParserPos pos) { + final BigDecimal value; try { - // The s maybe scientific notation string,e.g. 1.2E-3, - // we need to convert it to 0.0012 - s = new BigDecimal(s).toPlainString(); + value = new BigDecimal(s); } catch (NumberFormatException e) { throw SqlUtil.newContextException(pos, RESOURCE.invalidLiteral(s, "DECIMAL")); } - return SqlLiteral.createExactNumeric(s, pos); + if (!SqlUtil.isBoundedDecimal(value)) { + throw SqlUtil.newContextException(pos, + RESOURCE.invalidLiteral(s, "DECIMAL")); + } + return SqlLiteral.createExactNumeric(value.toPlainString(), pos); } public static SqlTimeLiteral parseTimeLiteral(String s, SqlParserPos pos) { diff --git a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java index 32f52812ce93..a64edcd6fb4d 100644 --- a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java +++ b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java @@ -1056,6 +1056,25 @@ private void checkLarge(int n) { .ok("SELECT 999"); } + /** Test case for [CALCITE-7731] + * Bound plain-notation expansion of DECIMAL literals to prevent parse-time + * OutOfMemoryError. */ + @Test void testDecimalLiteralWithOutOfRangeExponent() { + sql("select DECIMAL ^'1E2147483647'^") + .fails("(?s)Literal '1E2147483647' can not be parsed to type 'DECIMAL'.*"); + sql("select DECIMAL ^'1E-2147483647'^") + .fails("(?s)Literal '1E-2147483647' can not be parsed to type 'DECIMAL'.*"); + sql("select DECIMAL ^'1E1000000000'^") + .fails("(?s)Literal '1E1000000000' can not be parsed to type 'DECIMAL'.*"); + sql("select DECIMAL ^'-9.9E999999999'^") + .fails("(?s)Literal '-9.9E999999999' can not be parsed to type 'DECIMAL'.*"); + // Exponents within the bound still expand to plain notation. + sql("select DECIMAL '1E10'") + .ok("SELECT 10000000000"); + sql("select DECIMAL '1E-10'") + .ok("SELECT 0.0000000001"); + } + @Test void testDecimalWithScale() { sql("select cast(15 as decimal(3, 1))") .ok("SELECT CAST(15 AS DECIMAL(3, 1))"); From d240f2ff2211795515ff5f6ef1985767e0e60924 Mon Sep 17 00:00:00 2001 From: xuzifu666 <1206332514@qq.com> Date: Sat, 22 Aug 2026 18:48:43 +0800 Subject: [PATCH 492/562] [CALCITE-6126] Return type check fails with OVER --- .../test/java/org/apache/calcite/test/RelMetadataTest.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java b/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java index a40927c37a04..abb934457a0e 100644 --- a/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java @@ -2330,8 +2330,6 @@ private void checkColumnUniquenessForJoin(String sql) { .assertThatUniqueKeysAre(bitSetOf(0)); } -// TODO: Enable when CALCITE-6126 fixed -/* @Test void testOverByNonKey() { sql("select sal,\n" + "max(deptno) over (partition BY sal rows between 2 preceding and 0 following) maxDept,\n" @@ -2342,16 +2340,12 @@ private void checkColumnUniquenessForJoin(String sql) { .assertThatAreColumnsUnique(bitSetOf(2), is(false)) .assertThatUniqueKeysAre(); } -*/ -// TODO: Enable when CALCITE-6126 fixed -/* @Test void testOverNoPartitioning() { sql("select max(empno) over (rows between 2 preceding and 0 following) maxEmp from emp") .assertThatAreColumnsUnique(bitSetOf(0), is(false)) .assertThatUniqueKeysAre(); } -*/ @Test void testNoGroupBy() { sql("select max(sal), count(*) from emp") From 5d7f75d5d80ddacee58c618ee6d5ce2879a01026 Mon Sep 17 00:00:00 2001 From: Sergey Nuyanzin Date: Wed, 19 Aug 2026 21:08:31 +0200 Subject: [PATCH 493/562] [CALCITE-4581] The digest of TableScan should consider table hints --- .../apache/calcite/rel/logical/LogicalTableScan.java | 5 +++++ .../org/apache/calcite/test/SqlHintsConverterTest.java | 8 ++++++++ .../org/apache/calcite/test/SqlHintsConverterTest.xml | 10 ++++++++++ 3 files changed, 23 insertions(+) diff --git a/core/src/main/java/org/apache/calcite/rel/logical/LogicalTableScan.java b/core/src/main/java/org/apache/calcite/rel/logical/LogicalTableScan.java index 7424c007b8e1..46d117252450 100644 --- a/core/src/main/java/org/apache/calcite/rel/logical/LogicalTableScan.java +++ b/core/src/main/java/org/apache/calcite/rel/logical/LogicalTableScan.java @@ -23,6 +23,7 @@ import org.apache.calcite.rel.RelCollationTraitDef; import org.apache.calcite.rel.RelInput; import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.RelWriter; import org.apache.calcite.rel.core.TableScan; import org.apache.calcite.rel.hint.RelHint; import org.apache.calcite.schema.Table; @@ -94,6 +95,10 @@ public LogicalTableScan(RelInput input) { return this; } + @Override public RelWriter explainTerms(RelWriter pw) { + return super.explainTerms(pw).itemIf("hints", getHints(), !getHints().isEmpty()); + } + /** Creates a LogicalTableScan. * * @param cluster Cluster diff --git a/core/src/test/java/org/apache/calcite/test/SqlHintsConverterTest.java b/core/src/test/java/org/apache/calcite/test/SqlHintsConverterTest.java index 346e0c57e6b9..89f006526909 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlHintsConverterTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlHintsConverterTest.java @@ -681,6 +681,14 @@ public final Fixture sql(String sql) { + RelOptUtil.toString(newRel)); } + /** Test case for + * [CALCITE-4581] + * The digest of TableScan should consider table hints. */ + @Test void testTableScanHint() { + final String sql = "select * from emp /*+ index(idx1) */"; + sql(sql).ok(); + } + //~ Methods ---------------------------------------------------------------- private static boolean equalsStringList(List l, List r) { diff --git a/core/src/test/resources/org/apache/calcite/test/SqlHintsConverterTest.xml b/core/src/test/resources/org/apache/calcite/test/SqlHintsConverterTest.xml index d06db165d0e2..3252d2790fa5 100644 --- a/core/src/test/resources/org/apache/calcite/test/SqlHintsConverterTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/SqlHintsConverterTest.xml @@ -447,6 +447,16 @@ TableScan:[[PROPERTIES inheritPath:[] options:{K1=v1, K2=v2}]] + + + + + + + + From 91c7b2f64c53d80e3d3b2cec032ebe0578998fff Mon Sep 17 00:00:00 2001 From: Vladimir Sitnikov Date: Sun, 23 Aug 2026 22:11:05 +0300 Subject: [PATCH 494/562] [CALCITE-7736] Replace the Checker Framework with NullAway and JSpecify The Checker Framework verified nullness through a Gradle plugin of its own, a set of `.astub` files that patched the nullness of the JDK and of third-party libraries, and two dedicated CI jobs. NullAway runs as an Error Prone check instead, so it needs no separate plugin and no stub files: it ships nullness models for the JDK and for popular libraries, and JSpecify supplies the annotations. Build: * drop the `org.checkerframework` plugin and its configuration block, and delete the 48 `.astub` files * add `com.uber.nullaway:nullaway` to the `errorprone` configuration and configure it for JSpecify, including the experimental generics support (`JSpecifyExperimental`, `HandleWildcardGenerics`, `JSpecifyJDKModels`, `WarnOnGenericInferenceFailure`) * replace `org.checkerframework:checker-qual` with `org.jspecify:jspecify` * raise Error Prone to 2.50.0 and the Error Prone plugin to 5.1.0, which NullAway requires NullAway is an error in the projects listed in `nullawayProjects` and is off elsewhere, so a nullness problem fails one CI job rather than every test job. CI: * drop the two `CheckerFramework` jobs * fold nullness verification into the `errorprone` job, which moves to JDK 21 because Error Prone 2.43 and later require it This commit only moves the tooling; the source still carries Checker Framework annotations and is migrated by the commits that follow. Co-Authored-By: Claude Opus 5 --- .editorconfig | 3 - .github/workflows/main.yml | 47 +----- bom/build.gradle.kts | 2 +- build.gradle.kts | 136 ++++++++++++------ core/build.gradle.kts | 4 +- druid/build.gradle.kts | 2 +- elasticsearch/build.gradle.kts | 2 +- example/csv/build.gradle.kts | 2 +- example/function/build.gradle.kts | 2 +- file/build.gradle.kts | 2 +- geode/build.gradle.kts | 2 +- gradle.properties | 8 +- kafka/build.gradle.kts | 2 +- linq4j/build.gradle.kts | 5 +- piglet/build.gradle.kts | 2 +- plus/build.gradle.kts | 2 +- settings.gradle.kts | 1 - site/develop/index.md | 49 +++++-- .../config/checkerframework/Collection.astub | 27 ---- .../config/checkerframework/Constructor.astub | 23 --- src/main/config/checkerframework/Field.astub | 25 ---- .../checkerframework/InvocationHandler.astub | 23 --- src/main/config/checkerframework/List.astub | 31 ---- src/main/config/checkerframework/Map.astub | 35 ----- src/main/config/checkerframework/Method.astub | 23 --- .../checkerframework/MethodHandle.astub | 23 --- .../config/checkerframework/Objects.astub | 38 ----- src/main/config/checkerframework/Proxy.astub | 23 --- src/main/config/checkerframework/Set.astub | 27 ---- src/main/config/checkerframework/String.astub | 23 --- src/main/config/checkerframework/URI.astub | 24 ---- .../aggdesigner/Attribute.astub | 25 ---- .../checkerframework/aggdesigner/Table.astub | 23 --- .../avatica/AvaticaFactory.astub | 31 ---- .../avatica/AvaticaPreparedStatement.astub | 28 ---- .../avatica/AvaticaResultMetaData.astub | 26 ---- .../avatica/AvaticaResultSet.astub | 30 ---- .../avatica/AvaticaResultSetMetaData.astub | 30 ---- .../avatica/AvaticaSite.astub | 61 -------- .../avatica/AvaticaStatement.astub | 29 ---- .../avatica/ColumnMetaData.astub | 43 ------ .../avatica/ConnectionConfigImpl.astub | 25 ---- .../avatica/ConnectionProperty.astub | 33 ----- .../checkerframework/avatica/Handler.astub | 27 ---- .../checkerframework/avatica/Meta.astub | 43 ------ .../checkerframework/avatica/MetaImpl.astub | 64 --------- .../avatica/TimeUnitRange.astub | 25 ---- .../commons-dbcp2/BasicDataSource.astub | 29 ---- .../esri-geometry/OperatorBoundary.astub | 23 --- .../OperatorSimpleRelation.astub | 24 ---- .../checkerframework/guava/Function.astub | 30 ---- .../checkerframework/guava/Iterables.astub | 30 ---- .../checkerframework/guava/Ordering.astub | 32 ----- .../checkerframework/guava/Predicate.astub | 29 ---- .../jackson/ObjectMapper.astub | 25 ---- .../jackson/ObjectWriter.astub | 25 ---- .../janino/ClassBodyEvaluator.astub | 23 --- .../janino/IClassBodyEvaluator.astub | 23 --- .../janino/ISimpleCompiler.astub | 23 --- .../janino/JavaSourceClassLoader.astub | 35 ----- .../checkerframework/janino/Scanner.astub | 23 --- .../checkerframework/jdbc/Connection.astub | 23 --- .../jdbc/DatabaseMetaData.astub | 23 --- .../jsonpath/JacksonJsonProvider.astub | 23 --- .../checkerframework/slf4j/Logger.astub | 101 ------------- .../slf4j/MessageFormatter.astub | 23 --- testkit/build.gradle.kts | 2 +- 67 files changed, 152 insertions(+), 1578 deletions(-) delete mode 100644 src/main/config/checkerframework/Collection.astub delete mode 100644 src/main/config/checkerframework/Constructor.astub delete mode 100644 src/main/config/checkerframework/Field.astub delete mode 100644 src/main/config/checkerframework/InvocationHandler.astub delete mode 100644 src/main/config/checkerframework/List.astub delete mode 100644 src/main/config/checkerframework/Map.astub delete mode 100644 src/main/config/checkerframework/Method.astub delete mode 100644 src/main/config/checkerframework/MethodHandle.astub delete mode 100644 src/main/config/checkerframework/Objects.astub delete mode 100644 src/main/config/checkerframework/Proxy.astub delete mode 100644 src/main/config/checkerframework/Set.astub delete mode 100644 src/main/config/checkerframework/String.astub delete mode 100644 src/main/config/checkerframework/URI.astub delete mode 100644 src/main/config/checkerframework/aggdesigner/Attribute.astub delete mode 100644 src/main/config/checkerframework/aggdesigner/Table.astub delete mode 100644 src/main/config/checkerframework/avatica/AvaticaFactory.astub delete mode 100644 src/main/config/checkerframework/avatica/AvaticaPreparedStatement.astub delete mode 100644 src/main/config/checkerframework/avatica/AvaticaResultMetaData.astub delete mode 100644 src/main/config/checkerframework/avatica/AvaticaResultSet.astub delete mode 100644 src/main/config/checkerframework/avatica/AvaticaResultSetMetaData.astub delete mode 100644 src/main/config/checkerframework/avatica/AvaticaSite.astub delete mode 100644 src/main/config/checkerframework/avatica/AvaticaStatement.astub delete mode 100644 src/main/config/checkerframework/avatica/ColumnMetaData.astub delete mode 100644 src/main/config/checkerframework/avatica/ConnectionConfigImpl.astub delete mode 100644 src/main/config/checkerframework/avatica/ConnectionProperty.astub delete mode 100644 src/main/config/checkerframework/avatica/Handler.astub delete mode 100644 src/main/config/checkerframework/avatica/Meta.astub delete mode 100644 src/main/config/checkerframework/avatica/MetaImpl.astub delete mode 100644 src/main/config/checkerframework/avatica/TimeUnitRange.astub delete mode 100644 src/main/config/checkerframework/commons-dbcp2/BasicDataSource.astub delete mode 100644 src/main/config/checkerframework/esri-geometry/OperatorBoundary.astub delete mode 100644 src/main/config/checkerframework/esri-geometry/OperatorSimpleRelation.astub delete mode 100644 src/main/config/checkerframework/guava/Function.astub delete mode 100644 src/main/config/checkerframework/guava/Iterables.astub delete mode 100644 src/main/config/checkerframework/guava/Ordering.astub delete mode 100644 src/main/config/checkerframework/guava/Predicate.astub delete mode 100644 src/main/config/checkerframework/jackson/ObjectMapper.astub delete mode 100644 src/main/config/checkerframework/jackson/ObjectWriter.astub delete mode 100644 src/main/config/checkerframework/janino/ClassBodyEvaluator.astub delete mode 100644 src/main/config/checkerframework/janino/IClassBodyEvaluator.astub delete mode 100644 src/main/config/checkerframework/janino/ISimpleCompiler.astub delete mode 100644 src/main/config/checkerframework/janino/JavaSourceClassLoader.astub delete mode 100644 src/main/config/checkerframework/janino/Scanner.astub delete mode 100644 src/main/config/checkerframework/jdbc/Connection.astub delete mode 100644 src/main/config/checkerframework/jdbc/DatabaseMetaData.astub delete mode 100644 src/main/config/checkerframework/jsonpath/JacksonJsonProvider.astub delete mode 100644 src/main/config/checkerframework/slf4j/Logger.astub delete mode 100644 src/main/config/checkerframework/slf4j/MessageFormatter.astub diff --git a/.editorconfig b/.editorconfig index d7e51acbd562..5439c0bd7a59 100644 --- a/.editorconfig +++ b/.editorconfig @@ -20,9 +20,6 @@ ij_java_use_single_class_imports = true max_line_length = 100 ij_any_wrap_long_lines = true -[*.astub] -indent_size = 2 - [*.java] # Doc: https://youtrack.jetbrains.com/issue/IDEA-170643#focus=streamItem-27-3708697.0-0 # $ means "static" diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index df77ec38eefb..de8ef5f4b8db 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -343,16 +343,19 @@ jobs: errorprone-guava-latest: if: github.event.action != 'labeled' - name: 'ErrorProne (JDK 11, latest Guava)' + # This is the only job that verifies nullness, so a nullability problem fails here alone and + # the test jobs keep reporting test failures + name: 'ErrorProne and NullAway (JDK 21, latest Guava)' runs-on: ubuntu-latest steps: - uses: actions/checkout@v5 with: fetch-depth: 50 - - name: 'Set up JDK 11' + # Error Prone 2.43 and later require JDK 21, and NullAway requires JDK 17 + - name: 'Set up JDK 21' uses: actions/setup-java@v5 with: - java-version: 11 + java-version: 21 distribution: 'zulu' - uses: burrunan/gradle-cache-action@v1 name: Test @@ -362,44 +365,6 @@ jobs: # ErrorProne checks for Beta APIs, so use the latest supported Guava version arguments: --scan --no-parallel --no-daemon -Pguava.version=${{ env.GUAVA_MAX }} -PenableErrorprone classes - linux-checkerframework: - name: 'CheckerFramework (JDK 11)' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v5 - with: - fetch-depth: 50 - - name: 'Set up JDK 11' - uses: actions/setup-java@v5 - with: - java-version: 11 - distribution: 'zulu' - - name: 'Run CheckerFramework' - uses: burrunan/gradle-cache-action@v1 - with: - job-id: checkerframework-jdk11 - remote-build-cache-proxy-enabled: false - arguments: --scan --no-parallel --no-daemon -PenableCheckerframework :linq4j:classes :core:classes :server:classes - - linux-checkerframework-oldest-guava: - name: 'CheckerFramework (JDK 11, oldest Guava)' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v5 - with: - fetch-depth: 50 - - name: 'Set up JDK 11' - uses: actions/setup-java@v5 - with: - java-version: 11 - distribution: 'zulu' - - name: 'Run CheckerFramework' - uses: burrunan/gradle-cache-action@v1 - with: - job-id: checkerframework-jdk11 - remote-build-cache-proxy-enabled: false - arguments: --scan --no-parallel --no-daemon -Pguava.version=${{ env.GUAVA_MIN }} -PenableCheckerframework :linq4j:classes :core:classes :server:classes - linux-slow: # Run slow tests when the commit is on main or it is requested explicitly by adding an # appropriate label in the PR diff --git a/bom/build.gradle.kts b/bom/build.gradle.kts index 64cb532afdc3..e8d363581e68 100644 --- a/bom/build.gradle.kts +++ b/bom/build.gradle.kts @@ -71,7 +71,6 @@ dependencies { apiv("com.yahoo.datasketches:sketches-core") apiv("commons-codec:commons-codec") apiv("commons-io:commons-io") - apiv("org.checkerframework:checker-qual", "checkerframework") apiv("org.locationtech.jts:jts-core") apiv("org.locationtech.jts.io:jts-io-common") apiv("org.locationtech.proj4j:proj4j") @@ -148,6 +147,7 @@ dependencies { apiv("org.incava:java-diff") apiv("org.jboss:jandex") apiv("org.jooq:joou-java-6", "joou") + apiv("org.jspecify:jspecify") apiv("org.jsoup:jsoup") apiv("org.junit:junit-bom", "junit5") apiv("org.mockito:mockito-core", "mockito") diff --git a/build.gradle.kts b/build.gradle.kts index 4a4561672034..69291c9f3df1 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -42,7 +42,6 @@ plugins { calcite.buildext jacoco id("jacoco-report-aggregation") - id("org.checkerframework") apply false id("com.github.autostyle") id("org.nosphere.apache.rat") id("com.github.spotbugs") @@ -80,7 +79,6 @@ val lastEditYear by extra(lastEditYear()) // Do not enable spotbugs by default. Execute it only when -Pspotbugs is present val enableSpotBugs = props.bool("spotbugs") -val enableCheckerframework by props() val enableErrorprone by props() val enableDependencyAnalysis by props() val enableJacoco by props() @@ -94,6 +92,10 @@ val werror by props(true) // treat javac warnings as errors // Inherited from stage-vote-release-plugin: skipSign, useGpgCmd // Inherited from gradle-extensions-plugin: slowSuiteLogThreshold=0L, slowTestLogThreshold=2000L +// Projects whose main code NullAway verifies. The other projects are not annotated well enough +// yet, so NullAway would only produce noise there. +val nullawayProjects = listOf(":linq4j", ":core") + val hepLargePlanModeTestIncludes = mapOf( ":core" to listOf( "**/org/apache/calcite/test/HepPlannerTest.class", @@ -641,8 +643,10 @@ allprojects { replace("junit5: Assert.fail", "org.junit.Assert.fail", "org.junit.jupiter.api.Assertions.fail") } replaceRegex("side by side comments", "(\n\\s*+[*]*+/\n)(/[/*])", "\$1\n\$2") - replaceRegex("jsr305 nullable -> checkerframework", "javax\\.annotation\\.Nullable", "org.checkerframework.checker.nullness.qual.Nullable") - replaceRegex("jsr305 nonnull -> checkerframework", "javax\\.annotation\\.Nonnull", "org.checkerframework.checker.nullness.qual.NonNull") + replaceRegex("jsr305 nullable -> jspecify", "javax\\.annotation\\.Nullable", "org.jspecify.annotations.Nullable") + replaceRegex("jsr305 nonnull -> jspecify", "javax\\.annotation\\.Nonnull", "org.jspecify.annotations.NonNull") + replaceRegex("checkerframework nullable -> jspecify", "org\\.checkerframework\\.checker\\.nullness\\.qual\\.Nullable", "org.jspecify.annotations.Nullable") + replaceRegex("checkerframework nonnull -> jspecify", "org\\.checkerframework\\.checker\\.nullness\\.qual\\.NonNull", "org.jspecify.annotations.NonNull") importOrder( "org.apache.calcite.", "org.apache.", @@ -775,9 +779,14 @@ allprojects { apply(plugin = "net.ltgt.errorprone") dependencies { "errorprone"("com.google.errorprone:error_prone_core:${"errorprone".v}") + "errorprone"("com.uber.nullaway:nullaway:${"nullaway".v}") "annotationProcessor"("com.google.guava:guava-beta-checker:1.0") } + val nullawayEnabled = project.path in nullawayProjects tasks.withType().configureEach { + val mainCode = name == "compileJava" + // NullAway reports every error it finds, and javac hides all but the first 100 + options.compilerArgs.addAll(listOf("-Xmaxerrs", "10000")) options.errorprone { disableWarningsInGeneratedCode.set(true) errorproneArgs.add("-XepExcludedPaths:.*/javacc/.*") @@ -796,54 +805,87 @@ allprojects { ) // Analyze issues, and enable the check disable( + "AlreadyChecked", + "AnnotateFormatMethod", + "AssignmentExpression", "BigDecimalEquals", + "BooleanLiteral", + "ClassInitializationDeadlock", + "DoNotCall", "DoNotCallSuggester", - "StringSplitter" + "DuplicateBranches", + "EffectivelyPrivate", + "EnumOrdinal", + "ExposedPrivateType", + "FloggerArgumentToString", + "FormatStringShouldUsePlaceholders", + "InlineMeInliner", + "InlineMeSuggester", + "IntLiteralCast", + "InvalidLink", + "JavaDurationGetSecondsToToSeconds", + "JdkObsolete", + "LabelledBreakTarget", + "LenientFormatStringValidation", + "ListRemoveAmbiguous", + "MissingSummary", + "NonApiType", + "NonCanonicalType", + "NotJavadoc", + "RedundantControlFlow", + "ReturnValueIgnored", + "StaticAssignmentOfThrowable", + "StringSplitter", + "SuperCallToObjectMethod", + "UnnecessaryMethodReference", + "UnnecessaryStringBuilder", + "UnsafeReflectiveConstructionCast", + "UnusedMethod", + "UnusedVariable" ) + if (nullawayEnabled && mainCode) { + // Nullness errors must fail the build, so the annotations stay trustworthy + error("NullAway") + // Only @NullMarked code is analyzed, so an unannotated package is skipped + // rather than assumed non-null + option("NullAway:OnlyNullMarked", "true") + // JSpecify semantics, including nullness of generic type arguments + option("NullAway:JSpecifyMode", "true") + // Shorthand for HandleWildcardGenerics, JSpecifyJDKModels and + // WarnOnGenericInferenceFailure. They are spelled out below as well so it is + // clear which experiments Calcite relies on + option("NullAway:JSpecifyExperimental", "true") + // https://github.com/uber/NullAway/wiki/Configuration#handle-wildcard-generics + option("NullAway:HandleWildcardGenerics", "true") + // Nullness models for the JDK, which replace the CheckerFramework stub files + option("NullAway:JSpecifyJDKModels", "true") + // Report the calls where NullAway cannot infer the type arguments instead of + // silently assuming they are fine + option("NullAway:WarnOnGenericInferenceFailure", "true") + // Calcite runs its tests with assertions enabled + option("NullAway:AssertsEnabled", "true") + // Validate @Contract annotations rather than trusting them + option("NullAway:CheckContracts", "true") + // Check every override, not only the ones carrying @Override + option("NullAway:ExhaustiveOverride", "true") + option( + "NullAway:CastToNonNullMethod", + "org.apache.calcite.linq4j.Nullness.castNonNull" + ) + // Immutables-generated code is not annotated, and Calcite does not own it + option("NullAway:TreatGeneratedAsUnannotated", "true") + option( + "NullAway:CustomGeneratedCodeAnnotations", + "org.immutables.value.Generated" + ) + } else { + // Nullness is verified in a dedicated CI job only, so a nullness problem + // fails that job alone and leaves the test jobs reporting test failures + disable("NullAway") + } } } } - if (enableCheckerframework) { - apply(plugin = "org.checkerframework") - dependencies { - "checkerFramework"("org.checkerframework:checker:${"checkerframework".v}") - // CheckerFramework annotations might be used in the code as follows: - // dependencies { - // "compileOnly"("org.checkerframework:checker-qual") - // "testCompileOnly"("org.checkerframework:checker-qual") - // } - if (JavaVersion.current() == JavaVersion.VERSION_1_8) { - // only needed for JDK 8 - "checkerFrameworkAnnotatedJDK"("org.checkerframework:jdk8") - } - } - configure { - skipVersionCheck = true - // See https://checkerframework.org/manual/#introduction - checkers.add("org.checkerframework.checker.nullness.NullnessChecker") - // Below checkers take significant time and they do not provide much value :-/ - // checkers.add("org.checkerframework.checker.optional.OptionalChecker") - // checkers.add("org.checkerframework.checker.regex.RegexChecker") - // https://checkerframework.org/manual/#creating-debugging-options-progress - // extraJavacArgs.add("-Afilenames") - extraJavacArgs.addAll(listOf("-Xmaxerrs", "10000")) - // Consider Java assert statements for nullness and other checks - extraJavacArgs.add("-AassumeAssertionsAreEnabled") - // https://checkerframework.org/manual/#stub-using - extraJavacArgs.add("-Astubs=" + - fileTree("$rootDir/src/main/config/checkerframework") { - include("**/*.astub") - }.asPath - ) - if (project.path == ":core") { - extraJavacArgs.add("-AskipDefs=^org\\.apache\\.calcite\\.sql\\.parser\\.impl\\.") - } - if (project.path == ":server") { - extraJavacArgs.add("-AskipDefs=^org\\.apache\\.calcite\\.sql\\.parser\\.ddl\\.") - } - } - } - tasks { configureEach { manifest { @@ -882,7 +924,7 @@ allprojects { if (werror) { options.compilerArgs.add("-Werror") } - if (enableCheckerframework) { + if (enableErrorprone) { options.forkOptions.memoryMaximumSize = "2g" } excludeIntellijGenerated() diff --git a/core/build.gradle.kts b/core/build.gradle.kts index 086f114480e4..6681afc24c45 100644 --- a/core/build.gradle.kts +++ b/core/build.gradle.kts @@ -54,7 +54,7 @@ dependencies { api("com.google.guava:guava") api("org.apache.calcite.avatica:avatica-core") api("org.apiguardian:apiguardian-api") - api("org.checkerframework:checker-qual") + api("org.jspecify:jspecify") api("org.slf4j:slf4j-api") implementation("com.fasterxml.jackson.core:jackson-core") @@ -76,6 +76,8 @@ dependencies { implementation("org.codehaus.janino:commons-compiler") implementation("org.codehaus.janino:janino") annotationProcessor("org.immutables:value") + // The annotations are not retained at run time and the artifact never ships with Calcite, + // so its Java 11 bytecode is fine even though Calcite itself targets Java 8 compileOnly("org.immutables:value-annotations") compileOnly("com.google.code.findbugs:jsr305") testAnnotationProcessor("org.immutables:value") diff --git a/druid/build.gradle.kts b/druid/build.gradle.kts index 34cae5bd05ef..8b7230586ef2 100644 --- a/druid/build.gradle.kts +++ b/druid/build.gradle.kts @@ -27,7 +27,7 @@ dependencies { api("com.fasterxml.jackson.core:jackson-core") api("joda-time:joda-time") api("org.apache.calcite.avatica:avatica-core") - api("org.checkerframework:checker-qual") + api("org.jspecify:jspecify") api("org.slf4j:slf4j-api") implementation("com.fasterxml.jackson.core:jackson-databind") diff --git a/elasticsearch/build.gradle.kts b/elasticsearch/build.gradle.kts index 0ce6db032d23..5a544c2d57dd 100644 --- a/elasticsearch/build.gradle.kts +++ b/elasticsearch/build.gradle.kts @@ -33,7 +33,7 @@ dependencies { implementation("org.apache.httpcomponents:httpasyncclient") implementation("org.apache.httpcomponents:httpclient") implementation("org.apache.httpcomponents:httpcore") - implementation("org.checkerframework:checker-qual") + implementation("org.jspecify:jspecify") testImplementation("org.apache.logging.log4j:log4j-api") testImplementation("org.apache.logging.log4j:log4j-core") diff --git a/example/csv/build.gradle.kts b/example/csv/build.gradle.kts index 928f794ef058..09a931ff8af1 100644 --- a/example/csv/build.gradle.kts +++ b/example/csv/build.gradle.kts @@ -25,7 +25,7 @@ dependencies { api(project(":core")) api(project(":file")) api(project(":linq4j")) - api("org.checkerframework:checker-qual") + api("org.jspecify:jspecify") implementation("com.fasterxml.jackson.core:jackson-core") implementation("com.fasterxml.jackson.core:jackson-databind") diff --git a/example/function/build.gradle.kts b/example/function/build.gradle.kts index b030d0c8a657..f8489d914d18 100644 --- a/example/function/build.gradle.kts +++ b/example/function/build.gradle.kts @@ -17,7 +17,7 @@ dependencies { api(project(":core")) api(project(":linq4j")) - api("org.checkerframework:checker-qual") + api("org.jspecify:jspecify") testImplementation("sqlline:sqlline") testRuntimeOnly("org.apache.logging.log4j:log4j-slf4j-impl") diff --git a/file/build.gradle.kts b/file/build.gradle.kts index 04036a888eae..80f6ffad5100 100644 --- a/file/build.gradle.kts +++ b/file/build.gradle.kts @@ -24,7 +24,7 @@ plugins { dependencies { api(project(":core")) api(project(":linq4j")) - api("org.checkerframework:checker-qual") + api("org.jspecify:jspecify") implementation("com.google.guava:guava") implementation("com.joestelmach:natty") diff --git a/geode/build.gradle.kts b/geode/build.gradle.kts index 0ef17741679e..8b892b37d3a4 100644 --- a/geode/build.gradle.kts +++ b/geode/build.gradle.kts @@ -25,7 +25,7 @@ dependencies { api(project(":core")) api(project(":linq4j")) api("org.apache.geode:geode-core") - api("org.checkerframework:checker-qual") + api("org.jspecify:jspecify") api("org.slf4j:slf4j-api") implementation("com.google.guava:guava") diff --git a/gradle.properties b/gradle.properties index 0c5e83c5038c..10f464bf832d 100644 --- a/gradle.properties +++ b/gradle.properties @@ -42,7 +42,6 @@ calcite.avatica.version=1.28.0 # Plugins com.autonomousapps.dependency-analysis.version=0.71.0 -org.checkerframework.version=0.5.16 com.github.autostyle.version=3.2 com.github.johnrengelman.shadow.version=5.1.0 com.github.spotbugs.version=2.0.0 @@ -51,7 +50,7 @@ com.google.protobuf.version=0.8.10 de.thetaphi.forbiddenapis.version=3.10 jacoco.version=0.8.14 kotlin.version=2.3.20 -net.ltgt.errorprone.version=1.3.0 +net.ltgt.errorprone.version=5.1.0 me.champeau.jmh.version=0.7.2 org.jetbrains.gradle.plugin.idea-ext.version=1.4.1 org.nosphere.apache.rat.version=0.8.1 @@ -70,10 +69,10 @@ kotlin.stdlib.default.dependency=false # docker-maven-plugin.version=1.2.0 # Tools -checkerframework.version=3.10.0 checkstyle.version=8.28 spotbugs.version=3.1.11 -errorprone.version=2.5.1 +errorprone.version=2.50.0 +nullaway.version=0.14.0 # The property is used in https://github.com/wildfly/jandex regression testing, so avoid renaming jandex.version=3.5.3 @@ -134,6 +133,7 @@ joda-time.version=2.8.1 joou.version=0.9.5 json-path.version=2.10.0 json-smart.version=2.6.0 +jspecify.version=1.0.1 jsr305.version=3.0.2 jsoup.version=1.11.3 junit4.version=4.13.2 diff --git a/kafka/build.gradle.kts b/kafka/build.gradle.kts index 16abc9aab00a..f227739c04ac 100644 --- a/kafka/build.gradle.kts +++ b/kafka/build.gradle.kts @@ -18,7 +18,7 @@ dependencies { api(project(":core")) api(project(":linq4j")) api("org.apache.kafka:kafka-clients") - api("org.checkerframework:checker-qual") + api("org.jspecify:jspecify") implementation("com.google.guava:guava") diff --git a/linq4j/build.gradle.kts b/linq4j/build.gradle.kts index fd11eb7379bc..db097ad9deb9 100644 --- a/linq4j/build.gradle.kts +++ b/linq4j/build.gradle.kts @@ -16,7 +16,10 @@ */ dependencies { api("org.apiguardian:apiguardian-api") - api("org.checkerframework:checker-qual") + api("org.jspecify:jspecify") + + // The annotations are not retained at run time and the artifact never ships with Calcite, + // so its Java 11 bytecode is fine even though Calcite itself targets Java 8 implementation("com.google.guava:guava") implementation("org.apache.calcite.avatica:avatica-core") diff --git a/piglet/build.gradle.kts b/piglet/build.gradle.kts index 148c8777d9ee..820aa4f08b6f 100644 --- a/piglet/build.gradle.kts +++ b/piglet/build.gradle.kts @@ -31,7 +31,7 @@ dependencies { implementation("org.apache.calcite.avatica:avatica-core") implementation("org.apache.hadoop:hadoop-common") - implementation("org.checkerframework:checker-qual") + implementation("org.jspecify:jspecify") implementation("org.slf4j:slf4j-api") testImplementation(project(":testkit")) diff --git a/plus/build.gradle.kts b/plus/build.gradle.kts index 63fa9fcabd3e..2b84aac48763 100644 --- a/plus/build.gradle.kts +++ b/plus/build.gradle.kts @@ -19,7 +19,7 @@ dependencies { api(project(":linq4j")) api("net.hydromatic:quidem") api("org.apache.calcite.avatica:avatica-core") - api("org.checkerframework:checker-qual") + api("org.jspecify:jspecify") implementation("com.google.guava:guava") implementation("com.teradata.tpcds:tpcds") diff --git a/settings.gradle.kts b/settings.gradle.kts index fbc3502a7c29..cf0b390e6c24 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -22,7 +22,6 @@ pluginManagement { idv("com.gradle.develocity") idv("com.gradle.common-custom-user-data-gradle-plugin") idv("com.autonomousapps.dependency-analysis") - idv("org.checkerframework") idv("com.github.autostyle") idv("com.github.johnrengelman.shadow") idv("com.github.spotbugs") diff --git a/site/develop/index.md b/site/develop/index.md index de3221609a3c..ccc55352ee85 100644 --- a/site/develop/index.md +++ b/site/develop/index.md @@ -229,29 +229,50 @@ push it. ## Null safety -Apache Calcite uses the Checker Framework to avoid unexpected `NullPointerExceptions`. -You might find a detailed documentation at https://checkerframework.org/ +Apache Calcite annotates its code with [JSpecify](https://jspecify.dev/) and verifies it with +[NullAway](https://github.com/uber/NullAway) to avoid unexpected `NullPointerExceptions`. -Note: only main code is verified for now, so nullness annotation is not enforced in test code. +Only the main code of `calcite-linq4j` and `calcite-core` is verified for now. Every package in +those two declares `@NullMarked` in its `package-info.java`, which makes types non-nullable unless +they are annotated `@Nullable`. The other modules and all test code stay unmarked: `@NullMarked` +claims that a package is fully annotated, and nothing checks that claim there yet. -To execute the Checker Framework locally please use the following command: +NullAway skips a package that is not `@NullMarked`, so a missing annotation costs coverage without +saying a word. `LintTest.testLintNullMarked` fails when a package under one of the verified source +roots has no `package-info.java`, or has one that does not declare `@NullMarked`. - ./gradlew -PenableCheckerframework :linq4j:classes :core:classes +To verify one more module, add it to `nullawayProjects` in the root `build.gradle.kts` and to +`LintTest.NULL_MARKED_ROOTS`. + +To execute NullAway locally please use the following command: + + ./gradlew -PenableErrorprone :linq4j:classes :core:classes Here's a small introduction to null-safe programming: * By default, parameters, return values and fields are non-nullable, so refrain from using `@NonNull` * Local variables infer nullness from the expression, so you can write `Object v = ...` instead of `@Nullable Object v = ...` * Avoid the use of `javax.annotation.*` annotations. The annotations from `jsr305` do not support cases like `List<@Nullable String>` -so it is better to stick with `org.checkerframework.checker.nullness.qual.Nullable`. - Unfortunately, Guava (as of `29-jre`) has **both** `jsr305` and `checker-qual` dependencies at the same time, - so you might want to configure your IDE to exclude `javax.annotation.*` annotations from code completion. +so it is better to stick with `org.jspecify.annotations.Nullable`. + You might want to configure your IDE to exclude `javax.annotation.*` annotations from code completion. + +* A type variable is non-nullable unless it is declared with a nullable bound. Write + `` for a type variable that a caller may instantiate with a nullable + type. + +* For a method that returns null exactly when an argument is null, annotate it + `@Contract("!null, _ -> !null")` from `org.apache.calcite.linq4j.annotations`. The clause lists one entry + per parameter, `!null` for the ones the result depends on and `_` for the rest. NullAway verifies + the clause against the body, and a caller that passes a non-null argument gets a non-null result. + The annotation does not apply to a receiver parameter, to a varargs method, or to a type argument + such as `Enumerable<@Nullable T>`. -* The Checker Framework verifies code method by method. That means, it can't account for method execution order. +* NullAway verifies code method by method. That means, it can't account for method execution order. That is why `@Nullable` fields should be verified in each method where they are used. If you split logic into multiple methods, you might want verify null once, then pass it via non-nullable parameters. - For fields that start as null and become non-null later, use `@MonotonicNonNull`. - For fields that have already been checked against null, use `@RequiresNonNull`. + For fields that start as null and become non-null later, use `org.apache.calcite.linq4j.annotations.MonotonicNonNull`. + For fields that have already been checked against null, use `org.apache.calcite.linq4j.annotations.RequiresNonNull`. + Those annotations are `compileOnly` dependencies and are not retained at run time. * If you are absolutely sure the value is non-null, you might use `org.apache.calcite.linq4j.Nullness.castNonNull(T)`. The intention behind `castNonNull` is like `trustMeThisIsNeverNullHoweverTheVerifierCantTellYet(...)` @@ -259,9 +280,9 @@ so it is better to stick with `org.checkerframework.checker.nullness.qual.Nullab * If the expression is nullable, however, you need to pass it to a non-null method, use `Objects.requireNonNull`. It allows to have a better error message that includes context information. -* The Checker Framework comes with an annotated JDK, however, there might be invalid annotations. - In that cases, stub files can be placed to `/src/main/config/checkerframework` to override the annotations. - It is important the files have `.astub` extension otherwise they will be ignored. +* NullAway ships nullness models for the JDK and for popular libraries, so Calcite needs no stub + files of its own. When a third-party signature is modelled wrongly, add a + [library model](https://github.com/uber/NullAway/wiki/Library-Models) rather than a suppression. * In array types, a type annotation appears immediately before the type component (either the array or the array component) it refers to. This is explained in the [Java Language Specification](https://docs.oracle.com/javase/specs/jls/se8/html/jls-9.html#jls-9.7.4). diff --git a/src/main/config/checkerframework/Collection.astub b/src/main/config/checkerframework/Collection.astub deleted file mode 100644 index ba953e5e6a68..000000000000 --- a/src/main/config/checkerframework/Collection.astub +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package java.util; - -import java.util.function.Supplier; - -import org.checkerframework.checker.nullness.qual.*; - -interface Collection { - boolean contains(@Nullable Object o); - - boolean remove(@Nullable Object o); -} diff --git a/src/main/config/checkerframework/Constructor.astub b/src/main/config/checkerframework/Constructor.astub deleted file mode 100644 index 48818188520c..000000000000 --- a/src/main/config/checkerframework/Constructor.astub +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package java.lang.reflect; - -import org.checkerframework.checker.nullness.qual.*; - -class Constructor { - public @NonNull T newInstance(@Nullable Object... initargs); -} diff --git a/src/main/config/checkerframework/Field.astub b/src/main/config/checkerframework/Field.astub deleted file mode 100644 index b0c6f3daa938..000000000000 --- a/src/main/config/checkerframework/Field.astub +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package java.lang.reflect; - -import org.checkerframework.checker.nullness.qual.*; - -class Field { - @Nullable Object get(@Nullable Object obj); - - int getInt(@Nullable Object obj); -} diff --git a/src/main/config/checkerframework/InvocationHandler.astub b/src/main/config/checkerframework/InvocationHandler.astub deleted file mode 100644 index 4f3108f3b0f3..000000000000 --- a/src/main/config/checkerframework/InvocationHandler.astub +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package java.lang.reflect; - -import org.checkerframework.checker.nullness.qual.*; - -interface InvocationHandler { - @Nullable Object invoke(Object proxy, Method method, @Nullable Object[] args); -} diff --git a/src/main/config/checkerframework/List.astub b/src/main/config/checkerframework/List.astub deleted file mode 100644 index f458349a108c..000000000000 --- a/src/main/config/checkerframework/List.astub +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package java.util; - -import java.util.function.Supplier; - -import org.checkerframework.checker.nullness.qual.*; - -interface List { - boolean contains(@Nullable Object o); - - boolean remove(@Nullable Object o); - - int indexOf(@Nullable Object o); - - int lastIndexOf(@Nullable Object o); -} diff --git a/src/main/config/checkerframework/Map.astub b/src/main/config/checkerframework/Map.astub deleted file mode 100644 index 170cc7c33596..000000000000 --- a/src/main/config/checkerframework/Map.astub +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package java.util; - -import java.util.function.Supplier; - -import org.checkerframework.checker.nullness.qual.*; - -interface Map { - boolean containsKey(@Nullable Object o); - - boolean containsValue(@Nullable Object value); - - boolean remove(@Nullable Object key, @Nullable Object value); - - @Nullable V remove(@Nullable Object o); - - @Nullable V get(@Nullable Object key); - - V getOrDefault(@Nullable Object key, V defaultValue); -} diff --git a/src/main/config/checkerframework/Method.astub b/src/main/config/checkerframework/Method.astub deleted file mode 100644 index 8b488bb44b95..000000000000 --- a/src/main/config/checkerframework/Method.astub +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package java.lang.reflect; - -import org.checkerframework.checker.nullness.qual.*; - -class Method { - @Nullable Object invoke(@Nullable Object obj, @Nullable Object... args); -} diff --git a/src/main/config/checkerframework/MethodHandle.astub b/src/main/config/checkerframework/MethodHandle.astub deleted file mode 100644 index b6b938edfc43..000000000000 --- a/src/main/config/checkerframework/MethodHandle.astub +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package java.lang.invoke; - -import org.checkerframework.checker.nullness.qual.*; - -class MethodHandle { - @Nullable Object invokeWithArguments(@Nullable Object... arguments); -} diff --git a/src/main/config/checkerframework/Objects.astub b/src/main/config/checkerframework/Objects.astub deleted file mode 100644 index c785a5265adb..000000000000 --- a/src/main/config/checkerframework/Objects.astub +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package java.util; - -import java.util.function.Supplier; - -import org.checkerframework.checker.nullness.qual.*; - -class Objects { - /** - * The purpose of {@code requireNonNull} is to verify if given argument is non null. - * Unfortunately, checkerframework requires non-nullable arguments, so this stub overrides that. - * Then we can use {@code requireNonNull} for defensive programming, and the verifier won't - * complain. - */ - @EnsuresNonNull("#1") - public static @NonNull T requireNonNull(@Nullable T obj); - - @EnsuresNonNull("#1") - public static @NonNull T requireNonNull(@Nullable T obj, String message); - - @EnsuresNonNull("#1") - public static @NonNull T requireNonNull(@Nullable T obj, Supplier messageSupplier); -} diff --git a/src/main/config/checkerframework/Proxy.astub b/src/main/config/checkerframework/Proxy.astub deleted file mode 100644 index 108e54125dbb..000000000000 --- a/src/main/config/checkerframework/Proxy.astub +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package java.lang.reflect; - -import org.checkerframework.checker.nullness.qual.*; - -class Proxy { - Object newProxyInstance(@Nullable ClassLoader loader, Class[] interfaces, InvocationHandler h); -} diff --git a/src/main/config/checkerframework/Set.astub b/src/main/config/checkerframework/Set.astub deleted file mode 100644 index a3901540015e..000000000000 --- a/src/main/config/checkerframework/Set.astub +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package java.util; - -import java.util.function.Supplier; - -import org.checkerframework.checker.nullness.qual.*; - -interface Set { - boolean contains(@Nullable Object o); - - boolean remove(@Nullable Object o); -} diff --git a/src/main/config/checkerframework/String.astub b/src/main/config/checkerframework/String.astub deleted file mode 100644 index d35b0af309b7..000000000000 --- a/src/main/config/checkerframework/String.astub +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package java.lang; - -import org.checkerframework.checker.nullness.qual.*; - -class String { - String join(CharSequence delimiter, Iterable elements); -} diff --git a/src/main/config/checkerframework/URI.astub b/src/main/config/checkerframework/URI.astub deleted file mode 100644 index 9e20e50e022c..000000000000 --- a/src/main/config/checkerframework/URI.astub +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package java.net; - -import org.checkerframework.checker.nullness.qual.*; - -class URI { - public URI(@Nullable String scheme, @Nullable String host, @Nullable String path, - @Nullable String fragment); -} diff --git a/src/main/config/checkerframework/aggdesigner/Attribute.astub b/src/main/config/checkerframework/aggdesigner/Attribute.astub deleted file mode 100644 index d81e782ae598..000000000000 --- a/src/main/config/checkerframework/aggdesigner/Attribute.astub +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.pentaho.aggdes.model; - -import org.checkerframework.checker.nullness.qual.*; - -interface Attribute { - @Nullable tring getCandidateColumnName(); - - @Nullable String getDatatype(Dialect dialect); -} diff --git a/src/main/config/checkerframework/aggdesigner/Table.astub b/src/main/config/checkerframework/aggdesigner/Table.astub deleted file mode 100644 index f5922d500376..000000000000 --- a/src/main/config/checkerframework/aggdesigner/Table.astub +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.pentaho.aggdes.model; - -import org.checkerframework.checker.nullness.qual.*; - -interface Table { - @Nullable Table getParent(); -} diff --git a/src/main/config/checkerframework/avatica/AvaticaFactory.astub b/src/main/config/checkerframework/avatica/AvaticaFactory.astub deleted file mode 100644 index e84a514f2354..000000000000 --- a/src/main/config/checkerframework/avatica/AvaticaFactory.astub +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.calcite.avatica; - -import org.checkerframework.checker.nullness.qual.*; - -import java.sql.*; - -interface AvaticaFactory { - AvaticaStatement newStatement(AvaticaConnection connection, - Meta.@Nullable StatementHandle h, int resultSetType, - int resultSetConcurrency, int resultSetHoldability); - - AvaticaPreparedStatement newPreparedStatement(AvaticaConnection connection, - Meta.@Nullable StatementHandle h, Meta.Signature signature, - int resultSetType, int resultSetConcurrency, int resultSetHoldability); -} diff --git a/src/main/config/checkerframework/avatica/AvaticaPreparedStatement.astub b/src/main/config/checkerframework/avatica/AvaticaPreparedStatement.astub deleted file mode 100644 index 74cc35944e72..000000000000 --- a/src/main/config/checkerframework/avatica/AvaticaPreparedStatement.astub +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.calcite.avatica; - -import org.checkerframework.checker.nullness.qual.*; - -class AvaticaPreparedStatement { - AvaticaPreparedStatement(AvaticaConnection connection, - Meta.@Nullable StatementHandle h, - Meta.Signature signature, - int resultSetType, - int resultSetConcurrency, - int resultSetHoldability); -} diff --git a/src/main/config/checkerframework/avatica/AvaticaResultMetaData.astub b/src/main/config/checkerframework/avatica/AvaticaResultMetaData.astub deleted file mode 100644 index d43ac1aa3e54..000000000000 --- a/src/main/config/checkerframework/avatica/AvaticaResultMetaData.astub +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.calcite.avatica; - -import org.checkerframework.checker.nullness.qual.*; - -class AvaticaResultSetMetaData { - AvaticaResultSetMetaData( - AvaticaStatement statement, - @Nullable Object query, - Meta.Signature signature); -} diff --git a/src/main/config/checkerframework/avatica/AvaticaResultSet.astub b/src/main/config/checkerframework/avatica/AvaticaResultSet.astub deleted file mode 100644 index e9079d0f32ce..000000000000 --- a/src/main/config/checkerframework/avatica/AvaticaResultSet.astub +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.calcite.avatica; - -import org.checkerframework.checker.nullness.qual.*; - -import java.sql.*; - -class AvaticaResultSet { - AvaticaResultSet(AvaticaStatement statement, - @Nullable QueryState state, - Meta.Signature signature, - ResultSetMetaData resultSetMetaData, - TimeZone timeZone, - Meta.Frame firstFrame); -} diff --git a/src/main/config/checkerframework/avatica/AvaticaResultSetMetaData.astub b/src/main/config/checkerframework/avatica/AvaticaResultSetMetaData.astub deleted file mode 100644 index e9079d0f32ce..000000000000 --- a/src/main/config/checkerframework/avatica/AvaticaResultSetMetaData.astub +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.calcite.avatica; - -import org.checkerframework.checker.nullness.qual.*; - -import java.sql.*; - -class AvaticaResultSet { - AvaticaResultSet(AvaticaStatement statement, - @Nullable QueryState state, - Meta.Signature signature, - ResultSetMetaData resultSetMetaData, - TimeZone timeZone, - Meta.Frame firstFrame); -} diff --git a/src/main/config/checkerframework/avatica/AvaticaSite.astub b/src/main/config/checkerframework/avatica/AvaticaSite.astub deleted file mode 100644 index fceb680a42f3..000000000000 --- a/src/main/config/checkerframework/avatica/AvaticaSite.astub +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.calcite.avatica; - -import org.checkerframework.checker.nullness.qual.*; - -import java.sql.*; - -class AvaticaSite { - void setRowId(@Nullable RowId x); - - void setNString(@Nullable String o); - - void setNCharacterStream(@Nullable Reader value, long length); - - void setNClob(@Nullable NClob value); - - void setClob(@Nullable Reader reader, long length); - - void setBlob(@Nullable InputStream inputStream, long length); - - void setNClob(@Nullable Reader reader, long length); - - void setSQLXML(@Nullable SQLXML xmlObject); - - void setAsciiStream(@Nullable InputStream x, long length); - - void setBinaryStream(@Nullable InputStream x, long length); - - void setCharacterStream(@Nullable Reader reader, long length); - - void setAsciiStream(@Nullable InputStream x); - - void setBinaryStream(@Nullable InputStream x); - - void setCharacterStream(@Nullable Reader reader); - - void setNCharacterStream(@Nullable Reader value); - - void setClob(@Nullable Reader reader); - - void setBlob(@Nullable InputStream inputStream); - - void setNClob(@Nullable Reader reader); - - void setUnicodeStream(@Nullable InputStream x, int length); -} diff --git a/src/main/config/checkerframework/avatica/AvaticaStatement.astub b/src/main/config/checkerframework/avatica/AvaticaStatement.astub deleted file mode 100644 index 3ef5ce976bf4..000000000000 --- a/src/main/config/checkerframework/avatica/AvaticaStatement.astub +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.calcite.avatica; - -import org.checkerframework.checker.nullness.qual.*; - -class AvaticaStatement { - AvaticaStatement(AvaticaConnection connection, - Meta.@Nullable StatementHandle h, int resultSetType, int resultSetConcurrency, - int resultSetHoldability); - - AvaticaStatement(AvaticaConnection connection, - Meta.@Nullable StatementHandle h, int resultSetType, int resultSetConcurrency, - int resultSetHoldability, Meta.Signature signature); -} diff --git a/src/main/config/checkerframework/avatica/ColumnMetaData.astub b/src/main/config/checkerframework/avatica/ColumnMetaData.astub deleted file mode 100644 index 77ecc21fb74c..000000000000 --- a/src/main/config/checkerframework/avatica/ColumnMetaData.astub +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.calcite.avatica; - -import org.checkerframework.checker.nullness.qual.*; - -class ColumnMetaData { - ColumnMetaData( - int ordinal, - boolean autoIncrement, - boolean caseSensitive, - boolean searchable, - boolean currency, - int nullable, - boolean signed, - int displaySize, - String label, - @Nullable String columnName, - @Nullable String schemaName, - int precision, - int scale, - @Nullable String tableName, - @Nullable String catalogName, - AvaticaType type, - boolean readOnly, - boolean writable, - boolean definitelyWritable, - String columnClassName); -} diff --git a/src/main/config/checkerframework/avatica/ConnectionConfigImpl.astub b/src/main/config/checkerframework/avatica/ConnectionConfigImpl.astub deleted file mode 100644 index 0fe2baa53adf..000000000000 --- a/src/main/config/checkerframework/avatica/ConnectionConfigImpl.astub +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.calcite.avatica; - -import org.checkerframework.checker.nullness.qual.*; - -class ConnectionConfigImpl { - public static class PropEnv { - public @PolyNull T getPlugin(Class pluginClass, @PolyNull T defaultInstance); - } -} diff --git a/src/main/config/checkerframework/avatica/ConnectionProperty.astub b/src/main/config/checkerframework/avatica/ConnectionProperty.astub deleted file mode 100644 index 30846a3f0a7a..000000000000 --- a/src/main/config/checkerframework/avatica/ConnectionProperty.astub +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.calcite.avatica; - -import org.checkerframework.checker.nullness.qual.*; - -interface ConnectionProperty { - enum Type { - NONE; - - Class deduceValueClass(@Nullable Object defaultValue, @Nullable Class valueClass); - - boolean valid(@Nullable Object defaultValue, Class clazz); - } - - @Nullable Object defaultValue(); - - @Nullable Class valueClass(); -} diff --git a/src/main/config/checkerframework/avatica/Handler.astub b/src/main/config/checkerframework/avatica/Handler.astub deleted file mode 100644 index c57d936d0d40..000000000000 --- a/src/main/config/checkerframework/avatica/Handler.astub +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.calcite.avatica; - -import org.checkerframework.checker.nullness.qual.*; - -import java.sql.*; - -interface Handler { - void onStatementExecute( - AvaticaStatement statement, - @Nullable ResultSink resultSink); -} diff --git a/src/main/config/checkerframework/avatica/Meta.astub b/src/main/config/checkerframework/avatica/Meta.astub deleted file mode 100644 index af3e549be96a..000000000000 --- a/src/main/config/checkerframework/avatica/Meta.astub +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.calcite.avatica; - -import org.checkerframework.checker.nullness.qual.*; - -interface Meta { - class Signature { - public Signature(List columns, - @Nullable String sql, - List parameters, - Map internalParameters, - CursorFactory cursorFactory, - Meta.StatementType statementType); - } - - class CursorFactory { - CursorFactory deduce(List columns, @Nullable Class resultClazz); - } - - interface PrepareCallback { - void assign(Signature signature, @Nullable Frame firstFrame, long updateCount); - } - - class MetaResultSet { - static MetaResultSet create(String connectionId, int statementId, - boolean ownStatement, Signature signature, @Nullable Frame firstFrame, long updateCount); - } -} diff --git a/src/main/config/checkerframework/avatica/MetaImpl.astub b/src/main/config/checkerframework/avatica/MetaImpl.astub deleted file mode 100644 index 501f29b01228..000000000000 --- a/src/main/config/checkerframework/avatica/MetaImpl.astub +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.calcite.avatica; - -import org.checkerframework.checker.nullness.qual.*; - -class MetaImpl { - class MetaTable { - public MetaTable( - @Nullable String tableCat, - @Nullable String tableSchem, - @Nullable String tableName, - @Nullable String tableType); - } - - class MetaColumn { - MetaColumn( - String tableCat, - String tableSchem, - String tableName, - String columnName, - int dataType, - String typeName, - Integer columnSize, - @Nullable Integer decimalDigits, - Integer numPrecRadix, - int nullable, - Integer charOctetLength, - int ordinalPosition, - String isNullable); - } - - class MetaTypeInfo { - MetaTypeInfo( - String typeName, - int dataType, - Integer precision, - @Nullable String literalPrefix, - @Nullable String literalSuffix, - short nullable, - boolean caseSensitive, - short searchable, - boolean unsignedAttribute, - boolean fixedPrecScale, - boolean autoIncrement, - Short minimumScale, - Short maximumScale, - Integer numPrecRadix); - } -} diff --git a/src/main/config/checkerframework/avatica/TimeUnitRange.astub b/src/main/config/checkerframework/avatica/TimeUnitRange.astub deleted file mode 100644 index c5ed3c9dc92a..000000000000 --- a/src/main/config/checkerframework/avatica/TimeUnitRange.astub +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.calcite.avatica.util; - -import org.checkerframework.checker.nullness.qual.*; - -enum TimeUnitRange { - YEAR; - - public TimeUnitRange of(TimeUnit startUnit, @Nullable TimeUnit endUnit); -} diff --git a/src/main/config/checkerframework/commons-dbcp2/BasicDataSource.astub b/src/main/config/checkerframework/commons-dbcp2/BasicDataSource.astub deleted file mode 100644 index b6f49c6fd2c0..000000000000 --- a/src/main/config/checkerframework/commons-dbcp2/BasicDataSource.astub +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.commons.dbcp2; - -import org.checkerframework.checker.nullness.qual.*; - -interface BasicDataSource { - void setUrl(@Nullable String url); - - void setUsername(@Nullable String userName); - - void setPassword(@Nullable String password); - - void setDriverClassName(@Nullable String driverClassName); -} diff --git a/src/main/config/checkerframework/esri-geometry/OperatorBoundary.astub b/src/main/config/checkerframework/esri-geometry/OperatorBoundary.astub deleted file mode 100644 index a929df4c2756..000000000000 --- a/src/main/config/checkerframework/esri-geometry/OperatorBoundary.astub +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.esri.core.geometry; - -import org.checkerframework.checker.nullness.qual.*; - -interface OperatorBoundary { - Geometry execute(Geometry geom, @Nullable ProgressTracker progress_tracker); -} diff --git a/src/main/config/checkerframework/esri-geometry/OperatorSimpleRelation.astub b/src/main/config/checkerframework/esri-geometry/OperatorSimpleRelation.astub deleted file mode 100644 index f98ed02582fa..000000000000 --- a/src/main/config/checkerframework/esri-geometry/OperatorSimpleRelation.astub +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.esri.core.geometry; - -import org.checkerframework.checker.nullness.qual.*; - -interface OperatorSimpleRelation { - boolean execute(Geometry inputGeom1, Geometry inputGeom2, - SpatialReference sr, @Nullable ProgressTracker progressTracker); -} diff --git a/src/main/config/checkerframework/guava/Function.astub b/src/main/config/checkerframework/guava/Function.astub deleted file mode 100644 index 76f9d0aea34d..000000000000 --- a/src/main/config/checkerframework/guava/Function.astub +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.google.common.base; - -import org.checkerframework.checker.nullness.qual.*; - -/** - * Guava has {@code Nullable} argument and return value by default. - * Checkerframework cna infer nullability from the actual generic types. - * - * @param argument type - * @param return type - */ -public interface Function { - T apply(F input); -} diff --git a/src/main/config/checkerframework/guava/Iterables.astub b/src/main/config/checkerframework/guava/Iterables.astub deleted file mode 100644 index db302664618d..000000000000 --- a/src/main/config/checkerframework/guava/Iterables.astub +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.google.common.collect; - -import org.checkerframework.checker.nullness.qual.*; - -/** - * Guava has {@code Nullable} argument and return value by default. - * Checkerframework cna infer nullability from the actual generic types. - * - * @param argument type - * @param return type - */ -public class Iterables { - public static T[] toArray(Iterable iterable, Class type); -} diff --git a/src/main/config/checkerframework/guava/Ordering.astub b/src/main/config/checkerframework/guava/Ordering.astub deleted file mode 100644 index 820a66856bb5..000000000000 --- a/src/main/config/checkerframework/guava/Ordering.astub +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.google.common.collect; - -import org.checkerframework.checker.nullness.qual.*; - -/** - * Guava has {@code Nullable} argument and return value by default. - * Checkerframework cna infer nullability from the actual generic types. - * - * @param argument type - * @param return type - */ -public abstract class Ordering { - // The Checker Framework can infer nullness from generic itself, so we do not need - // "always nullable" from Guava. - public abstract int compare(T left, T right); -} diff --git a/src/main/config/checkerframework/guava/Predicate.astub b/src/main/config/checkerframework/guava/Predicate.astub deleted file mode 100644 index 577f270e38b5..000000000000 --- a/src/main/config/checkerframework/guava/Predicate.astub +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.google.common.base; - -import org.checkerframework.checker.nullness.qual.*; - -/** - * Guava has {@code Nullable} argument and return value by default. - * Checkerframework cna infer nullability from the actual generic types. - * - * @param argument type - */ -public interface Predicate { - boolean apply(T input); -} diff --git a/src/main/config/checkerframework/jackson/ObjectMapper.astub b/src/main/config/checkerframework/jackson/ObjectMapper.astub deleted file mode 100644 index a275a8bcc953..000000000000 --- a/src/main/config/checkerframework/jackson/ObjectMapper.astub +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.fasterxml.jackson.databind; - -import org.checkerframework.checker.nullness.qual.*; - -interface ObjectMapper { - String writeValueAsString(@Nullable Object value); - - byte[] writeValueAsBytes(@Nullable Object value); -} diff --git a/src/main/config/checkerframework/jackson/ObjectWriter.astub b/src/main/config/checkerframework/jackson/ObjectWriter.astub deleted file mode 100644 index 9a428d2c9f83..000000000000 --- a/src/main/config/checkerframework/jackson/ObjectWriter.astub +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.fasterxml.jackson.databind; - -import org.checkerframework.checker.nullness.qual.*; - -interface ObjectWriter { - String writeValueAsString(@Nullable Object value); - - byte[] writeValueAsBytes(@Nullable Object value); -} diff --git a/src/main/config/checkerframework/janino/ClassBodyEvaluator.astub b/src/main/config/checkerframework/janino/ClassBodyEvaluator.astub deleted file mode 100644 index 6d6222c4b87a..000000000000 --- a/src/main/config/checkerframework/janino/ClassBodyEvaluator.astub +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.codehaus.janino; - -import org.checkerframework.checker.nullness.qual.*; - -public class ClassBodyEvaluator extends Cookable implements IClassBodyEvaluator { - public void setParentClassLoader(@Nullable ClassLoader parentClassLoader); -} diff --git a/src/main/config/checkerframework/janino/IClassBodyEvaluator.astub b/src/main/config/checkerframework/janino/IClassBodyEvaluator.astub deleted file mode 100644 index a3b143cd3aab..000000000000 --- a/src/main/config/checkerframework/janino/IClassBodyEvaluator.astub +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.codehaus.commons.compiler; - -import org.checkerframework.checker.nullness.qual.*; - -public interface IClassBodyEvaluator extends ICookable { - void setParentClassLoader(@Nullable ClassLoader optionalParentClassLoader); -} diff --git a/src/main/config/checkerframework/janino/ISimpleCompiler.astub b/src/main/config/checkerframework/janino/ISimpleCompiler.astub deleted file mode 100644 index 3fa227320b2e..000000000000 --- a/src/main/config/checkerframework/janino/ISimpleCompiler.astub +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.codehaus.commons.compiler; - -import org.checkerframework.checker.nullness.qual.*; - -public interface ISimpleCompiler { - void setParentClassLoader(@Nullable ClassLoader optionalParentClassLoader); -} diff --git a/src/main/config/checkerframework/janino/JavaSourceClassLoader.astub b/src/main/config/checkerframework/janino/JavaSourceClassLoader.astub deleted file mode 100644 index 2e13a2b4b5f6..000000000000 --- a/src/main/config/checkerframework/janino/JavaSourceClassLoader.astub +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.codehaus.janino; - -import org.checkerframework.checker.nullness.qual.*; - -class JavaSourceClassLoader { - JavaSourceClassLoader( - ClassLoader parentClassLoader, - File @Nullable [] optionalSourcePath, - @Nullable String optionalCharacterEncoding - ); - - JavaSourceClassLoader( - ClassLoader parentClassLoader, - ResourceFinder sourceFinder, - @Nullable String optionalCharacterEncoding - ); - - protected @Nullable Map generateBytecodes(String name); -} diff --git a/src/main/config/checkerframework/janino/Scanner.astub b/src/main/config/checkerframework/janino/Scanner.astub deleted file mode 100644 index a2a1daee7e84..000000000000 --- a/src/main/config/checkerframework/janino/Scanner.astub +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.codehaus.janino; - -import org.checkerframework.checker.nullness.qual.*; - -class Scanner { - Scanner(@Nullable String optionalFileName, Reader in); -} diff --git a/src/main/config/checkerframework/jdbc/Connection.astub b/src/main/config/checkerframework/jdbc/Connection.astub deleted file mode 100644 index 6dd9bf72dc0e..000000000000 --- a/src/main/config/checkerframework/jdbc/Connection.astub +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package java.sql; - -import org.checkerframework.checker.nullness.qual.*; - -interface Connection { - @Nullable String getSchema() throws SQLException; -} diff --git a/src/main/config/checkerframework/jdbc/DatabaseMetaData.astub b/src/main/config/checkerframework/jdbc/DatabaseMetaData.astub deleted file mode 100644 index 212c72680bab..000000000000 --- a/src/main/config/checkerframework/jdbc/DatabaseMetaData.astub +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package java.sql; - -import org.checkerframework.checker.nullness.qual.*; - -interface DatabaseMetaData { - ResultSet getSchemas(@Nullable String catalog, @Nullable String schemaPattern); -} diff --git a/src/main/config/checkerframework/jsonpath/JacksonJsonProvider.astub b/src/main/config/checkerframework/jsonpath/JacksonJsonProvider.astub deleted file mode 100644 index 3d262171638b..000000000000 --- a/src/main/config/checkerframework/jsonpath/JacksonJsonProvider.astub +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.jayway.jsonpath.spi.json; - -import org.checkerframework.checker.nullness.qual.*; - -interface JacksonJsonProvider { - String toJson(@Nullable Object obj); -} diff --git a/src/main/config/checkerframework/slf4j/Logger.astub b/src/main/config/checkerframework/slf4j/Logger.astub deleted file mode 100644 index 7c709cef131b..000000000000 --- a/src/main/config/checkerframework/slf4j/Logger.astub +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.slf4j; - -import org.checkerframework.checker.nullness.qual.*; - -interface Logger { - public void trace(String format, @Nullable Object arg); - - public void trace(String format, @Nullable Object arg1, @Nullable Object arg2); - - public void trace(String format, @Nullable Object... arguments); - - public void trace(String msg, @Nullable Throwable t); - - public void trace(Marker marker, String format, @Nullable Object arg); - - public void trace(Marker marker, String format, @Nullable Object arg1, @Nullable Object arg2); - - public void trace(Marker marker, String format, @Nullable Object... arguments); - - public void trace(Marker marker, String msg, @Nullable Throwable t); - - public void debug(String format, @Nullable Object arg); - - public void debug(String format, @Nullable Object arg1, @Nullable Object arg2); - - public void debug(String format, @Nullable Object... arguments); - - public void debug(String msg, @Nullable Throwable t); - - public void debug(Marker marker, String format, @Nullable Object arg); - - public void debug(Marker marker, String format, @Nullable Object arg1, @Nullable Object arg2); - - public void debug(Marker marker, String format, @Nullable Object... arguments); - - public void debug(Marker marker, String msg, @Nullable Throwable t); - - public void info(String format, @Nullable Object arg); - - public void info(String format, @Nullable Object arg1, @Nullable Object arg2); - - public void info(String format, @Nullable Object... arguments); - - public void info(String msg, @Nullable Throwable t); - - public void info(Marker marker, String format, @Nullable Object arg); - - public void info(Marker marker, String format, @Nullable Object arg1, @Nullable Object arg2); - - public void info(Marker marker, String format, @Nullable Object... arguments); - - public void info(Marker marker, String msg, @Nullable Throwable t); - - public void warn(String format, @Nullable Object arg); - - public void warn(String format, @Nullable Object arg1, @Nullable Object arg2); - - public void warn(String format, @Nullable Object... arguments); - - public void warn(String msg, @Nullable Throwable t); - - public void warn(Marker marker, String format, @Nullable Object arg); - - public void warn(Marker marker, String format, @Nullable Object arg1, @Nullable Object arg2); - - public void warn(Marker marker, String format, @Nullable Object... arguments); - - public void warn(Marker marker, String msg, @Nullable Throwable t); - - public void error(String format, @Nullable Object arg); - - public void error(String format, @Nullable Object arg1, @Nullable Object arg2); - - public void error(String format, @Nullable Object... arguments); - - public void error(String msg, @Nullable Throwable t); - - public void error(Marker marker, String format, @Nullable Object arg); - - public void error(Marker marker, String format, @Nullable Object arg1, @Nullable Object arg2); - - public void error(Marker marker, String format, @Nullable Object... arguments); - - public void error(Marker marker, String msg, @Nullable Throwable t); -} diff --git a/src/main/config/checkerframework/slf4j/MessageFormatter.astub b/src/main/config/checkerframework/slf4j/MessageFormatter.astub deleted file mode 100644 index 8ef43aa609c2..000000000000 --- a/src/main/config/checkerframework/slf4j/MessageFormatter.astub +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.slf4j.helpers; - -import org.checkerframework.checker.nullness.qual.*; - -class MessageFormatter { - public FormattingTuple arrayFormat(String messagePattern, @Nullable Object[] argArray); -} diff --git a/testkit/build.gradle.kts b/testkit/build.gradle.kts index f164b4acfd30..3755e9c6c32c 100644 --- a/testkit/build.gradle.kts +++ b/testkit/build.gradle.kts @@ -20,7 +20,7 @@ plugins { dependencies { api(project(":core")) - api("org.checkerframework:checker-qual") + api("org.jspecify:jspecify") implementation(platform("org.junit:junit-bom")) implementation(kotlin("stdlib-jdk8")) From 28854224b781f4c1db0e59f4013c73faf82db970 Mon Sep 17 00:00:00 2001 From: Vladimir Sitnikov Date: Sun, 23 Aug 2026 22:11:05 +0300 Subject: [PATCH 495/562] [CALCITE-7736] Move @Nullable and @NonNull from the Checker Framework to JSpecify A purely mechanical rename of `org.checkerframework.checker.nullness.qual.Nullable` to `org.jspecify.annotations.Nullable`, and of the matching `NonNull`. Both annotations are `TYPE_USE`, so every existing position stays valid and no annotation moves. Autostyle rewrites either annotation to its JSpecify counterpart from now on, alongside the rule that already rewrote the jsr305 ones. The Checker Framework annotations that JSpecify does not define -- @PolyNull, @MonotonicNonNull, @RequiresNonNull and the rest -- are still here and are migrated by the next commit. Co-Authored-By: Claude Opus 5 --- .../java/org/apache/calcite/adapter/arrow/ArrowProject.java | 2 +- .../java/org/apache/calcite/adapter/arrow/ArrowRel.java | 2 +- .../java/org/apache/calcite/adapter/arrow/ArrowRules.java | 2 +- .../java/org/apache/calcite/adapter/arrow/ArrowSchema.java | 2 +- .../java/org/apache/calcite/adapter/arrow/ArrowTable.java | 2 +- .../org/apache/calcite/adapter/arrow/ArrowTranslator.java | 2 +- .../org/apache/calcite/adapter/arrow/ConditionToken.java | 2 +- .../org/apache/calcite/sql/babel/SqlBabelCreateTable.java | 2 +- .../test/java/org/apache/calcite/test/BabelParserTest.java | 2 +- .../test/java/org/apache/calcite/test/BabelQuidemTest.java | 2 +- .../src/test/java/org/apache/calcite/test/package-info.java | 2 +- .../calcite/adapter/cassandra/CassandraEnumerator.java | 2 +- .../apache/calcite/adapter/cassandra/CassandraFilter.java | 2 +- .../apache/calcite/adapter/cassandra/CassandraLimit.java | 2 +- .../apache/calcite/adapter/cassandra/CassandraProject.java | 2 +- .../org/apache/calcite/adapter/cassandra/CassandraRel.java | 2 +- .../apache/calcite/adapter/cassandra/CassandraRules.java | 2 +- .../org/apache/calcite/adapter/cassandra/CassandraSort.java | 2 +- .../apache/calcite/adapter/cassandra/CassandraTable.java | 2 +- .../calcite/adapter/cassandra/CassandraTableScan.java | 2 +- .../adapter/cassandra/CassandraToEnumerableConverter.java | 2 +- core/src/main/java/org/apache/calcite/DataContext.java | 2 +- core/src/main/java/org/apache/calcite/DataContexts.java | 2 +- .../java/org/apache/calcite/adapter/clone/ArrayTable.java | 2 +- .../java/org/apache/calcite/adapter/clone/CloneSchema.java | 2 +- .../java/org/apache/calcite/adapter/clone/ColumnLoader.java | 2 +- .../apache/calcite/adapter/enumerable/AggAddContext.java | 2 +- .../apache/calcite/adapter/enumerable/AggResultContext.java | 2 +- .../org/apache/calcite/adapter/enumerable/EnumUtils.java | 2 +- .../calcite/adapter/enumerable/EnumerableAggregate.java | 2 +- .../calcite/adapter/enumerable/EnumerableAggregateBase.java | 2 +- .../calcite/adapter/enumerable/EnumerableAggregateRule.java | 2 +- .../calcite/adapter/enumerable/EnumerableAsofJoin.java | 2 +- .../adapter/enumerable/EnumerableBatchNestedLoopJoin.java | 2 +- .../calcite/adapter/enumerable/EnumerableBindable.java | 2 +- .../apache/calcite/adapter/enumerable/EnumerableCalc.java | 2 +- .../adapter/enumerable/EnumerableConditionalCorrelate.java | 2 +- .../calcite/adapter/enumerable/EnumerableCorrelate.java | 2 +- .../apache/calcite/adapter/enumerable/EnumerableFilter.java | 2 +- .../calcite/adapter/enumerable/EnumerableHashJoin.java | 2 +- .../calcite/adapter/enumerable/EnumerableInterpretable.java | 2 +- .../calcite/adapter/enumerable/EnumerableInterpreter.java | 2 +- .../apache/calcite/adapter/enumerable/EnumerableLimit.java | 2 +- .../calcite/adapter/enumerable/EnumerableLimitSort.java | 2 +- .../apache/calcite/adapter/enumerable/EnumerableMatch.java | 2 +- .../calcite/adapter/enumerable/EnumerableMergeJoin.java | 2 +- .../calcite/adapter/enumerable/EnumerableMergeJoinRule.java | 2 +- .../adapter/enumerable/EnumerableNestedLoopJoin.java | 2 +- .../calcite/adapter/enumerable/EnumerableProject.java | 2 +- .../apache/calcite/adapter/enumerable/EnumerableRel.java | 2 +- .../calcite/adapter/enumerable/EnumerableRelFactories.java | 2 +- .../calcite/adapter/enumerable/EnumerableRepeatUnion.java | 2 +- .../apache/calcite/adapter/enumerable/EnumerableSort.java | 2 +- .../calcite/adapter/enumerable/EnumerableSortRule.java | 2 +- .../adapter/enumerable/EnumerableSortedAggregate.java | 2 +- .../adapter/enumerable/EnumerableSortedAggregateRule.java | 2 +- .../adapter/enumerable/EnumerableTableFunctionScan.java | 2 +- .../calcite/adapter/enumerable/EnumerableTableModify.java | 2 +- .../adapter/enumerable/EnumerableTableModifyRule.java | 2 +- .../calcite/adapter/enumerable/EnumerableTableScan.java | 2 +- .../calcite/adapter/enumerable/EnumerableTableScanRule.java | 2 +- .../calcite/adapter/enumerable/EnumerableTraitsUtils.java | 2 +- .../apache/calcite/adapter/enumerable/EnumerableValues.java | 2 +- .../apache/calcite/adapter/enumerable/EnumerableWindow.java | 2 +- .../apache/calcite/adapter/enumerable/JavaRowFormat.java | 2 +- .../org/apache/calcite/adapter/enumerable/PhysType.java | 2 +- .../org/apache/calcite/adapter/enumerable/PhysTypeImpl.java | 2 +- .../org/apache/calcite/adapter/enumerable/RexImpTable.java | 2 +- .../calcite/adapter/enumerable/RexImplementorTable.java | 2 +- .../calcite/adapter/enumerable/RexImplementorTables.java | 2 +- .../calcite/adapter/enumerable/RexToLixTranslator.java | 2 +- .../adapter/enumerable/impl/AggResultContextImpl.java | 2 +- .../org/apache/calcite/adapter/java/ReflectiveSchema.java | 2 +- .../org/apache/calcite/adapter/jdbc/JdbcBaseSchema.java | 2 +- .../org/apache/calcite/adapter/jdbc/JdbcCatalogSchema.java | 2 +- .../calcite/adapter/jdbc/JdbcCorrelationDataContext.java | 2 +- .../java/org/apache/calcite/adapter/jdbc/JdbcRules.java | 2 +- .../java/org/apache/calcite/adapter/jdbc/JdbcSchema.java | 2 +- .../java/org/apache/calcite/adapter/jdbc/JdbcTable.java | 2 +- .../calcite/adapter/jdbc/JdbcToEnumerableConverter.java | 2 +- .../calcite/adapter/jdbc/JdbcToEnumerableConverterRule.java | 2 +- .../java/org/apache/calcite/adapter/jdbc/JdbcUtils.java | 2 +- .../main/java/org/apache/calcite/adapter/package-info.java | 2 +- .../org/apache/calcite/config/CalciteConnectionConfig.java | 2 +- .../apache/calcite/config/CalciteConnectionConfigImpl.java | 2 +- .../apache/calcite/config/CalciteConnectionProperty.java | 2 +- .../org/apache/calcite/config/CalciteSystemProperty.java | 2 +- .../main/java/org/apache/calcite/config/package-info.java | 2 +- .../java/org/apache/calcite/interpreter/AggregateNode.java | 2 +- .../org/apache/calcite/interpreter/BindableConvention.java | 2 +- .../main/java/org/apache/calcite/interpreter/Bindables.java | 2 +- .../java/org/apache/calcite/interpreter/CollectNode.java | 2 +- .../main/java/org/apache/calcite/interpreter/Compiler.java | 2 +- .../main/java/org/apache/calcite/interpreter/Context.java | 2 +- .../apache/calcite/interpreter/InterpretableConverter.java | 2 +- .../org/apache/calcite/interpreter/InterpretableRel.java | 2 +- .../java/org/apache/calcite/interpreter/Interpreter.java | 2 +- .../java/org/apache/calcite/interpreter/Interpreters.java | 2 +- .../main/java/org/apache/calcite/interpreter/JoinNode.java | 2 +- .../calcite/interpreter/NoneToBindableConverterRule.java | 2 +- core/src/main/java/org/apache/calcite/interpreter/Row.java | 2 +- .../main/java/org/apache/calcite/interpreter/Scalar.java | 2 +- .../main/java/org/apache/calcite/interpreter/SortNode.java | 2 +- .../main/java/org/apache/calcite/interpreter/Source.java | 2 +- .../apache/calcite/interpreter/TableFunctionScanNode.java | 2 +- .../java/org/apache/calcite/interpreter/TableScanNode.java | 2 +- .../java/org/apache/calcite/interpreter/package-info.java | 2 +- .../java/org/apache/calcite/jdbc/CachingCalciteSchema.java | 2 +- .../java/org/apache/calcite/jdbc/CalciteConnection.java | 2 +- .../java/org/apache/calcite/jdbc/CalciteConnectionImpl.java | 2 +- .../main/java/org/apache/calcite/jdbc/CalciteFactory.java | 2 +- .../java/org/apache/calcite/jdbc/CalciteJdbc41Factory.java | 2 +- .../org/apache/calcite/jdbc/CalciteMetaColumnFactory.java | 2 +- .../apache/calcite/jdbc/CalciteMetaColumnFactoryImpl.java | 2 +- .../main/java/org/apache/calcite/jdbc/CalciteMetaImpl.java | 2 +- .../main/java/org/apache/calcite/jdbc/CalcitePrepare.java | 2 +- .../org/apache/calcite/jdbc/CalcitePreparedStatement.java | 2 +- .../main/java/org/apache/calcite/jdbc/CalciteSchema.java | 2 +- .../main/java/org/apache/calcite/jdbc/CalciteStatement.java | 2 +- core/src/main/java/org/apache/calcite/jdbc/Driver.java | 2 +- .../main/java/org/apache/calcite/jdbc/JavaCollation.java | 2 +- .../main/java/org/apache/calcite/jdbc/JavaRecordType.java | 2 +- .../java/org/apache/calcite/jdbc/JavaTypeFactoryImpl.java | 2 +- .../java/org/apache/calcite/jdbc/SimpleCalciteSchema.java | 2 +- .../src/main/java/org/apache/calcite/jdbc/package-info.java | 2 +- .../main/java/org/apache/calcite/materialize/Lattice.java | 2 +- .../java/org/apache/calcite/materialize/LatticeNode.java | 2 +- .../org/apache/calcite/materialize/LatticeSuggester.java | 2 +- .../java/org/apache/calcite/materialize/LatticeTable.java | 2 +- .../apache/calcite/materialize/MaterializationActor.java | 2 +- .../org/apache/calcite/materialize/MaterializationKey.java | 2 +- .../apache/calcite/materialize/MaterializationService.java | 2 +- .../java/org/apache/calcite/materialize/MutableNode.java | 2 +- core/src/main/java/org/apache/calcite/materialize/Path.java | 2 +- .../calcite/materialize/SqlLatticeStatisticProvider.java | 2 +- core/src/main/java/org/apache/calcite/materialize/Step.java | 2 +- .../main/java/org/apache/calcite/materialize/TileKey.java | 2 +- .../java/org/apache/calcite/materialize/TileSuggester.java | 2 +- .../java/org/apache/calcite/materialize/package-info.java | 2 +- .../main/java/org/apache/calcite/model/ClassNameFilter.java | 2 +- .../java/org/apache/calcite/model/JsonCustomSchema.java | 2 +- .../main/java/org/apache/calcite/model/JsonCustomTable.java | 2 +- .../main/java/org/apache/calcite/model/JsonFunction.java | 2 +- .../main/java/org/apache/calcite/model/JsonJdbcSchema.java | 2 +- .../src/main/java/org/apache/calcite/model/JsonLattice.java | 2 +- .../main/java/org/apache/calcite/model/JsonMapSchema.java | 2 +- .../java/org/apache/calcite/model/JsonMaterialization.java | 2 +- .../src/main/java/org/apache/calcite/model/JsonMeasure.java | 2 +- core/src/main/java/org/apache/calcite/model/JsonRoot.java | 2 +- core/src/main/java/org/apache/calcite/model/JsonSchema.java | 2 +- core/src/main/java/org/apache/calcite/model/JsonStream.java | 2 +- core/src/main/java/org/apache/calcite/model/JsonTable.java | 2 +- core/src/main/java/org/apache/calcite/model/JsonTile.java | 2 +- core/src/main/java/org/apache/calcite/model/JsonType.java | 2 +- core/src/main/java/org/apache/calcite/model/JsonView.java | 2 +- .../main/java/org/apache/calcite/model/ModelHandler.java | 2 +- .../main/java/org/apache/calcite/model/package-info.java | 2 +- .../java/org/apache/calcite/plan/AbstractRelOptPlanner.java | 2 +- core/src/main/java/org/apache/calcite/plan/Contexts.java | 2 +- core/src/main/java/org/apache/calcite/plan/Convention.java | 2 +- .../java/org/apache/calcite/plan/ConventionTraitDef.java | 2 +- .../java/org/apache/calcite/plan/RelCompositeTrait.java | 2 +- .../java/org/apache/calcite/plan/RelOptAbstractTable.java | 2 +- .../main/java/org/apache/calcite/plan/RelOptCluster.java | 2 +- .../main/java/org/apache/calcite/plan/RelOptCostImpl.java | 2 +- .../main/java/org/apache/calcite/plan/RelOptLattice.java | 2 +- .../main/java/org/apache/calcite/plan/RelOptListener.java | 2 +- .../java/org/apache/calcite/plan/RelOptMaterialization.java | 2 +- .../main/java/org/apache/calcite/plan/RelOptPlanner.java | 2 +- .../java/org/apache/calcite/plan/RelOptPredicateList.java | 2 +- core/src/main/java/org/apache/calcite/plan/RelOptQuery.java | 2 +- core/src/main/java/org/apache/calcite/plan/RelOptRule.java | 2 +- .../main/java/org/apache/calcite/plan/RelOptRuleCall.java | 2 +- .../java/org/apache/calcite/plan/RelOptRuleOperand.java | 2 +- .../src/main/java/org/apache/calcite/plan/RelOptSchema.java | 2 +- .../org/apache/calcite/plan/RelOptSchemaWithSampling.java | 2 +- core/src/main/java/org/apache/calcite/plan/RelOptTable.java | 2 +- core/src/main/java/org/apache/calcite/plan/RelOptUtil.java | 6 +++--- core/src/main/java/org/apache/calcite/plan/RelRule.java | 2 +- core/src/main/java/org/apache/calcite/plan/RelTrait.java | 2 +- core/src/main/java/org/apache/calcite/plan/RelTraitDef.java | 2 +- .../org/apache/calcite/plan/RelTraitPropagationVisitor.java | 2 +- core/src/main/java/org/apache/calcite/plan/RelTraitSet.java | 2 +- .../java/org/apache/calcite/plan/RexImplicationChecker.java | 2 +- .../main/java/org/apache/calcite/plan/SpoolRelOptTable.java | 2 +- .../java/org/apache/calcite/plan/SubstitutionVisitor.java | 2 +- .../main/java/org/apache/calcite/plan/TableAccessMap.java | 2 +- .../main/java/org/apache/calcite/plan/ViewExpanders.java | 2 +- .../java/org/apache/calcite/plan/VisitorDataContext.java | 2 +- .../java/org/apache/calcite/plan/hep/HepInstruction.java | 2 +- .../main/java/org/apache/calcite/plan/hep/HepPlanner.java | 2 +- .../main/java/org/apache/calcite/plan/hep/HepProgram.java | 2 +- .../org/apache/calcite/plan/hep/HepRelMetadataProvider.java | 2 +- .../main/java/org/apache/calcite/plan/hep/HepRelVertex.java | 2 +- .../main/java/org/apache/calcite/plan/hep/HepRuleCall.java | 2 +- .../src/main/java/org/apache/calcite/plan/package-info.java | 2 +- .../calcite/plan/visualizer/InputExcludedRelWriter.java | 2 +- .../apache/calcite/plan/visualizer/NodeUpdateHelper.java | 2 +- .../apache/calcite/plan/visualizer/RuleMatchVisualizer.java | 2 +- .../org/apache/calcite/plan/volcano/AbstractConverter.java | 2 +- .../main/java/org/apache/calcite/plan/volcano/Dumpers.java | 2 +- .../org/apache/calcite/plan/volcano/IterativeRuleQueue.java | 2 +- .../main/java/org/apache/calcite/plan/volcano/RelSet.java | 2 +- .../java/org/apache/calcite/plan/volcano/RelSubset.java | 2 +- .../org/apache/calcite/plan/volcano/TopDownRuleDriver.java | 2 +- .../org/apache/calcite/plan/volcano/TopDownRuleQueue.java | 2 +- .../java/org/apache/calcite/plan/volcano/VolcanoCost.java | 2 +- .../org/apache/calcite/plan/volcano/VolcanoPlanner.java | 2 +- .../calcite/plan/volcano/VolcanoRelMetadataProvider.java | 2 +- .../org/apache/calcite/plan/volcano/VolcanoRuleCall.java | 2 +- .../org/apache/calcite/prepare/CalciteCatalogReader.java | 2 +- .../java/org/apache/calcite/prepare/CalcitePrepareImpl.java | 2 +- .../main/java/org/apache/calcite/prepare/PlannerImpl.java | 2 +- core/src/main/java/org/apache/calcite/prepare/Prepare.java | 2 +- .../org/apache/calcite/prepare/QueryableRelBuilder.java | 2 +- .../java/org/apache/calcite/prepare/RelOptTableImpl.java | 2 +- .../main/java/org/apache/calcite/prepare/package-info.java | 2 +- core/src/main/java/org/apache/calcite/profile/Profiler.java | 2 +- .../main/java/org/apache/calcite/profile/ProfilerImpl.java | 2 +- .../java/org/apache/calcite/profile/SimpleProfiler.java | 2 +- .../main/java/org/apache/calcite/profile/package-info.java | 2 +- .../main/java/org/apache/calcite/rel/AbstractRelNode.java | 2 +- core/src/main/java/org/apache/calcite/rel/PhysicalNode.java | 2 +- .../main/java/org/apache/calcite/rel/RelCollationImpl.java | 2 +- .../java/org/apache/calcite/rel/RelCollationTraitDef.java | 2 +- .../calcite/rel/RelCommonExpressionBasicSuggester.java | 2 +- .../apache/calcite/rel/RelCommonExpressionSuggester.java | 2 +- .../org/apache/calcite/rel/RelDistributionTraitDef.java | 2 +- .../main/java/org/apache/calcite/rel/RelDistributions.java | 2 +- .../main/java/org/apache/calcite/rel/RelFieldCollation.java | 2 +- core/src/main/java/org/apache/calcite/rel/RelInput.java | 2 +- core/src/main/java/org/apache/calcite/rel/RelNode.java | 2 +- core/src/main/java/org/apache/calcite/rel/RelNodes.java | 2 +- .../java/org/apache/calcite/rel/RelValidityChecker.java | 2 +- core/src/main/java/org/apache/calcite/rel/RelVisitor.java | 2 +- core/src/main/java/org/apache/calcite/rel/RelWriter.java | 2 +- .../main/java/org/apache/calcite/rel/convert/Converter.java | 2 +- .../java/org/apache/calcite/rel/convert/ConverterImpl.java | 2 +- .../java/org/apache/calcite/rel/convert/ConverterRule.java | 2 +- .../org/apache/calcite/rel/convert/TraitMatchingRule.java | 2 +- .../main/java/org/apache/calcite/rel/core/Aggregate.java | 2 +- .../java/org/apache/calcite/rel/core/AggregateCall.java | 2 +- core/src/main/java/org/apache/calcite/rel/core/Calc.java | 2 +- core/src/main/java/org/apache/calcite/rel/core/Collect.java | 2 +- .../main/java/org/apache/calcite/rel/core/Correlate.java | 2 +- .../java/org/apache/calcite/rel/core/CorrelationId.java | 2 +- .../src/main/java/org/apache/calcite/rel/core/Exchange.java | 2 +- core/src/main/java/org/apache/calcite/rel/core/Filter.java | 2 +- core/src/main/java/org/apache/calcite/rel/core/Join.java | 2 +- core/src/main/java/org/apache/calcite/rel/core/Match.java | 2 +- core/src/main/java/org/apache/calcite/rel/core/Project.java | 2 +- .../main/java/org/apache/calcite/rel/core/RelFactories.java | 2 +- .../main/java/org/apache/calcite/rel/core/RepeatUnion.java | 2 +- .../src/main/java/org/apache/calcite/rel/core/Snapshot.java | 2 +- core/src/main/java/org/apache/calcite/rel/core/Sort.java | 2 +- .../java/org/apache/calcite/rel/core/TableFunctionScan.java | 2 +- .../main/java/org/apache/calcite/rel/core/TableModify.java | 2 +- .../main/java/org/apache/calcite/rel/core/TableScan.java | 2 +- core/src/main/java/org/apache/calcite/rel/core/Values.java | 2 +- core/src/main/java/org/apache/calcite/rel/core/Window.java | 2 +- .../org/apache/calcite/rel/externalize/RelDotWriter.java | 2 +- .../org/apache/calcite/rel/externalize/RelEnumTypes.java | 4 ++-- .../java/org/apache/calcite/rel/externalize/RelJson.java | 4 ++-- .../org/apache/calcite/rel/externalize/RelJsonReader.java | 2 +- .../org/apache/calcite/rel/externalize/RelJsonWriter.java | 2 +- .../org/apache/calcite/rel/externalize/RelWriterImpl.java | 2 +- .../org/apache/calcite/rel/externalize/RelXmlWriter.java | 2 +- .../main/java/org/apache/calcite/rel/hint/HintStrategy.java | 2 +- .../java/org/apache/calcite/rel/hint/HintStrategyTable.java | 2 +- core/src/main/java/org/apache/calcite/rel/hint/RelHint.java | 2 +- .../org/apache/calcite/rel/logical/LogicalAggregate.java | 2 +- .../org/apache/calcite/rel/logical/LogicalAsofJoin.java | 2 +- .../java/org/apache/calcite/rel/logical/LogicalFilter.java | 2 +- .../java/org/apache/calcite/rel/logical/LogicalJoin.java | 2 +- .../java/org/apache/calcite/rel/logical/LogicalMatch.java | 2 +- .../java/org/apache/calcite/rel/logical/LogicalProject.java | 2 +- .../org/apache/calcite/rel/logical/LogicalRepeatUnion.java | 2 +- .../java/org/apache/calcite/rel/logical/LogicalSort.java | 2 +- .../calcite/rel/logical/LogicalTableFunctionScan.java | 2 +- .../org/apache/calcite/rel/logical/LogicalTableModify.java | 2 +- .../java/org/apache/calcite/rel/logical/LogicalWindow.java | 2 +- .../org/apache/calcite/rel/metadata/BuiltInMetadata.java | 2 +- .../calcite/rel/metadata/CachingRelMetadataProvider.java | 2 +- .../calcite/rel/metadata/ChainedRelMetadataProvider.java | 2 +- .../calcite/rel/metadata/JaninoRelMetadataProvider.java | 2 +- .../org/apache/calcite/rel/metadata/MetadataFactory.java | 2 +- .../apache/calcite/rel/metadata/MetadataFactoryImpl.java | 2 +- .../java/org/apache/calcite/rel/metadata/NullSentinel.java | 2 +- .../calcite/rel/metadata/ReflectiveRelMetadataProvider.java | 2 +- .../org/apache/calcite/rel/metadata/RelColumnOrigin.java | 2 +- .../org/apache/calcite/rel/metadata/RelMdAllPredicates.java | 2 +- .../org/apache/calcite/rel/metadata/RelMdCollation.java | 2 +- .../org/apache/calcite/rel/metadata/RelMdColumnOrigins.java | 2 +- .../apache/calcite/rel/metadata/RelMdColumnUniqueness.java | 2 +- .../apache/calcite/rel/metadata/RelMdDistinctRowCount.java | 2 +- .../org/apache/calcite/rel/metadata/RelMdDistribution.java | 2 +- .../apache/calcite/rel/metadata/RelMdExplainVisibility.java | 2 +- .../apache/calcite/rel/metadata/RelMdExpressionLineage.java | 2 +- .../calcite/rel/metadata/RelMdFunctionalDependency.java | 2 +- .../apache/calcite/rel/metadata/RelMdLowerBoundCost.java | 2 +- .../org/apache/calcite/rel/metadata/RelMdMaxRowCount.java | 2 +- .../java/org/apache/calcite/rel/metadata/RelMdMeasure.java | 2 +- .../java/org/apache/calcite/rel/metadata/RelMdMemory.java | 2 +- .../org/apache/calcite/rel/metadata/RelMdMinRowCount.java | 2 +- .../org/apache/calcite/rel/metadata/RelMdNodeTypes.java | 2 +- .../calcite/rel/metadata/RelMdPercentageOriginalRows.java | 2 +- .../apache/calcite/rel/metadata/RelMdPopulationSize.java | 2 +- .../org/apache/calcite/rel/metadata/RelMdPredicates.java | 2 +- .../java/org/apache/calcite/rel/metadata/RelMdRowCount.java | 2 +- .../org/apache/calcite/rel/metadata/RelMdSelectivity.java | 2 +- .../java/org/apache/calcite/rel/metadata/RelMdSize.java | 2 +- .../apache/calcite/rel/metadata/RelMdTableReferences.java | 2 +- .../org/apache/calcite/rel/metadata/RelMdUniqueKeys.java | 2 +- .../java/org/apache/calcite/rel/metadata/RelMdUtil.java | 2 +- .../apache/calcite/rel/metadata/RelMetadataProvider.java | 2 +- .../org/apache/calcite/rel/metadata/RelMetadataQuery.java | 2 +- .../apache/calcite/rel/metadata/RelMetadataQueryBase.java | 2 +- .../org/apache/calcite/rel/metadata/UnboundMetadata.java | 2 +- .../calcite/rel/metadata/janino/DispatchGenerator.java | 2 +- .../metadata/janino/RelMetadataHandlerGeneratorUtil.java | 2 +- .../org/apache/calcite/rel/mutable/MutableAggregate.java | 2 +- .../java/org/apache/calcite/rel/mutable/MutableCalc.java | 2 +- .../java/org/apache/calcite/rel/mutable/MutableCollect.java | 2 +- .../org/apache/calcite/rel/mutable/MutableCorrelate.java | 2 +- .../org/apache/calcite/rel/mutable/MutableExchange.java | 2 +- .../java/org/apache/calcite/rel/mutable/MutableFilter.java | 2 +- .../java/org/apache/calcite/rel/mutable/MutableJoin.java | 2 +- .../java/org/apache/calcite/rel/mutable/MutableMatch.java | 2 +- .../java/org/apache/calcite/rel/mutable/MutableProject.java | 2 +- .../java/org/apache/calcite/rel/mutable/MutableRel.java | 2 +- .../org/apache/calcite/rel/mutable/MutableRelVisitor.java | 2 +- .../java/org/apache/calcite/rel/mutable/MutableRels.java | 2 +- .../java/org/apache/calcite/rel/mutable/MutableSample.java | 2 +- .../java/org/apache/calcite/rel/mutable/MutableScan.java | 2 +- .../java/org/apache/calcite/rel/mutable/MutableSetOp.java | 2 +- .../java/org/apache/calcite/rel/mutable/MutableSort.java | 2 +- .../calcite/rel/mutable/MutableTableFunctionScan.java | 2 +- .../org/apache/calcite/rel/mutable/MutableTableModify.java | 2 +- .../org/apache/calcite/rel/mutable/MutableUncollect.java | 2 +- .../java/org/apache/calcite/rel/mutable/MutableValues.java | 2 +- .../java/org/apache/calcite/rel/mutable/MutableWindow.java | 2 +- core/src/main/java/org/apache/calcite/rel/package-info.java | 2 +- .../org/apache/calcite/rel/rel2sql/RelToSqlConverter.java | 2 +- .../java/org/apache/calcite/rel/rel2sql/SqlImplementor.java | 2 +- .../apache/calcite/rel/rules/AggregateCaseToFilterRule.java | 2 +- .../rel/rules/AggregateExpandDistinctAggregatesRule.java | 2 +- .../rel/rules/AggregateExpandWithinDistinctRule.java | 2 +- .../calcite/rel/rules/AggregateJoinTransposeRule.java | 2 +- .../apache/calcite/rel/rules/AggregateProjectMergeRule.java | 2 +- .../rel/rules/AggregateReduceFunctionsOnGroupKeysRule.java | 2 +- .../calcite/rel/rules/AggregateReduceFunctionsRule.java | 2 +- .../org/apache/calcite/rel/rules/AggregateRemoveRule.java | 2 +- .../apache/calcite/rel/rules/AggregateStarTableRule.java | 2 +- .../java/org/apache/calcite/rel/rules/CalcRelSplitter.java | 2 +- .../java/org/apache/calcite/rel/rules/CoerceInputsRule.java | 2 +- .../java/org/apache/calcite/rel/rules/DateRangeRules.java | 2 +- core/src/main/java/org/apache/calcite/rel/rules/DpHyp.java | 2 +- .../calcite/rel/rules/ExpandDisjunctionForTableRule.java | 2 +- .../java/org/apache/calcite/rel/rules/FilterJoinRule.java | 2 +- .../apache/calcite/rel/rules/FilterMultiJoinMergeRule.java | 2 +- .../main/java/org/apache/calcite/rel/rules/HyperGraph.java | 2 +- .../java/org/apache/calcite/rel/rules/JoinCommuteRule.java | 2 +- .../apache/calcite/rel/rules/JoinProjectTransposeRule.java | 2 +- .../org/apache/calcite/rel/rules/JoinToMultiJoinRule.java | 2 +- .../java/org/apache/calcite/rel/rules/LoptMultiJoin.java | 2 +- .../org/apache/calcite/rel/rules/LoptOptimizeJoinRule.java | 2 +- .../org/apache/calcite/rel/rules/LoptSemiJoinOptimizer.java | 2 +- .../main/java/org/apache/calcite/rel/rules/MultiJoin.java | 2 +- .../calcite/rel/rules/MultiJoinOptimizeBushyRule.java | 2 +- .../apache/calcite/rel/rules/OuterJoinToAntiJoinRule.java | 2 +- .../calcite/rel/rules/ProjectFilterTransposeRule.java | 2 +- .../java/org/apache/calcite/rel/rules/PushProjector.java | 2 +- .../org/apache/calcite/rel/rules/ReduceDecimalsRule.java | 2 +- .../org/apache/calcite/rel/rules/ReduceExpressionsRule.java | 2 +- .../java/org/apache/calcite/rel/rules/SemiJoinRule.java | 2 +- .../org/apache/calcite/rel/rules/SetOpToFilterRule.java | 2 +- .../calcite/rel/rules/SingleValuesOptimizationRules.java | 2 +- .../org/apache/calcite/rel/rules/SortJoinTransposeRule.java | 2 +- .../java/org/apache/calcite/rel/rules/SpatialRules.java | 2 +- .../java/org/apache/calcite/rel/rules/ValuesReduceRule.java | 2 +- .../rules/materialize/MaterializedViewAggregateRule.java | 2 +- .../rel/rules/materialize/MaterializedViewJoinRule.java | 2 +- .../calcite/rel/rules/materialize/MaterializedViewRule.java | 2 +- .../org/apache/calcite/rel/type/DelegatingTypeSystem.java | 2 +- .../org/apache/calcite/rel/type/DynamicRecordTypeImpl.java | 2 +- .../main/java/org/apache/calcite/rel/type/RelDataType.java | 2 +- .../org/apache/calcite/rel/type/RelDataTypeFactory.java | 2 +- .../org/apache/calcite/rel/type/RelDataTypeFactoryImpl.java | 2 +- .../org/apache/calcite/rel/type/RelDataTypeFieldImpl.java | 2 +- .../java/org/apache/calcite/rel/type/RelDataTypeImpl.java | 2 +- .../java/org/apache/calcite/rel/type/RelDataTypeSystem.java | 2 +- .../org/apache/calcite/rel/type/RelDataTypeSystemImpl.java | 2 +- .../java/org/apache/calcite/rel/type/RelRecordType.java | 2 +- .../calcite/rel/type/SingleColumnAliasRelDataType.java | 2 +- .../main/java/org/apache/calcite/rel/type/TimeFrame.java | 2 +- .../main/java/org/apache/calcite/rel/type/TimeFrameSet.java | 2 +- .../main/java/org/apache/calcite/rel/type/TimeFrames.java | 2 +- core/src/main/java/org/apache/calcite/rex/LogicVisitor.java | 2 +- .../main/java/org/apache/calcite/rex/RexBiVisitorImpl.java | 2 +- core/src/main/java/org/apache/calcite/rex/RexBuilder.java | 2 +- core/src/main/java/org/apache/calcite/rex/RexCall.java | 2 +- .../main/java/org/apache/calcite/rex/RexCallBinding.java | 2 +- core/src/main/java/org/apache/calcite/rex/RexChecker.java | 2 +- .../main/java/org/apache/calcite/rex/RexCorrelVariable.java | 2 +- .../main/java/org/apache/calcite/rex/RexDynamicParam.java | 2 +- .../src/main/java/org/apache/calcite/rex/RexExecutable.java | 2 +- .../main/java/org/apache/calcite/rex/RexExecutorImpl.java | 2 +- .../main/java/org/apache/calcite/rex/RexFieldAccess.java | 2 +- core/src/main/java/org/apache/calcite/rex/RexInputRef.java | 2 +- .../main/java/org/apache/calcite/rex/RexInterpreter.java | 2 +- core/src/main/java/org/apache/calcite/rex/RexLambda.java | 2 +- core/src/main/java/org/apache/calcite/rex/RexLambdaRef.java | 2 +- core/src/main/java/org/apache/calcite/rex/RexLiteral.java | 2 +- core/src/main/java/org/apache/calcite/rex/RexLocalRef.java | 2 +- .../main/java/org/apache/calcite/rex/RexMultisetUtil.java | 2 +- core/src/main/java/org/apache/calcite/rex/RexNode.java | 2 +- .../java/org/apache/calcite/rex/RexNodeAndFieldIndex.java | 2 +- core/src/main/java/org/apache/calcite/rex/RexOver.java | 2 +- .../org/apache/calcite/rex/RexPermuteInputsShuttle.java | 2 +- core/src/main/java/org/apache/calcite/rex/RexProgram.java | 2 +- .../main/java/org/apache/calcite/rex/RexProgramBuilder.java | 2 +- core/src/main/java/org/apache/calcite/rex/RexRangeRef.java | 2 +- core/src/main/java/org/apache/calcite/rex/RexShuttle.java | 2 +- core/src/main/java/org/apache/calcite/rex/RexSimplify.java | 2 +- .../main/java/org/apache/calcite/rex/RexSqlConvertlet.java | 2 +- .../java/org/apache/calcite/rex/RexSqlConvertletTable.java | 2 +- .../apache/calcite/rex/RexSqlReflectiveConvertletTable.java | 2 +- .../apache/calcite/rex/RexSqlStandardConvertletTable.java | 2 +- core/src/main/java/org/apache/calcite/rex/RexSubQuery.java | 2 +- .../main/java/org/apache/calcite/rex/RexTableInputRef.java | 2 +- .../java/org/apache/calcite/rex/RexToSqlNodeConverter.java | 2 +- .../org/apache/calcite/rex/RexToSqlNodeConverterImpl.java | 2 +- .../main/java/org/apache/calcite/rex/RexUnaryBiVisitor.java | 2 +- core/src/main/java/org/apache/calcite/rex/RexUtil.java | 2 +- .../main/java/org/apache/calcite/rex/RexVisitorImpl.java | 2 +- core/src/main/java/org/apache/calcite/rex/RexWindow.java | 2 +- .../main/java/org/apache/calcite/rex/RexWindowBound.java | 2 +- .../main/java/org/apache/calcite/rex/RexWindowBounds.java | 2 +- core/src/main/java/org/apache/calcite/rex/package-info.java | 2 +- .../org/apache/calcite/runtime/AbstractImmutableList.java | 2 +- .../main/java/org/apache/calcite/runtime/ArrayBindable.java | 2 +- .../org/apache/calcite/runtime/ArrayEnumeratorCursor.java | 2 +- .../src/main/java/org/apache/calcite/runtime/Automaton.java | 2 +- .../org/apache/calcite/runtime/CalciteContextException.java | 2 +- .../java/org/apache/calcite/runtime/CalciteException.java | 2 +- .../java/org/apache/calcite/runtime/CalciteResource.java | 2 +- .../org/apache/calcite/runtime/CompressionFunctions.java | 2 +- core/src/main/java/org/apache/calcite/runtime/ConsList.java | 2 +- .../org/apache/calcite/runtime/DeterministicAutomaton.java | 2 +- .../main/java/org/apache/calcite/runtime/Enumerables.java | 2 +- .../src/main/java/org/apache/calcite/runtime/FlatLists.java | 4 ++-- .../java/org/apache/calcite/runtime/FunctionContexts.java | 2 +- .../src/main/java/org/apache/calcite/runtime/HttpUtils.java | 2 +- .../java/org/apache/calcite/runtime/ImmutablePairList.java | 2 +- .../main/java/org/apache/calcite/runtime/JsonFunctions.java | 2 +- core/src/main/java/org/apache/calcite/runtime/Like.java | 2 +- core/src/main/java/org/apache/calcite/runtime/MapEntry.java | 2 +- core/src/main/java/org/apache/calcite/runtime/Matcher.java | 2 +- core/src/main/java/org/apache/calcite/runtime/PairList.java | 2 +- .../src/main/java/org/apache/calcite/runtime/PairLists.java | 4 ++-- .../main/java/org/apache/calcite/runtime/PredicateImpl.java | 2 +- .../src/main/java/org/apache/calcite/runtime/Resources.java | 2 +- .../org/apache/calcite/runtime/ResultSetEnumerable.java | 2 +- .../org/apache/calcite/runtime/SpatialTypeFunctions.java | 2 +- .../main/java/org/apache/calcite/runtime/SqlFunctions.java | 2 +- .../src/main/java/org/apache/calcite/runtime/Utilities.java | 2 +- .../main/java/org/apache/calcite/runtime/XmlFunctions.java | 2 +- .../main/java/org/apache/calcite/runtime/package-info.java | 2 +- .../org/apache/calcite/runtime/rtti/BasicSqlTypeRtti.java | 2 +- .../org/apache/calcite/runtime/rtti/GenericSqlTypeRtti.java | 2 +- .../org/apache/calcite/runtime/rtti/RowSqlTypeRtti.java | 2 +- .../apache/calcite/runtime/rtti/RuntimeTypeInformation.java | 2 +- .../org/apache/calcite/runtime/variant/VariantNonNull.java | 2 +- .../org/apache/calcite/runtime/variant/VariantNull.java | 2 +- .../org/apache/calcite/runtime/variant/VariantSqlNull.java | 2 +- .../org/apache/calcite/runtime/variant/VariantSqlValue.java | 2 +- .../org/apache/calcite/runtime/variant/VariantValue.java | 2 +- .../java/org/apache/calcite/schema/FilterableTable.java | 2 +- .../java/org/apache/calcite/schema/FunctionContext.java | 2 +- .../java/org/apache/calcite/schema/ModifiableTable.java | 2 +- .../apache/calcite/schema/ProjectableFilterableTable.java | 2 +- .../main/java/org/apache/calcite/schema/ScannableTable.java | 2 +- core/src/main/java/org/apache/calcite/schema/Schema.java | 2 +- .../src/main/java/org/apache/calcite/schema/SchemaPlus.java | 2 +- core/src/main/java/org/apache/calcite/schema/Schemas.java | 2 +- core/src/main/java/org/apache/calcite/schema/Statistic.java | 2 +- .../src/main/java/org/apache/calcite/schema/Statistics.java | 2 +- core/src/main/java/org/apache/calcite/schema/Table.java | 2 +- .../main/java/org/apache/calcite/schema/TableFactory.java | 2 +- .../main/java/org/apache/calcite/schema/TableFunction.java | 2 +- .../src/main/java/org/apache/calcite/schema/TableMacro.java | 2 +- core/src/main/java/org/apache/calcite/schema/Wrapper.java | 2 +- .../java/org/apache/calcite/schema/impl/AbstractSchema.java | 2 +- .../java/org/apache/calcite/schema/impl/AbstractTable.java | 2 +- .../apache/calcite/schema/impl/AggregateFunctionImpl.java | 2 +- .../org/apache/calcite/schema/impl/DelegatingSchema.java | 2 +- .../org/apache/calcite/schema/impl/ListTransientTable.java | 2 +- .../org/apache/calcite/schema/impl/LongSchemaVersion.java | 2 +- .../apache/calcite/schema/impl/MaterializedViewTable.java | 2 +- .../org/apache/calcite/schema/impl/ModifiableViewTable.java | 2 +- .../apache/calcite/schema/impl/ReflectiveFunctionBase.java | 2 +- .../org/apache/calcite/schema/impl/ScalarFunctionImpl.java | 2 +- .../main/java/org/apache/calcite/schema/impl/StarTable.java | 2 +- .../org/apache/calcite/schema/impl/TableFunctionImpl.java | 2 +- .../java/org/apache/calcite/schema/impl/TableMacroImpl.java | 2 +- .../main/java/org/apache/calcite/schema/impl/ViewTable.java | 2 +- .../java/org/apache/calcite/schema/impl/ViewTableMacro.java | 2 +- .../apache/calcite/schema/lookup/CompatibilityLookup.java | 2 +- .../java/org/apache/calcite/schema/lookup/ConcatLookup.java | 2 +- .../java/org/apache/calcite/schema/lookup/EmptyLookup.java | 2 +- .../org/apache/calcite/schema/lookup/IgnoreCaseLookup.java | 2 +- .../apache/calcite/schema/lookup/LoadingCacheLookup.java | 2 +- .../main/java/org/apache/calcite/schema/lookup/Lookup.java | 2 +- .../org/apache/calcite/schema/lookup/NameMapLookup.java | 2 +- .../main/java/org/apache/calcite/schema/lookup/Named.java | 4 ++-- .../org/apache/calcite/schema/lookup/SnapshotLookup.java | 2 +- .../apache/calcite/schema/lookup/TransformingLookup.java | 2 +- .../java/org/apache/calcite/schema/lookup/package-info.java | 2 +- .../main/java/org/apache/calcite/schema/package-info.java | 2 +- .../org/apache/calcite/server/CalciteServerStatement.java | 2 +- .../main/java/org/apache/calcite/server/package-info.java | 2 +- .../org/apache/calcite/sql/ExplicitOperatorBinding.java | 2 +- .../main/java/org/apache/calcite/sql/SqlAggFunction.java | 2 +- core/src/main/java/org/apache/calcite/sql/SqlAlter.java | 2 +- core/src/main/java/org/apache/calcite/sql/SqlAsofJoin.java | 2 +- core/src/main/java/org/apache/calcite/sql/SqlBasicCall.java | 2 +- .../main/java/org/apache/calcite/sql/SqlBasicFunction.java | 2 +- .../java/org/apache/calcite/sql/SqlBasicTypeNameSpec.java | 2 +- .../main/java/org/apache/calcite/sql/SqlBinaryOperator.java | 2 +- .../src/main/java/org/apache/calcite/sql/SqlByRewriter.java | 2 +- core/src/main/java/org/apache/calcite/sql/SqlCall.java | 2 +- .../main/java/org/apache/calcite/sql/SqlCallBinding.java | 2 +- .../java/org/apache/calcite/sql/SqlCharStringLiteral.java | 2 +- core/src/main/java/org/apache/calcite/sql/SqlCollation.java | 2 +- .../org/apache/calcite/sql/SqlConstantValueAggFunction.java | 2 +- .../main/java/org/apache/calcite/sql/SqlDataTypeSpec.java | 2 +- core/src/main/java/org/apache/calcite/sql/SqlDelete.java | 2 +- .../main/java/org/apache/calcite/sql/SqlDescribeSchema.java | 2 +- .../main/java/org/apache/calcite/sql/SqlDescribeTable.java | 2 +- core/src/main/java/org/apache/calcite/sql/SqlDialect.java | 2 +- .../java/org/apache/calcite/sql/SqlDialectFactoryImpl.java | 2 +- .../main/java/org/apache/calcite/sql/SqlDynamicParam.java | 2 +- core/src/main/java/org/apache/calcite/sql/SqlExplain.java | 2 +- core/src/main/java/org/apache/calcite/sql/SqlFunction.java | 2 +- .../java/org/apache/calcite/sql/SqlFunctionalOperator.java | 2 +- .../org/apache/calcite/sql/SqlGroupedWindowFunction.java | 2 +- core/src/main/java/org/apache/calcite/sql/SqlHint.java | 2 +- .../src/main/java/org/apache/calcite/sql/SqlIdentifier.java | 2 +- .../main/java/org/apache/calcite/sql/SqlInfixOperator.java | 2 +- core/src/main/java/org/apache/calcite/sql/SqlInsert.java | 2 +- .../java/org/apache/calcite/sql/SqlInternalOperator.java | 2 +- .../java/org/apache/calcite/sql/SqlIntervalLiteral.java | 2 +- .../java/org/apache/calcite/sql/SqlIntervalQualifier.java | 2 +- .../java/org/apache/calcite/sql/SqlJdbcDataTypeName.java | 2 +- .../java/org/apache/calcite/sql/SqlJdbcFunctionCall.java | 2 +- core/src/main/java/org/apache/calcite/sql/SqlJoin.java | 2 +- core/src/main/java/org/apache/calcite/sql/SqlLambda.java | 2 +- core/src/main/java/org/apache/calcite/sql/SqlLiteral.java | 2 +- .../main/java/org/apache/calcite/sql/SqlMatchFunction.java | 2 +- .../main/java/org/apache/calcite/sql/SqlMatchRecognize.java | 2 +- core/src/main/java/org/apache/calcite/sql/SqlMerge.java | 2 +- core/src/main/java/org/apache/calcite/sql/SqlNode.java | 2 +- core/src/main/java/org/apache/calcite/sql/SqlNodeList.java | 2 +- .../org/apache/calcite/sql/SqlNullTreatmentOperator.java | 2 +- .../main/java/org/apache/calcite/sql/SqlNumericLiteral.java | 2 +- core/src/main/java/org/apache/calcite/sql/SqlOperator.java | 2 +- .../java/org/apache/calcite/sql/SqlOperatorBinding.java | 2 +- .../main/java/org/apache/calcite/sql/SqlOperatorTable.java | 2 +- core/src/main/java/org/apache/calcite/sql/SqlOrderBy.java | 2 +- core/src/main/java/org/apache/calcite/sql/SqlPivot.java | 2 +- .../java/org/apache/calcite/sql/SqlPostfixOperator.java | 2 +- .../main/java/org/apache/calcite/sql/SqlPrefixOperator.java | 2 +- core/src/main/java/org/apache/calcite/sql/SqlSelect.java | 2 +- .../main/java/org/apache/calcite/sql/SqlSelectOperator.java | 2 +- .../main/java/org/apache/calcite/sql/SqlSetOperator.java | 2 +- core/src/main/java/org/apache/calcite/sql/SqlSetOption.java | 2 +- .../apache/calcite/sql/SqlSetSemanticsTableOperator.java | 2 +- core/src/main/java/org/apache/calcite/sql/SqlSnapshot.java | 2 +- .../org/apache/calcite/sql/SqlSpatialTypeOperatorTable.java | 2 +- .../java/org/apache/calcite/sql/SqlSpecialOperator.java | 2 +- .../org/apache/calcite/sql/SqlSplittableAggFunction.java | 2 +- .../main/java/org/apache/calcite/sql/SqlStarExclude.java | 2 +- .../main/java/org/apache/calcite/sql/SqlStarReplace.java | 2 +- .../java/org/apache/calcite/sql/SqlStaticAggFunction.java | 2 +- core/src/main/java/org/apache/calcite/sql/SqlSyntax.java | 2 +- .../main/java/org/apache/calcite/sql/SqlTableFunction.java | 2 +- core/src/main/java/org/apache/calcite/sql/SqlTableRef.java | 2 +- core/src/main/java/org/apache/calcite/sql/SqlUnpivot.java | 2 +- .../java/org/apache/calcite/sql/SqlUnresolvedFunction.java | 2 +- core/src/main/java/org/apache/calcite/sql/SqlUpdate.java | 2 +- core/src/main/java/org/apache/calcite/sql/SqlUtil.java | 2 +- core/src/main/java/org/apache/calcite/sql/SqlWindow.java | 2 +- .../java/org/apache/calcite/sql/SqlWindowTableFunction.java | 2 +- core/src/main/java/org/apache/calcite/sql/SqlWith.java | 2 +- core/src/main/java/org/apache/calcite/sql/SqlWithItem.java | 2 +- core/src/main/java/org/apache/calcite/sql/SqlWriter.java | 2 +- .../main/java/org/apache/calcite/sql/SqlWriterConfig.java | 2 +- .../java/org/apache/calcite/sql/TableCharacteristic.java | 2 +- .../main/java/org/apache/calcite/sql/advise/SqlAdvisor.java | 2 +- .../calcite/sql/advise/SqlAdvisorGetHintsFunction.java | 2 +- .../calcite/sql/advise/SqlAdvisorGetHintsFunction2.java | 2 +- .../java/org/apache/calcite/sql/advise/SqlAdvisorHint.java | 2 +- .../java/org/apache/calcite/sql/advise/SqlAdvisorHint2.java | 2 +- .../java/org/apache/calcite/sql/advise/SqlSimpleParser.java | 2 +- .../org/apache/calcite/sql/ddl/SqlAttributeDefinition.java | 2 +- .../java/org/apache/calcite/sql/ddl/SqlCheckConstraint.java | 2 +- .../org/apache/calcite/sql/ddl/SqlColumnDeclaration.java | 2 +- .../org/apache/calcite/sql/ddl/SqlCreateForeignSchema.java | 2 +- .../java/org/apache/calcite/sql/ddl/SqlCreateFunction.java | 2 +- .../apache/calcite/sql/ddl/SqlCreateMaterializedView.java | 2 +- .../java/org/apache/calcite/sql/ddl/SqlCreateSchema.java | 2 +- .../java/org/apache/calcite/sql/ddl/SqlCreateTable.java | 2 +- .../java/org/apache/calcite/sql/ddl/SqlCreateTableLike.java | 2 +- .../main/java/org/apache/calcite/sql/ddl/SqlCreateType.java | 2 +- .../main/java/org/apache/calcite/sql/ddl/SqlCreateView.java | 2 +- .../java/org/apache/calcite/sql/ddl/SqlDropFunction.java | 2 +- .../org/apache/calcite/sql/ddl/SqlDropMaterializedView.java | 2 +- .../main/java/org/apache/calcite/sql/ddl/SqlDropSchema.java | 2 +- .../main/java/org/apache/calcite/sql/ddl/SqlDropTable.java | 2 +- .../main/java/org/apache/calcite/sql/ddl/SqlDropType.java | 2 +- .../main/java/org/apache/calcite/sql/ddl/SqlDropView.java | 2 +- .../java/org/apache/calcite/sql/ddl/SqlKeyConstraint.java | 2 +- .../java/org/apache/calcite/sql/ddl/SqlTruncateTable.java | 2 +- .../org/apache/calcite/sql/dialect/BigQuerySqlDialect.java | 2 +- .../apache/calcite/sql/dialect/ClickHouseSqlDialect.java | 2 +- .../org/apache/calcite/sql/dialect/DorisSqlDialect.java | 2 +- .../org/apache/calcite/sql/dialect/ExasolSqlDialect.java | 2 +- .../org/apache/calcite/sql/dialect/FireboltSqlDialect.java | 2 +- .../java/org/apache/calcite/sql/dialect/HiveSqlDialect.java | 2 +- .../org/apache/calcite/sql/dialect/HsqldbSqlDialect.java | 2 +- .../apache/calcite/sql/dialect/JethroDataSqlDialect.java | 2 +- .../org/apache/calcite/sql/dialect/MssqlSqlDialect.java | 2 +- .../org/apache/calcite/sql/dialect/MysqlSqlDialect.java | 2 +- .../org/apache/calcite/sql/dialect/OracleSqlDialect.java | 2 +- .../org/apache/calcite/sql/dialect/PhoenixSqlDialect.java | 2 +- .../apache/calcite/sql/dialect/PostgresqlSqlDialect.java | 2 +- .../org/apache/calcite/sql/dialect/PrestoSqlDialect.java | 2 +- .../org/apache/calcite/sql/dialect/RedshiftSqlDialect.java | 2 +- .../org/apache/calcite/sql/dialect/SnowflakeSqlDialect.java | 2 +- .../org/apache/calcite/sql/dialect/SparkSqlDialect.java | 2 +- .../org/apache/calcite/sql/dialect/SqliteSqlDialect.java | 2 +- .../org/apache/calcite/sql/dialect/StarRocksSqlDialect.java | 2 +- .../org/apache/calcite/sql/dialect/SybaseSqlDialect.java | 2 +- .../org/apache/calcite/sql/dialect/TrinoSqlDialect.java | 2 +- .../org/apache/calcite/sql/dialect/VerticaSqlDialect.java | 2 +- .../apache/calcite/sql/fun/SqlAbstractGroupFunction.java | 2 +- .../org/apache/calcite/sql/fun/SqlAnyValueAggFunction.java | 2 +- .../java/org/apache/calcite/sql/fun/SqlAvgAggFunction.java | 2 +- .../org/apache/calcite/sql/fun/SqlBasicAggFunction.java | 2 +- .../org/apache/calcite/sql/fun/SqlBitOpAggFunction.java | 2 +- .../java/org/apache/calcite/sql/fun/SqlCallFactory.java | 2 +- core/src/main/java/org/apache/calcite/sql/fun/SqlCase.java | 2 +- .../java/org/apache/calcite/sql/fun/SqlCaseOperator.java | 2 +- .../org/apache/calcite/sql/fun/SqlCoalesceFunction.java | 2 +- .../java/org/apache/calcite/sql/fun/SqlConvertFunction.java | 2 +- .../org/apache/calcite/sql/fun/SqlCountAggFunction.java | 2 +- .../java/org/apache/calcite/sql/fun/SqlFloorFunction.java | 2 +- .../org/apache/calcite/sql/fun/SqlGroupingFunction.java | 2 +- .../org/apache/calcite/sql/fun/SqlInternalOperators.java | 2 +- .../apache/calcite/sql/fun/SqlJsonArrayAggAggFunction.java | 2 +- .../org/apache/calcite/sql/fun/SqlJsonArrayFunction.java | 2 +- .../org/apache/calcite/sql/fun/SqlJsonDepthFunction.java | 2 +- .../org/apache/calcite/sql/fun/SqlJsonModifyFunction.java | 2 +- .../org/apache/calcite/sql/fun/SqlJsonObjectFunction.java | 2 +- .../org/apache/calcite/sql/fun/SqlJsonPrettyFunction.java | 2 +- .../org/apache/calcite/sql/fun/SqlJsonQueryFunction.java | 2 +- .../org/apache/calcite/sql/fun/SqlJsonTypeFunction.java | 2 +- .../org/apache/calcite/sql/fun/SqlJsonValueFunction.java | 2 +- .../main/java/org/apache/calcite/sql/fun/SqlLibrary.java | 2 +- .../org/apache/calcite/sql/fun/SqlLibraryOperators.java | 2 +- .../org/apache/calcite/sql/fun/SqlLiteralAggFunction.java | 2 +- .../org/apache/calcite/sql/fun/SqlMapValueConstructor.java | 2 +- .../org/apache/calcite/sql/fun/SqlMinMaxAggFunction.java | 2 +- .../apache/calcite/sql/fun/SqlMonotonicUnaryFunction.java | 2 +- .../apache/calcite/sql/fun/SqlMultisetValueConstructor.java | 2 +- .../org/apache/calcite/sql/fun/SqlOverlapsOperator.java | 2 +- .../org/apache/calcite/sql/fun/SqlQuantifyOperator.java | 2 +- .../java/org/apache/calcite/sql/fun/SqlRowOperator.java | 2 +- .../apache/calcite/sql/fun/SqlSingleValueAggFunction.java | 2 +- .../org/apache/calcite/sql/fun/SqlSpatialTypeFunctions.java | 2 +- .../org/apache/calcite/sql/fun/SqlStdOperatorTable.java | 2 +- .../java/org/apache/calcite/sql/fun/SqlSumAggFunction.java | 2 +- .../calcite/sql/fun/SqlSumEmptyIsZeroAggFunction.java | 2 +- .../org/apache/calcite/sql/fun/SqlTimestampAddFunction.java | 2 +- .../java/org/apache/calcite/sql/fun/SqlTrimFunction.java | 2 +- core/src/main/java/org/apache/calcite/sql/package-info.java | 2 +- core/src/main/java/org/apache/calcite/sql/parser/Span.java | 2 +- .../apache/calcite/sql/parser/SqlAbstractParserImpl.java | 2 +- .../java/org/apache/calcite/sql/parser/SqlParserPos.java | 2 +- .../java/org/apache/calcite/sql/parser/SqlParserUtil.java | 2 +- .../java/org/apache/calcite/sql/parser/StringAndPos.java | 2 +- .../java/org/apache/calcite/sql/pretty/SqlPrettyWriter.java | 2 +- .../java/org/apache/calcite/sql/type/AbstractSqlType.java | 2 +- .../main/java/org/apache/calcite/sql/type/ArraySqlType.java | 2 +- .../calcite/sql/type/AssignableOperandTypeChecker.java | 2 +- .../main/java/org/apache/calcite/sql/type/BasicSqlType.java | 2 +- .../calcite/sql/type/CompositeOperandTypeChecker.java | 2 +- .../calcite/sql/type/CompositeSingleOperandTypeChecker.java | 2 +- .../apache/calcite/sql/type/CursorReturnTypeInference.java | 2 +- .../calcite/sql/type/JavaToSqlTypeConversionRules.java | 2 +- .../main/java/org/apache/calcite/sql/type/MapSqlType.java | 2 +- .../apache/calcite/sql/type/MatchReturnTypeInference.java | 2 +- .../java/org/apache/calcite/sql/type/MultisetSqlType.java | 2 +- .../java/org/apache/calcite/sql/type/ObjectSqlType.java | 2 +- .../java/org/apache/calcite/sql/type/OperandHandlers.java | 2 +- .../main/java/org/apache/calcite/sql/type/OperandTypes.java | 2 +- .../main/java/org/apache/calcite/sql/type/ReturnTypes.java | 2 +- .../org/apache/calcite/sql/type/SameOperandTypeChecker.java | 2 +- .../sql/type/SameOperandTypeExceptLastOperandChecker.java | 2 +- .../org/apache/calcite/sql/type/SqlOperandTypeChecker.java | 2 +- .../org/apache/calcite/sql/type/SqlReturnTypeInference.java | 2 +- .../calcite/sql/type/SqlReturnTypeInferenceChain.java | 2 +- .../calcite/sql/type/SqlTypeExplicitPrecedenceList.java | 2 +- .../org/apache/calcite/sql/type/SqlTypeFactoryImpl.java | 2 +- .../java/org/apache/calcite/sql/type/SqlTypeFamily.java | 2 +- .../main/java/org/apache/calcite/sql/type/SqlTypeName.java | 2 +- .../apache/calcite/sql/type/SqlTypeTransformCascade.java | 2 +- .../main/java/org/apache/calcite/sql/type/SqlTypeUtil.java | 2 +- .../calcite/sql/type/TableFunctionReturnTypeInference.java | 2 +- .../apache/calcite/sql/util/ChainedSqlOperatorTable.java | 2 +- core/src/main/java/org/apache/calcite/sql/util/IdPair.java | 2 +- .../org/apache/calcite/sql/util/ListSqlOperatorTable.java | 2 +- .../apache/calcite/sql/util/ReflectiveSqlOperatorTable.java | 2 +- .../java/org/apache/calcite/sql/util/SqlBasicVisitor.java | 2 +- .../main/java/org/apache/calcite/sql/util/SqlShuttle.java | 2 +- .../main/java/org/apache/calcite/sql/util/SqlString.java | 2 +- .../org/apache/calcite/sql/validate/AbstractNamespace.java | 2 +- .../java/org/apache/calcite/sql/validate/AggFinder.java | 2 +- .../java/org/apache/calcite/sql/validate/AggVisitor.java | 2 +- .../org/apache/calcite/sql/validate/AliasNamespace.java | 2 +- .../org/apache/calcite/sql/validate/CollectNamespace.java | 2 +- .../java/org/apache/calcite/sql/validate/CollectScope.java | 2 +- .../apache/calcite/sql/validate/DelegatingNamespace.java | 2 +- .../org/apache/calcite/sql/validate/DelegatingScope.java | 2 +- .../sql/validate/DelegatingSqlValidatorCatalogReader.java | 2 +- .../java/org/apache/calcite/sql/validate/EmptyScope.java | 2 +- .../org/apache/calcite/sql/validate/FieldNamespace.java | 2 +- .../apache/calcite/sql/validate/IdentifierNamespace.java | 2 +- .../java/org/apache/calcite/sql/validate/JoinNamespace.java | 2 +- .../java/org/apache/calcite/sql/validate/JoinScope.java | 2 +- .../org/apache/calcite/sql/validate/LambdaNamespace.java | 2 +- .../java/org/apache/calcite/sql/validate/ListScope.java | 2 +- .../calcite/sql/validate/MatchRecognizeNamespace.java | 2 +- .../java/org/apache/calcite/sql/validate/MeasureScope.java | 2 +- .../java/org/apache/calcite/sql/validate/OrderByScope.java | 2 +- .../org/apache/calcite/sql/validate/ParameterNamespace.java | 2 +- .../org/apache/calcite/sql/validate/ParameterScope.java | 2 +- .../org/apache/calcite/sql/validate/ProcedureNamespace.java | 2 +- .../org/apache/calcite/sql/validate/SchemaNamespace.java | 2 +- .../java/org/apache/calcite/sql/validate/SelectScope.java | 2 +- .../java/org/apache/calcite/sql/validate/SemanticTable.java | 2 +- .../org/apache/calcite/sql/validate/SetopNamespace.java | 2 +- .../org/apache/calcite/sql/validate/SqlLambdaScope.java | 2 +- .../org/apache/calcite/sql/validate/SqlMonikerImpl.java | 2 +- .../org/apache/calcite/sql/validate/SqlNameMatcher.java | 2 +- .../org/apache/calcite/sql/validate/SqlNameMatchers.java | 2 +- .../java/org/apache/calcite/sql/validate/SqlQualified.java | 2 +- .../org/apache/calcite/sql/validate/SqlScopedShuttle.java | 2 +- .../calcite/sql/validate/SqlUserDefinedAggFunction.java | 2 +- .../apache/calcite/sql/validate/SqlUserDefinedFunction.java | 2 +- .../calcite/sql/validate/SqlUserDefinedTableFunction.java | 2 +- .../calcite/sql/validate/SqlUserDefinedTableMacro.java | 2 +- .../java/org/apache/calcite/sql/validate/SqlValidator.java | 2 +- .../calcite/sql/validate/SqlValidatorCatalogReader.java | 2 +- .../org/apache/calcite/sql/validate/SqlValidatorImpl.java | 4 ++-- .../apache/calcite/sql/validate/SqlValidatorNamespace.java | 2 +- .../org/apache/calcite/sql/validate/SqlValidatorScope.java | 2 +- .../org/apache/calcite/sql/validate/SqlValidatorUtil.java | 2 +- .../apache/calcite/sql/validate/SqlValidatorWithHints.java | 2 +- .../apache/calcite/sql/validate/SqlWithItemTableRef.java | 2 +- .../calcite/sql/validate/TableConstructorNamespace.java | 2 +- .../org/apache/calcite/sql/validate/TableNamespace.java | 2 +- .../org/apache/calcite/sql/validate/UnnestNamespace.java | 2 +- .../org/apache/calcite/sql/validate/WithItemNamespace.java | 2 +- .../calcite/sql/validate/WithItemRecursiveNamespace.java | 2 +- .../java/org/apache/calcite/sql/validate/WithNamespace.java | 2 +- .../org/apache/calcite/sql/validate/WithRecursiveScope.java | 2 +- .../java/org/apache/calcite/sql/validate/WithScope.java | 2 +- .../calcite/sql/validate/implicit/AbstractTypeCoercion.java | 2 +- .../apache/calcite/sql/validate/implicit/TypeCoercion.java | 2 +- .../calcite/sql/validate/implicit/TypeCoercionImpl.java | 2 +- .../main/java/org/apache/calcite/sql2rel/AggConverter.java | 2 +- .../apache/calcite/sql2rel/CorrelateProjectExtractor.java | 2 +- .../calcite/sql2rel/InitializerExpressionFactory.java | 2 +- .../calcite/sql2rel/NullInitializerExpressionFactory.java | 2 +- .../apache/calcite/sql2rel/ReflectiveConvertletTable.java | 2 +- .../java/org/apache/calcite/sql2rel/RelDecorrelator.java | 2 +- .../java/org/apache/calcite/sql2rel/RelFieldTrimmer.java | 2 +- .../apache/calcite/sql2rel/RelStructuredTypeFlattener.java | 2 +- .../org/apache/calcite/sql2rel/SqlRexConvertletTable.java | 2 +- .../java/org/apache/calcite/sql2rel/SqlToRelConverter.java | 2 +- .../org/apache/calcite/sql2rel/StandardConvertletTable.java | 2 +- .../java/org/apache/calcite/sql2rel/SubQueryConverter.java | 2 +- .../apache/calcite/sql2rel/TopDownGeneralDecorrelator.java | 2 +- .../main/java/org/apache/calcite/sql2rel/package-info.java | 2 +- .../java/org/apache/calcite/statistic/package-info.java | 2 +- .../main/java/org/apache/calcite/tools/FrameworkConfig.java | 2 +- core/src/main/java/org/apache/calcite/tools/Frameworks.java | 2 +- core/src/main/java/org/apache/calcite/tools/Hoist.java | 2 +- .../main/java/org/apache/calcite/tools/PigRelBuilder.java | 2 +- core/src/main/java/org/apache/calcite/tools/RelBuilder.java | 2 +- .../java/org/apache/calcite/tools/RelBuilderFactory.java | 2 +- core/src/main/java/org/apache/calcite/tools/RuleSets.java | 2 +- .../main/java/org/apache/calcite/tools/package-info.java | 2 +- core/src/main/java/org/apache/calcite/util/Arrow.java | 2 +- .../org/apache/calcite/util/BarfingInvocationHandler.java | 2 +- core/src/main/java/org/apache/calcite/util/BitString.java | 2 +- .../src/main/java/org/apache/calcite/util/BlackholeMap.java | 2 +- .../main/java/org/apache/calcite/util/BuiltInMethod.java | 2 +- core/src/main/java/org/apache/calcite/util/ChunkList.java | 2 +- .../src/main/java/org/apache/calcite/util/CompositeMap.java | 2 +- .../main/java/org/apache/calcite/util/ConversionUtil.java | 2 +- core/src/main/java/org/apache/calcite/util/DateString.java | 2 +- .../apache/calcite/util/DelegatingInvocationHandler.java | 2 +- core/src/main/java/org/apache/calcite/util/Filterator.java | 2 +- core/src/main/java/org/apache/calcite/util/Glossary.java | 2 +- core/src/main/java/org/apache/calcite/util/Holder.java | 2 +- .../main/java/org/apache/calcite/util/ImmutableBitSet.java | 2 +- .../main/java/org/apache/calcite/util/ImmutableIntList.java | 2 +- .../java/org/apache/calcite/util/ImmutableNullableList.java | 2 +- .../java/org/apache/calcite/util/ImmutableNullableSet.java | 4 ++-- .../java/org/apache/calcite/util/IntegerIntervalSet.java | 2 +- core/src/main/java/org/apache/calcite/util/JdbcType.java | 2 +- .../src/main/java/org/apache/calcite/util/JdbcTypeImpl.java | 2 +- core/src/main/java/org/apache/calcite/util/JsonBuilder.java | 2 +- core/src/main/java/org/apache/calcite/util/Litmus.java | 2 +- .../java/org/apache/calcite/util/MonotonicSupplier.java | 2 +- core/src/main/java/org/apache/calcite/util/NameMap.java | 2 +- .../src/main/java/org/apache/calcite/util/NameMultimap.java | 2 +- core/src/main/java/org/apache/calcite/util/NameSet.java | 2 +- core/src/main/java/org/apache/calcite/util/NlsString.java | 2 +- core/src/main/java/org/apache/calcite/util/NumberUtil.java | 2 +- core/src/main/java/org/apache/calcite/util/Pair.java | 2 +- .../java/org/apache/calcite/util/PartiallyOrderedSet.java | 2 +- core/src/main/java/org/apache/calcite/util/Permutation.java | 2 +- .../org/apache/calcite/util/PrecedenceClimbingParser.java | 2 +- core/src/main/java/org/apache/calcite/util/RangeSets.java | 2 +- core/src/main/java/org/apache/calcite/util/ReflectUtil.java | 2 +- .../org/apache/calcite/util/ReflectiveVisitDispatcher.java | 2 +- core/src/main/java/org/apache/calcite/util/Sarg.java | 2 +- .../org/apache/calcite/util/SimpleNamespaceContext.java | 2 +- core/src/main/java/org/apache/calcite/util/Source.java | 2 +- core/src/main/java/org/apache/calcite/util/Sources.java | 2 +- core/src/main/java/org/apache/calcite/util/Template.java | 2 +- core/src/main/java/org/apache/calcite/util/TimeString.java | 2 +- .../org/apache/calcite/util/TimeWithTimeZoneString.java | 2 +- .../main/java/org/apache/calcite/util/TimestampString.java | 2 +- .../apache/calcite/util/TimestampWithTimeZoneString.java | 2 +- .../main/java/org/apache/calcite/util/TryThreadLocal.java | 4 ++-- core/src/main/java/org/apache/calcite/util/Util.java | 2 +- core/src/main/java/org/apache/calcite/util/XmlOutput.java | 2 +- .../util/format/postgresql/CompiledDateTimeFormat.java | 2 +- .../util/format/postgresql/PostgresqlDateTimeFormatter.java | 2 +- .../apache/calcite/util/graph/AttributedDirectedGraph.java | 2 +- .../org/apache/calcite/util/graph/DefaultDirectedGraph.java | 2 +- .../java/org/apache/calcite/util/graph/DefaultEdge.java | 2 +- .../java/org/apache/calcite/util/graph/DirectedGraph.java | 2 +- .../java/org/apache/calcite/util/javac/JaninoCompiler.java | 2 +- .../main/java/org/apache/calcite/util/mapping/IntPair.java | 2 +- .../main/java/org/apache/calcite/util/mapping/Mappings.java | 2 +- .../src/main/java/org/apache/calcite/util/package-info.java | 2 +- .../java/org/apache/calcite/util/trace/CalciteLogger.java | 2 +- .../org/apache/calcite/util/trace/CalciteTimingTracer.java | 2 +- .../java/org/apache/calcite/util/trace/CalciteTrace.java | 2 +- .../adapter/enumerable/EnumerableCustomAggregateTest.java | 2 +- .../calcite/adapter/enumerable/RexImplementorTableTest.java | 2 +- .../org/apache/calcite/adapter/generate/RangeTable.java | 2 +- .../org/apache/calcite/jdbc/CalciteRemoteDriverTest.java | 2 +- .../CustomMaterializedViewRecognitionRuleTest.java | 2 +- .../test/java/org/apache/calcite/plan/RelWriterTest.java | 2 +- .../calcite/plan/volcano/CollationConversionTest.java | 2 +- .../java/org/apache/calcite/plan/volcano/ComboRuleTest.java | 2 +- .../calcite/plan/volcano/MultipleTraitConversionTest.java | 2 +- .../java/org/apache/calcite/plan/volcano/PlannerTests.java | 2 +- .../apache/calcite/plan/volcano/TraitConversionTest.java | 2 +- .../apache/calcite/plan/volcano/TraitPropagationTest.java | 2 +- .../calcite/plan/volcano/VolcanoPlannerTraitTest.java | 2 +- .../apache/calcite/prepare/LookupOperatorOverloadsTest.java | 2 +- .../apache/calcite/rel/logical/ToLogicalConverterTest.java | 2 +- .../apache/calcite/rel/rel2sql/RelToSqlConverterTest.java | 2 +- .../test/java/org/apache/calcite/rex/RexProgramTest.java | 2 +- .../java/org/apache/calcite/runtime/EnumerablesTest.java | 2 +- .../java/org/apache/calcite/schema/lookup/FakeLookup.java | 2 +- .../apache/calcite/schema/lookup/IgnoreCaseLookupTest.java | 2 +- .../java/org/apache/calcite/schemas/HrClusteredSchema.java | 2 +- .../java/org/apache/calcite/sql/test/SqlAdvisorTest.java | 2 +- .../org/apache/calcite/sql/test/SqlPrettyWriterFixture.java | 2 +- .../org/apache/calcite/sql/test/SqlPrettyWriterTest.java | 2 +- .../org/apache/calcite/sql/type/RelDataTypeSystemTest.java | 2 +- .../apache/calcite/sql/validate/LexCaseSensitiveTest.java | 2 +- .../java/org/apache/calcite/sql/validate/LexEscapeTest.java | 2 +- .../calcite/sql2rel/CorrelateProjectExtractorTest.java | 2 +- .../org/apache/calcite/sql2rel/RelDecorrelatorTest.java | 2 +- .../org/apache/calcite/sql2rel/RelFieldTrimmerTest.java | 2 +- .../java/org/apache/calcite/test/CollectionTypeTest.java | 2 +- .../test/java/org/apache/calcite/test/CoreQuidemTest.java | 2 +- .../test/java/org/apache/calcite/test/HepPlannerTest.java | 2 +- .../test/java/org/apache/calcite/test/InterpreterTest.java | 2 +- core/src/test/java/org/apache/calcite/test/JdbcTest.java | 2 +- core/src/test/java/org/apache/calcite/test/LintTest.java | 2 +- .../java/org/apache/calcite/test/MaterializationTest.java | 2 +- .../org/apache/calcite/test/MaterializedViewFixture.java | 2 +- .../test/java/org/apache/calcite/test/RelBuilderTest.java | 2 +- .../test/java/org/apache/calcite/test/RelMetadataTest.java | 4 ++-- .../test/java/org/apache/calcite/test/RelOptRulesTest.java | 2 +- .../org/apache/calcite/test/RuleMatchVisualizerTest.java | 2 +- .../java/org/apache/calcite/test/ScannableTableTest.java | 2 +- .../java/org/apache/calcite/test/SqlHintsConverterTest.java | 2 +- core/src/test/java/org/apache/calcite/test/SqlTestGen.java | 2 +- .../java/org/apache/calcite/test/SqlToRelConverterTest.java | 2 +- .../org/apache/calcite/test/SqlValidatorFeatureTest.java | 2 +- .../test/java/org/apache/calcite/test/SqlValidatorTest.java | 2 +- .../java/org/apache/calcite/test/SqlXmlFunctionsTest.java | 2 +- .../test/java/org/apache/calcite/test/TCatalogReader.java | 2 +- .../test/java/org/apache/calcite/test/TopDownOptTest.java | 2 +- .../org/apache/calcite/test/TypeCoercionConverterTest.java | 2 +- .../test/java/org/apache/calcite/test/TypeCoercionTest.java | 2 +- .../test/concurrent/ConcurrentTestCommandExecutor.java | 2 +- .../test/concurrent/ConcurrentTestCommandGenerator.java | 2 +- .../test/concurrent/ConcurrentTestCommandScript.java | 2 +- .../test/concurrent/ConcurrentTestPluginCommand.java | 2 +- .../test/java/org/apache/calcite/tools/FrameworksTest.java | 2 +- core/src/test/java/org/apache/calcite/util/TestUnsafe.java | 2 +- core/src/test/java/org/apache/calcite/util/UtilTest.java | 4 ++-- .../org/apache/calcite/util/graph/DirectedGraphTest.java | 2 +- .../calcite/adapter/druid/BinaryOperatorConversion.java | 2 +- .../calcite/adapter/druid/CeilOperatorConversion.java | 2 +- .../apache/calcite/adapter/druid/DefaultDimensionSpec.java | 2 +- .../org/apache/calcite/adapter/druid/DimensionSpec.java | 2 +- .../calcite/adapter/druid/DirectOperatorConversion.java | 2 +- .../apache/calcite/adapter/druid/DruidConnectionImpl.java | 2 +- .../apache/calcite/adapter/druid/DruidDateTimeUtils.java | 2 +- .../org/apache/calcite/adapter/druid/DruidExpressions.java | 2 +- .../org/apache/calcite/adapter/druid/DruidJsonFilter.java | 2 +- .../java/org/apache/calcite/adapter/druid/DruidQuery.java | 2 +- .../java/org/apache/calcite/adapter/druid/DruidRules.java | 2 +- .../java/org/apache/calcite/adapter/druid/DruidSchema.java | 2 +- .../apache/calcite/adapter/druid/DruidSqlCastConverter.java | 2 +- .../calcite/adapter/druid/DruidSqlOperatorConverter.java | 2 +- .../java/org/apache/calcite/adapter/druid/DruidTable.java | 2 +- .../org/apache/calcite/adapter/druid/DruidTableFactory.java | 2 +- .../calcite/adapter/druid/ExtractOperatorConversion.java | 2 +- .../calcite/adapter/druid/ExtractionDimensionSpec.java | 2 +- .../calcite/adapter/druid/FloorOperatorConversion.java | 2 +- .../apache/calcite/adapter/druid/NaryOperatorConverter.java | 2 +- .../calcite/adapter/druid/SubstringOperatorConversion.java | 2 +- .../calcite/adapter/druid/TimeExtractionFunction.java | 2 +- .../adapter/druid/UnaryPrefixOperatorConversion.java | 2 +- .../adapter/druid/UnarySuffixOperatorConversion.java | 2 +- .../adapter/elasticsearch/ElasticsearchAggregate.java | 2 +- .../calcite/adapter/elasticsearch/ElasticsearchFilter.java | 2 +- .../calcite/adapter/elasticsearch/ElasticsearchJson.java | 2 +- .../calcite/adapter/elasticsearch/ElasticsearchMapping.java | 2 +- .../calcite/adapter/elasticsearch/ElasticsearchProject.java | 2 +- .../calcite/adapter/elasticsearch/ElasticsearchRules.java | 2 +- .../calcite/adapter/elasticsearch/ElasticsearchSchema.java | 2 +- .../calcite/adapter/elasticsearch/ElasticsearchSort.java | 2 +- .../adapter/elasticsearch/ElasticsearchTableScan.java | 2 +- .../elasticsearch/ElasticsearchToEnumerableConverter.java | 2 +- .../org/apache/calcite/adapter/csv/CsvFilterableTable.java | 2 +- .../apache/calcite/adapter/csv/CsvProjectTableScanRule.java | 2 +- .../org/apache/calcite/adapter/csv/CsvScannableTable.java | 2 +- .../main/java/org/apache/calcite/adapter/csv/CsvSchema.java | 2 +- .../apache/calcite/adapter/csv/CsvStreamScannableTable.java | 2 +- .../apache/calcite/adapter/csv/CsvStreamTableFactory.java | 2 +- .../main/java/org/apache/calcite/adapter/csv/CsvTable.java | 2 +- .../org/apache/calcite/adapter/csv/CsvTableFactory.java | 2 +- .../java/org/apache/calcite/adapter/csv/CsvTableScan.java | 2 +- .../apache/calcite/adapter/csv/CsvTranslatableTable.java | 2 +- .../csv/src/test/java/org/apache/calcite/test/CsvTest.java | 2 +- .../java/org/apache/calcite/example/maze/MazeTable.java | 2 +- .../java/org/apache/calcite/adapter/file/CsvEnumerator.java | 2 +- .../org/apache/calcite/adapter/file/CsvStreamReader.java | 2 +- .../main/java/org/apache/calcite/adapter/file/CsvTable.java | 2 +- .../org/apache/calcite/adapter/file/CsvTableFactory.java | 2 +- .../java/org/apache/calcite/adapter/file/CsvTableScan.java | 2 +- .../apache/calcite/adapter/file/CsvTranslatableTable.java | 2 +- .../org/apache/calcite/adapter/file/FileEnumerator.java | 2 +- .../java/org/apache/calcite/adapter/file/FileFieldType.java | 2 +- .../java/org/apache/calcite/adapter/file/FileReader.java | 2 +- .../org/apache/calcite/adapter/file/FileRowConverter.java | 2 +- .../java/org/apache/calcite/adapter/file/FileSchema.java | 2 +- .../java/org/apache/calcite/adapter/file/FileTable.java | 2 +- .../org/apache/calcite/adapter/file/JsonEnumerator.java | 2 +- .../org/apache/calcite/adapter/file/JsonScannableTable.java | 2 +- .../java/org/apache/calcite/adapter/file/JsonTable.java | 2 +- .../org/apache/calcite/adapter/file/FileAdapterTests.java | 2 +- .../apache/calcite/adapter/geode/rel/GeodeAggregate.java | 2 +- .../apache/calcite/adapter/geode/rel/GeodeEnumerator.java | 2 +- .../org/apache/calcite/adapter/geode/rel/GeodeFilter.java | 2 +- .../org/apache/calcite/adapter/geode/rel/GeodeProject.java | 2 +- .../org/apache/calcite/adapter/geode/rel/GeodeRules.java | 2 +- .../org/apache/calcite/adapter/geode/rel/GeodeSort.java | 2 +- .../org/apache/calcite/adapter/geode/rel/GeodeTable.java | 2 +- .../apache/calcite/adapter/geode/rel/GeodeTableScan.java | 2 +- .../adapter/geode/rel/GeodeToEnumerableConverter.java | 2 +- .../calcite/adapter/geode/simple/GeodeSimpleEnumerator.java | 2 +- .../adapter/geode/simple/GeodeSimpleScannableTable.java | 2 +- .../org/apache/calcite/adapter/geode/util/GeodeUtils.java | 2 +- .../org/apache/calcite/adapter/innodb/IndexCondition.java | 2 +- .../org/apache/calcite/adapter/innodb/InnodbEnumerator.java | 2 +- .../org/apache/calcite/adapter/innodb/InnodbFilter.java | 2 +- .../calcite/adapter/innodb/InnodbFilterTranslator.java | 2 +- .../org/apache/calcite/adapter/innodb/InnodbProject.java | 2 +- .../java/org/apache/calcite/adapter/innodb/InnodbSort.java | 2 +- .../org/apache/calcite/adapter/innodb/InnodbTableScan.java | 2 +- .../calcite/adapter/innodb/InnodbToEnumerableConverter.java | 2 +- .../apache/calcite/adapter/innodb/InnodbAdapterTest.java | 2 +- .../calcite/adapter/kafka/KafkaMessageEnumerator.java | 2 +- .../org/apache/calcite/adapter/kafka/KafkaStreamTable.java | 2 +- .../org/apache/calcite/adapter/kafka/KafkaTableFactory.java | 2 +- .../main/java/org/apache/calcite/linq4j/BaseQueryable.java | 2 +- .../java/org/apache/calcite/linq4j/DefaultEnumerable.java | 2 +- .../java/org/apache/calcite/linq4j/DefaultQueryable.java | 2 +- .../java/org/apache/calcite/linq4j/EnumerableDefaults.java | 2 +- .../apache/calcite/linq4j/EnumerableOrderedQueryable.java | 2 +- .../java/org/apache/calcite/linq4j/EnumerableQueryable.java | 2 +- .../java/org/apache/calcite/linq4j/ExtendedEnumerable.java | 2 +- .../java/org/apache/calcite/linq4j/ExtendedQueryable.java | 2 +- .../main/java/org/apache/calcite/linq4j/GroupingImpl.java | 2 +- linq4j/src/main/java/org/apache/calcite/linq4j/Linq4j.java | 2 +- .../src/main/java/org/apache/calcite/linq4j/LookupImpl.java | 2 +- .../java/org/apache/calcite/linq4j/MemoryEnumerator.java | 2 +- .../main/java/org/apache/calcite/linq4j/MemoryFactory.java | 2 +- .../org/apache/calcite/linq4j/MergeUnionEnumerator.java | 2 +- .../main/java/org/apache/calcite/linq4j/ModularInteger.java | 2 +- .../src/main/java/org/apache/calcite/linq4j/Nullness.java | 4 ++-- linq4j/src/main/java/org/apache/calcite/linq4j/Ord.java | 2 +- .../java/org/apache/calcite/linq4j/QueryableDefaults.java | 2 +- .../java/org/apache/calcite/linq4j/QueryableFactory.java | 2 +- .../java/org/apache/calcite/linq4j/QueryableRecorder.java | 2 +- .../main/java/org/apache/calcite/linq4j/RawQueryable.java | 2 +- .../java/org/apache/calcite/linq4j/function/Functions.java | 2 +- .../main/java/org/apache/calcite/linq4j/package-info.java | 2 +- .../java/org/apache/calcite/linq4j/tree/AbstractNode.java | 2 +- .../apache/calcite/linq4j/tree/ArrayLengthRecordField.java | 2 +- .../org/apache/calcite/linq4j/tree/BinaryExpression.java | 2 +- .../java/org/apache/calcite/linq4j/tree/BlockBuilder.java | 2 +- .../java/org/apache/calcite/linq4j/tree/BlockStatement.java | 2 +- .../java/org/apache/calcite/linq4j/tree/CatchBlock.java | 2 +- .../org/apache/calcite/linq4j/tree/ClassDeclaration.java | 2 +- .../apache/calcite/linq4j/tree/ClassDeclarationFinder.java | 2 +- .../apache/calcite/linq4j/tree/ConditionalExpression.java | 2 +- .../apache/calcite/linq4j/tree/ConditionalStatement.java | 2 +- .../org/apache/calcite/linq4j/tree/ConstantExpression.java | 2 +- .../org/apache/calcite/linq4j/tree/ConstantUntypedNull.java | 2 +- .../apache/calcite/linq4j/tree/ConstructorDeclaration.java | 2 +- .../apache/calcite/linq4j/tree/DeclarationStatement.java | 2 +- .../calcite/linq4j/tree/DeterministicCodeOptimizer.java | 2 +- .../main/java/org/apache/calcite/linq4j/tree/Evaluator.java | 2 +- .../java/org/apache/calcite/linq4j/tree/ExpressionType.java | 2 +- .../org/apache/calcite/linq4j/tree/ExpressionWriter.java | 2 +- .../java/org/apache/calcite/linq4j/tree/Expressions.java | 2 +- .../org/apache/calcite/linq4j/tree/FieldDeclaration.java | 2 +- .../org/apache/calcite/linq4j/tree/ForEachStatement.java | 2 +- .../java/org/apache/calcite/linq4j/tree/ForStatement.java | 2 +- .../org/apache/calcite/linq4j/tree/FunctionExpression.java | 2 +- .../java/org/apache/calcite/linq4j/tree/GotoStatement.java | 2 +- .../org/apache/calcite/linq4j/tree/IndexExpression.java | 2 +- .../java/org/apache/calcite/linq4j/tree/LabelStatement.java | 2 +- .../java/org/apache/calcite/linq4j/tree/LabelTarget.java | 2 +- .../org/apache/calcite/linq4j/tree/MemberExpression.java | 2 +- .../apache/calcite/linq4j/tree/MethodCallExpression.java | 2 +- .../org/apache/calcite/linq4j/tree/MethodDeclaration.java | 2 +- .../org/apache/calcite/linq4j/tree/NewArrayExpression.java | 2 +- .../java/org/apache/calcite/linq4j/tree/NewExpression.java | 2 +- .../org/apache/calcite/linq4j/tree/OptimizeShuttle.java | 2 +- .../org/apache/calcite/linq4j/tree/ParameterExpression.java | 2 +- .../main/java/org/apache/calcite/linq4j/tree/Primitive.java | 2 +- .../java/org/apache/calcite/linq4j/tree/PseudoField.java | 2 +- .../apache/calcite/linq4j/tree/ReflectedPseudoField.java | 2 +- .../main/java/org/apache/calcite/linq4j/tree/Shuttle.java | 2 +- .../org/apache/calcite/linq4j/tree/TernaryExpression.java | 2 +- .../java/org/apache/calcite/linq4j/tree/ThrowStatement.java | 2 +- .../java/org/apache/calcite/linq4j/tree/TryStatement.java | 2 +- .../apache/calcite/linq4j/tree/TypeBinaryExpression.java | 2 +- .../src/main/java/org/apache/calcite/linq4j/tree/Types.java | 2 +- .../org/apache/calcite/linq4j/tree/UnaryExpression.java | 2 +- .../java/org/apache/calcite/linq4j/tree/UnsignedType.java | 2 +- .../java/org/apache/calcite/linq4j/tree/VisitorImpl.java | 2 +- .../java/org/apache/calcite/linq4j/tree/WhileStatement.java | 2 +- .../java/org/apache/calcite/linq4j/util/Compatible.java | 2 +- .../org/apache/calcite/linq4j/test/BlockBuilderTest.java | 2 +- .../java/org/apache/calcite/linq4j/test/ExpressionTest.java | 2 +- .../apache/calcite/linq4j/test/JoinPreserveOrderTest.java | 2 +- .../java/org/apache/calcite/linq4j/test/LimitSortTest.java | 2 +- .../calcite/linq4j/tree/IdentifierValidationTest.java | 2 +- .../org/apache/calcite/adapter/mongodb/MongoAggregate.java | 2 +- .../org/apache/calcite/adapter/mongodb/MongoEnumerator.java | 2 +- .../org/apache/calcite/adapter/mongodb/MongoFilter.java | 2 +- .../org/apache/calcite/adapter/mongodb/MongoProject.java | 2 +- .../java/org/apache/calcite/adapter/mongodb/MongoRel.java | 2 +- .../java/org/apache/calcite/adapter/mongodb/MongoSort.java | 2 +- .../org/apache/calcite/adapter/mongodb/MongoTableScan.java | 2 +- .../calcite/adapter/mongodb/MongoToEnumerableConverter.java | 2 +- .../org/apache/calcite/adapter/pig/PigTableFactory.java | 2 +- .../src/main/java/org/apache/calcite/piglet/PigTable.java | 2 +- .../calcite/adapter/os/AbstractBaseScannableTable.java | 2 +- .../org/apache/calcite/adapter/os/CpuInfoTableFunction.java | 2 +- .../org/apache/calcite/adapter/os/CpuTimeTableFunction.java | 2 +- .../java/org/apache/calcite/adapter/os/DuTableFunction.java | 2 +- .../org/apache/calcite/adapter/os/FilesTableFunction.java | 2 +- .../apache/calcite/adapter/os/GitCommitsTableFunction.java | 2 +- .../calcite/adapter/os/InterfaceAddressesTableFunction.java | 2 +- .../calcite/adapter/os/InterfaceDetailsTableFunction.java | 2 +- .../apache/calcite/adapter/os/JavaInfoTableFunction.java | 2 +- .../org/apache/calcite/adapter/os/JpsTableFunction.java | 2 +- .../apache/calcite/adapter/os/MemoryInfoTableFunction.java | 2 +- .../org/apache/calcite/adapter/os/MountsTableFunction.java | 2 +- .../apache/calcite/adapter/os/OsVersionTableFunction.java | 2 +- .../java/org/apache/calcite/adapter/os/PsTableFunction.java | 2 +- .../org/apache/calcite/adapter/os/StdinTableFunction.java | 2 +- .../apache/calcite/adapter/os/SystemInfoTableFunction.java | 2 +- .../org/apache/calcite/adapter/os/VmstatTableFunction.java | 2 +- .../java/org/apache/calcite/adapter/tpcds/TpcdsSchema.java | 2 +- .../apache/calcite/chinook/PreferredAlbumsTableFactory.java | 2 +- .../apache/calcite/chinook/PreferredGenresTableFactory.java | 2 +- .../java/org/apache/calcite/adapter/tpcds/TpcdsTest.java | 2 +- .../java/org/apache/calcite/adapter/redis/RedisSchema.java | 2 +- .../java/org/apache/calcite/adapter/redis/RedisTable.java | 2 +- .../org/apache/calcite/adapter/redis/RedisTableFactory.java | 2 +- .../org/apache/calcite/server/AbstractModifiableTable.java | 2 +- .../org/apache/calcite/server/MaterializedViewTable.java | 2 +- .../java/org/apache/calcite/server/MutableArrayTable.java | 2 +- .../java/org/apache/calcite/server/ServerDdlExecutor.java | 2 +- .../test/java/org/apache/calcite/test/ServerParserTest.java | 2 +- .../calcite/adapter/spark/EnumerableToSparkConverter.java | 2 +- .../java/org/apache/calcite/adapter/spark/HttpServer.java | 2 +- .../apache/calcite/adapter/spark/JdbcToSparkConverter.java | 2 +- .../java/org/apache/calcite/adapter/spark/SparkRules.java | 2 +- .../calcite/adapter/spark/SparkToEnumerableConverter.java | 2 +- .../org/apache/calcite/adapter/splunk/SplunkDriver.java | 2 +- .../java/org/apache/calcite/adapter/splunk/SplunkQuery.java | 2 +- .../java/org/apache/calcite/adapter/splunk/SplunkTable.java | 2 +- .../calcite/adapter/splunk/search/SplunkConnection.java | 2 +- .../calcite/adapter/splunk/search/SplunkConnectionImpl.java | 2 +- .../org/apache/calcite/sql/parser/SqlParserFixture.java | 2 +- .../org/apache/calcite/sql/parser/SqlParserListFixture.java | 2 +- .../java/org/apache/calcite/sql/parser/SqlParserTest.java | 2 +- .../java/org/apache/calcite/sql/test/AbstractSqlTester.java | 2 +- .../org/apache/calcite/sql/test/SqlOperatorFixture.java | 2 +- .../java/org/apache/calcite/sql/test/SqlTestFactory.java | 2 +- .../main/java/org/apache/calcite/sql/test/SqlTester.java | 2 +- .../src/main/java/org/apache/calcite/sql/test/SqlTests.java | 2 +- .../org/apache/calcite/test/AbstractModifiableTable.java | 2 +- .../main/java/org/apache/calcite/test/CalciteAssert.java | 2 +- .../main/java/org/apache/calcite/test/ConnectionSpec.java | 2 +- .../main/java/org/apache/calcite/test/DiffRepository.java | 2 +- .../src/main/java/org/apache/calcite/test/DiffTestCase.java | 2 +- .../main/java/org/apache/calcite/test/MockDdlExecutor.java | 2 +- .../java/org/apache/calcite/test/MockRelOptPlanner.java | 2 +- .../java/org/apache/calcite/test/MockSqlOperatorTable.java | 2 +- .../src/main/java/org/apache/calcite/test/QuidemTest.java | 2 +- .../calcite/test/ReflectiveSchemaWithoutRowCount.java | 2 +- .../main/java/org/apache/calcite/test/RelOptFixture.java | 2 +- .../org/apache/calcite/test/SqlOperatorFixtureImpl.java | 2 +- .../java/org/apache/calcite/test/SqlOperatorFixtures.java | 2 +- .../main/java/org/apache/calcite/test/SqlOperatorTest.java | 2 +- .../main/java/org/apache/calcite/test/SqlRuntimeTester.java | 2 +- .../main/java/org/apache/calcite/test/SqlToRelFixture.java | 2 +- .../org/apache/calcite/test/catalog/MockCatalogReader.java | 2 +- .../calcite/test/catalog/MockCatalogReaderDynamic.java | 2 +- .../calcite/test/catalog/MockCatalogReaderExtended.java | 2 +- .../calcite/test/catalog/MockCatalogReaderSimple.java | 2 +- .../calcite/test/schemata/bookstore/BookstoreSchema.java | 2 +- .../calcite/test/schemata/catchall/CatchallSchema.java | 2 +- .../test/schemata/countries/CountriesTableFunction.java | 2 +- .../test/schemata/countries/StatesTableFunction.java | 2 +- .../calcite/test/schemata/foodmart/FoodmartSchema.java | 2 +- .../org/apache/calcite/test/schemata/hr/Department.java | 2 +- .../org/apache/calcite/test/schemata/hr/DepartmentPlus.java | 2 +- .../java/org/apache/calcite/test/schemata/hr/Employee.java | 2 +- .../java/org/apache/calcite/test/schemata/hr/Event.java | 2 +- .../org/apache/calcite/test/schemata/hr/NullableTest.java | 2 +- .../test/schemata/orderstream/BaseOrderStreamTable.java | 2 +- .../orderstream/InfiniteOrdersStreamTableFactory.java | 2 +- .../test/schemata/orderstream/InfiniteOrdersTable.java | 2 +- .../test/schemata/orderstream/OrdersHistoryTable.java | 2 +- .../test/schemata/orderstream/OrdersStreamTableFactory.java | 2 +- .../calcite/test/schemata/orderstream/OrdersTable.java | 2 +- .../calcite/test/schemata/orderstream/ProductsTable.java | 2 +- .../test/schemata/orderstream/ProductsTableFactory.java | 2 +- .../test/schemata/orderstream/ProductsTemporalTable.java | 2 +- testkit/src/main/java/org/apache/calcite/util/Smalls.java | 2 +- 1184 files changed, 1197 insertions(+), 1197 deletions(-) diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowProject.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowProject.java index 8fee390d543f..959fb772f0eb 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowProject.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowProject.java @@ -30,7 +30,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowRel.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowRel.java index 944c17d867dd..a06e71270749 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowRel.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowRel.java @@ -21,7 +21,7 @@ import org.apache.calcite.rel.RelNode; import org.apache.calcite.util.ImmutableIntList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowRules.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowRules.java index 3da1527f1014..00d3cf25a9ba 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowRules.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowRules.java @@ -35,8 +35,8 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; import org.immutables.value.Value; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowSchema.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowSchema.java index 510adfb8428f..053080ac0b5d 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowSchema.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowSchema.java @@ -29,7 +29,7 @@ import com.google.common.base.Suppliers; import com.google.common.collect.ImmutableMap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTable.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTable.java index 2585f5d156ea..95ee7867a5b4 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTable.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTable.java @@ -44,7 +44,7 @@ import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.Schema; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.io.File; import java.io.FileInputStream; diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTranslator.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTranslator.java index 2b4229598153..25debb0ebf16 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTranslator.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTranslator.java @@ -29,7 +29,7 @@ import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.util.DateString; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.text.SimpleDateFormat; import java.util.ArrayList; diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ConditionToken.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ConditionToken.java index c5b690840add..57401f73139a 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ConditionToken.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ConditionToken.java @@ -18,7 +18,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/babel/src/main/java/org/apache/calcite/sql/babel/SqlBabelCreateTable.java b/babel/src/main/java/org/apache/calcite/sql/babel/SqlBabelCreateTable.java index 652f0a92e372..3a05fd9ed295 100644 --- a/babel/src/main/java/org/apache/calcite/sql/babel/SqlBabelCreateTable.java +++ b/babel/src/main/java/org/apache/calcite/sql/babel/SqlBabelCreateTable.java @@ -28,7 +28,7 @@ import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.util.ImmutableNullableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/babel/src/test/java/org/apache/calcite/test/BabelParserTest.java b/babel/src/test/java/org/apache/calcite/test/BabelParserTest.java index 48172f7d7fe4..c3c79517c055 100644 --- a/babel/src/test/java/org/apache/calcite/test/BabelParserTest.java +++ b/babel/src/test/java/org/apache/calcite/test/BabelParserTest.java @@ -34,7 +34,7 @@ import com.google.common.base.Throwables; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; diff --git a/babel/src/test/java/org/apache/calcite/test/BabelQuidemTest.java b/babel/src/test/java/org/apache/calcite/test/BabelQuidemTest.java index f826d01e4512..d4d6efb84f23 100644 --- a/babel/src/test/java/org/apache/calcite/test/BabelQuidemTest.java +++ b/babel/src/test/java/org/apache/calcite/test/BabelQuidemTest.java @@ -30,7 +30,7 @@ import net.hydromatic.quidem.CommandHandler; import net.hydromatic.quidem.Quidem; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.BeforeEach; import java.sql.Connection; diff --git a/babel/src/test/java/org/apache/calcite/test/package-info.java b/babel/src/test/java/org/apache/calcite/test/package-info.java index c06f789d71ca..f5580cb7a5f2 100644 --- a/babel/src/test/java/org/apache/calcite/test/package-info.java +++ b/babel/src/test/java/org/apache/calcite/test/package-info.java @@ -23,6 +23,6 @@ @DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.RETURN) package org.apache.calcite.test; -import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.framework.qual.DefaultQualifier; import org.checkerframework.framework.qual.TypeUseLocation; +import org.jspecify.annotations.NonNull; diff --git a/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraEnumerator.java b/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraEnumerator.java index 7d5d0baa04a4..f24ccd973a3c 100644 --- a/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraEnumerator.java +++ b/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraEnumerator.java @@ -29,7 +29,7 @@ import com.datastax.oss.driver.api.core.data.TupleValue; import com.datastax.oss.driver.api.core.type.codec.registry.CodecRegistry; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.nio.ByteBuffer; import java.time.Instant; diff --git a/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraFilter.java b/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraFilter.java index 8da8cff99186..378742e21dd8 100644 --- a/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraFilter.java +++ b/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraFilter.java @@ -39,7 +39,7 @@ import org.apache.calcite.util.TimestampWithTimeZoneString; import org.apache.calcite.util.Util; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.text.SimpleDateFormat; import java.util.ArrayList; diff --git a/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraLimit.java b/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraLimit.java index ee5f6aa77652..c9a92bed8133 100644 --- a/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraLimit.java +++ b/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraLimit.java @@ -27,7 +27,7 @@ import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraProject.java b/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraProject.java index 0ee42eec05ae..48197b014cfb 100644 --- a/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraProject.java +++ b/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraProject.java @@ -30,7 +30,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.LinkedHashMap; import java.util.List; diff --git a/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraRel.java b/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraRel.java index adb20bac9b6a..e80973c81d59 100644 --- a/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraRel.java +++ b/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraRel.java @@ -20,7 +20,7 @@ import org.apache.calcite.plan.RelOptTable; import org.apache.calcite.rel.RelNode; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.LinkedHashMap; diff --git a/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraRules.java b/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraRules.java index db5d83960d0c..08bd9d636442 100644 --- a/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraRules.java +++ b/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraRules.java @@ -40,8 +40,8 @@ import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.validate.SqlValidatorUtil; -import org.checkerframework.checker.nullness.qual.Nullable; import org.immutables.value.Value; +import org.jspecify.annotations.Nullable; import java.util.HashSet; import java.util.List; diff --git a/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraSort.java b/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraSort.java index d897cce52c00..109be8848ca3 100644 --- a/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraSort.java +++ b/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraSort.java @@ -28,7 +28,7 @@ import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.rex.RexNode; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraTable.java b/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraTable.java index c28ada76e964..bba5bdad07fe 100644 --- a/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraTable.java +++ b/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraTable.java @@ -42,7 +42,7 @@ import com.datastax.oss.driver.api.core.cql.ResultSet; import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Iterator; import java.util.List; diff --git a/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraTableScan.java b/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraTableScan.java index 25d29792433f..8a0785dff612 100644 --- a/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraTableScan.java +++ b/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraTableScan.java @@ -27,7 +27,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraToEnumerableConverter.java b/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraToEnumerableConverter.java index f99951a7a5a4..fc0a28a610e7 100644 --- a/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraToEnumerableConverter.java +++ b/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraToEnumerableConverter.java @@ -41,7 +41,7 @@ import org.apache.calcite.util.Pair; import org.apache.calcite.util.Util; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.AbstractList; import java.util.ArrayList; diff --git a/core/src/main/java/org/apache/calcite/DataContext.java b/core/src/main/java/org/apache/calcite/DataContext.java index eeca324268e4..82f0823ee424 100644 --- a/core/src/main/java/org/apache/calcite/DataContext.java +++ b/core/src/main/java/org/apache/calcite/DataContext.java @@ -26,7 +26,7 @@ import com.google.common.base.CaseFormat; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.io.InputStream; import java.io.OutputStream; diff --git a/core/src/main/java/org/apache/calcite/DataContexts.java b/core/src/main/java/org/apache/calcite/DataContexts.java index b7e7f3fea2b0..03d0316f711d 100644 --- a/core/src/main/java/org/apache/calcite/DataContexts.java +++ b/core/src/main/java/org/apache/calcite/DataContexts.java @@ -23,7 +23,7 @@ import com.google.common.collect.ImmutableMap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.io.Serializable; import java.util.Map; diff --git a/core/src/main/java/org/apache/calcite/adapter/clone/ArrayTable.java b/core/src/main/java/org/apache/calcite/adapter/clone/ArrayTable.java index fabbe6ea8d71..cfda64687a44 100644 --- a/core/src/main/java/org/apache/calcite/adapter/clone/ArrayTable.java +++ b/core/src/main/java/org/apache/calcite/adapter/clone/ArrayTable.java @@ -40,7 +40,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Array; import java.lang.reflect.Type; diff --git a/core/src/main/java/org/apache/calcite/adapter/clone/CloneSchema.java b/core/src/main/java/org/apache/calcite/adapter/clone/CloneSchema.java index 87cd8666ba98..fd7ced538dd4 100644 --- a/core/src/main/java/org/apache/calcite/adapter/clone/CloneSchema.java +++ b/core/src/main/java/org/apache/calcite/adapter/clone/CloneSchema.java @@ -39,7 +39,7 @@ import com.google.common.base.Suppliers; import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Type; import java.util.LinkedHashMap; diff --git a/core/src/main/java/org/apache/calcite/adapter/clone/ColumnLoader.java b/core/src/main/java/org/apache/calcite/adapter/clone/ColumnLoader.java index 3d78f3548ba1..8e7af9cec577 100644 --- a/core/src/main/java/org/apache/calcite/adapter/clone/ColumnLoader.java +++ b/core/src/main/java/org/apache/calcite/adapter/clone/ColumnLoader.java @@ -28,7 +28,7 @@ import org.apache.calcite.util.Util; import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Type; import java.sql.Date; diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/AggAddContext.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/AggAddContext.java index 68d1e1be47fc..00c8d2a9bd6f 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/AggAddContext.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/AggAddContext.java @@ -19,7 +19,7 @@ import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.rex.RexNode; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/AggResultContext.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/AggResultContext.java index 2c4b1b4bba4f..52e9c16d2d97 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/AggResultContext.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/AggResultContext.java @@ -19,7 +19,7 @@ import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.rel.core.AggregateCall; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Information for a call to diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java index fd60a29ae750..4c0dfafb7575 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java @@ -65,7 +65,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Method; import java.lang.reflect.Modifier; diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableAggregate.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableAggregate.java index 0429168a8fd5..a59a345c129a 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableAggregate.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableAggregate.java @@ -36,7 +36,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Type; import java.util.ArrayList; diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableAggregateBase.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableAggregateBase.java index ae2f939988aa..6718af296b34 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableAggregateBase.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableAggregateBase.java @@ -47,7 +47,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Type; import java.util.ArrayList; diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableAggregateRule.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableAggregateRule.java index 55474083886f..86a761686be9 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableAggregateRule.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableAggregateRule.java @@ -25,7 +25,7 @@ import org.apache.calcite.rel.core.AggregateCall; import org.apache.calcite.rel.logical.LogicalAggregate; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Rule to convert a {@link LogicalAggregate} to an {@link EnumerableAggregate}. diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableAsofJoin.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableAsofJoin.java index d7cb7b4cdb7e..d5489bcdcd0d 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableAsofJoin.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableAsofJoin.java @@ -44,7 +44,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableBatchNestedLoopJoin.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableBatchNestedLoopJoin.java index 11a3620c8c2e..e595126f200e 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableBatchNestedLoopJoin.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableBatchNestedLoopJoin.java @@ -42,7 +42,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Modifier; import java.lang.reflect.Type; diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableBindable.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableBindable.java index 666e8802d415..c2b4a28fb824 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableBindable.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableBindable.java @@ -35,7 +35,7 @@ import com.google.common.collect.ImmutableMap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableCalc.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableCalc.java index 03cd0e419a0d..564c524dd674 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableCalc.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableCalc.java @@ -51,7 +51,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Modifier; import java.lang.reflect.Type; diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableConditionalCorrelate.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableConditionalCorrelate.java index 02b51a624926..bd3ec415c594 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableConditionalCorrelate.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableConditionalCorrelate.java @@ -38,7 +38,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Modifier; import java.lang.reflect.Type; diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableCorrelate.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableCorrelate.java index e5adff42d9de..2d6004057aa8 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableCorrelate.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableCorrelate.java @@ -37,7 +37,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Modifier; import java.lang.reflect.Type; diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableFilter.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableFilter.java index f4943ce8917c..46221dbf6c66 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableFilter.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableFilter.java @@ -32,7 +32,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableHashJoin.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableHashJoin.java index 6fc9a9ba3d07..3ed56bfcb72c 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableHashJoin.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableHashJoin.java @@ -42,7 +42,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Method; import java.util.ArrayList; diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableInterpretable.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableInterpretable.java index 7ed858b754b5..df9bd78b2ea5 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableInterpretable.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableInterpretable.java @@ -47,11 +47,11 @@ import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; -import org.checkerframework.checker.nullness.qual.Nullable; import org.codehaus.commons.compiler.CompileException; import org.codehaus.commons.compiler.CompilerFactoryFactory; import org.codehaus.commons.compiler.ICompilerFactory; import org.codehaus.commons.compiler.ISimpleCompiler; +import org.jspecify.annotations.Nullable; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Modifier; diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableInterpreter.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableInterpreter.java index ae2c88b3e93a..4262cc25c548 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableInterpreter.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableInterpreter.java @@ -30,7 +30,7 @@ import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.util.BuiltInMethod; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimit.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimit.java index 96a04be1b862..90d45d051501 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimit.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimit.java @@ -35,7 +35,7 @@ import org.apache.calcite.rex.RexNode; import org.apache.calcite.util.BuiltInMethod; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimitSort.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimitSort.java index 68759436d0fd..6abe360c64ed 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimitSort.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimitSort.java @@ -28,7 +28,7 @@ import org.apache.calcite.util.BuiltInMethod; import org.apache.calcite.util.Pair; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.math.BigDecimal; diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMatch.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMatch.java index 77d86cf5d729..1287d088d946 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMatch.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMatch.java @@ -47,7 +47,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Modifier; import java.lang.reflect.Type; diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeJoin.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeJoin.java index fe65699e0c12..e4222961fbf6 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeJoin.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeJoin.java @@ -54,7 +54,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Type; import java.util.ArrayList; diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeJoinRule.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeJoinRule.java index f78423edbde5..5078caa7c08b 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeJoinRule.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeJoinRule.java @@ -33,7 +33,7 @@ import org.apache.calcite.rex.RexUtil; import org.apache.calcite.sql.fun.SqlStdOperatorTable; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.Arrays; diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableNestedLoopJoin.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableNestedLoopJoin.java index 545329726fb0..27f9c2925220 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableNestedLoopJoin.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableNestedLoopJoin.java @@ -39,7 +39,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; import java.util.Set; diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableProject.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableProject.java index 8b4c0c5ded53..04f13314ce47 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableProject.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableProject.java @@ -31,7 +31,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableRel.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableRel.java index 2136b1f727ca..75ddf9ef53a8 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableRel.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableRel.java @@ -22,7 +22,7 @@ import org.apache.calcite.rel.PhysicalNode; import org.apache.calcite.util.Pair; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableRelFactories.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableRelFactories.java index 38adfd50a118..43501a38a6cf 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableRelFactories.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableRelFactories.java @@ -26,7 +26,7 @@ import org.apache.calcite.rex.RexUtil; import org.apache.calcite.sql.validate.SqlValidatorUtil; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; import java.util.Set; diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableRepeatUnion.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableRepeatUnion.java index 2cee0be2c46b..5d40346879e7 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableRepeatUnion.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableRepeatUnion.java @@ -30,7 +30,7 @@ import org.apache.calcite.util.BuiltInMethod; import org.apache.calcite.util.Util; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableSort.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableSort.java index be23b6602e89..36ea8865298b 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableSort.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableSort.java @@ -28,7 +28,7 @@ import org.apache.calcite.util.BuiltInMethod; import org.apache.calcite.util.Pair; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** Implementation of {@link org.apache.calcite.rel.core.Sort} in * {@link org.apache.calcite.adapter.enumerable.EnumerableConvention enumerable calling convention}. */ diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableSortRule.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableSortRule.java index 14a9c8671a01..d8179e524cf8 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableSortRule.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableSortRule.java @@ -21,7 +21,7 @@ import org.apache.calcite.rel.convert.ConverterRule; import org.apache.calcite.rel.core.Sort; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Rule to convert an {@link org.apache.calcite.rel.core.Sort} to an diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableSortedAggregate.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableSortedAggregate.java index 29997dd3b720..f28cf3917039 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableSortedAggregate.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableSortedAggregate.java @@ -42,7 +42,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Type; import java.util.ArrayList; diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableSortedAggregateRule.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableSortedAggregateRule.java index 30f76e28def9..cbc851d4b4b5 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableSortedAggregateRule.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableSortedAggregateRule.java @@ -25,7 +25,7 @@ import org.apache.calcite.rel.logical.LogicalAggregate; import org.apache.calcite.util.ImmutableIntList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Rule to convert a {@link LogicalAggregate} to an {@link EnumerableSortedAggregate}. diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableTableFunctionScan.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableTableFunctionScan.java index c9a74e8ae6cd..536b4f99b4e9 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableTableFunctionScan.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableTableFunctionScan.java @@ -36,7 +36,7 @@ import org.apache.calcite.sql.validate.SqlConformanceEnum; import org.apache.calcite.sql.validate.SqlUserDefinedTableFunction; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Method; import java.lang.reflect.Type; diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableTableModify.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableTableModify.java index 3f60240fd994..49528837b52b 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableTableModify.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableTableModify.java @@ -36,7 +36,7 @@ import org.apache.calcite.schema.ModifiableTable; import org.apache.calcite.util.BuiltInMethod; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Modifier; import java.util.ArrayDeque; diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableTableModifyRule.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableTableModifyRule.java index 4e111fe2248a..28fa65fa996a 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableTableModifyRule.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableTableModifyRule.java @@ -24,7 +24,7 @@ import org.apache.calcite.rel.logical.LogicalTableModify; import org.apache.calcite.schema.ModifiableTable; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** Planner rule that converts a {@link LogicalTableModify} to an {@link EnumerableTableModify}. * You may provide a custom config to convert other nodes that extend {@link TableModify}. diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableTableScan.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableTableScan.java index ace76188b135..bba5984a4c14 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableTableScan.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableTableScan.java @@ -51,7 +51,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Type; import java.util.ArrayList; diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableTableScanRule.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableTableScanRule.java index c89e803c2c28..c99533da77e7 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableTableScanRule.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableTableScanRule.java @@ -25,7 +25,7 @@ import org.apache.calcite.schema.QueryableTable; import org.apache.calcite.schema.Table; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** Planner rule that converts a {@link LogicalTableScan} to an {@link EnumerableTableScan}. * You may provide a custom config to convert other nodes that extend {@link TableScan}. diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableTraitsUtils.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableTraitsUtils.java index cc16ef98ed93..36072e91d25c 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableTraitsUtils.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableTraitsUtils.java @@ -39,7 +39,7 @@ import com.google.common.collect.ImmutableList; import org.apiguardian.api.API; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableValues.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableValues.java index c732d14319dd..9ca1e64b0479 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableValues.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableValues.java @@ -42,7 +42,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Ordering; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Type; import java.util.ArrayList; diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java index e57d3dd9e8c3..b07ccd55b426 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java @@ -57,7 +57,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Modifier; import java.lang.reflect.Type; diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/JavaRowFormat.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/JavaRowFormat.java index cfbb2f7e4f0a..ea8802a003e1 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/JavaRowFormat.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/JavaRowFormat.java @@ -32,7 +32,7 @@ import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.util.BuiltInMethod; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Type; import java.util.ArrayList; diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/PhysType.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/PhysType.java index c50d6237dfe5..7c9e5bcca7eb 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/PhysType.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/PhysType.java @@ -24,7 +24,7 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.util.Pair; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Type; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/PhysTypeImpl.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/PhysTypeImpl.java index 2efbb25d22bb..bcf4b6e078c9 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/PhysTypeImpl.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/PhysTypeImpl.java @@ -40,7 +40,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Modifier; import java.lang.reflect.Type; diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java index 6ef0182b301c..65997cdaebde 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java @@ -87,11 +87,11 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; -import org.checkerframework.checker.nullness.qual.Nullable; import org.joou.UByte; import org.joou.UInteger; import org.joou.ULong; import org.joou.UShort; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImplementorTable.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImplementorTable.java index 9c135bf83a09..b33e35338ce9 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImplementorTable.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImplementorTable.java @@ -22,7 +22,7 @@ import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.SqlWindowTableFunction; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Provides the implementor that generates code for calls to an operator. diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImplementorTables.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImplementorTables.java index d5ff234a0e19..1613c04f9e9f 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImplementorTables.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImplementorTables.java @@ -25,7 +25,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java index 35373aca3e39..37c4978d94e3 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java @@ -76,7 +76,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.locationtech.jts.geom.Geometry; import java.lang.reflect.Constructor; diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/impl/AggResultContextImpl.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/impl/AggResultContextImpl.java index 17a6843ef459..db65d3e1ce05 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/impl/AggResultContextImpl.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/impl/AggResultContextImpl.java @@ -25,7 +25,7 @@ import org.apache.calcite.rel.core.AggregateCall; import org.apache.calcite.sql.validate.SqlConformanceEnum; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/adapter/java/ReflectiveSchema.java b/core/src/main/java/org/apache/calcite/adapter/java/ReflectiveSchema.java index 07e29b2b2233..190647b25a97 100644 --- a/core/src/main/java/org/apache/calcite/adapter/java/ReflectiveSchema.java +++ b/core/src/main/java/org/apache/calcite/adapter/java/ReflectiveSchema.java @@ -54,7 +54,7 @@ import com.google.common.collect.Multimap; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; diff --git a/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcBaseSchema.java b/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcBaseSchema.java index 6c1e328d5106..cc0e97982866 100644 --- a/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcBaseSchema.java +++ b/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcBaseSchema.java @@ -27,7 +27,7 @@ import org.apache.calcite.schema.lookup.LikePattern; import org.apache.calcite.schema.lookup.Lookup; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Collection; import java.util.Collections; diff --git a/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcCatalogSchema.java b/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcCatalogSchema.java index 54330d327fac..c276e67145bb 100644 --- a/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcCatalogSchema.java +++ b/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcCatalogSchema.java @@ -36,7 +36,7 @@ import com.google.common.base.Suppliers; import com.google.common.collect.ImmutableSet; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.sql.Connection; import java.sql.ResultSet; diff --git a/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcCorrelationDataContext.java b/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcCorrelationDataContext.java index 802e490d83fc..8d3555e3c53a 100644 --- a/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcCorrelationDataContext.java +++ b/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcCorrelationDataContext.java @@ -21,7 +21,7 @@ import org.apache.calcite.linq4j.QueryProvider; import org.apache.calcite.schema.SchemaPlus; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import static java.lang.Integer.parseInt; diff --git a/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcRules.java b/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcRules.java index 9a35cf7cce2d..a4f5a0d17e7b 100644 --- a/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcRules.java +++ b/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcRules.java @@ -73,7 +73,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.slf4j.Logger; import java.util.ArrayList; diff --git a/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcSchema.java b/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcSchema.java index 7d1cb8864afa..533741b00d6c 100644 --- a/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcSchema.java +++ b/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcSchema.java @@ -48,7 +48,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Ordering; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcTable.java b/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcTable.java index 8d14f95e0e8b..0b3bff721ed2 100644 --- a/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcTable.java +++ b/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcTable.java @@ -56,7 +56,7 @@ import com.google.common.base.Suppliers; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.sql.SQLException; import java.util.ArrayList; diff --git a/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcToEnumerableConverter.java b/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcToEnumerableConverter.java index da2380077f14..d4b4dac008b4 100644 --- a/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcToEnumerableConverter.java +++ b/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcToEnumerableConverter.java @@ -50,7 +50,7 @@ import org.apache.calcite.sql.util.SqlString; import org.apache.calcite.util.BuiltInMethod; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Method; import java.lang.reflect.Modifier; diff --git a/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcToEnumerableConverterRule.java b/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcToEnumerableConverterRule.java index d676f9e1550a..d60c88c79f66 100644 --- a/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcToEnumerableConverterRule.java +++ b/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcToEnumerableConverterRule.java @@ -21,7 +21,7 @@ import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.convert.ConverterRule; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Rule to convert a relational expression from diff --git a/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcUtils.java b/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcUtils.java index ef48b505b51c..5243ca6da207 100644 --- a/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcUtils.java +++ b/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcUtils.java @@ -33,7 +33,7 @@ import com.google.common.cache.LoadingCache; import com.google.common.primitives.Ints; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.sql.Connection; import java.sql.DatabaseMetaData; diff --git a/core/src/main/java/org/apache/calcite/adapter/package-info.java b/core/src/main/java/org/apache/calcite/adapter/package-info.java index daf4a7d886f9..b9faaec327f5 100644 --- a/core/src/main/java/org/apache/calcite/adapter/package-info.java +++ b/core/src/main/java/org/apache/calcite/adapter/package-info.java @@ -42,6 +42,6 @@ @DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.RETURN) package org.apache.calcite.adapter; -import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.framework.qual.DefaultQualifier; import org.checkerframework.framework.qual.TypeUseLocation; +import org.jspecify.annotations.NonNull; diff --git a/core/src/main/java/org/apache/calcite/config/CalciteConnectionConfig.java b/core/src/main/java/org/apache/calcite/config/CalciteConnectionConfig.java index 52c8d4cd5663..4f3770386554 100644 --- a/core/src/main/java/org/apache/calcite/config/CalciteConnectionConfig.java +++ b/core/src/main/java/org/apache/calcite/config/CalciteConnectionConfig.java @@ -22,8 +22,8 @@ import org.apache.calcite.model.JsonSchema; import org.apache.calcite.sql.validate.SqlConformance; -import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.checker.nullness.qual.PolyNull; +import org.jspecify.annotations.Nullable; import java.util.Properties; diff --git a/core/src/main/java/org/apache/calcite/config/CalciteConnectionConfigImpl.java b/core/src/main/java/org/apache/calcite/config/CalciteConnectionConfigImpl.java index 221259cef959..05b9329edbb1 100644 --- a/core/src/main/java/org/apache/calcite/config/CalciteConnectionConfigImpl.java +++ b/core/src/main/java/org/apache/calcite/config/CalciteConnectionConfigImpl.java @@ -27,8 +27,8 @@ import org.apache.calcite.sql.validate.SqlConformance; import org.apache.calcite.sql.validate.SqlConformanceEnum; -import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.checker.nullness.qual.PolyNull; +import org.jspecify.annotations.Nullable; import java.util.List; import java.util.Properties; diff --git a/core/src/main/java/org/apache/calcite/config/CalciteConnectionProperty.java b/core/src/main/java/org/apache/calcite/config/CalciteConnectionProperty.java index 32079be2cfbc..6f9a3d032981 100644 --- a/core/src/main/java/org/apache/calcite/config/CalciteConnectionProperty.java +++ b/core/src/main/java/org/apache/calcite/config/CalciteConnectionProperty.java @@ -22,7 +22,7 @@ import org.apache.calcite.model.JsonSchema; import org.apache.calcite.sql.validate.SqlConformanceEnum; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.HashMap; import java.util.Locale; diff --git a/core/src/main/java/org/apache/calcite/config/CalciteSystemProperty.java b/core/src/main/java/org/apache/calcite/config/CalciteSystemProperty.java index 0d14fed45b84..a5ef2b7b4f2f 100644 --- a/core/src/main/java/org/apache/calcite/config/CalciteSystemProperty.java +++ b/core/src/main/java/org/apache/calcite/config/CalciteSystemProperty.java @@ -18,7 +18,7 @@ import com.google.common.collect.ImmutableSet; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.io.File; import java.io.IOException; diff --git a/core/src/main/java/org/apache/calcite/config/package-info.java b/core/src/main/java/org/apache/calcite/config/package-info.java index 7cca08157a6a..167617c0b88c 100644 --- a/core/src/main/java/org/apache/calcite/config/package-info.java +++ b/core/src/main/java/org/apache/calcite/config/package-info.java @@ -23,6 +23,6 @@ @DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.RETURN) package org.apache.calcite.config; -import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.framework.qual.DefaultQualifier; import org.checkerframework.framework.qual.TypeUseLocation; +import org.jspecify.annotations.NonNull; diff --git a/core/src/main/java/org/apache/calcite/interpreter/AggregateNode.java b/core/src/main/java/org/apache/calcite/interpreter/AggregateNode.java index 684baad8d493..593c853a011c 100644 --- a/core/src/main/java/org/apache/calcite/interpreter/AggregateNode.java +++ b/core/src/main/java/org/apache/calcite/interpreter/AggregateNode.java @@ -49,7 +49,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; diff --git a/core/src/main/java/org/apache/calcite/interpreter/BindableConvention.java b/core/src/main/java/org/apache/calcite/interpreter/BindableConvention.java index eea419d82f89..f36e4839562c 100644 --- a/core/src/main/java/org/apache/calcite/interpreter/BindableConvention.java +++ b/core/src/main/java/org/apache/calcite/interpreter/BindableConvention.java @@ -24,7 +24,7 @@ import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.RelNode; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Calling convention that returns results as an diff --git a/core/src/main/java/org/apache/calcite/interpreter/Bindables.java b/core/src/main/java/org/apache/calcite/interpreter/Bindables.java index f3a4d22ba12e..c303cc226c02 100644 --- a/core/src/main/java/org/apache/calcite/interpreter/Bindables.java +++ b/core/src/main/java/org/apache/calcite/interpreter/Bindables.java @@ -79,8 +79,8 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import org.checkerframework.checker.nullness.qual.Nullable; import org.immutables.value.Value; +import org.jspecify.annotations.Nullable; import java.util.List; import java.util.Map; diff --git a/core/src/main/java/org/apache/calcite/interpreter/CollectNode.java b/core/src/main/java/org/apache/calcite/interpreter/CollectNode.java index a72bae3fe58b..d57346f656d1 100644 --- a/core/src/main/java/org/apache/calcite/interpreter/CollectNode.java +++ b/core/src/main/java/org/apache/calcite/interpreter/CollectNode.java @@ -20,7 +20,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/interpreter/Compiler.java b/core/src/main/java/org/apache/calcite/interpreter/Compiler.java index 41a5d5d73766..da014955cbe1 100644 --- a/core/src/main/java/org/apache/calcite/interpreter/Compiler.java +++ b/core/src/main/java/org/apache/calcite/interpreter/Compiler.java @@ -22,7 +22,7 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexNode; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/interpreter/Context.java b/core/src/main/java/org/apache/calcite/interpreter/Context.java index 09a7efdf66f2..0225cd6bcf48 100644 --- a/core/src/main/java/org/apache/calcite/interpreter/Context.java +++ b/core/src/main/java/org/apache/calcite/interpreter/Context.java @@ -19,7 +19,7 @@ import org.apache.calcite.DataContext; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Context for executing a scalar expression in an interpreter. diff --git a/core/src/main/java/org/apache/calcite/interpreter/InterpretableConverter.java b/core/src/main/java/org/apache/calcite/interpreter/InterpretableConverter.java index 26e431828a3f..e89b8c062b02 100644 --- a/core/src/main/java/org/apache/calcite/interpreter/InterpretableConverter.java +++ b/core/src/main/java/org/apache/calcite/interpreter/InterpretableConverter.java @@ -25,7 +25,7 @@ import org.apache.calcite.rel.convert.ConverterImpl; import org.apache.calcite.runtime.ArrayBindable; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/interpreter/InterpretableRel.java b/core/src/main/java/org/apache/calcite/interpreter/InterpretableRel.java index e0e8dd745f5a..f546456a936a 100644 --- a/core/src/main/java/org/apache/calcite/interpreter/InterpretableRel.java +++ b/core/src/main/java/org/apache/calcite/interpreter/InterpretableRel.java @@ -20,7 +20,7 @@ import org.apache.calcite.jdbc.CalcitePrepare; import org.apache.calcite.rel.RelNode; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.HashMap; import java.util.LinkedHashMap; diff --git a/core/src/main/java/org/apache/calcite/interpreter/Interpreter.java b/core/src/main/java/org/apache/calcite/interpreter/Interpreter.java index 8bc82646a7ff..3fb48ef62fe8 100644 --- a/core/src/main/java/org/apache/calcite/interpreter/Interpreter.java +++ b/core/src/main/java/org/apache/calcite/interpreter/Interpreter.java @@ -51,7 +51,7 @@ import org.checkerframework.checker.initialization.qual.NotOnlyInitialized; import org.checkerframework.checker.initialization.qual.UnknownInitialization; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayDeque; import java.util.ArrayList; diff --git a/core/src/main/java/org/apache/calcite/interpreter/Interpreters.java b/core/src/main/java/org/apache/calcite/interpreter/Interpreters.java index 9bb42719d8a7..d2ef6c97fcf2 100644 --- a/core/src/main/java/org/apache/calcite/interpreter/Interpreters.java +++ b/core/src/main/java/org/apache/calcite/interpreter/Interpreters.java @@ -21,7 +21,7 @@ import org.apache.calcite.rel.RelNode; import org.apache.calcite.runtime.ArrayBindable; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Utilities relating to {@link org.apache.calcite.interpreter.Interpreter} diff --git a/core/src/main/java/org/apache/calcite/interpreter/JoinNode.java b/core/src/main/java/org/apache/calcite/interpreter/JoinNode.java index f3c5e483349d..a87f8e001ee0 100644 --- a/core/src/main/java/org/apache/calcite/interpreter/JoinNode.java +++ b/core/src/main/java/org/apache/calcite/interpreter/JoinNode.java @@ -21,7 +21,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.HashSet; diff --git a/core/src/main/java/org/apache/calcite/interpreter/NoneToBindableConverterRule.java b/core/src/main/java/org/apache/calcite/interpreter/NoneToBindableConverterRule.java index fd4198b05621..6fb1e25a5b38 100644 --- a/core/src/main/java/org/apache/calcite/interpreter/NoneToBindableConverterRule.java +++ b/core/src/main/java/org/apache/calcite/interpreter/NoneToBindableConverterRule.java @@ -21,7 +21,7 @@ import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.convert.ConverterRule; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Rule to convert a relational expression from diff --git a/core/src/main/java/org/apache/calcite/interpreter/Row.java b/core/src/main/java/org/apache/calcite/interpreter/Row.java index 073851d3b707..25d107c3396c 100644 --- a/core/src/main/java/org/apache/calcite/interpreter/Row.java +++ b/core/src/main/java/org/apache/calcite/interpreter/Row.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.interpreter; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Arrays; diff --git a/core/src/main/java/org/apache/calcite/interpreter/Scalar.java b/core/src/main/java/org/apache/calcite/interpreter/Scalar.java index a1beda01ef34..2f19cedd1e35 100644 --- a/core/src/main/java/org/apache/calcite/interpreter/Scalar.java +++ b/core/src/main/java/org/apache/calcite/interpreter/Scalar.java @@ -18,7 +18,7 @@ import org.apache.calcite.DataContext; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.function.Function; diff --git a/core/src/main/java/org/apache/calcite/interpreter/SortNode.java b/core/src/main/java/org/apache/calcite/interpreter/SortNode.java index 0f393a3e68d8..a49f9435cd41 100644 --- a/core/src/main/java/org/apache/calcite/interpreter/SortNode.java +++ b/core/src/main/java/org/apache/calcite/interpreter/SortNode.java @@ -28,7 +28,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Ordering; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.math.BigDecimal; import java.math.RoundingMode; diff --git a/core/src/main/java/org/apache/calcite/interpreter/Source.java b/core/src/main/java/org/apache/calcite/interpreter/Source.java index f6c49e398e5b..b6e91fc13dac 100644 --- a/core/src/main/java/org/apache/calcite/interpreter/Source.java +++ b/core/src/main/java/org/apache/calcite/interpreter/Source.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.interpreter; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Source of rows. diff --git a/core/src/main/java/org/apache/calcite/interpreter/TableFunctionScanNode.java b/core/src/main/java/org/apache/calcite/interpreter/TableFunctionScanNode.java index 749040e88ca6..255144a4c866 100644 --- a/core/src/main/java/org/apache/calcite/interpreter/TableFunctionScanNode.java +++ b/core/src/main/java/org/apache/calcite/interpreter/TableFunctionScanNode.java @@ -30,7 +30,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Interpreter node that implements a diff --git a/core/src/main/java/org/apache/calcite/interpreter/TableScanNode.java b/core/src/main/java/org/apache/calcite/interpreter/TableScanNode.java index 98ece3e24f26..b2fe5e487cdc 100644 --- a/core/src/main/java/org/apache/calcite/interpreter/TableScanNode.java +++ b/core/src/main/java/org/apache/calcite/interpreter/TableScanNode.java @@ -45,7 +45,7 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Lists; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Field; import java.lang.reflect.Type; diff --git a/core/src/main/java/org/apache/calcite/interpreter/package-info.java b/core/src/main/java/org/apache/calcite/interpreter/package-info.java index a4b0f3537a19..06de1b6bcdfb 100644 --- a/core/src/main/java/org/apache/calcite/interpreter/package-info.java +++ b/core/src/main/java/org/apache/calcite/interpreter/package-info.java @@ -27,6 +27,6 @@ @DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.RETURN) package org.apache.calcite.interpreter; -import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.framework.qual.DefaultQualifier; import org.checkerframework.framework.qual.TypeUseLocation; +import org.jspecify.annotations.NonNull; diff --git a/core/src/main/java/org/apache/calcite/jdbc/CachingCalciteSchema.java b/core/src/main/java/org/apache/calcite/jdbc/CachingCalciteSchema.java index 0350934f164e..bec2ff118294 100644 --- a/core/src/main/java/org/apache/calcite/jdbc/CachingCalciteSchema.java +++ b/core/src/main/java/org/apache/calcite/jdbc/CachingCalciteSchema.java @@ -32,7 +32,7 @@ import com.google.common.collect.ImmutableSortedMap; import com.google.common.collect.ImmutableSortedSet; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Collection; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/jdbc/CalciteConnection.java b/core/src/main/java/org/apache/calcite/jdbc/CalciteConnection.java index 285d96664174..20117b0c568f 100644 --- a/core/src/main/java/org/apache/calcite/jdbc/CalciteConnection.java +++ b/core/src/main/java/org/apache/calcite/jdbc/CalciteConnection.java @@ -22,7 +22,7 @@ import org.apache.calcite.linq4j.QueryProvider; import org.apache.calcite.schema.SchemaPlus; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.sql.Connection; import java.sql.SQLException; diff --git a/core/src/main/java/org/apache/calcite/jdbc/CalciteConnectionImpl.java b/core/src/main/java/org/apache/calcite/jdbc/CalciteConnectionImpl.java index ac2fbc127f51..b3d5a69a3f6c 100644 --- a/core/src/main/java/org/apache/calcite/jdbc/CalciteConnectionImpl.java +++ b/core/src/main/java/org/apache/calcite/jdbc/CalciteConnectionImpl.java @@ -72,7 +72,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Type; import java.sql.ResultSet; diff --git a/core/src/main/java/org/apache/calcite/jdbc/CalciteFactory.java b/core/src/main/java/org/apache/calcite/jdbc/CalciteFactory.java index 9100da4acc09..0e4fdcb143cb 100644 --- a/core/src/main/java/org/apache/calcite/jdbc/CalciteFactory.java +++ b/core/src/main/java/org/apache/calcite/jdbc/CalciteFactory.java @@ -21,7 +21,7 @@ import org.apache.calcite.avatica.AvaticaFactory; import org.apache.calcite.avatica.UnregisteredDriver; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Properties; diff --git a/core/src/main/java/org/apache/calcite/jdbc/CalciteJdbc41Factory.java b/core/src/main/java/org/apache/calcite/jdbc/CalciteJdbc41Factory.java index a4a2ee218228..04873007907f 100644 --- a/core/src/main/java/org/apache/calcite/jdbc/CalciteJdbc41Factory.java +++ b/core/src/main/java/org/apache/calcite/jdbc/CalciteJdbc41Factory.java @@ -27,7 +27,7 @@ import org.apache.calcite.avatica.QueryState; import org.apache.calcite.avatica.UnregisteredDriver; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.io.InputStream; import java.io.Reader; diff --git a/core/src/main/java/org/apache/calcite/jdbc/CalciteMetaColumnFactory.java b/core/src/main/java/org/apache/calcite/jdbc/CalciteMetaColumnFactory.java index 1f9d1a487b9e..9972729049b7 100644 --- a/core/src/main/java/org/apache/calcite/jdbc/CalciteMetaColumnFactory.java +++ b/core/src/main/java/org/apache/calcite/jdbc/CalciteMetaColumnFactory.java @@ -19,7 +19,7 @@ import org.apache.calcite.avatica.MetaImpl.MetaColumn; import org.apache.calcite.schema.Table; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/jdbc/CalciteMetaColumnFactoryImpl.java b/core/src/main/java/org/apache/calcite/jdbc/CalciteMetaColumnFactoryImpl.java index be320626cf12..877df6a7e34d 100644 --- a/core/src/main/java/org/apache/calcite/jdbc/CalciteMetaColumnFactoryImpl.java +++ b/core/src/main/java/org/apache/calcite/jdbc/CalciteMetaColumnFactoryImpl.java @@ -19,7 +19,7 @@ import org.apache.calcite.avatica.MetaImpl.MetaColumn; import org.apache.calcite.schema.Table; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** Default implementation of CalciteMetaColumnFactoryImpl. */ public class CalciteMetaColumnFactoryImpl diff --git a/core/src/main/java/org/apache/calcite/jdbc/CalciteMetaImpl.java b/core/src/main/java/org/apache/calcite/jdbc/CalciteMetaImpl.java index e294643fca2a..0c4d0df1c198 100644 --- a/core/src/main/java/org/apache/calcite/jdbc/CalciteMetaImpl.java +++ b/core/src/main/java/org/apache/calcite/jdbc/CalciteMetaImpl.java @@ -68,7 +68,7 @@ import com.google.common.primitives.Ints; import com.google.common.primitives.Longs; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Field; import java.sql.Connection; diff --git a/core/src/main/java/org/apache/calcite/jdbc/CalcitePrepare.java b/core/src/main/java/org/apache/calcite/jdbc/CalcitePrepare.java index ec37fd4c7a39..3c051292a7e3 100644 --- a/core/src/main/java/org/apache/calcite/jdbc/CalcitePrepare.java +++ b/core/src/main/java/org/apache/calcite/jdbc/CalcitePrepare.java @@ -50,7 +50,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; diff --git a/core/src/main/java/org/apache/calcite/jdbc/CalcitePreparedStatement.java b/core/src/main/java/org/apache/calcite/jdbc/CalcitePreparedStatement.java index 9b30ba7f8065..1c5f00e9304d 100644 --- a/core/src/main/java/org/apache/calcite/jdbc/CalcitePreparedStatement.java +++ b/core/src/main/java/org/apache/calcite/jdbc/CalcitePreparedStatement.java @@ -19,7 +19,7 @@ import org.apache.calcite.avatica.AvaticaPreparedStatement; import org.apache.calcite.avatica.Meta; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.sql.SQLException; diff --git a/core/src/main/java/org/apache/calcite/jdbc/CalciteSchema.java b/core/src/main/java/org/apache/calcite/jdbc/CalciteSchema.java index 53cffca3712e..0e69b4de05dd 100644 --- a/core/src/main/java/org/apache/calcite/jdbc/CalciteSchema.java +++ b/core/src/main/java/org/apache/calcite/jdbc/CalciteSchema.java @@ -42,7 +42,7 @@ import com.google.common.collect.ImmutableSortedMap; import com.google.common.collect.ImmutableSortedSet; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.Collection; diff --git a/core/src/main/java/org/apache/calcite/jdbc/CalciteStatement.java b/core/src/main/java/org/apache/calcite/jdbc/CalciteStatement.java index 9b06b7fc2a62..8267813ba1a4 100644 --- a/core/src/main/java/org/apache/calcite/jdbc/CalciteStatement.java +++ b/core/src/main/java/org/apache/calcite/jdbc/CalciteStatement.java @@ -22,7 +22,7 @@ import org.apache.calcite.linq4j.Queryable; import org.apache.calcite.server.CalciteServerStatement; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.sql.SQLException; diff --git a/core/src/main/java/org/apache/calcite/jdbc/Driver.java b/core/src/main/java/org/apache/calcite/jdbc/Driver.java index 5db1f15864f3..ce0a7b8434b1 100644 --- a/core/src/main/java/org/apache/calcite/jdbc/Driver.java +++ b/core/src/main/java/org/apache/calcite/jdbc/Driver.java @@ -36,7 +36,7 @@ import org.apache.calcite.util.JsonBuilder; import org.apache.calcite.util.Util; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.io.IOException; import java.sql.SQLException; diff --git a/core/src/main/java/org/apache/calcite/jdbc/JavaCollation.java b/core/src/main/java/org/apache/calcite/jdbc/JavaCollation.java index 421d6a840439..dede84d21c22 100644 --- a/core/src/main/java/org/apache/calcite/jdbc/JavaCollation.java +++ b/core/src/main/java/org/apache/calcite/jdbc/JavaCollation.java @@ -19,7 +19,7 @@ import org.apache.calcite.sql.SqlCollation; import org.checkerframework.checker.initialization.qual.UnderInitialization; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.nio.charset.Charset; import java.text.Collator; diff --git a/core/src/main/java/org/apache/calcite/jdbc/JavaRecordType.java b/core/src/main/java/org/apache/calcite/jdbc/JavaRecordType.java index 2183c2ac28bc..cd3d80e6ffe4 100644 --- a/core/src/main/java/org/apache/calcite/jdbc/JavaRecordType.java +++ b/core/src/main/java/org/apache/calcite/jdbc/JavaRecordType.java @@ -19,7 +19,7 @@ import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.rel.type.RelRecordType; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; import java.util.Objects; diff --git a/core/src/main/java/org/apache/calcite/jdbc/JavaTypeFactoryImpl.java b/core/src/main/java/org/apache/calcite/jdbc/JavaTypeFactoryImpl.java index 32aed69e3d6e..55d90c6b1b6f 100644 --- a/core/src/main/java/org/apache/calcite/jdbc/JavaTypeFactoryImpl.java +++ b/core/src/main/java/org/apache/calcite/jdbc/JavaTypeFactoryImpl.java @@ -37,11 +37,11 @@ import org.apache.calcite.util.Pair; import org.apache.calcite.util.Util; -import org.checkerframework.checker.nullness.qual.Nullable; import org.joou.UByte; import org.joou.UInteger; import org.joou.ULong; import org.joou.UShort; +import org.jspecify.annotations.Nullable; import org.locationtech.jts.geom.Geometry; import java.lang.reflect.Field; diff --git a/core/src/main/java/org/apache/calcite/jdbc/SimpleCalciteSchema.java b/core/src/main/java/org/apache/calcite/jdbc/SimpleCalciteSchema.java index 2d52edf60093..8885a68d1600 100644 --- a/core/src/main/java/org/apache/calcite/jdbc/SimpleCalciteSchema.java +++ b/core/src/main/java/org/apache/calcite/jdbc/SimpleCalciteSchema.java @@ -30,7 +30,7 @@ import com.google.common.collect.ImmutableSortedMap; import com.google.common.collect.ImmutableSortedSet; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Collection; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/jdbc/package-info.java b/core/src/main/java/org/apache/calcite/jdbc/package-info.java index 9a8221276703..5fa5d6969262 100644 --- a/core/src/main/java/org/apache/calcite/jdbc/package-info.java +++ b/core/src/main/java/org/apache/calcite/jdbc/package-info.java @@ -23,6 +23,6 @@ @DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.RETURN) package org.apache.calcite.jdbc; -import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.framework.qual.DefaultQualifier; import org.checkerframework.framework.qual.TypeUseLocation; +import org.jspecify.annotations.NonNull; diff --git a/core/src/main/java/org/apache/calcite/materialize/Lattice.java b/core/src/main/java/org/apache/calcite/materialize/Lattice.java index 720407315002..f6bc5f1032f5 100644 --- a/core/src/main/java/org/apache/calcite/materialize/Lattice.java +++ b/core/src/main/java/org/apache/calcite/materialize/Lattice.java @@ -67,8 +67,8 @@ import org.checkerframework.checker.initialization.qual.UnknownInitialization; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; -import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.checker.nullness.qual.RequiresNonNull; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.Arrays; diff --git a/core/src/main/java/org/apache/calcite/materialize/LatticeNode.java b/core/src/main/java/org/apache/calcite/materialize/LatticeNode.java index 6e8225db15fe..87c2fe816065 100644 --- a/core/src/main/java/org/apache/calcite/materialize/LatticeNode.java +++ b/core/src/main/java/org/apache/calcite/materialize/LatticeNode.java @@ -22,7 +22,7 @@ import com.google.common.collect.ImmutableList; import org.checkerframework.checker.initialization.qual.Initialized; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/materialize/LatticeSuggester.java b/core/src/main/java/org/apache/calcite/materialize/LatticeSuggester.java index 1edb9063c51d..3520772baa2f 100644 --- a/core/src/main/java/org/apache/calcite/materialize/LatticeSuggester.java +++ b/core/src/main/java/org/apache/calcite/materialize/LatticeSuggester.java @@ -55,7 +55,7 @@ import com.google.common.collect.LinkedListMultimap; import com.google.common.collect.Multimap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.Collection; diff --git a/core/src/main/java/org/apache/calcite/materialize/LatticeTable.java b/core/src/main/java/org/apache/calcite/materialize/LatticeTable.java index 5e5bdaeb7deb..daa132ef7a61 100644 --- a/core/src/main/java/org/apache/calcite/materialize/LatticeTable.java +++ b/core/src/main/java/org/apache/calcite/materialize/LatticeTable.java @@ -20,7 +20,7 @@ import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.util.Util; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import static java.util.Objects.requireNonNull; diff --git a/core/src/main/java/org/apache/calcite/materialize/MaterializationActor.java b/core/src/main/java/org/apache/calcite/materialize/MaterializationActor.java index 0475fc40aa1d..8ff7dd915615 100644 --- a/core/src/main/java/org/apache/calcite/materialize/MaterializationActor.java +++ b/core/src/main/java/org/apache/calcite/materialize/MaterializationActor.java @@ -22,7 +22,7 @@ import com.google.common.collect.HashMultimap; import com.google.common.collect.Multimap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.HashMap; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/materialize/MaterializationKey.java b/core/src/main/java/org/apache/calcite/materialize/MaterializationKey.java index 3661a7f45f49..7a206eae0135 100644 --- a/core/src/main/java/org/apache/calcite/materialize/MaterializationKey.java +++ b/core/src/main/java/org/apache/calcite/materialize/MaterializationKey.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.materialize; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.io.Serializable; import java.util.UUID; diff --git a/core/src/main/java/org/apache/calcite/materialize/MaterializationService.java b/core/src/main/java/org/apache/calcite/materialize/MaterializationService.java index 195f50fb9171..6ac184d177ee 100644 --- a/core/src/main/java/org/apache/calcite/materialize/MaterializationService.java +++ b/core/src/main/java/org/apache/calcite/materialize/MaterializationService.java @@ -41,7 +41,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Type; import java.util.ArrayList; diff --git a/core/src/main/java/org/apache/calcite/materialize/MutableNode.java b/core/src/main/java/org/apache/calcite/materialize/MutableNode.java index 278c946966d0..67bdfb851cd2 100644 --- a/core/src/main/java/org/apache/calcite/materialize/MutableNode.java +++ b/core/src/main/java/org/apache/calcite/materialize/MutableNode.java @@ -20,7 +20,7 @@ import com.google.common.collect.Ordering; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.Comparator; diff --git a/core/src/main/java/org/apache/calcite/materialize/Path.java b/core/src/main/java/org/apache/calcite/materialize/Path.java index 68ee4f5fc053..e02187639442 100644 --- a/core/src/main/java/org/apache/calcite/materialize/Path.java +++ b/core/src/main/java/org/apache/calcite/materialize/Path.java @@ -18,7 +18,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/materialize/SqlLatticeStatisticProvider.java b/core/src/main/java/org/apache/calcite/materialize/SqlLatticeStatisticProvider.java index bb42685dffa6..9a1239319ef1 100644 --- a/core/src/main/java/org/apache/calcite/materialize/SqlLatticeStatisticProvider.java +++ b/core/src/main/java/org/apache/calcite/materialize/SqlLatticeStatisticProvider.java @@ -25,7 +25,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/materialize/Step.java b/core/src/main/java/org/apache/calcite/materialize/Step.java index ecbabb32e20b..99c8105cd4a3 100644 --- a/core/src/main/java/org/apache/calcite/materialize/Step.java +++ b/core/src/main/java/org/apache/calcite/materialize/Step.java @@ -26,7 +26,7 @@ import org.checkerframework.checker.initialization.qual.NotOnlyInitialized; import org.checkerframework.checker.initialization.qual.UnderInitialization; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; import java.util.Objects; diff --git a/core/src/main/java/org/apache/calcite/materialize/TileKey.java b/core/src/main/java/org/apache/calcite/materialize/TileKey.java index 6a16ecb2244c..c1425d9b33ef 100644 --- a/core/src/main/java/org/apache/calcite/materialize/TileKey.java +++ b/core/src/main/java/org/apache/calcite/materialize/TileKey.java @@ -20,7 +20,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Objects; diff --git a/core/src/main/java/org/apache/calcite/materialize/TileSuggester.java b/core/src/main/java/org/apache/calcite/materialize/TileSuggester.java index e5c19de71987..5cad260b7b8c 100644 --- a/core/src/main/java/org/apache/calcite/materialize/TileSuggester.java +++ b/core/src/main/java/org/apache/calcite/materialize/TileSuggester.java @@ -21,7 +21,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.pentaho.aggdes.algorithm.Algorithm; import org.pentaho.aggdes.algorithm.Progress; import org.pentaho.aggdes.algorithm.Result; diff --git a/core/src/main/java/org/apache/calcite/materialize/package-info.java b/core/src/main/java/org/apache/calcite/materialize/package-info.java index 4939c315ce97..2bcd6aae5cea 100644 --- a/core/src/main/java/org/apache/calcite/materialize/package-info.java +++ b/core/src/main/java/org/apache/calcite/materialize/package-info.java @@ -37,6 +37,6 @@ @DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.RETURN) package org.apache.calcite.materialize; -import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.framework.qual.DefaultQualifier; import org.checkerframework.framework.qual.TypeUseLocation; +import org.jspecify.annotations.NonNull; diff --git a/core/src/main/java/org/apache/calcite/model/ClassNameFilter.java b/core/src/main/java/org/apache/calcite/model/ClassNameFilter.java index 0c37c2c5cbea..51dbc81f42e4 100644 --- a/core/src/main/java/org/apache/calcite/model/ClassNameFilter.java +++ b/core/src/main/java/org/apache/calcite/model/ClassNameFilter.java @@ -21,7 +21,7 @@ import com.google.common.collect.ImmutableList; import org.apiguardian.api.API; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; diff --git a/core/src/main/java/org/apache/calcite/model/JsonCustomSchema.java b/core/src/main/java/org/apache/calcite/model/JsonCustomSchema.java index dee32b6ec80b..793787972764 100644 --- a/core/src/main/java/org/apache/calcite/model/JsonCustomSchema.java +++ b/core/src/main/java/org/apache/calcite/model/JsonCustomSchema.java @@ -19,7 +19,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; import java.util.Map; diff --git a/core/src/main/java/org/apache/calcite/model/JsonCustomTable.java b/core/src/main/java/org/apache/calcite/model/JsonCustomTable.java index ef70381da75e..5ee555f28553 100644 --- a/core/src/main/java/org/apache/calcite/model/JsonCustomTable.java +++ b/core/src/main/java/org/apache/calcite/model/JsonCustomTable.java @@ -19,7 +19,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Map; diff --git a/core/src/main/java/org/apache/calcite/model/JsonFunction.java b/core/src/main/java/org/apache/calcite/model/JsonFunction.java index 0adf4f886961..2805e3a4ff81 100644 --- a/core/src/main/java/org/apache/calcite/model/JsonFunction.java +++ b/core/src/main/java/org/apache/calcite/model/JsonFunction.java @@ -19,7 +19,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/model/JsonJdbcSchema.java b/core/src/main/java/org/apache/calcite/model/JsonJdbcSchema.java index 99edce2b22e3..4dbe2d6b49ba 100644 --- a/core/src/main/java/org/apache/calcite/model/JsonJdbcSchema.java +++ b/core/src/main/java/org/apache/calcite/model/JsonJdbcSchema.java @@ -19,7 +19,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/model/JsonLattice.java b/core/src/main/java/org/apache/calcite/model/JsonLattice.java index 0f98143d8d20..f6666cea6ad6 100644 --- a/core/src/main/java/org/apache/calcite/model/JsonLattice.java +++ b/core/src/main/java/org/apache/calcite/model/JsonLattice.java @@ -20,7 +20,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/model/JsonMapSchema.java b/core/src/main/java/org/apache/calcite/model/JsonMapSchema.java index 98d06754ae1b..1b967c88f593 100644 --- a/core/src/main/java/org/apache/calcite/model/JsonMapSchema.java +++ b/core/src/main/java/org/apache/calcite/model/JsonMapSchema.java @@ -19,7 +19,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/model/JsonMaterialization.java b/core/src/main/java/org/apache/calcite/model/JsonMaterialization.java index 55969be47f5a..be74e71840b1 100644 --- a/core/src/main/java/org/apache/calcite/model/JsonMaterialization.java +++ b/core/src/main/java/org/apache/calcite/model/JsonMaterialization.java @@ -19,7 +19,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/model/JsonMeasure.java b/core/src/main/java/org/apache/calcite/model/JsonMeasure.java index 8485eb35f127..4705a3069a46 100644 --- a/core/src/main/java/org/apache/calcite/model/JsonMeasure.java +++ b/core/src/main/java/org/apache/calcite/model/JsonMeasure.java @@ -19,7 +19,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import static java.util.Objects.requireNonNull; diff --git a/core/src/main/java/org/apache/calcite/model/JsonRoot.java b/core/src/main/java/org/apache/calcite/model/JsonRoot.java index 0ca140e9cca6..9ba616a378c7 100644 --- a/core/src/main/java/org/apache/calcite/model/JsonRoot.java +++ b/core/src/main/java/org/apache/calcite/model/JsonRoot.java @@ -19,7 +19,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/model/JsonSchema.java b/core/src/main/java/org/apache/calcite/model/JsonSchema.java index bb820ebf43b7..cec7782e5e99 100644 --- a/core/src/main/java/org/apache/calcite/model/JsonSchema.java +++ b/core/src/main/java/org/apache/calcite/model/JsonSchema.java @@ -19,7 +19,7 @@ import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/model/JsonStream.java b/core/src/main/java/org/apache/calcite/model/JsonStream.java index 886fd9143d79..43dc42955fc2 100644 --- a/core/src/main/java/org/apache/calcite/model/JsonStream.java +++ b/core/src/main/java/org/apache/calcite/model/JsonStream.java @@ -19,7 +19,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Information about whether a table allows streaming. diff --git a/core/src/main/java/org/apache/calcite/model/JsonTable.java b/core/src/main/java/org/apache/calcite/model/JsonTable.java index f6d977b54c0a..fb09bb75ab51 100644 --- a/core/src/main/java/org/apache/calcite/model/JsonTable.java +++ b/core/src/main/java/org/apache/calcite/model/JsonTable.java @@ -19,7 +19,7 @@ import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/model/JsonTile.java b/core/src/main/java/org/apache/calcite/model/JsonTile.java index 33e087096f1d..86ed32b7e50f 100644 --- a/core/src/main/java/org/apache/calcite/model/JsonTile.java +++ b/core/src/main/java/org/apache/calcite/model/JsonTile.java @@ -20,7 +20,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/model/JsonType.java b/core/src/main/java/org/apache/calcite/model/JsonType.java index bc3f7fd30e45..3eda71fd85fc 100644 --- a/core/src/main/java/org/apache/calcite/model/JsonType.java +++ b/core/src/main/java/org/apache/calcite/model/JsonType.java @@ -19,7 +19,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/model/JsonView.java b/core/src/main/java/org/apache/calcite/model/JsonView.java index 9c84f430c0ad..87b9448fb5a2 100644 --- a/core/src/main/java/org/apache/calcite/model/JsonView.java +++ b/core/src/main/java/org/apache/calcite/model/JsonView.java @@ -19,7 +19,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/model/ModelHandler.java b/core/src/main/java/org/apache/calcite/model/ModelHandler.java index 46661065cc28..322cea082d4f 100644 --- a/core/src/main/java/org/apache/calcite/model/ModelHandler.java +++ b/core/src/main/java/org/apache/calcite/model/ModelHandler.java @@ -52,7 +52,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.io.File; import java.io.IOException; diff --git a/core/src/main/java/org/apache/calcite/model/package-info.java b/core/src/main/java/org/apache/calcite/model/package-info.java index fdedb4c3ab8a..0c2993c4e649 100644 --- a/core/src/main/java/org/apache/calcite/model/package-info.java +++ b/core/src/main/java/org/apache/calcite/model/package-info.java @@ -38,6 +38,6 @@ @DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.RETURN) package org.apache.calcite.model; -import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.framework.qual.DefaultQualifier; import org.checkerframework.framework.qual.TypeUseLocation; +import org.jspecify.annotations.NonNull; diff --git a/core/src/main/java/org/apache/calcite/plan/AbstractRelOptPlanner.java b/core/src/main/java/org/apache/calcite/plan/AbstractRelOptPlanner.java index 654f25689395..d37233963bd9 100644 --- a/core/src/main/java/org/apache/calcite/plan/AbstractRelOptPlanner.java +++ b/core/src/main/java/org/apache/calcite/plan/AbstractRelOptPlanner.java @@ -32,8 +32,8 @@ import org.checkerframework.checker.initialization.qual.UnknownInitialization; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; -import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.dataflow.qual.Pure; +import org.jspecify.annotations.Nullable; import org.slf4j.Logger; import java.text.NumberFormat; diff --git a/core/src/main/java/org/apache/calcite/plan/Contexts.java b/core/src/main/java/org/apache/calcite/plan/Contexts.java index 9ffc22c63ccb..732e291ec6a6 100644 --- a/core/src/main/java/org/apache/calcite/plan/Contexts.java +++ b/core/src/main/java/org/apache/calcite/plan/Contexts.java @@ -20,7 +20,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/plan/Convention.java b/core/src/main/java/org/apache/calcite/plan/Convention.java index 47bfe99e39b6..dec4d88d20f6 100644 --- a/core/src/main/java/org/apache/calcite/plan/Convention.java +++ b/core/src/main/java/org/apache/calcite/plan/Convention.java @@ -19,7 +19,7 @@ import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.RelFactories; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Calling convention trait. diff --git a/core/src/main/java/org/apache/calcite/plan/ConventionTraitDef.java b/core/src/main/java/org/apache/calcite/plan/ConventionTraitDef.java index a68f5a22bc19..5c14ba8aaa4d 100644 --- a/core/src/main/java/org/apache/calcite/plan/ConventionTraitDef.java +++ b/core/src/main/java/org/apache/calcite/plan/ConventionTraitDef.java @@ -32,7 +32,7 @@ import com.google.common.collect.Multimap; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/plan/RelCompositeTrait.java b/core/src/main/java/org/apache/calcite/plan/RelCompositeTrait.java index 1a0f840af0ce..a347867f6089 100644 --- a/core/src/main/java/org/apache/calcite/plan/RelCompositeTrait.java +++ b/core/src/main/java/org/apache/calcite/plan/RelCompositeTrait.java @@ -19,7 +19,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Ordering; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Arrays; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/plan/RelOptAbstractTable.java b/core/src/main/java/org/apache/calcite/plan/RelOptAbstractTable.java index efd9aec68ef0..f434246c4b2e 100644 --- a/core/src/main/java/org/apache/calcite/plan/RelOptAbstractTable.java +++ b/core/src/main/java/org/apache/calcite/plan/RelOptAbstractTable.java @@ -31,7 +31,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Collections; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/plan/RelOptCluster.java b/core/src/main/java/org/apache/calcite/plan/RelOptCluster.java index a5c9d3395ef2..aacdf8aa6a9a 100644 --- a/core/src/main/java/org/apache/calcite/plan/RelOptCluster.java +++ b/core/src/main/java/org/apache/calcite/plan/RelOptCluster.java @@ -31,7 +31,7 @@ import org.checkerframework.checker.initialization.qual.UnknownInitialization; import org.checkerframework.checker.nullness.qual.EnsuresNonNull; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.HashMap; import java.util.Map; diff --git a/core/src/main/java/org/apache/calcite/plan/RelOptCostImpl.java b/core/src/main/java/org/apache/calcite/plan/RelOptCostImpl.java index 66a1080ca0a7..102417f6e6ca 100644 --- a/core/src/main/java/org/apache/calcite/plan/RelOptCostImpl.java +++ b/core/src/main/java/org/apache/calcite/plan/RelOptCostImpl.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.plan; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * RelOptCostImpl provides a default implementation for the {@link RelOptCost} diff --git a/core/src/main/java/org/apache/calcite/plan/RelOptLattice.java b/core/src/main/java/org/apache/calcite/plan/RelOptLattice.java index e5d4988643f1..568093986ffe 100644 --- a/core/src/main/java/org/apache/calcite/plan/RelOptLattice.java +++ b/core/src/main/java/org/apache/calcite/plan/RelOptLattice.java @@ -25,7 +25,7 @@ import org.apache.calcite.util.ImmutableBitSet; import org.apache.calcite.util.Pair; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/plan/RelOptListener.java b/core/src/main/java/org/apache/calcite/plan/RelOptListener.java index 92c5e6226c7d..1ded48a840f0 100644 --- a/core/src/main/java/org/apache/calcite/plan/RelOptListener.java +++ b/core/src/main/java/org/apache/calcite/plan/RelOptListener.java @@ -18,7 +18,7 @@ import org.apache.calcite.rel.RelNode; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.EventListener; import java.util.EventObject; diff --git a/core/src/main/java/org/apache/calcite/plan/RelOptMaterialization.java b/core/src/main/java/org/apache/calcite/plan/RelOptMaterialization.java index 6cc5569bc731..d2f7de83221a 100644 --- a/core/src/main/java/org/apache/calcite/plan/RelOptMaterialization.java +++ b/core/src/main/java/org/apache/calcite/plan/RelOptMaterialization.java @@ -38,7 +38,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/plan/RelOptPlanner.java b/core/src/main/java/org/apache/calcite/plan/RelOptPlanner.java index 5341819ce9dd..ed795c8e7d32 100644 --- a/core/src/main/java/org/apache/calcite/plan/RelOptPlanner.java +++ b/core/src/main/java/org/apache/calcite/plan/RelOptPlanner.java @@ -25,7 +25,7 @@ import org.apache.calcite.util.CancelFlag; import org.apache.calcite.util.trace.CalciteTrace; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.slf4j.Logger; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/plan/RelOptPredicateList.java b/core/src/main/java/org/apache/calcite/plan/RelOptPredicateList.java index 30caf49e59aa..18455a5579e1 100644 --- a/core/src/main/java/org/apache/calcite/plan/RelOptPredicateList.java +++ b/core/src/main/java/org/apache/calcite/plan/RelOptPredicateList.java @@ -27,7 +27,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Collection; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/plan/RelOptQuery.java b/core/src/main/java/org/apache/calcite/plan/RelOptQuery.java index c617e3fb0cab..8778211a95c3 100644 --- a/core/src/main/java/org/apache/calcite/plan/RelOptQuery.java +++ b/core/src/main/java/org/apache/calcite/plan/RelOptQuery.java @@ -21,7 +21,7 @@ import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rex.RexBuilder; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.HashMap; import java.util.Map; diff --git a/core/src/main/java/org/apache/calcite/plan/RelOptRule.java b/core/src/main/java/org/apache/calcite/plan/RelOptRule.java index 8a6a002fb44b..5e6f72b8597c 100644 --- a/core/src/main/java/org/apache/calcite/plan/RelOptRule.java +++ b/core/src/main/java/org/apache/calcite/plan/RelOptRule.java @@ -27,7 +27,7 @@ import com.google.common.collect.Lists; import org.checkerframework.checker.initialization.qual.UnderInitialization; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/plan/RelOptRuleCall.java b/core/src/main/java/org/apache/calcite/plan/RelOptRuleCall.java index b39aad73e230..6d14ea9ded4a 100644 --- a/core/src/main/java/org/apache/calcite/plan/RelOptRuleCall.java +++ b/core/src/main/java/org/apache/calcite/plan/RelOptRuleCall.java @@ -26,7 +26,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.slf4j.Logger; import java.util.HashMap; diff --git a/core/src/main/java/org/apache/calcite/plan/RelOptRuleOperand.java b/core/src/main/java/org/apache/calcite/plan/RelOptRuleOperand.java index bfdd51abcfcb..a11b4b51f0a9 100644 --- a/core/src/main/java/org/apache/calcite/plan/RelOptRuleOperand.java +++ b/core/src/main/java/org/apache/calcite/plan/RelOptRuleOperand.java @@ -23,7 +23,7 @@ import org.checkerframework.checker.initialization.qual.NotOnlyInitialized; import org.checkerframework.checker.initialization.qual.UnknownInitialization; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; import java.util.Objects; diff --git a/core/src/main/java/org/apache/calcite/plan/RelOptSchema.java b/core/src/main/java/org/apache/calcite/plan/RelOptSchema.java index 75ff4dbd3b9c..6a1d18b95126 100644 --- a/core/src/main/java/org/apache/calcite/plan/RelOptSchema.java +++ b/core/src/main/java/org/apache/calcite/plan/RelOptSchema.java @@ -18,7 +18,7 @@ import org.apache.calcite.rel.type.RelDataTypeFactory; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/plan/RelOptSchemaWithSampling.java b/core/src/main/java/org/apache/calcite/plan/RelOptSchemaWithSampling.java index a4cc4e29b3b2..2b817142b6ec 100644 --- a/core/src/main/java/org/apache/calcite/plan/RelOptSchemaWithSampling.java +++ b/core/src/main/java/org/apache/calcite/plan/RelOptSchemaWithSampling.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.plan; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/plan/RelOptTable.java b/core/src/main/java/org/apache/calcite/plan/RelOptTable.java index 7fb90677c85a..9e1ee6bd4f3a 100644 --- a/core/src/main/java/org/apache/calcite/plan/RelOptTable.java +++ b/core/src/main/java/org/apache/calcite/plan/RelOptTable.java @@ -30,7 +30,7 @@ import org.apache.calcite.schema.Wrapper; import org.apache.calcite.util.ImmutableBitSet; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java b/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java index 42e86b5c4be5..e3b7e32784e6 100644 --- a/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java +++ b/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java @@ -116,8 +116,8 @@ import org.checkerframework.checker.initialization.qual.NotOnlyInitialized; import org.checkerframework.checker.initialization.qual.UnknownInitialization; -import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.checker.nullness.qual.PolyNull; +import org.jspecify.annotations.Nullable; import java.io.PrintWriter; import java.io.StringWriter; @@ -3584,7 +3584,7 @@ class JoinCounter extends RelVisitor { int joinCount; @Override public void visit(RelNode node, int ordinal, - @org.checkerframework.checker.nullness.qual.Nullable RelNode parent) { + @org.jspecify.annotations.Nullable RelNode parent) { if (node instanceof Join) { ++joinCount; } @@ -4541,7 +4541,7 @@ private static class VariableSetVisitor extends RelVisitor { @Override public void visit( RelNode p, int ordinal, - @org.checkerframework.checker.nullness.qual.Nullable RelNode parent) { + @org.jspecify.annotations.Nullable RelNode parent) { super.visit(p, ordinal, parent); p.collectVariablesUsed(variables); diff --git a/core/src/main/java/org/apache/calcite/plan/RelRule.java b/core/src/main/java/org/apache/calcite/plan/RelRule.java index 4e7f655d3746..4511c35d92ea 100644 --- a/core/src/main/java/org/apache/calcite/plan/RelRule.java +++ b/core/src/main/java/org/apache/calcite/plan/RelRule.java @@ -22,7 +22,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.immutables.value.Value; import java.util.ArrayList; diff --git a/core/src/main/java/org/apache/calcite/plan/RelTrait.java b/core/src/main/java/org/apache/calcite/plan/RelTrait.java index c0e29793e309..cee0661a901d 100644 --- a/core/src/main/java/org/apache/calcite/plan/RelTrait.java +++ b/core/src/main/java/org/apache/calcite/plan/RelTrait.java @@ -20,7 +20,7 @@ import org.apache.calcite.rel.core.Project; import org.apache.calcite.util.mapping.Mappings; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * RelTrait represents the manifestation of a relational expression trait within diff --git a/core/src/main/java/org/apache/calcite/plan/RelTraitDef.java b/core/src/main/java/org/apache/calcite/plan/RelTraitDef.java index a413787fb0e2..f722569aa939 100644 --- a/core/src/main/java/org/apache/calcite/plan/RelTraitDef.java +++ b/core/src/main/java/org/apache/calcite/plan/RelTraitDef.java @@ -22,7 +22,7 @@ import com.google.common.collect.Interner; import com.google.common.collect.Interners; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * RelTraitDef represents a class of {@link RelTrait}s. Implementations of diff --git a/core/src/main/java/org/apache/calcite/plan/RelTraitPropagationVisitor.java b/core/src/main/java/org/apache/calcite/plan/RelTraitPropagationVisitor.java index c2280ae85c30..3c8ea3a6e3c6 100644 --- a/core/src/main/java/org/apache/calcite/plan/RelTraitPropagationVisitor.java +++ b/core/src/main/java/org/apache/calcite/plan/RelTraitPropagationVisitor.java @@ -20,7 +20,7 @@ import org.apache.calcite.rel.RelVisitor; import org.apache.calcite.util.Util; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * RelTraitPropagationVisitor traverses a RelNode and its unregistered diff --git a/core/src/main/java/org/apache/calcite/plan/RelTraitSet.java b/core/src/main/java/org/apache/calcite/plan/RelTraitSet.java index fff7eea4f93b..607535fc6194 100644 --- a/core/src/main/java/org/apache/calcite/plan/RelTraitSet.java +++ b/core/src/main/java/org/apache/calcite/plan/RelTraitSet.java @@ -24,7 +24,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.AbstractList; import java.util.Arrays; diff --git a/core/src/main/java/org/apache/calcite/plan/RexImplicationChecker.java b/core/src/main/java/org/apache/calcite/plan/RexImplicationChecker.java index 1b0788062324..d243a6cabb15 100644 --- a/core/src/main/java/org/apache/calcite/plan/RexImplicationChecker.java +++ b/core/src/main/java/org/apache/calcite/plan/RexImplicationChecker.java @@ -37,7 +37,7 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.slf4j.LoggerFactory; import java.util.HashMap; diff --git a/core/src/main/java/org/apache/calcite/plan/SpoolRelOptTable.java b/core/src/main/java/org/apache/calcite/plan/SpoolRelOptTable.java index 0909b951c456..ae9ed7e950af 100644 --- a/core/src/main/java/org/apache/calcite/plan/SpoolRelOptTable.java +++ b/core/src/main/java/org/apache/calcite/plan/SpoolRelOptTable.java @@ -33,7 +33,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Collections; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/plan/SubstitutionVisitor.java b/core/src/main/java/org/apache/calcite/plan/SubstitutionVisitor.java index 5a76cb99b3e7..537902de6fb5 100644 --- a/core/src/main/java/org/apache/calcite/plan/SubstitutionVisitor.java +++ b/core/src/main/java/org/apache/calcite/plan/SubstitutionVisitor.java @@ -74,7 +74,7 @@ import com.google.common.collect.Multimap; import com.google.common.collect.Sets; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.Arrays; diff --git a/core/src/main/java/org/apache/calcite/plan/TableAccessMap.java b/core/src/main/java/org/apache/calcite/plan/TableAccessMap.java index 59dc10c9257c..8075b890b659 100644 --- a/core/src/main/java/org/apache/calcite/plan/TableAccessMap.java +++ b/core/src/main/java/org/apache/calcite/plan/TableAccessMap.java @@ -20,7 +20,7 @@ import org.apache.calcite.rel.RelVisitor; import org.apache.calcite.rel.core.TableModify; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Collections; import java.util.HashMap; diff --git a/core/src/main/java/org/apache/calcite/plan/ViewExpanders.java b/core/src/main/java/org/apache/calcite/plan/ViewExpanders.java index b3d96cd07fe5..a358ad892119 100644 --- a/core/src/main/java/org/apache/calcite/plan/ViewExpanders.java +++ b/core/src/main/java/org/apache/calcite/plan/ViewExpanders.java @@ -22,7 +22,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/plan/VisitorDataContext.java b/core/src/main/java/org/apache/calcite/plan/VisitorDataContext.java index 388a0748837b..a563218a2027 100644 --- a/core/src/main/java/org/apache/calcite/plan/VisitorDataContext.java +++ b/core/src/main/java/org/apache/calcite/plan/VisitorDataContext.java @@ -35,7 +35,7 @@ import org.apache.calcite.util.TimestampString; import org.apache.calcite.util.trace.CalciteLogger; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.slf4j.LoggerFactory; import java.math.BigDecimal; diff --git a/core/src/main/java/org/apache/calcite/plan/hep/HepInstruction.java b/core/src/main/java/org/apache/calcite/plan/hep/HepInstruction.java index 9d8edeb2dcc3..7bbfcfa48762 100644 --- a/core/src/main/java/org/apache/calcite/plan/hep/HepInstruction.java +++ b/core/src/main/java/org/apache/calcite/plan/hep/HepInstruction.java @@ -21,7 +21,7 @@ import com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Collection; import java.util.HashSet; diff --git a/core/src/main/java/org/apache/calcite/plan/hep/HepPlanner.java b/core/src/main/java/org/apache/calcite/plan/hep/HepPlanner.java index d2a39d4a43c9..df28ae14d36a 100644 --- a/core/src/main/java/org/apache/calcite/plan/hep/HepPlanner.java +++ b/core/src/main/java/org/apache/calcite/plan/hep/HepPlanner.java @@ -58,7 +58,7 @@ import com.google.common.collect.Multimap; import com.google.common.collect.Sets; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayDeque; import java.util.ArrayList; diff --git a/core/src/main/java/org/apache/calcite/plan/hep/HepProgram.java b/core/src/main/java/org/apache/calcite/plan/hep/HepProgram.java index 9b57abddcb7d..8972f36e2703 100644 --- a/core/src/main/java/org/apache/calcite/plan/hep/HepProgram.java +++ b/core/src/main/java/org/apache/calcite/plan/hep/HepProgram.java @@ -18,7 +18,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.HashMap; diff --git a/core/src/main/java/org/apache/calcite/plan/hep/HepRelMetadataProvider.java b/core/src/main/java/org/apache/calcite/plan/hep/HepRelMetadataProvider.java index 423d298906e8..90520fab860e 100644 --- a/core/src/main/java/org/apache/calcite/plan/hep/HepRelMetadataProvider.java +++ b/core/src/main/java/org/apache/calcite/plan/hep/HepRelMetadataProvider.java @@ -27,7 +27,7 @@ import com.google.common.collect.ImmutableMultimap; import com.google.common.collect.Multimap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Method; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/plan/hep/HepRelVertex.java b/core/src/main/java/org/apache/calcite/plan/hep/HepRelVertex.java index 887a507dd775..2155568ff5aa 100644 --- a/core/src/main/java/org/apache/calcite/plan/hep/HepRelVertex.java +++ b/core/src/main/java/org/apache/calcite/plan/hep/HepRelVertex.java @@ -26,7 +26,7 @@ import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.rel.type.RelDataType; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/plan/hep/HepRuleCall.java b/core/src/main/java/org/apache/calcite/plan/hep/HepRuleCall.java index bc2e3db8a806..2779fcb296ff 100644 --- a/core/src/main/java/org/apache/calcite/plan/hep/HepRuleCall.java +++ b/core/src/main/java/org/apache/calcite/plan/hep/HepRuleCall.java @@ -23,7 +23,7 @@ import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.rel.RelNode; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/plan/package-info.java b/core/src/main/java/org/apache/calcite/plan/package-info.java index 5cd8a60aeba2..a96143efe292 100644 --- a/core/src/main/java/org/apache/calcite/plan/package-info.java +++ b/core/src/main/java/org/apache/calcite/plan/package-info.java @@ -24,6 +24,6 @@ @DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.RETURN) package org.apache.calcite.plan; -import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.framework.qual.DefaultQualifier; import org.checkerframework.framework.qual.TypeUseLocation; +import org.jspecify.annotations.NonNull; diff --git a/core/src/main/java/org/apache/calcite/plan/visualizer/InputExcludedRelWriter.java b/core/src/main/java/org/apache/calcite/plan/visualizer/InputExcludedRelWriter.java index c3a0f894abd5..ba7c5c695d7a 100644 --- a/core/src/main/java/org/apache/calcite/plan/visualizer/InputExcludedRelWriter.java +++ b/core/src/main/java/org/apache/calcite/plan/visualizer/InputExcludedRelWriter.java @@ -21,7 +21,7 @@ import org.apache.calcite.sql.SqlExplainLevel; import org.apache.calcite.util.Pair; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.LinkedHashMap; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/plan/visualizer/NodeUpdateHelper.java b/core/src/main/java/org/apache/calcite/plan/visualizer/NodeUpdateHelper.java index 4b4e56d784ca..47991482f707 100644 --- a/core/src/main/java/org/apache/calcite/plan/visualizer/NodeUpdateHelper.java +++ b/core/src/main/java/org/apache/calcite/plan/visualizer/NodeUpdateHelper.java @@ -18,7 +18,7 @@ import org.apache.calcite.rel.RelNode; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Collections; import java.util.LinkedHashMap; diff --git a/core/src/main/java/org/apache/calcite/plan/visualizer/RuleMatchVisualizer.java b/core/src/main/java/org/apache/calcite/plan/visualizer/RuleMatchVisualizer.java index 57bf1a990683..61f04b9ff1bb 100644 --- a/core/src/main/java/org/apache/calcite/plan/visualizer/RuleMatchVisualizer.java +++ b/core/src/main/java/org/apache/calcite/plan/visualizer/RuleMatchVisualizer.java @@ -32,7 +32,7 @@ import com.fasterxml.jackson.core.util.Separators; import com.fasterxml.jackson.databind.ObjectMapper; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.io.IOException; import java.io.InputStream; diff --git a/core/src/main/java/org/apache/calcite/plan/volcano/AbstractConverter.java b/core/src/main/java/org/apache/calcite/plan/volcano/AbstractConverter.java index 23f85c888ec9..b4e2e774a7f0 100644 --- a/core/src/main/java/org/apache/calcite/plan/volcano/AbstractConverter.java +++ b/core/src/main/java/org/apache/calcite/plan/volcano/AbstractConverter.java @@ -30,8 +30,8 @@ import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.tools.RelBuilderFactory; -import org.checkerframework.checker.nullness.qual.Nullable; import org.immutables.value.Value; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/plan/volcano/Dumpers.java b/core/src/main/java/org/apache/calcite/plan/volcano/Dumpers.java index 44560dde7ec8..386c5b11f9e9 100644 --- a/core/src/main/java/org/apache/calcite/plan/volcano/Dumpers.java +++ b/core/src/main/java/org/apache/calcite/plan/volcano/Dumpers.java @@ -26,7 +26,7 @@ import com.google.common.collect.Ordering; import org.apiguardian.api.API; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.io.PrintWriter; import java.io.StringWriter; diff --git a/core/src/main/java/org/apache/calcite/plan/volcano/IterativeRuleQueue.java b/core/src/main/java/org/apache/calcite/plan/volcano/IterativeRuleQueue.java index 711f92e31931..9b07c64f7b17 100644 --- a/core/src/main/java/org/apache/calcite/plan/volcano/IterativeRuleQueue.java +++ b/core/src/main/java/org/apache/calcite/plan/volcano/IterativeRuleQueue.java @@ -23,7 +23,7 @@ import com.google.common.collect.HashMultimap; import com.google.common.collect.Multimap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.slf4j.Logger; import java.io.PrintWriter; diff --git a/core/src/main/java/org/apache/calcite/plan/volcano/RelSet.java b/core/src/main/java/org/apache/calcite/plan/volcano/RelSet.java index d715f37d27d4..58db11bbb9ba 100644 --- a/core/src/main/java/org/apache/calcite/plan/volcano/RelSet.java +++ b/core/src/main/java/org/apache/calcite/plan/volcano/RelSet.java @@ -33,7 +33,7 @@ import com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.slf4j.Logger; import java.util.ArrayList; diff --git a/core/src/main/java/org/apache/calcite/plan/volcano/RelSubset.java b/core/src/main/java/org/apache/calcite/plan/volcano/RelSubset.java index 8fa18de241c1..68609a5c7082 100644 --- a/core/src/main/java/org/apache/calcite/plan/volcano/RelSubset.java +++ b/core/src/main/java/org/apache/calcite/plan/volcano/RelSubset.java @@ -43,7 +43,7 @@ import org.apiguardian.api.API; import org.checkerframework.checker.initialization.qual.UnderInitialization; import org.checkerframework.checker.nullness.qual.EnsuresNonNull; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.slf4j.Logger; import java.io.PrintWriter; diff --git a/core/src/main/java/org/apache/calcite/plan/volcano/TopDownRuleDriver.java b/core/src/main/java/org/apache/calcite/plan/volcano/TopDownRuleDriver.java index 6940b0b1ae50..e4be6d4a2f5b 100644 --- a/core/src/main/java/org/apache/calcite/plan/volcano/TopDownRuleDriver.java +++ b/core/src/main/java/org/apache/calcite/plan/volcano/TopDownRuleDriver.java @@ -25,7 +25,7 @@ import org.apache.calcite.util.Pair; import org.apache.calcite.util.trace.CalciteTrace; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.slf4j.Logger; import java.util.ArrayDeque; diff --git a/core/src/main/java/org/apache/calcite/plan/volcano/TopDownRuleQueue.java b/core/src/main/java/org/apache/calcite/plan/volcano/TopDownRuleQueue.java index 4b92b381aebf..0f696cda14e4 100644 --- a/core/src/main/java/org/apache/calcite/plan/volcano/TopDownRuleQueue.java +++ b/core/src/main/java/org/apache/calcite/plan/volcano/TopDownRuleQueue.java @@ -19,7 +19,7 @@ import org.apache.calcite.rel.RelNode; import org.apache.calcite.util.Pair; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayDeque; import java.util.Deque; diff --git a/core/src/main/java/org/apache/calcite/plan/volcano/VolcanoCost.java b/core/src/main/java/org/apache/calcite/plan/volcano/VolcanoCost.java index dd4f0023528b..da98dc2526ca 100644 --- a/core/src/main/java/org/apache/calcite/plan/volcano/VolcanoCost.java +++ b/core/src/main/java/org/apache/calcite/plan/volcano/VolcanoCost.java @@ -20,7 +20,7 @@ import org.apache.calcite.plan.RelOptCostFactory; import org.apache.calcite.plan.RelOptUtil; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Objects; diff --git a/core/src/main/java/org/apache/calcite/plan/volcano/VolcanoPlanner.java b/core/src/main/java/org/apache/calcite/plan/volcano/VolcanoPlanner.java index 8f4cf379b41e..54cc9c46f271 100644 --- a/core/src/main/java/org/apache/calcite/plan/volcano/VolcanoPlanner.java +++ b/core/src/main/java/org/apache/calcite/plan/volcano/VolcanoPlanner.java @@ -63,10 +63,10 @@ import org.apiguardian.api.API; import org.checkerframework.checker.nullness.qual.EnsuresNonNull; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; -import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.checker.nullness.qual.PolyNull; import org.checkerframework.checker.nullness.qual.RequiresNonNull; import org.checkerframework.dataflow.qual.Pure; +import org.jspecify.annotations.Nullable; import java.io.PrintWriter; import java.io.StringWriter; diff --git a/core/src/main/java/org/apache/calcite/plan/volcano/VolcanoRelMetadataProvider.java b/core/src/main/java/org/apache/calcite/plan/volcano/VolcanoRelMetadataProvider.java index 266aec0a1442..9ff9361c4412 100644 --- a/core/src/main/java/org/apache/calcite/plan/volcano/VolcanoRelMetadataProvider.java +++ b/core/src/main/java/org/apache/calcite/plan/volcano/VolcanoRelMetadataProvider.java @@ -27,7 +27,7 @@ import com.google.common.collect.ImmutableMultimap; import com.google.common.collect.Multimap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Method; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/plan/volcano/VolcanoRuleCall.java b/core/src/main/java/org/apache/calcite/plan/volcano/VolcanoRuleCall.java index a4b84907126a..4c0f5b9c3a99 100644 --- a/core/src/main/java/org/apache/calcite/plan/volcano/VolcanoRuleCall.java +++ b/core/src/main/java/org/apache/calcite/plan/volcano/VolcanoRuleCall.java @@ -30,7 +30,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.Arrays; diff --git a/core/src/main/java/org/apache/calcite/prepare/CalciteCatalogReader.java b/core/src/main/java/org/apache/calcite/prepare/CalciteCatalogReader.java index df5e7ade813d..129a09985472 100644 --- a/core/src/main/java/org/apache/calcite/prepare/CalciteCatalogReader.java +++ b/core/src/main/java/org/apache/calcite/prepare/CalciteCatalogReader.java @@ -68,7 +68,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.Collection; diff --git a/core/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java b/core/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java index 06e27e361cff..f44178c8a6c8 100644 --- a/core/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java +++ b/core/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java @@ -121,7 +121,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Type; import java.math.BigDecimal; diff --git a/core/src/main/java/org/apache/calcite/prepare/PlannerImpl.java b/core/src/main/java/org/apache/calcite/prepare/PlannerImpl.java index 5fece8fa9770..046e77a4ee16 100644 --- a/core/src/main/java/org/apache/calcite/prepare/PlannerImpl.java +++ b/core/src/main/java/org/apache/calcite/prepare/PlannerImpl.java @@ -62,7 +62,7 @@ import com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.EnsuresNonNull; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.io.Reader; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/prepare/Prepare.java b/core/src/main/java/org/apache/calcite/prepare/Prepare.java index eac4fa6dd347..b3284a86f36a 100644 --- a/core/src/main/java/org/apache/calcite/prepare/Prepare.java +++ b/core/src/main/java/org/apache/calcite/prepare/Prepare.java @@ -70,7 +70,7 @@ import com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.slf4j.Logger; import java.lang.reflect.Type; diff --git a/core/src/main/java/org/apache/calcite/prepare/QueryableRelBuilder.java b/core/src/main/java/org/apache/calcite/prepare/QueryableRelBuilder.java index 1835555c7156..c2544c181b82 100644 --- a/core/src/main/java/org/apache/calcite/prepare/QueryableRelBuilder.java +++ b/core/src/main/java/org/apache/calcite/prepare/QueryableRelBuilder.java @@ -51,8 +51,8 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.checker.nullness.qual.PolyNull; +import org.jspecify.annotations.Nullable; import java.math.BigDecimal; import java.util.Comparator; diff --git a/core/src/main/java/org/apache/calcite/prepare/RelOptTableImpl.java b/core/src/main/java/org/apache/calcite/prepare/RelOptTableImpl.java index 8f1dc3e5f513..3a5a63120432 100644 --- a/core/src/main/java/org/apache/calcite/prepare/RelOptTableImpl.java +++ b/core/src/main/java/org/apache/calcite/prepare/RelOptTableImpl.java @@ -61,7 +61,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.AbstractList; import java.util.Collection; diff --git a/core/src/main/java/org/apache/calcite/prepare/package-info.java b/core/src/main/java/org/apache/calcite/prepare/package-info.java index 29062836cafd..98fb70f64c24 100644 --- a/core/src/main/java/org/apache/calcite/prepare/package-info.java +++ b/core/src/main/java/org/apache/calcite/prepare/package-info.java @@ -23,6 +23,6 @@ @DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.RETURN) package org.apache.calcite.prepare; -import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.framework.qual.DefaultQualifier; import org.checkerframework.framework.qual.TypeUseLocation; +import org.jspecify.annotations.NonNull; diff --git a/core/src/main/java/org/apache/calcite/profile/Profiler.java b/core/src/main/java/org/apache/calcite/profile/Profiler.java index 32f9f6b9b4e2..26d8e98f2f1a 100644 --- a/core/src/main/java/org/apache/calcite/profile/Profiler.java +++ b/core/src/main/java/org/apache/calcite/profile/Profiler.java @@ -25,7 +25,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSortedSet; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.math.BigDecimal; import java.math.MathContext; diff --git a/core/src/main/java/org/apache/calcite/profile/ProfilerImpl.java b/core/src/main/java/org/apache/calcite/profile/ProfilerImpl.java index b37a30680002..6a9337c6af73 100644 --- a/core/src/main/java/org/apache/calcite/profile/ProfilerImpl.java +++ b/core/src/main/java/org/apache/calcite/profile/ProfilerImpl.java @@ -33,7 +33,7 @@ import com.google.common.collect.Ordering; import com.yahoo.sketches.hll.HllSketch; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; diff --git a/core/src/main/java/org/apache/calcite/profile/SimpleProfiler.java b/core/src/main/java/org/apache/calcite/profile/SimpleProfiler.java index 2b4c413c5e31..117f93489f83 100644 --- a/core/src/main/java/org/apache/calcite/profile/SimpleProfiler.java +++ b/core/src/main/java/org/apache/calcite/profile/SimpleProfiler.java @@ -28,8 +28,8 @@ import com.google.common.collect.Iterables; import org.checkerframework.checker.initialization.qual.UnknownInitialization; -import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.checker.nullness.qual.RequiresNonNull; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.BitSet; diff --git a/core/src/main/java/org/apache/calcite/profile/package-info.java b/core/src/main/java/org/apache/calcite/profile/package-info.java index 8a708b7db24a..7d92e45e36d0 100644 --- a/core/src/main/java/org/apache/calcite/profile/package-info.java +++ b/core/src/main/java/org/apache/calcite/profile/package-info.java @@ -23,6 +23,6 @@ @DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.RETURN) package org.apache.calcite.profile; -import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.framework.qual.DefaultQualifier; import org.checkerframework.framework.qual.TypeUseLocation; +import org.jspecify.annotations.NonNull; diff --git a/core/src/main/java/org/apache/calcite/rel/AbstractRelNode.java b/core/src/main/java/org/apache/calcite/rel/AbstractRelNode.java index 0786798474d8..2dbc3ebf2dba 100644 --- a/core/src/main/java/org/apache/calcite/rel/AbstractRelNode.java +++ b/core/src/main/java/org/apache/calcite/rel/AbstractRelNode.java @@ -44,8 +44,8 @@ import org.apiguardian.api.API; import org.checkerframework.checker.initialization.qual.UnknownInitialization; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; -import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.dataflow.qual.Pure; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.Collections; diff --git a/core/src/main/java/org/apache/calcite/rel/PhysicalNode.java b/core/src/main/java/org/apache/calcite/rel/PhysicalNode.java index 50da278f27f5..4070a66bff43 100644 --- a/core/src/main/java/org/apache/calcite/rel/PhysicalNode.java +++ b/core/src/main/java/org/apache/calcite/rel/PhysicalNode.java @@ -23,7 +23,7 @@ import org.apache.calcite.rel.core.Sort; import org.apache.calcite.util.Pair; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/rel/RelCollationImpl.java b/core/src/main/java/org/apache/calcite/rel/RelCollationImpl.java index 19219f741a10..73abb7bce971 100644 --- a/core/src/main/java/org/apache/calcite/rel/RelCollationImpl.java +++ b/core/src/main/java/org/apache/calcite/rel/RelCollationImpl.java @@ -29,7 +29,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.UnmodifiableIterator; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Iterator; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/rel/RelCollationTraitDef.java b/core/src/main/java/org/apache/calcite/rel/RelCollationTraitDef.java index d60864362bc2..c27c9896eaa5 100644 --- a/core/src/main/java/org/apache/calcite/rel/RelCollationTraitDef.java +++ b/core/src/main/java/org/apache/calcite/rel/RelCollationTraitDef.java @@ -22,7 +22,7 @@ import org.apache.calcite.rel.core.Sort; import org.apache.calcite.rel.logical.LogicalSort; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Definition of the ordering trait. diff --git a/core/src/main/java/org/apache/calcite/rel/RelCommonExpressionBasicSuggester.java b/core/src/main/java/org/apache/calcite/rel/RelCommonExpressionBasicSuggester.java index 4c74237aeedc..67d311b473ba 100644 --- a/core/src/main/java/org/apache/calcite/rel/RelCommonExpressionBasicSuggester.java +++ b/core/src/main/java/org/apache/calcite/rel/RelCommonExpressionBasicSuggester.java @@ -25,7 +25,7 @@ import org.apache.calcite.rel.rules.CommonRelSubExprRegisterRule; import org.apiguardian.api.API; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Collection; import java.util.stream.Collectors; diff --git a/core/src/main/java/org/apache/calcite/rel/RelCommonExpressionSuggester.java b/core/src/main/java/org/apache/calcite/rel/RelCommonExpressionSuggester.java index 872a9554091e..f58b4adc7b12 100644 --- a/core/src/main/java/org/apache/calcite/rel/RelCommonExpressionSuggester.java +++ b/core/src/main/java/org/apache/calcite/rel/RelCommonExpressionSuggester.java @@ -19,7 +19,7 @@ import org.apache.calcite.plan.Context; import org.apiguardian.api.API; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Collection; diff --git a/core/src/main/java/org/apache/calcite/rel/RelDistributionTraitDef.java b/core/src/main/java/org/apache/calcite/rel/RelDistributionTraitDef.java index 5fc2ed5bf576..cb657eeead53 100644 --- a/core/src/main/java/org/apache/calcite/rel/RelDistributionTraitDef.java +++ b/core/src/main/java/org/apache/calcite/rel/RelDistributionTraitDef.java @@ -22,7 +22,7 @@ import org.apache.calcite.rel.core.Exchange; import org.apache.calcite.rel.logical.LogicalExchange; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Definition of the distribution trait. diff --git a/core/src/main/java/org/apache/calcite/rel/RelDistributions.java b/core/src/main/java/org/apache/calcite/rel/RelDistributions.java index 7917c9f58f6d..3a4c72b5550a 100644 --- a/core/src/main/java/org/apache/calcite/rel/RelDistributions.java +++ b/core/src/main/java/org/apache/calcite/rel/RelDistributions.java @@ -26,7 +26,7 @@ import com.google.common.collect.Ordering; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Collection; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/rel/RelFieldCollation.java b/core/src/main/java/org/apache/calcite/rel/RelFieldCollation.java index 985ae902caca..193601e791b7 100644 --- a/core/src/main/java/org/apache/calcite/rel/RelFieldCollation.java +++ b/core/src/main/java/org/apache/calcite/rel/RelFieldCollation.java @@ -18,7 +18,7 @@ import org.apache.calcite.sql.validate.SqlMonotonicity; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Objects; diff --git a/core/src/main/java/org/apache/calcite/rel/RelInput.java b/core/src/main/java/org/apache/calcite/rel/RelInput.java index c3d8e41b570c..bcfebc8ca9a2 100644 --- a/core/src/main/java/org/apache/calcite/rel/RelInput.java +++ b/core/src/main/java/org/apache/calcite/rel/RelInput.java @@ -27,7 +27,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.math.BigDecimal; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/rel/RelNode.java b/core/src/main/java/org/apache/calcite/rel/RelNode.java index 5eeece98223f..f5d1bd672a63 100644 --- a/core/src/main/java/org/apache/calcite/rel/RelNode.java +++ b/core/src/main/java/org/apache/calcite/rel/RelNode.java @@ -34,8 +34,8 @@ import org.apiguardian.api.API; import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf; -import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.dataflow.qual.Pure; +import org.jspecify.annotations.Nullable; import java.util.List; import java.util.Set; diff --git a/core/src/main/java/org/apache/calcite/rel/RelNodes.java b/core/src/main/java/org/apache/calcite/rel/RelNodes.java index 0565fa0292a4..865091140135 100644 --- a/core/src/main/java/org/apache/calcite/rel/RelNodes.java +++ b/core/src/main/java/org/apache/calcite/rel/RelNodes.java @@ -27,7 +27,7 @@ import com.google.common.collect.Ordering; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Comparator; import java.util.function.BiConsumer; diff --git a/core/src/main/java/org/apache/calcite/rel/RelValidityChecker.java b/core/src/main/java/org/apache/calcite/rel/RelValidityChecker.java index f30fa3238d43..9be60346865b 100644 --- a/core/src/main/java/org/apache/calcite/rel/RelValidityChecker.java +++ b/core/src/main/java/org/apache/calcite/rel/RelValidityChecker.java @@ -21,7 +21,7 @@ import com.google.common.collect.ImmutableSet; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayDeque; import java.util.Deque; diff --git a/core/src/main/java/org/apache/calcite/rel/RelVisitor.java b/core/src/main/java/org/apache/calcite/rel/RelVisitor.java index c355eba747c8..4cfef79808f8 100644 --- a/core/src/main/java/org/apache/calcite/rel/RelVisitor.java +++ b/core/src/main/java/org/apache/calcite/rel/RelVisitor.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.rel; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * A RelVisitor is a Visitor role in the diff --git a/core/src/main/java/org/apache/calcite/rel/RelWriter.java b/core/src/main/java/org/apache/calcite/rel/RelWriter.java index 6af6e1117875..c382aeaebff8 100644 --- a/core/src/main/java/org/apache/calcite/rel/RelWriter.java +++ b/core/src/main/java/org/apache/calcite/rel/RelWriter.java @@ -19,7 +19,7 @@ import org.apache.calcite.sql.SqlExplainLevel; import org.apache.calcite.util.Pair; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/rel/convert/Converter.java b/core/src/main/java/org/apache/calcite/rel/convert/Converter.java index cc0eaa8ee72d..d412459f6716 100644 --- a/core/src/main/java/org/apache/calcite/rel/convert/Converter.java +++ b/core/src/main/java/org/apache/calcite/rel/convert/Converter.java @@ -20,7 +20,7 @@ import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.RelNode; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * A relational expression implements the interface Converter to diff --git a/core/src/main/java/org/apache/calcite/rel/convert/ConverterImpl.java b/core/src/main/java/org/apache/calcite/rel/convert/ConverterImpl.java index 623e1acc92ff..e7574a607ab1 100644 --- a/core/src/main/java/org/apache/calcite/rel/convert/ConverterImpl.java +++ b/core/src/main/java/org/apache/calcite/rel/convert/ConverterImpl.java @@ -25,7 +25,7 @@ import org.apache.calcite.rel.SingleRel; import org.apache.calcite.rel.metadata.RelMetadataQuery; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Abstract implementation of {@link Converter}. diff --git a/core/src/main/java/org/apache/calcite/rel/convert/ConverterRule.java b/core/src/main/java/org/apache/calcite/rel/convert/ConverterRule.java index 10e11d2b71ae..b948bccee9cf 100644 --- a/core/src/main/java/org/apache/calcite/rel/convert/ConverterRule.java +++ b/core/src/main/java/org/apache/calcite/rel/convert/ConverterRule.java @@ -25,8 +25,8 @@ import org.apache.calcite.rel.RelNode; import org.apache.calcite.tools.RelBuilderFactory; -import org.checkerframework.checker.nullness.qual.Nullable; import org.immutables.value.Value; +import org.jspecify.annotations.Nullable; import java.util.Locale; import java.util.Objects; diff --git a/core/src/main/java/org/apache/calcite/rel/convert/TraitMatchingRule.java b/core/src/main/java/org/apache/calcite/rel/convert/TraitMatchingRule.java index abd05a935e11..23c2ac3115f6 100644 --- a/core/src/main/java/org/apache/calcite/rel/convert/TraitMatchingRule.java +++ b/core/src/main/java/org/apache/calcite/rel/convert/TraitMatchingRule.java @@ -25,8 +25,8 @@ import org.apache.calcite.rel.core.RelFactories; import org.apache.calcite.tools.RelBuilderFactory; -import org.checkerframework.checker.nullness.qual.Nullable; import org.immutables.value.Value; +import org.jspecify.annotations.Nullable; /** * TraitMatchingRule adapts a converter rule, restricting it to fire only when diff --git a/core/src/main/java/org/apache/calcite/rel/core/Aggregate.java b/core/src/main/java/org/apache/calcite/rel/core/Aggregate.java index c019d53e9909..ef8538faa1db 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/Aggregate.java +++ b/core/src/main/java/org/apache/calcite/rel/core/Aggregate.java @@ -49,7 +49,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.math.IntMath; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.Collections; diff --git a/core/src/main/java/org/apache/calcite/rel/core/AggregateCall.java b/core/src/main/java/org/apache/calcite/rel/core/AggregateCall.java index 47164f94def9..b222cfc4f557 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/AggregateCall.java +++ b/core/src/main/java/org/apache/calcite/rel/core/AggregateCall.java @@ -35,7 +35,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; import java.util.Objects; diff --git a/core/src/main/java/org/apache/calcite/rel/core/Calc.java b/core/src/main/java/org/apache/calcite/rel/core/Calc.java index 0e1e2e61965c..7b5725881d03 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/Calc.java +++ b/core/src/main/java/org/apache/calcite/rel/core/Calc.java @@ -43,7 +43,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/rel/core/Collect.java b/core/src/main/java/org/apache/calcite/rel/core/Collect.java index d2f89eeec3ba..48547f54301f 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/Collect.java +++ b/core/src/main/java/org/apache/calcite/rel/core/Collect.java @@ -33,7 +33,7 @@ import com.google.common.collect.Iterables; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/rel/core/Correlate.java b/core/src/main/java/org/apache/calcite/rel/core/Correlate.java index c6a994e922eb..55351399c56f 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/Correlate.java +++ b/core/src/main/java/org/apache/calcite/rel/core/Correlate.java @@ -37,7 +37,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; import java.util.Set; diff --git a/core/src/main/java/org/apache/calcite/rel/core/CorrelationId.java b/core/src/main/java/org/apache/calcite/rel/core/CorrelationId.java index 8deaa947b69a..8faa343e07b1 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/CorrelationId.java +++ b/core/src/main/java/org/apache/calcite/rel/core/CorrelationId.java @@ -18,7 +18,7 @@ import com.google.common.collect.ImmutableSet; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Set; diff --git a/core/src/main/java/org/apache/calcite/rel/core/Exchange.java b/core/src/main/java/org/apache/calcite/rel/core/Exchange.java index b6a59f9e29d4..d0ef389de77e 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/Exchange.java +++ b/core/src/main/java/org/apache/calcite/rel/core/Exchange.java @@ -30,7 +30,7 @@ import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.util.Util; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/rel/core/Filter.java b/core/src/main/java/org/apache/calcite/rel/core/Filter.java index 27aafd375285..f1fd9a6b1195 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/Filter.java +++ b/core/src/main/java/org/apache/calcite/rel/core/Filter.java @@ -42,7 +42,7 @@ import org.apiguardian.api.API; import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; import java.util.Objects; diff --git a/core/src/main/java/org/apache/calcite/rel/core/Join.java b/core/src/main/java/org/apache/calcite/rel/core/Join.java index 999c4639f9c7..a5ecdaea4a53 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/Join.java +++ b/core/src/main/java/org/apache/calcite/rel/core/Join.java @@ -43,7 +43,7 @@ import org.apiguardian.api.API; import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Collections; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/rel/core/Match.java b/core/src/main/java/org/apache/calcite/rel/core/Match.java index a0d106b2096b..09bfd9a1765e 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/Match.java +++ b/core/src/main/java/org/apache/calcite/rel/core/Match.java @@ -39,7 +39,7 @@ import com.google.common.collect.ImmutableSortedMap; import com.google.common.collect.ImmutableSortedSet; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.HashSet; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/rel/core/Project.java b/core/src/main/java/org/apache/calcite/rel/core/Project.java index 1de67c9228f0..2fd3ab7fdc11 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/Project.java +++ b/core/src/main/java/org/apache/calcite/rel/core/Project.java @@ -50,7 +50,7 @@ import org.apiguardian.api.API; import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.HashSet; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/rel/core/RelFactories.java b/core/src/main/java/org/apache/calcite/rel/core/RelFactories.java index e2a096762f77..b351cdbeab4b 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/RelFactories.java +++ b/core/src/main/java/org/apache/calcite/rel/core/RelFactories.java @@ -64,7 +64,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Type; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/rel/core/RepeatUnion.java b/core/src/main/java/org/apache/calcite/rel/core/RepeatUnion.java index 5c31b4f189a8..62e881af2ec3 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/RepeatUnion.java +++ b/core/src/main/java/org/apache/calcite/rel/core/RepeatUnion.java @@ -28,7 +28,7 @@ import org.apache.calcite.schema.TransientTable; import org.apache.calcite.util.Util; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/rel/core/Snapshot.java b/core/src/main/java/org/apache/calcite/rel/core/Snapshot.java index a7c387c66375..89df44b05663 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/Snapshot.java +++ b/core/src/main/java/org/apache/calcite/rel/core/Snapshot.java @@ -33,7 +33,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/rel/core/Sort.java b/core/src/main/java/org/apache/calcite/rel/core/Sort.java index 9271a6210718..a4d70b52fc99 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/Sort.java +++ b/core/src/main/java/org/apache/calcite/rel/core/Sort.java @@ -38,7 +38,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Collections; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/rel/core/TableFunctionScan.java b/core/src/main/java/org/apache/calcite/rel/core/TableFunctionScan.java index 4e1ad50b7343..140fcf40863a 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/TableFunctionScan.java +++ b/core/src/main/java/org/apache/calcite/rel/core/TableFunctionScan.java @@ -35,7 +35,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Type; import java.util.ArrayList; diff --git a/core/src/main/java/org/apache/calcite/rel/core/TableModify.java b/core/src/main/java/org/apache/calcite/rel/core/TableModify.java index 83dd8106476e..6c222c7079e7 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/TableModify.java +++ b/core/src/main/java/org/apache/calcite/rel/core/TableModify.java @@ -39,7 +39,7 @@ import org.apache.calcite.sql.type.SqlTypeUtil; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/rel/core/TableScan.java b/core/src/main/java/org/apache/calcite/rel/core/TableScan.java index 0d6c4a9cdd04..5a7ab319c739 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/TableScan.java +++ b/core/src/main/java/org/apache/calcite/rel/core/TableScan.java @@ -40,7 +40,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/rel/core/Values.java b/core/src/main/java/org/apache/calcite/rel/core/Values.java index e45648933892..2b3777a8c3b6 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/Values.java +++ b/core/src/main/java/org/apache/calcite/rel/core/Values.java @@ -37,7 +37,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Collections; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/rel/core/Window.java b/core/src/main/java/org/apache/calcite/rel/core/Window.java index 705e349b8465..ed28583a9aaf 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/Window.java +++ b/core/src/main/java/org/apache/calcite/rel/core/Window.java @@ -52,8 +52,8 @@ import com.google.common.collect.ImmutableList; import org.checkerframework.checker.initialization.qual.UnderInitialization; -import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.checker.nullness.qual.RequiresNonNull; +import org.jspecify.annotations.Nullable; import java.util.AbstractList; import java.util.Collections; diff --git a/core/src/main/java/org/apache/calcite/rel/externalize/RelDotWriter.java b/core/src/main/java/org/apache/calcite/rel/externalize/RelDotWriter.java index f354bda24637..bf0ef3c4a5cf 100644 --- a/core/src/main/java/org/apache/calcite/rel/externalize/RelDotWriter.java +++ b/core/src/main/java/org/apache/calcite/rel/externalize/RelDotWriter.java @@ -26,8 +26,8 @@ import com.google.common.collect.HashMultimap; import com.google.common.collect.Multimap; -import org.checkerframework.checker.nullness.qual.Nullable; import org.immutables.value.Value; +import org.jspecify.annotations.Nullable; import java.io.PrintWriter; import java.util.ArrayList; diff --git a/core/src/main/java/org/apache/calcite/rel/externalize/RelEnumTypes.java b/core/src/main/java/org/apache/calcite/rel/externalize/RelEnumTypes.java index b4ad2d1b900a..80ebdce17a57 100644 --- a/core/src/main/java/org/apache/calcite/rel/externalize/RelEnumTypes.java +++ b/core/src/main/java/org/apache/calcite/rel/externalize/RelEnumTypes.java @@ -34,8 +34,8 @@ import com.google.common.collect.ImmutableMap; -import org.checkerframework.checker.nullness.qual.NonNull; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.NonNull; +import org.jspecify.annotations.Nullable; import static org.apache.calcite.linq4j.Nullness.castNonNull; diff --git a/core/src/main/java/org/apache/calcite/rel/externalize/RelJson.java b/core/src/main/java/org/apache/calcite/rel/externalize/RelJson.java index 8fa5abd149d8..8029933a2a3a 100644 --- a/core/src/main/java/org/apache/calcite/rel/externalize/RelJson.java +++ b/core/src/main/java/org/apache/calcite/rel/externalize/RelJson.java @@ -87,9 +87,9 @@ import com.google.common.collect.Range; import com.google.common.collect.RangeSet; -import org.checkerframework.checker.nullness.qual.NonNull; -import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.checker.nullness.qual.PolyNull; +import org.jspecify.annotations.NonNull; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; diff --git a/core/src/main/java/org/apache/calcite/rel/externalize/RelJsonReader.java b/core/src/main/java/org/apache/calcite/rel/externalize/RelJsonReader.java index 2cdd785fe835..e5334d940136 100644 --- a/core/src/main/java/org/apache/calcite/rel/externalize/RelJsonReader.java +++ b/core/src/main/java/org/apache/calcite/rel/externalize/RelJsonReader.java @@ -43,7 +43,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.io.IOException; import java.lang.reflect.Constructor; diff --git a/core/src/main/java/org/apache/calcite/rel/externalize/RelJsonWriter.java b/core/src/main/java/org/apache/calcite/rel/externalize/RelJsonWriter.java index 85d275f79d6c..26f1db978973 100644 --- a/core/src/main/java/org/apache/calcite/rel/externalize/RelJsonWriter.java +++ b/core/src/main/java/org/apache/calcite/rel/externalize/RelJsonWriter.java @@ -24,7 +24,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.IdentityHashMap; diff --git a/core/src/main/java/org/apache/calcite/rel/externalize/RelWriterImpl.java b/core/src/main/java/org/apache/calcite/rel/externalize/RelWriterImpl.java index 25af2ed3af35..f8cd466292b7 100644 --- a/core/src/main/java/org/apache/calcite/rel/externalize/RelWriterImpl.java +++ b/core/src/main/java/org/apache/calcite/rel/externalize/RelWriterImpl.java @@ -28,7 +28,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.io.PrintWriter; import java.util.ArrayList; diff --git a/core/src/main/java/org/apache/calcite/rel/externalize/RelXmlWriter.java b/core/src/main/java/org/apache/calcite/rel/externalize/RelXmlWriter.java index b11098240e40..fef5fbf0eaa5 100644 --- a/core/src/main/java/org/apache/calcite/rel/externalize/RelXmlWriter.java +++ b/core/src/main/java/org/apache/calcite/rel/externalize/RelXmlWriter.java @@ -21,7 +21,7 @@ import org.apache.calcite.util.Pair; import org.apache.calcite.util.XmlOutput; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.io.PrintWriter; import java.util.ArrayList; diff --git a/core/src/main/java/org/apache/calcite/rel/hint/HintStrategy.java b/core/src/main/java/org/apache/calcite/rel/hint/HintStrategy.java index 4f28fd5a9405..1841654158e6 100644 --- a/core/src/main/java/org/apache/calcite/rel/hint/HintStrategy.java +++ b/core/src/main/java/org/apache/calcite/rel/hint/HintStrategy.java @@ -21,7 +21,7 @@ import com.google.common.collect.ImmutableSet; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import static java.util.Objects.requireNonNull; diff --git a/core/src/main/java/org/apache/calcite/rel/hint/HintStrategyTable.java b/core/src/main/java/org/apache/calcite/rel/hint/HintStrategyTable.java index a8de62f186ea..ddeff7b6d2fc 100644 --- a/core/src/main/java/org/apache/calcite/rel/hint/HintStrategyTable.java +++ b/core/src/main/java/org/apache/calcite/rel/hint/HintStrategyTable.java @@ -24,7 +24,7 @@ import com.google.common.collect.ImmutableMap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.slf4j.Logger; import java.util.HashMap; diff --git a/core/src/main/java/org/apache/calcite/rel/hint/RelHint.java b/core/src/main/java/org/apache/calcite/rel/hint/RelHint.java index 73ac9d146d7b..fcbb4716000b 100644 --- a/core/src/main/java/org/apache/calcite/rel/hint/RelHint.java +++ b/core/src/main/java/org/apache/calcite/rel/hint/RelHint.java @@ -21,7 +21,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.Arrays; diff --git a/core/src/main/java/org/apache/calcite/rel/logical/LogicalAggregate.java b/core/src/main/java/org/apache/calcite/rel/logical/LogicalAggregate.java index d60cd228bcdb..96f648389f70 100644 --- a/core/src/main/java/org/apache/calcite/rel/logical/LogicalAggregate.java +++ b/core/src/main/java/org/apache/calcite/rel/logical/LogicalAggregate.java @@ -29,7 +29,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/rel/logical/LogicalAsofJoin.java b/core/src/main/java/org/apache/calcite/rel/logical/LogicalAsofJoin.java index f6c31282fdb0..d6935607e384 100644 --- a/core/src/main/java/org/apache/calcite/rel/logical/LogicalAsofJoin.java +++ b/core/src/main/java/org/apache/calcite/rel/logical/LogicalAsofJoin.java @@ -32,7 +32,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/rel/logical/LogicalFilter.java b/core/src/main/java/org/apache/calcite/rel/logical/LogicalFilter.java index ca67a1d0e174..9703f76fbf74 100644 --- a/core/src/main/java/org/apache/calcite/rel/logical/LogicalFilter.java +++ b/core/src/main/java/org/apache/calcite/rel/logical/LogicalFilter.java @@ -36,7 +36,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; import java.util.Objects; diff --git a/core/src/main/java/org/apache/calcite/rel/logical/LogicalJoin.java b/core/src/main/java/org/apache/calcite/rel/logical/LogicalJoin.java index 9504f01e5534..ee24239828bf 100644 --- a/core/src/main/java/org/apache/calcite/rel/logical/LogicalJoin.java +++ b/core/src/main/java/org/apache/calcite/rel/logical/LogicalJoin.java @@ -33,7 +33,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; import java.util.Objects; diff --git a/core/src/main/java/org/apache/calcite/rel/logical/LogicalMatch.java b/core/src/main/java/org/apache/calcite/rel/logical/LogicalMatch.java index 800a7bdee772..3dde4007f7cc 100644 --- a/core/src/main/java/org/apache/calcite/rel/logical/LogicalMatch.java +++ b/core/src/main/java/org/apache/calcite/rel/logical/LogicalMatch.java @@ -27,7 +27,7 @@ import org.apache.calcite.rex.RexNode; import org.apache.calcite.util.ImmutableBitSet; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; import java.util.Map; diff --git a/core/src/main/java/org/apache/calcite/rel/logical/LogicalProject.java b/core/src/main/java/org/apache/calcite/rel/logical/LogicalProject.java index c1db7f223b81..9195bf9a924b 100644 --- a/core/src/main/java/org/apache/calcite/rel/logical/LogicalProject.java +++ b/core/src/main/java/org/apache/calcite/rel/logical/LogicalProject.java @@ -38,7 +38,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; import java.util.Set; diff --git a/core/src/main/java/org/apache/calcite/rel/logical/LogicalRepeatUnion.java b/core/src/main/java/org/apache/calcite/rel/logical/LogicalRepeatUnion.java index 9b5e099290fc..7c6d20f85419 100644 --- a/core/src/main/java/org/apache/calcite/rel/logical/LogicalRepeatUnion.java +++ b/core/src/main/java/org/apache/calcite/rel/logical/LogicalRepeatUnion.java @@ -25,7 +25,7 @@ import org.apache.calcite.rel.RelShuttle; import org.apache.calcite.rel.core.RepeatUnion; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/rel/logical/LogicalSort.java b/core/src/main/java/org/apache/calcite/rel/logical/LogicalSort.java index 31b8b4b006d8..1396a0cf4e52 100644 --- a/core/src/main/java/org/apache/calcite/rel/logical/LogicalSort.java +++ b/core/src/main/java/org/apache/calcite/rel/logical/LogicalSort.java @@ -28,7 +28,7 @@ import org.apache.calcite.rel.hint.RelHint; import org.apache.calcite.rex.RexNode; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Collections; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/rel/logical/LogicalTableFunctionScan.java b/core/src/main/java/org/apache/calcite/rel/logical/LogicalTableFunctionScan.java index 07765eb6bf40..7672154f9de9 100644 --- a/core/src/main/java/org/apache/calcite/rel/logical/LogicalTableFunctionScan.java +++ b/core/src/main/java/org/apache/calcite/rel/logical/LogicalTableFunctionScan.java @@ -30,7 +30,7 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexNode; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Type; import java.util.Collections; diff --git a/core/src/main/java/org/apache/calcite/rel/logical/LogicalTableModify.java b/core/src/main/java/org/apache/calcite/rel/logical/LogicalTableModify.java index 042547529dca..08230adc9225 100644 --- a/core/src/main/java/org/apache/calcite/rel/logical/LogicalTableModify.java +++ b/core/src/main/java/org/apache/calcite/rel/logical/LogicalTableModify.java @@ -27,7 +27,7 @@ import org.apache.calcite.rel.core.TableModify; import org.apache.calcite.rex.RexNode; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/rel/logical/LogicalWindow.java b/core/src/main/java/org/apache/calcite/rel/logical/LogicalWindow.java index 16b247974b3d..a2f63da0dac7 100644 --- a/core/src/main/java/org/apache/calcite/rel/logical/LogicalWindow.java +++ b/core/src/main/java/org/apache/calcite/rel/logical/LogicalWindow.java @@ -45,7 +45,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Multimap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.AbstractList; import java.util.ArrayList; diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/BuiltInMetadata.java b/core/src/main/java/org/apache/calcite/rel/metadata/BuiltInMetadata.java index 9c571dfddcd3..e896751f2216 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/BuiltInMetadata.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/BuiltInMetadata.java @@ -37,7 +37,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Multimap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; import java.util.Set; diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/CachingRelMetadataProvider.java b/core/src/main/java/org/apache/calcite/rel/metadata/CachingRelMetadataProvider.java index 399c4b67aba5..c835a7118d18 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/CachingRelMetadataProvider.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/CachingRelMetadataProvider.java @@ -22,7 +22,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Multimap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.InvocationHandler; import java.lang.reflect.InvocationTargetException; diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/ChainedRelMetadataProvider.java b/core/src/main/java/org/apache/calcite/rel/metadata/ChainedRelMetadataProvider.java index acfc83bda18a..116e4d36f3cf 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/ChainedRelMetadataProvider.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/ChainedRelMetadataProvider.java @@ -23,7 +23,7 @@ import com.google.common.collect.ImmutableMultimap; import com.google.common.collect.Multimap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.InvocationHandler; import java.lang.reflect.InvocationTargetException; diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/JaninoRelMetadataProvider.java b/core/src/main/java/org/apache/calcite/rel/metadata/JaninoRelMetadataProvider.java index 135b11e11083..199440d444a6 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/JaninoRelMetadataProvider.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/JaninoRelMetadataProvider.java @@ -30,11 +30,11 @@ import com.google.common.util.concurrent.UncheckedExecutionException; import org.apiguardian.api.API; -import org.checkerframework.checker.nullness.qual.Nullable; import org.codehaus.commons.compiler.CompileException; import org.codehaus.commons.compiler.CompilerFactoryFactory; import org.codehaus.commons.compiler.ICompilerFactory; import org.codehaus.commons.compiler.ISimpleCompiler; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/MetadataFactory.java b/core/src/main/java/org/apache/calcite/rel/metadata/MetadataFactory.java index 157d6a3e7566..63c8cb2f1227 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/MetadataFactory.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/MetadataFactory.java @@ -18,7 +18,7 @@ import org.apache.calcite.rel.RelNode; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Source of metadata about relational expressions. diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/MetadataFactoryImpl.java b/core/src/main/java/org/apache/calcite/rel/metadata/MetadataFactoryImpl.java index c0f725af29be..854c3784f8f6 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/MetadataFactoryImpl.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/MetadataFactoryImpl.java @@ -25,7 +25,7 @@ import com.google.common.cache.LoadingCache; import com.google.common.util.concurrent.UncheckedExecutionException; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.concurrent.ExecutionException; diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/NullSentinel.java b/core/src/main/java/org/apache/calcite/rel/metadata/NullSentinel.java index c559ef8a7c3a..df6412e1a714 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/NullSentinel.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/NullSentinel.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.rel.metadata; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** Placeholder for null values. */ public enum NullSentinel { diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/ReflectiveRelMetadataProvider.java b/core/src/main/java/org/apache/calcite/rel/metadata/ReflectiveRelMetadataProvider.java index 31caa39ed24a..e8129979627c 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/ReflectiveRelMetadataProvider.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/ReflectiveRelMetadataProvider.java @@ -29,7 +29,7 @@ import com.google.common.collect.ImmutableMultimap; import com.google.common.collect.Multimap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelColumnOrigin.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelColumnOrigin.java index 974501b50bdd..85bbe6b858e5 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelColumnOrigin.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelColumnOrigin.java @@ -18,7 +18,7 @@ import org.apache.calcite.plan.RelOptTable; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * RelColumnOrigin is a data structure describing one of the origins of an diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdAllPredicates.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdAllPredicates.java index d86a61fad8d4..d86e98f179db 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdAllPredicates.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdAllPredicates.java @@ -49,7 +49,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Multimap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Collection; import java.util.HashMap; diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdCollation.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdCollation.java index e6ae979b1ab3..2a867a78a5a2 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdCollation.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdCollation.java @@ -62,7 +62,7 @@ import com.google.common.collect.Multimap; import com.google.common.collect.Ordering; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.Collection; diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdColumnOrigins.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdColumnOrigins.java index a5e6da109af2..3d8fbc1c43bb 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdColumnOrigins.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdColumnOrigins.java @@ -42,8 +42,8 @@ import org.apache.calcite.rex.RexVisitorImpl; import org.apache.calcite.util.Util; -import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.checker.nullness.qual.PolyNull; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.HashSet; diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdColumnUniqueness.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdColumnUniqueness.java index e69da277451c..8fb570305b8c 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdColumnUniqueness.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdColumnUniqueness.java @@ -58,7 +58,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.HashSet; diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdDistinctRowCount.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdDistinctRowCount.java index 7058dcbb4dac..f33f977993e2 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdDistinctRowCount.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdDistinctRowCount.java @@ -39,7 +39,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.HashSet; diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdDistribution.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdDistribution.java index 6bbae1b78d4e..32018b79c87b 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdDistribution.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdDistribution.java @@ -39,7 +39,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdExplainVisibility.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdExplainVisibility.java index a06b2842a139..66d3764dc53a 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdExplainVisibility.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdExplainVisibility.java @@ -20,7 +20,7 @@ import org.apache.calcite.rel.core.TableScan; import org.apache.calcite.sql.SqlExplainLevel; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * RelMdExplainVisibility supplies a default implementation of diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdExpressionLineage.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdExpressionLineage.java index 3025ee1d4a53..bf5888615516 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdExpressionLineage.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdExpressionLineage.java @@ -52,7 +52,7 @@ import com.google.common.collect.Multimap; import org.checkerframework.checker.nullness.qual.KeyFor; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.Collection; diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdFunctionalDependency.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdFunctionalDependency.java index 15679c63c701..469bd412a529 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdFunctionalDependency.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdFunctionalDependency.java @@ -39,7 +39,7 @@ import org.apache.calcite.util.ImmutableBitSet; import org.apache.calcite.util.mapping.Mappings; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.HashMap; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdLowerBoundCost.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdLowerBoundCost.java index b54f2d0708cf..5678e245b441 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdLowerBoundCost.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdLowerBoundCost.java @@ -22,7 +22,7 @@ import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.metadata.BuiltInMetadata.LowerBoundCost; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Default implementations of the diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdMaxRowCount.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdMaxRowCount.java index 869f1ad50e78..a52e234da3a8 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdMaxRowCount.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdMaxRowCount.java @@ -39,7 +39,7 @@ import org.apache.calcite.util.Bug; import org.apache.calcite.util.Util; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import static org.apache.calcite.rel.metadata.RelMdUtil.literalValueApproximatedByDouble; diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdMeasure.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdMeasure.java index ce27b879cb49..7e8a7c9dc355 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdMeasure.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdMeasure.java @@ -34,7 +34,7 @@ import org.apache.calcite.sql.type.SqlTypeUtil; import org.apache.calcite.tools.RelBuilder; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdMemory.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdMemory.java index b8333fc4c0f7..4d316d192330 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdMemory.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdMemory.java @@ -18,7 +18,7 @@ import org.apache.calcite.rel.RelNode; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Default implementations of the diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdMinRowCount.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdMinRowCount.java index 298e77f2d8e4..fcc082ac1116 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdMinRowCount.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdMinRowCount.java @@ -37,7 +37,7 @@ import org.apache.calcite.util.Bug; import org.apache.calcite.util.Util; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import static org.apache.calcite.rel.metadata.RelMdUtil.literalValueApproximatedByDouble; diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdNodeTypes.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdNodeTypes.java index 60aa05f9f247..c22836d9dade 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdNodeTypes.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdNodeTypes.java @@ -40,7 +40,7 @@ import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.Multimap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * RelMdNodeTypeCount supplies a default implementation of diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdPercentageOriginalRows.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdPercentageOriginalRows.java index e2ec6d94b8bb..63e7123552ef 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdPercentageOriginalRows.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdPercentageOriginalRows.java @@ -27,8 +27,8 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.checker.nullness.qual.PolyNull; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdPopulationSize.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdPopulationSize.java index bbd4c0fa8298..e800c4e5c5fe 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdPopulationSize.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdPopulationSize.java @@ -31,7 +31,7 @@ import org.apache.calcite.rex.RexUtil; import org.apache.calcite.util.ImmutableBitSet; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdPredicates.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdPredicates.java index e04e2098ff78..b2330bff7718 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdPredicates.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdPredicates.java @@ -71,7 +71,7 @@ import com.google.common.collect.RangeSet; import com.google.common.collect.TreeRangeSet; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.Arrays; diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdRowCount.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdRowCount.java index 3e7824e1aac6..9cd8400a20ba 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdRowCount.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdRowCount.java @@ -41,7 +41,7 @@ import org.apache.calcite.util.NumberUtil; import org.apache.calcite.util.Util; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import static org.apache.calcite.rel.metadata.RelMdUtil.literalValueApproximatedByDouble; diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdSelectivity.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdSelectivity.java index 5faaeaa85d3d..4397fba1d7b0 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdSelectivity.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdSelectivity.java @@ -35,7 +35,7 @@ import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.util.ImmutableBitSet; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdSize.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdSize.java index 675aa1e3639e..33a540ba736c 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdSize.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdSize.java @@ -48,7 +48,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdTableReferences.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdTableReferences.java index ebcbdc187ddb..1c17541fd80c 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdTableReferences.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdTableReferences.java @@ -37,7 +37,7 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Multimap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Collection; import java.util.HashMap; diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdUniqueKeys.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdUniqueKeys.java index c42fc812317d..da0c6086e0fb 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdUniqueKeys.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdUniqueKeys.java @@ -49,7 +49,7 @@ import com.google.common.collect.Multimap; import com.google.common.collect.Sets; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.HashSet; diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdUtil.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdUtil.java index ab7dab016b4b..d8ad2f9a1fc1 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdUtil.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdUtil.java @@ -49,8 +49,8 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.checker.nullness.qual.PolyNull; +import org.jspecify.annotations.Nullable; import java.math.BigDecimal; import java.util.ArrayList; diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMetadataProvider.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMetadataProvider.java index 26a92709e18a..562640ae78d7 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMetadataProvider.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMetadataProvider.java @@ -20,7 +20,7 @@ import com.google.common.collect.Multimap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Method; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMetadataQuery.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMetadataQuery.java index aeeefa9763a3..0f2adec9e408 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMetadataQuery.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMetadataQuery.java @@ -35,7 +35,7 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Multimap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Collections; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMetadataQueryBase.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMetadataQueryBase.java index 37a2e5a127f4..f4cc78fea47b 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMetadataQueryBase.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMetadataQueryBase.java @@ -21,7 +21,7 @@ import com.google.common.collect.HashBasedTable; import com.google.common.collect.Table; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Proxy; import java.util.Map; diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/UnboundMetadata.java b/core/src/main/java/org/apache/calcite/rel/metadata/UnboundMetadata.java index 0f3131668644..a4ac28ccc047 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/UnboundMetadata.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/UnboundMetadata.java @@ -18,7 +18,7 @@ import org.apache.calcite.rel.RelNode; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Metadata that needs to be bound to a {@link RelNode} and diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/janino/DispatchGenerator.java b/core/src/main/java/org/apache/calcite/rel/metadata/janino/DispatchGenerator.java index 4c661f476ad5..7c8f83e46efd 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/janino/DispatchGenerator.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/janino/DispatchGenerator.java @@ -22,7 +22,7 @@ import com.google.common.collect.ImmutableSet; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Method; import java.util.ArrayDeque; diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/janino/RelMetadataHandlerGeneratorUtil.java b/core/src/main/java/org/apache/calcite/rel/metadata/janino/RelMetadataHandlerGeneratorUtil.java index 0fd6103cfd0f..9c0350967ac8 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/janino/RelMetadataHandlerGeneratorUtil.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/janino/RelMetadataHandlerGeneratorUtil.java @@ -20,8 +20,8 @@ import org.apache.calcite.rel.metadata.MetadataDef; import org.apache.calcite.rel.metadata.MetadataHandler; -import org.checkerframework.checker.nullness.qual.Nullable; import org.immutables.value.Value; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Method; import java.util.LinkedHashMap; diff --git a/core/src/main/java/org/apache/calcite/rel/mutable/MutableAggregate.java b/core/src/main/java/org/apache/calcite/rel/mutable/MutableAggregate.java index 92593be2f11e..b77c046b6b5d 100644 --- a/core/src/main/java/org/apache/calcite/rel/mutable/MutableAggregate.java +++ b/core/src/main/java/org/apache/calcite/rel/mutable/MutableAggregate.java @@ -23,7 +23,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; import java.util.Objects; diff --git a/core/src/main/java/org/apache/calcite/rel/mutable/MutableCalc.java b/core/src/main/java/org/apache/calcite/rel/mutable/MutableCalc.java index 1d172c5f2cb5..f82285d5b778 100644 --- a/core/src/main/java/org/apache/calcite/rel/mutable/MutableCalc.java +++ b/core/src/main/java/org/apache/calcite/rel/mutable/MutableCalc.java @@ -18,7 +18,7 @@ import org.apache.calcite.rex.RexProgram; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Objects; diff --git a/core/src/main/java/org/apache/calcite/rel/mutable/MutableCollect.java b/core/src/main/java/org/apache/calcite/rel/mutable/MutableCollect.java index 0782d2f89fa1..da3576bfddce 100644 --- a/core/src/main/java/org/apache/calcite/rel/mutable/MutableCollect.java +++ b/core/src/main/java/org/apache/calcite/rel/mutable/MutableCollect.java @@ -18,7 +18,7 @@ import org.apache.calcite.rel.type.RelDataType; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Objects; diff --git a/core/src/main/java/org/apache/calcite/rel/mutable/MutableCorrelate.java b/core/src/main/java/org/apache/calcite/rel/mutable/MutableCorrelate.java index cba7890c8d45..25bbdf0302b7 100644 --- a/core/src/main/java/org/apache/calcite/rel/mutable/MutableCorrelate.java +++ b/core/src/main/java/org/apache/calcite/rel/mutable/MutableCorrelate.java @@ -21,7 +21,7 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.util.ImmutableBitSet; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Objects; diff --git a/core/src/main/java/org/apache/calcite/rel/mutable/MutableExchange.java b/core/src/main/java/org/apache/calcite/rel/mutable/MutableExchange.java index 72669a1cdbf7..e9bdad320e2b 100644 --- a/core/src/main/java/org/apache/calcite/rel/mutable/MutableExchange.java +++ b/core/src/main/java/org/apache/calcite/rel/mutable/MutableExchange.java @@ -18,7 +18,7 @@ import org.apache.calcite.rel.RelDistribution; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Objects; diff --git a/core/src/main/java/org/apache/calcite/rel/mutable/MutableFilter.java b/core/src/main/java/org/apache/calcite/rel/mutable/MutableFilter.java index 783e8d9301f5..b5fe93290843 100644 --- a/core/src/main/java/org/apache/calcite/rel/mutable/MutableFilter.java +++ b/core/src/main/java/org/apache/calcite/rel/mutable/MutableFilter.java @@ -18,7 +18,7 @@ import org.apache.calcite.rex.RexNode; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Objects; diff --git a/core/src/main/java/org/apache/calcite/rel/mutable/MutableJoin.java b/core/src/main/java/org/apache/calcite/rel/mutable/MutableJoin.java index ccc6a093fbec..d1a462487f24 100644 --- a/core/src/main/java/org/apache/calcite/rel/mutable/MutableJoin.java +++ b/core/src/main/java/org/apache/calcite/rel/mutable/MutableJoin.java @@ -21,7 +21,7 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexNode; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Objects; import java.util.Set; diff --git a/core/src/main/java/org/apache/calcite/rel/mutable/MutableMatch.java b/core/src/main/java/org/apache/calcite/rel/mutable/MutableMatch.java index a55b2a71cb94..21bd863358c0 100644 --- a/core/src/main/java/org/apache/calcite/rel/mutable/MutableMatch.java +++ b/core/src/main/java/org/apache/calcite/rel/mutable/MutableMatch.java @@ -21,7 +21,7 @@ import org.apache.calcite.rex.RexNode; import org.apache.calcite.util.ImmutableBitSet; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Map; import java.util.Objects; diff --git a/core/src/main/java/org/apache/calcite/rel/mutable/MutableProject.java b/core/src/main/java/org/apache/calcite/rel/mutable/MutableProject.java index 9254ce096d96..f9f7068fb5e7 100644 --- a/core/src/main/java/org/apache/calcite/rel/mutable/MutableProject.java +++ b/core/src/main/java/org/apache/calcite/rel/mutable/MutableProject.java @@ -25,7 +25,7 @@ import org.apache.calcite.util.Pair; import org.apache.calcite.util.mapping.Mappings; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; import java.util.Objects; diff --git a/core/src/main/java/org/apache/calcite/rel/mutable/MutableRel.java b/core/src/main/java/org/apache/calcite/rel/mutable/MutableRel.java index 71f6790d6404..1609ab194abe 100644 --- a/core/src/main/java/org/apache/calcite/rel/mutable/MutableRel.java +++ b/core/src/main/java/org/apache/calcite/rel/mutable/MutableRel.java @@ -24,7 +24,7 @@ import com.google.common.base.Equivalence; import com.google.common.collect.Lists; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/rel/mutable/MutableRelVisitor.java b/core/src/main/java/org/apache/calcite/rel/mutable/MutableRelVisitor.java index 9fe5020ae969..9c5d80ee06cf 100644 --- a/core/src/main/java/org/apache/calcite/rel/mutable/MutableRelVisitor.java +++ b/core/src/main/java/org/apache/calcite/rel/mutable/MutableRelVisitor.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.rel.mutable; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** Visitor over {@link MutableRel}. */ public class MutableRelVisitor { diff --git a/core/src/main/java/org/apache/calcite/rel/mutable/MutableRels.java b/core/src/main/java/org/apache/calcite/rel/mutable/MutableRels.java index 092d45c8fe9a..4b18bafafa02 100644 --- a/core/src/main/java/org/apache/calcite/rel/mutable/MutableRels.java +++ b/core/src/main/java/org/apache/calcite/rel/mutable/MutableRels.java @@ -61,7 +61,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.AbstractList; import java.util.ArrayList; diff --git a/core/src/main/java/org/apache/calcite/rel/mutable/MutableSample.java b/core/src/main/java/org/apache/calcite/rel/mutable/MutableSample.java index 7345fdbf9b80..8b623bb544d9 100644 --- a/core/src/main/java/org/apache/calcite/rel/mutable/MutableSample.java +++ b/core/src/main/java/org/apache/calcite/rel/mutable/MutableSample.java @@ -18,7 +18,7 @@ import org.apache.calcite.plan.RelOptSamplingParameters; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Objects; diff --git a/core/src/main/java/org/apache/calcite/rel/mutable/MutableScan.java b/core/src/main/java/org/apache/calcite/rel/mutable/MutableScan.java index e2450e725835..3da4075692a1 100644 --- a/core/src/main/java/org/apache/calcite/rel/mutable/MutableScan.java +++ b/core/src/main/java/org/apache/calcite/rel/mutable/MutableScan.java @@ -19,7 +19,7 @@ import org.apache.calcite.plan.RelOptTable; import org.apache.calcite.rel.core.TableScan; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; import java.util.Objects; diff --git a/core/src/main/java/org/apache/calcite/rel/mutable/MutableSetOp.java b/core/src/main/java/org/apache/calcite/rel/mutable/MutableSetOp.java index 500a56bcfbb5..37a4b3dc704a 100644 --- a/core/src/main/java/org/apache/calcite/rel/mutable/MutableSetOp.java +++ b/core/src/main/java/org/apache/calcite/rel/mutable/MutableSetOp.java @@ -19,7 +19,7 @@ import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.rel.type.RelDataType; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; import java.util.Objects; diff --git a/core/src/main/java/org/apache/calcite/rel/mutable/MutableSort.java b/core/src/main/java/org/apache/calcite/rel/mutable/MutableSort.java index 904c6ce64bb0..23f1a29471a1 100644 --- a/core/src/main/java/org/apache/calcite/rel/mutable/MutableSort.java +++ b/core/src/main/java/org/apache/calcite/rel/mutable/MutableSort.java @@ -19,7 +19,7 @@ import org.apache.calcite.rel.RelCollation; import org.apache.calcite.rex.RexNode; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Objects; diff --git a/core/src/main/java/org/apache/calcite/rel/mutable/MutableTableFunctionScan.java b/core/src/main/java/org/apache/calcite/rel/mutable/MutableTableFunctionScan.java index 7135c962a23a..c7adaf356bd0 100644 --- a/core/src/main/java/org/apache/calcite/rel/mutable/MutableTableFunctionScan.java +++ b/core/src/main/java/org/apache/calcite/rel/mutable/MutableTableFunctionScan.java @@ -21,7 +21,7 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexNode; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Type; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/rel/mutable/MutableTableModify.java b/core/src/main/java/org/apache/calcite/rel/mutable/MutableTableModify.java index d84047356b05..b46829660655 100644 --- a/core/src/main/java/org/apache/calcite/rel/mutable/MutableTableModify.java +++ b/core/src/main/java/org/apache/calcite/rel/mutable/MutableTableModify.java @@ -22,7 +22,7 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexNode; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; import java.util.Objects; diff --git a/core/src/main/java/org/apache/calcite/rel/mutable/MutableUncollect.java b/core/src/main/java/org/apache/calcite/rel/mutable/MutableUncollect.java index 0dc09b2a001e..484bec868028 100644 --- a/core/src/main/java/org/apache/calcite/rel/mutable/MutableUncollect.java +++ b/core/src/main/java/org/apache/calcite/rel/mutable/MutableUncollect.java @@ -18,7 +18,7 @@ import org.apache.calcite.rel.type.RelDataType; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Objects; diff --git a/core/src/main/java/org/apache/calcite/rel/mutable/MutableValues.java b/core/src/main/java/org/apache/calcite/rel/mutable/MutableValues.java index 2f921bc1b696..150e3cd3f812 100644 --- a/core/src/main/java/org/apache/calcite/rel/mutable/MutableValues.java +++ b/core/src/main/java/org/apache/calcite/rel/mutable/MutableValues.java @@ -18,7 +18,7 @@ import org.apache.calcite.rel.core.Values; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** Mutable equivalent of {@link org.apache.calcite.rel.core.Values}. */ public class MutableValues extends MutableLeafRel { diff --git a/core/src/main/java/org/apache/calcite/rel/mutable/MutableWindow.java b/core/src/main/java/org/apache/calcite/rel/mutable/MutableWindow.java index 013d9534764d..aaff45b5be43 100644 --- a/core/src/main/java/org/apache/calcite/rel/mutable/MutableWindow.java +++ b/core/src/main/java/org/apache/calcite/rel/mutable/MutableWindow.java @@ -20,7 +20,7 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexLiteral; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; import java.util.Objects; diff --git a/core/src/main/java/org/apache/calcite/rel/package-info.java b/core/src/main/java/org/apache/calcite/rel/package-info.java index d4cd83282028..6612057eaacd 100644 --- a/core/src/main/java/org/apache/calcite/rel/package-info.java +++ b/core/src/main/java/org/apache/calcite/rel/package-info.java @@ -40,6 +40,6 @@ @DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.RETURN) package org.apache.calcite.rel; -import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.framework.qual.DefaultQualifier; import org.checkerframework.framework.qual.TypeUseLocation; +import org.jspecify.annotations.NonNull; diff --git a/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java b/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java index 7806ab5e75ef..0b4270c0d793 100644 --- a/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java +++ b/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java @@ -105,7 +105,7 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Ordering; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayDeque; import java.util.ArrayList; diff --git a/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java b/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java index a7a52579112c..c9b82b3eecdb 100644 --- a/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java +++ b/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java @@ -125,7 +125,7 @@ import com.google.common.collect.RangeSet; import org.checkerframework.checker.initialization.qual.UnknownInitialization; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.math.BigDecimal; import java.util.AbstractList; diff --git a/core/src/main/java/org/apache/calcite/rel/rules/AggregateCaseToFilterRule.java b/core/src/main/java/org/apache/calcite/rel/rules/AggregateCaseToFilterRule.java index 7ac593adad6f..1bfd733ca068 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/AggregateCaseToFilterRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/AggregateCaseToFilterRule.java @@ -39,8 +39,8 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; import org.immutables.value.Value; +import org.jspecify.annotations.Nullable; import java.math.BigDecimal; import java.util.ArrayList; diff --git a/core/src/main/java/org/apache/calcite/rel/rules/AggregateExpandDistinctAggregatesRule.java b/core/src/main/java/org/apache/calcite/rel/rules/AggregateExpandDistinctAggregatesRule.java index afb4edbf8e4a..b9abcc7a1b8b 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/AggregateExpandDistinctAggregatesRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/AggregateExpandDistinctAggregatesRule.java @@ -48,8 +48,8 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Lists; -import org.checkerframework.checker.nullness.qual.Nullable; import org.immutables.value.Value; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.Collection; diff --git a/core/src/main/java/org/apache/calcite/rel/rules/AggregateExpandWithinDistinctRule.java b/core/src/main/java/org/apache/calcite/rel/rules/AggregateExpandWithinDistinctRule.java index 111cf0c6b6f6..46f4e5994e2b 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/AggregateExpandWithinDistinctRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/AggregateExpandWithinDistinctRule.java @@ -36,8 +36,8 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Multimap; -import org.checkerframework.checker.nullness.qual.Nullable; import org.immutables.value.Value; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.HashMap; diff --git a/core/src/main/java/org/apache/calcite/rel/rules/AggregateJoinTransposeRule.java b/core/src/main/java/org/apache/calcite/rel/rules/AggregateJoinTransposeRule.java index c56f61ba1388..08334d1db4c2 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/AggregateJoinTransposeRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/AggregateJoinTransposeRule.java @@ -47,8 +47,8 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; import org.immutables.value.Value; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.BitSet; diff --git a/core/src/main/java/org/apache/calcite/rel/rules/AggregateProjectMergeRule.java b/core/src/main/java/org/apache/calcite/rel/rules/AggregateProjectMergeRule.java index 2068b0e40c83..205938d082fb 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/AggregateProjectMergeRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/AggregateProjectMergeRule.java @@ -34,8 +34,8 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; import org.immutables.value.Value; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.HashMap; diff --git a/core/src/main/java/org/apache/calcite/rel/rules/AggregateReduceFunctionsOnGroupKeysRule.java b/core/src/main/java/org/apache/calcite/rel/rules/AggregateReduceFunctionsOnGroupKeysRule.java index c31e731c0773..1fefb100e4d0 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/AggregateReduceFunctionsOnGroupKeysRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/AggregateReduceFunctionsOnGroupKeysRule.java @@ -36,8 +36,8 @@ import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.tools.RelBuilder; -import org.checkerframework.checker.nullness.qual.Nullable; import org.immutables.value.Value; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/rel/rules/AggregateReduceFunctionsRule.java b/core/src/main/java/org/apache/calcite/rel/rules/AggregateReduceFunctionsRule.java index 498d6e70470f..d0e1a86bab03 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/AggregateReduceFunctionsRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/AggregateReduceFunctionsRule.java @@ -45,8 +45,8 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import org.checkerframework.checker.nullness.qual.Nullable; import org.immutables.value.Value; +import org.jspecify.annotations.Nullable; import java.math.BigDecimal; import java.util.ArrayList; diff --git a/core/src/main/java/org/apache/calcite/rel/rules/AggregateRemoveRule.java b/core/src/main/java/org/apache/calcite/rel/rules/AggregateRemoveRule.java index c4d406af2195..4eef37a0e65a 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/AggregateRemoveRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/AggregateRemoveRule.java @@ -34,8 +34,8 @@ import org.apache.calcite.tools.RelBuilder; import org.apache.calcite.tools.RelBuilderFactory; -import org.checkerframework.checker.nullness.qual.Nullable; import org.immutables.value.Value; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/rel/rules/AggregateStarTableRule.java b/core/src/main/java/org/apache/calcite/rel/rules/AggregateStarTableRule.java index 962fa5f64f44..0aae7872fa71 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/AggregateStarTableRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/AggregateStarTableRule.java @@ -46,8 +46,8 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; import org.immutables.value.Value; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/rel/rules/CalcRelSplitter.java b/core/src/main/java/org/apache/calcite/rel/rules/CalcRelSplitter.java index 17e6333dc223..88704f914c0b 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/CalcRelSplitter.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/CalcRelSplitter.java @@ -48,7 +48,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.primitives.Ints; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.slf4j.Logger; import java.io.PrintWriter; diff --git a/core/src/main/java/org/apache/calcite/rel/rules/CoerceInputsRule.java b/core/src/main/java/org/apache/calcite/rel/rules/CoerceInputsRule.java index c6c211cb1e4d..aa51529ad0ae 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/CoerceInputsRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/CoerceInputsRule.java @@ -24,8 +24,8 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.tools.RelBuilderFactory; -import org.checkerframework.checker.nullness.qual.Nullable; import org.immutables.value.Value; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/rel/rules/DateRangeRules.java b/core/src/main/java/org/apache/calcite/rel/rules/DateRangeRules.java index ae0d62e0fcd8..d25b579de7f0 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/DateRangeRules.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/DateRangeRules.java @@ -52,8 +52,8 @@ import com.google.common.collect.RangeSet; import com.google.common.collect.TreeRangeSet; -import org.checkerframework.checker.nullness.qual.Nullable; import org.immutables.value.Value; +import org.jspecify.annotations.Nullable; import java.util.ArrayDeque; import java.util.ArrayList; diff --git a/core/src/main/java/org/apache/calcite/rel/rules/DpHyp.java b/core/src/main/java/org/apache/calcite/rel/rules/DpHyp.java index 10402db68efb..7dfa72f463d1 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/DpHyp.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/DpHyp.java @@ -30,7 +30,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.slf4j.Logger; import java.util.ArrayList; diff --git a/core/src/main/java/org/apache/calcite/rel/rules/ExpandDisjunctionForTableRule.java b/core/src/main/java/org/apache/calcite/rel/rules/ExpandDisjunctionForTableRule.java index a48495efc275..ceaddff6afb5 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/ExpandDisjunctionForTableRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/ExpandDisjunctionForTableRule.java @@ -33,8 +33,8 @@ import com.google.common.collect.Lists; -import org.checkerframework.checker.nullness.qual.Nullable; import org.immutables.value.Value; +import org.jspecify.annotations.Nullable; import java.util.HashMap; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/rel/rules/FilterJoinRule.java b/core/src/main/java/org/apache/calcite/rel/rules/FilterJoinRule.java index d4e1473ccbfc..73e3c25daede 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/FilterJoinRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/FilterJoinRule.java @@ -39,8 +39,8 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; -import org.checkerframework.checker.nullness.qual.Nullable; import org.immutables.value.Value; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.Iterator; diff --git a/core/src/main/java/org/apache/calcite/rel/rules/FilterMultiJoinMergeRule.java b/core/src/main/java/org/apache/calcite/rel/rules/FilterMultiJoinMergeRule.java index b3d70b0eca0a..62d00f854cb6 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/FilterMultiJoinMergeRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/FilterMultiJoinMergeRule.java @@ -24,8 +24,8 @@ import org.apache.calcite.rex.RexUtil; import org.apache.calcite.tools.RelBuilderFactory; -import org.checkerframework.checker.nullness.qual.Nullable; import org.immutables.value.Value; +import org.jspecify.annotations.Nullable; import java.util.Arrays; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/rel/rules/HyperGraph.java b/core/src/main/java/org/apache/calcite/rel/rules/HyperGraph.java index 3e785a19bdfe..3f4e720be48b 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/HyperGraph.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/HyperGraph.java @@ -36,7 +36,7 @@ import com.google.common.collect.Lists; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.BitSet; diff --git a/core/src/main/java/org/apache/calcite/rel/rules/JoinCommuteRule.java b/core/src/main/java/org/apache/calcite/rel/rules/JoinCommuteRule.java index 012a00eb838b..4fd5a0e0429e 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/JoinCommuteRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/JoinCommuteRule.java @@ -35,8 +35,8 @@ import org.apache.calcite.tools.RelBuilder; import org.apache.calcite.tools.RelBuilderFactory; -import org.checkerframework.checker.nullness.qual.Nullable; import org.immutables.value.Value; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/rel/rules/JoinProjectTransposeRule.java b/core/src/main/java/org/apache/calcite/rel/rules/JoinProjectTransposeRule.java index e82f57e96b0f..f4313507ceec 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/JoinProjectTransposeRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/JoinProjectTransposeRule.java @@ -41,8 +41,8 @@ import org.apache.calcite.tools.RelBuilderFactory; import org.apache.calcite.util.Pair; -import org.checkerframework.checker.nullness.qual.Nullable; import org.immutables.value.Value; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.Collections; diff --git a/core/src/main/java/org/apache/calcite/rel/rules/JoinToMultiJoinRule.java b/core/src/main/java/org/apache/calcite/rel/rules/JoinToMultiJoinRule.java index 8de5032bbe01..ec34f6e619d1 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/JoinToMultiJoinRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/JoinToMultiJoinRule.java @@ -36,8 +36,8 @@ import com.google.common.collect.ImmutableMap; -import org.checkerframework.checker.nullness.qual.Nullable; import org.immutables.value.Value; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.Arrays; diff --git a/core/src/main/java/org/apache/calcite/rel/rules/LoptMultiJoin.java b/core/src/main/java/org/apache/calcite/rel/rules/LoptMultiJoin.java index f76f74e75e5a..ae6a91d9827e 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/LoptMultiJoin.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/LoptMultiJoin.java @@ -38,8 +38,8 @@ import org.checkerframework.checker.initialization.qual.UnderInitialization; import org.checkerframework.checker.initialization.qual.UnknownInitialization; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; -import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.checker.nullness.qual.RequiresNonNull; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.BitSet; diff --git a/core/src/main/java/org/apache/calcite/rel/rules/LoptOptimizeJoinRule.java b/core/src/main/java/org/apache/calcite/rel/rules/LoptOptimizeJoinRule.java index 16cb367ac2fe..662ada2bf637 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/LoptOptimizeJoinRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/LoptOptimizeJoinRule.java @@ -48,8 +48,8 @@ import org.apache.calcite.util.mapping.IntPair; import org.checkerframework.checker.nullness.qual.KeyFor; -import org.checkerframework.checker.nullness.qual.Nullable; import org.immutables.value.Value; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.BitSet; diff --git a/core/src/main/java/org/apache/calcite/rel/rules/LoptSemiJoinOptimizer.java b/core/src/main/java/org/apache/calcite/rel/rules/LoptSemiJoinOptimizer.java index 724884641dec..fb5914183773 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/LoptSemiJoinOptimizer.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/LoptSemiJoinOptimizer.java @@ -42,7 +42,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Ordering; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.Comparator; diff --git a/core/src/main/java/org/apache/calcite/rel/rules/MultiJoin.java b/core/src/main/java/org/apache/calcite/rel/rules/MultiJoin.java index 49000fe2fb65..1a74c810bcd2 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/MultiJoin.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/MultiJoin.java @@ -35,7 +35,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.HashMap; diff --git a/core/src/main/java/org/apache/calcite/rel/rules/MultiJoinOptimizeBushyRule.java b/core/src/main/java/org/apache/calcite/rel/rules/MultiJoinOptimizeBushyRule.java index 6336b997e742..f6e8abd03700 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/MultiJoinOptimizeBushyRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/MultiJoinOptimizeBushyRule.java @@ -38,8 +38,8 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; import org.immutables.value.Value; +import org.jspecify.annotations.Nullable; import java.io.PrintWriter; import java.util.ArrayList; diff --git a/core/src/main/java/org/apache/calcite/rel/rules/OuterJoinToAntiJoinRule.java b/core/src/main/java/org/apache/calcite/rel/rules/OuterJoinToAntiJoinRule.java index 2b005633704d..93f09393254d 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/OuterJoinToAntiJoinRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/OuterJoinToAntiJoinRule.java @@ -35,8 +35,8 @@ import org.apache.calcite.tools.RelBuilder; import org.apache.calcite.util.ImmutableBitSet; -import org.checkerframework.checker.nullness.qual.Nullable; import org.immutables.value.Value; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/rel/rules/ProjectFilterTransposeRule.java b/core/src/main/java/org/apache/calcite/rel/rules/ProjectFilterTransposeRule.java index ad8a58d8fe59..c449e9126efc 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/ProjectFilterTransposeRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/ProjectFilterTransposeRule.java @@ -36,8 +36,8 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import org.checkerframework.checker.nullness.qual.Nullable; import org.immutables.value.Value; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.LinkedHashSet; diff --git a/core/src/main/java/org/apache/calcite/rel/rules/PushProjector.java b/core/src/main/java/org/apache/calcite/rel/rules/PushProjector.java index 9dc5172896cb..2bbcb54235b5 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/PushProjector.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/PushProjector.java @@ -44,7 +44,7 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.BitSet; diff --git a/core/src/main/java/org/apache/calcite/rel/rules/ReduceDecimalsRule.java b/core/src/main/java/org/apache/calcite/rel/rules/ReduceDecimalsRule.java index 8918bf55f9e8..54aba9f91aba 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/ReduceDecimalsRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/ReduceDecimalsRule.java @@ -44,8 +44,8 @@ import com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; -import org.checkerframework.checker.nullness.qual.Nullable; import org.immutables.value.Value; +import org.jspecify.annotations.Nullable; import java.math.BigDecimal; import java.math.BigInteger; diff --git a/core/src/main/java/org/apache/calcite/rel/rules/ReduceExpressionsRule.java b/core/src/main/java/org/apache/calcite/rel/rules/ReduceExpressionsRule.java index db93802118b6..3fc1a6fcd614 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/ReduceExpressionsRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/ReduceExpressionsRule.java @@ -75,8 +75,8 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; -import org.checkerframework.checker.nullness.qual.Nullable; import org.immutables.value.Value; +import org.jspecify.annotations.Nullable; import java.util.ArrayDeque; import java.util.ArrayList; diff --git a/core/src/main/java/org/apache/calcite/rel/rules/SemiJoinRule.java b/core/src/main/java/org/apache/calcite/rel/rules/SemiJoinRule.java index 8b011ed08bf1..d78c367d12a4 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/SemiJoinRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/SemiJoinRule.java @@ -34,8 +34,8 @@ import org.apache.calcite.util.ImmutableBitSet; import org.apache.calcite.util.ImmutableIntList; -import org.checkerframework.checker.nullness.qual.Nullable; import org.immutables.value.Value; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/rel/rules/SetOpToFilterRule.java b/core/src/main/java/org/apache/calcite/rel/rules/SetOpToFilterRule.java index 848cc6c797fa..7f4ca4e25165 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/SetOpToFilterRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/SetOpToFilterRule.java @@ -30,8 +30,8 @@ import org.apache.calcite.tools.RelBuilder; import org.apache.calcite.util.Pair; -import org.checkerframework.checker.nullness.qual.Nullable; import org.immutables.value.Value; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.LinkedHashMap; diff --git a/core/src/main/java/org/apache/calcite/rel/rules/SingleValuesOptimizationRules.java b/core/src/main/java/org/apache/calcite/rel/rules/SingleValuesOptimizationRules.java index d0e53be44d12..554dc37153b8 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/SingleValuesOptimizationRules.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/SingleValuesOptimizationRules.java @@ -36,8 +36,8 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; import org.immutables.value.Value; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/rel/rules/SortJoinTransposeRule.java b/core/src/main/java/org/apache/calcite/rel/rules/SortJoinTransposeRule.java index dfb2d8401d7c..0a7493f63739 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/SortJoinTransposeRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/SortJoinTransposeRule.java @@ -34,8 +34,8 @@ import org.apache.calcite.rex.RexNode; import org.apache.calcite.tools.RelBuilderFactory; -import org.checkerframework.checker.nullness.qual.Nullable; import org.immutables.value.Value; +import org.jspecify.annotations.Nullable; import java.math.BigDecimal; diff --git a/core/src/main/java/org/apache/calcite/rel/rules/SpatialRules.java b/core/src/main/java/org/apache/calcite/rel/rules/SpatialRules.java index 1b003de36b47..ef943c04b502 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/SpatialRules.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/SpatialRules.java @@ -37,8 +37,8 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; import org.immutables.value.Value; +import org.jspecify.annotations.Nullable; import org.locationtech.jts.geom.Envelope; import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.geom.Point; diff --git a/core/src/main/java/org/apache/calcite/rel/rules/ValuesReduceRule.java b/core/src/main/java/org/apache/calcite/rel/rules/ValuesReduceRule.java index acd80a81d977..514c39c8508b 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/ValuesReduceRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/ValuesReduceRule.java @@ -38,8 +38,8 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; import org.immutables.value.Value; +import org.jspecify.annotations.Nullable; import org.slf4j.Logger; import java.util.ArrayList; diff --git a/core/src/main/java/org/apache/calcite/rel/rules/materialize/MaterializedViewAggregateRule.java b/core/src/main/java/org/apache/calcite/rel/rules/materialize/MaterializedViewAggregateRule.java index 538723661d60..15f3618728a3 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/materialize/MaterializedViewAggregateRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/materialize/MaterializedViewAggregateRule.java @@ -66,8 +66,8 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Multimap; -import org.checkerframework.checker.nullness.qual.Nullable; import org.immutables.value.Value; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.Collection; diff --git a/core/src/main/java/org/apache/calcite/rel/rules/materialize/MaterializedViewJoinRule.java b/core/src/main/java/org/apache/calcite/rel/rules/materialize/MaterializedViewJoinRule.java index 818b784c4a34..e84ffddc6a92 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/materialize/MaterializedViewJoinRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/materialize/MaterializedViewJoinRule.java @@ -39,7 +39,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Multimap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.Collection; diff --git a/core/src/main/java/org/apache/calcite/rel/rules/materialize/MaterializedViewRule.java b/core/src/main/java/org/apache/calcite/rel/rules/materialize/MaterializedViewRule.java index 0ce1de86da65..20215f7796f5 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/materialize/MaterializedViewRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/materialize/MaterializedViewRule.java @@ -64,7 +64,7 @@ import com.google.common.collect.Multimap; import com.google.common.collect.Sets; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.Collection; diff --git a/core/src/main/java/org/apache/calcite/rel/type/DelegatingTypeSystem.java b/core/src/main/java/org/apache/calcite/rel/type/DelegatingTypeSystem.java index ec220e8ef7e8..fff71064b979 100644 --- a/core/src/main/java/org/apache/calcite/rel/type/DelegatingTypeSystem.java +++ b/core/src/main/java/org/apache/calcite/rel/type/DelegatingTypeSystem.java @@ -18,7 +18,7 @@ import org.apache.calcite.sql.type.SqlTypeName; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.math.RoundingMode; diff --git a/core/src/main/java/org/apache/calcite/rel/type/DynamicRecordTypeImpl.java b/core/src/main/java/org/apache/calcite/rel/type/DynamicRecordTypeImpl.java index 22084195b4ac..cd601e64c352 100644 --- a/core/src/main/java/org/apache/calcite/rel/type/DynamicRecordTypeImpl.java +++ b/core/src/main/java/org/apache/calcite/rel/type/DynamicRecordTypeImpl.java @@ -23,7 +23,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/rel/type/RelDataType.java b/core/src/main/java/org/apache/calcite/rel/type/RelDataType.java index 071a1152a108..50c212d16c74 100644 --- a/core/src/main/java/org/apache/calcite/rel/type/RelDataType.java +++ b/core/src/main/java/org/apache/calcite/rel/type/RelDataType.java @@ -24,8 +24,8 @@ import org.apache.calcite.sql.type.SqlTypeUtil; import org.apiguardian.api.API; -import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.dataflow.qual.Pure; +import org.jspecify.annotations.Nullable; import java.nio.charset.Charset; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeFactory.java b/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeFactory.java index 5912f78ac7cf..5d97590edc21 100644 --- a/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeFactory.java +++ b/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeFactory.java @@ -25,7 +25,7 @@ import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.sql.validate.SqlValidatorUtil; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.nio.charset.Charset; import java.util.ArrayList; diff --git a/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeFactoryImpl.java b/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeFactoryImpl.java index c0d90578fa7a..42d99c140a77 100644 --- a/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeFactoryImpl.java +++ b/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeFactoryImpl.java @@ -38,7 +38,7 @@ import com.google.common.collect.Interner; import com.google.common.collect.Interners; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Field; import java.nio.charset.Charset; diff --git a/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeFieldImpl.java b/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeFieldImpl.java index f11f9ec24c98..759451086d98 100644 --- a/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeFieldImpl.java +++ b/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeFieldImpl.java @@ -18,7 +18,7 @@ import org.apache.calcite.sql.type.SqlTypeName; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.io.Serializable; import java.util.Objects; diff --git a/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeImpl.java b/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeImpl.java index d896ece7e9bd..b4e0c32345b9 100644 --- a/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeImpl.java +++ b/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeImpl.java @@ -30,7 +30,7 @@ import com.google.common.collect.Iterables; import org.checkerframework.checker.initialization.qual.UnknownInitialization; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.io.Serializable; import java.nio.charset.Charset; diff --git a/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeSystem.java b/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeSystem.java index c9d6186747be..54a6d454865a 100644 --- a/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeSystem.java +++ b/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeSystem.java @@ -20,7 +20,7 @@ import org.apache.calcite.sql.type.SqlTypeUtil; import org.apache.calcite.util.Glossary; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.math.RoundingMode; diff --git a/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeSystemImpl.java b/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeSystemImpl.java index 9bd698d8303a..f790135ac512 100644 --- a/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeSystemImpl.java +++ b/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeSystemImpl.java @@ -22,7 +22,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.math.RoundingMode; diff --git a/core/src/main/java/org/apache/calcite/rel/type/RelRecordType.java b/core/src/main/java/org/apache/calcite/rel/type/RelRecordType.java index a63b107ca0cb..5af790d01af1 100644 --- a/core/src/main/java/org/apache/calcite/rel/type/RelRecordType.java +++ b/core/src/main/java/org/apache/calcite/rel/type/RelRecordType.java @@ -21,7 +21,7 @@ import com.google.common.collect.ImmutableMap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.io.Serializable; import java.util.HashMap; diff --git a/core/src/main/java/org/apache/calcite/rel/type/SingleColumnAliasRelDataType.java b/core/src/main/java/org/apache/calcite/rel/type/SingleColumnAliasRelDataType.java index 28e0406f6717..8839ce1ab4f3 100644 --- a/core/src/main/java/org/apache/calcite/rel/type/SingleColumnAliasRelDataType.java +++ b/core/src/main/java/org/apache/calcite/rel/type/SingleColumnAliasRelDataType.java @@ -21,7 +21,7 @@ import org.apache.calcite.sql.SqlIntervalQualifier; import org.apache.calcite.sql.type.SqlTypeName; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.nio.charset.Charset; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/rel/type/TimeFrame.java b/core/src/main/java/org/apache/calcite/rel/type/TimeFrame.java index 1ea43217d76a..8274632f37cd 100644 --- a/core/src/main/java/org/apache/calcite/rel/type/TimeFrame.java +++ b/core/src/main/java/org/apache/calcite/rel/type/TimeFrame.java @@ -20,7 +20,7 @@ import org.apache.commons.math3.fraction.BigFraction; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** Time frame. * diff --git a/core/src/main/java/org/apache/calcite/rel/type/TimeFrameSet.java b/core/src/main/java/org/apache/calcite/rel/type/TimeFrameSet.java index 07869b8504a5..e9c5fb27b64c 100644 --- a/core/src/main/java/org/apache/calcite/rel/type/TimeFrameSet.java +++ b/core/src/main/java/org/apache/calcite/rel/type/TimeFrameSet.java @@ -29,7 +29,7 @@ import com.google.common.collect.ImmutableMultimap; import com.google.common.collect.Iterables; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.math.BigInteger; import java.util.NavigableMap; diff --git a/core/src/main/java/org/apache/calcite/rel/type/TimeFrames.java b/core/src/main/java/org/apache/calcite/rel/type/TimeFrames.java index f69e66f3c6e4..762dc5ca3194 100644 --- a/core/src/main/java/org/apache/calcite/rel/type/TimeFrames.java +++ b/core/src/main/java/org/apache/calcite/rel/type/TimeFrames.java @@ -31,7 +31,7 @@ import com.google.common.collect.ImmutableMultimap; import com.google.common.collect.Iterables; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.math.BigInteger; import java.util.Calendar; diff --git a/core/src/main/java/org/apache/calcite/rex/LogicVisitor.java b/core/src/main/java/org/apache/calcite/rex/LogicVisitor.java index b18613b104aa..8467a6a3ed87 100644 --- a/core/src/main/java/org/apache/calcite/rex/LogicVisitor.java +++ b/core/src/main/java/org/apache/calcite/rex/LogicVisitor.java @@ -20,7 +20,7 @@ import com.google.common.collect.Iterables; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Collection; import java.util.Collections; diff --git a/core/src/main/java/org/apache/calcite/rex/RexBiVisitorImpl.java b/core/src/main/java/org/apache/calcite/rex/RexBiVisitorImpl.java index 0c8f220bff29..527da65b52b7 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexBiVisitorImpl.java +++ b/core/src/main/java/org/apache/calcite/rex/RexBiVisitorImpl.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.rex; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Default implementation of {@link RexBiVisitor}, which visits each node but diff --git a/core/src/main/java/org/apache/calcite/rex/RexBuilder.java b/core/src/main/java/org/apache/calcite/rex/RexBuilder.java index efcb80f20367..b146de2042cd 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexBuilder.java +++ b/core/src/main/java/org/apache/calcite/rex/RexBuilder.java @@ -67,8 +67,8 @@ import com.google.common.collect.RangeSet; import com.google.common.collect.TreeRangeSet; -import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.checker.nullness.qual.PolyNull; +import org.jspecify.annotations.Nullable; import org.locationtech.jts.geom.Geometry; import java.math.BigDecimal; diff --git a/core/src/main/java/org/apache/calcite/rex/RexCall.java b/core/src/main/java/org/apache/calcite/rex/RexCall.java index 27d93189c527..e2bfce84d65f 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexCall.java +++ b/core/src/main/java/org/apache/calcite/rex/RexCall.java @@ -30,7 +30,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/rex/RexCallBinding.java b/core/src/main/java/org/apache/calcite/rex/RexCallBinding.java index 66df15f55f65..4b9280a2acd2 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexCallBinding.java +++ b/core/src/main/java/org/apache/calcite/rex/RexCallBinding.java @@ -31,7 +31,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/rex/RexChecker.java b/core/src/main/java/org/apache/calcite/rex/RexChecker.java index 467910218984..590058ebdda1 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexChecker.java +++ b/core/src/main/java/org/apache/calcite/rex/RexChecker.java @@ -22,7 +22,7 @@ import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.util.Litmus; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/rex/RexCorrelVariable.java b/core/src/main/java/org/apache/calcite/rex/RexCorrelVariable.java index c0ea81a48542..ee576085f73b 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexCorrelVariable.java +++ b/core/src/main/java/org/apache/calcite/rex/RexCorrelVariable.java @@ -20,7 +20,7 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.sql.SqlKind; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Objects; diff --git a/core/src/main/java/org/apache/calcite/rex/RexDynamicParam.java b/core/src/main/java/org/apache/calcite/rex/RexDynamicParam.java index 105b65324cfc..10269b5729f2 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexDynamicParam.java +++ b/core/src/main/java/org/apache/calcite/rex/RexDynamicParam.java @@ -19,7 +19,7 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.sql.SqlKind; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Objects; diff --git a/core/src/main/java/org/apache/calcite/rex/RexExecutable.java b/core/src/main/java/org/apache/calcite/rex/RexExecutable.java index 8828654c24fe..ae2a80521450 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexExecutable.java +++ b/core/src/main/java/org/apache/calcite/rex/RexExecutable.java @@ -22,10 +22,10 @@ import org.apache.calcite.runtime.Utilities; import org.apache.calcite.util.Pair; -import org.checkerframework.checker.nullness.qual.Nullable; import org.codehaus.commons.compiler.CompileException; import org.codehaus.janino.ClassBodyEvaluator; import org.codehaus.janino.Scanner; +import org.jspecify.annotations.Nullable; import java.io.IOException; import java.io.Serializable; diff --git a/core/src/main/java/org/apache/calcite/rex/RexExecutorImpl.java b/core/src/main/java/org/apache/calcite/rex/RexExecutorImpl.java index 7f06abf58c2f..12c82cfc1ace 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexExecutorImpl.java +++ b/core/src/main/java/org/apache/calcite/rex/RexExecutorImpl.java @@ -41,7 +41,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Modifier; import java.lang.reflect.Type; diff --git a/core/src/main/java/org/apache/calcite/rex/RexFieldAccess.java b/core/src/main/java/org/apache/calcite/rex/RexFieldAccess.java index dbc61ca152a7..da42d57aca43 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexFieldAccess.java +++ b/core/src/main/java/org/apache/calcite/rex/RexFieldAccess.java @@ -20,7 +20,7 @@ import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.sql.SqlKind; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import static com.google.common.base.Preconditions.checkArgument; diff --git a/core/src/main/java/org/apache/calcite/rex/RexInputRef.java b/core/src/main/java/org/apache/calcite/rex/RexInputRef.java index 606ac807ccdb..98f50214fe35 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexInputRef.java +++ b/core/src/main/java/org/apache/calcite/rex/RexInputRef.java @@ -22,7 +22,7 @@ import org.apache.calcite.sql.SqlKind; import org.apache.calcite.util.Pair; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/rex/RexInterpreter.java b/core/src/main/java/org/apache/calcite/rex/RexInterpreter.java index 549efa5387f6..53a9378db2e4 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexInterpreter.java +++ b/core/src/main/java/org/apache/calcite/rex/RexInterpreter.java @@ -34,7 +34,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.RangeSet; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.math.BigDecimal; import java.math.BigInteger; diff --git a/core/src/main/java/org/apache/calcite/rex/RexLambda.java b/core/src/main/java/org/apache/calcite/rex/RexLambda.java index 2803b84b782d..fae0c4184b93 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexLambda.java +++ b/core/src/main/java/org/apache/calcite/rex/RexLambda.java @@ -22,7 +22,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; import java.util.Objects; diff --git a/core/src/main/java/org/apache/calcite/rex/RexLambdaRef.java b/core/src/main/java/org/apache/calcite/rex/RexLambdaRef.java index 8a8784609476..092154f91b7f 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexLambdaRef.java +++ b/core/src/main/java/org/apache/calcite/rex/RexLambdaRef.java @@ -19,7 +19,7 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.sql.SqlKind; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Objects; diff --git a/core/src/main/java/org/apache/calcite/rex/RexLiteral.java b/core/src/main/java/org/apache/calcite/rex/RexLiteral.java index 545e81995d0d..1dfb332fc03d 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexLiteral.java +++ b/core/src/main/java/org/apache/calcite/rex/RexLiteral.java @@ -48,10 +48,10 @@ import com.google.common.collect.ImmutableList; import org.checkerframework.checker.initialization.qual.UnknownInitialization; -import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.checker.nullness.qual.PolyNull; import org.checkerframework.checker.nullness.qual.RequiresNonNull; import org.checkerframework.dataflow.qual.Pure; +import org.jspecify.annotations.Nullable; import org.locationtech.jts.geom.Geometry; import java.io.PrintWriter; diff --git a/core/src/main/java/org/apache/calcite/rex/RexLocalRef.java b/core/src/main/java/org/apache/calcite/rex/RexLocalRef.java index 81092bcd630e..d6a337389435 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexLocalRef.java +++ b/core/src/main/java/org/apache/calcite/rex/RexLocalRef.java @@ -19,7 +19,7 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.sql.SqlKind; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; import java.util.Objects; diff --git a/core/src/main/java/org/apache/calcite/rex/RexMultisetUtil.java b/core/src/main/java/org/apache/calcite/rex/RexMultisetUtil.java index 2534ed4d8a84..47252a1cde5a 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexMultisetUtil.java +++ b/core/src/main/java/org/apache/calcite/rex/RexMultisetUtil.java @@ -22,7 +22,7 @@ import com.google.common.collect.ImmutableSet; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; import java.util.Set; diff --git a/core/src/main/java/org/apache/calcite/rex/RexNode.java b/core/src/main/java/org/apache/calcite/rex/RexNode.java index 9db8c867a598..4440a6c31852 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexNode.java +++ b/core/src/main/java/org/apache/calcite/rex/RexNode.java @@ -20,7 +20,7 @@ import org.apache.calcite.sql.SqlKind; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Collection; diff --git a/core/src/main/java/org/apache/calcite/rex/RexNodeAndFieldIndex.java b/core/src/main/java/org/apache/calcite/rex/RexNodeAndFieldIndex.java index b6b5e6c72d28..f84f0d8dbe8b 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexNodeAndFieldIndex.java +++ b/core/src/main/java/org/apache/calcite/rex/RexNodeAndFieldIndex.java @@ -18,7 +18,7 @@ import org.apache.calcite.rel.type.RelDataType; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Objects; diff --git a/core/src/main/java/org/apache/calcite/rex/RexOver.java b/core/src/main/java/org/apache/calcite/rex/RexOver.java index 0b01838b385a..4770ae40b460 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexOver.java +++ b/core/src/main/java/org/apache/calcite/rex/RexOver.java @@ -23,7 +23,7 @@ import org.apache.calcite.util.ControlFlowException; import org.apache.calcite.util.Util; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; import java.util.Objects; diff --git a/core/src/main/java/org/apache/calcite/rex/RexPermuteInputsShuttle.java b/core/src/main/java/org/apache/calcite/rex/RexPermuteInputsShuttle.java index 0906a6610da7..4f019a2dc51a 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexPermuteInputsShuttle.java +++ b/core/src/main/java/org/apache/calcite/rex/RexPermuteInputsShuttle.java @@ -22,7 +22,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/rex/RexProgram.java b/core/src/main/java/org/apache/calcite/rex/RexProgram.java index 9c391cd7f898..53b3f846471b 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexProgram.java +++ b/core/src/main/java/org/apache/calcite/rex/RexProgram.java @@ -43,8 +43,8 @@ import org.checkerframework.checker.initialization.qual.UnknownInitialization; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; -import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.dataflow.qual.Pure; +import org.jspecify.annotations.Nullable; import java.io.PrintWriter; import java.io.StringWriter; diff --git a/core/src/main/java/org/apache/calcite/rex/RexProgramBuilder.java b/core/src/main/java/org/apache/calcite/rex/RexProgramBuilder.java index ebeb12aa609e..37a9c472decc 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexProgramBuilder.java +++ b/core/src/main/java/org/apache/calcite/rex/RexProgramBuilder.java @@ -24,7 +24,7 @@ import org.apache.calcite.util.Litmus; import org.apache.calcite.util.Pair; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.HashMap; diff --git a/core/src/main/java/org/apache/calcite/rex/RexRangeRef.java b/core/src/main/java/org/apache/calcite/rex/RexRangeRef.java index 1abac5d19aee..8d2a06bc3bd5 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexRangeRef.java +++ b/core/src/main/java/org/apache/calcite/rex/RexRangeRef.java @@ -18,7 +18,7 @@ import org.apache.calcite.rel.type.RelDataType; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Objects; diff --git a/core/src/main/java/org/apache/calcite/rex/RexShuttle.java b/core/src/main/java/org/apache/calcite/rex/RexShuttle.java index d99ae12630fa..e28db54864ab 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexShuttle.java +++ b/core/src/main/java/org/apache/calcite/rex/RexShuttle.java @@ -20,8 +20,8 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.checker.nullness.qual.PolyNull; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/rex/RexSimplify.java b/core/src/main/java/org/apache/calcite/rex/RexSimplify.java index fe9f7b5ca06e..8696a57e7bb2 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexSimplify.java +++ b/core/src/main/java/org/apache/calcite/rex/RexSimplify.java @@ -53,7 +53,7 @@ import com.google.common.collect.Sets; import com.google.common.collect.TreeRangeSet; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.math.BigDecimal; import java.util.ArrayList; diff --git a/core/src/main/java/org/apache/calcite/rex/RexSqlConvertlet.java b/core/src/main/java/org/apache/calcite/rex/RexSqlConvertlet.java index 8db99b3a2cc1..809c7ffcd9a1 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexSqlConvertlet.java +++ b/core/src/main/java/org/apache/calcite/rex/RexSqlConvertlet.java @@ -18,7 +18,7 @@ import org.apache.calcite.sql.SqlNode; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Converts a {@link RexNode} expression into a {@link SqlNode} expression. diff --git a/core/src/main/java/org/apache/calcite/rex/RexSqlConvertletTable.java b/core/src/main/java/org/apache/calcite/rex/RexSqlConvertletTable.java index 05ed1c6c567a..f51287bfb5ee 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexSqlConvertletTable.java +++ b/core/src/main/java/org/apache/calcite/rex/RexSqlConvertletTable.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.rex; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Collection of {@link RexSqlConvertlet}s. diff --git a/core/src/main/java/org/apache/calcite/rex/RexSqlReflectiveConvertletTable.java b/core/src/main/java/org/apache/calcite/rex/RexSqlReflectiveConvertletTable.java index fdaf3eb83dce..7c7cea2d2e53 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexSqlReflectiveConvertletTable.java +++ b/core/src/main/java/org/apache/calcite/rex/RexSqlReflectiveConvertletTable.java @@ -18,7 +18,7 @@ import org.apache.calcite.sql.SqlOperator; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.HashMap; import java.util.Map; diff --git a/core/src/main/java/org/apache/calcite/rex/RexSqlStandardConvertletTable.java b/core/src/main/java/org/apache/calcite/rex/RexSqlStandardConvertletTable.java index 18b3c9de68a0..5dec70be8417 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexSqlStandardConvertletTable.java +++ b/core/src/main/java/org/apache/calcite/rex/RexSqlStandardConvertletTable.java @@ -28,7 +28,7 @@ import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.sql.type.SqlTypeUtil; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/rex/RexSubQuery.java b/core/src/main/java/org/apache/calcite/rex/RexSubQuery.java index 5a9cad2a970b..9bf31c32bffe 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexSubQuery.java +++ b/core/src/main/java/org/apache/calcite/rex/RexSubQuery.java @@ -30,7 +30,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; import java.util.Objects; diff --git a/core/src/main/java/org/apache/calcite/rex/RexTableInputRef.java b/core/src/main/java/org/apache/calcite/rex/RexTableInputRef.java index a27c73273cbe..51adef917eca 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexTableInputRef.java +++ b/core/src/main/java/org/apache/calcite/rex/RexTableInputRef.java @@ -20,7 +20,7 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.sql.SqlKind; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; import java.util.Objects; diff --git a/core/src/main/java/org/apache/calcite/rex/RexToSqlNodeConverter.java b/core/src/main/java/org/apache/calcite/rex/RexToSqlNodeConverter.java index f5d6b3c078a1..0b470ffb5754 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexToSqlNodeConverter.java +++ b/core/src/main/java/org/apache/calcite/rex/RexToSqlNodeConverter.java @@ -20,7 +20,7 @@ import org.apache.calcite.sql.SqlLiteral; import org.apache.calcite.sql.SqlNode; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Converts expressions from {@link RexNode} to {@link SqlNode}. diff --git a/core/src/main/java/org/apache/calcite/rex/RexToSqlNodeConverterImpl.java b/core/src/main/java/org/apache/calcite/rex/RexToSqlNodeConverterImpl.java index d18d7745ae24..2f0f1eadd67d 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexToSqlNodeConverterImpl.java +++ b/core/src/main/java/org/apache/calcite/rex/RexToSqlNodeConverterImpl.java @@ -25,7 +25,7 @@ import org.apache.calcite.util.TimeString; import org.apache.calcite.util.TimestampString; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import static java.util.Objects.requireNonNull; diff --git a/core/src/main/java/org/apache/calcite/rex/RexUnaryBiVisitor.java b/core/src/main/java/org/apache/calcite/rex/RexUnaryBiVisitor.java index e59e3b06d921..4df853145e0a 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexUnaryBiVisitor.java +++ b/core/src/main/java/org/apache/calcite/rex/RexUnaryBiVisitor.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.rex; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Default implementation of a {@link RexBiVisitor} whose payload and return diff --git a/core/src/main/java/org/apache/calcite/rex/RexUtil.java b/core/src/main/java/org/apache/calcite/rex/RexUtil.java index 99045f0a3cb6..b262194ca624 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexUtil.java +++ b/core/src/main/java/org/apache/calcite/rex/RexUtil.java @@ -63,7 +63,7 @@ import com.google.common.collect.Range; import org.apiguardian.api.API; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.math.BigDecimal; import java.math.RoundingMode; diff --git a/core/src/main/java/org/apache/calcite/rex/RexVisitorImpl.java b/core/src/main/java/org/apache/calcite/rex/RexVisitorImpl.java index d6d5931d9768..d1c3a911a542 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexVisitorImpl.java +++ b/core/src/main/java/org/apache/calcite/rex/RexVisitorImpl.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.rex; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/rex/RexWindow.java b/core/src/main/java/org/apache/calcite/rex/RexWindow.java index 3c39f4706443..11b6df2e7d57 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexWindow.java +++ b/core/src/main/java/org/apache/calcite/rex/RexWindow.java @@ -20,7 +20,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/rex/RexWindowBound.java b/core/src/main/java/org/apache/calcite/rex/RexWindowBound.java index 0072db9c1a53..58f0c159c89d 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexWindowBound.java +++ b/core/src/main/java/org/apache/calcite/rex/RexWindowBound.java @@ -19,8 +19,8 @@ import org.apache.calcite.sql.SqlNode; import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf; -import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.dataflow.qual.Pure; +import org.jspecify.annotations.Nullable; /** * Abstracts "XX PRECEDING/FOLLOWING" and "CURRENT ROW" bounds for windowed diff --git a/core/src/main/java/org/apache/calcite/rex/RexWindowBounds.java b/core/src/main/java/org/apache/calcite/rex/RexWindowBounds.java index 37c209e1abd7..7ae667b73666 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexWindowBounds.java +++ b/core/src/main/java/org/apache/calcite/rex/RexWindowBounds.java @@ -22,7 +22,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Objects; diff --git a/core/src/main/java/org/apache/calcite/rex/package-info.java b/core/src/main/java/org/apache/calcite/rex/package-info.java index 4467d8c65ca7..be7479293d29 100644 --- a/core/src/main/java/org/apache/calcite/rex/package-info.java +++ b/core/src/main/java/org/apache/calcite/rex/package-info.java @@ -83,6 +83,6 @@ @DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.RETURN) package org.apache.calcite.rex; -import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.framework.qual.DefaultQualifier; import org.checkerframework.framework.qual.TypeUseLocation; +import org.jspecify.annotations.NonNull; diff --git a/core/src/main/java/org/apache/calcite/runtime/AbstractImmutableList.java b/core/src/main/java/org/apache/calcite/runtime/AbstractImmutableList.java index 54e2092d1575..d438a025b288 100644 --- a/core/src/main/java/org/apache/calcite/runtime/AbstractImmutableList.java +++ b/core/src/main/java/org/apache/calcite/runtime/AbstractImmutableList.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.runtime; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Collection; import java.util.Iterator; diff --git a/core/src/main/java/org/apache/calcite/runtime/ArrayBindable.java b/core/src/main/java/org/apache/calcite/runtime/ArrayBindable.java index 52815518a27e..48173e1a048d 100644 --- a/core/src/main/java/org/apache/calcite/runtime/ArrayBindable.java +++ b/core/src/main/java/org/apache/calcite/runtime/ArrayBindable.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.runtime; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Extension to {@link Bindable} that returns rows that are arrays of objects. diff --git a/core/src/main/java/org/apache/calcite/runtime/ArrayEnumeratorCursor.java b/core/src/main/java/org/apache/calcite/runtime/ArrayEnumeratorCursor.java index 219df429ffcf..d30967b04a38 100644 --- a/core/src/main/java/org/apache/calcite/runtime/ArrayEnumeratorCursor.java +++ b/core/src/main/java/org/apache/calcite/runtime/ArrayEnumeratorCursor.java @@ -18,7 +18,7 @@ import org.apache.calcite.linq4j.Enumerator; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Implementation of {@link org.apache.calcite.avatica.util.Cursor} on top of an diff --git a/core/src/main/java/org/apache/calcite/runtime/Automaton.java b/core/src/main/java/org/apache/calcite/runtime/Automaton.java index 86832cfc4000..abc58d70a8f3 100644 --- a/core/src/main/java/org/apache/calcite/runtime/Automaton.java +++ b/core/src/main/java/org/apache/calcite/runtime/Automaton.java @@ -21,7 +21,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Objects; diff --git a/core/src/main/java/org/apache/calcite/runtime/CalciteContextException.java b/core/src/main/java/org/apache/calcite/runtime/CalciteContextException.java index 3c00f8c633b1..53b4f363e101 100644 --- a/core/src/main/java/org/apache/calcite/runtime/CalciteContextException.java +++ b/core/src/main/java/org/apache/calcite/runtime/CalciteContextException.java @@ -21,7 +21,7 @@ // dependencies on other Calcite code. import org.checkerframework.checker.initialization.qual.UnknownInitialization; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import static java.util.Objects.requireNonNull; diff --git a/core/src/main/java/org/apache/calcite/runtime/CalciteException.java b/core/src/main/java/org/apache/calcite/runtime/CalciteException.java index fb491b9f0c2a..f417a413b975 100644 --- a/core/src/main/java/org/apache/calcite/runtime/CalciteException.java +++ b/core/src/main/java/org/apache/calcite/runtime/CalciteException.java @@ -18,7 +18,7 @@ import org.apache.calcite.config.CalciteSystemProperty; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java b/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java index cc27cb5c2005..2a0a0cac8a5d 100644 --- a/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java +++ b/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java @@ -18,7 +18,7 @@ import org.apache.calcite.sql.validate.SqlValidatorException; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import static org.apache.calcite.runtime.Resources.BaseMessage; import static org.apache.calcite.runtime.Resources.ExInst; diff --git a/core/src/main/java/org/apache/calcite/runtime/CompressionFunctions.java b/core/src/main/java/org/apache/calcite/runtime/CompressionFunctions.java index 59bac5d5fd6c..2dbedea77330 100644 --- a/core/src/main/java/org/apache/calcite/runtime/CompressionFunctions.java +++ b/core/src/main/java/org/apache/calcite/runtime/CompressionFunctions.java @@ -18,7 +18,7 @@ import org.apache.calcite.avatica.util.ByteString; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.io.ByteArrayOutputStream; import java.io.IOException; diff --git a/core/src/main/java/org/apache/calcite/runtime/ConsList.java b/core/src/main/java/org/apache/calcite/runtime/ConsList.java index f564a606f346..1954a6526b81 100644 --- a/core/src/main/java/org/apache/calcite/runtime/ConsList.java +++ b/core/src/main/java/org/apache/calcite/runtime/ConsList.java @@ -18,8 +18,8 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.checker.nullness.qual.PolyNull; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.Arrays; diff --git a/core/src/main/java/org/apache/calcite/runtime/DeterministicAutomaton.java b/core/src/main/java/org/apache/calcite/runtime/DeterministicAutomaton.java index 6bdaefb7821d..0f2363234935 100644 --- a/core/src/main/java/org/apache/calcite/runtime/DeterministicAutomaton.java +++ b/core/src/main/java/org/apache/calcite/runtime/DeterministicAutomaton.java @@ -19,7 +19,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.HashSet; import java.util.Objects; diff --git a/core/src/main/java/org/apache/calcite/runtime/Enumerables.java b/core/src/main/java/org/apache/calcite/runtime/Enumerables.java index 1f2c537ce10a..efa4f0331174 100644 --- a/core/src/main/java/org/apache/calcite/runtime/Enumerables.java +++ b/core/src/main/java/org/apache/calcite/runtime/Enumerables.java @@ -22,7 +22,7 @@ import org.apache.calcite.linq4j.Enumerator; import org.apache.calcite.linq4j.function.Function1; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayDeque; import java.util.Deque; diff --git a/core/src/main/java/org/apache/calcite/runtime/FlatLists.java b/core/src/main/java/org/apache/calcite/runtime/FlatLists.java index 872bc76a49a6..68d1453322bb 100644 --- a/core/src/main/java/org/apache/calcite/runtime/FlatLists.java +++ b/core/src/main/java/org/apache/calcite/runtime/FlatLists.java @@ -21,9 +21,9 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import org.checkerframework.checker.nullness.qual.NonNull; -import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.checker.nullness.qual.PolyNull; +import org.jspecify.annotations.NonNull; +import org.jspecify.annotations.Nullable; import java.util.AbstractList; import java.util.ArrayList; diff --git a/core/src/main/java/org/apache/calcite/runtime/FunctionContexts.java b/core/src/main/java/org/apache/calcite/runtime/FunctionContexts.java index e975b69680f9..457d57f75bc0 100644 --- a/core/src/main/java/org/apache/calcite/runtime/FunctionContexts.java +++ b/core/src/main/java/org/apache/calcite/runtime/FunctionContexts.java @@ -21,7 +21,7 @@ import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.schema.FunctionContext; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Runtime support for {@link org.apache.calcite.schema.FunctionContext}. diff --git a/core/src/main/java/org/apache/calcite/runtime/HttpUtils.java b/core/src/main/java/org/apache/calcite/runtime/HttpUtils.java index fb524bb7e032..a3af68ea2963 100644 --- a/core/src/main/java/org/apache/calcite/runtime/HttpUtils.java +++ b/core/src/main/java/org/apache/calcite/runtime/HttpUtils.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.runtime; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.io.IOException; import java.io.InputStream; diff --git a/core/src/main/java/org/apache/calcite/runtime/ImmutablePairList.java b/core/src/main/java/org/apache/calcite/runtime/ImmutablePairList.java index f285338675f4..2ea23448d834 100644 --- a/core/src/main/java/org/apache/calcite/runtime/ImmutablePairList.java +++ b/core/src/main/java/org/apache/calcite/runtime/ImmutablePairList.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.runtime; -import org.checkerframework.checker.nullness.qual.NonNull; +import org.jspecify.annotations.NonNull; import java.util.ArrayList; import java.util.Collection; diff --git a/core/src/main/java/org/apache/calcite/runtime/JsonFunctions.java b/core/src/main/java/org/apache/calcite/runtime/JsonFunctions.java index f5ce8dc479bd..c6eb7868693c 100644 --- a/core/src/main/java/org/apache/calcite/runtime/JsonFunctions.java +++ b/core/src/main/java/org/apache/calcite/runtime/JsonFunctions.java @@ -41,7 +41,7 @@ import com.jayway.jsonpath.spi.mapper.MappingProvider; import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.Arrays; diff --git a/core/src/main/java/org/apache/calcite/runtime/Like.java b/core/src/main/java/org/apache/calcite/runtime/Like.java index 376e7ca58db3..0dafc46ea8ee 100644 --- a/core/src/main/java/org/apache/calcite/runtime/Like.java +++ b/core/src/main/java/org/apache/calcite/runtime/Like.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.runtime; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Arrays; import java.util.Locale; diff --git a/core/src/main/java/org/apache/calcite/runtime/MapEntry.java b/core/src/main/java/org/apache/calcite/runtime/MapEntry.java index 8b1a3a1354af..1021d76b5ac5 100644 --- a/core/src/main/java/org/apache/calcite/runtime/MapEntry.java +++ b/core/src/main/java/org/apache/calcite/runtime/MapEntry.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.runtime; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Map; import java.util.Objects; diff --git a/core/src/main/java/org/apache/calcite/runtime/Matcher.java b/core/src/main/java/org/apache/calcite/runtime/Matcher.java index b0fdb17ffb42..347a6887dc12 100644 --- a/core/src/main/java/org/apache/calcite/runtime/Matcher.java +++ b/core/src/main/java/org/apache/calcite/runtime/Matcher.java @@ -24,7 +24,7 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Arrays; import java.util.Collection; diff --git a/core/src/main/java/org/apache/calcite/runtime/PairList.java b/core/src/main/java/org/apache/calcite/runtime/PairList.java index d4fa9aaa2bf5..b7be90f6173a 100644 --- a/core/src/main/java/org/apache/calcite/runtime/PairList.java +++ b/core/src/main/java/org/apache/calcite/runtime/PairList.java @@ -20,7 +20,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.Collection; diff --git a/core/src/main/java/org/apache/calcite/runtime/PairLists.java b/core/src/main/java/org/apache/calcite/runtime/PairLists.java index 51a7c4a0ed3d..1e5fb9196c97 100644 --- a/core/src/main/java/org/apache/calcite/runtime/PairLists.java +++ b/core/src/main/java/org/apache/calcite/runtime/PairLists.java @@ -22,8 +22,8 @@ import com.google.common.collect.ImmutableMap; import com.google.errorprone.annotations.CanIgnoreReturnValue; -import org.checkerframework.checker.nullness.qual.NonNull; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.NonNull; +import org.jspecify.annotations.Nullable; import java.util.AbstractList; import java.util.Arrays; diff --git a/core/src/main/java/org/apache/calcite/runtime/PredicateImpl.java b/core/src/main/java/org/apache/calcite/runtime/PredicateImpl.java index f68134e0dd2b..fa4aab552545 100644 --- a/core/src/main/java/org/apache/calcite/runtime/PredicateImpl.java +++ b/core/src/main/java/org/apache/calcite/runtime/PredicateImpl.java @@ -18,7 +18,7 @@ import com.google.common.base.Predicate; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Abstract implementation of {@link com.google.common.base.Predicate}. diff --git a/core/src/main/java/org/apache/calcite/runtime/Resources.java b/core/src/main/java/org/apache/calcite/runtime/Resources.java index b0ec8c9922b6..01c88e1e9426 100644 --- a/core/src/main/java/org/apache/calcite/runtime/Resources.java +++ b/core/src/main/java/org/apache/calcite/runtime/Resources.java @@ -17,7 +17,7 @@ package org.apache.calcite.runtime; import org.checkerframework.checker.initialization.qual.UnderInitialization; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.checkerframework.checker.nullness.qual.PolyNull; import org.checkerframework.checker.nullness.qual.RequiresNonNull; diff --git a/core/src/main/java/org/apache/calcite/runtime/ResultSetEnumerable.java b/core/src/main/java/org/apache/calcite/runtime/ResultSetEnumerable.java index 5d60b6552f38..8a0a08033b11 100644 --- a/core/src/main/java/org/apache/calcite/runtime/ResultSetEnumerable.java +++ b/core/src/main/java/org/apache/calcite/runtime/ResultSetEnumerable.java @@ -26,7 +26,7 @@ import org.apache.calcite.linq4j.tree.Primitive; import org.apache.calcite.util.Static; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/core/src/main/java/org/apache/calcite/runtime/SpatialTypeFunctions.java b/core/src/main/java/org/apache/calcite/runtime/SpatialTypeFunctions.java index 1942ef22d910..9270e6b17407 100644 --- a/core/src/main/java/org/apache/calcite/runtime/SpatialTypeFunctions.java +++ b/core/src/main/java/org/apache/calcite/runtime/SpatialTypeFunctions.java @@ -26,7 +26,7 @@ import org.apache.calcite.linq4j.function.Strict; import org.apache.calcite.runtime.SpatialTypeUtils.SpatialType; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.locationtech.jts.algorithm.InteriorPoint; import org.locationtech.jts.algorithm.MinimumBoundingCircle; import org.locationtech.jts.algorithm.MinimumDiameter; diff --git a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java index 65a26e05d6c0..eaba8ad81d9e 100644 --- a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java +++ b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java @@ -73,13 +73,13 @@ import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.UncheckedExecutionException; -import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.checker.nullness.qual.PolyNull; import org.joou.UByte; import org.joou.UInteger; import org.joou.ULong; import org.joou.UShort; import org.joou.Unsigned; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Field; import java.math.BigDecimal; diff --git a/core/src/main/java/org/apache/calcite/runtime/Utilities.java b/core/src/main/java/org/apache/calcite/runtime/Utilities.java index 3409af722480..186630240add 100644 --- a/core/src/main/java/org/apache/calcite/runtime/Utilities.java +++ b/core/src/main/java/org/apache/calcite/runtime/Utilities.java @@ -18,7 +18,7 @@ import org.apache.calcite.linq4j.EnumerableDefaults; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.text.Collator; import java.util.Comparator; diff --git a/core/src/main/java/org/apache/calcite/runtime/XmlFunctions.java b/core/src/main/java/org/apache/calcite/runtime/XmlFunctions.java index 00660a1b4643..c3682b58245c 100644 --- a/core/src/main/java/org/apache/calcite/runtime/XmlFunctions.java +++ b/core/src/main/java/org/apache/calcite/runtime/XmlFunctions.java @@ -20,7 +20,7 @@ import org.apache.calcite.util.TryThreadLocal; import org.apache.calcite.util.Util; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.w3c.dom.Node; import org.w3c.dom.NodeList; import org.xml.sax.InputSource; diff --git a/core/src/main/java/org/apache/calcite/runtime/package-info.java b/core/src/main/java/org/apache/calcite/runtime/package-info.java index 6ab59fedc4f6..b645be030955 100644 --- a/core/src/main/java/org/apache/calcite/runtime/package-info.java +++ b/core/src/main/java/org/apache/calcite/runtime/package-info.java @@ -23,6 +23,6 @@ @DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.RETURN) package org.apache.calcite.runtime; -import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.framework.qual.DefaultQualifier; import org.checkerframework.framework.qual.TypeUseLocation; +import org.jspecify.annotations.NonNull; diff --git a/core/src/main/java/org/apache/calcite/runtime/rtti/BasicSqlTypeRtti.java b/core/src/main/java/org/apache/calcite/runtime/rtti/BasicSqlTypeRtti.java index 00aa444a7b91..3a0aad78e4ea 100644 --- a/core/src/main/java/org/apache/calcite/runtime/rtti/BasicSqlTypeRtti.java +++ b/core/src/main/java/org/apache/calcite/runtime/rtti/BasicSqlTypeRtti.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.runtime.rtti; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Objects; diff --git a/core/src/main/java/org/apache/calcite/runtime/rtti/GenericSqlTypeRtti.java b/core/src/main/java/org/apache/calcite/runtime/rtti/GenericSqlTypeRtti.java index 7c8893b3a862..89ce364612ae 100644 --- a/core/src/main/java/org/apache/calcite/runtime/rtti/GenericSqlTypeRtti.java +++ b/core/src/main/java/org/apache/calcite/runtime/rtti/GenericSqlTypeRtti.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.runtime.rtti; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Arrays; diff --git a/core/src/main/java/org/apache/calcite/runtime/rtti/RowSqlTypeRtti.java b/core/src/main/java/org/apache/calcite/runtime/rtti/RowSqlTypeRtti.java index 035c7aeb0294..882207afe869 100644 --- a/core/src/main/java/org/apache/calcite/runtime/rtti/RowSqlTypeRtti.java +++ b/core/src/main/java/org/apache/calcite/runtime/rtti/RowSqlTypeRtti.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.runtime.rtti; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Arrays; import java.util.Map; diff --git a/core/src/main/java/org/apache/calcite/runtime/rtti/RuntimeTypeInformation.java b/core/src/main/java/org/apache/calcite/runtime/rtti/RuntimeTypeInformation.java index 53e7c69301c4..a9187dd9b4d3 100644 --- a/core/src/main/java/org/apache/calcite/runtime/rtti/RuntimeTypeInformation.java +++ b/core/src/main/java/org/apache/calcite/runtime/rtti/RuntimeTypeInformation.java @@ -22,7 +22,7 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeField; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.AbstractMap; diff --git a/core/src/main/java/org/apache/calcite/runtime/variant/VariantNonNull.java b/core/src/main/java/org/apache/calcite/runtime/variant/VariantNonNull.java index 9c288e3058cc..d022cad04563 100644 --- a/core/src/main/java/org/apache/calcite/runtime/variant/VariantNonNull.java +++ b/core/src/main/java/org/apache/calcite/runtime/variant/VariantNonNull.java @@ -23,11 +23,11 @@ import org.apache.calcite.runtime.rtti.RowSqlTypeRtti; import org.apache.calcite.runtime.rtti.RuntimeTypeInformation; -import org.checkerframework.checker.nullness.qual.Nullable; import org.joou.UByte; import org.joou.UInteger; import org.joou.ULong; import org.joou.UShort; +import org.jspecify.annotations.Nullable; import java.math.BigDecimal; import java.math.BigInteger; diff --git a/core/src/main/java/org/apache/calcite/runtime/variant/VariantNull.java b/core/src/main/java/org/apache/calcite/runtime/variant/VariantNull.java index a11dbbed6a57..aeba450775b1 100644 --- a/core/src/main/java/org/apache/calcite/runtime/variant/VariantNull.java +++ b/core/src/main/java/org/apache/calcite/runtime/variant/VariantNull.java @@ -18,7 +18,7 @@ import org.apache.calcite.runtime.rtti.RuntimeTypeInformation; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** The VARIANT type has its own notion of null, which is * different from the SQL NULL value. For example, two variant nulls are equal diff --git a/core/src/main/java/org/apache/calcite/runtime/variant/VariantSqlNull.java b/core/src/main/java/org/apache/calcite/runtime/variant/VariantSqlNull.java index 85d424461dc3..4e04bd0a7b9e 100644 --- a/core/src/main/java/org/apache/calcite/runtime/variant/VariantSqlNull.java +++ b/core/src/main/java/org/apache/calcite/runtime/variant/VariantSqlNull.java @@ -18,7 +18,7 @@ import org.apache.calcite.runtime.rtti.RuntimeTypeInformation; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Objects; diff --git a/core/src/main/java/org/apache/calcite/runtime/variant/VariantSqlValue.java b/core/src/main/java/org/apache/calcite/runtime/variant/VariantSqlValue.java index 36dc86a26da1..7e02f73d380d 100644 --- a/core/src/main/java/org/apache/calcite/runtime/variant/VariantSqlValue.java +++ b/core/src/main/java/org/apache/calcite/runtime/variant/VariantSqlValue.java @@ -18,7 +18,7 @@ import org.apache.calcite.runtime.rtti.RuntimeTypeInformation; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.math.RoundingMode; diff --git a/core/src/main/java/org/apache/calcite/runtime/variant/VariantValue.java b/core/src/main/java/org/apache/calcite/runtime/variant/VariantValue.java index 483b34db668b..247c0f9f5c86 100644 --- a/core/src/main/java/org/apache/calcite/runtime/variant/VariantValue.java +++ b/core/src/main/java/org/apache/calcite/runtime/variant/VariantValue.java @@ -18,7 +18,7 @@ import org.apache.calcite.runtime.rtti.RuntimeTypeInformation; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** Base class for the runtime support for values of the VARIANT SQL type. */ public abstract class VariantValue { diff --git a/core/src/main/java/org/apache/calcite/schema/FilterableTable.java b/core/src/main/java/org/apache/calcite/schema/FilterableTable.java index 718fcdba84b0..dc693455eb87 100644 --- a/core/src/main/java/org/apache/calcite/schema/FilterableTable.java +++ b/core/src/main/java/org/apache/calcite/schema/FilterableTable.java @@ -20,7 +20,7 @@ import org.apache.calcite.linq4j.Enumerable; import org.apache.calcite.rex.RexNode; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/schema/FunctionContext.java b/core/src/main/java/org/apache/calcite/schema/FunctionContext.java index 172e2e9c0125..a949b1c1f46f 100644 --- a/core/src/main/java/org/apache/calcite/schema/FunctionContext.java +++ b/core/src/main/java/org/apache/calcite/schema/FunctionContext.java @@ -19,7 +19,7 @@ import org.apache.calcite.linq4j.function.Experimental; import org.apache.calcite.rel.type.RelDataTypeFactory; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Information about a function call that is passed to the constructor of a diff --git a/core/src/main/java/org/apache/calcite/schema/ModifiableTable.java b/core/src/main/java/org/apache/calcite/schema/ModifiableTable.java index 3b49b54805a9..de7c5ebe5c31 100644 --- a/core/src/main/java/org/apache/calcite/schema/ModifiableTable.java +++ b/core/src/main/java/org/apache/calcite/schema/ModifiableTable.java @@ -23,7 +23,7 @@ import org.apache.calcite.rel.core.TableModify; import org.apache.calcite.rex.RexNode; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Collection; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/schema/ProjectableFilterableTable.java b/core/src/main/java/org/apache/calcite/schema/ProjectableFilterableTable.java index fa8fab08f3af..f20981531853 100644 --- a/core/src/main/java/org/apache/calcite/schema/ProjectableFilterableTable.java +++ b/core/src/main/java/org/apache/calcite/schema/ProjectableFilterableTable.java @@ -20,7 +20,7 @@ import org.apache.calcite.linq4j.Enumerable; import org.apache.calcite.rex.RexNode; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/schema/ScannableTable.java b/core/src/main/java/org/apache/calcite/schema/ScannableTable.java index 31b3c6896c88..2b95ca6ca6a6 100644 --- a/core/src/main/java/org/apache/calcite/schema/ScannableTable.java +++ b/core/src/main/java/org/apache/calcite/schema/ScannableTable.java @@ -19,7 +19,7 @@ import org.apache.calcite.DataContext; import org.apache.calcite.linq4j.Enumerable; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Table that can be scanned without creating an intermediate relational diff --git a/core/src/main/java/org/apache/calcite/schema/Schema.java b/core/src/main/java/org/apache/calcite/schema/Schema.java index 13f3fdb90715..af844a93908a 100644 --- a/core/src/main/java/org/apache/calcite/schema/Schema.java +++ b/core/src/main/java/org/apache/calcite/schema/Schema.java @@ -22,7 +22,7 @@ import org.apache.calcite.schema.lookup.LikePattern; import org.apache.calcite.schema.lookup.Lookup; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Collection; import java.util.Set; diff --git a/core/src/main/java/org/apache/calcite/schema/SchemaPlus.java b/core/src/main/java/org/apache/calcite/schema/SchemaPlus.java index c27edf09f0ef..bf8fab970b55 100644 --- a/core/src/main/java/org/apache/calcite/schema/SchemaPlus.java +++ b/core/src/main/java/org/apache/calcite/schema/SchemaPlus.java @@ -22,7 +22,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Extension to the {@link Schema} interface. diff --git a/core/src/main/java/org/apache/calcite/schema/Schemas.java b/core/src/main/java/org/apache/calcite/schema/Schemas.java index 8abff8dd8af0..edd8ed2867c4 100644 --- a/core/src/main/java/org/apache/calcite/schema/Schemas.java +++ b/core/src/main/java/org/apache/calcite/schema/Schemas.java @@ -47,7 +47,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Type; import java.sql.Connection; diff --git a/core/src/main/java/org/apache/calcite/schema/Statistic.java b/core/src/main/java/org/apache/calcite/schema/Statistic.java index 7d494f28358d..696e5bf4223c 100644 --- a/core/src/main/java/org/apache/calcite/schema/Statistic.java +++ b/core/src/main/java/org/apache/calcite/schema/Statistic.java @@ -21,7 +21,7 @@ import org.apache.calcite.rel.RelReferentialConstraint; import org.apache.calcite.util.ImmutableBitSet; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/schema/Statistics.java b/core/src/main/java/org/apache/calcite/schema/Statistics.java index 85de175989e5..4d43345ac7d7 100644 --- a/core/src/main/java/org/apache/calcite/schema/Statistics.java +++ b/core/src/main/java/org/apache/calcite/schema/Statistics.java @@ -22,7 +22,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/schema/Table.java b/core/src/main/java/org/apache/calcite/schema/Table.java index bd422ff027f6..274992999a77 100644 --- a/core/src/main/java/org/apache/calcite/schema/Table.java +++ b/core/src/main/java/org/apache/calcite/schema/Table.java @@ -22,7 +22,7 @@ import org.apache.calcite.sql.SqlCall; import org.apache.calcite.sql.SqlNode; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Table. diff --git a/core/src/main/java/org/apache/calcite/schema/TableFactory.java b/core/src/main/java/org/apache/calcite/schema/TableFactory.java index 011cd63498a6..9676c2e05390 100644 --- a/core/src/main/java/org/apache/calcite/schema/TableFactory.java +++ b/core/src/main/java/org/apache/calcite/schema/TableFactory.java @@ -18,7 +18,7 @@ import org.apache.calcite.rel.type.RelDataType; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Map; diff --git a/core/src/main/java/org/apache/calcite/schema/TableFunction.java b/core/src/main/java/org/apache/calcite/schema/TableFunction.java index 495224ea4dc4..461d757b2cb1 100644 --- a/core/src/main/java/org/apache/calcite/schema/TableFunction.java +++ b/core/src/main/java/org/apache/calcite/schema/TableFunction.java @@ -19,7 +19,7 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Type; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/schema/TableMacro.java b/core/src/main/java/org/apache/calcite/schema/TableMacro.java index 84c3c0b697ad..2eb929deb708 100644 --- a/core/src/main/java/org/apache/calcite/schema/TableMacro.java +++ b/core/src/main/java/org/apache/calcite/schema/TableMacro.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.schema; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/schema/Wrapper.java b/core/src/main/java/org/apache/calcite/schema/Wrapper.java index 4b1bff4a3f18..6265c860f3da 100644 --- a/core/src/main/java/org/apache/calcite/schema/Wrapper.java +++ b/core/src/main/java/org/apache/calcite/schema/Wrapper.java @@ -17,7 +17,7 @@ package org.apache.calcite.schema; import org.apiguardian.api.API; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Optional; diff --git a/core/src/main/java/org/apache/calcite/schema/impl/AbstractSchema.java b/core/src/main/java/org/apache/calcite/schema/impl/AbstractSchema.java index 3b8f9d6a74c8..da1feef7f29f 100644 --- a/core/src/main/java/org/apache/calcite/schema/impl/AbstractSchema.java +++ b/core/src/main/java/org/apache/calcite/schema/impl/AbstractSchema.java @@ -33,7 +33,7 @@ import com.google.common.collect.ImmutableMultimap; import com.google.common.collect.Multimap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Collection; import java.util.Map; diff --git a/core/src/main/java/org/apache/calcite/schema/impl/AbstractTable.java b/core/src/main/java/org/apache/calcite/schema/impl/AbstractTable.java index 7f7d7b0ae92c..52f0806b6027 100644 --- a/core/src/main/java/org/apache/calcite/schema/impl/AbstractTable.java +++ b/core/src/main/java/org/apache/calcite/schema/impl/AbstractTable.java @@ -25,7 +25,7 @@ import org.apache.calcite.sql.SqlCall; import org.apache.calcite.sql.SqlNode; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Abstract base class for implementing {@link Table}. diff --git a/core/src/main/java/org/apache/calcite/schema/impl/AggregateFunctionImpl.java b/core/src/main/java/org/apache/calcite/schema/impl/AggregateFunctionImpl.java index e0a37fa86def..abf0e6a98a2c 100644 --- a/core/src/main/java/org/apache/calcite/schema/impl/AggregateFunctionImpl.java +++ b/core/src/main/java/org/apache/calcite/schema/impl/AggregateFunctionImpl.java @@ -27,7 +27,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Method; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/schema/impl/DelegatingSchema.java b/core/src/main/java/org/apache/calcite/schema/impl/DelegatingSchema.java index 55bfb8b7bf87..c5490dcbb554 100644 --- a/core/src/main/java/org/apache/calcite/schema/impl/DelegatingSchema.java +++ b/core/src/main/java/org/apache/calcite/schema/impl/DelegatingSchema.java @@ -26,7 +26,7 @@ import org.apache.calcite.schema.lookup.LikePattern; import org.apache.calcite.schema.lookup.Lookup; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Collection; import java.util.Set; diff --git a/core/src/main/java/org/apache/calcite/schema/impl/ListTransientTable.java b/core/src/main/java/org/apache/calcite/schema/impl/ListTransientTable.java index f04c9372b7a0..27da0d6d3a63 100644 --- a/core/src/main/java/org/apache/calcite/schema/impl/ListTransientTable.java +++ b/core/src/main/java/org/apache/calcite/schema/impl/ListTransientTable.java @@ -41,7 +41,7 @@ import org.apache.calcite.schema.Schemas; import org.apache.calcite.schema.TransientTable; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Type; import java.util.ArrayList; diff --git a/core/src/main/java/org/apache/calcite/schema/impl/LongSchemaVersion.java b/core/src/main/java/org/apache/calcite/schema/impl/LongSchemaVersion.java index dc60dacbe6e0..f17e728a089c 100644 --- a/core/src/main/java/org/apache/calcite/schema/impl/LongSchemaVersion.java +++ b/core/src/main/java/org/apache/calcite/schema/impl/LongSchemaVersion.java @@ -18,7 +18,7 @@ import org.apache.calcite.schema.SchemaVersion; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** Implementation of SchemaVersion that uses a long value as representation. */ public class LongSchemaVersion implements SchemaVersion { diff --git a/core/src/main/java/org/apache/calcite/schema/impl/MaterializedViewTable.java b/core/src/main/java/org/apache/calcite/schema/impl/MaterializedViewTable.java index 125693ee9a8f..b686b494c309 100644 --- a/core/src/main/java/org/apache/calcite/schema/impl/MaterializedViewTable.java +++ b/core/src/main/java/org/apache/calcite/schema/impl/MaterializedViewTable.java @@ -30,7 +30,7 @@ import org.apache.calcite.schema.Table; import org.apache.calcite.schema.TranslatableTable; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Type; import java.sql.DriverManager; diff --git a/core/src/main/java/org/apache/calcite/schema/impl/ModifiableViewTable.java b/core/src/main/java/org/apache/calcite/schema/impl/ModifiableViewTable.java index aee998c0eb36..5b74e4285612 100644 --- a/core/src/main/java/org/apache/calcite/schema/impl/ModifiableViewTable.java +++ b/core/src/main/java/org/apache/calcite/schema/impl/ModifiableViewTable.java @@ -40,7 +40,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Type; import java.util.ArrayList; diff --git a/core/src/main/java/org/apache/calcite/schema/impl/ReflectiveFunctionBase.java b/core/src/main/java/org/apache/calcite/schema/impl/ReflectiveFunctionBase.java index c6dd6d7fe491..b265a0c106d3 100644 --- a/core/src/main/java/org/apache/calcite/schema/impl/ReflectiveFunctionBase.java +++ b/core/src/main/java/org/apache/calcite/schema/impl/ReflectiveFunctionBase.java @@ -25,7 +25,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Constructor; import java.lang.reflect.Method; diff --git a/core/src/main/java/org/apache/calcite/schema/impl/ScalarFunctionImpl.java b/core/src/main/java/org/apache/calcite/schema/impl/ScalarFunctionImpl.java index 1559a5c3918d..674b4232fbbb 100644 --- a/core/src/main/java/org/apache/calcite/schema/impl/ScalarFunctionImpl.java +++ b/core/src/main/java/org/apache/calcite/schema/impl/ScalarFunctionImpl.java @@ -32,7 +32,7 @@ import com.google.common.collect.ImmutableMultimap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Method; diff --git a/core/src/main/java/org/apache/calcite/schema/impl/StarTable.java b/core/src/main/java/org/apache/calcite/schema/impl/StarTable.java index e8dad3b66269..04c0fa5b5c0d 100644 --- a/core/src/main/java/org/apache/calcite/schema/impl/StarTable.java +++ b/core/src/main/java/org/apache/calcite/schema/impl/StarTable.java @@ -37,7 +37,7 @@ import com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/schema/impl/TableFunctionImpl.java b/core/src/main/java/org/apache/calcite/schema/impl/TableFunctionImpl.java index 8d510093b084..288995ae1a94 100644 --- a/core/src/main/java/org/apache/calcite/schema/impl/TableFunctionImpl.java +++ b/core/src/main/java/org/apache/calcite/schema/impl/TableFunctionImpl.java @@ -34,7 +34,7 @@ import org.apache.calcite.schema.TableFunction; import org.apache.calcite.util.BuiltInMethod; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; diff --git a/core/src/main/java/org/apache/calcite/schema/impl/TableMacroImpl.java b/core/src/main/java/org/apache/calcite/schema/impl/TableMacroImpl.java index 527ff288c9bb..b65a8a6289e6 100644 --- a/core/src/main/java/org/apache/calcite/schema/impl/TableMacroImpl.java +++ b/core/src/main/java/org/apache/calcite/schema/impl/TableMacroImpl.java @@ -19,7 +19,7 @@ import org.apache.calcite.schema.TableMacro; import org.apache.calcite.schema.TranslatableTable; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; diff --git a/core/src/main/java/org/apache/calcite/schema/impl/ViewTable.java b/core/src/main/java/org/apache/calcite/schema/impl/ViewTable.java index 2f683520d863..a2683f4b2486 100644 --- a/core/src/main/java/org/apache/calcite/schema/impl/ViewTable.java +++ b/core/src/main/java/org/apache/calcite/schema/impl/ViewTable.java @@ -35,7 +35,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Type; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/schema/impl/ViewTableMacro.java b/core/src/main/java/org/apache/calcite/schema/impl/ViewTableMacro.java index 6fdf2a9654ea..45f7a4abb455 100644 --- a/core/src/main/java/org/apache/calcite/schema/impl/ViewTableMacro.java +++ b/core/src/main/java/org/apache/calcite/schema/impl/ViewTableMacro.java @@ -28,7 +28,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Type; import java.util.Collections; diff --git a/core/src/main/java/org/apache/calcite/schema/lookup/CompatibilityLookup.java b/core/src/main/java/org/apache/calcite/schema/lookup/CompatibilityLookup.java index 6ab340e9af30..23b868873bc0 100644 --- a/core/src/main/java/org/apache/calcite/schema/lookup/CompatibilityLookup.java +++ b/core/src/main/java/org/apache/calcite/schema/lookup/CompatibilityLookup.java @@ -18,7 +18,7 @@ import org.apache.calcite.linq4j.function.Predicate1; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Set; import java.util.function.Function; diff --git a/core/src/main/java/org/apache/calcite/schema/lookup/ConcatLookup.java b/core/src/main/java/org/apache/calcite/schema/lookup/ConcatLookup.java index cb09d2342fd7..465553a12d18 100644 --- a/core/src/main/java/org/apache/calcite/schema/lookup/ConcatLookup.java +++ b/core/src/main/java/org/apache/calcite/schema/lookup/ConcatLookup.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.schema.lookup; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Set; import java.util.stream.Collectors; diff --git a/core/src/main/java/org/apache/calcite/schema/lookup/EmptyLookup.java b/core/src/main/java/org/apache/calcite/schema/lookup/EmptyLookup.java index 0730a7856abf..1b480e0070f4 100644 --- a/core/src/main/java/org/apache/calcite/schema/lookup/EmptyLookup.java +++ b/core/src/main/java/org/apache/calcite/schema/lookup/EmptyLookup.java @@ -18,7 +18,7 @@ import com.google.common.collect.ImmutableSet; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Set; diff --git a/core/src/main/java/org/apache/calcite/schema/lookup/IgnoreCaseLookup.java b/core/src/main/java/org/apache/calcite/schema/lookup/IgnoreCaseLookup.java index a22e7accfce3..9d8d7243f6fd 100644 --- a/core/src/main/java/org/apache/calcite/schema/lookup/IgnoreCaseLookup.java +++ b/core/src/main/java/org/apache/calcite/schema/lookup/IgnoreCaseLookup.java @@ -19,7 +19,7 @@ import org.apache.calcite.util.LazyReference; import org.apache.calcite.util.NameMap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Map; import java.util.Set; diff --git a/core/src/main/java/org/apache/calcite/schema/lookup/LoadingCacheLookup.java b/core/src/main/java/org/apache/calcite/schema/lookup/LoadingCacheLookup.java index 8f8adaa5cb69..2772d4664602 100644 --- a/core/src/main/java/org/apache/calcite/schema/lookup/LoadingCacheLookup.java +++ b/core/src/main/java/org/apache/calcite/schema/lookup/LoadingCacheLookup.java @@ -21,7 +21,7 @@ import com.google.common.cache.LoadingCache; import com.google.common.util.concurrent.UncheckedExecutionException; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.time.Duration; import java.util.Set; diff --git a/core/src/main/java/org/apache/calcite/schema/lookup/Lookup.java b/core/src/main/java/org/apache/calcite/schema/lookup/Lookup.java index 40b42dab6db4..f98d99490f63 100644 --- a/core/src/main/java/org/apache/calcite/schema/lookup/Lookup.java +++ b/core/src/main/java/org/apache/calcite/schema/lookup/Lookup.java @@ -18,7 +18,7 @@ import org.apache.calcite.util.NameMap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Set; import java.util.function.BiFunction; diff --git a/core/src/main/java/org/apache/calcite/schema/lookup/NameMapLookup.java b/core/src/main/java/org/apache/calcite/schema/lookup/NameMapLookup.java index 7231eea6696b..3be365529e96 100644 --- a/core/src/main/java/org/apache/calcite/schema/lookup/NameMapLookup.java +++ b/core/src/main/java/org/apache/calcite/schema/lookup/NameMapLookup.java @@ -19,7 +19,7 @@ import org.apache.calcite.linq4j.function.Predicate1; import org.apache.calcite.util.NameMap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Map; import java.util.Set; diff --git a/core/src/main/java/org/apache/calcite/schema/lookup/Named.java b/core/src/main/java/org/apache/calcite/schema/lookup/Named.java index ec9c672f2fa0..544224a3cacd 100644 --- a/core/src/main/java/org/apache/calcite/schema/lookup/Named.java +++ b/core/src/main/java/org/apache/calcite/schema/lookup/Named.java @@ -16,8 +16,8 @@ */ package org.apache.calcite.schema.lookup; -import org.checkerframework.checker.nullness.qual.NonNull; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.NonNull; +import org.jspecify.annotations.Nullable; import static java.util.Objects.requireNonNull; diff --git a/core/src/main/java/org/apache/calcite/schema/lookup/SnapshotLookup.java b/core/src/main/java/org/apache/calcite/schema/lookup/SnapshotLookup.java index 9f89cfa8903c..5823a4d1d3c0 100644 --- a/core/src/main/java/org/apache/calcite/schema/lookup/SnapshotLookup.java +++ b/core/src/main/java/org/apache/calcite/schema/lookup/SnapshotLookup.java @@ -19,7 +19,7 @@ import org.apache.calcite.util.LazyReference; import org.apache.calcite.util.NameMap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Set; diff --git a/core/src/main/java/org/apache/calcite/schema/lookup/TransformingLookup.java b/core/src/main/java/org/apache/calcite/schema/lookup/TransformingLookup.java index 384b53e5d983..83191aeee772 100644 --- a/core/src/main/java/org/apache/calcite/schema/lookup/TransformingLookup.java +++ b/core/src/main/java/org/apache/calcite/schema/lookup/TransformingLookup.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.schema.lookup; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Set; import java.util.function.BiFunction; diff --git a/core/src/main/java/org/apache/calcite/schema/lookup/package-info.java b/core/src/main/java/org/apache/calcite/schema/lookup/package-info.java index edd5b38d382b..fd3821cc6587 100644 --- a/core/src/main/java/org/apache/calcite/schema/lookup/package-info.java +++ b/core/src/main/java/org/apache/calcite/schema/lookup/package-info.java @@ -26,6 +26,6 @@ @DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.RETURN) package org.apache.calcite.schema.lookup; -import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.framework.qual.DefaultQualifier; import org.checkerframework.framework.qual.TypeUseLocation; +import org.jspecify.annotations.NonNull; diff --git a/core/src/main/java/org/apache/calcite/schema/package-info.java b/core/src/main/java/org/apache/calcite/schema/package-info.java index e6eccdc15910..cda5149c6249 100644 --- a/core/src/main/java/org/apache/calcite/schema/package-info.java +++ b/core/src/main/java/org/apache/calcite/schema/package-info.java @@ -27,6 +27,6 @@ @DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.RETURN) package org.apache.calcite.schema; -import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.framework.qual.DefaultQualifier; import org.checkerframework.framework.qual.TypeUseLocation; +import org.jspecify.annotations.NonNull; diff --git a/core/src/main/java/org/apache/calcite/server/CalciteServerStatement.java b/core/src/main/java/org/apache/calcite/server/CalciteServerStatement.java index 91e6f0865340..439d307527b3 100644 --- a/core/src/main/java/org/apache/calcite/server/CalciteServerStatement.java +++ b/core/src/main/java/org/apache/calcite/server/CalciteServerStatement.java @@ -20,7 +20,7 @@ import org.apache.calcite.jdbc.CalciteConnection; import org.apache.calcite.jdbc.CalcitePrepare; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Iterator; diff --git a/core/src/main/java/org/apache/calcite/server/package-info.java b/core/src/main/java/org/apache/calcite/server/package-info.java index 1d9a89498818..84cfd70fa2c3 100644 --- a/core/src/main/java/org/apache/calcite/server/package-info.java +++ b/core/src/main/java/org/apache/calcite/server/package-info.java @@ -23,6 +23,6 @@ @DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.RETURN) package org.apache.calcite.server; -import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.framework.qual.DefaultQualifier; import org.checkerframework.framework.qual.TypeUseLocation; +import org.jspecify.annotations.NonNull; diff --git a/core/src/main/java/org/apache/calcite/sql/ExplicitOperatorBinding.java b/core/src/main/java/org/apache/calcite/sql/ExplicitOperatorBinding.java index c66730f354a9..431e527a7ea6 100644 --- a/core/src/main/java/org/apache/calcite/sql/ExplicitOperatorBinding.java +++ b/core/src/main/java/org/apache/calcite/sql/ExplicitOperatorBinding.java @@ -23,7 +23,7 @@ import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.sql.validate.SqlValidatorException; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlAggFunction.java b/core/src/main/java/org/apache/calcite/sql/SqlAggFunction.java index 609bfee6d517..849f0968840e 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlAggFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlAggFunction.java @@ -29,7 +29,7 @@ import org.apache.calcite.util.Optionality; import org.apache.calcite.util.Static; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlAlter.java b/core/src/main/java/org/apache/calcite/sql/SqlAlter.java index e7f232bbc245..dd31eb527488 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlAlter.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlAlter.java @@ -18,7 +18,7 @@ import org.apache.calcite.sql.parser.SqlParserPos; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Base class for an ALTER statements parse tree nodes. The portion of the diff --git a/core/src/main/java/org/apache/calcite/sql/SqlAsofJoin.java b/core/src/main/java/org/apache/calcite/sql/SqlAsofJoin.java index 14f9723d37d8..286e2df1df62 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlAsofJoin.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlAsofJoin.java @@ -21,7 +21,7 @@ import org.apache.calcite.util.ImmutableNullableList; import org.apache.calcite.util.Util; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; import java.util.function.UnaryOperator; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlBasicCall.java b/core/src/main/java/org/apache/calcite/sql/SqlBasicCall.java index 8279924af1a0..a49c278404d5 100755 --- a/core/src/main/java/org/apache/calcite/sql/SqlBasicCall.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlBasicCall.java @@ -19,7 +19,7 @@ import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.util.ImmutableNullableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlBasicFunction.java b/core/src/main/java/org/apache/calcite/sql/SqlBasicFunction.java index fa511d447685..fa677b26d5a1 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlBasicFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlBasicFunction.java @@ -25,7 +25,7 @@ import org.apache.calcite.sql.validate.SqlValidator; import org.apache.calcite.sql.validate.SqlValidatorScope; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.function.Function; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlBasicTypeNameSpec.java b/core/src/main/java/org/apache/calcite/sql/SqlBasicTypeNameSpec.java index 7213352e6dc9..dc51fad5253a 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlBasicTypeNameSpec.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlBasicTypeNameSpec.java @@ -24,7 +24,7 @@ import org.apache.calcite.sql.validate.SqlValidator; import org.apache.calcite.util.Litmus; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.nio.charset.Charset; import java.util.Objects; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlBinaryOperator.java b/core/src/main/java/org/apache/calcite/sql/SqlBinaryOperator.java index 514c61fe099d..3bdf92e31e01 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlBinaryOperator.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlBinaryOperator.java @@ -28,7 +28,7 @@ import org.apache.calcite.util.Litmus; import org.apache.calcite.util.Util; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.math.BigDecimal; import java.nio.charset.Charset; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlByRewriter.java b/core/src/main/java/org/apache/calcite/sql/SqlByRewriter.java index b8d6cc910e2e..45b587365155 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlByRewriter.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlByRewriter.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.sql; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlCall.java b/core/src/main/java/org/apache/calcite/sql/SqlCall.java index 1747272f621a..31aec4fa3559 100755 --- a/core/src/main/java/org/apache/calcite/sql/SqlCall.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlCall.java @@ -27,8 +27,8 @@ import org.apache.calcite.sql.validate.SqlValidatorScope; import org.apache.calcite.util.Litmus; -import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.dataflow.qual.Pure; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.Collection; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlCallBinding.java b/core/src/main/java/org/apache/calcite/sql/SqlCallBinding.java index 95e403901ce7..8f1d1f020fab 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlCallBinding.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlCallBinding.java @@ -44,7 +44,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.math.BigDecimal; import java.util.ArrayList; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlCharStringLiteral.java b/core/src/main/java/org/apache/calcite/sql/SqlCharStringLiteral.java index 2f720f774b18..205998bc1dcd 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlCharStringLiteral.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlCharStringLiteral.java @@ -22,7 +22,7 @@ import org.apache.calcite.util.NlsString; import org.apache.calcite.util.Util; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlCollation.java b/core/src/main/java/org/apache/calcite/sql/SqlCollation.java index 5f6e4805f764..7282ff431039 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlCollation.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlCollation.java @@ -29,8 +29,8 @@ import com.fasterxml.jackson.annotation.JsonProperty; import org.checkerframework.checker.initialization.qual.UnderInitialization; -import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.dataflow.qual.Pure; +import org.jspecify.annotations.Nullable; import java.io.Serializable; import java.nio.charset.Charset; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlConstantValueAggFunction.java b/core/src/main/java/org/apache/calcite/sql/SqlConstantValueAggFunction.java index 6091e7763695..62f040ca9ca6 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlConstantValueAggFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlConstantValueAggFunction.java @@ -20,7 +20,7 @@ import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexNode; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Aggregate function that returns a constant value when applied to constant diff --git a/core/src/main/java/org/apache/calcite/sql/SqlDataTypeSpec.java b/core/src/main/java/org/apache/calcite/sql/SqlDataTypeSpec.java index a1e68becc9a1..db1e05122d16 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlDataTypeSpec.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlDataTypeSpec.java @@ -25,7 +25,7 @@ import org.apache.calcite.sql.validate.SqlValidatorScope; import org.apache.calcite.util.Litmus; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Objects; import java.util.TimeZone; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlDelete.java b/core/src/main/java/org/apache/calcite/sql/SqlDelete.java index 7b3cf41c5cfe..4d085dcca565 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlDelete.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlDelete.java @@ -22,7 +22,7 @@ import org.apache.calcite.sql.validate.SqlValidatorScope; import org.apache.calcite.util.ImmutableNullableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlDescribeSchema.java b/core/src/main/java/org/apache/calcite/sql/SqlDescribeSchema.java index bdf2cda65193..34fa69f97e4d 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlDescribeSchema.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlDescribeSchema.java @@ -19,7 +19,7 @@ import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.util.ImmutableNullableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlDescribeTable.java b/core/src/main/java/org/apache/calcite/sql/SqlDescribeTable.java index b956bebc00dc..68150c251224 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlDescribeTable.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlDescribeTable.java @@ -19,7 +19,7 @@ import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.util.ImmutableNullableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlDialect.java b/core/src/main/java/org/apache/calcite/sql/SqlDialect.java index eef1c386d594..7deaba883411 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlDialect.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlDialect.java @@ -47,8 +47,8 @@ import com.google.common.base.Suppliers; import com.google.common.collect.ImmutableSet; -import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.dataflow.qual.Pure; +import org.jspecify.annotations.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlDialectFactoryImpl.java b/core/src/main/java/org/apache/calcite/sql/SqlDialectFactoryImpl.java index 3b8de0d65c0c..0d1f238dafa3 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlDialectFactoryImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlDialectFactoryImpl.java @@ -56,7 +56,7 @@ import org.apache.calcite.sql.dialect.TrinoSqlDialect; import org.apache.calcite.sql.dialect.VerticaSqlDialect; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.sql.DatabaseMetaData; import java.sql.SQLException; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlDynamicParam.java b/core/src/main/java/org/apache/calcite/sql/SqlDynamicParam.java index 76e50c6b3410..eb149f2ba339 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlDynamicParam.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlDynamicParam.java @@ -23,7 +23,7 @@ import org.apache.calcite.sql.validate.SqlValidatorScope; import org.apache.calcite.util.Litmus; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * A SqlDynamicParam represents a dynamic parameter marker in an diff --git a/core/src/main/java/org/apache/calcite/sql/SqlExplain.java b/core/src/main/java/org/apache/calcite/sql/SqlExplain.java index d072a6ceacd1..ab9718fd42bc 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlExplain.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlExplain.java @@ -19,8 +19,8 @@ import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.util.ImmutableNullableList; -import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.dataflow.qual.Pure; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlFunction.java b/core/src/main/java/org/apache/calcite/sql/SqlFunction.java index 156150c41940..afb7d10f06c5 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlFunction.java @@ -30,8 +30,8 @@ import org.apache.calcite.sql.validate.implicit.TypeCoercion; import org.apache.calcite.util.Util; -import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.dataflow.qual.Pure; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlFunctionalOperator.java b/core/src/main/java/org/apache/calcite/sql/SqlFunctionalOperator.java index 7f7f36ebc477..8273f7a65381 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlFunctionalOperator.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlFunctionalOperator.java @@ -20,7 +20,7 @@ import org.apache.calcite.sql.type.SqlOperandTypeInference; import org.apache.calcite.sql.type.SqlReturnTypeInference; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * SqlFunctionalOperator is a base class for special operators which use diff --git a/core/src/main/java/org/apache/calcite/sql/SqlGroupedWindowFunction.java b/core/src/main/java/org/apache/calcite/sql/SqlGroupedWindowFunction.java index 2a9192595c97..4dfc3fd31918 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlGroupedWindowFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlGroupedWindowFunction.java @@ -24,7 +24,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlHint.java b/core/src/main/java/org/apache/calcite/sql/SqlHint.java index 8c61828daec9..cb4d5cc51d75 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlHint.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlHint.java @@ -21,7 +21,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.HashMap; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlIdentifier.java b/core/src/main/java/org/apache/calcite/sql/SqlIdentifier.java index e3214579d3f3..11c7fcf8e881 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlIdentifier.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlIdentifier.java @@ -28,8 +28,8 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.dataflow.qual.Pure; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlInfixOperator.java b/core/src/main/java/org/apache/calcite/sql/SqlInfixOperator.java index 49aa1f0f737b..fd5742851cce 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlInfixOperator.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlInfixOperator.java @@ -21,7 +21,7 @@ import org.apache.calcite.sql.type.SqlOperandTypeInference; import org.apache.calcite.sql.type.SqlReturnTypeInference; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * A generalization of a binary operator to involve several (two or more) diff --git a/core/src/main/java/org/apache/calcite/sql/SqlInsert.java b/core/src/main/java/org/apache/calcite/sql/SqlInsert.java index f0d3ba3e55a0..c37f7726c510 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlInsert.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlInsert.java @@ -21,8 +21,8 @@ import org.apache.calcite.sql.validate.SqlValidatorScope; import org.apache.calcite.util.ImmutableNullableList; -import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.dataflow.qual.Pure; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlInternalOperator.java b/core/src/main/java/org/apache/calcite/sql/SqlInternalOperator.java index 70c590650afc..16057a714e8e 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlInternalOperator.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlInternalOperator.java @@ -25,7 +25,7 @@ import org.apache.calcite.sql.validate.SqlValidator; import org.apache.calcite.sql.validate.SqlValidatorScope; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Generic operator for nodes with internal syntax. diff --git a/core/src/main/java/org/apache/calcite/sql/SqlIntervalLiteral.java b/core/src/main/java/org/apache/calcite/sql/SqlIntervalLiteral.java index 0f311ce51341..b099a230045b 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlIntervalLiteral.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlIntervalLiteral.java @@ -20,7 +20,7 @@ import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.util.Litmus; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Objects; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlIntervalQualifier.java b/core/src/main/java/org/apache/calcite/sql/SqlIntervalQualifier.java index 5f81211b994e..c07ca28b47f9 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlIntervalQualifier.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlIntervalQualifier.java @@ -32,7 +32,7 @@ import com.google.common.collect.ImmutableSet; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.math.BigDecimal; import java.util.Locale; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlJdbcDataTypeName.java b/core/src/main/java/org/apache/calcite/sql/SqlJdbcDataTypeName.java index cf6227941639..5c15194bf137 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlJdbcDataTypeName.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlJdbcDataTypeName.java @@ -20,7 +20,7 @@ import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.sql.type.SqlTypeName; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import static com.google.common.base.Preconditions.checkArgument; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlJdbcFunctionCall.java b/core/src/main/java/org/apache/calcite/sql/SqlJdbcFunctionCall.java index 17691a3a950b..73a20faf642a 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlJdbcFunctionCall.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlJdbcFunctionCall.java @@ -27,7 +27,7 @@ import com.google.common.collect.ImmutableMap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Map; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlJoin.java b/core/src/main/java/org/apache/calcite/sql/SqlJoin.java index f948a7d668d4..a67774450971 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlJoin.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlJoin.java @@ -22,7 +22,7 @@ import org.apache.calcite.util.ImmutableNullableList; import org.apache.calcite.util.Util; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; import java.util.function.UnaryOperator; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlLambda.java b/core/src/main/java/org/apache/calcite/sql/SqlLambda.java index 696753b4f4e5..555fda118bac 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlLambda.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlLambda.java @@ -23,7 +23,7 @@ import org.apache.calcite.sql.validate.SqlValidatorScope; import org.apache.calcite.util.UnmodifiableArrayList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlLiteral.java b/core/src/main/java/org/apache/calcite/sql/SqlLiteral.java index d0af5ba65228..a5160d3abc40 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlLiteral.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlLiteral.java @@ -42,7 +42,7 @@ import org.apache.calcite.util.TimestampWithTimeZoneString; import org.apache.calcite.util.Util; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.math.BigDecimal; import java.nio.charset.Charset; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlMatchFunction.java b/core/src/main/java/org/apache/calcite/sql/SqlMatchFunction.java index 7c8f3a5673b2..18becfed7f89 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlMatchFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlMatchFunction.java @@ -20,7 +20,7 @@ import org.apache.calcite.sql.type.SqlOperandTypeInference; import org.apache.calcite.sql.type.SqlReturnTypeInference; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** Base class for all functions used in MATCH_RECOGNIZE. */ public class SqlMatchFunction extends SqlFunction { diff --git a/core/src/main/java/org/apache/calcite/sql/SqlMatchRecognize.java b/core/src/main/java/org/apache/calcite/sql/SqlMatchRecognize.java index ffd37c04d2c7..d45b4721e6a4 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlMatchRecognize.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlMatchRecognize.java @@ -23,7 +23,7 @@ import org.apache.calcite.sql.validate.SqlValidatorScope; import org.apache.calcite.util.ImmutableNullableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlMerge.java b/core/src/main/java/org/apache/calcite/sql/SqlMerge.java index df450974a56b..f533041d99eb 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlMerge.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlMerge.java @@ -23,8 +23,8 @@ import org.apache.calcite.util.ImmutableNullableList; import org.apache.calcite.util.Pair; -import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.dataflow.qual.Pure; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlNode.java b/core/src/main/java/org/apache/calcite/sql/SqlNode.java index fa642101b460..c35806ad67cd 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlNode.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlNode.java @@ -28,7 +28,7 @@ import org.apache.calcite.util.Litmus; import org.apache.calcite.util.Util; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.Collection; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlNodeList.java b/core/src/main/java/org/apache/calcite/sql/SqlNodeList.java index 507b74a6916d..06ff47ad3879 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlNodeList.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlNodeList.java @@ -24,7 +24,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.Collection; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlNullTreatmentOperator.java b/core/src/main/java/org/apache/calcite/sql/SqlNullTreatmentOperator.java index 28ad44956c38..fa83e6535d30 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlNullTreatmentOperator.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlNullTreatmentOperator.java @@ -23,7 +23,7 @@ import org.apache.calcite.sql.validate.SqlValidatorScope; import org.apache.calcite.util.ImmutableNullableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import static com.google.common.base.Preconditions.checkArgument; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlNumericLiteral.java b/core/src/main/java/org/apache/calcite/sql/SqlNumericLiteral.java index 575388ed5621..949d68cc654f 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlNumericLiteral.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlNumericLiteral.java @@ -22,8 +22,8 @@ import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.util.Util; -import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.dataflow.qual.Pure; +import org.jspecify.annotations.Nullable; import java.math.BigDecimal; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlOperator.java b/core/src/main/java/org/apache/calcite/sql/SqlOperator.java index dc8f09770997..b9a240bcb43c 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlOperator.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlOperator.java @@ -41,8 +41,8 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; -import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.dataflow.qual.Pure; +import org.jspecify.annotations.Nullable; import java.util.List; import java.util.Objects; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlOperatorBinding.java b/core/src/main/java/org/apache/calcite/sql/SqlOperatorBinding.java index 43e938d0e65d..cf2241265382 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlOperatorBinding.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlOperatorBinding.java @@ -28,7 +28,7 @@ import org.apache.calcite.sql.validate.SqlValidatorException; import org.apache.calcite.util.NlsString; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.AbstractList; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlOperatorTable.java b/core/src/main/java/org/apache/calcite/sql/SqlOperatorTable.java index 7bd58045f717..1851a6641097 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlOperatorTable.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlOperatorTable.java @@ -18,7 +18,7 @@ import org.apache.calcite.sql.validate.SqlNameMatcher; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlOrderBy.java b/core/src/main/java/org/apache/calcite/sql/SqlOrderBy.java index c0c88240f80e..6ba5ee872af3 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlOrderBy.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlOrderBy.java @@ -19,7 +19,7 @@ import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.util.ImmutableNullableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlPivot.java b/core/src/main/java/org/apache/calcite/sql/SqlPivot.java index 2d0f3652de10..c7bc8c586a77 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlPivot.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlPivot.java @@ -26,7 +26,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.HashSet; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlPostfixOperator.java b/core/src/main/java/org/apache/calcite/sql/SqlPostfixOperator.java index b4e172a032b5..4d95298524b7 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlPostfixOperator.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlPostfixOperator.java @@ -25,7 +25,7 @@ import org.apache.calcite.util.Litmus; import org.apache.calcite.util.Util; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import static org.apache.calcite.linq4j.Nullness.castNonNull; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlPrefixOperator.java b/core/src/main/java/org/apache/calcite/sql/SqlPrefixOperator.java index a7e2a144b1ca..9f316d4d1ab9 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlPrefixOperator.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlPrefixOperator.java @@ -26,7 +26,7 @@ import org.apache.calcite.util.Litmus; import org.apache.calcite.util.Util; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import static org.apache.calcite.sql.type.NonNullableAccessors.getCharset; import static org.apache.calcite.sql.type.NonNullableAccessors.getCollation; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlSelect.java b/core/src/main/java/org/apache/calcite/sql/SqlSelect.java index ad542c31a005..4d8f09975831 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlSelect.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlSelect.java @@ -23,8 +23,8 @@ import org.apache.calcite.util.ImmutableNullableList; import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf; -import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.dataflow.qual.Pure; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlSelectOperator.java b/core/src/main/java/org/apache/calcite/sql/SqlSelectOperator.java index 1c1a8edfe8c3..4dbce066e598 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlSelectOperator.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlSelectOperator.java @@ -21,7 +21,7 @@ import org.apache.calcite.sql.util.SqlBasicVisitor; import org.apache.calcite.sql.util.SqlVisitor; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlSetOperator.java b/core/src/main/java/org/apache/calcite/sql/SqlSetOperator.java index 0ce69276f71d..059c68e6c169 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlSetOperator.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlSetOperator.java @@ -24,7 +24,7 @@ import org.apache.calcite.sql.validate.SqlValidator; import org.apache.calcite.sql.validate.SqlValidatorScope; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * SqlSetOperator represents a relational set theory operator (UNION, INTERSECT, diff --git a/core/src/main/java/org/apache/calcite/sql/SqlSetOption.java b/core/src/main/java/org/apache/calcite/sql/SqlSetOption.java index 689ba20642ba..a1030afdcec1 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlSetOption.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlSetOption.java @@ -21,7 +21,7 @@ import org.apache.calcite.sql.validate.SqlValidatorScope; import org.apache.calcite.util.ImmutableNullableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlSetSemanticsTableOperator.java b/core/src/main/java/org/apache/calcite/sql/SqlSetSemanticsTableOperator.java index f33003c74318..a5caded1d0fc 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlSetSemanticsTableOperator.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlSetSemanticsTableOperator.java @@ -21,7 +21,7 @@ import org.apache.calcite.sql.validate.SqlValidator; import org.apache.calcite.sql.validate.SqlValidatorScope; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlSnapshot.java b/core/src/main/java/org/apache/calcite/sql/SqlSnapshot.java index adf9f27f9fe2..d7b6ad835c4e 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlSnapshot.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlSnapshot.java @@ -21,7 +21,7 @@ import org.apache.calcite.sql.util.SqlVisitor; import org.apache.calcite.util.ImmutableNullableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlSpatialTypeOperatorTable.java b/core/src/main/java/org/apache/calcite/sql/SqlSpatialTypeOperatorTable.java index 6d0f920d2ae1..c8d7c8774d66 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlSpatialTypeOperatorTable.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlSpatialTypeOperatorTable.java @@ -33,7 +33,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlSpecialOperator.java b/core/src/main/java/org/apache/calcite/sql/SqlSpecialOperator.java index a37ec2ba9979..755e95f5e344 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlSpecialOperator.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlSpecialOperator.java @@ -23,7 +23,7 @@ import org.apache.calcite.util.PrecedenceClimbingParser; import org.apache.calcite.util.Util; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.function.Predicate; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlSplittableAggFunction.java b/core/src/main/java/org/apache/calcite/sql/SqlSplittableAggFunction.java index d5bdeb00f334..0d19d275d2e3 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlSplittableAggFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlSplittableAggFunction.java @@ -33,7 +33,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.math.BigDecimal; import java.util.ArrayList; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlStarExclude.java b/core/src/main/java/org/apache/calcite/sql/SqlStarExclude.java index fc1d7f298889..17fa21947064 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlStarExclude.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlStarExclude.java @@ -20,7 +20,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlStarReplace.java b/core/src/main/java/org/apache/calcite/sql/SqlStarReplace.java index 28b5f5ab4243..f7775af0e3b6 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlStarReplace.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlStarReplace.java @@ -20,7 +20,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlStaticAggFunction.java b/core/src/main/java/org/apache/calcite/sql/SqlStaticAggFunction.java index 191e80550ed8..46b645fac063 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlStaticAggFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlStaticAggFunction.java @@ -23,7 +23,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** Aggregate function whose value may be a constant expression, based on * only the contents of the GROUP BY clause. */ diff --git a/core/src/main/java/org/apache/calcite/sql/SqlSyntax.java b/core/src/main/java/org/apache/calcite/sql/SqlSyntax.java index 826891794d29..d7157b2a31cf 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlSyntax.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlSyntax.java @@ -20,7 +20,7 @@ import org.apache.calcite.util.Util; import org.checkerframework.checker.initialization.qual.NotOnlyInitialized; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Enumeration of possible syntactic types of {@link SqlOperator operators}. diff --git a/core/src/main/java/org/apache/calcite/sql/SqlTableFunction.java b/core/src/main/java/org/apache/calcite/sql/SqlTableFunction.java index bef0b7d12703..a28db757e5e2 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlTableFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlTableFunction.java @@ -18,7 +18,7 @@ import org.apache.calcite.sql.type.SqlReturnTypeInference; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * A function that returns a table. diff --git a/core/src/main/java/org/apache/calcite/sql/SqlTableRef.java b/core/src/main/java/org/apache/calcite/sql/SqlTableRef.java index e4665c116c82..b891e53b3f5f 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlTableRef.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlTableRef.java @@ -20,7 +20,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlUnpivot.java b/core/src/main/java/org/apache/calcite/sql/SqlUnpivot.java index 732e6d4e6a19..0087fe3cd6c5 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlUnpivot.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlUnpivot.java @@ -22,7 +22,7 @@ import org.apache.calcite.util.ImmutableNullableList; import org.apache.calcite.util.Util; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.HashSet; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlUnresolvedFunction.java b/core/src/main/java/org/apache/calcite/sql/SqlUnresolvedFunction.java index d1f3d796a9bd..31c12db6dc98 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlUnresolvedFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlUnresolvedFunction.java @@ -23,7 +23,7 @@ import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.apache.calcite.sql.type.SqlTypeName; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlUpdate.java b/core/src/main/java/org/apache/calcite/sql/SqlUpdate.java index 0f743540d18e..60c0e2463cf2 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlUpdate.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlUpdate.java @@ -23,8 +23,8 @@ import org.apache.calcite.util.ImmutableNullableList; import org.apache.calcite.util.Pair; -import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.dataflow.qual.Pure; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlUtil.java b/core/src/main/java/org/apache/calcite/sql/SqlUtil.java index e6bccfe51e3e..c14171fab120 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlUtil.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlUtil.java @@ -56,8 +56,8 @@ import com.google.common.collect.Iterators; import com.google.common.collect.Lists; -import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.checker.nullness.qual.PolyNull; +import org.jspecify.annotations.Nullable; import java.math.BigDecimal; import java.nio.charset.Charset; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlWindow.java b/core/src/main/java/org/apache/calcite/sql/SqlWindow.java index 66c4f8d7b766..01b561a8af71 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlWindow.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlWindow.java @@ -36,8 +36,8 @@ import com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf; -import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.dataflow.qual.Pure; +import org.jspecify.annotations.Nullable; import java.math.BigDecimal; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlWindowTableFunction.java b/core/src/main/java/org/apache/calcite/sql/SqlWindowTableFunction.java index 601b38299450..f4f21491536c 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlWindowTableFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlWindowTableFunction.java @@ -31,7 +31,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Collections; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlWith.java b/core/src/main/java/org/apache/calcite/sql/SqlWith.java index 647e02e2bc11..89e036ba3d52 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlWith.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlWith.java @@ -22,7 +22,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlWithItem.java b/core/src/main/java/org/apache/calcite/sql/SqlWithItem.java index 89fe2e8d9cdc..01f5059a3f1d 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlWithItem.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlWithItem.java @@ -19,7 +19,7 @@ import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.util.ImmutableNullableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlWriter.java b/core/src/main/java/org/apache/calcite/sql/SqlWriter.java index 9682f9c99f33..cbc987980015 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlWriter.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlWriter.java @@ -19,8 +19,8 @@ import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.util.SqlString; -import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.dataflow.qual.Pure; +import org.jspecify.annotations.Nullable; import java.util.function.Consumer; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlWriterConfig.java b/core/src/main/java/org/apache/calcite/sql/SqlWriterConfig.java index 038201bbd150..8e7434353556 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlWriterConfig.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlWriterConfig.java @@ -18,8 +18,8 @@ import org.apache.calcite.sql.pretty.SqlPrettyWriter; -import org.checkerframework.checker.nullness.qual.Nullable; import org.immutables.value.Value; +import org.jspecify.annotations.Nullable; /** Configuration for {@link SqlWriter} and {@link SqlPrettyWriter}. */ @Value.Immutable diff --git a/core/src/main/java/org/apache/calcite/sql/TableCharacteristic.java b/core/src/main/java/org/apache/calcite/sql/TableCharacteristic.java index 245a965494bc..c0f0ad899b11 100644 --- a/core/src/main/java/org/apache/calcite/sql/TableCharacteristic.java +++ b/core/src/main/java/org/apache/calcite/sql/TableCharacteristic.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.sql; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Objects; diff --git a/core/src/main/java/org/apache/calcite/sql/advise/SqlAdvisor.java b/core/src/main/java/org/apache/calcite/sql/advise/SqlAdvisor.java index c830dc2cd728..0d773ae67b71 100644 --- a/core/src/main/java/org/apache/calcite/sql/advise/SqlAdvisor.java +++ b/core/src/main/java/org/apache/calcite/sql/advise/SqlAdvisor.java @@ -39,7 +39,7 @@ import com.google.common.collect.Lists; import org.checkerframework.checker.nullness.qual.EnsuresNonNull; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.slf4j.Logger; import java.util.ArrayList; diff --git a/core/src/main/java/org/apache/calcite/sql/advise/SqlAdvisorGetHintsFunction.java b/core/src/main/java/org/apache/calcite/sql/advise/SqlAdvisorGetHintsFunction.java index d1b05764e893..b705a02c5730 100644 --- a/core/src/main/java/org/apache/calcite/sql/advise/SqlAdvisorGetHintsFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/advise/SqlAdvisorGetHintsFunction.java @@ -36,7 +36,7 @@ import com.google.common.collect.Iterables; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Method; import java.lang.reflect.Type; diff --git a/core/src/main/java/org/apache/calcite/sql/advise/SqlAdvisorGetHintsFunction2.java b/core/src/main/java/org/apache/calcite/sql/advise/SqlAdvisorGetHintsFunction2.java index 0ec33fda81b3..7fbbce4eddec 100644 --- a/core/src/main/java/org/apache/calcite/sql/advise/SqlAdvisorGetHintsFunction2.java +++ b/core/src/main/java/org/apache/calcite/sql/advise/SqlAdvisorGetHintsFunction2.java @@ -36,7 +36,7 @@ import com.google.common.collect.Iterables; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Method; import java.lang.reflect.Type; diff --git a/core/src/main/java/org/apache/calcite/sql/advise/SqlAdvisorHint.java b/core/src/main/java/org/apache/calcite/sql/advise/SqlAdvisorHint.java index 9746fc62265e..ae07a5016f6a 100644 --- a/core/src/main/java/org/apache/calcite/sql/advise/SqlAdvisorHint.java +++ b/core/src/main/java/org/apache/calcite/sql/advise/SqlAdvisorHint.java @@ -18,7 +18,7 @@ import org.apache.calcite.sql.validate.SqlMoniker; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/advise/SqlAdvisorHint2.java b/core/src/main/java/org/apache/calcite/sql/advise/SqlAdvisorHint2.java index bf9a405d20cb..73cff70c3e9e 100644 --- a/core/src/main/java/org/apache/calcite/sql/advise/SqlAdvisorHint2.java +++ b/core/src/main/java/org/apache/calcite/sql/advise/SqlAdvisorHint2.java @@ -18,7 +18,7 @@ import org.apache.calcite.sql.validate.SqlMoniker; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * This class is used to return values for diff --git a/core/src/main/java/org/apache/calcite/sql/advise/SqlSimpleParser.java b/core/src/main/java/org/apache/calcite/sql/advise/SqlSimpleParser.java index 86d4c88e941c..61696d544419 100644 --- a/core/src/main/java/org/apache/calcite/sql/advise/SqlSimpleParser.java +++ b/core/src/main/java/org/apache/calcite/sql/advise/SqlSimpleParser.java @@ -19,7 +19,7 @@ import org.apache.calcite.avatica.util.Quoting; import org.apache.calcite.sql.parser.SqlParser; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.EnumSet; diff --git a/core/src/main/java/org/apache/calcite/sql/ddl/SqlAttributeDefinition.java b/core/src/main/java/org/apache/calcite/sql/ddl/SqlAttributeDefinition.java index 494c262959a0..f1a796f3bda6 100644 --- a/core/src/main/java/org/apache/calcite/sql/ddl/SqlAttributeDefinition.java +++ b/core/src/main/java/org/apache/calcite/sql/ddl/SqlAttributeDefinition.java @@ -30,7 +30,7 @@ import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.util.ImmutableNullableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/ddl/SqlCheckConstraint.java b/core/src/main/java/org/apache/calcite/sql/ddl/SqlCheckConstraint.java index 8106f2d1e81c..0bc6ba734a67 100644 --- a/core/src/main/java/org/apache/calcite/sql/ddl/SqlCheckConstraint.java +++ b/core/src/main/java/org/apache/calcite/sql/ddl/SqlCheckConstraint.java @@ -27,7 +27,7 @@ import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.util.ImmutableNullableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/ddl/SqlColumnDeclaration.java b/core/src/main/java/org/apache/calcite/sql/ddl/SqlColumnDeclaration.java index d57fcb502b2b..aa54f671a7e8 100644 --- a/core/src/main/java/org/apache/calcite/sql/ddl/SqlColumnDeclaration.java +++ b/core/src/main/java/org/apache/calcite/sql/ddl/SqlColumnDeclaration.java @@ -29,7 +29,7 @@ import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.util.ImmutableNullableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateForeignSchema.java b/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateForeignSchema.java index e7a4993d8d8d..eabd59fd1775 100644 --- a/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateForeignSchema.java +++ b/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateForeignSchema.java @@ -32,7 +32,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.AbstractList; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateFunction.java b/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateFunction.java index fb859b2013d0..9c74ebec37be 100644 --- a/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateFunction.java @@ -32,7 +32,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateMaterializedView.java b/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateMaterializedView.java index 3894cc276971..3a28c8aeb8ac 100644 --- a/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateMaterializedView.java +++ b/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateMaterializedView.java @@ -29,7 +29,7 @@ import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.util.ImmutableNullableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateSchema.java b/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateSchema.java index d661daadc219..b028b888d06c 100644 --- a/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateSchema.java +++ b/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateSchema.java @@ -28,7 +28,7 @@ import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.util.ImmutableNullableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateTable.java b/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateTable.java index 51d4469dcf42..76c296c965ad 100644 --- a/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateTable.java +++ b/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateTable.java @@ -29,7 +29,7 @@ import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.util.ImmutableNullableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateTableLike.java b/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateTableLike.java index 225ae2d8d855..559f53344635 100644 --- a/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateTableLike.java +++ b/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateTableLike.java @@ -30,7 +30,7 @@ import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.util.ImmutableNullableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.HashSet; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateType.java b/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateType.java index 7e4dc1f632d8..500456b6bae5 100644 --- a/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateType.java +++ b/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateType.java @@ -30,7 +30,7 @@ import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.util.ImmutableNullableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateView.java b/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateView.java index 53a688acfe2c..131f11d86984 100644 --- a/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateView.java +++ b/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateView.java @@ -29,7 +29,7 @@ import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.util.ImmutableNullableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/ddl/SqlDropFunction.java b/core/src/main/java/org/apache/calcite/sql/ddl/SqlDropFunction.java index a8601ba9d0f8..1bd792efd871 100644 --- a/core/src/main/java/org/apache/calcite/sql/ddl/SqlDropFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/ddl/SqlDropFunction.java @@ -25,7 +25,7 @@ import org.apache.calcite.sql.SqlSpecialOperator; import org.apache.calcite.sql.parser.SqlParserPos; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import static java.util.Objects.requireNonNull; diff --git a/core/src/main/java/org/apache/calcite/sql/ddl/SqlDropMaterializedView.java b/core/src/main/java/org/apache/calcite/sql/ddl/SqlDropMaterializedView.java index 09807cbc96a8..5f1d735f323a 100644 --- a/core/src/main/java/org/apache/calcite/sql/ddl/SqlDropMaterializedView.java +++ b/core/src/main/java/org/apache/calcite/sql/ddl/SqlDropMaterializedView.java @@ -25,7 +25,7 @@ import org.apache.calcite.sql.SqlSpecialOperator; import org.apache.calcite.sql.parser.SqlParserPos; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import static java.util.Objects.requireNonNull; diff --git a/core/src/main/java/org/apache/calcite/sql/ddl/SqlDropSchema.java b/core/src/main/java/org/apache/calcite/sql/ddl/SqlDropSchema.java index 95689c29ef25..0820240abcac 100644 --- a/core/src/main/java/org/apache/calcite/sql/ddl/SqlDropSchema.java +++ b/core/src/main/java/org/apache/calcite/sql/ddl/SqlDropSchema.java @@ -29,7 +29,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/ddl/SqlDropTable.java b/core/src/main/java/org/apache/calcite/sql/ddl/SqlDropTable.java index 314686f1e310..7510471b5da5 100644 --- a/core/src/main/java/org/apache/calcite/sql/ddl/SqlDropTable.java +++ b/core/src/main/java/org/apache/calcite/sql/ddl/SqlDropTable.java @@ -25,7 +25,7 @@ import org.apache.calcite.sql.SqlSpecialOperator; import org.apache.calcite.sql.parser.SqlParserPos; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import static java.util.Objects.requireNonNull; diff --git a/core/src/main/java/org/apache/calcite/sql/ddl/SqlDropType.java b/core/src/main/java/org/apache/calcite/sql/ddl/SqlDropType.java index 06ecf7427038..6162b70f4c55 100644 --- a/core/src/main/java/org/apache/calcite/sql/ddl/SqlDropType.java +++ b/core/src/main/java/org/apache/calcite/sql/ddl/SqlDropType.java @@ -25,7 +25,7 @@ import org.apache.calcite.sql.SqlSpecialOperator; import org.apache.calcite.sql.parser.SqlParserPos; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import static java.util.Objects.requireNonNull; diff --git a/core/src/main/java/org/apache/calcite/sql/ddl/SqlDropView.java b/core/src/main/java/org/apache/calcite/sql/ddl/SqlDropView.java index ff2d669a6810..8fca63c87f3d 100644 --- a/core/src/main/java/org/apache/calcite/sql/ddl/SqlDropView.java +++ b/core/src/main/java/org/apache/calcite/sql/ddl/SqlDropView.java @@ -25,7 +25,7 @@ import org.apache.calcite.sql.SqlSpecialOperator; import org.apache.calcite.sql.parser.SqlParserPos; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import static java.util.Objects.requireNonNull; diff --git a/core/src/main/java/org/apache/calcite/sql/ddl/SqlKeyConstraint.java b/core/src/main/java/org/apache/calcite/sql/ddl/SqlKeyConstraint.java index 37ad08be223e..a7423cc67fdd 100644 --- a/core/src/main/java/org/apache/calcite/sql/ddl/SqlKeyConstraint.java +++ b/core/src/main/java/org/apache/calcite/sql/ddl/SqlKeyConstraint.java @@ -28,7 +28,7 @@ import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.util.ImmutableNullableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/ddl/SqlTruncateTable.java b/core/src/main/java/org/apache/calcite/sql/ddl/SqlTruncateTable.java index 1c633c819f72..08bff251fdce 100644 --- a/core/src/main/java/org/apache/calcite/sql/ddl/SqlTruncateTable.java +++ b/core/src/main/java/org/apache/calcite/sql/ddl/SqlTruncateTable.java @@ -29,7 +29,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/dialect/BigQuerySqlDialect.java b/core/src/main/java/org/apache/calcite/sql/dialect/BigQuerySqlDialect.java index 13314a1aeda1..ba56730c32db 100644 --- a/core/src/main/java/org/apache/calcite/sql/dialect/BigQuerySqlDialect.java +++ b/core/src/main/java/org/apache/calcite/sql/dialect/BigQuerySqlDialect.java @@ -48,7 +48,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Arrays; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/dialect/ClickHouseSqlDialect.java b/core/src/main/java/org/apache/calcite/sql/dialect/ClickHouseSqlDialect.java index fd5a719e1a38..49c51ab8bd63 100644 --- a/core/src/main/java/org/apache/calcite/sql/dialect/ClickHouseSqlDialect.java +++ b/core/src/main/java/org/apache/calcite/sql/dialect/ClickHouseSqlDialect.java @@ -45,7 +45,7 @@ import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.util.RelToSqlConverterUtil; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Locale; diff --git a/core/src/main/java/org/apache/calcite/sql/dialect/DorisSqlDialect.java b/core/src/main/java/org/apache/calcite/sql/dialect/DorisSqlDialect.java index 3f7e10d00265..ba2c27fa750f 100644 --- a/core/src/main/java/org/apache/calcite/sql/dialect/DorisSqlDialect.java +++ b/core/src/main/java/org/apache/calcite/sql/dialect/DorisSqlDialect.java @@ -25,7 +25,7 @@ import org.apache.calcite.sql.SqlWriter; import org.apache.calcite.sql.fun.SqlFloorFunction; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import static org.apache.calcite.util.RelToSqlConverterUtil.unparseSparkArrayAndMap; diff --git a/core/src/main/java/org/apache/calcite/sql/dialect/ExasolSqlDialect.java b/core/src/main/java/org/apache/calcite/sql/dialect/ExasolSqlDialect.java index e3122b41a165..5a47ccbb0399 100644 --- a/core/src/main/java/org/apache/calcite/sql/dialect/ExasolSqlDialect.java +++ b/core/src/main/java/org/apache/calcite/sql/dialect/ExasolSqlDialect.java @@ -24,7 +24,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; import java.util.Locale; diff --git a/core/src/main/java/org/apache/calcite/sql/dialect/FireboltSqlDialect.java b/core/src/main/java/org/apache/calcite/sql/dialect/FireboltSqlDialect.java index 0b2d9e0e3490..12eee4a05dd1 100644 --- a/core/src/main/java/org/apache/calcite/sql/dialect/FireboltSqlDialect.java +++ b/core/src/main/java/org/apache/calcite/sql/dialect/FireboltSqlDialect.java @@ -39,7 +39,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Arrays; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/dialect/HiveSqlDialect.java b/core/src/main/java/org/apache/calcite/sql/dialect/HiveSqlDialect.java index 689ecc1ecf60..41893b1227d9 100644 --- a/core/src/main/java/org/apache/calcite/sql/dialect/HiveSqlDialect.java +++ b/core/src/main/java/org/apache/calcite/sql/dialect/HiveSqlDialect.java @@ -40,7 +40,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import static org.apache.calcite.util.RelToSqlConverterUtil.unparseSparkArrayAndMap; diff --git a/core/src/main/java/org/apache/calcite/sql/dialect/HsqldbSqlDialect.java b/core/src/main/java/org/apache/calcite/sql/dialect/HsqldbSqlDialect.java index e5b47913589a..1ae53bdaa60e 100644 --- a/core/src/main/java/org/apache/calcite/sql/dialect/HsqldbSqlDialect.java +++ b/core/src/main/java/org/apache/calcite/sql/dialect/HsqldbSqlDialect.java @@ -33,7 +33,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * A SqlDialect implementation for the Hsqldb database. diff --git a/core/src/main/java/org/apache/calcite/sql/dialect/JethroDataSqlDialect.java b/core/src/main/java/org/apache/calcite/sql/dialect/JethroDataSqlDialect.java index 0dddb6de0522..c1e2c377931a 100644 --- a/core/src/main/java/org/apache/calcite/sql/dialect/JethroDataSqlDialect.java +++ b/core/src/main/java/org/apache/calcite/sql/dialect/JethroDataSqlDialect.java @@ -28,7 +28,7 @@ import com.google.common.collect.LinkedHashMultimap; import com.google.common.collect.Multimap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.sql.Connection; import java.sql.DatabaseMetaData; diff --git a/core/src/main/java/org/apache/calcite/sql/dialect/MssqlSqlDialect.java b/core/src/main/java/org/apache/calcite/sql/dialect/MssqlSqlDialect.java index 964fa03f8f91..f93ab2e09ccf 100644 --- a/core/src/main/java/org/apache/calcite/sql/dialect/MssqlSqlDialect.java +++ b/core/src/main/java/org/apache/calcite/sql/dialect/MssqlSqlDialect.java @@ -46,7 +46,7 @@ import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.type.SqlTypeName; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import static org.apache.calcite.util.RelToSqlConverterUtil.unparseBoolLiteralToCondition; diff --git a/core/src/main/java/org/apache/calcite/sql/dialect/MysqlSqlDialect.java b/core/src/main/java/org/apache/calcite/sql/dialect/MysqlSqlDialect.java index 1f114a155331..539a982b4474 100644 --- a/core/src/main/java/org/apache/calcite/sql/dialect/MysqlSqlDialect.java +++ b/core/src/main/java/org/apache/calcite/sql/dialect/MysqlSqlDialect.java @@ -51,7 +51,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; import java.util.Locale; diff --git a/core/src/main/java/org/apache/calcite/sql/dialect/OracleSqlDialect.java b/core/src/main/java/org/apache/calcite/sql/dialect/OracleSqlDialect.java index 9aab15f5e247..a8e0b3098328 100644 --- a/core/src/main/java/org/apache/calcite/sql/dialect/OracleSqlDialect.java +++ b/core/src/main/java/org/apache/calcite/sql/dialect/OracleSqlDialect.java @@ -41,7 +41,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/dialect/PhoenixSqlDialect.java b/core/src/main/java/org/apache/calcite/sql/dialect/PhoenixSqlDialect.java index c3cebb92ae42..dfb22f323385 100644 --- a/core/src/main/java/org/apache/calcite/sql/dialect/PhoenixSqlDialect.java +++ b/core/src/main/java/org/apache/calcite/sql/dialect/PhoenixSqlDialect.java @@ -27,7 +27,7 @@ import org.apache.calcite.sql.type.AbstractSqlType; import org.apache.calcite.sql.type.SqlTypeName; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * A SqlDialect implementation for the Apache Phoenix database. diff --git a/core/src/main/java/org/apache/calcite/sql/dialect/PostgresqlSqlDialect.java b/core/src/main/java/org/apache/calcite/sql/dialect/PostgresqlSqlDialect.java index 19342fa4059f..4bb2137e3e64 100644 --- a/core/src/main/java/org/apache/calcite/sql/dialect/PostgresqlSqlDialect.java +++ b/core/src/main/java/org/apache/calcite/sql/dialect/PostgresqlSqlDialect.java @@ -52,7 +52,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; import java.util.Locale; diff --git a/core/src/main/java/org/apache/calcite/sql/dialect/PrestoSqlDialect.java b/core/src/main/java/org/apache/calcite/sql/dialect/PrestoSqlDialect.java index 1d8479aa5811..a3aeb7b15910 100644 --- a/core/src/main/java/org/apache/calcite/sql/dialect/PrestoSqlDialect.java +++ b/core/src/main/java/org/apache/calcite/sql/dialect/PrestoSqlDialect.java @@ -49,7 +49,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/dialect/RedshiftSqlDialect.java b/core/src/main/java/org/apache/calcite/sql/dialect/RedshiftSqlDialect.java index 7caf2ab2be24..5da2ff007a2b 100644 --- a/core/src/main/java/org/apache/calcite/sql/dialect/RedshiftSqlDialect.java +++ b/core/src/main/java/org/apache/calcite/sql/dialect/RedshiftSqlDialect.java @@ -28,7 +28,7 @@ import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.sql.type.SqlTypeName; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * A SqlDialect implementation for the Redshift database. diff --git a/core/src/main/java/org/apache/calcite/sql/dialect/SnowflakeSqlDialect.java b/core/src/main/java/org/apache/calcite/sql/dialect/SnowflakeSqlDialect.java index 9deb0836eeea..d736a2aa9eea 100644 --- a/core/src/main/java/org/apache/calcite/sql/dialect/SnowflakeSqlDialect.java +++ b/core/src/main/java/org/apache/calcite/sql/dialect/SnowflakeSqlDialect.java @@ -29,7 +29,7 @@ import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.sql.type.SqlTypeName; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * A SqlDialect implementation for the Snowflake database. diff --git a/core/src/main/java/org/apache/calcite/sql/dialect/SparkSqlDialect.java b/core/src/main/java/org/apache/calcite/sql/dialect/SparkSqlDialect.java index 6f6e3d3917b6..baaab0ef2ddb 100644 --- a/core/src/main/java/org/apache/calcite/sql/dialect/SparkSqlDialect.java +++ b/core/src/main/java/org/apache/calcite/sql/dialect/SparkSqlDialect.java @@ -41,7 +41,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import static org.apache.calcite.util.RelToSqlConverterUtil.unparseHiveTrim; import static org.apache.calcite.util.RelToSqlConverterUtil.unparseSparkArrayAndMap; diff --git a/core/src/main/java/org/apache/calcite/sql/dialect/SqliteSqlDialect.java b/core/src/main/java/org/apache/calcite/sql/dialect/SqliteSqlDialect.java index 833ac5c10d96..53bc71cb03c7 100644 --- a/core/src/main/java/org/apache/calcite/sql/dialect/SqliteSqlDialect.java +++ b/core/src/main/java/org/apache/calcite/sql/dialect/SqliteSqlDialect.java @@ -35,7 +35,7 @@ import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.util.RelToSqlConverterUtil; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * A SqliteSqlDialect implementation for the SQLite database. diff --git a/core/src/main/java/org/apache/calcite/sql/dialect/StarRocksSqlDialect.java b/core/src/main/java/org/apache/calcite/sql/dialect/StarRocksSqlDialect.java index a877623bb2a2..dc2fed680d71 100644 --- a/core/src/main/java/org/apache/calcite/sql/dialect/StarRocksSqlDialect.java +++ b/core/src/main/java/org/apache/calcite/sql/dialect/StarRocksSqlDialect.java @@ -39,7 +39,7 @@ import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.util.RelToSqlConverterUtil; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import static org.apache.calcite.util.RelToSqlConverterUtil.unparseHiveTrim; diff --git a/core/src/main/java/org/apache/calcite/sql/dialect/SybaseSqlDialect.java b/core/src/main/java/org/apache/calcite/sql/dialect/SybaseSqlDialect.java index f62e1d820d61..48d17b48a3f7 100644 --- a/core/src/main/java/org/apache/calcite/sql/dialect/SybaseSqlDialect.java +++ b/core/src/main/java/org/apache/calcite/sql/dialect/SybaseSqlDialect.java @@ -20,7 +20,7 @@ import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlWriter; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import static java.util.Objects.requireNonNull; diff --git a/core/src/main/java/org/apache/calcite/sql/dialect/TrinoSqlDialect.java b/core/src/main/java/org/apache/calcite/sql/dialect/TrinoSqlDialect.java index 5e5beac9e3ad..d2d6abc0b82f 100644 --- a/core/src/main/java/org/apache/calcite/sql/dialect/TrinoSqlDialect.java +++ b/core/src/main/java/org/apache/calcite/sql/dialect/TrinoSqlDialect.java @@ -25,7 +25,7 @@ import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.util.RelToSqlConverterUtil; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * A SqlDialect implementation for the Trino database. diff --git a/core/src/main/java/org/apache/calcite/sql/dialect/VerticaSqlDialect.java b/core/src/main/java/org/apache/calcite/sql/dialect/VerticaSqlDialect.java index 1b28d9e10986..17d6b037c811 100644 --- a/core/src/main/java/org/apache/calcite/sql/dialect/VerticaSqlDialect.java +++ b/core/src/main/java/org/apache/calcite/sql/dialect/VerticaSqlDialect.java @@ -23,7 +23,7 @@ import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.SqlWriter; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlAbstractGroupFunction.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlAbstractGroupFunction.java index a4aa00477c7b..13aecbd3d9ba 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlAbstractGroupFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlAbstractGroupFunction.java @@ -34,7 +34,7 @@ import org.apache.calcite.util.Optionality; import org.apache.calcite.util.Static; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import static java.util.Objects.requireNonNull; diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlAnyValueAggFunction.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlAnyValueAggFunction.java index f83e4f129fd1..284223fa767d 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlAnyValueAggFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlAnyValueAggFunction.java @@ -24,7 +24,7 @@ import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.util.Optionality; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import static com.google.common.base.Preconditions.checkArgument; diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlAvgAggFunction.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlAvgAggFunction.java index 93ccca514150..b25916ecd82e 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlAvgAggFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlAvgAggFunction.java @@ -27,7 +27,7 @@ import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.util.Optionality; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import static com.google.common.base.Preconditions.checkArgument; diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlBasicAggFunction.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlBasicAggFunction.java index 76b7e4de4489..6576553d0cc6 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlBasicAggFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlBasicAggFunction.java @@ -32,7 +32,7 @@ import org.apache.calcite.sql.validate.SqlValidatorScope; import org.apache.calcite.util.Optionality; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import static java.util.Objects.requireNonNull; diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlBitOpAggFunction.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlBitOpAggFunction.java index 2f60a1185979..b908da9923a4 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlBitOpAggFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlBitOpAggFunction.java @@ -24,7 +24,7 @@ import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.util.Optionality; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import static com.google.common.base.Preconditions.checkArgument; diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlCallFactory.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlCallFactory.java index 015f482b7c4b..550bf6c5c79e 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlCallFactory.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlCallFactory.java @@ -22,7 +22,7 @@ import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.parser.SqlParserPos; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * A factory for creating {@link org.apache.calcite.sql.SqlCall}. diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlCase.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlCase.java index 26ba031a28d4..6cc2277438f2 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlCase.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlCase.java @@ -25,7 +25,7 @@ import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.util.UnmodifiableArrayList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlCaseOperator.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlCaseOperator.java index 4d5cbf9a6dc4..403200956012 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlCaseOperator.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlCaseOperator.java @@ -47,7 +47,7 @@ import com.google.common.collect.Iterables; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlCoalesceFunction.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlCoalesceFunction.java index 4a3fc60282e7..1a437191a162 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlCoalesceFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlCoalesceFunction.java @@ -35,7 +35,7 @@ import org.apache.calcite.sql.validate.implicit.TypeCoercion; import org.apache.calcite.util.Util; -import org.checkerframework.checker.nullness.qual.NonNull; +import org.jspecify.annotations.NonNull; import java.util.ArrayList; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlConvertFunction.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlConvertFunction.java index a57e59aa908c..c0a23611a23a 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlConvertFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlConvertFunction.java @@ -41,7 +41,7 @@ import org.apache.calcite.sql.validate.SqlValidator; import org.apache.calcite.sql.validate.SqlValidatorScope; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.nio.charset.Charset; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlCountAggFunction.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlCountAggFunction.java index 3007ad346593..7429365f6559 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlCountAggFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlCountAggFunction.java @@ -35,7 +35,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlFloorFunction.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlFloorFunction.java index 784cf4d22734..18ddbe98bc46 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlFloorFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlFloorFunction.java @@ -37,7 +37,7 @@ import org.apache.calcite.sql.validate.SqlValidator; import org.apache.calcite.sql.validate.SqlValidatorScope; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import static com.google.common.base.Preconditions.checkArgument; diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlGroupingFunction.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlGroupingFunction.java index caf4aefc871a..499611ef60f4 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlGroupingFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlGroupingFunction.java @@ -28,7 +28,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.math.BigDecimal; diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlInternalOperators.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlInternalOperators.java index d1429f501b04..bb15edb98493 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlInternalOperators.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlInternalOperators.java @@ -34,7 +34,7 @@ import org.apache.calcite.sql.type.SqlTypeTransforms; import org.apache.calcite.util.Litmus; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlJsonArrayAggAggFunction.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlJsonArrayAggAggFunction.java index cce6f38ff2bd..c491fb9197ff 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlJsonArrayAggAggFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlJsonArrayAggAggFunction.java @@ -34,7 +34,7 @@ import org.apache.calcite.sql.validate.SqlValidatorScope; import org.apache.calcite.util.Optionality; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import static java.util.Objects.requireNonNull; diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlJsonArrayFunction.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlJsonArrayFunction.java index a0fa2f2766ed..edbdd3f39bfa 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlJsonArrayFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlJsonArrayFunction.java @@ -33,7 +33,7 @@ import org.apache.calcite.sql.type.SqlOperandTypeChecker; import org.apache.calcite.sql.validate.SqlValidator; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Locale; diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlJsonDepthFunction.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlJsonDepthFunction.java index 91a017bca6a6..5deb0386d5f8 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlJsonDepthFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlJsonDepthFunction.java @@ -28,7 +28,7 @@ import org.apache.calcite.sql.type.SqlTypeTransforms; import org.apache.calcite.sql.validate.SqlValidator; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * The JSON_DEPTH function. diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlJsonModifyFunction.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlJsonModifyFunction.java index ba8896bf7c72..e3060af3608a 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlJsonModifyFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlJsonModifyFunction.java @@ -31,7 +31,7 @@ import org.apache.calcite.sql.type.SqlTypeUtil; import org.apache.calcite.sql.validate.SqlValidator; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Locale; diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlJsonObjectFunction.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlJsonObjectFunction.java index 61d8393b39b7..be69ecdf3c07 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlJsonObjectFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlJsonObjectFunction.java @@ -36,7 +36,7 @@ import org.apache.calcite.sql.type.SqlTypeUtil; import org.apache.calcite.sql.validate.SqlValidator; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Locale; diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlJsonPrettyFunction.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlJsonPrettyFunction.java index ab20f71f5e0a..00ef8ceb68be 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlJsonPrettyFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlJsonPrettyFunction.java @@ -28,7 +28,7 @@ import org.apache.calcite.sql.type.SqlTypeTransforms; import org.apache.calcite.sql.validate.SqlValidator; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * The JSON_TYPE function. diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlJsonQueryFunction.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlJsonQueryFunction.java index ba3d0c31bf93..6984718b2aef 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlJsonQueryFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlJsonQueryFunction.java @@ -39,7 +39,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlJsonTypeFunction.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlJsonTypeFunction.java index 12929993659d..1fb64bff210b 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlJsonTypeFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlJsonTypeFunction.java @@ -29,7 +29,7 @@ import org.apache.calcite.sql.type.SqlTypeTransforms; import org.apache.calcite.sql.validate.SqlValidator; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * The JSON_TYPE function. diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlJsonValueFunction.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlJsonValueFunction.java index 005b4f46134a..cf4e53713f5c 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlJsonValueFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlJsonValueFunction.java @@ -37,7 +37,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.Arrays; diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlLibrary.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlLibrary.java index 2bdb1e8535d0..dc127ebc18c4 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlLibrary.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlLibrary.java @@ -22,7 +22,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Arrays; import java.util.LinkedHashSet; diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java index 174f562a805e..50495094fc16 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java @@ -54,7 +54,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlLiteralAggFunction.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlLiteralAggFunction.java index 007f708f6e9d..c6d6d61997b1 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlLiteralAggFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlLiteralAggFunction.java @@ -29,7 +29,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * The {@code LITERAL_AGG} aggregate function. diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlMapValueConstructor.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlMapValueConstructor.java index e7c013247242..9ad150bf7129 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlMapValueConstructor.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlMapValueConstructor.java @@ -26,7 +26,7 @@ import org.apache.calcite.util.Pair; import org.apache.calcite.util.Util; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlMinMaxAggFunction.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlMinMaxAggFunction.java index 2196a3647c2d..a14db7292076 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlMinMaxAggFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlMinMaxAggFunction.java @@ -29,7 +29,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlMonotonicUnaryFunction.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlMonotonicUnaryFunction.java index 22e9cd58591b..4c17467d1ee1 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlMonotonicUnaryFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlMonotonicUnaryFunction.java @@ -25,7 +25,7 @@ import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.apache.calcite.sql.validate.SqlMonotonicity; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Base class for unary operators such as FLOOR/CEIL which are monotonic for diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlMultisetValueConstructor.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlMultisetValueConstructor.java index 99ee65736f87..f48144c2be9d 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlMultisetValueConstructor.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlMultisetValueConstructor.java @@ -31,7 +31,7 @@ import org.apache.calcite.sql.type.SqlOperandTypeInference; import org.apache.calcite.sql.type.SqlTypeUtil; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlOverlapsOperator.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlOverlapsOperator.java index 550b6c65c512..775d572e8a7c 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlOverlapsOperator.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlOverlapsOperator.java @@ -33,7 +33,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * SqlOverlapsOperator represents the SQL:1999 standard {@code OVERLAPS} diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlQuantifyOperator.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlQuantifyOperator.java index 041d7abf09bb..872dda927a30 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlQuantifyOperator.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlQuantifyOperator.java @@ -32,7 +32,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlRowOperator.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlRowOperator.java index eaa77f03d55e..c2abcdc30942 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlRowOperator.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlRowOperator.java @@ -31,7 +31,7 @@ import org.apache.calcite.sql.validate.SqlValidatorScope; import org.apache.calcite.util.ImmutableNullableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlSingleValueAggFunction.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlSingleValueAggFunction.java index 18973d2afe8d..bdf2f0484e3a 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlSingleValueAggFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlSingleValueAggFunction.java @@ -28,7 +28,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlSpatialTypeFunctions.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlSpatialTypeFunctions.java index 296ca3de0d5f..61c994caff98 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlSpatialTypeFunctions.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlSpatialTypeFunctions.java @@ -35,7 +35,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.locationtech.jts.geom.Geometry; import java.math.BigDecimal; diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java index d50bc7299e67..f22b64021a4d 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java @@ -81,7 +81,7 @@ import com.google.common.base.Suppliers; import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; import java.util.function.BiConsumer; diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlSumAggFunction.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlSumAggFunction.java index ba77d818f2ff..3dbd165265f5 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlSumAggFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlSumAggFunction.java @@ -28,7 +28,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlSumEmptyIsZeroAggFunction.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlSumEmptyIsZeroAggFunction.java index f3dbf54dae99..dea9b4f9ac32 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlSumEmptyIsZeroAggFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlSumEmptyIsZeroAggFunction.java @@ -29,7 +29,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlTimestampAddFunction.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlTimestampAddFunction.java index ec886d201398..b825537750a4 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlTimestampAddFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlTimestampAddFunction.java @@ -34,7 +34,7 @@ import com.google.common.collect.ImmutableMap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Map; diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlTrimFunction.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlTrimFunction.java index 7373bbc90029..4cda5ecc653c 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlTrimFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlTrimFunction.java @@ -35,7 +35,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Arrays; diff --git a/core/src/main/java/org/apache/calcite/sql/package-info.java b/core/src/main/java/org/apache/calcite/sql/package-info.java index 27a8d137dd45..c71b95dfb1ec 100644 --- a/core/src/main/java/org/apache/calcite/sql/package-info.java +++ b/core/src/main/java/org/apache/calcite/sql/package-info.java @@ -97,6 +97,6 @@ @DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.RETURN) package org.apache.calcite.sql; -import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.framework.qual.DefaultQualifier; import org.checkerframework.framework.qual.TypeUseLocation; +import org.jspecify.annotations.NonNull; diff --git a/core/src/main/java/org/apache/calcite/sql/parser/Span.java b/core/src/main/java/org/apache/calcite/sql/parser/Span.java index c682769e6f6a..c8a257123105 100644 --- a/core/src/main/java/org/apache/calcite/sql/parser/Span.java +++ b/core/src/main/java/org/apache/calcite/sql/parser/Span.java @@ -19,7 +19,7 @@ import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlNodeList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.Collection; diff --git a/core/src/main/java/org/apache/calcite/sql/parser/SqlAbstractParserImpl.java b/core/src/main/java/org/apache/calcite/sql/parser/SqlAbstractParserImpl.java index 389a6fa56152..9977e504f6e8 100644 --- a/core/src/main/java/org/apache/calcite/sql/parser/SqlAbstractParserImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/parser/SqlAbstractParserImpl.java @@ -38,7 +38,7 @@ import com.google.common.collect.Iterables; import org.checkerframework.checker.initialization.qual.UnderInitialization; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.io.Reader; import java.io.StringReader; diff --git a/core/src/main/java/org/apache/calcite/sql/parser/SqlParserPos.java b/core/src/main/java/org/apache/calcite/sql/parser/SqlParserPos.java index ed74719ab332..ba0c508a7cbb 100644 --- a/core/src/main/java/org/apache/calcite/sql/parser/SqlParserPos.java +++ b/core/src/main/java/org/apache/calcite/sql/parser/SqlParserPos.java @@ -20,7 +20,7 @@ import com.google.common.collect.Lists; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.io.Serializable; import java.util.Collection; diff --git a/core/src/main/java/org/apache/calcite/sql/parser/SqlParserUtil.java b/core/src/main/java/org/apache/calcite/sql/parser/SqlParserUtil.java index 3c0b6bc99b6d..ca47e878b6ff 100644 --- a/core/src/main/java/org/apache/calcite/sql/parser/SqlParserUtil.java +++ b/core/src/main/java/org/apache/calcite/sql/parser/SqlParserUtil.java @@ -57,7 +57,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.slf4j.Logger; import java.io.StringReader; diff --git a/core/src/main/java/org/apache/calcite/sql/parser/StringAndPos.java b/core/src/main/java/org/apache/calcite/sql/parser/StringAndPos.java index c2d8a23ae2e9..43103637ead1 100644 --- a/core/src/main/java/org/apache/calcite/sql/parser/StringAndPos.java +++ b/core/src/main/java/org/apache/calcite/sql/parser/StringAndPos.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.sql.parser; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Objects; diff --git a/core/src/main/java/org/apache/calcite/sql/pretty/SqlPrettyWriter.java b/core/src/main/java/org/apache/calcite/sql/pretty/SqlPrettyWriter.java index 95fdfd27c221..3216aeaaf34b 100644 --- a/core/src/main/java/org/apache/calcite/sql/pretty/SqlPrettyWriter.java +++ b/core/src/main/java/org/apache/calcite/sql/pretty/SqlPrettyWriter.java @@ -31,7 +31,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.slf4j.LoggerFactory; import java.io.PrintWriter; diff --git a/core/src/main/java/org/apache/calcite/sql/type/AbstractSqlType.java b/core/src/main/java/org/apache/calcite/sql/type/AbstractSqlType.java index 16962b08560f..7e66042c3aea 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/AbstractSqlType.java +++ b/core/src/main/java/org/apache/calcite/sql/type/AbstractSqlType.java @@ -22,7 +22,7 @@ import org.apache.calcite.rel.type.RelDataTypeImpl; import org.apache.calcite.rel.type.RelDataTypePrecedenceList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.io.Serializable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/type/ArraySqlType.java b/core/src/main/java/org/apache/calcite/sql/type/ArraySqlType.java index 7d1d1afc470f..8569280be9f5 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/ArraySqlType.java +++ b/core/src/main/java/org/apache/calcite/sql/type/ArraySqlType.java @@ -20,7 +20,7 @@ import org.apache.calcite.rel.type.RelDataTypeFamily; import org.apache.calcite.rel.type.RelDataTypePrecedenceList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Objects; diff --git a/core/src/main/java/org/apache/calcite/sql/type/AssignableOperandTypeChecker.java b/core/src/main/java/org/apache/calcite/sql/type/AssignableOperandTypeChecker.java index 9f65e9ee470d..7f74e3b1e97c 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/AssignableOperandTypeChecker.java +++ b/core/src/main/java/org/apache/calcite/sql/type/AssignableOperandTypeChecker.java @@ -26,7 +26,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/type/BasicSqlType.java b/core/src/main/java/org/apache/calcite/sql/type/BasicSqlType.java index b322c4eaa82e..2a79981d9a01 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/BasicSqlType.java +++ b/core/src/main/java/org/apache/calcite/sql/type/BasicSqlType.java @@ -20,7 +20,7 @@ import org.apache.calcite.sql.SqlCollation; import org.apache.calcite.util.SerializableCharset; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.nio.charset.Charset; diff --git a/core/src/main/java/org/apache/calcite/sql/type/CompositeOperandTypeChecker.java b/core/src/main/java/org/apache/calcite/sql/type/CompositeOperandTypeChecker.java index e3a509bf571e..5eb719cc6869 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/CompositeOperandTypeChecker.java +++ b/core/src/main/java/org/apache/calcite/sql/type/CompositeOperandTypeChecker.java @@ -27,8 +27,8 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; -import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.checker.nullness.qual.UnknownKeyFor; +import org.jspecify.annotations.Nullable; import java.util.AbstractList; import java.util.ArrayList; diff --git a/core/src/main/java/org/apache/calcite/sql/type/CompositeSingleOperandTypeChecker.java b/core/src/main/java/org/apache/calcite/sql/type/CompositeSingleOperandTypeChecker.java index 4f21ff154475..a1b6afecfe0a 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/CompositeSingleOperandTypeChecker.java +++ b/core/src/main/java/org/apache/calcite/sql/type/CompositeSingleOperandTypeChecker.java @@ -22,7 +22,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Allows multiple diff --git a/core/src/main/java/org/apache/calcite/sql/type/CursorReturnTypeInference.java b/core/src/main/java/org/apache/calcite/sql/type/CursorReturnTypeInference.java index c4cc272a8a58..c891860de17c 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/CursorReturnTypeInference.java +++ b/core/src/main/java/org/apache/calcite/sql/type/CursorReturnTypeInference.java @@ -19,7 +19,7 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.sql.SqlOperatorBinding; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Returns the rowtype of a cursor of the operand at a particular 0-based diff --git a/core/src/main/java/org/apache/calcite/sql/type/JavaToSqlTypeConversionRules.java b/core/src/main/java/org/apache/calcite/sql/type/JavaToSqlTypeConversionRules.java index 08fb73c231ee..0824a0bfb0b3 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/JavaToSqlTypeConversionRules.java +++ b/core/src/main/java/org/apache/calcite/sql/type/JavaToSqlTypeConversionRules.java @@ -21,7 +21,7 @@ import com.google.common.collect.ImmutableMap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.locationtech.jts.geom.Geometry; import java.math.BigDecimal; diff --git a/core/src/main/java/org/apache/calcite/sql/type/MapSqlType.java b/core/src/main/java/org/apache/calcite/sql/type/MapSqlType.java index 9ad216057577..faaf015ff4ab 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/MapSqlType.java +++ b/core/src/main/java/org/apache/calcite/sql/type/MapSqlType.java @@ -19,7 +19,7 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFamily; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Objects; diff --git a/core/src/main/java/org/apache/calcite/sql/type/MatchReturnTypeInference.java b/core/src/main/java/org/apache/calcite/sql/type/MatchReturnTypeInference.java index 1426c51c3ec9..613e206f905c 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/MatchReturnTypeInference.java +++ b/core/src/main/java/org/apache/calcite/sql/type/MatchReturnTypeInference.java @@ -21,7 +21,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/type/MultisetSqlType.java b/core/src/main/java/org/apache/calcite/sql/type/MultisetSqlType.java index cbea4062c2ea..e00b9a294519 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/MultisetSqlType.java +++ b/core/src/main/java/org/apache/calcite/sql/type/MultisetSqlType.java @@ -20,7 +20,7 @@ import org.apache.calcite.rel.type.RelDataTypeFamily; import org.apache.calcite.rel.type.RelDataTypePrecedenceList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Objects; diff --git a/core/src/main/java/org/apache/calcite/sql/type/ObjectSqlType.java b/core/src/main/java/org/apache/calcite/sql/type/ObjectSqlType.java index b354688449d0..6fe5bc6f4f05 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/ObjectSqlType.java +++ b/core/src/main/java/org/apache/calcite/sql/type/ObjectSqlType.java @@ -21,7 +21,7 @@ import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.sql.SqlIdentifier; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/type/OperandHandlers.java b/core/src/main/java/org/apache/calcite/sql/type/OperandHandlers.java index b45560390cfc..96f022ff96b4 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/OperandHandlers.java +++ b/core/src/main/java/org/apache/calcite/sql/type/OperandHandlers.java @@ -24,7 +24,7 @@ import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.validate.SqlValidator; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/type/OperandTypes.java b/core/src/main/java/org/apache/calcite/sql/type/OperandTypes.java index 114b11cf169d..fb55e214e2b2 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/OperandTypes.java +++ b/core/src/main/java/org/apache/calcite/sql/type/OperandTypes.java @@ -46,7 +46,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.math.BigDecimal; import java.util.ArrayList; diff --git a/core/src/main/java/org/apache/calcite/sql/type/ReturnTypes.java b/core/src/main/java/org/apache/calcite/sql/type/ReturnTypes.java index 2ad00299f8e2..35a986aed3fe 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/ReturnTypes.java +++ b/core/src/main/java/org/apache/calcite/sql/type/ReturnTypes.java @@ -36,7 +36,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.AbstractList; import java.util.ArrayList; diff --git a/core/src/main/java/org/apache/calcite/sql/type/SameOperandTypeChecker.java b/core/src/main/java/org/apache/calcite/sql/type/SameOperandTypeChecker.java index 80b3ce469f62..baba7a3a6701 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/SameOperandTypeChecker.java +++ b/core/src/main/java/org/apache/calcite/sql/type/SameOperandTypeChecker.java @@ -27,7 +27,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Collections; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/type/SameOperandTypeExceptLastOperandChecker.java b/core/src/main/java/org/apache/calcite/sql/type/SameOperandTypeExceptLastOperandChecker.java index 808a29b59a32..6a41b3f7e871 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/SameOperandTypeExceptLastOperandChecker.java +++ b/core/src/main/java/org/apache/calcite/sql/type/SameOperandTypeExceptLastOperandChecker.java @@ -24,7 +24,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Collections; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/type/SqlOperandTypeChecker.java b/core/src/main/java/org/apache/calcite/sql/type/SqlOperandTypeChecker.java index 2689f2a0dc28..0d4da9e0bde0 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/SqlOperandTypeChecker.java +++ b/core/src/main/java/org/apache/calcite/sql/type/SqlOperandTypeChecker.java @@ -20,7 +20,7 @@ import org.apache.calcite.sql.SqlOperandCountRange; import org.apache.calcite.sql.SqlOperator; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.function.BiFunction; diff --git a/core/src/main/java/org/apache/calcite/sql/type/SqlReturnTypeInference.java b/core/src/main/java/org/apache/calcite/sql/type/SqlReturnTypeInference.java index d832f96c6184..f7b6d119787c 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/SqlReturnTypeInference.java +++ b/core/src/main/java/org/apache/calcite/sql/type/SqlReturnTypeInference.java @@ -20,7 +20,7 @@ import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.SqlOperatorBinding; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Strategy interface to infer the type of an operator call from the type of the diff --git a/core/src/main/java/org/apache/calcite/sql/type/SqlReturnTypeInferenceChain.java b/core/src/main/java/org/apache/calcite/sql/type/SqlReturnTypeInferenceChain.java index 3ed9875b189b..b32d69de1ffe 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/SqlReturnTypeInferenceChain.java +++ b/core/src/main/java/org/apache/calcite/sql/type/SqlReturnTypeInferenceChain.java @@ -21,7 +21,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import static com.google.common.base.Preconditions.checkArgument; diff --git a/core/src/main/java/org/apache/calcite/sql/type/SqlTypeExplicitPrecedenceList.java b/core/src/main/java/org/apache/calcite/sql/type/SqlTypeExplicitPrecedenceList.java index da87c5166ce4..3707174dc7df 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/SqlTypeExplicitPrecedenceList.java +++ b/core/src/main/java/org/apache/calcite/sql/type/SqlTypeExplicitPrecedenceList.java @@ -24,7 +24,7 @@ import com.google.common.collect.ImmutableMap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Arrays; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/type/SqlTypeFactoryImpl.java b/core/src/main/java/org/apache/calcite/sql/type/SqlTypeFactoryImpl.java index 544f9ca708da..3d9c49d01059 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/SqlTypeFactoryImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/type/SqlTypeFactoryImpl.java @@ -25,7 +25,7 @@ import org.apache.calcite.sql.SqlIntervalQualifier; import org.apache.calcite.util.Util; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.nio.charset.Charset; import java.util.ArrayList; diff --git a/core/src/main/java/org/apache/calcite/sql/type/SqlTypeFamily.java b/core/src/main/java/org/apache/calcite/sql/type/SqlTypeFamily.java index 396932555a09..6ff982de7e19 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/SqlTypeFamily.java +++ b/core/src/main/java/org/apache/calcite/sql/type/SqlTypeFamily.java @@ -27,7 +27,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.sql.Types; import java.util.Collection; diff --git a/core/src/main/java/org/apache/calcite/sql/type/SqlTypeName.java b/core/src/main/java/org/apache/calcite/sql/type/SqlTypeName.java index 73557737ca50..65b9b7c89c8c 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/SqlTypeName.java +++ b/core/src/main/java/org/apache/calcite/sql/type/SqlTypeName.java @@ -29,7 +29,7 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Sets; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.math.BigDecimal; import java.sql.Types; diff --git a/core/src/main/java/org/apache/calcite/sql/type/SqlTypeTransformCascade.java b/core/src/main/java/org/apache/calcite/sql/type/SqlTypeTransformCascade.java index ae7bdee528ab..39ddec4280df 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/SqlTypeTransformCascade.java +++ b/core/src/main/java/org/apache/calcite/sql/type/SqlTypeTransformCascade.java @@ -21,7 +21,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import static com.google.common.base.Preconditions.checkArgument; diff --git a/core/src/main/java/org/apache/calcite/sql/type/SqlTypeUtil.java b/core/src/main/java/org/apache/calcite/sql/type/SqlTypeUtil.java index 5de00a50b4c8..e15885cc0e89 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/SqlTypeUtil.java +++ b/core/src/main/java/org/apache/calcite/sql/type/SqlTypeUtil.java @@ -49,7 +49,7 @@ import org.apiguardian.api.API; import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.math.BigDecimal; import java.math.BigInteger; diff --git a/core/src/main/java/org/apache/calcite/sql/type/TableFunctionReturnTypeInference.java b/core/src/main/java/org/apache/calcite/sql/type/TableFunctionReturnTypeInference.java index 5e54a19bad94..9220c2efc43c 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/TableFunctionReturnTypeInference.java +++ b/core/src/main/java/org/apache/calcite/sql/type/TableFunctionReturnTypeInference.java @@ -24,8 +24,8 @@ import org.apache.calcite.sql.SqlOperatorBinding; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; -import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.checker.nullness.qual.RequiresNonNull; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.HashSet; diff --git a/core/src/main/java/org/apache/calcite/sql/util/ChainedSqlOperatorTable.java b/core/src/main/java/org/apache/calcite/sql/util/ChainedSqlOperatorTable.java index 2cc599d81748..4fbf1400e2fe 100644 --- a/core/src/main/java/org/apache/calcite/sql/util/ChainedSqlOperatorTable.java +++ b/core/src/main/java/org/apache/calcite/sql/util/ChainedSqlOperatorTable.java @@ -25,7 +25,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/util/IdPair.java b/core/src/main/java/org/apache/calcite/sql/util/IdPair.java index 1ecbdf457ef7..3870b3421685 100644 --- a/core/src/main/java/org/apache/calcite/sql/util/IdPair.java +++ b/core/src/main/java/org/apache/calcite/sql/util/IdPair.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.sql.util; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; import java.util.Map; diff --git a/core/src/main/java/org/apache/calcite/sql/util/ListSqlOperatorTable.java b/core/src/main/java/org/apache/calcite/sql/util/ListSqlOperatorTable.java index 05229e4eb5e6..522d31f9f8e8 100644 --- a/core/src/main/java/org/apache/calcite/sql/util/ListSqlOperatorTable.java +++ b/core/src/main/java/org/apache/calcite/sql/util/ListSqlOperatorTable.java @@ -27,7 +27,7 @@ import com.google.common.collect.ImmutableSet; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/util/ReflectiveSqlOperatorTable.java b/core/src/main/java/org/apache/calcite/sql/util/ReflectiveSqlOperatorTable.java index 545afc73ae99..be37a23d19bd 100644 --- a/core/src/main/java/org/apache/calcite/sql/util/ReflectiveSqlOperatorTable.java +++ b/core/src/main/java/org/apache/calcite/sql/util/ReflectiveSqlOperatorTable.java @@ -30,7 +30,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Field; import java.util.ArrayList; diff --git a/core/src/main/java/org/apache/calcite/sql/util/SqlBasicVisitor.java b/core/src/main/java/org/apache/calcite/sql/util/SqlBasicVisitor.java index 1a4fa559a415..effc9edcb564 100644 --- a/core/src/main/java/org/apache/calcite/sql/util/SqlBasicVisitor.java +++ b/core/src/main/java/org/apache/calcite/sql/util/SqlBasicVisitor.java @@ -25,7 +25,7 @@ import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlNodeList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Basic implementation of {@link SqlVisitor} which does nothing at each node. diff --git a/core/src/main/java/org/apache/calcite/sql/util/SqlShuttle.java b/core/src/main/java/org/apache/calcite/sql/util/SqlShuttle.java index 6ca2000642e4..5e948f6943dc 100644 --- a/core/src/main/java/org/apache/calcite/sql/util/SqlShuttle.java +++ b/core/src/main/java/org/apache/calcite/sql/util/SqlShuttle.java @@ -25,7 +25,7 @@ import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlNodeList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/util/SqlString.java b/core/src/main/java/org/apache/calcite/sql/util/SqlString.java index 1c105b8c91d6..3848f4c78b2d 100644 --- a/core/src/main/java/org/apache/calcite/sql/util/SqlString.java +++ b/core/src/main/java/org/apache/calcite/sql/util/SqlString.java @@ -20,8 +20,8 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.dataflow.qual.Pure; +import org.jspecify.annotations.Nullable; import static java.util.Objects.requireNonNull; diff --git a/core/src/main/java/org/apache/calcite/sql/validate/AbstractNamespace.java b/core/src/main/java/org/apache/calcite/sql/validate/AbstractNamespace.java index 4bfa1a80e014..d15148b107f1 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/AbstractNamespace.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/AbstractNamespace.java @@ -25,7 +25,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/validate/AggFinder.java b/core/src/main/java/org/apache/calcite/sql/validate/AggFinder.java index 6a988401bfff..30bbba8c320d 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/AggFinder.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/AggFinder.java @@ -22,7 +22,7 @@ import org.apache.calcite.sql.SqlOperatorTable; import org.apache.calcite.util.Util; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.Iterator; diff --git a/core/src/main/java/org/apache/calcite/sql/validate/AggVisitor.java b/core/src/main/java/org/apache/calcite/sql/validate/AggVisitor.java index c8583e512019..2b1f51e9598f 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/AggVisitor.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/AggVisitor.java @@ -26,7 +26,7 @@ import org.apache.calcite.sql.fun.SqlAbstractGroupFunction; import org.apache.calcite.sql.util.SqlBasicVisitor; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/validate/AliasNamespace.java b/core/src/main/java/org/apache/calcite/sql/validate/AliasNamespace.java index 99f84154e77b..f1b5bc88cc4c 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/AliasNamespace.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/AliasNamespace.java @@ -33,7 +33,7 @@ import org.apache.calcite.util.Pair; import org.apache.calcite.util.Util; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/validate/CollectNamespace.java b/core/src/main/java/org/apache/calcite/sql/validate/CollectNamespace.java index f2f153129402..cdba5b3edacd 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/CollectNamespace.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/CollectNamespace.java @@ -21,7 +21,7 @@ import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlNode; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Namespace for COLLECT and TABLE constructs. diff --git a/core/src/main/java/org/apache/calcite/sql/validate/CollectScope.java b/core/src/main/java/org/apache/calcite/sql/validate/CollectScope.java index 17678bdd76d4..0fdcc50707a7 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/CollectScope.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/CollectScope.java @@ -19,7 +19,7 @@ import org.apache.calcite.sql.SqlCall; import org.apache.calcite.sql.SqlNode; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * The name-resolution context for expression inside a multiset call. The diff --git a/core/src/main/java/org/apache/calcite/sql/validate/DelegatingNamespace.java b/core/src/main/java/org/apache/calcite/sql/validate/DelegatingNamespace.java index 21f59a3dfde3..f00939874d3b 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/DelegatingNamespace.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/DelegatingNamespace.java @@ -21,7 +21,7 @@ import org.apache.calcite.sql.SqlNode; import org.apache.calcite.util.Pair; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/validate/DelegatingScope.java b/core/src/main/java/org/apache/calcite/sql/validate/DelegatingScope.java index 16a0e3a53b4d..fd10d5c4b418 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/DelegatingScope.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/DelegatingScope.java @@ -37,7 +37,7 @@ import com.google.common.base.Suppliers; import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.Arrays; diff --git a/core/src/main/java/org/apache/calcite/sql/validate/DelegatingSqlValidatorCatalogReader.java b/core/src/main/java/org/apache/calcite/sql/validate/DelegatingSqlValidatorCatalogReader.java index 6f9b7531cbd2..c996133d766d 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/DelegatingSqlValidatorCatalogReader.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/DelegatingSqlValidatorCatalogReader.java @@ -19,7 +19,7 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.sql.SqlIdentifier; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/validate/EmptyScope.java b/core/src/main/java/org/apache/calcite/sql/validate/EmptyScope.java index 70b2954ae58c..d6eac25afc03 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/EmptyScope.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/EmptyScope.java @@ -38,7 +38,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.Collection; diff --git a/core/src/main/java/org/apache/calcite/sql/validate/FieldNamespace.java b/core/src/main/java/org/apache/calcite/sql/validate/FieldNamespace.java index 15a0c0a02762..cf1389e39638 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/FieldNamespace.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/FieldNamespace.java @@ -20,7 +20,7 @@ import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.sql.SqlNode; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import static java.util.Objects.requireNonNull; diff --git a/core/src/main/java/org/apache/calcite/sql/validate/IdentifierNamespace.java b/core/src/main/java/org/apache/calcite/sql/validate/IdentifierNamespace.java index f23ddec087ec..17c6fe5c4468 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/IdentifierNamespace.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/IdentifierNamespace.java @@ -29,7 +29,7 @@ import com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/validate/JoinNamespace.java b/core/src/main/java/org/apache/calcite/sql/validate/JoinNamespace.java index c533796acd20..94e93fbd8ba2 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/JoinNamespace.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/JoinNamespace.java @@ -21,7 +21,7 @@ import org.apache.calcite.sql.SqlJoin; import org.apache.calcite.sql.SqlNode; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Namespace representing the row type produced by joining two relations. diff --git a/core/src/main/java/org/apache/calcite/sql/validate/JoinScope.java b/core/src/main/java/org/apache/calcite/sql/validate/JoinScope.java index f03f11201cde..f54847175871 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/JoinScope.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/JoinScope.java @@ -20,7 +20,7 @@ import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlWindow; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import static org.apache.calcite.sql.JoinType.LEFT_ANTI_JOIN; import static org.apache.calcite.sql.JoinType.LEFT_SEMI_JOIN; diff --git a/core/src/main/java/org/apache/calcite/sql/validate/LambdaNamespace.java b/core/src/main/java/org/apache/calcite/sql/validate/LambdaNamespace.java index 216f77fe3140..3781349d6335 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/LambdaNamespace.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/LambdaNamespace.java @@ -20,7 +20,7 @@ import org.apache.calcite.sql.SqlLambda; import org.apache.calcite.sql.SqlNode; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import static java.util.Objects.requireNonNull; diff --git a/core/src/main/java/org/apache/calcite/sql/validate/ListScope.java b/core/src/main/java/org/apache/calcite/sql/validate/ListScope.java index 81a2a070cfaa..2df3697aa0de 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/ListScope.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/ListScope.java @@ -25,7 +25,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.Collection; diff --git a/core/src/main/java/org/apache/calcite/sql/validate/MatchRecognizeNamespace.java b/core/src/main/java/org/apache/calcite/sql/validate/MatchRecognizeNamespace.java index 91b4e70e3925..d70026918bae 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/MatchRecognizeNamespace.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/MatchRecognizeNamespace.java @@ -20,7 +20,7 @@ import org.apache.calcite.sql.SqlMatchRecognize; import org.apache.calcite.sql.SqlNode; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import static java.util.Objects.requireNonNull; diff --git a/core/src/main/java/org/apache/calcite/sql/validate/MeasureScope.java b/core/src/main/java/org/apache/calcite/sql/validate/MeasureScope.java index e43dd3ab8799..28dee4f770b1 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/MeasureScope.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/MeasureScope.java @@ -24,7 +24,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.HashSet; diff --git a/core/src/main/java/org/apache/calcite/sql/validate/OrderByScope.java b/core/src/main/java/org/apache/calcite/sql/validate/OrderByScope.java index b8e58222ad97..76145a94f00f 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/OrderByScope.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/OrderByScope.java @@ -23,7 +23,7 @@ import org.apache.calcite.sql.SqlNodeList; import org.apache.calcite.sql.SqlSelect; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/validate/ParameterNamespace.java b/core/src/main/java/org/apache/calcite/sql/validate/ParameterNamespace.java index 9a2c54e31d05..52acc97022c4 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/ParameterNamespace.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/ParameterNamespace.java @@ -19,7 +19,7 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.sql.SqlNode; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Namespace representing the type of a dynamic parameter. diff --git a/core/src/main/java/org/apache/calcite/sql/validate/ParameterScope.java b/core/src/main/java/org/apache/calcite/sql/validate/ParameterScope.java index 249996be63e5..ee26d29a9058 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/ParameterScope.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/ParameterScope.java @@ -21,7 +21,7 @@ import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlNode; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Map; diff --git a/core/src/main/java/org/apache/calcite/sql/validate/ProcedureNamespace.java b/core/src/main/java/org/apache/calcite/sql/validate/ProcedureNamespace.java index 9d97a252efa5..0c65fffde82e 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/ProcedureNamespace.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/ProcedureNamespace.java @@ -25,7 +25,7 @@ import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.apache.calcite.sql.type.SqlTypeName; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import static java.util.Objects.requireNonNull; diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SchemaNamespace.java b/core/src/main/java/org/apache/calcite/sql/validate/SchemaNamespace.java index 2dce1178c650..62321d952641 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SchemaNamespace.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SchemaNamespace.java @@ -23,7 +23,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SelectScope.java b/core/src/main/java/org/apache/calcite/sql/validate/SelectScope.java index 376a7c6482f0..b61212b627ad 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SelectScope.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SelectScope.java @@ -28,7 +28,7 @@ import org.apache.calcite.util.Pair; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SemanticTable.java b/core/src/main/java/org/apache/calcite/sql/validate/SemanticTable.java index 6b39af7065c7..c874ad44a7bc 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SemanticTable.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SemanticTable.java @@ -18,7 +18,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SetopNamespace.java b/core/src/main/java/org/apache/calcite/sql/validate/SetopNamespace.java index 58c381e80c31..53c537bab8c5 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SetopNamespace.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SetopNamespace.java @@ -22,7 +22,7 @@ import org.apache.calcite.sql.SqlNode; import org.apache.calcite.util.Util; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import static org.apache.calcite.util.Static.RESOURCE; diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlLambdaScope.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlLambdaScope.java index a4ba93d56ab0..574b7d55fe66 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlLambdaScope.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlLambdaScope.java @@ -22,7 +22,7 @@ import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.type.SqlTypeName; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.HashMap; import java.util.Map; diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlMonikerImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlMonikerImpl.java index 0eff356b6b6b..b7c2c4212392 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlMonikerImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlMonikerImpl.java @@ -22,7 +22,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; import java.util.Objects; diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlNameMatcher.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlNameMatcher.java index dee642efa3f1..c3e4df91c5d5 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlNameMatcher.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlNameMatcher.java @@ -22,7 +22,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; import java.util.Map; diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlNameMatchers.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlNameMatchers.java index c01b7187acdb..3ebb511a16d8 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlNameMatchers.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlNameMatchers.java @@ -23,7 +23,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.LinkedHashSet; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlQualified.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlQualified.java index 2ec1099a230b..f41e9e8515f2 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlQualified.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlQualified.java @@ -19,7 +19,7 @@ import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.util.Util; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlScopedShuttle.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlScopedShuttle.java index ca4d55ff7a9a..073a47b5628e 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlScopedShuttle.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlScopedShuttle.java @@ -21,7 +21,7 @@ import org.apache.calcite.sql.util.SqlShuttle; import org.apache.calcite.sql.util.SqlVisitor; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayDeque; import java.util.Deque; diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlUserDefinedAggFunction.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlUserDefinedAggFunction.java index d24d2d6df45d..5af6140ee3c7 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlUserDefinedAggFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlUserDefinedAggFunction.java @@ -29,7 +29,7 @@ import org.apache.calcite.util.Optionality; import org.apache.calcite.util.Util; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * User-defined aggregate function. diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlUserDefinedFunction.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlUserDefinedFunction.java index d75514ca614d..6d54c876b25f 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlUserDefinedFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlUserDefinedFunction.java @@ -30,7 +30,7 @@ import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.apache.calcite.util.Util; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlUserDefinedTableFunction.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlUserDefinedTableFunction.java index 5b524daa772f..1a3880d552be 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlUserDefinedTableFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlUserDefinedTableFunction.java @@ -30,7 +30,7 @@ import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.apache.calcite.util.Util; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Type; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlUserDefinedTableMacro.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlUserDefinedTableMacro.java index 7a8419d828ba..0ff9d1faac52 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlUserDefinedTableMacro.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlUserDefinedTableMacro.java @@ -35,7 +35,7 @@ import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.apache.calcite.util.Util; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidator.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidator.java index 94b90c6789bc..6750235c2651 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidator.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidator.java @@ -53,9 +53,9 @@ import org.apache.calcite.sql.validate.implicit.TypeCoercions; import org.apiguardian.api.API; -import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.dataflow.qual.Pure; import org.immutables.value.Value; +import org.jspecify.annotations.Nullable; import java.util.List; import java.util.Map; diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorCatalogReader.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorCatalogReader.java index 65b12a4d5367..991928d1a275 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorCatalogReader.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorCatalogReader.java @@ -23,7 +23,7 @@ import org.apache.calcite.schema.Wrapper; import org.apache.calcite.sql.SqlIdentifier; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java index 6642a1522b76..7932caf0f7d7 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java @@ -134,11 +134,11 @@ import org.apiguardian.api.API; import org.checkerframework.checker.initialization.qual.UnknownInitialization; import org.checkerframework.checker.nullness.qual.KeyFor; -import org.checkerframework.checker.nullness.qual.NonNull; -import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.checker.nullness.qual.PolyNull; import org.checkerframework.checker.nullness.qual.RequiresNonNull; import org.checkerframework.dataflow.qual.Pure; +import org.jspecify.annotations.NonNull; +import org.jspecify.annotations.Nullable; import org.slf4j.Logger; import java.math.BigDecimal; diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorNamespace.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorNamespace.java index ed857859ba39..23a0f6ceba44 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorNamespace.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorNamespace.java @@ -21,8 +21,8 @@ import org.apache.calcite.sql.SqlNode; import org.apache.calcite.util.Pair; -import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.dataflow.qual.Pure; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorScope.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorScope.java index bbf7dca02ae2..d7493181a1f8 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorScope.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorScope.java @@ -31,7 +31,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.Collection; diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorUtil.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorUtil.java index b4465ab4345a..ace6909646d3 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorUtil.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorUtil.java @@ -70,7 +70,7 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Sets; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.nio.charset.Charset; import java.util.ArrayList; diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorWithHints.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorWithHints.java index d24a218ccf51..184c8ad1e00c 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorWithHints.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorWithHints.java @@ -20,7 +20,7 @@ import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.parser.SqlParserPos; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlWithItemTableRef.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlWithItemTableRef.java index dee61285db14..277ea2c5a5c1 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlWithItemTableRef.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlWithItemTableRef.java @@ -27,7 +27,7 @@ import org.apache.calcite.sql.SqlWithItem; import org.apache.calcite.sql.parser.SqlParserPos; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import static java.util.Objects.requireNonNull; diff --git a/core/src/main/java/org/apache/calcite/sql/validate/TableConstructorNamespace.java b/core/src/main/java/org/apache/calcite/sql/validate/TableConstructorNamespace.java index 58715a6327b9..5c0e2c360137 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/TableConstructorNamespace.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/TableConstructorNamespace.java @@ -20,7 +20,7 @@ import org.apache.calcite.sql.SqlCall; import org.apache.calcite.sql.SqlNode; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import static org.apache.calcite.util.Static.RESOURCE; diff --git a/core/src/main/java/org/apache/calcite/sql/validate/TableNamespace.java b/core/src/main/java/org/apache/calcite/sql/validate/TableNamespace.java index 0b4b38336eca..d6bba2b8444b 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/TableNamespace.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/TableNamespace.java @@ -33,7 +33,7 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; import java.util.Map; diff --git a/core/src/main/java/org/apache/calcite/sql/validate/UnnestNamespace.java b/core/src/main/java/org/apache/calcite/sql/validate/UnnestNamespace.java index 34b6be2a30b0..71c8856fb467 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/UnnestNamespace.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/UnnestNamespace.java @@ -22,7 +22,7 @@ import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlUnnestOperator; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import static java.util.Objects.requireNonNull; diff --git a/core/src/main/java/org/apache/calcite/sql/validate/WithItemNamespace.java b/core/src/main/java/org/apache/calcite/sql/validate/WithItemNamespace.java index dc41d92d3c8e..a8e3c4e06f4a 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/WithItemNamespace.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/WithItemNamespace.java @@ -25,7 +25,7 @@ import org.apache.calcite.sql.SqlWithItem; import org.apache.calcite.util.Pair; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** Very similar to {@link AliasNamespace}. */ class WithItemNamespace extends AbstractNamespace { diff --git a/core/src/main/java/org/apache/calcite/sql/validate/WithItemRecursiveNamespace.java b/core/src/main/java/org/apache/calcite/sql/validate/WithItemRecursiveNamespace.java index 3b92ecc6bc6f..ca771da4fcf2 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/WithItemRecursiveNamespace.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/WithItemRecursiveNamespace.java @@ -23,7 +23,7 @@ import org.apache.calcite.sql.SqlWithItem; import org.apache.calcite.sql.parser.SqlParserPos; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** Very similar to {@link WithItemNamespace} but created only for RECURSIVE queries. */ class WithItemRecursiveNamespace extends WithItemNamespace { diff --git a/core/src/main/java/org/apache/calcite/sql/validate/WithNamespace.java b/core/src/main/java/org/apache/calcite/sql/validate/WithNamespace.java index 286b91d3cce5..ba5ab2dfc5c5 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/WithNamespace.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/WithNamespace.java @@ -22,7 +22,7 @@ import org.apache.calcite.sql.SqlWithItem; import org.apache.calcite.util.Util; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import static java.util.Objects.requireNonNull; diff --git a/core/src/main/java/org/apache/calcite/sql/validate/WithRecursiveScope.java b/core/src/main/java/org/apache/calcite/sql/validate/WithRecursiveScope.java index 1fb33e643446..a023ce9c8e62 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/WithRecursiveScope.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/WithRecursiveScope.java @@ -22,7 +22,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/validate/WithScope.java b/core/src/main/java/org/apache/calcite/sql/validate/WithScope.java index 7d6229895751..22c8dc71a4a1 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/WithScope.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/WithScope.java @@ -22,7 +22,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/validate/implicit/AbstractTypeCoercion.java b/core/src/main/java/org/apache/calcite/sql/validate/implicit/AbstractTypeCoercion.java index 22f6e16c61f1..4266a2a196ca 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/implicit/AbstractTypeCoercion.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/implicit/AbstractTypeCoercion.java @@ -50,7 +50,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.nio.charset.Charset; import java.util.ArrayList; diff --git a/core/src/main/java/org/apache/calcite/sql/validate/implicit/TypeCoercion.java b/core/src/main/java/org/apache/calcite/sql/validate/implicit/TypeCoercion.java index f3d1d528fd99..ae7203c312da 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/implicit/TypeCoercion.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/implicit/TypeCoercion.java @@ -24,7 +24,7 @@ import org.apache.calcite.sql.type.SqlTypeFamily; import org.apache.calcite.sql.validate.SqlValidatorScope; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/validate/implicit/TypeCoercionImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/implicit/TypeCoercionImpl.java index 46dc3af5ed80..57186fe646cf 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/implicit/TypeCoercionImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/implicit/TypeCoercionImpl.java @@ -44,7 +44,7 @@ import org.apache.calcite.sql.validate.SqlValidatorScope; import org.apache.calcite.util.Util; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.math.BigDecimal; import java.util.AbstractList; diff --git a/core/src/main/java/org/apache/calcite/sql2rel/AggConverter.java b/core/src/main/java/org/apache/calcite/sql2rel/AggConverter.java index 0193b17c3e00..6d77076484d6 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/AggConverter.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/AggConverter.java @@ -55,7 +55,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.HashMap; diff --git a/core/src/main/java/org/apache/calcite/sql2rel/CorrelateProjectExtractor.java b/core/src/main/java/org/apache/calcite/sql2rel/CorrelateProjectExtractor.java index 127f4e487941..f903db1893ae 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/CorrelateProjectExtractor.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/CorrelateProjectExtractor.java @@ -40,7 +40,7 @@ import org.apache.calcite.util.ImmutableBitSet; import org.apiguardian.api.API; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.HashMap; diff --git a/core/src/main/java/org/apache/calcite/sql2rel/InitializerExpressionFactory.java b/core/src/main/java/org/apache/calcite/sql2rel/InitializerExpressionFactory.java index e4e632fa53bb..9641255e3024 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/InitializerExpressionFactory.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/InitializerExpressionFactory.java @@ -23,7 +23,7 @@ import org.apache.calcite.schema.ColumnStrategy; import org.apache.calcite.sql.SqlFunction; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; import java.util.function.BiFunction; diff --git a/core/src/main/java/org/apache/calcite/sql2rel/NullInitializerExpressionFactory.java b/core/src/main/java/org/apache/calcite/sql2rel/NullInitializerExpressionFactory.java index 13ac72c37ba1..28c856079a9d 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/NullInitializerExpressionFactory.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/NullInitializerExpressionFactory.java @@ -23,7 +23,7 @@ import org.apache.calcite.schema.ColumnStrategy; import org.apache.calcite.sql.SqlFunction; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; import java.util.function.BiFunction; diff --git a/core/src/main/java/org/apache/calcite/sql2rel/ReflectiveConvertletTable.java b/core/src/main/java/org/apache/calcite/sql2rel/ReflectiveConvertletTable.java index 1b7077d53a7f..ed1028b463eb 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/ReflectiveConvertletTable.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/ReflectiveConvertletTable.java @@ -23,8 +23,8 @@ import org.apache.calcite.sql.parser.SqlParserPos; import org.checkerframework.checker.initialization.qual.UnderInitialization; -import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.checker.nullness.qual.RequiresNonNull; +import org.jspecify.annotations.Nullable; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; diff --git a/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java b/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java index fbf949ac189a..7edfc2a4fd3e 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java @@ -110,8 +110,8 @@ import com.google.common.collect.Sets; import com.google.common.collect.SortedSetMultimap; -import org.checkerframework.checker.nullness.qual.Nullable; import org.immutables.value.Value; +import org.jspecify.annotations.Nullable; import org.slf4j.Logger; import java.math.BigDecimal; diff --git a/core/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java b/core/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java index d47adb17b5e0..7210d4f397e8 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java @@ -79,7 +79,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.math.BigDecimal; import java.util.ArrayList; diff --git a/core/src/main/java/org/apache/calcite/sql2rel/RelStructuredTypeFlattener.java b/core/src/main/java/org/apache/calcite/sql2rel/RelStructuredTypeFlattener.java index 3ab73813eb28..da142b295923 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/RelStructuredTypeFlattener.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/RelStructuredTypeFlattener.java @@ -86,8 +86,8 @@ import com.google.common.collect.Lists; import com.google.common.collect.SortedSetMultimap; -import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.common.value.qual.MinLen; +import org.jspecify.annotations.Nullable; import java.util.ArrayDeque; import java.util.ArrayList; diff --git a/core/src/main/java/org/apache/calcite/sql2rel/SqlRexConvertletTable.java b/core/src/main/java/org/apache/calcite/sql2rel/SqlRexConvertletTable.java index 92562019c327..2b35467d0f1a 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/SqlRexConvertletTable.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/SqlRexConvertletTable.java @@ -18,7 +18,7 @@ import org.apache.calcite.sql.SqlCall; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Collection of {@link SqlRexConvertlet}s. diff --git a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java index c486e8ccded7..3e73ccdf52b2 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java @@ -194,8 +194,8 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; -import org.checkerframework.checker.nullness.qual.Nullable; import org.immutables.value.Value; +import org.jspecify.annotations.Nullable; import org.slf4j.Logger; import java.lang.reflect.Type; diff --git a/core/src/main/java/org/apache/calcite/sql2rel/StandardConvertletTable.java b/core/src/main/java/org/apache/calcite/sql2rel/StandardConvertletTable.java index bff49f3924ea..fa98c332c945 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/StandardConvertletTable.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/StandardConvertletTable.java @@ -87,7 +87,7 @@ import com.google.common.collect.ImmutableList; import org.checkerframework.checker.initialization.qual.UnknownInitialization; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.math.BigDecimal; import java.math.RoundingMode; diff --git a/core/src/main/java/org/apache/calcite/sql2rel/SubQueryConverter.java b/core/src/main/java/org/apache/calcite/sql2rel/SubQueryConverter.java index f1c00864d3c9..fa6096f0d372 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/SubQueryConverter.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/SubQueryConverter.java @@ -19,7 +19,7 @@ import org.apache.calcite.rex.RexNode; import org.apache.calcite.sql.SqlCall; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * SubQueryConverter provides the interface for classes that convert sub-queries diff --git a/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java b/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java index ead86414f228..3df3e9671004 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java @@ -63,7 +63,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.Collections; diff --git a/core/src/main/java/org/apache/calcite/sql2rel/package-info.java b/core/src/main/java/org/apache/calcite/sql2rel/package-info.java index 7e442a1288bc..93b9d97d4a22 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/package-info.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/package-info.java @@ -23,6 +23,6 @@ @DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.RETURN) package org.apache.calcite.sql2rel; -import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.framework.qual.DefaultQualifier; import org.checkerframework.framework.qual.TypeUseLocation; +import org.jspecify.annotations.NonNull; diff --git a/core/src/main/java/org/apache/calcite/statistic/package-info.java b/core/src/main/java/org/apache/calcite/statistic/package-info.java index 016ed58efcea..67012cd687b7 100644 --- a/core/src/main/java/org/apache/calcite/statistic/package-info.java +++ b/core/src/main/java/org/apache/calcite/statistic/package-info.java @@ -25,6 +25,6 @@ @DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.RETURN) package org.apache.calcite.statistic; -import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.framework.qual.DefaultQualifier; import org.checkerframework.framework.qual.TypeUseLocation; +import org.jspecify.annotations.NonNull; diff --git a/core/src/main/java/org/apache/calcite/tools/FrameworkConfig.java b/core/src/main/java/org/apache/calcite/tools/FrameworkConfig.java index 68f727a318ff..b3b579e2f2e3 100644 --- a/core/src/main/java/org/apache/calcite/tools/FrameworkConfig.java +++ b/core/src/main/java/org/apache/calcite/tools/FrameworkConfig.java @@ -32,7 +32,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Interface that describes how to configure planning sessions generated diff --git a/core/src/main/java/org/apache/calcite/tools/Frameworks.java b/core/src/main/java/org/apache/calcite/tools/Frameworks.java index 2af5c7447e16..fa0a8966122f 100644 --- a/core/src/main/java/org/apache/calcite/tools/Frameworks.java +++ b/core/src/main/java/org/apache/calcite/tools/Frameworks.java @@ -46,7 +46,7 @@ import com.google.common.base.Suppliers; import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.sql.Connection; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/tools/Hoist.java b/core/src/main/java/org/apache/calcite/tools/Hoist.java index 1de2cb6c9af5..8d654439d796 100644 --- a/core/src/main/java/org/apache/calcite/tools/Hoist.java +++ b/core/src/main/java/org/apache/calcite/tools/Hoist.java @@ -28,8 +28,8 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; -import org.checkerframework.checker.nullness.qual.Nullable; import org.immutables.value.Value; +import org.jspecify.annotations.Nullable; import java.sql.PreparedStatement; import java.util.ArrayList; diff --git a/core/src/main/java/org/apache/calcite/tools/PigRelBuilder.java b/core/src/main/java/org/apache/calcite/tools/PigRelBuilder.java index 27024aad8cf4..78b0e99a376c 100644 --- a/core/src/main/java/org/apache/calcite/tools/PigRelBuilder.java +++ b/core/src/main/java/org/apache/calcite/tools/PigRelBuilder.java @@ -30,7 +30,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java index 97d2e3080dfe..8a356136796f 100644 --- a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java +++ b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java @@ -136,8 +136,8 @@ import com.google.common.collect.Multiset; import com.google.common.collect.Sets; -import org.checkerframework.checker.nullness.qual.Nullable; import org.immutables.value.Value; +import org.jspecify.annotations.Nullable; import java.math.BigDecimal; import java.util.AbstractList; diff --git a/core/src/main/java/org/apache/calcite/tools/RelBuilderFactory.java b/core/src/main/java/org/apache/calcite/tools/RelBuilderFactory.java index 57f716a8eef6..b0f08131c708 100644 --- a/core/src/main/java/org/apache/calcite/tools/RelBuilderFactory.java +++ b/core/src/main/java/org/apache/calcite/tools/RelBuilderFactory.java @@ -21,7 +21,7 @@ import org.apache.calcite.plan.RelOptSchema; import org.apache.calcite.rel.core.RelFactories; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** A partially-created RelBuilder. * diff --git a/core/src/main/java/org/apache/calcite/tools/RuleSets.java b/core/src/main/java/org/apache/calcite/tools/RuleSets.java index 8ad896148d21..2e282aca4eec 100644 --- a/core/src/main/java/org/apache/calcite/tools/RuleSets.java +++ b/core/src/main/java/org/apache/calcite/tools/RuleSets.java @@ -20,7 +20,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Iterator; diff --git a/core/src/main/java/org/apache/calcite/tools/package-info.java b/core/src/main/java/org/apache/calcite/tools/package-info.java index 91e9a3b6f617..5b37f6e36cfb 100644 --- a/core/src/main/java/org/apache/calcite/tools/package-info.java +++ b/core/src/main/java/org/apache/calcite/tools/package-info.java @@ -23,6 +23,6 @@ @DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.RETURN) package org.apache.calcite.tools; -import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.framework.qual.DefaultQualifier; import org.checkerframework.framework.qual.TypeUseLocation; +import org.jspecify.annotations.NonNull; diff --git a/core/src/main/java/org/apache/calcite/util/Arrow.java b/core/src/main/java/org/apache/calcite/util/Arrow.java index 01584264c90a..3e01a178c8a6 100644 --- a/core/src/main/java/org/apache/calcite/util/Arrow.java +++ b/core/src/main/java/org/apache/calcite/util/Arrow.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.util; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Objects; diff --git a/core/src/main/java/org/apache/calcite/util/BarfingInvocationHandler.java b/core/src/main/java/org/apache/calcite/util/BarfingInvocationHandler.java index 50db7bb4bfc2..1bec8d98139c 100644 --- a/core/src/main/java/org/apache/calcite/util/BarfingInvocationHandler.java +++ b/core/src/main/java/org/apache/calcite/util/BarfingInvocationHandler.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.util; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.InvocationHandler; import java.lang.reflect.Method; diff --git a/core/src/main/java/org/apache/calcite/util/BitString.java b/core/src/main/java/org/apache/calcite/util/BitString.java index 1d9459f7c90e..5190859c96da 100644 --- a/core/src/main/java/org/apache/calcite/util/BitString.java +++ b/core/src/main/java/org/apache/calcite/util/BitString.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.util; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.math.BigInteger; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/util/BlackholeMap.java b/core/src/main/java/org/apache/calcite/util/BlackholeMap.java index f22c72a2d029..430278bf9461 100644 --- a/core/src/main/java/org/apache/calcite/util/BlackholeMap.java +++ b/core/src/main/java/org/apache/calcite/util/BlackholeMap.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.util; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.AbstractMap; import java.util.AbstractSet; diff --git a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java index 948b62f28816..390efb81ee01 100644 --- a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java +++ b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java @@ -128,7 +128,7 @@ import com.google.common.collect.ImmutableMap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Constructor; import java.lang.reflect.Field; diff --git a/core/src/main/java/org/apache/calcite/util/ChunkList.java b/core/src/main/java/org/apache/calcite/util/ChunkList.java index 7f7ea16250ef..46aa606e326b 100644 --- a/core/src/main/java/org/apache/calcite/util/ChunkList.java +++ b/core/src/main/java/org/apache/calcite/util/ChunkList.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.util; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.AbstractSequentialList; import java.util.Arrays; diff --git a/core/src/main/java/org/apache/calcite/util/CompositeMap.java b/core/src/main/java/org/apache/calcite/util/CompositeMap.java index b5f7acd0d7c9..aca080f57267 100644 --- a/core/src/main/java/org/apache/calcite/util/CompositeMap.java +++ b/core/src/main/java/org/apache/calcite/util/CompositeMap.java @@ -20,7 +20,7 @@ import com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.KeyFor; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Collection; import java.util.LinkedHashSet; diff --git a/core/src/main/java/org/apache/calcite/util/ConversionUtil.java b/core/src/main/java/org/apache/calcite/util/ConversionUtil.java index 72c418db3e87..beab62aa6136 100644 --- a/core/src/main/java/org/apache/calcite/util/ConversionUtil.java +++ b/core/src/main/java/org/apache/calcite/util/ConversionUtil.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.util; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.nio.ByteOrder; import java.text.NumberFormat; diff --git a/core/src/main/java/org/apache/calcite/util/DateString.java b/core/src/main/java/org/apache/calcite/util/DateString.java index 86497047f195..41c6672301be 100644 --- a/core/src/main/java/org/apache/calcite/util/DateString.java +++ b/core/src/main/java/org/apache/calcite/util/DateString.java @@ -22,7 +22,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Calendar; import java.util.regex.Pattern; diff --git a/core/src/main/java/org/apache/calcite/util/DelegatingInvocationHandler.java b/core/src/main/java/org/apache/calcite/util/DelegatingInvocationHandler.java index 30e2db00dd70..bc305a62c287 100644 --- a/core/src/main/java/org/apache/calcite/util/DelegatingInvocationHandler.java +++ b/core/src/main/java/org/apache/calcite/util/DelegatingInvocationHandler.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.util; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.InvocationHandler; import java.lang.reflect.InvocationTargetException; diff --git a/core/src/main/java/org/apache/calcite/util/Filterator.java b/core/src/main/java/org/apache/calcite/util/Filterator.java index c3c441031ef8..adda2f08ae36 100644 --- a/core/src/main/java/org/apache/calcite/util/Filterator.java +++ b/core/src/main/java/org/apache/calcite/util/Filterator.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.util; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Iterator; import java.util.NoSuchElementException; diff --git a/core/src/main/java/org/apache/calcite/util/Glossary.java b/core/src/main/java/org/apache/calcite/util/Glossary.java index ddd6a723ef10..6b41f2537afe 100644 --- a/core/src/main/java/org/apache/calcite/util/Glossary.java +++ b/core/src/main/java/org/apache/calcite/util/Glossary.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.util; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * A collection of terms. diff --git a/core/src/main/java/org/apache/calcite/util/Holder.java b/core/src/main/java/org/apache/calcite/util/Holder.java index 78aedcd3eebd..2124562bba5c 100644 --- a/core/src/main/java/org/apache/calcite/util/Holder.java +++ b/core/src/main/java/org/apache/calcite/util/Holder.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.util; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.function.UnaryOperator; diff --git a/core/src/main/java/org/apache/calcite/util/ImmutableBitSet.java b/core/src/main/java/org/apache/calcite/util/ImmutableBitSet.java index 29aac2d42120..521e60c5b7a2 100644 --- a/core/src/main/java/org/apache/calcite/util/ImmutableBitSet.java +++ b/core/src/main/java/org/apache/calcite/util/ImmutableBitSet.java @@ -25,9 +25,9 @@ import com.google.common.collect.Ordering; import org.checkerframework.checker.initialization.qual.UnderInitialization; -import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.checker.nullness.qual.RequiresNonNull; import org.checkerframework.dataflow.qual.Pure; +import org.jspecify.annotations.Nullable; import java.io.Serializable; import java.nio.LongBuffer; diff --git a/core/src/main/java/org/apache/calcite/util/ImmutableIntList.java b/core/src/main/java/org/apache/calcite/util/ImmutableIntList.java index c8448aa9bcbc..ba5cddd4a8bd 100644 --- a/core/src/main/java/org/apache/calcite/util/ImmutableIntList.java +++ b/core/src/main/java/org/apache/calcite/util/ImmutableIntList.java @@ -25,7 +25,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.UnmodifiableListIterator; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Array; import java.util.ArrayList; diff --git a/core/src/main/java/org/apache/calcite/util/ImmutableNullableList.java b/core/src/main/java/org/apache/calcite/util/ImmutableNullableList.java index 34b2ce948d1b..6fe64eeba65d 100644 --- a/core/src/main/java/org/apache/calcite/util/ImmutableNullableList.java +++ b/core/src/main/java/org/apache/calcite/util/ImmutableNullableList.java @@ -20,7 +20,7 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Iterators; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.AbstractList; import java.util.ArrayList; diff --git a/core/src/main/java/org/apache/calcite/util/ImmutableNullableSet.java b/core/src/main/java/org/apache/calcite/util/ImmutableNullableSet.java index 4c886a7c1515..c9a21af250e8 100644 --- a/core/src/main/java/org/apache/calcite/util/ImmutableNullableSet.java +++ b/core/src/main/java/org/apache/calcite/util/ImmutableNullableSet.java @@ -23,8 +23,8 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Iterators; -import org.checkerframework.checker.nullness.qual.NonNull; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.NonNull; +import org.jspecify.annotations.Nullable; import java.util.AbstractSet; import java.util.ArrayList; diff --git a/core/src/main/java/org/apache/calcite/util/IntegerIntervalSet.java b/core/src/main/java/org/apache/calcite/util/IntegerIntervalSet.java index a2972daed655..0bb3a7f38bb9 100644 --- a/core/src/main/java/org/apache/calcite/util/IntegerIntervalSet.java +++ b/core/src/main/java/org/apache/calcite/util/IntegerIntervalSet.java @@ -19,7 +19,7 @@ import org.apache.calcite.linq4j.Enumerator; import org.apache.calcite.linq4j.Linq4j; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.AbstractSet; import java.util.Iterator; diff --git a/core/src/main/java/org/apache/calcite/util/JdbcType.java b/core/src/main/java/org/apache/calcite/util/JdbcType.java index eb5d26475074..f495e6aea8d0 100644 --- a/core/src/main/java/org/apache/calcite/util/JdbcType.java +++ b/core/src/main/java/org/apache/calcite/util/JdbcType.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.util; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.math.BigDecimal; import java.sql.ResultSet; diff --git a/core/src/main/java/org/apache/calcite/util/JdbcTypeImpl.java b/core/src/main/java/org/apache/calcite/util/JdbcTypeImpl.java index aa8d4853de30..3ae906b0cbc6 100644 --- a/core/src/main/java/org/apache/calcite/util/JdbcTypeImpl.java +++ b/core/src/main/java/org/apache/calcite/util/JdbcTypeImpl.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.util; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.math.BigDecimal; import java.sql.ResultSet; diff --git a/core/src/main/java/org/apache/calcite/util/JsonBuilder.java b/core/src/main/java/org/apache/calcite/util/JsonBuilder.java index 600dbd14591d..f814c3a49cb8 100644 --- a/core/src/main/java/org/apache/calcite/util/JsonBuilder.java +++ b/core/src/main/java/org/apache/calcite/util/JsonBuilder.java @@ -20,7 +20,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.LinkedHashMap; diff --git a/core/src/main/java/org/apache/calcite/util/Litmus.java b/core/src/main/java/org/apache/calcite/util/Litmus.java index 863e14964b61..4ac0fadcdecf 100644 --- a/core/src/main/java/org/apache/calcite/util/Litmus.java +++ b/core/src/main/java/org/apache/calcite/util/Litmus.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.util; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.slf4j.helpers.MessageFormatter; /** diff --git a/core/src/main/java/org/apache/calcite/util/MonotonicSupplier.java b/core/src/main/java/org/apache/calcite/util/MonotonicSupplier.java index 861623589f25..f993bfa5a467 100644 --- a/core/src/main/java/org/apache/calcite/util/MonotonicSupplier.java +++ b/core/src/main/java/org/apache/calcite/util/MonotonicSupplier.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.util; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.function.Consumer; import java.util.function.Supplier; diff --git a/core/src/main/java/org/apache/calcite/util/NameMap.java b/core/src/main/java/org/apache/calcite/util/NameMap.java index a41e3cdd3d25..342fc2c58b53 100644 --- a/core/src/main/java/org/apache/calcite/util/NameMap.java +++ b/core/src/main/java/org/apache/calcite/util/NameMap.java @@ -20,7 +20,7 @@ import com.google.common.collect.ImmutableSortedMap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Collections; import java.util.Map; diff --git a/core/src/main/java/org/apache/calcite/util/NameMultimap.java b/core/src/main/java/org/apache/calcite/util/NameMultimap.java index d37f5366c98e..25e04e38e030 100644 --- a/core/src/main/java/org/apache/calcite/util/NameMultimap.java +++ b/core/src/main/java/org/apache/calcite/util/NameMultimap.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.util; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.Collection; diff --git a/core/src/main/java/org/apache/calcite/util/NameSet.java b/core/src/main/java/org/apache/calcite/util/NameSet.java index 26621ee868ad..b024b2b5f0cd 100644 --- a/core/src/main/java/org/apache/calcite/util/NameSet.java +++ b/core/src/main/java/org/apache/calcite/util/NameSet.java @@ -18,7 +18,7 @@ import com.google.common.collect.Maps; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Collection; import java.util.Collections; diff --git a/core/src/main/java/org/apache/calcite/util/NlsString.java b/core/src/main/java/org/apache/calcite/util/NlsString.java index b00041186476..9dc578389bfc 100644 --- a/core/src/main/java/org/apache/calcite/util/NlsString.java +++ b/core/src/main/java/org/apache/calcite/util/NlsString.java @@ -29,8 +29,8 @@ import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; -import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.dataflow.qual.Pure; +import org.jspecify.annotations.Nullable; import java.nio.ByteBuffer; import java.nio.charset.CharacterCodingException; diff --git a/core/src/main/java/org/apache/calcite/util/NumberUtil.java b/core/src/main/java/org/apache/calcite/util/NumberUtil.java index cef004ad3eec..fb9bd1c8d833 100644 --- a/core/src/main/java/org/apache/calcite/util/NumberUtil.java +++ b/core/src/main/java/org/apache/calcite/util/NumberUtil.java @@ -16,8 +16,8 @@ */ package org.apache.calcite.util; -import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.checker.nullness.qual.PolyNull; +import org.jspecify.annotations.Nullable; import java.math.BigDecimal; import java.math.BigInteger; diff --git a/core/src/main/java/org/apache/calcite/util/Pair.java b/core/src/main/java/org/apache/calcite/util/Pair.java index 15f5676740ed..88c27ff21808 100644 --- a/core/src/main/java/org/apache/calcite/util/Pair.java +++ b/core/src/main/java/org/apache/calcite/util/Pair.java @@ -18,7 +18,7 @@ import org.apache.calcite.runtime.PairList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.io.Serializable; import java.util.AbstractList; diff --git a/core/src/main/java/org/apache/calcite/util/PartiallyOrderedSet.java b/core/src/main/java/org/apache/calcite/util/PartiallyOrderedSet.java index 0a307e1e41ca..bf7caab180a9 100644 --- a/core/src/main/java/org/apache/calcite/util/PartiallyOrderedSet.java +++ b/core/src/main/java/org/apache/calcite/util/PartiallyOrderedSet.java @@ -20,7 +20,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.AbstractSet; import java.util.ArrayDeque; diff --git a/core/src/main/java/org/apache/calcite/util/Permutation.java b/core/src/main/java/org/apache/calcite/util/Permutation.java index fd187ce23ec6..a592b318d6c9 100644 --- a/core/src/main/java/org/apache/calcite/util/Permutation.java +++ b/core/src/main/java/org/apache/calcite/util/Permutation.java @@ -22,8 +22,8 @@ import org.apache.calcite.util.mapping.Mappings; import org.checkerframework.checker.initialization.qual.UnknownInitialization; -import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.checker.nullness.qual.RequiresNonNull; +import org.jspecify.annotations.Nullable; import java.util.Arrays; import java.util.Iterator; diff --git a/core/src/main/java/org/apache/calcite/util/PrecedenceClimbingParser.java b/core/src/main/java/org/apache/calcite/util/PrecedenceClimbingParser.java index 7eb5ed698914..62c71bf7edd0 100644 --- a/core/src/main/java/org/apache/calcite/util/PrecedenceClimbingParser.java +++ b/core/src/main/java/org/apache/calcite/util/PrecedenceClimbingParser.java @@ -20,7 +20,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.AbstractList; import java.util.ArrayList; diff --git a/core/src/main/java/org/apache/calcite/util/RangeSets.java b/core/src/main/java/org/apache/calcite/util/RangeSets.java index 743b170cc74b..54d0924ab4dd 100644 --- a/core/src/main/java/org/apache/calcite/util/RangeSets.java +++ b/core/src/main/java/org/apache/calcite/util/RangeSets.java @@ -22,7 +22,7 @@ import com.google.common.collect.RangeSet; import com.google.common.collect.TreeRangeSet; -import org.checkerframework.checker.nullness.qual.NonNull; +import org.jspecify.annotations.NonNull; import java.util.Iterator; import java.util.Set; diff --git a/core/src/main/java/org/apache/calcite/util/ReflectUtil.java b/core/src/main/java/org/apache/calcite/util/ReflectUtil.java index a621743b0d11..1ad3789098f6 100644 --- a/core/src/main/java/org/apache/calcite/util/ReflectUtil.java +++ b/core/src/main/java/org/apache/calcite/util/ReflectUtil.java @@ -21,7 +21,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.annotation.Annotation; import java.lang.reflect.InvocationTargetException; diff --git a/core/src/main/java/org/apache/calcite/util/ReflectiveVisitDispatcher.java b/core/src/main/java/org/apache/calcite/util/ReflectiveVisitDispatcher.java index 6f55e0267180..99ed1bae116f 100644 --- a/core/src/main/java/org/apache/calcite/util/ReflectiveVisitDispatcher.java +++ b/core/src/main/java/org/apache/calcite/util/ReflectiveVisitDispatcher.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.util; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Method; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/util/Sarg.java b/core/src/main/java/org/apache/calcite/util/Sarg.java index 89e4c6e9a1c8..b25762c6de4f 100644 --- a/core/src/main/java/org/apache/calcite/util/Sarg.java +++ b/core/src/main/java/org/apache/calcite/util/Sarg.java @@ -25,7 +25,7 @@ import com.google.common.collect.Range; import com.google.common.collect.RangeSet; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.function.BiConsumer; diff --git a/core/src/main/java/org/apache/calcite/util/SimpleNamespaceContext.java b/core/src/main/java/org/apache/calcite/util/SimpleNamespaceContext.java index 68bab8f134f5..53446f1f71a0 100644 --- a/core/src/main/java/org/apache/calcite/util/SimpleNamespaceContext.java +++ b/core/src/main/java/org/apache/calcite/util/SimpleNamespaceContext.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.util; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Collections; import java.util.HashMap; diff --git a/core/src/main/java/org/apache/calcite/util/Source.java b/core/src/main/java/org/apache/calcite/util/Source.java index 39b535b157b8..378f88059f9a 100644 --- a/core/src/main/java/org/apache/calcite/util/Source.java +++ b/core/src/main/java/org/apache/calcite/util/Source.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.util; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.io.File; import java.io.IOException; diff --git a/core/src/main/java/org/apache/calcite/util/Sources.java b/core/src/main/java/org/apache/calcite/util/Sources.java index 406b1e6ddc57..d93bbb72c8b4 100644 --- a/core/src/main/java/org/apache/calcite/util/Sources.java +++ b/core/src/main/java/org/apache/calcite/util/Sources.java @@ -18,7 +18,7 @@ import com.google.common.io.CharSource; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.io.File; import java.io.IOException; diff --git a/core/src/main/java/org/apache/calcite/util/Template.java b/core/src/main/java/org/apache/calcite/util/Template.java index 11047261af4f..7165dcdcfe57 100644 --- a/core/src/main/java/org/apache/calcite/util/Template.java +++ b/core/src/main/java/org/apache/calcite/util/Template.java @@ -18,7 +18,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.text.MessageFormat; import java.util.ArrayList; diff --git a/core/src/main/java/org/apache/calcite/util/TimeString.java b/core/src/main/java/org/apache/calcite/util/TimeString.java index 856ad167b4a0..6f16e93854e9 100644 --- a/core/src/main/java/org/apache/calcite/util/TimeString.java +++ b/core/src/main/java/org/apache/calcite/util/TimeString.java @@ -22,7 +22,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Strings; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Calendar; import java.util.regex.Pattern; diff --git a/core/src/main/java/org/apache/calcite/util/TimeWithTimeZoneString.java b/core/src/main/java/org/apache/calcite/util/TimeWithTimeZoneString.java index 2087b79af631..08b86e58dbf8 100644 --- a/core/src/main/java/org/apache/calcite/util/TimeWithTimeZoneString.java +++ b/core/src/main/java/org/apache/calcite/util/TimeWithTimeZoneString.java @@ -18,7 +18,7 @@ import org.apache.calcite.avatica.util.DateTimeUtils; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.text.SimpleDateFormat; import java.util.Calendar; diff --git a/core/src/main/java/org/apache/calcite/util/TimestampString.java b/core/src/main/java/org/apache/calcite/util/TimestampString.java index a4b16b2896d9..3f3abdb96275 100644 --- a/core/src/main/java/org/apache/calcite/util/TimestampString.java +++ b/core/src/main/java/org/apache/calcite/util/TimestampString.java @@ -22,7 +22,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Strings; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Calendar; import java.util.TimeZone; diff --git a/core/src/main/java/org/apache/calcite/util/TimestampWithTimeZoneString.java b/core/src/main/java/org/apache/calcite/util/TimestampWithTimeZoneString.java index 52e9eb6d4f05..e9648e651ec9 100644 --- a/core/src/main/java/org/apache/calcite/util/TimestampWithTimeZoneString.java +++ b/core/src/main/java/org/apache/calcite/util/TimestampWithTimeZoneString.java @@ -18,7 +18,7 @@ import org.apache.calcite.avatica.util.DateTimeUtils; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.text.SimpleDateFormat; import java.util.Calendar; diff --git a/core/src/main/java/org/apache/calcite/util/TryThreadLocal.java b/core/src/main/java/org/apache/calcite/util/TryThreadLocal.java index 12e702753ca3..33a90e2e71d5 100644 --- a/core/src/main/java/org/apache/calcite/util/TryThreadLocal.java +++ b/core/src/main/java/org/apache/calcite/util/TryThreadLocal.java @@ -16,8 +16,8 @@ */ package org.apache.calcite.util; -import org.checkerframework.checker.nullness.qual.NonNull; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.NonNull; +import org.jspecify.annotations.Nullable; import java.util.function.Supplier; diff --git a/core/src/main/java/org/apache/calcite/util/Util.java b/core/src/main/java/org/apache/calcite/util/Util.java index 05b9927192ed..63a1bd48b576 100644 --- a/core/src/main/java/org/apache/calcite/util/Util.java +++ b/core/src/main/java/org/apache/calcite/util/Util.java @@ -43,9 +43,9 @@ import com.google.common.collect.Sets; import org.apiguardian.api.API; -import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.checker.nullness.qual.PolyNull; import org.checkerframework.dataflow.qual.Pure; +import org.jspecify.annotations.Nullable; import org.slf4j.Logger; import java.io.BufferedReader; diff --git a/core/src/main/java/org/apache/calcite/util/XmlOutput.java b/core/src/main/java/org/apache/calcite/util/XmlOutput.java index f802621a5a42..94b9136a9c92 100644 --- a/core/src/main/java/org/apache/calcite/util/XmlOutput.java +++ b/core/src/main/java/org/apache/calcite/util/XmlOutput.java @@ -18,7 +18,7 @@ import com.google.common.collect.Lists; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.io.PrintWriter; import java.io.Writer; diff --git a/core/src/main/java/org/apache/calcite/util/format/postgresql/CompiledDateTimeFormat.java b/core/src/main/java/org/apache/calcite/util/format/postgresql/CompiledDateTimeFormat.java index 0941f8f6cb10..932915b97636 100644 --- a/core/src/main/java/org/apache/calcite/util/format/postgresql/CompiledDateTimeFormat.java +++ b/core/src/main/java/org/apache/calcite/util/format/postgresql/CompiledDateTimeFormat.java @@ -22,7 +22,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.text.ParseException; import java.text.ParsePosition; diff --git a/core/src/main/java/org/apache/calcite/util/format/postgresql/PostgresqlDateTimeFormatter.java b/core/src/main/java/org/apache/calcite/util/format/postgresql/PostgresqlDateTimeFormatter.java index b62f24895742..4c5195c87933 100644 --- a/core/src/main/java/org/apache/calcite/util/format/postgresql/PostgresqlDateTimeFormatter.java +++ b/core/src/main/java/org/apache/calcite/util/format/postgresql/PostgresqlDateTimeFormatter.java @@ -34,7 +34,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.text.ParseException; import java.text.ParsePosition; diff --git a/core/src/main/java/org/apache/calcite/util/graph/AttributedDirectedGraph.java b/core/src/main/java/org/apache/calcite/util/graph/AttributedDirectedGraph.java index 15c15a70c3d0..a8d759e7942e 100644 --- a/core/src/main/java/org/apache/calcite/util/graph/AttributedDirectedGraph.java +++ b/core/src/main/java/org/apache/calcite/util/graph/AttributedDirectedGraph.java @@ -19,7 +19,7 @@ import org.apache.calcite.util.Util; import org.checkerframework.checker.initialization.qual.UnknownInitialization; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/util/graph/DefaultDirectedGraph.java b/core/src/main/java/org/apache/calcite/util/graph/DefaultDirectedGraph.java index 71c14ca89ef4..bc1aa18d95d8 100644 --- a/core/src/main/java/org/apache/calcite/util/graph/DefaultDirectedGraph.java +++ b/core/src/main/java/org/apache/calcite/util/graph/DefaultDirectedGraph.java @@ -21,7 +21,7 @@ import org.apiguardian.api.API; import org.checkerframework.checker.initialization.qual.NotOnlyInitialized; import org.checkerframework.checker.initialization.qual.UnknownInitialization; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.Collection; diff --git a/core/src/main/java/org/apache/calcite/util/graph/DefaultEdge.java b/core/src/main/java/org/apache/calcite/util/graph/DefaultEdge.java index 4fb5449c0148..bc007b1a2bc0 100644 --- a/core/src/main/java/org/apache/calcite/util/graph/DefaultEdge.java +++ b/core/src/main/java/org/apache/calcite/util/graph/DefaultEdge.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.util.graph; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import static java.util.Objects.requireNonNull; diff --git a/core/src/main/java/org/apache/calcite/util/graph/DirectedGraph.java b/core/src/main/java/org/apache/calcite/util/graph/DirectedGraph.java index eb15a96b7afe..a1449395a1f0 100644 --- a/core/src/main/java/org/apache/calcite/util/graph/DirectedGraph.java +++ b/core/src/main/java/org/apache/calcite/util/graph/DirectedGraph.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.util.graph; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Collection; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/util/javac/JaninoCompiler.java b/core/src/main/java/org/apache/calcite/util/javac/JaninoCompiler.java index 68ab65df09b2..ca11fe06e400 100644 --- a/core/src/main/java/org/apache/calcite/util/javac/JaninoCompiler.java +++ b/core/src/main/java/org/apache/calcite/util/javac/JaninoCompiler.java @@ -18,11 +18,11 @@ import org.apache.calcite.config.CalciteSystemProperty; -import org.checkerframework.checker.nullness.qual.Nullable; import org.codehaus.commons.compiler.util.resource.MapResourceFinder; import org.codehaus.commons.compiler.util.resource.ResourceFinder; import org.codehaus.janino.JavaSourceClassLoader; import org.codehaus.janino.util.ClassFile; +import org.jspecify.annotations.Nullable; import java.io.File; import java.io.FileOutputStream; diff --git a/core/src/main/java/org/apache/calcite/util/mapping/IntPair.java b/core/src/main/java/org/apache/calcite/util/mapping/IntPair.java index 1c64c9e978b1..c2a016476bae 100644 --- a/core/src/main/java/org/apache/calcite/util/mapping/IntPair.java +++ b/core/src/main/java/org/apache/calcite/util/mapping/IntPair.java @@ -22,7 +22,7 @@ import com.google.common.base.Function; import com.google.common.collect.Ordering; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.AbstractList; import java.util.Comparator; diff --git a/core/src/main/java/org/apache/calcite/util/mapping/Mappings.java b/core/src/main/java/org/apache/calcite/util/mapping/Mappings.java index 0139473bc50d..b24d766d88f8 100644 --- a/core/src/main/java/org/apache/calcite/util/mapping/Mappings.java +++ b/core/src/main/java/org/apache/calcite/util/mapping/Mappings.java @@ -26,7 +26,7 @@ import com.google.errorprone.annotations.CheckReturnValue; import org.checkerframework.checker.initialization.qual.UnknownInitialization; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.AbstractList; import java.util.ArrayList; diff --git a/core/src/main/java/org/apache/calcite/util/package-info.java b/core/src/main/java/org/apache/calcite/util/package-info.java index 714aa1190c82..23eb05e65e2e 100644 --- a/core/src/main/java/org/apache/calcite/util/package-info.java +++ b/core/src/main/java/org/apache/calcite/util/package-info.java @@ -23,6 +23,6 @@ @DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.RETURN) package org.apache.calcite.util; -import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.framework.qual.DefaultQualifier; import org.checkerframework.framework.qual.TypeUseLocation; +import org.jspecify.annotations.NonNull; diff --git a/core/src/main/java/org/apache/calcite/util/trace/CalciteLogger.java b/core/src/main/java/org/apache/calcite/util/trace/CalciteLogger.java index 50888a43a04b..556477def10c 100644 --- a/core/src/main/java/org/apache/calcite/util/trace/CalciteLogger.java +++ b/core/src/main/java/org/apache/calcite/util/trace/CalciteLogger.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.util.trace; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.slf4j.Logger; import static java.util.Objects.requireNonNull; diff --git a/core/src/main/java/org/apache/calcite/util/trace/CalciteTimingTracer.java b/core/src/main/java/org/apache/calcite/util/trace/CalciteTimingTracer.java index 26101a3f572a..992d70b93380 100644 --- a/core/src/main/java/org/apache/calcite/util/trace/CalciteTimingTracer.java +++ b/core/src/main/java/org/apache/calcite/util/trace/CalciteTimingTracer.java @@ -18,7 +18,7 @@ import org.apache.calcite.util.NumberUtil; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.slf4j.Logger; import java.text.DecimalFormat; diff --git a/core/src/main/java/org/apache/calcite/util/trace/CalciteTrace.java b/core/src/main/java/org/apache/calcite/util/trace/CalciteTrace.java index 75c6a3dfb8c6..7aca0fe00cf6 100644 --- a/core/src/main/java/org/apache/calcite/util/trace/CalciteTrace.java +++ b/core/src/main/java/org/apache/calcite/util/trace/CalciteTrace.java @@ -24,7 +24,7 @@ import org.apache.calcite.prepare.Prepare; import org.apache.calcite.rel.rules.DpHyp; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/core/src/test/java/org/apache/calcite/adapter/enumerable/EnumerableCustomAggregateTest.java b/core/src/test/java/org/apache/calcite/adapter/enumerable/EnumerableCustomAggregateTest.java index 8e68b547574e..81b3cd6b878b 100644 --- a/core/src/test/java/org/apache/calcite/adapter/enumerable/EnumerableCustomAggregateTest.java +++ b/core/src/test/java/org/apache/calcite/adapter/enumerable/EnumerableCustomAggregateTest.java @@ -53,7 +53,7 @@ import com.google.common.collect.ImmutableMap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.Test; import java.util.ArrayList; diff --git a/core/src/test/java/org/apache/calcite/adapter/enumerable/RexImplementorTableTest.java b/core/src/test/java/org/apache/calcite/adapter/enumerable/RexImplementorTableTest.java index 6fb3039a61d0..ead6d31f9ccb 100644 --- a/core/src/test/java/org/apache/calcite/adapter/enumerable/RexImplementorTableTest.java +++ b/core/src/test/java/org/apache/calcite/adapter/enumerable/RexImplementorTableTest.java @@ -42,7 +42,7 @@ import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.validate.SqlConformanceEnum; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.Test; import java.lang.reflect.Type; diff --git a/core/src/test/java/org/apache/calcite/adapter/generate/RangeTable.java b/core/src/test/java/org/apache/calcite/adapter/generate/RangeTable.java index 46eb8afd74e0..d284b7344a9d 100644 --- a/core/src/test/java/org/apache/calcite/adapter/generate/RangeTable.java +++ b/core/src/test/java/org/apache/calcite/adapter/generate/RangeTable.java @@ -27,7 +27,7 @@ import org.apache.calcite.schema.impl.AbstractTableQueryable; import org.apache.calcite.sql.type.SqlTypeName; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Map; import java.util.NoSuchElementException; diff --git a/core/src/test/java/org/apache/calcite/jdbc/CalciteRemoteDriverTest.java b/core/src/test/java/org/apache/calcite/jdbc/CalciteRemoteDriverTest.java index 4c15a1476ad6..578586207b28 100644 --- a/core/src/test/java/org/apache/calcite/jdbc/CalciteRemoteDriverTest.java +++ b/core/src/test/java/org/apache/calcite/jdbc/CalciteRemoteDriverTest.java @@ -34,7 +34,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Disabled; diff --git a/core/src/test/java/org/apache/calcite/materialize/CustomMaterializedViewRecognitionRuleTest.java b/core/src/test/java/org/apache/calcite/materialize/CustomMaterializedViewRecognitionRuleTest.java index 7a485ccecbb0..ff17880ef347 100644 --- a/core/src/test/java/org/apache/calcite/materialize/CustomMaterializedViewRecognitionRuleTest.java +++ b/core/src/test/java/org/apache/calcite/materialize/CustomMaterializedViewRecognitionRuleTest.java @@ -47,7 +47,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.Test; import java.util.ArrayList; diff --git a/core/src/test/java/org/apache/calcite/plan/RelWriterTest.java b/core/src/test/java/org/apache/calcite/plan/RelWriterTest.java index a107afd17f0d..29147b9df6c4 100644 --- a/core/src/test/java/org/apache/calcite/plan/RelWriterTest.java +++ b/core/src/test/java/org/apache/calcite/plan/RelWriterTest.java @@ -86,8 +86,8 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; -import org.checkerframework.checker.nullness.qual.Nullable; import org.hamcrest.Matcher; +import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; diff --git a/core/src/test/java/org/apache/calcite/plan/volcano/CollationConversionTest.java b/core/src/test/java/org/apache/calcite/plan/volcano/CollationConversionTest.java index 7dacef208ac9..df12fb4dac09 100644 --- a/core/src/test/java/org/apache/calcite/plan/volcano/CollationConversionTest.java +++ b/core/src/test/java/org/apache/calcite/plan/volcano/CollationConversionTest.java @@ -37,8 +37,8 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; import org.immutables.value.Value; +import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.Test; import java.util.List; diff --git a/core/src/test/java/org/apache/calcite/plan/volcano/ComboRuleTest.java b/core/src/test/java/org/apache/calcite/plan/volcano/ComboRuleTest.java index 60178619c7a3..04ff33acc00f 100644 --- a/core/src/test/java/org/apache/calcite/plan/volcano/ComboRuleTest.java +++ b/core/src/test/java/org/apache/calcite/plan/volcano/ComboRuleTest.java @@ -29,8 +29,8 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; import org.immutables.value.Value; +import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.Test; import java.util.List; diff --git a/core/src/test/java/org/apache/calcite/plan/volcano/MultipleTraitConversionTest.java b/core/src/test/java/org/apache/calcite/plan/volcano/MultipleTraitConversionTest.java index 0322311f143a..6b96f24fed20 100644 --- a/core/src/test/java/org/apache/calcite/plan/volcano/MultipleTraitConversionTest.java +++ b/core/src/test/java/org/apache/calcite/plan/volcano/MultipleTraitConversionTest.java @@ -30,7 +30,7 @@ import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.util.ImmutableIntList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.Test; import java.util.List; diff --git a/core/src/test/java/org/apache/calcite/plan/volcano/PlannerTests.java b/core/src/test/java/org/apache/calcite/plan/volcano/PlannerTests.java index 9e46f13f4864..cb0e52502afe 100644 --- a/core/src/test/java/org/apache/calcite/plan/volcano/PlannerTests.java +++ b/core/src/test/java/org/apache/calcite/plan/volcano/PlannerTests.java @@ -35,8 +35,8 @@ import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.sql.type.SqlTypeFactoryImpl; -import org.checkerframework.checker.nullness.qual.Nullable; import org.immutables.value.Value; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/test/java/org/apache/calcite/plan/volcano/TraitConversionTest.java b/core/src/test/java/org/apache/calcite/plan/volcano/TraitConversionTest.java index b66d50fba931..1b1ebbe77d2f 100644 --- a/core/src/test/java/org/apache/calcite/plan/volcano/TraitConversionTest.java +++ b/core/src/test/java/org/apache/calcite/plan/volcano/TraitConversionTest.java @@ -30,8 +30,8 @@ import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.metadata.RelMetadataQuery; -import org.checkerframework.checker.nullness.qual.Nullable; import org.immutables.value.Value; +import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.Test; import java.util.List; diff --git a/core/src/test/java/org/apache/calcite/plan/volcano/TraitPropagationTest.java b/core/src/test/java/org/apache/calcite/plan/volcano/TraitPropagationTest.java index 0caa7ec3e598..0e8f9e084e47 100644 --- a/core/src/test/java/org/apache/calcite/plan/volcano/TraitPropagationTest.java +++ b/core/src/test/java/org/apache/calcite/plan/volcano/TraitPropagationTest.java @@ -74,8 +74,8 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import org.checkerframework.checker.nullness.qual.Nullable; import org.immutables.value.Value; +import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.Test; import java.sql.Connection; diff --git a/core/src/test/java/org/apache/calcite/plan/volcano/VolcanoPlannerTraitTest.java b/core/src/test/java/org/apache/calcite/plan/volcano/VolcanoPlannerTraitTest.java index c3e9c23a3013..e75d1951f749 100644 --- a/core/src/test/java/org/apache/calcite/plan/volcano/VolcanoPlannerTraitTest.java +++ b/core/src/test/java/org/apache/calcite/plan/volcano/VolcanoPlannerTraitTest.java @@ -44,8 +44,8 @@ import com.google.common.collect.HashMultimap; import com.google.common.collect.Multimap; -import org.checkerframework.checker.nullness.qual.Nullable; import org.immutables.value.Value; +import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; diff --git a/core/src/test/java/org/apache/calcite/prepare/LookupOperatorOverloadsTest.java b/core/src/test/java/org/apache/calcite/prepare/LookupOperatorOverloadsTest.java index a05dbfc82633..0224e8a47240 100644 --- a/core/src/test/java/org/apache/calcite/prepare/LookupOperatorOverloadsTest.java +++ b/core/src/test/java/org/apache/calcite/prepare/LookupOperatorOverloadsTest.java @@ -44,7 +44,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Multimap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.Test; import java.sql.Connection; diff --git a/core/src/test/java/org/apache/calcite/rel/logical/ToLogicalConverterTest.java b/core/src/test/java/org/apache/calcite/rel/logical/ToLogicalConverterTest.java index fb7deb5b8a30..7dc5275f840f 100644 --- a/core/src/test/java/org/apache/calcite/rel/logical/ToLogicalConverterTest.java +++ b/core/src/test/java/org/apache/calcite/rel/logical/ToLogicalConverterTest.java @@ -43,7 +43,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.Test; import java.util.Locale; diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java index 8e2b6efb53b0..b354f6c9538b 100644 --- a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java +++ b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java @@ -112,7 +112,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.Test; import org.opentest4j.TestAbortedException; diff --git a/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java b/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java index ca063c48559b..d7282f89255e 100644 --- a/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java +++ b/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java @@ -57,8 +57,8 @@ import com.google.common.collect.Range; import com.google.common.collect.RangeSet; -import org.checkerframework.checker.nullness.qual.Nullable; import org.hamcrest.Matcher; +import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; diff --git a/core/src/test/java/org/apache/calcite/runtime/EnumerablesTest.java b/core/src/test/java/org/apache/calcite/runtime/EnumerablesTest.java index 575a8bfec651..19e7c51bb813 100644 --- a/core/src/test/java/org/apache/calcite/runtime/EnumerablesTest.java +++ b/core/src/test/java/org/apache/calcite/runtime/EnumerablesTest.java @@ -27,7 +27,7 @@ import com.google.common.collect.Lists; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; diff --git a/core/src/test/java/org/apache/calcite/schema/lookup/FakeLookup.java b/core/src/test/java/org/apache/calcite/schema/lookup/FakeLookup.java index c0eb4d9bc788..f0014e8d5ee9 100644 --- a/core/src/test/java/org/apache/calcite/schema/lookup/FakeLookup.java +++ b/core/src/test/java/org/apache/calcite/schema/lookup/FakeLookup.java @@ -18,7 +18,7 @@ import org.apache.calcite.linq4j.function.Predicate1; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.HashMap; import java.util.Locale; diff --git a/core/src/test/java/org/apache/calcite/schema/lookup/IgnoreCaseLookupTest.java b/core/src/test/java/org/apache/calcite/schema/lookup/IgnoreCaseLookupTest.java index 95aef8473888..2c30c8cd4116 100644 --- a/core/src/test/java/org/apache/calcite/schema/lookup/IgnoreCaseLookupTest.java +++ b/core/src/test/java/org/apache/calcite/schema/lookup/IgnoreCaseLookupTest.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.schema.lookup; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.Test; import java.util.Collections; diff --git a/core/src/test/java/org/apache/calcite/schemas/HrClusteredSchema.java b/core/src/test/java/org/apache/calcite/schemas/HrClusteredSchema.java index 08a644304f58..87cd7b532cbb 100644 --- a/core/src/test/java/org/apache/calcite/schemas/HrClusteredSchema.java +++ b/core/src/test/java/org/apache/calcite/schemas/HrClusteredSchema.java @@ -34,7 +34,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.Arrays; diff --git a/core/src/test/java/org/apache/calcite/sql/test/SqlAdvisorTest.java b/core/src/test/java/org/apache/calcite/sql/test/SqlAdvisorTest.java index 01488fcc4bad..0cff350fa63d 100644 --- a/core/src/test/java/org/apache/calcite/sql/test/SqlAdvisorTest.java +++ b/core/src/test/java/org/apache/calcite/sql/test/SqlAdvisorTest.java @@ -29,7 +29,7 @@ import com.google.common.collect.ImmutableMap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; diff --git a/core/src/test/java/org/apache/calcite/sql/test/SqlPrettyWriterFixture.java b/core/src/test/java/org/apache/calcite/sql/test/SqlPrettyWriterFixture.java index d92eef1badd7..3270a52e92fa 100644 --- a/core/src/test/java/org/apache/calcite/sql/test/SqlPrettyWriterFixture.java +++ b/core/src/test/java/org/apache/calcite/sql/test/SqlPrettyWriterFixture.java @@ -26,7 +26,7 @@ import org.apache.calcite.test.DiffRepository; import org.apache.calcite.util.Litmus; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.io.PrintWriter; import java.io.StringWriter; diff --git a/core/src/test/java/org/apache/calcite/sql/test/SqlPrettyWriterTest.java b/core/src/test/java/org/apache/calcite/sql/test/SqlPrettyWriterTest.java index 4bef9b541d46..26cbd51d7284 100644 --- a/core/src/test/java/org/apache/calcite/sql/test/SqlPrettyWriterTest.java +++ b/core/src/test/java/org/apache/calcite/sql/test/SqlPrettyWriterTest.java @@ -26,7 +26,7 @@ import org.apache.calcite.sql.pretty.SqlPrettyWriter; import org.apache.calcite.test.DiffRepository; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; diff --git a/core/src/test/java/org/apache/calcite/sql/type/RelDataTypeSystemTest.java b/core/src/test/java/org/apache/calcite/sql/type/RelDataTypeSystemTest.java index 58d56c99dd1d..c5858d7f7334 100644 --- a/core/src/test/java/org/apache/calcite/sql/type/RelDataTypeSystemTest.java +++ b/core/src/test/java/org/apache/calcite/sql/type/RelDataTypeSystemTest.java @@ -34,7 +34,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.Test; import java.util.AbstractMap; diff --git a/core/src/test/java/org/apache/calcite/sql/validate/LexCaseSensitiveTest.java b/core/src/test/java/org/apache/calcite/sql/validate/LexCaseSensitiveTest.java index 67e91454915a..665af0929148 100644 --- a/core/src/test/java/org/apache/calcite/sql/validate/LexCaseSensitiveTest.java +++ b/core/src/test/java/org/apache/calcite/sql/validate/LexCaseSensitiveTest.java @@ -33,7 +33,7 @@ import org.apache.calcite.tools.Programs; import org.apache.calcite.tools.ValidationException; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.Test; import java.util.List; diff --git a/core/src/test/java/org/apache/calcite/sql/validate/LexEscapeTest.java b/core/src/test/java/org/apache/calcite/sql/validate/LexEscapeTest.java index 64f5a0a75e52..a311d3bf716d 100644 --- a/core/src/test/java/org/apache/calcite/sql/validate/LexEscapeTest.java +++ b/core/src/test/java/org/apache/calcite/sql/validate/LexEscapeTest.java @@ -40,7 +40,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.Test; import java.util.List; diff --git a/core/src/test/java/org/apache/calcite/sql2rel/CorrelateProjectExtractorTest.java b/core/src/test/java/org/apache/calcite/sql2rel/CorrelateProjectExtractorTest.java index da0a17296b3c..0bee7d0ef1bd 100644 --- a/core/src/test/java/org/apache/calcite/sql2rel/CorrelateProjectExtractorTest.java +++ b/core/src/test/java/org/apache/calcite/sql2rel/CorrelateProjectExtractorTest.java @@ -29,7 +29,7 @@ import org.apache.calcite.tools.RelBuilder; import org.apache.calcite.util.Holder; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.Test; import java.util.List; diff --git a/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java b/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java index 80ef7f588ade..8c2d424133f3 100644 --- a/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java +++ b/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java @@ -51,7 +51,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.Test; import java.util.Collections; diff --git a/core/src/test/java/org/apache/calcite/sql2rel/RelFieldTrimmerTest.java b/core/src/test/java/org/apache/calcite/sql2rel/RelFieldTrimmerTest.java index 535dc07b1602..c1a23161e154 100644 --- a/core/src/test/java/org/apache/calcite/sql2rel/RelFieldTrimmerTest.java +++ b/core/src/test/java/org/apache/calcite/sql2rel/RelFieldTrimmerTest.java @@ -47,7 +47,7 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.Test; import java.util.Collections; diff --git a/core/src/test/java/org/apache/calcite/test/CollectionTypeTest.java b/core/src/test/java/org/apache/calcite/test/CollectionTypeTest.java index 1ea33de0b20f..8db230f4a6fb 100644 --- a/core/src/test/java/org/apache/calcite/test/CollectionTypeTest.java +++ b/core/src/test/java/org/apache/calcite/test/CollectionTypeTest.java @@ -34,7 +34,7 @@ import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.type.SqlTypeName; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.Test; import java.sql.Connection; diff --git a/core/src/test/java/org/apache/calcite/test/CoreQuidemTest.java b/core/src/test/java/org/apache/calcite/test/CoreQuidemTest.java index bbb547b85833..70b9c2468f19 100644 --- a/core/src/test/java/org/apache/calcite/test/CoreQuidemTest.java +++ b/core/src/test/java/org/apache/calcite/test/CoreQuidemTest.java @@ -28,7 +28,7 @@ import net.hydromatic.quidem.CommandHandler; import net.hydromatic.quidem.Quidem; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.sql.Connection; import java.util.Collection; diff --git a/core/src/test/java/org/apache/calcite/test/HepPlannerTest.java b/core/src/test/java/org/apache/calcite/test/HepPlannerTest.java index 1729817e368d..1940d7237653 100644 --- a/core/src/test/java/org/apache/calcite/test/HepPlannerTest.java +++ b/core/src/test/java/org/apache/calcite/test/HepPlannerTest.java @@ -37,7 +37,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Test; diff --git a/core/src/test/java/org/apache/calcite/test/InterpreterTest.java b/core/src/test/java/org/apache/calcite/test/InterpreterTest.java index c138056e727a..e552642afdb8 100644 --- a/core/src/test/java/org/apache/calcite/test/InterpreterTest.java +++ b/core/src/test/java/org/apache/calcite/test/InterpreterTest.java @@ -52,7 +52,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; diff --git a/core/src/test/java/org/apache/calcite/test/JdbcTest.java b/core/src/test/java/org/apache/calcite/test/JdbcTest.java index a75983c76a98..7d5154a90a8b 100644 --- a/core/src/test/java/org/apache/calcite/test/JdbcTest.java +++ b/core/src/test/java/org/apache/calcite/test/JdbcTest.java @@ -106,11 +106,11 @@ import com.google.common.collect.LinkedListMultimap; import com.google.common.collect.Multimap; -import org.checkerframework.checker.nullness.qual.Nullable; import org.hamcrest.Matcher; import org.hamcrest.comparator.ComparatorMatcherBuilder; import org.hamcrest.number.OrderingComparison; import org.hsqldb.jdbcDriver; +import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; diff --git a/core/src/test/java/org/apache/calcite/test/LintTest.java b/core/src/test/java/org/apache/calcite/test/LintTest.java index ef0db448315d..0714beb1a71b 100644 --- a/core/src/test/java/org/apache/calcite/test/LintTest.java +++ b/core/src/test/java/org/apache/calcite/test/LintTest.java @@ -27,7 +27,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.Test; import java.io.File; diff --git a/core/src/test/java/org/apache/calcite/test/MaterializationTest.java b/core/src/test/java/org/apache/calcite/test/MaterializationTest.java index c91f0e75eceb..ad951f286d3f 100644 --- a/core/src/test/java/org/apache/calcite/test/MaterializationTest.java +++ b/core/src/test/java/org/apache/calcite/test/MaterializationTest.java @@ -44,7 +44,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Ordering; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; diff --git a/core/src/test/java/org/apache/calcite/test/MaterializedViewFixture.java b/core/src/test/java/org/apache/calcite/test/MaterializedViewFixture.java index f0269081be26..1cdac2410220 100644 --- a/core/src/test/java/org/apache/calcite/test/MaterializedViewFixture.java +++ b/core/src/test/java/org/apache/calcite/test/MaterializedViewFixture.java @@ -21,7 +21,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.function.Predicate; diff --git a/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java b/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java index ebe8990c1ddc..c3e1290edec5 100644 --- a/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java @@ -103,9 +103,9 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; -import org.checkerframework.checker.nullness.qual.Nullable; import org.hamcrest.FeatureMatcher; import org.hamcrest.Matcher; +import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; diff --git a/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java b/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java index abb934457a0e..efccc070148d 100644 --- a/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java @@ -125,8 +125,8 @@ import com.google.common.collect.Multimap; import com.google.common.collect.Sets; -import org.checkerframework.checker.nullness.qual.NonNull; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.NonNull; +import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index 76ff0a831335..385fe5950c31 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -148,8 +148,8 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import org.checkerframework.checker.nullness.qual.Nullable; import org.immutables.value.Value; +import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; diff --git a/core/src/test/java/org/apache/calcite/test/RuleMatchVisualizerTest.java b/core/src/test/java/org/apache/calcite/test/RuleMatchVisualizerTest.java index 7a1e3fd9a0ea..b4f59e9c8e4b 100644 --- a/core/src/test/java/org/apache/calcite/test/RuleMatchVisualizerTest.java +++ b/core/src/test/java/org/apache/calcite/test/RuleMatchVisualizerTest.java @@ -26,7 +26,7 @@ import org.apache.calcite.rel.RelCollationTraitDef; import org.apache.calcite.rel.rules.CoreRules; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Test; diff --git a/core/src/test/java/org/apache/calcite/test/ScannableTableTest.java b/core/src/test/java/org/apache/calcite/test/ScannableTableTest.java index 976f5abcd60f..fb0be08d2d54 100644 --- a/core/src/test/java/org/apache/calcite/test/ScannableTableTest.java +++ b/core/src/test/java/org/apache/calcite/test/ScannableTableTest.java @@ -45,7 +45,7 @@ import com.google.common.collect.ImmutableMap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.Test; import java.math.BigDecimal; diff --git a/core/src/test/java/org/apache/calcite/test/SqlHintsConverterTest.java b/core/src/test/java/org/apache/calcite/test/SqlHintsConverterTest.java index 89f006526909..47e2eca7a6ac 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlHintsConverterTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlHintsConverterTest.java @@ -75,8 +75,8 @@ import org.apache.calcite.util.Litmus; import org.apache.calcite.util.Util; -import org.checkerframework.checker.nullness.qual.Nullable; import org.immutables.value.Value; +import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; diff --git a/core/src/test/java/org/apache/calcite/test/SqlTestGen.java b/core/src/test/java/org/apache/calcite/test/SqlTestGen.java index 0e7ef7dee2eb..6710a1a7f445 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlTestGen.java +++ b/core/src/test/java/org/apache/calcite/test/SqlTestGen.java @@ -24,7 +24,7 @@ import org.apache.calcite.util.TestUtil; import org.apache.calcite.util.Util; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.io.File; import java.io.PrintWriter; diff --git a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java index d56b7d54375b..f48454851db4 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java @@ -60,7 +60,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorFeatureTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorFeatureTest.java index 24cdf1935d1b..7efaf22dacd4 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorFeatureTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorFeatureTest.java @@ -25,7 +25,7 @@ import org.apache.calcite.sql.validate.SqlValidatorCatalogReader; import org.apache.calcite.sql.validate.SqlValidatorImpl; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.Test; import static org.apache.calcite.util.Static.RESOURCE; diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index 854a5ce8fff2..d97c78af61ea 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -81,7 +81,7 @@ import com.google.common.collect.Ordering; import com.google.common.collect.Sets; -import org.checkerframework.checker.nullness.qual.NonNull; +import org.jspecify.annotations.NonNull; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; diff --git a/core/src/test/java/org/apache/calcite/test/SqlXmlFunctionsTest.java b/core/src/test/java/org/apache/calcite/test/SqlXmlFunctionsTest.java index 046d93570591..03494835a190 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlXmlFunctionsTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlXmlFunctionsTest.java @@ -21,8 +21,8 @@ import org.apache.calcite.runtime.XmlFunctions; import org.apache.calcite.util.BuiltInMethod; -import org.checkerframework.checker.nullness.qual.Nullable; import org.hamcrest.Matcher; +import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; diff --git a/core/src/test/java/org/apache/calcite/test/TCatalogReader.java b/core/src/test/java/org/apache/calcite/test/TCatalogReader.java index c6ebd03869dd..b63028e60bbc 100644 --- a/core/src/test/java/org/apache/calcite/test/TCatalogReader.java +++ b/core/src/test/java/org/apache/calcite/test/TCatalogReader.java @@ -19,7 +19,7 @@ import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.test.catalog.MockCatalogReader; -import org.checkerframework.checker.nullness.qual.NonNull; +import org.jspecify.annotations.NonNull; /** A catalog reader with tables "T1" and "T2" whose schema contains all * test data types. */ diff --git a/core/src/test/java/org/apache/calcite/test/TopDownOptTest.java b/core/src/test/java/org/apache/calcite/test/TopDownOptTest.java index 19b9d2063917..18606417063e 100644 --- a/core/src/test/java/org/apache/calcite/test/TopDownOptTest.java +++ b/core/src/test/java/org/apache/calcite/test/TopDownOptTest.java @@ -24,7 +24,7 @@ import org.apache.calcite.rel.rules.CoreRules; import org.apache.calcite.rel.rules.JoinPushThroughJoinRule; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Test; diff --git a/core/src/test/java/org/apache/calcite/test/TypeCoercionConverterTest.java b/core/src/test/java/org/apache/calcite/test/TypeCoercionConverterTest.java index a12a07d5e57b..2aba21cf7784 100644 --- a/core/src/test/java/org/apache/calcite/test/TypeCoercionConverterTest.java +++ b/core/src/test/java/org/apache/calcite/test/TypeCoercionConverterTest.java @@ -18,7 +18,7 @@ import org.apache.calcite.sql.validate.implicit.TypeCoercion; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Test; diff --git a/core/src/test/java/org/apache/calcite/test/TypeCoercionTest.java b/core/src/test/java/org/apache/calcite/test/TypeCoercionTest.java index 32db4091bfe6..392c34dba3a2 100644 --- a/core/src/test/java/org/apache/calcite/test/TypeCoercionTest.java +++ b/core/src/test/java/org/apache/calcite/test/TypeCoercionTest.java @@ -33,7 +33,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.Test; import java.util.List; diff --git a/core/src/test/java/org/apache/calcite/test/concurrent/ConcurrentTestCommandExecutor.java b/core/src/test/java/org/apache/calcite/test/concurrent/ConcurrentTestCommandExecutor.java index 5a514d451e7d..62c1d31cd8b1 100644 --- a/core/src/test/java/org/apache/calcite/test/concurrent/ConcurrentTestCommandExecutor.java +++ b/core/src/test/java/org/apache/calcite/test/concurrent/ConcurrentTestCommandExecutor.java @@ -18,7 +18,7 @@ import org.apache.calcite.util.Unsafe; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.io.PrintStream; import java.sql.Connection; diff --git a/core/src/test/java/org/apache/calcite/test/concurrent/ConcurrentTestCommandGenerator.java b/core/src/test/java/org/apache/calcite/test/concurrent/ConcurrentTestCommandGenerator.java index 6e146e1a7d5f..7a95a1ab4100 100644 --- a/core/src/test/java/org/apache/calcite/test/concurrent/ConcurrentTestCommandGenerator.java +++ b/core/src/test/java/org/apache/calcite/test/concurrent/ConcurrentTestCommandGenerator.java @@ -21,7 +21,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.io.PrintStream; import java.math.BigDecimal; diff --git a/core/src/test/java/org/apache/calcite/test/concurrent/ConcurrentTestCommandScript.java b/core/src/test/java/org/apache/calcite/test/concurrent/ConcurrentTestCommandScript.java index b61e145f477d..b2cc9d8a28be 100644 --- a/core/src/test/java/org/apache/calcite/test/concurrent/ConcurrentTestCommandScript.java +++ b/core/src/test/java/org/apache/calcite/test/concurrent/ConcurrentTestCommandScript.java @@ -21,7 +21,7 @@ import org.apache.calcite.util.Unsafe; import org.apache.calcite.util.Util; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.io.BufferedReader; import java.io.BufferedWriter; diff --git a/core/src/test/java/org/apache/calcite/test/concurrent/ConcurrentTestPluginCommand.java b/core/src/test/java/org/apache/calcite/test/concurrent/ConcurrentTestPluginCommand.java index 4f49736d7f97..279f90b132d0 100644 --- a/core/src/test/java/org/apache/calcite/test/concurrent/ConcurrentTestPluginCommand.java +++ b/core/src/test/java/org/apache/calcite/test/concurrent/ConcurrentTestPluginCommand.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.test.concurrent; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.sql.Connection; import java.sql.Statement; diff --git a/core/src/test/java/org/apache/calcite/tools/FrameworksTest.java b/core/src/test/java/org/apache/calcite/tools/FrameworksTest.java index b413cc8b0125..cb6f3d040c2b 100644 --- a/core/src/test/java/org/apache/calcite/tools/FrameworksTest.java +++ b/core/src/test/java/org/apache/calcite/tools/FrameworksTest.java @@ -75,7 +75,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.Test; import java.lang.reflect.Type; diff --git a/core/src/test/java/org/apache/calcite/util/TestUnsafe.java b/core/src/test/java/org/apache/calcite/util/TestUnsafe.java index 71b4d83d5a63..e83b66bb328d 100644 --- a/core/src/test/java/org/apache/calcite/util/TestUnsafe.java +++ b/core/src/test/java/org/apache/calcite/util/TestUnsafe.java @@ -18,7 +18,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.slf4j.Logger; import java.io.BufferedInputStream; diff --git a/core/src/test/java/org/apache/calcite/util/UtilTest.java b/core/src/test/java/org/apache/calcite/util/UtilTest.java index c7cf33a66c0a..130a1949e0bf 100644 --- a/core/src/test/java/org/apache/calcite/util/UtilTest.java +++ b/core/src/test/java/org/apache/calcite/util/UtilTest.java @@ -49,13 +49,13 @@ import com.google.common.collect.Lists; import com.google.common.primitives.Ints; -import org.checkerframework.checker.nullness.qual.NonNull; -import org.checkerframework.checker.nullness.qual.Nullable; import org.hamcrest.Description; import org.hamcrest.FeatureMatcher; import org.hamcrest.Matcher; import org.hamcrest.StringDescription; import org.hamcrest.TypeSafeMatcher; +import org.jspecify.annotations.NonNull; +import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.Test; import org.locationtech.jts.util.Assert; diff --git a/core/src/test/java/org/apache/calcite/util/graph/DirectedGraphTest.java b/core/src/test/java/org/apache/calcite/util/graph/DirectedGraphTest.java index dae679bdadeb..355971ee382e 100644 --- a/core/src/test/java/org/apache/calcite/util/graph/DirectedGraphTest.java +++ b/core/src/test/java/org/apache/calcite/util/graph/DirectedGraphTest.java @@ -21,7 +21,7 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.Test; import java.util.ArrayList; diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/BinaryOperatorConversion.java b/druid/src/main/java/org/apache/calcite/adapter/druid/BinaryOperatorConversion.java index ca8a4c5ee345..8bcaa679dafd 100644 --- a/druid/src/main/java/org/apache/calcite/adapter/druid/BinaryOperatorConversion.java +++ b/druid/src/main/java/org/apache/calcite/adapter/druid/BinaryOperatorConversion.java @@ -21,7 +21,7 @@ import org.apache.calcite.rex.RexNode; import org.apache.calcite.sql.SqlOperator; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/CeilOperatorConversion.java b/druid/src/main/java/org/apache/calcite/adapter/druid/CeilOperatorConversion.java index bece7eee8cb1..94cc4b37191c 100644 --- a/druid/src/main/java/org/apache/calcite/adapter/druid/CeilOperatorConversion.java +++ b/druid/src/main/java/org/apache/calcite/adapter/druid/CeilOperatorConversion.java @@ -26,7 +26,7 @@ import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.type.SqlTypeName; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.TimeZone; diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/DefaultDimensionSpec.java b/druid/src/main/java/org/apache/calcite/adapter/druid/DefaultDimensionSpec.java index e4be8c6d2722..d2c10fa5f676 100644 --- a/druid/src/main/java/org/apache/calcite/adapter/druid/DefaultDimensionSpec.java +++ b/druid/src/main/java/org/apache/calcite/adapter/druid/DefaultDimensionSpec.java @@ -18,7 +18,7 @@ import com.fasterxml.jackson.core.JsonGenerator; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.io.IOException; diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/DimensionSpec.java b/druid/src/main/java/org/apache/calcite/adapter/druid/DimensionSpec.java index 333c8768b1a4..10d702b2f7e0 100644 --- a/druid/src/main/java/org/apache/calcite/adapter/druid/DimensionSpec.java +++ b/druid/src/main/java/org/apache/calcite/adapter/druid/DimensionSpec.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.adapter.druid; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Interface for Druid DimensionSpec. diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/DirectOperatorConversion.java b/druid/src/main/java/org/apache/calcite/adapter/druid/DirectOperatorConversion.java index 8cc5964419e3..e0d76b4870a7 100644 --- a/druid/src/main/java/org/apache/calcite/adapter/druid/DirectOperatorConversion.java +++ b/druid/src/main/java/org/apache/calcite/adapter/druid/DirectOperatorConversion.java @@ -21,7 +21,7 @@ import org.apache.calcite.rex.RexNode; import org.apache.calcite.sql.SqlOperator; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidConnectionImpl.java b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidConnectionImpl.java index 5fb37199b726..dedadefd60ab 100644 --- a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidConnectionImpl.java +++ b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidConnectionImpl.java @@ -38,8 +38,8 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Interval; +import org.jspecify.annotations.Nullable; import java.io.ByteArrayInputStream; import java.io.IOException; diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidDateTimeUtils.java b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidDateTimeUtils.java index e44b41702de0..e6b530b29e40 100644 --- a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidDateTimeUtils.java +++ b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidDateTimeUtils.java @@ -37,10 +37,10 @@ import com.google.common.collect.Range; import com.google.common.collect.TreeRangeSet; -import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Interval; import org.joda.time.Period; import org.joda.time.chrono.ISOChronology; +import org.jspecify.annotations.Nullable; import org.slf4j.Logger; import java.util.ArrayList; diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidExpressions.java b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidExpressions.java index cbdf146ea1d1..d0010e363164 100644 --- a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidExpressions.java +++ b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidExpressions.java @@ -31,7 +31,7 @@ import com.google.common.io.BaseEncoding; import com.google.common.primitives.Chars; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.math.BigDecimal; import java.util.ArrayList; diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidJsonFilter.java b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidJsonFilter.java index 6d2dc814dd9b..e8ed7c33baae 100644 --- a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidJsonFilter.java +++ b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidJsonFilter.java @@ -32,7 +32,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.io.IOException; import java.text.SimpleDateFormat; diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidQuery.java b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidQuery.java index e6aa27b6bdaf..2d89420a13ee 100644 --- a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidQuery.java +++ b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidQuery.java @@ -73,8 +73,8 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Maps; -import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Interval; +import org.jspecify.annotations.Nullable; import java.io.IOException; import java.io.StringWriter; diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java index ff64d44ed2c1..814b716c1cf7 100644 --- a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java +++ b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java @@ -57,9 +57,9 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; -import org.checkerframework.checker.nullness.qual.Nullable; import org.immutables.value.Value; import org.joda.time.Interval; +import org.jspecify.annotations.Nullable; import org.slf4j.Logger; import java.util.ArrayList; diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidSchema.java b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidSchema.java index 4174aff76ddb..a555f321667d 100644 --- a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidSchema.java +++ b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidSchema.java @@ -26,7 +26,7 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.HashMap; import java.util.LinkedHashMap; diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidSqlCastConverter.java b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidSqlCastConverter.java index d33fff097727..67f725f14d1b 100644 --- a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidSqlCastConverter.java +++ b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidSqlCastConverter.java @@ -26,8 +26,8 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Period; +import org.jspecify.annotations.Nullable; import java.util.TimeZone; diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidSqlOperatorConverter.java b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidSqlOperatorConverter.java index f84a2ad63184..ac335bf85e23 100644 --- a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidSqlOperatorConverter.java +++ b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidSqlOperatorConverter.java @@ -20,7 +20,7 @@ import org.apache.calcite.rex.RexNode; import org.apache.calcite.sql.SqlOperator; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Defines how to convert a {@link RexNode} with a given Calcite SQL operator to diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidTable.java b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidTable.java index 349e1132821a..925cab5d7b85 100644 --- a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidTable.java +++ b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidTable.java @@ -41,10 +41,10 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.DateTime; import org.joda.time.Interval; import org.joda.time.chrono.ISOChronology; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidTableFactory.java b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidTableFactory.java index 20acc9a9eb27..b4174ab1e642 100644 --- a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidTableFactory.java +++ b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidTableFactory.java @@ -25,9 +25,9 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Interval; import org.joda.time.chrono.ISOChronology; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.HashMap; diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/ExtractOperatorConversion.java b/druid/src/main/java/org/apache/calcite/adapter/druid/ExtractOperatorConversion.java index 77fd833e5f48..777ff3a6e5a0 100644 --- a/druid/src/main/java/org/apache/calcite/adapter/druid/ExtractOperatorConversion.java +++ b/druid/src/main/java/org/apache/calcite/adapter/druid/ExtractOperatorConversion.java @@ -28,7 +28,7 @@ import com.google.common.collect.ImmutableMap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Map; import java.util.TimeZone; diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/ExtractionDimensionSpec.java b/druid/src/main/java/org/apache/calcite/adapter/druid/ExtractionDimensionSpec.java index 8284eafb74a8..b5f5cb08e5c5 100644 --- a/druid/src/main/java/org/apache/calcite/adapter/druid/ExtractionDimensionSpec.java +++ b/druid/src/main/java/org/apache/calcite/adapter/druid/ExtractionDimensionSpec.java @@ -18,7 +18,7 @@ import com.fasterxml.jackson.core.JsonGenerator; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.io.IOException; diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/FloorOperatorConversion.java b/druid/src/main/java/org/apache/calcite/adapter/druid/FloorOperatorConversion.java index 89240d61a64e..ba77be893422 100644 --- a/druid/src/main/java/org/apache/calcite/adapter/druid/FloorOperatorConversion.java +++ b/druid/src/main/java/org/apache/calcite/adapter/druid/FloorOperatorConversion.java @@ -24,7 +24,7 @@ import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.type.SqlTypeName; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.TimeZone; diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/NaryOperatorConverter.java b/druid/src/main/java/org/apache/calcite/adapter/druid/NaryOperatorConverter.java index 4de4067a31a6..405d9e93aa64 100644 --- a/druid/src/main/java/org/apache/calcite/adapter/druid/NaryOperatorConverter.java +++ b/druid/src/main/java/org/apache/calcite/adapter/druid/NaryOperatorConverter.java @@ -21,7 +21,7 @@ import org.apache.calcite.rex.RexNode; import org.apache.calcite.sql.SqlOperator; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/SubstringOperatorConversion.java b/druid/src/main/java/org/apache/calcite/adapter/druid/SubstringOperatorConversion.java index 1675bddd79ba..81587fe269bb 100644 --- a/druid/src/main/java/org/apache/calcite/adapter/druid/SubstringOperatorConversion.java +++ b/druid/src/main/java/org/apache/calcite/adapter/druid/SubstringOperatorConversion.java @@ -24,7 +24,7 @@ import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.fun.SqlStdOperatorTable; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Converts Calcite SUBSTRING call to Druid Expression when possible. diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/TimeExtractionFunction.java b/druid/src/main/java/org/apache/calcite/adapter/druid/TimeExtractionFunction.java index 5bb861948033..6d9e6e21a150 100644 --- a/druid/src/main/java/org/apache/calcite/adapter/druid/TimeExtractionFunction.java +++ b/druid/src/main/java/org/apache/calcite/adapter/druid/TimeExtractionFunction.java @@ -28,7 +28,7 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.io.IOException; import java.util.Locale; diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/UnaryPrefixOperatorConversion.java b/druid/src/main/java/org/apache/calcite/adapter/druid/UnaryPrefixOperatorConversion.java index 2f37eefe6671..a7c76ee5710d 100644 --- a/druid/src/main/java/org/apache/calcite/adapter/druid/UnaryPrefixOperatorConversion.java +++ b/druid/src/main/java/org/apache/calcite/adapter/druid/UnaryPrefixOperatorConversion.java @@ -23,7 +23,7 @@ import com.google.common.collect.Iterables; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/UnarySuffixOperatorConversion.java b/druid/src/main/java/org/apache/calcite/adapter/druid/UnarySuffixOperatorConversion.java index 3d555eb6d097..4a5e0bc9de50 100644 --- a/druid/src/main/java/org/apache/calcite/adapter/druid/UnarySuffixOperatorConversion.java +++ b/druid/src/main/java/org/apache/calcite/adapter/druid/UnarySuffixOperatorConversion.java @@ -23,7 +23,7 @@ import com.google.common.collect.Iterables; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchAggregate.java b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchAggregate.java index 7d8ab37fe5f9..28d2783735ac 100644 --- a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchAggregate.java +++ b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchAggregate.java @@ -35,7 +35,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.EnumSet; diff --git a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchFilter.java b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchFilter.java index 4383d6017c0e..123024980279 100644 --- a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchFilter.java +++ b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchFilter.java @@ -30,7 +30,7 @@ import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.ObjectMapper; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.io.IOException; import java.io.StringWriter; diff --git a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchJson.java b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchJson.java index de0d36b98c8f..be04172422c7 100644 --- a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchJson.java +++ b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchJson.java @@ -30,7 +30,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableSet; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.io.IOException; import java.time.Duration; diff --git a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchMapping.java b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchMapping.java index 3bcac313055b..9a0baac8c5ec 100644 --- a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchMapping.java +++ b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchMapping.java @@ -20,7 +20,7 @@ import com.fasterxml.jackson.databind.node.JsonNodeFactory; import com.google.common.collect.ImmutableMap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.time.LocalDate; import java.time.ZoneOffset; diff --git a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java index 640b225434e6..093d0f1fdba3 100644 --- a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java +++ b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java @@ -31,7 +31,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchRules.java b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchRules.java index 3c44992b0a48..90dea1a52924 100644 --- a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchRules.java +++ b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchRules.java @@ -44,7 +44,7 @@ import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.sql.validate.SqlValidatorUtil; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.AbstractList; import java.util.List; diff --git a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchSchema.java b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchSchema.java index ec41851c430a..5b22f4f44b91 100644 --- a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchSchema.java +++ b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchSchema.java @@ -25,10 +25,10 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Sets; -import org.checkerframework.checker.nullness.qual.Nullable; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.client.RestClient; +import org.jspecify.annotations.Nullable; import java.io.IOException; import java.io.InputStream; diff --git a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchSort.java b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchSort.java index e7af28627978..44db02061f23 100644 --- a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchSort.java +++ b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchSort.java @@ -29,7 +29,7 @@ import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchTableScan.java b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchTableScan.java index 43e1aff0f0b7..f2e7ce34702d 100644 --- a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchTableScan.java +++ b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchTableScan.java @@ -30,7 +30,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchToEnumerableConverter.java b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchToEnumerableConverter.java index a2c2f5b6cedc..9a808256d1c3 100644 --- a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchToEnumerableConverter.java +++ b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchToEnumerableConverter.java @@ -37,7 +37,7 @@ import org.apache.calcite.util.BuiltInMethod; import org.apache.calcite.util.Pair; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.AbstractList; import java.util.List; diff --git a/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvFilterableTable.java b/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvFilterableTable.java index 212aff0ce5d1..e8be1b736538 100644 --- a/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvFilterableTable.java +++ b/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvFilterableTable.java @@ -33,7 +33,7 @@ import org.apache.calcite.util.ImmutableIntList; import org.apache.calcite.util.Source; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; diff --git a/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvProjectTableScanRule.java b/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvProjectTableScanRule.java index 4dcdf2e817a5..2649d1f5473a 100644 --- a/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvProjectTableScanRule.java +++ b/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvProjectTableScanRule.java @@ -22,8 +22,8 @@ import org.apache.calcite.rex.RexInputRef; import org.apache.calcite.rex.RexNode; -import org.checkerframework.checker.nullness.qual.Nullable; import org.immutables.value.Value; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvScannableTable.java b/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvScannableTable.java index 046825bd35db..3e45b911742c 100644 --- a/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvScannableTable.java +++ b/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvScannableTable.java @@ -28,7 +28,7 @@ import org.apache.calcite.util.ImmutableIntList; import org.apache.calcite.util.Source; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; diff --git a/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvSchema.java b/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvSchema.java index e612aa83e4a5..3153b31417b8 100644 --- a/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvSchema.java +++ b/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvSchema.java @@ -24,7 +24,7 @@ import com.google.common.collect.ImmutableMap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.io.File; import java.util.Map; diff --git a/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvStreamScannableTable.java b/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvStreamScannableTable.java index 98ab6a03bcbf..680e39b415eb 100644 --- a/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvStreamScannableTable.java +++ b/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvStreamScannableTable.java @@ -30,7 +30,7 @@ import org.apache.calcite.util.ImmutableIntList; import org.apache.calcite.util.Source; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; diff --git a/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvStreamTableFactory.java b/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvStreamTableFactory.java index cb524dcf3f95..615f91f20c3f 100644 --- a/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvStreamTableFactory.java +++ b/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvStreamTableFactory.java @@ -25,7 +25,7 @@ import org.apache.calcite.util.Source; import org.apache.calcite.util.Sources; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.io.File; import java.util.Map; diff --git a/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvTable.java b/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvTable.java index aac56bb38ce4..46f04b04c3ea 100644 --- a/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvTable.java +++ b/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvTable.java @@ -24,7 +24,7 @@ import org.apache.calcite.schema.impl.AbstractTable; import org.apache.calcite.util.Source; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvTableFactory.java b/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvTableFactory.java index fed7ddb338e6..d7efbe97e392 100644 --- a/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvTableFactory.java +++ b/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvTableFactory.java @@ -25,7 +25,7 @@ import org.apache.calcite.util.Source; import org.apache.calcite.util.Sources; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.io.File; import java.util.Map; diff --git a/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvTableScan.java b/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvTableScan.java index daa0a3070a78..aca9aeb3a42a 100644 --- a/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvTableScan.java +++ b/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvTableScan.java @@ -39,7 +39,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvTranslatableTable.java b/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvTranslatableTable.java index 1da1992b6b14..8382079cfa5a 100644 --- a/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvTranslatableTable.java +++ b/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvTranslatableTable.java @@ -35,7 +35,7 @@ import org.apache.calcite.util.ImmutableIntList; import org.apache.calcite.util.Source; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Type; import java.util.concurrent.atomic.AtomicBoolean; diff --git a/example/csv/src/test/java/org/apache/calcite/test/CsvTest.java b/example/csv/src/test/java/org/apache/calcite/test/CsvTest.java index 2236345f06ab..aff26774d6f4 100644 --- a/example/csv/src/test/java/org/apache/calcite/test/CsvTest.java +++ b/example/csv/src/test/java/org/apache/calcite/test/CsvTest.java @@ -28,7 +28,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Ordering; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; diff --git a/example/function/src/main/java/org/apache/calcite/example/maze/MazeTable.java b/example/function/src/main/java/org/apache/calcite/example/maze/MazeTable.java index 1068bf25cbbf..d715666ea88d 100644 --- a/example/function/src/main/java/org/apache/calcite/example/maze/MazeTable.java +++ b/example/function/src/main/java/org/apache/calcite/example/maze/MazeTable.java @@ -28,7 +28,7 @@ import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.util.Util; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.io.PrintWriter; import java.util.Random; diff --git a/file/src/main/java/org/apache/calcite/adapter/file/CsvEnumerator.java b/file/src/main/java/org/apache/calcite/adapter/file/CsvEnumerator.java index 99951e85cdbe..f54e4202da6a 100644 --- a/file/src/main/java/org/apache/calcite/adapter/file/CsvEnumerator.java +++ b/file/src/main/java/org/apache/calcite/adapter/file/CsvEnumerator.java @@ -31,7 +31,7 @@ import com.google.common.annotations.VisibleForTesting; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.slf4j.LoggerFactory; import java.io.IOException; diff --git a/file/src/main/java/org/apache/calcite/adapter/file/CsvStreamReader.java b/file/src/main/java/org/apache/calcite/adapter/file/CsvStreamReader.java index e9113c7d1ac1..88796e524557 100644 --- a/file/src/main/java/org/apache/calcite/adapter/file/CsvStreamReader.java +++ b/file/src/main/java/org/apache/calcite/adapter/file/CsvStreamReader.java @@ -25,7 +25,7 @@ import au.com.bytecode.opencsv.CSVParser; import au.com.bytecode.opencsv.CSVReader; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.io.Closeable; import java.io.IOException; diff --git a/file/src/main/java/org/apache/calcite/adapter/file/CsvTable.java b/file/src/main/java/org/apache/calcite/adapter/file/CsvTable.java index 4b4c718d41a5..f325cc225518 100644 --- a/file/src/main/java/org/apache/calcite/adapter/file/CsvTable.java +++ b/file/src/main/java/org/apache/calcite/adapter/file/CsvTable.java @@ -23,7 +23,7 @@ import org.apache.calcite.schema.impl.AbstractTable; import org.apache.calcite.util.Source; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/file/src/main/java/org/apache/calcite/adapter/file/CsvTableFactory.java b/file/src/main/java/org/apache/calcite/adapter/file/CsvTableFactory.java index fa37d156532c..78ada9185ab1 100644 --- a/file/src/main/java/org/apache/calcite/adapter/file/CsvTableFactory.java +++ b/file/src/main/java/org/apache/calcite/adapter/file/CsvTableFactory.java @@ -27,7 +27,7 @@ import au.com.bytecode.opencsv.CSVParser; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.io.File; import java.util.Map; diff --git a/file/src/main/java/org/apache/calcite/adapter/file/CsvTableScan.java b/file/src/main/java/org/apache/calcite/adapter/file/CsvTableScan.java index 36b2ab332f34..941c315ac8fc 100644 --- a/file/src/main/java/org/apache/calcite/adapter/file/CsvTableScan.java +++ b/file/src/main/java/org/apache/calcite/adapter/file/CsvTableScan.java @@ -43,7 +43,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/file/src/main/java/org/apache/calcite/adapter/file/CsvTranslatableTable.java b/file/src/main/java/org/apache/calcite/adapter/file/CsvTranslatableTable.java index 7f81defebe01..ce4c7824caca 100644 --- a/file/src/main/java/org/apache/calcite/adapter/file/CsvTranslatableTable.java +++ b/file/src/main/java/org/apache/calcite/adapter/file/CsvTranslatableTable.java @@ -34,7 +34,7 @@ import org.apache.calcite.util.ImmutableIntList; import org.apache.calcite.util.Source; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Type; import java.util.concurrent.atomic.AtomicBoolean; diff --git a/file/src/main/java/org/apache/calcite/adapter/file/FileEnumerator.java b/file/src/main/java/org/apache/calcite/adapter/file/FileEnumerator.java index 6aa480f9f332..638393326cfa 100644 --- a/file/src/main/java/org/apache/calcite/adapter/file/FileEnumerator.java +++ b/file/src/main/java/org/apache/calcite/adapter/file/FileEnumerator.java @@ -18,8 +18,8 @@ import org.apache.calcite.linq4j.Enumerator; -import org.checkerframework.checker.nullness.qual.Nullable; import org.jsoup.select.Elements; +import org.jspecify.annotations.Nullable; import java.util.Iterator; diff --git a/file/src/main/java/org/apache/calcite/adapter/file/FileFieldType.java b/file/src/main/java/org/apache/calcite/adapter/file/FileFieldType.java index 64f50f4eb4c4..71caff6057a2 100644 --- a/file/src/main/java/org/apache/calcite/adapter/file/FileFieldType.java +++ b/file/src/main/java/org/apache/calcite/adapter/file/FileFieldType.java @@ -22,7 +22,7 @@ import com.google.common.collect.ImmutableMap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Map; diff --git a/file/src/main/java/org/apache/calcite/adapter/file/FileReader.java b/file/src/main/java/org/apache/calcite/adapter/file/FileReader.java index ca164852733d..632dceac5e08 100644 --- a/file/src/main/java/org/apache/calcite/adapter/file/FileReader.java +++ b/file/src/main/java/org/apache/calcite/adapter/file/FileReader.java @@ -18,11 +18,11 @@ import org.apache.calcite.util.Source; -import org.checkerframework.checker.nullness.qual.Nullable; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; +import org.jspecify.annotations.Nullable; import java.io.IOException; import java.nio.charset.Charset; diff --git a/file/src/main/java/org/apache/calcite/adapter/file/FileRowConverter.java b/file/src/main/java/org/apache/calcite/adapter/file/FileRowConverter.java index b4e3986e1534..94bb6362f6a7 100644 --- a/file/src/main/java/org/apache/calcite/adapter/file/FileRowConverter.java +++ b/file/src/main/java/org/apache/calcite/adapter/file/FileRowConverter.java @@ -25,9 +25,9 @@ import com.joestelmach.natty.DateGroup; import com.joestelmach.natty.Parser; -import org.checkerframework.checker.nullness.qual.Nullable; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; +import org.jspecify.annotations.Nullable; import java.text.NumberFormat; import java.text.ParseException; diff --git a/file/src/main/java/org/apache/calcite/adapter/file/FileSchema.java b/file/src/main/java/org/apache/calcite/adapter/file/FileSchema.java index f66effb4a32e..ef8616bc1cbd 100644 --- a/file/src/main/java/org/apache/calcite/adapter/file/FileSchema.java +++ b/file/src/main/java/org/apache/calcite/adapter/file/FileSchema.java @@ -28,7 +28,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.io.File; import java.util.List; diff --git a/file/src/main/java/org/apache/calcite/adapter/file/FileTable.java b/file/src/main/java/org/apache/calcite/adapter/file/FileTable.java index ef3f959b81d9..b3a2aba3acff 100644 --- a/file/src/main/java/org/apache/calcite/adapter/file/FileTable.java +++ b/file/src/main/java/org/apache/calcite/adapter/file/FileTable.java @@ -37,7 +37,7 @@ import org.apache.calcite.schema.impl.AbstractTableQueryable; import org.apache.calcite.util.Source; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; import java.util.Map; diff --git a/file/src/main/java/org/apache/calcite/adapter/file/JsonEnumerator.java b/file/src/main/java/org/apache/calcite/adapter/file/JsonEnumerator.java index 256c8e0f587c..c5e975f1e381 100644 --- a/file/src/main/java/org/apache/calcite/adapter/file/JsonEnumerator.java +++ b/file/src/main/java/org/apache/calcite/adapter/file/JsonEnumerator.java @@ -27,7 +27,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.exc.MismatchedInputException; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.Arrays; diff --git a/file/src/main/java/org/apache/calcite/adapter/file/JsonScannableTable.java b/file/src/main/java/org/apache/calcite/adapter/file/JsonScannableTable.java index f0f23739635d..6de4d5c07454 100644 --- a/file/src/main/java/org/apache/calcite/adapter/file/JsonScannableTable.java +++ b/file/src/main/java/org/apache/calcite/adapter/file/JsonScannableTable.java @@ -24,7 +24,7 @@ import org.apache.calcite.schema.ScannableTable; import org.apache.calcite.util.Source; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Table based on a JSON file. diff --git a/file/src/main/java/org/apache/calcite/adapter/file/JsonTable.java b/file/src/main/java/org/apache/calcite/adapter/file/JsonTable.java index e26f81d8c325..8d8084ed3eef 100644 --- a/file/src/main/java/org/apache/calcite/adapter/file/JsonTable.java +++ b/file/src/main/java/org/apache/calcite/adapter/file/JsonTable.java @@ -24,7 +24,7 @@ import org.apache.calcite.schema.impl.AbstractTable; import org.apache.calcite.util.Source; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/file/src/test/java/org/apache/calcite/adapter/file/FileAdapterTests.java b/file/src/test/java/org/apache/calcite/adapter/file/FileAdapterTests.java index 7caea64e9d84..c9a2ef2d5aa6 100644 --- a/file/src/test/java/org/apache/calcite/adapter/file/FileAdapterTests.java +++ b/file/src/test/java/org/apache/calcite/adapter/file/FileAdapterTests.java @@ -22,7 +22,7 @@ import com.google.common.collect.Ordering; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.io.PrintStream; import java.net.URL; diff --git a/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeAggregate.java b/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeAggregate.java index baf850e0f707..0b9041087618 100644 --- a/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeAggregate.java +++ b/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeAggregate.java @@ -32,7 +32,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeEnumerator.java b/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeEnumerator.java index b70d1e31528b..93e28a3013c6 100644 --- a/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeEnumerator.java +++ b/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeEnumerator.java @@ -25,7 +25,7 @@ import org.apache.geode.cache.query.SelectResults; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeFilter.java b/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeFilter.java index 47d43728c620..b1cf8f5ac6b2 100644 --- a/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeFilter.java +++ b/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeFilter.java @@ -37,7 +37,7 @@ import org.apache.calcite.util.TimestampString; import org.apache.calcite.util.Util; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.Collections; diff --git a/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeProject.java b/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeProject.java index 9cdcfa3eaf49..800d3fd85de6 100644 --- a/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeProject.java +++ b/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeProject.java @@ -31,7 +31,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.LinkedHashMap; import java.util.List; diff --git a/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeRules.java b/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeRules.java index b8beb854e0f6..e06d522b41fb 100644 --- a/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeRules.java +++ b/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeRules.java @@ -42,8 +42,8 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; import org.immutables.value.Value; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeSort.java b/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeSort.java index c9b33252d980..fa150e978a26 100644 --- a/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeSort.java +++ b/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeSort.java @@ -28,7 +28,7 @@ import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeTable.java b/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeTable.java index 584b5cacaeb8..83075a7db007 100644 --- a/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeTable.java +++ b/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeTable.java @@ -46,7 +46,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeTableScan.java b/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeTableScan.java index 60650b17a739..f720a4a30d51 100644 --- a/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeTableScan.java +++ b/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeTableScan.java @@ -27,7 +27,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeToEnumerableConverter.java b/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeToEnumerableConverter.java index 84af3ec4c0e8..ef8e3a445fcf 100644 --- a/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeToEnumerableConverter.java +++ b/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeToEnumerableConverter.java @@ -40,7 +40,7 @@ import org.apache.calcite.util.Pair; import org.apache.calcite.util.Util; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Method; import java.util.AbstractList; diff --git a/geode/src/main/java/org/apache/calcite/adapter/geode/simple/GeodeSimpleEnumerator.java b/geode/src/main/java/org/apache/calcite/adapter/geode/simple/GeodeSimpleEnumerator.java index 6e24e71ef32d..341edf72b72e 100644 --- a/geode/src/main/java/org/apache/calcite/adapter/geode/simple/GeodeSimpleEnumerator.java +++ b/geode/src/main/java/org/apache/calcite/adapter/geode/simple/GeodeSimpleEnumerator.java @@ -22,7 +22,7 @@ import org.apache.geode.cache.query.QueryService; import org.apache.geode.cache.query.SelectResults; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Iterator; diff --git a/geode/src/main/java/org/apache/calcite/adapter/geode/simple/GeodeSimpleScannableTable.java b/geode/src/main/java/org/apache/calcite/adapter/geode/simple/GeodeSimpleScannableTable.java index 1594b30e27d3..a415ecfac57f 100644 --- a/geode/src/main/java/org/apache/calcite/adapter/geode/simple/GeodeSimpleScannableTable.java +++ b/geode/src/main/java/org/apache/calcite/adapter/geode/simple/GeodeSimpleScannableTable.java @@ -27,7 +27,7 @@ import org.apache.geode.cache.client.ClientCache; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import static org.apache.calcite.adapter.geode.util.GeodeUtils.convertToRowValues; diff --git a/geode/src/main/java/org/apache/calcite/adapter/geode/util/GeodeUtils.java b/geode/src/main/java/org/apache/calcite/adapter/geode/util/GeodeUtils.java index 2aa212dc5a92..b9066de66b7a 100644 --- a/geode/src/main/java/org/apache/calcite/adapter/geode/util/GeodeUtils.java +++ b/geode/src/main/java/org/apache/calcite/adapter/geode/util/GeodeUtils.java @@ -33,7 +33,7 @@ import org.apache.geode.pdx.PdxInstance; import org.apache.geode.pdx.ReflectionBasedAutoSerializer; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/innodb/src/main/java/org/apache/calcite/adapter/innodb/IndexCondition.java b/innodb/src/main/java/org/apache/calcite/adapter/innodb/IndexCondition.java index 358790e64e23..5a2cadbcd401 100644 --- a/innodb/src/main/java/org/apache/calcite/adapter/innodb/IndexCondition.java +++ b/innodb/src/main/java/org/apache/calcite/adapter/innodb/IndexCondition.java @@ -25,7 +25,7 @@ import com.alibaba.innodb.java.reader.comparator.ComparisonOperator; import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbEnumerator.java b/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbEnumerator.java index 4125466b367b..2d27d9a474d5 100644 --- a/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbEnumerator.java +++ b/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbEnumerator.java @@ -26,7 +26,7 @@ import com.alibaba.innodb.java.reader.page.index.GenericRecord; import com.alibaba.innodb.java.reader.util.Utils; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.sql.Date; import java.sql.Time; diff --git a/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbFilter.java b/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbFilter.java index 37c5c57c8ced..a5a146b95cc4 100644 --- a/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbFilter.java +++ b/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbFilter.java @@ -29,7 +29,7 @@ import com.alibaba.innodb.java.reader.schema.TableDef; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import static java.util.Objects.requireNonNull; diff --git a/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbFilterTranslator.java b/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbFilterTranslator.java index 72e7bc348355..59e5332dfaa0 100644 --- a/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbFilterTranslator.java +++ b/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbFilterTranslator.java @@ -37,7 +37,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Multimap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.Collection; diff --git a/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbProject.java b/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbProject.java index f05c30091c76..acde92946419 100644 --- a/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbProject.java +++ b/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbProject.java @@ -30,7 +30,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.LinkedHashMap; import java.util.List; diff --git a/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbSort.java b/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbSort.java index 44fa6da3fe8f..f3a01c2d1bdb 100644 --- a/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbSort.java +++ b/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbSort.java @@ -27,7 +27,7 @@ import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.rex.RexNode; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbTableScan.java b/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbTableScan.java index 5b38e2eb4c48..f1d5390993fd 100644 --- a/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbTableScan.java +++ b/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbTableScan.java @@ -33,7 +33,7 @@ import com.alibaba.innodb.java.reader.Constants; import com.alibaba.innodb.java.reader.schema.KeyMeta; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; import java.util.Optional; diff --git a/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbToEnumerableConverter.java b/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbToEnumerableConverter.java index a30273ac41ca..92d798a6453c 100644 --- a/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbToEnumerableConverter.java +++ b/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbToEnumerableConverter.java @@ -42,7 +42,7 @@ import com.alibaba.innodb.java.reader.comparator.ComparisonOperator; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.AbstractList; import java.util.Collections; diff --git a/innodb/src/test/java/org/apache/calcite/adapter/innodb/InnodbAdapterTest.java b/innodb/src/test/java/org/apache/calcite/adapter/innodb/InnodbAdapterTest.java index f7a1f484fd2c..0b8c04338d2e 100644 --- a/innodb/src/test/java/org/apache/calcite/adapter/innodb/InnodbAdapterTest.java +++ b/innodb/src/test/java/org/apache/calcite/adapter/innodb/InnodbAdapterTest.java @@ -30,7 +30,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; diff --git a/kafka/src/main/java/org/apache/calcite/adapter/kafka/KafkaMessageEnumerator.java b/kafka/src/main/java/org/apache/calcite/adapter/kafka/KafkaMessageEnumerator.java index adfd999bdfa3..f33c85e4a51c 100644 --- a/kafka/src/main/java/org/apache/calcite/adapter/kafka/KafkaMessageEnumerator.java +++ b/kafka/src/main/java/org/apache/calcite/adapter/kafka/KafkaMessageEnumerator.java @@ -23,7 +23,7 @@ import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.consumer.ConsumerRecords; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.time.Duration; import java.util.ArrayDeque; diff --git a/kafka/src/main/java/org/apache/calcite/adapter/kafka/KafkaStreamTable.java b/kafka/src/main/java/org/apache/calcite/adapter/kafka/KafkaStreamTable.java index f73941c9418a..8a3954b3a532 100644 --- a/kafka/src/main/java/org/apache/calcite/adapter/kafka/KafkaStreamTable.java +++ b/kafka/src/main/java/org/apache/calcite/adapter/kafka/KafkaStreamTable.java @@ -39,7 +39,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Collections; import java.util.Properties; diff --git a/kafka/src/main/java/org/apache/calcite/adapter/kafka/KafkaTableFactory.java b/kafka/src/main/java/org/apache/calcite/adapter/kafka/KafkaTableFactory.java index f6cbce53539c..efeb20aba260 100644 --- a/kafka/src/main/java/org/apache/calcite/adapter/kafka/KafkaTableFactory.java +++ b/kafka/src/main/java/org/apache/calcite/adapter/kafka/KafkaTableFactory.java @@ -23,7 +23,7 @@ import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.consumer.OffsetResetStrategy; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.InvocationTargetException; import java.util.Locale; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/BaseQueryable.java b/linq4j/src/main/java/org/apache/calcite/linq4j/BaseQueryable.java index b707fad12d45..37b5cbdca775 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/BaseQueryable.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/BaseQueryable.java @@ -18,7 +18,7 @@ import org.apache.calcite.linq4j.tree.Expression; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Type; import java.util.Iterator; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/DefaultEnumerable.java b/linq4j/src/main/java/org/apache/calcite/linq4j/DefaultEnumerable.java index c02001b346d4..23ce85ef3b0d 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/DefaultEnumerable.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/DefaultEnumerable.java @@ -34,8 +34,8 @@ import org.apache.calcite.linq4j.function.Predicate1; import org.apache.calcite.linq4j.function.Predicate2; -import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.checker.nullness.qual.PolyNull; +import org.jspecify.annotations.Nullable; import java.math.BigDecimal; import java.util.Collection; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/DefaultQueryable.java b/linq4j/src/main/java/org/apache/calcite/linq4j/DefaultQueryable.java index 65c79bacb553..d60bba500bf1 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/DefaultQueryable.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/DefaultQueryable.java @@ -33,7 +33,7 @@ import org.apache.calcite.linq4j.function.Predicate2; import org.apache.calcite.linq4j.tree.FunctionExpression; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.math.BigDecimal; import java.util.Comparator; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java b/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java index 0eca4b6d5283..326989182111 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java @@ -42,10 +42,10 @@ import org.apiguardian.api.API; import org.checkerframework.checker.nullness.qual.KeyFor; -import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.checker.nullness.qual.PolyNull; import org.checkerframework.dataflow.qual.Pure; import org.checkerframework.framework.qual.HasQualifierParameter; +import org.jspecify.annotations.Nullable; import java.math.BigDecimal; import java.math.RoundingMode; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableOrderedQueryable.java b/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableOrderedQueryable.java index 2684c0246d78..53b1dbf762d2 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableOrderedQueryable.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableOrderedQueryable.java @@ -20,7 +20,7 @@ import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.linq4j.tree.FunctionExpression; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Comparator; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableQueryable.java b/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableQueryable.java index 7a37a57c5f04..0c777684556d 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableQueryable.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableQueryable.java @@ -34,7 +34,7 @@ import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.linq4j.tree.FunctionExpression; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Type; import java.math.BigDecimal; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedEnumerable.java b/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedEnumerable.java index 362372464ac4..578d30ef0ffd 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedEnumerable.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedEnumerable.java @@ -34,9 +34,9 @@ import org.apache.calcite.linq4j.function.Predicate1; import org.apache.calcite.linq4j.function.Predicate2; -import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.checker.nullness.qual.PolyNull; import org.checkerframework.framework.qual.Covariant; +import org.jspecify.annotations.Nullable; import java.math.BigDecimal; import java.util.Collection; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedQueryable.java b/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedQueryable.java index f30b22482b8a..0e16304eec0c 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedQueryable.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedQueryable.java @@ -33,8 +33,8 @@ import org.apache.calcite.linq4j.function.Predicate2; import org.apache.calcite.linq4j.tree.FunctionExpression; -import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.framework.qual.Covariant; +import org.jspecify.annotations.Nullable; import java.math.BigDecimal; import java.util.Comparator; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/GroupingImpl.java b/linq4j/src/main/java/org/apache/calcite/linq4j/GroupingImpl.java index f294bc2c4036..0e7a23572791 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/GroupingImpl.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/GroupingImpl.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.linq4j; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; import java.util.Map; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/Linq4j.java b/linq4j/src/main/java/org/apache/calcite/linq4j/Linq4j.java index c59303d4fdc0..7e0f377bc627 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/Linq4j.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/Linq4j.java @@ -18,7 +18,7 @@ import org.apache.calcite.linq4j.function.Function1; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Method; import java.math.BigDecimal; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/LookupImpl.java b/linq4j/src/main/java/org/apache/calcite/linq4j/LookupImpl.java index 7d2a83821389..9411ecbaa83d 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/LookupImpl.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/LookupImpl.java @@ -19,7 +19,7 @@ import org.apache.calcite.linq4j.function.Function2; import org.checkerframework.checker.nullness.qual.KeyFor; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.AbstractCollection; import java.util.AbstractMap; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/MemoryEnumerator.java b/linq4j/src/main/java/org/apache/calcite/linq4j/MemoryEnumerator.java index c09661cde38b..41250f5b5413 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/MemoryEnumerator.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/MemoryEnumerator.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.linq4j; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.concurrent.atomic.AtomicInteger; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/MemoryFactory.java b/linq4j/src/main/java/org/apache/calcite/linq4j/MemoryFactory.java index 421a52a98d7b..78b4e83229cf 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/MemoryFactory.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/MemoryFactory.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.linq4j; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Arrays; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/MergeUnionEnumerator.java b/linq4j/src/main/java/org/apache/calcite/linq4j/MergeUnionEnumerator.java index d708f67858c3..86a30ea1bb30 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/MergeUnionEnumerator.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/MergeUnionEnumerator.java @@ -20,8 +20,8 @@ import org.apache.calcite.linq4j.function.Function1; import org.checkerframework.checker.initialization.qual.UnknownInitialization; -import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.checker.nullness.qual.RequiresNonNull; +import org.jspecify.annotations.Nullable; import java.util.Comparator; import java.util.HashSet; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/ModularInteger.java b/linq4j/src/main/java/org/apache/calcite/linq4j/ModularInteger.java index 638b1f5f913c..1bc7d02260d5 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/ModularInteger.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/ModularInteger.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.linq4j; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import static com.google.common.base.Preconditions.checkArgument; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/Nullness.java b/linq4j/src/main/java/org/apache/calcite/linq4j/Nullness.java index 55cf8c4f694c..6ad855ea33d9 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/Nullness.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/Nullness.java @@ -18,9 +18,9 @@ import org.checkerframework.checker.initialization.qual.UnderInitialization; import org.checkerframework.checker.nullness.qual.EnsuresNonNull; -import org.checkerframework.checker.nullness.qual.NonNull; -import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.dataflow.qual.Pure; +import org.jspecify.annotations.NonNull; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/Ord.java b/linq4j/src/main/java/org/apache/calcite/linq4j/Ord.java index f30c7ba25643..fbc95d5a61f0 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/Ord.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/Ord.java @@ -18,7 +18,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.AbstractList; import java.util.Iterator; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/QueryableDefaults.java b/linq4j/src/main/java/org/apache/calcite/linq4j/QueryableDefaults.java index e3d96a3aec62..eab9a894aeef 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/QueryableDefaults.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/QueryableDefaults.java @@ -36,7 +36,7 @@ import org.apache.calcite.linq4j.tree.Expressions; import org.apache.calcite.linq4j.tree.FunctionExpression; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Type; import java.math.BigDecimal; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/QueryableFactory.java b/linq4j/src/main/java/org/apache/calcite/linq4j/QueryableFactory.java index 2207c64db34f..c7a322ce1329 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/QueryableFactory.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/QueryableFactory.java @@ -33,9 +33,9 @@ import org.apache.calcite.linq4j.function.Predicate2; import org.apache.calcite.linq4j.tree.FunctionExpression; -import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.checker.nullness.qual.PolyNull; import org.checkerframework.framework.qual.Covariant; +import org.jspecify.annotations.Nullable; import java.math.BigDecimal; import java.util.Comparator; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/QueryableRecorder.java b/linq4j/src/main/java/org/apache/calcite/linq4j/QueryableRecorder.java index 8e0dd8f3a33e..fff571251240 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/QueryableRecorder.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/QueryableRecorder.java @@ -33,9 +33,9 @@ import org.apache.calcite.linq4j.function.Predicate2; import org.apache.calcite.linq4j.tree.FunctionExpression; -import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.checker.nullness.qual.PolyNull; import org.checkerframework.framework.qual.Covariant; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Type; import java.math.BigDecimal; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/RawQueryable.java b/linq4j/src/main/java/org/apache/calcite/linq4j/RawQueryable.java index 05ecf85692aa..69398463ba1f 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/RawQueryable.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/RawQueryable.java @@ -18,8 +18,8 @@ import org.apache.calcite.linq4j.tree.Expression; -import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.framework.qual.Covariant; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Type; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/function/Functions.java b/linq4j/src/main/java/org/apache/calcite/linq4j/function/Functions.java index 89ee4699e1e4..63eab93e8904 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/function/Functions.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/function/Functions.java @@ -16,9 +16,9 @@ */ package org.apache.calcite.linq4j.function; -import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.framework.qual.DefaultQualifier; import org.checkerframework.framework.qual.TypeUseLocation; +import org.jspecify.annotations.Nullable; import java.io.Serializable; import java.lang.reflect.Type; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/package-info.java b/linq4j/src/main/java/org/apache/calcite/linq4j/package-info.java index 726df9815271..6a13f8f258d1 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/package-info.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/package-info.java @@ -23,6 +23,6 @@ @DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.RETURN) package org.apache.calcite.linq4j; -import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.framework.qual.DefaultQualifier; import org.checkerframework.framework.qual.TypeUseLocation; +import org.jspecify.annotations.NonNull; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/AbstractNode.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/AbstractNode.java index a60241123862..348a9d556e23 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/AbstractNode.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/AbstractNode.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.linq4j.tree; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Type; import java.util.Objects; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ArrayLengthRecordField.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ArrayLengthRecordField.java index 5780054c4eb2..adeb36e1eeaa 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ArrayLengthRecordField.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ArrayLengthRecordField.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.linq4j.tree; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Array; import java.lang.reflect.Type; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/BinaryExpression.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/BinaryExpression.java index 495ef3517953..0876ed0bc073 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/BinaryExpression.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/BinaryExpression.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.linq4j.tree; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Type; import java.util.Objects; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/BlockBuilder.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/BlockBuilder.java index 5d811ea68c02..ed9a562d0685 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/BlockBuilder.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/BlockBuilder.java @@ -16,8 +16,8 @@ */ package org.apache.calcite.linq4j.tree; -import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.checker.nullness.qual.PolyNull; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Modifier; import java.lang.reflect.Type; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/BlockStatement.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/BlockStatement.java index b55908ec00bf..11d0372c2798 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/BlockStatement.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/BlockStatement.java @@ -17,7 +17,7 @@ package org.apache.calcite.linq4j.tree; import org.checkerframework.checker.initialization.qual.UnderInitialization; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Type; import java.util.HashSet; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/CatchBlock.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/CatchBlock.java index 21aff57d96f7..08430d1ed11f 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/CatchBlock.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/CatchBlock.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.linq4j.tree; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Objects; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ClassDeclaration.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ClassDeclaration.java index 184961e354f1..2c34c99c995b 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ClassDeclaration.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ClassDeclaration.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.linq4j.tree; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Modifier; import java.lang.reflect.Type; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ClassDeclarationFinder.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ClassDeclarationFinder.java index b95253477292..7c725686b808 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ClassDeclarationFinder.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ClassDeclarationFinder.java @@ -18,7 +18,7 @@ import org.apache.calcite.linq4j.function.Function1; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ConditionalExpression.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ConditionalExpression.java index 6fbcc1ef1f3e..267c8da6d6eb 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ConditionalExpression.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ConditionalExpression.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.linq4j.tree; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Type; import java.util.List; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ConditionalStatement.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ConditionalStatement.java index c061c4fbe3a6..db6b52c6149d 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ConditionalStatement.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ConditionalStatement.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.linq4j.tree; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; import java.util.Objects; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ConstantExpression.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ConstantExpression.java index 73002c0e1432..ddc377a00709 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ConstantExpression.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ConstantExpression.java @@ -18,7 +18,7 @@ import org.apache.calcite.linq4j.util.Compatible; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Constructor; import java.lang.reflect.Field; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ConstantUntypedNull.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ConstantUntypedNull.java index 204f82d8e901..d435ae839b3a 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ConstantUntypedNull.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ConstantUntypedNull.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.linq4j.tree; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Represents a constant null of unknown type diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ConstructorDeclaration.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ConstructorDeclaration.java index 10d61963c2eb..bae7ad7613c5 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ConstructorDeclaration.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ConstructorDeclaration.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.linq4j.tree; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Modifier; import java.lang.reflect.Type; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/DeclarationStatement.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/DeclarationStatement.java index 3795df638658..787c8dad0d9e 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/DeclarationStatement.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/DeclarationStatement.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.linq4j.tree; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Modifier; import java.util.Objects; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/DeterministicCodeOptimizer.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/DeterministicCodeOptimizer.java index 453fa69addec..b182ee968a9f 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/DeterministicCodeOptimizer.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/DeterministicCodeOptimizer.java @@ -21,7 +21,7 @@ import com.google.common.collect.ImmutableSet; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Constructor; import java.lang.reflect.Method; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/Evaluator.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/Evaluator.java index f6cc41b7dd2c..164a3531fdb2 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/Evaluator.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/Evaluator.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.linq4j.tree; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ExpressionType.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ExpressionType.java index bcc87c480af5..7b0668f3978b 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ExpressionType.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ExpressionType.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.linq4j.tree; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** Analogous to LINQ's System.Linq.Expressions.ExpressionType. */ public enum ExpressionType { diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ExpressionWriter.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ExpressionWriter.java index 8bc640bac6e4..616814ba6408 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ExpressionWriter.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ExpressionWriter.java @@ -18,7 +18,7 @@ import org.apache.calcite.avatica.util.Spacer; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Type; import java.util.Iterator; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/Expressions.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/Expressions.java index 5ba55093b651..fd4dd28af7c5 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/Expressions.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/Expressions.java @@ -26,7 +26,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Constructor; import java.lang.reflect.Field; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/FieldDeclaration.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/FieldDeclaration.java index f0813af67cd8..97633f2b73c4 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/FieldDeclaration.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/FieldDeclaration.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.linq4j.tree; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Modifier; import java.util.Objects; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ForEachStatement.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ForEachStatement.java index eeb286a7a32e..f2f3137ea9ae 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ForEachStatement.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ForEachStatement.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.linq4j.tree; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Objects; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ForStatement.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ForStatement.java index 987f37da2863..29e6105c79b1 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ForStatement.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ForStatement.java @@ -18,7 +18,7 @@ import org.apache.calcite.linq4j.Ord; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; import java.util.Objects; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/FunctionExpression.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/FunctionExpression.java index 932b904b6a8f..9dca4808f7fc 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/FunctionExpression.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/FunctionExpression.java @@ -22,7 +22,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Method; import java.lang.reflect.Proxy; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/GotoStatement.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/GotoStatement.java index 32c660136240..1bce13c7682c 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/GotoStatement.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/GotoStatement.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.linq4j.tree; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Objects; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/IndexExpression.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/IndexExpression.java index 75dc90e23a5b..04ddd91709ad 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/IndexExpression.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/IndexExpression.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.linq4j.tree; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; import java.util.Objects; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/LabelStatement.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/LabelStatement.java index f861325b37bb..62e6cf394d50 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/LabelStatement.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/LabelStatement.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.linq4j.tree; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Objects; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/LabelTarget.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/LabelTarget.java index 16679ab02343..a82b075c2577 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/LabelTarget.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/LabelTarget.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.linq4j.tree; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Objects; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/MemberExpression.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/MemberExpression.java index 54d0d7361815..139eb45ca826 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/MemberExpression.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/MemberExpression.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.linq4j.tree; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Field; import java.lang.reflect.Modifier; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/MethodCallExpression.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/MethodCallExpression.java index 8115934458d4..a3e76e2934c0 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/MethodCallExpression.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/MethodCallExpression.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.linq4j.tree; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/MethodDeclaration.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/MethodDeclaration.java index 444da129d5a6..3f97b34aafa2 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/MethodDeclaration.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/MethodDeclaration.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.linq4j.tree; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Modifier; import java.lang.reflect.Type; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/NewArrayExpression.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/NewArrayExpression.java index b8a920ff2a37..86edb90f9972 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/NewArrayExpression.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/NewArrayExpression.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.linq4j.tree; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Type; import java.util.List; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/NewExpression.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/NewExpression.java index 5cbe06824c43..b51fd123a202 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/NewExpression.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/NewExpression.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.linq4j.tree; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Type; import java.util.List; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/OptimizeShuttle.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/OptimizeShuttle.java index a5f04cdc3541..0de70567d898 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/OptimizeShuttle.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/OptimizeShuttle.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.linq4j.tree; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Method; import java.lang.reflect.Modifier; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ParameterExpression.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ParameterExpression.java index 9e3c05d982ae..ffe330979cab 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ParameterExpression.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ParameterExpression.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.linq4j.tree; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Modifier; import java.lang.reflect.Type; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/Primitive.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/Primitive.java index a35fba35a015..a52183f555bd 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/Primitive.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/Primitive.java @@ -17,11 +17,11 @@ package org.apache.calcite.linq4j.tree; import org.apiguardian.api.API; -import org.checkerframework.checker.nullness.qual.Nullable; import org.joou.UByte; import org.joou.UInteger; import org.joou.ULong; import org.joou.UShort; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Array; import java.lang.reflect.Field; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/PseudoField.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/PseudoField.java index 93f781954a7d..d41c8840b8a8 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/PseudoField.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/PseudoField.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.linq4j.tree; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Type; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ReflectedPseudoField.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ReflectedPseudoField.java index 8bfde58e2f1c..4699924fdc12 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ReflectedPseudoField.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ReflectedPseudoField.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.linq4j.tree; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Field; import java.lang.reflect.Type; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/Shuttle.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/Shuttle.java index 8e5533c81936..ca8134a6b9ba 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/Shuttle.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/Shuttle.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.linq4j.tree; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; import java.util.Objects; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/TernaryExpression.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/TernaryExpression.java index a05504c73231..f54cad02f0cb 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/TernaryExpression.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/TernaryExpression.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.linq4j.tree; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Type; import java.util.Objects; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ThrowStatement.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ThrowStatement.java index b9298824efb2..8d535043e7da 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ThrowStatement.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ThrowStatement.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.linq4j.tree; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Objects; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/TryStatement.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/TryStatement.java index 95c06f5abf05..8a19ee1738d9 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/TryStatement.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/TryStatement.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.linq4j.tree; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/TypeBinaryExpression.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/TypeBinaryExpression.java index 9e2e44e27713..16b4bf689737 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/TypeBinaryExpression.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/TypeBinaryExpression.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.linq4j.tree; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Type; import java.util.Objects; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/Types.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/Types.java index e3a2a8390ac9..ea9548086a24 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/Types.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/Types.java @@ -20,7 +20,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Array; import java.lang.reflect.Constructor; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/UnaryExpression.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/UnaryExpression.java index 6c04bf73c0c4..af9623490ff9 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/UnaryExpression.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/UnaryExpression.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.linq4j.tree; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Type; import java.util.Objects; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/UnsignedType.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/UnsignedType.java index 32622315f770..08fc8862276c 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/UnsignedType.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/UnsignedType.java @@ -16,12 +16,12 @@ */ package org.apache.calcite.linq4j.tree; -import org.checkerframework.checker.nullness.qual.Nullable; import org.joou.UByte; import org.joou.UInteger; import org.joou.ULong; import org.joou.UShort; import org.joou.Unsigned; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Type; import java.math.BigDecimal; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/VisitorImpl.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/VisitorImpl.java index 18025e3c791f..43264c13810f 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/VisitorImpl.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/VisitorImpl.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.linq4j.tree; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/WhileStatement.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/WhileStatement.java index e828a1aa5bb3..ddd73bff37f6 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/WhileStatement.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/WhileStatement.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.linq4j.tree; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Objects; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/util/Compatible.java b/linq4j/src/main/java/org/apache/calcite/linq4j/util/Compatible.java index 63781e0641ca..9daeec2f9483 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/util/Compatible.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/util/Compatible.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.linq4j.util; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.invoke.MethodHandle; import java.lang.invoke.MethodHandles; diff --git a/linq4j/src/test/java/org/apache/calcite/linq4j/test/BlockBuilderTest.java b/linq4j/src/test/java/org/apache/calcite/linq4j/test/BlockBuilderTest.java index 5cc772c0b545..09a5e46a0661 100644 --- a/linq4j/src/test/java/org/apache/calcite/linq4j/test/BlockBuilderTest.java +++ b/linq4j/src/test/java/org/apache/calcite/linq4j/test/BlockBuilderTest.java @@ -25,7 +25,7 @@ import org.apache.calcite.linq4j.tree.ParameterExpression; import org.apache.calcite.linq4j.tree.Shuttle; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; diff --git a/linq4j/src/test/java/org/apache/calcite/linq4j/test/ExpressionTest.java b/linq4j/src/test/java/org/apache/calcite/linq4j/test/ExpressionTest.java index 3482e2b86797..50e9012d40c7 100644 --- a/linq4j/src/test/java/org/apache/calcite/linq4j/test/ExpressionTest.java +++ b/linq4j/src/test/java/org/apache/calcite/linq4j/test/ExpressionTest.java @@ -38,7 +38,7 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; diff --git a/linq4j/src/test/java/org/apache/calcite/linq4j/test/JoinPreserveOrderTest.java b/linq4j/src/test/java/org/apache/calcite/linq4j/test/JoinPreserveOrderTest.java index 0ec28abd4a9f..00af6ec08fa9 100644 --- a/linq4j/src/test/java/org/apache/calcite/linq4j/test/JoinPreserveOrderTest.java +++ b/linq4j/src/test/java/org/apache/calcite/linq4j/test/JoinPreserveOrderTest.java @@ -23,7 +23,7 @@ import org.apache.calcite.linq4j.function.Function1; import org.apache.calcite.linq4j.function.Function2; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; diff --git a/linq4j/src/test/java/org/apache/calcite/linq4j/test/LimitSortTest.java b/linq4j/src/test/java/org/apache/calcite/linq4j/test/LimitSortTest.java index 77e0eb8ca81a..05df73225eae 100644 --- a/linq4j/src/test/java/org/apache/calcite/linq4j/test/LimitSortTest.java +++ b/linq4j/src/test/java/org/apache/calcite/linq4j/test/LimitSortTest.java @@ -21,7 +21,7 @@ import org.apache.calcite.linq4j.Linq4j; import org.apache.calcite.linq4j.function.Function1; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.Test; import java.util.Comparator; diff --git a/linq4j/src/test/java/org/apache/calcite/linq4j/tree/IdentifierValidationTest.java b/linq4j/src/test/java/org/apache/calcite/linq4j/tree/IdentifierValidationTest.java index 3040e00f0638..062ff63e7f68 100644 --- a/linq4j/src/test/java/org/apache/calcite/linq4j/tree/IdentifierValidationTest.java +++ b/linq4j/src/test/java/org/apache/calcite/linq4j/tree/IdentifierValidationTest.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.linq4j.tree; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.Test; import java.lang.reflect.Modifier; diff --git a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoAggregate.java b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoAggregate.java index 4e368607de52..0388e0770f47 100644 --- a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoAggregate.java +++ b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoAggregate.java @@ -31,7 +31,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.AbstractList; import java.util.ArrayList; diff --git a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoEnumerator.java b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoEnumerator.java index b654364b3be4..b8de7e0597b4 100644 --- a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoEnumerator.java +++ b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoEnumerator.java @@ -28,7 +28,7 @@ import org.bson.Document; import org.bson.types.Binary; import org.bson.types.Decimal128; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.math.BigDecimal; import java.util.Date; diff --git a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoFilter.java b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoFilter.java index 93f96d79b079..ad1330c02246 100644 --- a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoFilter.java +++ b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoFilter.java @@ -40,7 +40,7 @@ import com.google.common.collect.HashMultimap; import com.google.common.collect.Multimap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.Collection; diff --git a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoProject.java b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoProject.java index a0d795aa0d8f..79f060b006a1 100644 --- a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoProject.java +++ b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoProject.java @@ -32,7 +32,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoRel.java b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoRel.java index 68a86daf0c6e..c05f1d718815 100644 --- a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoRel.java +++ b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoRel.java @@ -22,7 +22,7 @@ import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.runtime.PairList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Relational expression that uses Mongo calling convention. diff --git a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoSort.java b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoSort.java index fd79235c4aba..45d7d7e2cc03 100644 --- a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoSort.java +++ b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoSort.java @@ -30,7 +30,7 @@ import org.apache.calcite.rex.RexNode; import org.apache.calcite.util.Util; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoTableScan.java b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoTableScan.java index f8bff5a0f692..ffffe6a9f578 100644 --- a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoTableScan.java +++ b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoTableScan.java @@ -29,7 +29,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoToEnumerableConverter.java b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoToEnumerableConverter.java index 116bd6e5dd1c..daa0faf8fd03 100644 --- a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoToEnumerableConverter.java +++ b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoToEnumerableConverter.java @@ -40,7 +40,7 @@ import org.apache.calcite.util.Pair; import org.apache.calcite.util.Util; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.AbstractList; import java.util.List; diff --git a/pig/src/main/java/org/apache/calcite/adapter/pig/PigTableFactory.java b/pig/src/main/java/org/apache/calcite/adapter/pig/PigTableFactory.java index 6ebf30f0ad4c..78544bbe49f4 100644 --- a/pig/src/main/java/org/apache/calcite/adapter/pig/PigTableFactory.java +++ b/pig/src/main/java/org/apache/calcite/adapter/pig/PigTableFactory.java @@ -21,7 +21,7 @@ import org.apache.calcite.schema.SchemaPlus; import org.apache.calcite.schema.TableFactory; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.io.File; import java.util.List; diff --git a/piglet/src/main/java/org/apache/calcite/piglet/PigTable.java b/piglet/src/main/java/org/apache/calcite/piglet/PigTable.java index 94a8f7427f5a..d57fe4acf2a7 100644 --- a/piglet/src/main/java/org/apache/calcite/piglet/PigTable.java +++ b/piglet/src/main/java/org/apache/calcite/piglet/PigTable.java @@ -31,7 +31,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/plus/src/main/java/org/apache/calcite/adapter/os/AbstractBaseScannableTable.java b/plus/src/main/java/org/apache/calcite/adapter/os/AbstractBaseScannableTable.java index 9f1bb186b1d2..4a3465606e17 100644 --- a/plus/src/main/java/org/apache/calcite/adapter/os/AbstractBaseScannableTable.java +++ b/plus/src/main/java/org/apache/calcite/adapter/os/AbstractBaseScannableTable.java @@ -27,7 +27,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Abstract base class for implementations of OS table functions. diff --git a/plus/src/main/java/org/apache/calcite/adapter/os/CpuInfoTableFunction.java b/plus/src/main/java/org/apache/calcite/adapter/os/CpuInfoTableFunction.java index a42d897a39df..e92c3feb51b5 100644 --- a/plus/src/main/java/org/apache/calcite/adapter/os/CpuInfoTableFunction.java +++ b/plus/src/main/java/org/apache/calcite/adapter/os/CpuInfoTableFunction.java @@ -25,7 +25,7 @@ import org.apache.calcite.schema.ScannableTable; import org.apache.calcite.sql.type.SqlTypeName; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Table function that executes the OS "cpu_info". diff --git a/plus/src/main/java/org/apache/calcite/adapter/os/CpuTimeTableFunction.java b/plus/src/main/java/org/apache/calcite/adapter/os/CpuTimeTableFunction.java index e76ececcaf53..a303143ce367 100644 --- a/plus/src/main/java/org/apache/calcite/adapter/os/CpuTimeTableFunction.java +++ b/plus/src/main/java/org/apache/calcite/adapter/os/CpuTimeTableFunction.java @@ -25,7 +25,7 @@ import org.apache.calcite.schema.ScannableTable; import org.apache.calcite.sql.type.SqlTypeName; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Table function that executes the OS "cpu_info". diff --git a/plus/src/main/java/org/apache/calcite/adapter/os/DuTableFunction.java b/plus/src/main/java/org/apache/calcite/adapter/os/DuTableFunction.java index f1847bfab75b..80f6995a13bd 100644 --- a/plus/src/main/java/org/apache/calcite/adapter/os/DuTableFunction.java +++ b/plus/src/main/java/org/apache/calcite/adapter/os/DuTableFunction.java @@ -23,7 +23,7 @@ import org.apache.calcite.schema.ScannableTable; import org.apache.calcite.sql.type.SqlTypeName; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Table function that executes the OS "du" ("disk usage") command diff --git a/plus/src/main/java/org/apache/calcite/adapter/os/FilesTableFunction.java b/plus/src/main/java/org/apache/calcite/adapter/os/FilesTableFunction.java index cd7ca9e8b8d1..6e8162f730ee 100644 --- a/plus/src/main/java/org/apache/calcite/adapter/os/FilesTableFunction.java +++ b/plus/src/main/java/org/apache/calcite/adapter/os/FilesTableFunction.java @@ -29,7 +29,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.math.BigDecimal; import java.util.Arrays; diff --git a/plus/src/main/java/org/apache/calcite/adapter/os/GitCommitsTableFunction.java b/plus/src/main/java/org/apache/calcite/adapter/os/GitCommitsTableFunction.java index 2e589572914b..a56dced2b0fc 100644 --- a/plus/src/main/java/org/apache/calcite/adapter/os/GitCommitsTableFunction.java +++ b/plus/src/main/java/org/apache/calcite/adapter/os/GitCommitsTableFunction.java @@ -25,7 +25,7 @@ import org.apache.calcite.schema.ScannableTable; import org.apache.calcite.sql.type.SqlTypeName; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.NoSuchElementException; diff --git a/plus/src/main/java/org/apache/calcite/adapter/os/InterfaceAddressesTableFunction.java b/plus/src/main/java/org/apache/calcite/adapter/os/InterfaceAddressesTableFunction.java index 6c566fc34ec9..30cb54c66f84 100644 --- a/plus/src/main/java/org/apache/calcite/adapter/os/InterfaceAddressesTableFunction.java +++ b/plus/src/main/java/org/apache/calcite/adapter/os/InterfaceAddressesTableFunction.java @@ -25,7 +25,7 @@ import org.apache.calcite.schema.ScannableTable; import org.apache.calcite.sql.type.SqlTypeName; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Table function that executes the OS "interface_addresses". diff --git a/plus/src/main/java/org/apache/calcite/adapter/os/InterfaceDetailsTableFunction.java b/plus/src/main/java/org/apache/calcite/adapter/os/InterfaceDetailsTableFunction.java index 507e8ab2c9db..9b2964bc6471 100644 --- a/plus/src/main/java/org/apache/calcite/adapter/os/InterfaceDetailsTableFunction.java +++ b/plus/src/main/java/org/apache/calcite/adapter/os/InterfaceDetailsTableFunction.java @@ -25,7 +25,7 @@ import org.apache.calcite.schema.ScannableTable; import org.apache.calcite.sql.type.SqlTypeName; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Table function that executes the OS "interface_details". diff --git a/plus/src/main/java/org/apache/calcite/adapter/os/JavaInfoTableFunction.java b/plus/src/main/java/org/apache/calcite/adapter/os/JavaInfoTableFunction.java index adacb0b5c3c6..75f52a77aa65 100644 --- a/plus/src/main/java/org/apache/calcite/adapter/os/JavaInfoTableFunction.java +++ b/plus/src/main/java/org/apache/calcite/adapter/os/JavaInfoTableFunction.java @@ -25,7 +25,7 @@ import org.apache.calcite.schema.ScannableTable; import org.apache.calcite.sql.type.SqlTypeName; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Table function that executes the OS "java_info". diff --git a/plus/src/main/java/org/apache/calcite/adapter/os/JpsTableFunction.java b/plus/src/main/java/org/apache/calcite/adapter/os/JpsTableFunction.java index 1d4ae1c2c5ec..bc1a03a042a2 100644 --- a/plus/src/main/java/org/apache/calcite/adapter/os/JpsTableFunction.java +++ b/plus/src/main/java/org/apache/calcite/adapter/os/JpsTableFunction.java @@ -23,7 +23,7 @@ import org.apache.calcite.schema.ScannableTable; import org.apache.calcite.sql.type.SqlTypeName; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Table function that executes the OS "jps" ("Java Virtual Machine Process diff --git a/plus/src/main/java/org/apache/calcite/adapter/os/MemoryInfoTableFunction.java b/plus/src/main/java/org/apache/calcite/adapter/os/MemoryInfoTableFunction.java index 5047f3f885f2..d1c4d66b5348 100644 --- a/plus/src/main/java/org/apache/calcite/adapter/os/MemoryInfoTableFunction.java +++ b/plus/src/main/java/org/apache/calcite/adapter/os/MemoryInfoTableFunction.java @@ -25,7 +25,7 @@ import org.apache.calcite.schema.ScannableTable; import org.apache.calcite.sql.type.SqlTypeName; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Table function that executes the OS "memory_info". diff --git a/plus/src/main/java/org/apache/calcite/adapter/os/MountsTableFunction.java b/plus/src/main/java/org/apache/calcite/adapter/os/MountsTableFunction.java index 3beb4b230e41..3f488f623441 100644 --- a/plus/src/main/java/org/apache/calcite/adapter/os/MountsTableFunction.java +++ b/plus/src/main/java/org/apache/calcite/adapter/os/MountsTableFunction.java @@ -25,7 +25,7 @@ import org.apache.calcite.schema.ScannableTable; import org.apache.calcite.sql.type.SqlTypeName; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Table function that executes the OS "mounts". diff --git a/plus/src/main/java/org/apache/calcite/adapter/os/OsVersionTableFunction.java b/plus/src/main/java/org/apache/calcite/adapter/os/OsVersionTableFunction.java index 961e89597b79..0c78bbb341d6 100644 --- a/plus/src/main/java/org/apache/calcite/adapter/os/OsVersionTableFunction.java +++ b/plus/src/main/java/org/apache/calcite/adapter/os/OsVersionTableFunction.java @@ -25,7 +25,7 @@ import org.apache.calcite.schema.ScannableTable; import org.apache.calcite.sql.type.SqlTypeName; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Table function that executes the OS "os_version". diff --git a/plus/src/main/java/org/apache/calcite/adapter/os/PsTableFunction.java b/plus/src/main/java/org/apache/calcite/adapter/os/PsTableFunction.java index 635b55100da5..2337bbf27c2e 100644 --- a/plus/src/main/java/org/apache/calcite/adapter/os/PsTableFunction.java +++ b/plus/src/main/java/org/apache/calcite/adapter/os/PsTableFunction.java @@ -30,7 +30,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; import java.util.regex.Matcher; diff --git a/plus/src/main/java/org/apache/calcite/adapter/os/StdinTableFunction.java b/plus/src/main/java/org/apache/calcite/adapter/os/StdinTableFunction.java index 493bccbe99d7..53e7424ecf1b 100644 --- a/plus/src/main/java/org/apache/calcite/adapter/os/StdinTableFunction.java +++ b/plus/src/main/java/org/apache/calcite/adapter/os/StdinTableFunction.java @@ -25,7 +25,7 @@ import org.apache.calcite.schema.ScannableTable; import org.apache.calcite.sql.type.SqlTypeName; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.io.BufferedReader; import java.io.IOException; diff --git a/plus/src/main/java/org/apache/calcite/adapter/os/SystemInfoTableFunction.java b/plus/src/main/java/org/apache/calcite/adapter/os/SystemInfoTableFunction.java index 36e406454882..f01aedb127d0 100644 --- a/plus/src/main/java/org/apache/calcite/adapter/os/SystemInfoTableFunction.java +++ b/plus/src/main/java/org/apache/calcite/adapter/os/SystemInfoTableFunction.java @@ -25,7 +25,7 @@ import org.apache.calcite.schema.ScannableTable; import org.apache.calcite.sql.type.SqlTypeName; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Table function that executes the OS "system_info". diff --git a/plus/src/main/java/org/apache/calcite/adapter/os/VmstatTableFunction.java b/plus/src/main/java/org/apache/calcite/adapter/os/VmstatTableFunction.java index 2a8f5e8ae196..87eb474df478 100644 --- a/plus/src/main/java/org/apache/calcite/adapter/os/VmstatTableFunction.java +++ b/plus/src/main/java/org/apache/calcite/adapter/os/VmstatTableFunction.java @@ -27,7 +27,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/plus/src/main/java/org/apache/calcite/adapter/tpcds/TpcdsSchema.java b/plus/src/main/java/org/apache/calcite/adapter/tpcds/TpcdsSchema.java index 20ae7d314521..153e8da71124 100644 --- a/plus/src/main/java/org/apache/calcite/adapter/tpcds/TpcdsSchema.java +++ b/plus/src/main/java/org/apache/calcite/adapter/tpcds/TpcdsSchema.java @@ -43,7 +43,7 @@ import com.teradata.tpcds.column.Column; import com.teradata.tpcds.column.ColumnType; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.math.BigDecimal; import java.util.ArrayList; diff --git a/plus/src/main/java/org/apache/calcite/chinook/PreferredAlbumsTableFactory.java b/plus/src/main/java/org/apache/calcite/chinook/PreferredAlbumsTableFactory.java index cb9086a7403c..d49d6805ef92 100644 --- a/plus/src/main/java/org/apache/calcite/chinook/PreferredAlbumsTableFactory.java +++ b/plus/src/main/java/org/apache/calcite/chinook/PreferredAlbumsTableFactory.java @@ -30,7 +30,7 @@ import com.google.common.collect.DiscreteDomain; import com.google.common.collect.Range; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Map; diff --git a/plus/src/main/java/org/apache/calcite/chinook/PreferredGenresTableFactory.java b/plus/src/main/java/org/apache/calcite/chinook/PreferredGenresTableFactory.java index abfada547f71..f7a8a317d44e 100644 --- a/plus/src/main/java/org/apache/calcite/chinook/PreferredGenresTableFactory.java +++ b/plus/src/main/java/org/apache/calcite/chinook/PreferredGenresTableFactory.java @@ -30,7 +30,7 @@ import com.google.common.collect.DiscreteDomain; import com.google.common.collect.Range; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Map; diff --git a/plus/src/test/java/org/apache/calcite/adapter/tpcds/TpcdsTest.java b/plus/src/test/java/org/apache/calcite/adapter/tpcds/TpcdsTest.java index 096e5e582c69..54654907811e 100644 --- a/plus/src/test/java/org/apache/calcite/adapter/tpcds/TpcdsTest.java +++ b/plus/src/test/java/org/apache/calcite/adapter/tpcds/TpcdsTest.java @@ -32,7 +32,7 @@ import net.hydromatic.tpcds.query.Query; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; diff --git a/redis/src/main/java/org/apache/calcite/adapter/redis/RedisSchema.java b/redis/src/main/java/org/apache/calcite/adapter/redis/RedisSchema.java index 42c623895bc8..41038af92f66 100644 --- a/redis/src/main/java/org/apache/calcite/adapter/redis/RedisSchema.java +++ b/redis/src/main/java/org/apache/calcite/adapter/redis/RedisSchema.java @@ -25,7 +25,7 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.Arrays; diff --git a/redis/src/main/java/org/apache/calcite/adapter/redis/RedisTable.java b/redis/src/main/java/org/apache/calcite/adapter/redis/RedisTable.java index 30f57a0d4549..c5e562f2af5b 100644 --- a/redis/src/main/java/org/apache/calcite/adapter/redis/RedisTable.java +++ b/redis/src/main/java/org/apache/calcite/adapter/redis/RedisTable.java @@ -30,7 +30,7 @@ import com.google.common.collect.ImmutableMap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/redis/src/main/java/org/apache/calcite/adapter/redis/RedisTableFactory.java b/redis/src/main/java/org/apache/calcite/adapter/redis/RedisTableFactory.java index de4042c71c60..97e5d0938c9c 100644 --- a/redis/src/main/java/org/apache/calcite/adapter/redis/RedisTableFactory.java +++ b/redis/src/main/java/org/apache/calcite/adapter/redis/RedisTableFactory.java @@ -23,7 +23,7 @@ import org.apache.calcite.schema.Table; import org.apache.calcite.schema.TableFactory; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Map; diff --git a/server/src/main/java/org/apache/calcite/server/AbstractModifiableTable.java b/server/src/main/java/org/apache/calcite/server/AbstractModifiableTable.java index 7677eba4efa5..dbb066142286 100644 --- a/server/src/main/java/org/apache/calcite/server/AbstractModifiableTable.java +++ b/server/src/main/java/org/apache/calcite/server/AbstractModifiableTable.java @@ -26,7 +26,7 @@ import org.apache.calcite.schema.ModifiableTable; import org.apache.calcite.schema.impl.AbstractTable; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/server/src/main/java/org/apache/calcite/server/MaterializedViewTable.java b/server/src/main/java/org/apache/calcite/server/MaterializedViewTable.java index fe72308a9d69..5915514ccc5d 100644 --- a/server/src/main/java/org/apache/calcite/server/MaterializedViewTable.java +++ b/server/src/main/java/org/apache/calcite/server/MaterializedViewTable.java @@ -21,7 +21,7 @@ import org.apache.calcite.schema.Schema; import org.apache.calcite.sql2rel.NullInitializerExpressionFactory; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** A table that implements a materialized view. */ class MaterializedViewTable diff --git a/server/src/main/java/org/apache/calcite/server/MutableArrayTable.java b/server/src/main/java/org/apache/calcite/server/MutableArrayTable.java index b022ce3120eb..95456bc4948c 100644 --- a/server/src/main/java/org/apache/calcite/server/MutableArrayTable.java +++ b/server/src/main/java/org/apache/calcite/server/MutableArrayTable.java @@ -30,7 +30,7 @@ import org.apache.calcite.schema.impl.AbstractTableQueryable; import org.apache.calcite.sql2rel.InitializerExpressionFactory; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Type; import java.util.ArrayList; diff --git a/server/src/main/java/org/apache/calcite/server/ServerDdlExecutor.java b/server/src/main/java/org/apache/calcite/server/ServerDdlExecutor.java index 8bd55b91519f..0235f0e4a699 100644 --- a/server/src/main/java/org/apache/calcite/server/ServerDdlExecutor.java +++ b/server/src/main/java/org/apache/calcite/server/ServerDdlExecutor.java @@ -90,7 +90,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.io.Reader; import java.sql.PreparedStatement; diff --git a/server/src/test/java/org/apache/calcite/test/ServerParserTest.java b/server/src/test/java/org/apache/calcite/test/ServerParserTest.java index 99fa4f37d664..6316b7fe06a0 100644 --- a/server/src/test/java/org/apache/calcite/test/ServerParserTest.java +++ b/server/src/test/java/org/apache/calcite/test/ServerParserTest.java @@ -36,7 +36,7 @@ import org.apache.calcite.sql.parser.ddl.SqlDdlParserImpl; import org.apache.calcite.sql.util.SqlShuttle; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.Test; import java.util.function.BiConsumer; diff --git a/spark/src/main/java/org/apache/calcite/adapter/spark/EnumerableToSparkConverter.java b/spark/src/main/java/org/apache/calcite/adapter/spark/EnumerableToSparkConverter.java index 60f48db8e9b3..45ce57f684bf 100644 --- a/spark/src/main/java/org/apache/calcite/adapter/spark/EnumerableToSparkConverter.java +++ b/spark/src/main/java/org/apache/calcite/adapter/spark/EnumerableToSparkConverter.java @@ -32,7 +32,7 @@ import org.apache.calcite.rel.convert.ConverterImpl; import org.apache.calcite.rel.metadata.RelMetadataQuery; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/spark/src/main/java/org/apache/calcite/adapter/spark/HttpServer.java b/spark/src/main/java/org/apache/calcite/adapter/spark/HttpServer.java index aa0c5cb41ce3..d8b3227e8379 100644 --- a/spark/src/main/java/org/apache/calcite/adapter/spark/HttpServer.java +++ b/spark/src/main/java/org/apache/calcite/adapter/spark/HttpServer.java @@ -16,7 +16,6 @@ */ package org.apache.calcite.adapter.spark; -import org.checkerframework.checker.nullness.qual.Nullable; import org.eclipse.jetty.server.Connector; import org.eclipse.jetty.server.Handler; import org.eclipse.jetty.server.Server; @@ -25,6 +24,7 @@ import org.eclipse.jetty.server.handler.HandlerList; import org.eclipse.jetty.server.handler.ResourceHandler; import org.eclipse.jetty.util.thread.QueuedThreadPool; +import org.jspecify.annotations.Nullable; import java.io.File; import java.io.IOException; diff --git a/spark/src/main/java/org/apache/calcite/adapter/spark/JdbcToSparkConverter.java b/spark/src/main/java/org/apache/calcite/adapter/spark/JdbcToSparkConverter.java index aef790879335..fe8bff1986bf 100644 --- a/spark/src/main/java/org/apache/calcite/adapter/spark/JdbcToSparkConverter.java +++ b/spark/src/main/java/org/apache/calcite/adapter/spark/JdbcToSparkConverter.java @@ -40,7 +40,7 @@ import org.apache.calcite.sql.SqlDialect; import org.apache.calcite.util.BuiltInMethod; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/spark/src/main/java/org/apache/calcite/adapter/spark/SparkRules.java b/spark/src/main/java/org/apache/calcite/adapter/spark/SparkRules.java index eb38a6dc17f2..51bab088d1f2 100644 --- a/spark/src/main/java/org/apache/calcite/adapter/spark/SparkRules.java +++ b/spark/src/main/java/org/apache/calcite/adapter/spark/SparkRules.java @@ -66,7 +66,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import scala.Tuple2; diff --git a/spark/src/main/java/org/apache/calcite/adapter/spark/SparkToEnumerableConverter.java b/spark/src/main/java/org/apache/calcite/adapter/spark/SparkToEnumerableConverter.java index cfdc180ebaad..edc4b893dcf8 100644 --- a/spark/src/main/java/org/apache/calcite/adapter/spark/SparkToEnumerableConverter.java +++ b/spark/src/main/java/org/apache/calcite/adapter/spark/SparkToEnumerableConverter.java @@ -36,7 +36,7 @@ import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.sql.validate.SqlConformance; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/splunk/src/main/java/org/apache/calcite/adapter/splunk/SplunkDriver.java b/splunk/src/main/java/org/apache/calcite/adapter/splunk/SplunkDriver.java index 765e94c40e7a..26df5e6f2244 100644 --- a/splunk/src/main/java/org/apache/calcite/adapter/splunk/SplunkDriver.java +++ b/splunk/src/main/java/org/apache/calcite/adapter/splunk/SplunkDriver.java @@ -24,7 +24,7 @@ import org.apache.calcite.linq4j.Enumerator; import org.apache.calcite.schema.SchemaPlus; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.net.URI; import java.net.URL; diff --git a/splunk/src/main/java/org/apache/calcite/adapter/splunk/SplunkQuery.java b/splunk/src/main/java/org/apache/calcite/adapter/splunk/SplunkQuery.java index 2b9c848d3825..470d02b5c1c1 100644 --- a/splunk/src/main/java/org/apache/calcite/adapter/splunk/SplunkQuery.java +++ b/splunk/src/main/java/org/apache/calcite/adapter/splunk/SplunkQuery.java @@ -21,7 +21,7 @@ import org.apache.calcite.linq4j.AbstractEnumerable; import org.apache.calcite.linq4j.Enumerator; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.HashMap; import java.util.List; diff --git a/splunk/src/main/java/org/apache/calcite/adapter/splunk/SplunkTable.java b/splunk/src/main/java/org/apache/calcite/adapter/splunk/SplunkTable.java index 4c27074f58f9..fcb42a3c0738 100644 --- a/splunk/src/main/java/org/apache/calcite/adapter/splunk/SplunkTable.java +++ b/splunk/src/main/java/org/apache/calcite/adapter/splunk/SplunkTable.java @@ -29,7 +29,7 @@ import org.apache.calcite.schema.TranslatableTable; import org.apache.calcite.schema.impl.AbstractTableQueryable; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/splunk/src/main/java/org/apache/calcite/adapter/splunk/search/SplunkConnection.java b/splunk/src/main/java/org/apache/calcite/adapter/splunk/search/SplunkConnection.java index 6374f8d78c08..8631e9d25e08 100644 --- a/splunk/src/main/java/org/apache/calcite/adapter/splunk/search/SplunkConnection.java +++ b/splunk/src/main/java/org/apache/calcite/adapter/splunk/search/SplunkConnection.java @@ -18,7 +18,7 @@ import org.apache.calcite.linq4j.Enumerator; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; import java.util.Map; diff --git a/splunk/src/main/java/org/apache/calcite/adapter/splunk/search/SplunkConnectionImpl.java b/splunk/src/main/java/org/apache/calcite/adapter/splunk/search/SplunkConnectionImpl.java index b8498c8a7ffa..1ff97d5a36e2 100644 --- a/splunk/src/main/java/org/apache/calcite/adapter/splunk/search/SplunkConnectionImpl.java +++ b/splunk/src/main/java/org/apache/calcite/adapter/splunk/search/SplunkConnectionImpl.java @@ -24,7 +24,7 @@ import au.com.bytecode.opencsv.CSVReader; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserFixture.java b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserFixture.java index 34fc7ac1140e..2d775f2db7c3 100644 --- a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserFixture.java +++ b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserFixture.java @@ -25,8 +25,8 @@ import org.apache.calcite.sql.validate.SqlConformance; import org.apache.calcite.sql.validate.SqlConformanceEnum; -import org.checkerframework.checker.nullness.qual.Nullable; import org.hamcrest.Matcher; +import org.jspecify.annotations.Nullable; import java.util.List; import java.util.function.Consumer; diff --git a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserListFixture.java b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserListFixture.java index fd5e83eb2dd9..3a1b5a6f63ff 100644 --- a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserListFixture.java +++ b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserListFixture.java @@ -21,7 +21,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.function.UnaryOperator; diff --git a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java index a64edcd6fb4d..4d5c9e806d49 100644 --- a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java +++ b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java @@ -56,11 +56,11 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableSortedSet; -import org.checkerframework.checker.nullness.qual.Nullable; import org.hamcrest.BaseMatcher; import org.hamcrest.CustomTypeSafeMatcher; import org.hamcrest.Description; import org.hamcrest.Matcher; +import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; diff --git a/testkit/src/main/java/org/apache/calcite/sql/test/AbstractSqlTester.java b/testkit/src/main/java/org/apache/calcite/sql/test/AbstractSqlTester.java index 00132a93118f..39b0b1e3fec3 100644 --- a/testkit/src/main/java/org/apache/calcite/sql/test/AbstractSqlTester.java +++ b/testkit/src/main/java/org/apache/calcite/sql/test/AbstractSqlTester.java @@ -50,8 +50,8 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; import org.hamcrest.Matcher; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.Collection; diff --git a/testkit/src/main/java/org/apache/calcite/sql/test/SqlOperatorFixture.java b/testkit/src/main/java/org/apache/calcite/sql/test/SqlOperatorFixture.java index 94436699b216..e2a0fb30ec5c 100644 --- a/testkit/src/main/java/org/apache/calcite/sql/test/SqlOperatorFixture.java +++ b/testkit/src/main/java/org/apache/calcite/sql/test/SqlOperatorFixture.java @@ -37,7 +37,7 @@ import org.apache.calcite.test.Matchers; import org.apache.calcite.util.Bug; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/testkit/src/main/java/org/apache/calcite/sql/test/SqlTestFactory.java b/testkit/src/main/java/org/apache/calcite/sql/test/SqlTestFactory.java index 9ecc8429c485..c2a65c6003d3 100644 --- a/testkit/src/main/java/org/apache/calcite/sql/test/SqlTestFactory.java +++ b/testkit/src/main/java/org/apache/calcite/sql/test/SqlTestFactory.java @@ -52,7 +52,7 @@ import com.google.common.base.Suppliers; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; import java.util.function.Supplier; diff --git a/testkit/src/main/java/org/apache/calcite/sql/test/SqlTester.java b/testkit/src/main/java/org/apache/calcite/sql/test/SqlTester.java index c47839db623d..90d5e8a2c708 100644 --- a/testkit/src/main/java/org/apache/calcite/sql/test/SqlTester.java +++ b/testkit/src/main/java/org/apache/calcite/sql/test/SqlTester.java @@ -27,7 +27,7 @@ import org.apache.calcite.test.DiffRepository; import org.apache.calcite.util.Pair; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.sql.ResultSet; import java.util.function.Consumer; diff --git a/testkit/src/main/java/org/apache/calcite/sql/test/SqlTests.java b/testkit/src/main/java/org/apache/calcite/sql/test/SqlTests.java index 2cfaf8fc872d..021321c27350 100644 --- a/testkit/src/main/java/org/apache/calcite/sql/test/SqlTests.java +++ b/testkit/src/main/java/org/apache/calcite/sql/test/SqlTests.java @@ -29,7 +29,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Arrays; import java.util.List; diff --git a/testkit/src/main/java/org/apache/calcite/test/AbstractModifiableTable.java b/testkit/src/main/java/org/apache/calcite/test/AbstractModifiableTable.java index 59b9016a7b61..b7ab79cc5e7a 100644 --- a/testkit/src/main/java/org/apache/calcite/test/AbstractModifiableTable.java +++ b/testkit/src/main/java/org/apache/calcite/test/AbstractModifiableTable.java @@ -26,7 +26,7 @@ import org.apache.calcite.schema.ModifiableTable; import org.apache.calcite.schema.impl.AbstractTable; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/testkit/src/main/java/org/apache/calcite/test/CalciteAssert.java b/testkit/src/main/java/org/apache/calcite/test/CalciteAssert.java index aa27c7b77a8c..6a551a5957d0 100644 --- a/testkit/src/main/java/org/apache/calcite/test/CalciteAssert.java +++ b/testkit/src/main/java/org/apache/calcite/test/CalciteAssert.java @@ -103,8 +103,8 @@ import net.hydromatic.steelwheels.data.hsqldb.SteelwheelsHsqldb; import org.apiguardian.api.API; -import org.checkerframework.checker.nullness.qual.Nullable; import org.hamcrest.Matcher; +import org.jspecify.annotations.Nullable; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; diff --git a/testkit/src/main/java/org/apache/calcite/test/ConnectionSpec.java b/testkit/src/main/java/org/apache/calcite/test/ConnectionSpec.java index c354fcfbfa4f..e712ea197094 100644 --- a/testkit/src/main/java/org/apache/calcite/test/ConnectionSpec.java +++ b/testkit/src/main/java/org/apache/calcite/test/ConnectionSpec.java @@ -18,7 +18,7 @@ import com.google.errorprone.annotations.Immutable; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** Information necessary to create a JDBC connection. * diff --git a/testkit/src/main/java/org/apache/calcite/test/DiffRepository.java b/testkit/src/main/java/org/apache/calcite/test/DiffRepository.java index db60757f4bd9..7ca26a1df08a 100644 --- a/testkit/src/main/java/org/apache/calcite/test/DiffRepository.java +++ b/testkit/src/main/java/org/apache/calcite/test/DiffRepository.java @@ -29,7 +29,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSortedSet; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.Assertions; import org.opentest4j.AssertionFailedError; import org.w3c.dom.CDATASection; diff --git a/testkit/src/main/java/org/apache/calcite/test/DiffTestCase.java b/testkit/src/main/java/org/apache/calcite/test/DiffTestCase.java index 57b9962a55c5..64fa1c6503c5 100644 --- a/testkit/src/main/java/org/apache/calcite/test/DiffTestCase.java +++ b/testkit/src/main/java/org/apache/calcite/test/DiffTestCase.java @@ -20,9 +20,9 @@ import org.apache.calcite.util.TestUtil; import org.apache.calcite.util.Util; -import org.checkerframework.checker.nullness.qual.Nullable; import org.incava.diff.Diff; import org.incava.diff.Difference; +import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; diff --git a/testkit/src/main/java/org/apache/calcite/test/MockDdlExecutor.java b/testkit/src/main/java/org/apache/calcite/test/MockDdlExecutor.java index 8d744d829bee..6c0215e38a21 100644 --- a/testkit/src/main/java/org/apache/calcite/test/MockDdlExecutor.java +++ b/testkit/src/main/java/org/apache/calcite/test/MockDdlExecutor.java @@ -65,7 +65,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Type; import java.sql.PreparedStatement; diff --git a/testkit/src/main/java/org/apache/calcite/test/MockRelOptPlanner.java b/testkit/src/main/java/org/apache/calcite/test/MockRelOptPlanner.java index 74fcf5cec36a..40549361543c 100644 --- a/testkit/src/main/java/org/apache/calcite/test/MockRelOptPlanner.java +++ b/testkit/src/main/java/org/apache/calcite/test/MockRelOptPlanner.java @@ -32,7 +32,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.Collections; diff --git a/testkit/src/main/java/org/apache/calcite/test/MockSqlOperatorTable.java b/testkit/src/main/java/org/apache/calcite/test/MockSqlOperatorTable.java index a2eaae23d2ab..7f73e74b9821 100644 --- a/testkit/src/main/java/org/apache/calcite/test/MockSqlOperatorTable.java +++ b/testkit/src/main/java/org/apache/calcite/test/MockSqlOperatorTable.java @@ -55,7 +55,7 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Lists; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; import java.util.Map; diff --git a/testkit/src/main/java/org/apache/calcite/test/QuidemTest.java b/testkit/src/main/java/org/apache/calcite/test/QuidemTest.java index e9d18930d408..ac21cf102490 100644 --- a/testkit/src/main/java/org/apache/calcite/test/QuidemTest.java +++ b/testkit/src/main/java/org/apache/calcite/test/QuidemTest.java @@ -59,7 +59,7 @@ import net.hydromatic.quidem.CommandHandler; import net.hydromatic.quidem.Quidem; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; diff --git a/testkit/src/main/java/org/apache/calcite/test/ReflectiveSchemaWithoutRowCount.java b/testkit/src/main/java/org/apache/calcite/test/ReflectiveSchemaWithoutRowCount.java index d87173a87197..2410d194a1b3 100644 --- a/testkit/src/main/java/org/apache/calcite/test/ReflectiveSchemaWithoutRowCount.java +++ b/testkit/src/main/java/org/apache/calcite/test/ReflectiveSchemaWithoutRowCount.java @@ -18,7 +18,7 @@ import org.apache.calcite.adapter.java.ReflectiveSchema; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * A ReflectiveSchema that does not return row count statistics. diff --git a/testkit/src/main/java/org/apache/calcite/test/RelOptFixture.java b/testkit/src/main/java/org/apache/calcite/test/RelOptFixture.java index 68a6111d0629..d65d3f2fb42f 100644 --- a/testkit/src/main/java/org/apache/calcite/test/RelOptFixture.java +++ b/testkit/src/main/java/org/apache/calcite/test/RelOptFixture.java @@ -48,7 +48,7 @@ import com.google.common.collect.ImmutableMap; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.List; diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorFixtureImpl.java b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorFixtureImpl.java index 0f0ec67344c9..b491d6f7290f 100644 --- a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorFixtureImpl.java +++ b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorFixtureImpl.java @@ -30,8 +30,8 @@ import org.apache.calcite.sql.validate.SqlValidator; import org.apache.calcite.util.JdbcType; -import org.checkerframework.checker.nullness.qual.Nullable; import org.hamcrest.Matcher; +import org.jspecify.annotations.Nullable; import java.util.List; import java.util.function.UnaryOperator; diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorFixtures.java b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorFixtures.java index d6affd622e92..dc703d15afdd 100644 --- a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorFixtures.java +++ b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorFixtures.java @@ -19,7 +19,7 @@ import org.apache.calcite.sql.test.SqlOperatorFixture; import org.apache.calcite.util.DelegatingInvocationHandler; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Proxy; import java.util.regex.Pattern; diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java index 6082c27638df..fe5c32b7adaa 100644 --- a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java +++ b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java @@ -87,7 +87,7 @@ import com.google.common.base.Throwables; import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlRuntimeTester.java b/testkit/src/main/java/org/apache/calcite/test/SqlRuntimeTester.java index 8c641b3a98ee..e484fbcd99b2 100644 --- a/testkit/src/main/java/org/apache/calcite/test/SqlRuntimeTester.java +++ b/testkit/src/main/java/org/apache/calcite/test/SqlRuntimeTester.java @@ -24,7 +24,7 @@ import org.apache.calcite.sql.test.SqlTests; import org.apache.calcite.sql.validate.SqlValidator; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import static org.junit.jupiter.api.Assertions.assertNotNull; diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlToRelFixture.java b/testkit/src/main/java/org/apache/calcite/test/SqlToRelFixture.java index 784cb1eba1dd..a6840f6a2021 100644 --- a/testkit/src/main/java/org/apache/calcite/test/SqlToRelFixture.java +++ b/testkit/src/main/java/org/apache/calcite/test/SqlToRelFixture.java @@ -29,7 +29,7 @@ import org.apache.calcite.test.catalog.MockCatalogReaderExtended; import org.apache.calcite.util.TestUtil; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.function.Predicate; import java.util.function.UnaryOperator; diff --git a/testkit/src/main/java/org/apache/calcite/test/catalog/MockCatalogReader.java b/testkit/src/main/java/org/apache/calcite/test/catalog/MockCatalogReader.java index 87328381fe49..e0e84ae923ee 100644 --- a/testkit/src/main/java/org/apache/calcite/test/catalog/MockCatalogReader.java +++ b/testkit/src/main/java/org/apache/calcite/test/catalog/MockCatalogReader.java @@ -95,7 +95,7 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Type; import java.util.AbstractList; diff --git a/testkit/src/main/java/org/apache/calcite/test/catalog/MockCatalogReaderDynamic.java b/testkit/src/main/java/org/apache/calcite/test/catalog/MockCatalogReaderDynamic.java index 5b859e578c80..2a7ef23ae267 100644 --- a/testkit/src/main/java/org/apache/calcite/test/catalog/MockCatalogReaderDynamic.java +++ b/testkit/src/main/java/org/apache/calcite/test/catalog/MockCatalogReaderDynamic.java @@ -23,7 +23,7 @@ import org.apache.calcite.schema.impl.ViewTable; import org.apache.calcite.sql.type.SqlTypeName; -import org.checkerframework.checker.nullness.qual.NonNull; +import org.jspecify.annotations.NonNull; import java.util.Arrays; import java.util.Collections; diff --git a/testkit/src/main/java/org/apache/calcite/test/catalog/MockCatalogReaderExtended.java b/testkit/src/main/java/org/apache/calcite/test/catalog/MockCatalogReaderExtended.java index 3a70e306aedf..fccf51db3e69 100644 --- a/testkit/src/main/java/org/apache/calcite/test/catalog/MockCatalogReaderExtended.java +++ b/testkit/src/main/java/org/apache/calcite/test/catalog/MockCatalogReaderExtended.java @@ -35,7 +35,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.NonNull; +import org.jspecify.annotations.NonNull; import java.util.ArrayList; import java.util.Arrays; diff --git a/testkit/src/main/java/org/apache/calcite/test/catalog/MockCatalogReaderSimple.java b/testkit/src/main/java/org/apache/calcite/test/catalog/MockCatalogReaderSimple.java index 1f7be1141091..e464c3855c5c 100644 --- a/testkit/src/main/java/org/apache/calcite/test/catalog/MockCatalogReaderSimple.java +++ b/testkit/src/main/java/org/apache/calcite/test/catalog/MockCatalogReaderSimple.java @@ -35,7 +35,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.NonNull; +import org.jspecify.annotations.NonNull; import java.math.BigDecimal; import java.util.Arrays; diff --git a/testkit/src/main/java/org/apache/calcite/test/schemata/bookstore/BookstoreSchema.java b/testkit/src/main/java/org/apache/calcite/test/schemata/bookstore/BookstoreSchema.java index 84c334c48bdd..55e8fd3c1ae4 100644 --- a/testkit/src/main/java/org/apache/calcite/test/schemata/bookstore/BookstoreSchema.java +++ b/testkit/src/main/java/org/apache/calcite/test/schemata/bookstore/BookstoreSchema.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.test.schemata.bookstore; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.math.BigDecimal; import java.util.Arrays; diff --git a/testkit/src/main/java/org/apache/calcite/test/schemata/catchall/CatchallSchema.java b/testkit/src/main/java/org/apache/calcite/test/schemata/catchall/CatchallSchema.java index df1c8dbb9a4d..2aad0b3c4b8c 100644 --- a/testkit/src/main/java/org/apache/calcite/test/schemata/catchall/CatchallSchema.java +++ b/testkit/src/main/java/org/apache/calcite/test/schemata/catchall/CatchallSchema.java @@ -23,7 +23,7 @@ import org.apache.calcite.test.schemata.hr.Employee; import org.apache.calcite.test.schemata.hr.HrSchema; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.lang.reflect.Field; import java.math.BigDecimal; diff --git a/testkit/src/main/java/org/apache/calcite/test/schemata/countries/CountriesTableFunction.java b/testkit/src/main/java/org/apache/calcite/test/schemata/countries/CountriesTableFunction.java index a918a69a5d6c..4a5a945590a2 100644 --- a/testkit/src/main/java/org/apache/calcite/test/schemata/countries/CountriesTableFunction.java +++ b/testkit/src/main/java/org/apache/calcite/test/schemata/countries/CountriesTableFunction.java @@ -33,7 +33,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** A table function that returns all countries in the world. * diff --git a/testkit/src/main/java/org/apache/calcite/test/schemata/countries/StatesTableFunction.java b/testkit/src/main/java/org/apache/calcite/test/schemata/countries/StatesTableFunction.java index 69e6e95a7a6f..3c30cc876966 100644 --- a/testkit/src/main/java/org/apache/calcite/test/schemata/countries/StatesTableFunction.java +++ b/testkit/src/main/java/org/apache/calcite/test/schemata/countries/StatesTableFunction.java @@ -33,7 +33,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** A table function that returns states and their boundaries; also national * parks. diff --git a/testkit/src/main/java/org/apache/calcite/test/schemata/foodmart/FoodmartSchema.java b/testkit/src/main/java/org/apache/calcite/test/schemata/foodmart/FoodmartSchema.java index 9c818709c581..4ff4ef972a7f 100644 --- a/testkit/src/main/java/org/apache/calcite/test/schemata/foodmart/FoodmartSchema.java +++ b/testkit/src/main/java/org/apache/calcite/test/schemata/foodmart/FoodmartSchema.java @@ -18,7 +18,7 @@ import org.apache.calcite.test.CalciteAssert; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Objects; diff --git a/testkit/src/main/java/org/apache/calcite/test/schemata/hr/Department.java b/testkit/src/main/java/org/apache/calcite/test/schemata/hr/Department.java index a3bdc6002bef..f1a7cf47d2ef 100644 --- a/testkit/src/main/java/org/apache/calcite/test/schemata/hr/Department.java +++ b/testkit/src/main/java/org/apache/calcite/test/schemata/hr/Department.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.test.schemata.hr; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.List; import java.util.Objects; diff --git a/testkit/src/main/java/org/apache/calcite/test/schemata/hr/DepartmentPlus.java b/testkit/src/main/java/org/apache/calcite/test/schemata/hr/DepartmentPlus.java index f93b3916dc78..83e968306b0e 100644 --- a/testkit/src/main/java/org/apache/calcite/test/schemata/hr/DepartmentPlus.java +++ b/testkit/src/main/java/org/apache/calcite/test/schemata/hr/DepartmentPlus.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.test.schemata.hr; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.sql.Timestamp; import java.util.List; diff --git a/testkit/src/main/java/org/apache/calcite/test/schemata/hr/Employee.java b/testkit/src/main/java/org/apache/calcite/test/schemata/hr/Employee.java index 24864e810908..c3e045df4fc9 100644 --- a/testkit/src/main/java/org/apache/calcite/test/schemata/hr/Employee.java +++ b/testkit/src/main/java/org/apache/calcite/test/schemata/hr/Employee.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.test.schemata.hr; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Objects; diff --git a/testkit/src/main/java/org/apache/calcite/test/schemata/hr/Event.java b/testkit/src/main/java/org/apache/calcite/test/schemata/hr/Event.java index 31cf4c616e48..269125a83a7f 100644 --- a/testkit/src/main/java/org/apache/calcite/test/schemata/hr/Event.java +++ b/testkit/src/main/java/org/apache/calcite/test/schemata/hr/Event.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.test.schemata.hr; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.sql.Timestamp; import java.util.Objects; diff --git a/testkit/src/main/java/org/apache/calcite/test/schemata/hr/NullableTest.java b/testkit/src/main/java/org/apache/calcite/test/schemata/hr/NullableTest.java index baec4d397cfc..d6b87b97f63e 100644 --- a/testkit/src/main/java/org/apache/calcite/test/schemata/hr/NullableTest.java +++ b/testkit/src/main/java/org/apache/calcite/test/schemata/hr/NullableTest.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.test.schemata.hr; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Objects; diff --git a/testkit/src/main/java/org/apache/calcite/test/schemata/orderstream/BaseOrderStreamTable.java b/testkit/src/main/java/org/apache/calcite/test/schemata/orderstream/BaseOrderStreamTable.java index c07f8e4e95d2..94e5b84fbb5e 100644 --- a/testkit/src/main/java/org/apache/calcite/test/schemata/orderstream/BaseOrderStreamTable.java +++ b/testkit/src/main/java/org/apache/calcite/test/schemata/orderstream/BaseOrderStreamTable.java @@ -31,7 +31,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Base table for the Orders table. Manages the base schema used for the test tables and common diff --git a/testkit/src/main/java/org/apache/calcite/test/schemata/orderstream/InfiniteOrdersStreamTableFactory.java b/testkit/src/main/java/org/apache/calcite/test/schemata/orderstream/InfiniteOrdersStreamTableFactory.java index 4686a0500b15..20f7db39e768 100644 --- a/testkit/src/main/java/org/apache/calcite/test/schemata/orderstream/InfiniteOrdersStreamTableFactory.java +++ b/testkit/src/main/java/org/apache/calcite/test/schemata/orderstream/InfiniteOrdersStreamTableFactory.java @@ -21,7 +21,7 @@ import org.apache.calcite.schema.Table; import org.apache.calcite.schema.TableFactory; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Map; diff --git a/testkit/src/main/java/org/apache/calcite/test/schemata/orderstream/InfiniteOrdersTable.java b/testkit/src/main/java/org/apache/calcite/test/schemata/orderstream/InfiniteOrdersTable.java index 2917c04f34e1..0c01004a2d48 100644 --- a/testkit/src/main/java/org/apache/calcite/test/schemata/orderstream/InfiniteOrdersTable.java +++ b/testkit/src/main/java/org/apache/calcite/test/schemata/orderstream/InfiniteOrdersTable.java @@ -22,7 +22,7 @@ import org.apache.calcite.schema.StreamableTable; import org.apache.calcite.schema.Table; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Iterator; diff --git a/testkit/src/main/java/org/apache/calcite/test/schemata/orderstream/OrdersHistoryTable.java b/testkit/src/main/java/org/apache/calcite/test/schemata/orderstream/OrdersHistoryTable.java index c5c5e92ce9f5..dca1a92a5825 100644 --- a/testkit/src/main/java/org/apache/calcite/test/schemata/orderstream/OrdersHistoryTable.java +++ b/testkit/src/main/java/org/apache/calcite/test/schemata/orderstream/OrdersHistoryTable.java @@ -22,7 +22,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** Table representing the history of the ORDERS stream. */ public class OrdersHistoryTable extends BaseOrderStreamTable { diff --git a/testkit/src/main/java/org/apache/calcite/test/schemata/orderstream/OrdersStreamTableFactory.java b/testkit/src/main/java/org/apache/calcite/test/schemata/orderstream/OrdersStreamTableFactory.java index b8d6202b53db..e305ef0a88d8 100644 --- a/testkit/src/main/java/org/apache/calcite/test/schemata/orderstream/OrdersStreamTableFactory.java +++ b/testkit/src/main/java/org/apache/calcite/test/schemata/orderstream/OrdersStreamTableFactory.java @@ -24,7 +24,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Map; diff --git a/testkit/src/main/java/org/apache/calcite/test/schemata/orderstream/OrdersTable.java b/testkit/src/main/java/org/apache/calcite/test/schemata/orderstream/OrdersTable.java index cf1c0e2aa700..36bafa2b990d 100644 --- a/testkit/src/main/java/org/apache/calcite/test/schemata/orderstream/OrdersTable.java +++ b/testkit/src/main/java/org/apache/calcite/test/schemata/orderstream/OrdersTable.java @@ -27,7 +27,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Table representing the ORDERS stream. diff --git a/testkit/src/main/java/org/apache/calcite/test/schemata/orderstream/ProductsTable.java b/testkit/src/main/java/org/apache/calcite/test/schemata/orderstream/ProductsTable.java index a0c3dec42433..b2ef4d259709 100644 --- a/testkit/src/main/java/org/apache/calcite/test/schemata/orderstream/ProductsTable.java +++ b/testkit/src/main/java/org/apache/calcite/test/schemata/orderstream/ProductsTable.java @@ -33,7 +33,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Table representing the PRODUCTS relation. diff --git a/testkit/src/main/java/org/apache/calcite/test/schemata/orderstream/ProductsTableFactory.java b/testkit/src/main/java/org/apache/calcite/test/schemata/orderstream/ProductsTableFactory.java index d306fc4f61b8..f3005cd8cf9d 100644 --- a/testkit/src/main/java/org/apache/calcite/test/schemata/orderstream/ProductsTableFactory.java +++ b/testkit/src/main/java/org/apache/calcite/test/schemata/orderstream/ProductsTableFactory.java @@ -23,7 +23,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.util.Map; diff --git a/testkit/src/main/java/org/apache/calcite/test/schemata/orderstream/ProductsTemporalTable.java b/testkit/src/main/java/org/apache/calcite/test/schemata/orderstream/ProductsTemporalTable.java index 92a7d6e7e45b..42fd5656e54b 100644 --- a/testkit/src/main/java/org/apache/calcite/test/schemata/orderstream/ProductsTemporalTable.java +++ b/testkit/src/main/java/org/apache/calcite/test/schemata/orderstream/ProductsTemporalTable.java @@ -30,7 +30,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; /** * Table representing the PRODUCTS_TEMPORAL temporal table. diff --git a/testkit/src/main/java/org/apache/calcite/util/Smalls.java b/testkit/src/main/java/org/apache/calcite/util/Smalls.java index 0184a0f1681a..20e210c7bb22 100644 --- a/testkit/src/main/java/org/apache/calcite/util/Smalls.java +++ b/testkit/src/main/java/org/apache/calcite/util/Smalls.java @@ -65,7 +65,7 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.jspecify.annotations.Nullable; import java.io.IOException; import java.lang.reflect.Method; From 9a6e73c4bfc4752c997ee1d2a64bfd3026626933 Mon Sep 17 00:00:00 2001 From: Vladimir Sitnikov Date: Sun, 23 Aug 2026 22:11:06 +0300 Subject: [PATCH 496/562] [CALCITE-7736] Declare @NullMarked on the packages that NullAway verifies `@DefaultQualifier(NonNull, {FIELD, PARAMETER, RETURN})` is how the Checker Framework said "types are non-null unless annotated". JSpecify says it with `@NullMarked`, which covers every type position rather than three of them. Only `calcite-linq4j` and `calcite-core` are verified, so only their main packages get the annotation: `@NullMarked` claims that a package is fully annotated, and in an unverified module nothing backs that claim. The 23 `package-info.java` files that carried `@DefaultQualifier` are converted, and the remaining packages of those two modules get the annotation. `babel`'s test `package-info.java` also carried `@DefaultQualifier`. It loses the annotation rather than gaining `@NullMarked`, because `checker-qual` is gone and test code is not verified. NullAway skips a package that is not `@NullMarked`, so a missing annotation costs coverage without saying a word. `LintTest.testLintNullMarked` walks the source roots in `NULL_MARKED_ROOTS` and fails when a package there has no `package-info.java`, or has one that does not declare `@NullMarked`. Widen that list together with `nullawayProjects`. Co-Authored-By: Claude Opus 5 --- .../org/apache/calcite/test/package-info.java | 7 -- .../calcite/adapter/clone/package-info.java | 3 + .../adapter/enumerable/impl/package-info.java | 3 + .../adapter/enumerable/package-info.java | 3 + .../calcite/adapter/java/package-info.java | 3 + .../calcite/adapter/jdbc/package-info.java | 3 + .../apache/calcite/adapter/package-info.java | 8 +-- .../apache/calcite/config/package-info.java | 8 +-- .../calcite/interpreter/package-info.java | 8 +-- .../org/apache/calcite/jdbc/package-info.java | 8 +-- .../calcite/materialize/package-info.java | 8 +-- .../apache/calcite/model/package-info.java | 8 +-- .../java/org/apache/calcite/package-info.java | 3 + .../apache/calcite/plan/hep/package-info.java | 3 + .../org/apache/calcite/plan/package-info.java | 8 +-- .../calcite/plan/visualizer/package-info.java | 3 + .../calcite/plan/volcano/package-info.java | 3 + .../apache/calcite/prepare/package-info.java | 8 +-- .../apache/calcite/profile/package-info.java | 8 +-- .../calcite/rel/convert/package-info.java | 3 + .../apache/calcite/rel/core/package-info.java | 3 + .../calcite/rel/externalize/package-info.java | 3 + .../apache/calcite/rel/hint/package-info.java | 3 + .../calcite/rel/logical/package-info.java | 3 + .../rel/metadata/janino/package-info.java | 8 +-- .../calcite/rel/metadata/package-info.java | 3 + .../calcite/rel/mutable/package-info.java | 3 + .../org/apache/calcite/rel/package-info.java | 8 +-- .../calcite/rel/rel2sql/package-info.java | 3 + .../rel/rules/materialize/package-info.java | 3 + .../calcite/rel/rules/package-info.java | 3 + .../calcite/rel/stream/package-info.java | 3 + .../apache/calcite/rel/type/package-info.java | 3 + .../org/apache/calcite/rex/package-info.java | 8 +-- .../apache/calcite/runtime/package-info.java | 8 +-- .../calcite/runtime/rtti/package-info.java | 3 + .../calcite/runtime/variant/package-info.java | 3 + .../calcite/schema/impl/package-info.java | 3 + .../calcite/schema/lookup/package-info.java | 8 +-- .../apache/calcite/schema/package-info.java | 8 +-- .../apache/calcite/server/package-info.java | 8 +-- .../calcite/sql/advise/package-info.java | 3 + .../apache/calcite/sql/ddl/package-info.java | 3 + .../calcite/sql/dialect/package-info.java | 3 + .../apache/calcite/sql/fun/package-info.java | 3 + .../org/apache/calcite/sql/package-info.java | 8 +-- .../calcite/sql/parser/impl/package-info.java | 3 + .../calcite/sql/parser/package-info.java | 3 + .../calcite/sql/pretty/package-info.java | 3 + .../apache/calcite/sql/type/package-info.java | 3 + .../apache/calcite/sql/util/package-info.java | 3 + .../sql/validate/implicit/package-info.java | 3 + .../calcite/sql/validate/package-info.java | 3 + .../apache/calcite/sql2rel/package-info.java | 8 +-- .../calcite/statistic/package-info.java | 8 +-- .../apache/calcite/tools/package-info.java | 8 +-- .../calcite/util/format/package-info.java | 3 + .../format/compiled/package-info.java | 3 + .../postgresql/format/package-info.java | 3 + .../util/format/postgresql/package-info.java | 3 + .../calcite/util/graph/package-info.java | 3 + .../calcite/util/javac/package-info.java | 3 + .../calcite/util/mapping/package-info.java | 3 + .../org/apache/calcite/util/package-info.java | 8 +-- .../calcite/util/trace/package-info.java | 3 + .../org/apache/calcite/test/LintTest.java | 68 +++++++++++++++++++ .../calcite/linq4j/function/package-info.java | 3 + .../apache/calcite/linq4j/package-info.java | 8 +-- .../calcite/linq4j/tree/package-info.java | 3 + .../calcite/linq4j/util/package-info.java | 3 + 70 files changed, 251 insertions(+), 138 deletions(-) diff --git a/babel/src/test/java/org/apache/calcite/test/package-info.java b/babel/src/test/java/org/apache/calcite/test/package-info.java index f5580cb7a5f2..ca3131428577 100644 --- a/babel/src/test/java/org/apache/calcite/test/package-info.java +++ b/babel/src/test/java/org/apache/calcite/test/package-info.java @@ -18,11 +18,4 @@ /** * Tests for Calcite. */ -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.FIELD) -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.PARAMETER) -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.RETURN) package org.apache.calcite.test; - -import org.checkerframework.framework.qual.DefaultQualifier; -import org.checkerframework.framework.qual.TypeUseLocation; -import org.jspecify.annotations.NonNull; diff --git a/core/src/main/java/org/apache/calcite/adapter/clone/package-info.java b/core/src/main/java/org/apache/calcite/adapter/clone/package-info.java index d2f9429783e5..cdd1af71a2ed 100644 --- a/core/src/main/java/org/apache/calcite/adapter/clone/package-info.java +++ b/core/src/main/java/org/apache/calcite/adapter/clone/package-info.java @@ -18,4 +18,7 @@ /** * Provides utility classes. */ +@NullMarked package org.apache.calcite.adapter.clone; + +import org.jspecify.annotations.NullMarked; diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/impl/package-info.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/impl/package-info.java index 722e56d6b0fc..74249c98146a 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/impl/package-info.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/impl/package-info.java @@ -18,4 +18,7 @@ /** * Calcite-specific classes for implementation of regular and window aggregates. */ +@NullMarked package org.apache.calcite.adapter.enumerable.impl; + +import org.jspecify.annotations.NullMarked; diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/package-info.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/package-info.java index 2b626c35a850..8ed908082c81 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/package-info.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/package-info.java @@ -18,4 +18,7 @@ /** * Query optimizer rules for Java calling convention. */ +@NullMarked package org.apache.calcite.adapter.enumerable; + +import org.jspecify.annotations.NullMarked; diff --git a/core/src/main/java/org/apache/calcite/adapter/java/package-info.java b/core/src/main/java/org/apache/calcite/adapter/java/package-info.java index fdd167c439fa..39044809ae7c 100644 --- a/core/src/main/java/org/apache/calcite/adapter/java/package-info.java +++ b/core/src/main/java/org/apache/calcite/adapter/java/package-info.java @@ -19,4 +19,7 @@ * Query provider based on Java in-memory data * structures. */ +@NullMarked package org.apache.calcite.adapter.java; + +import org.jspecify.annotations.NullMarked; diff --git a/core/src/main/java/org/apache/calcite/adapter/jdbc/package-info.java b/core/src/main/java/org/apache/calcite/adapter/jdbc/package-info.java index ecabc575d463..bbd21b91d4ab 100644 --- a/core/src/main/java/org/apache/calcite/adapter/jdbc/package-info.java +++ b/core/src/main/java/org/apache/calcite/adapter/jdbc/package-info.java @@ -18,4 +18,7 @@ /** * Query provider based on a JDBC data source. */ +@NullMarked package org.apache.calcite.adapter.jdbc; + +import org.jspecify.annotations.NullMarked; diff --git a/core/src/main/java/org/apache/calcite/adapter/package-info.java b/core/src/main/java/org/apache/calcite/adapter/package-info.java index b9faaec327f5..f5d6085b72c2 100644 --- a/core/src/main/java/org/apache/calcite/adapter/package-info.java +++ b/core/src/main/java/org/apache/calcite/adapter/package-info.java @@ -37,11 +37,7 @@ * * */ -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.FIELD) -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.PARAMETER) -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.RETURN) +@NullMarked package org.apache.calcite.adapter; -import org.checkerframework.framework.qual.DefaultQualifier; -import org.checkerframework.framework.qual.TypeUseLocation; -import org.jspecify.annotations.NonNull; +import org.jspecify.annotations.NullMarked; diff --git a/core/src/main/java/org/apache/calcite/config/package-info.java b/core/src/main/java/org/apache/calcite/config/package-info.java index 167617c0b88c..15198da0f7b4 100644 --- a/core/src/main/java/org/apache/calcite/config/package-info.java +++ b/core/src/main/java/org/apache/calcite/config/package-info.java @@ -18,11 +18,7 @@ /** * Configuration. */ -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.FIELD) -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.PARAMETER) -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.RETURN) +@NullMarked package org.apache.calcite.config; -import org.checkerframework.framework.qual.DefaultQualifier; -import org.checkerframework.framework.qual.TypeUseLocation; -import org.jspecify.annotations.NonNull; +import org.jspecify.annotations.NullMarked; diff --git a/core/src/main/java/org/apache/calcite/interpreter/package-info.java b/core/src/main/java/org/apache/calcite/interpreter/package-info.java index 06de1b6bcdfb..736a93b502b5 100644 --- a/core/src/main/java/org/apache/calcite/interpreter/package-info.java +++ b/core/src/main/java/org/apache/calcite/interpreter/package-info.java @@ -22,11 +22,7 @@ * preparation time is less, and so the total prepare + execute time is * competitive for queries over small data sets. */ -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.FIELD) -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.PARAMETER) -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.RETURN) +@NullMarked package org.apache.calcite.interpreter; -import org.checkerframework.framework.qual.DefaultQualifier; -import org.checkerframework.framework.qual.TypeUseLocation; -import org.jspecify.annotations.NonNull; +import org.jspecify.annotations.NullMarked; diff --git a/core/src/main/java/org/apache/calcite/jdbc/package-info.java b/core/src/main/java/org/apache/calcite/jdbc/package-info.java index 5fa5d6969262..997b39288b6f 100644 --- a/core/src/main/java/org/apache/calcite/jdbc/package-info.java +++ b/core/src/main/java/org/apache/calcite/jdbc/package-info.java @@ -18,11 +18,7 @@ /** * JDBC driver for Calcite. */ -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.FIELD) -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.PARAMETER) -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.RETURN) +@NullMarked package org.apache.calcite.jdbc; -import org.checkerframework.framework.qual.DefaultQualifier; -import org.checkerframework.framework.qual.TypeUseLocation; -import org.jspecify.annotations.NonNull; +import org.jspecify.annotations.NullMarked; diff --git a/core/src/main/java/org/apache/calcite/materialize/package-info.java b/core/src/main/java/org/apache/calcite/materialize/package-info.java index 2bcd6aae5cea..1554ece9caf9 100644 --- a/core/src/main/java/org/apache/calcite/materialize/package-info.java +++ b/core/src/main/java/org/apache/calcite/materialize/package-info.java @@ -32,11 +32,7 @@ * instantiating materializations from the intermediate results of queries, and * recognize what materializations would be useful based on actual query load. */ -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.FIELD) -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.PARAMETER) -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.RETURN) +@NullMarked package org.apache.calcite.materialize; -import org.checkerframework.framework.qual.DefaultQualifier; -import org.checkerframework.framework.qual.TypeUseLocation; -import org.jspecify.annotations.NonNull; +import org.jspecify.annotations.NullMarked; diff --git a/core/src/main/java/org/apache/calcite/model/package-info.java b/core/src/main/java/org/apache/calcite/model/package-info.java index 0c2993c4e649..13ae91f4b968 100644 --- a/core/src/main/java/org/apache/calcite/model/package-info.java +++ b/core/src/main/java/org/apache/calcite/model/package-info.java @@ -33,11 +33,7 @@ *

      There are several examples of schemas in the * tutorial. */ -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.FIELD) -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.PARAMETER) -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.RETURN) +@NullMarked package org.apache.calcite.model; -import org.checkerframework.framework.qual.DefaultQualifier; -import org.checkerframework.framework.qual.TypeUseLocation; -import org.jspecify.annotations.NonNull; +import org.jspecify.annotations.NullMarked; diff --git a/core/src/main/java/org/apache/calcite/package-info.java b/core/src/main/java/org/apache/calcite/package-info.java index add98daeb3e8..a5ba6cd989d8 100644 --- a/core/src/main/java/org/apache/calcite/package-info.java +++ b/core/src/main/java/org/apache/calcite/package-info.java @@ -19,4 +19,7 @@ * Main package for Calcite, the dynamic data management platform. */ @CalciteImmutable +@NullMarked package org.apache.calcite; + +import org.jspecify.annotations.NullMarked; diff --git a/core/src/main/java/org/apache/calcite/plan/hep/package-info.java b/core/src/main/java/org/apache/calcite/plan/hep/package-info.java index c388091ef224..840a71c588b5 100644 --- a/core/src/main/java/org/apache/calcite/plan/hep/package-info.java +++ b/core/src/main/java/org/apache/calcite/plan/hep/package-info.java @@ -19,4 +19,7 @@ * Provides a heuristic planner implementation for the interfaces in * {@link org.apache.calcite.plan}. */ +@NullMarked package org.apache.calcite.plan.hep; + +import org.jspecify.annotations.NullMarked; diff --git a/core/src/main/java/org/apache/calcite/plan/package-info.java b/core/src/main/java/org/apache/calcite/plan/package-info.java index a96143efe292..3515618763a8 100644 --- a/core/src/main/java/org/apache/calcite/plan/package-info.java +++ b/core/src/main/java/org/apache/calcite/plan/package-info.java @@ -19,11 +19,7 @@ * Defines interfaces for constructing rule-based optimizers of * relational expressions. */ -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.FIELD) -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.PARAMETER) -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.RETURN) +@NullMarked package org.apache.calcite.plan; -import org.checkerframework.framework.qual.DefaultQualifier; -import org.checkerframework.framework.qual.TypeUseLocation; -import org.jspecify.annotations.NonNull; +import org.jspecify.annotations.NullMarked; diff --git a/core/src/main/java/org/apache/calcite/plan/visualizer/package-info.java b/core/src/main/java/org/apache/calcite/plan/visualizer/package-info.java index 4f23efbcfc5a..f41225b70b49 100644 --- a/core/src/main/java/org/apache/calcite/plan/visualizer/package-info.java +++ b/core/src/main/java/org/apache/calcite/plan/visualizer/package-info.java @@ -20,4 +20,7 @@ * * @see org.apache.calcite.plan.visualizer.RuleMatchVisualizer */ +@NullMarked package org.apache.calcite.plan.visualizer; + +import org.jspecify.annotations.NullMarked; diff --git a/core/src/main/java/org/apache/calcite/plan/volcano/package-info.java b/core/src/main/java/org/apache/calcite/plan/volcano/package-info.java index 8605f1def88a..45b0d704b504 100644 --- a/core/src/main/java/org/apache/calcite/plan/volcano/package-info.java +++ b/core/src/main/java/org/apache/calcite/plan/volcano/package-info.java @@ -270,4 +270,7 @@ * McKenna * (1993). */ +@NullMarked package org.apache.calcite.plan.volcano; + +import org.jspecify.annotations.NullMarked; diff --git a/core/src/main/java/org/apache/calcite/prepare/package-info.java b/core/src/main/java/org/apache/calcite/prepare/package-info.java index 98fb70f64c24..c339d51517da 100644 --- a/core/src/main/java/org/apache/calcite/prepare/package-info.java +++ b/core/src/main/java/org/apache/calcite/prepare/package-info.java @@ -18,11 +18,7 @@ /** * Preparation of queries (parsing, planning and implementation). */ -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.FIELD) -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.PARAMETER) -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.RETURN) +@NullMarked package org.apache.calcite.prepare; -import org.checkerframework.framework.qual.DefaultQualifier; -import org.checkerframework.framework.qual.TypeUseLocation; -import org.jspecify.annotations.NonNull; +import org.jspecify.annotations.NullMarked; diff --git a/core/src/main/java/org/apache/calcite/profile/package-info.java b/core/src/main/java/org/apache/calcite/profile/package-info.java index 7d92e45e36d0..7a2ccabf515d 100644 --- a/core/src/main/java/org/apache/calcite/profile/package-info.java +++ b/core/src/main/java/org/apache/calcite/profile/package-info.java @@ -18,11 +18,7 @@ /** * Utilities to analyze data sets. */ -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.FIELD) -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.PARAMETER) -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.RETURN) +@NullMarked package org.apache.calcite.profile; -import org.checkerframework.framework.qual.DefaultQualifier; -import org.checkerframework.framework.qual.TypeUseLocation; -import org.jspecify.annotations.NonNull; +import org.jspecify.annotations.NullMarked; diff --git a/core/src/main/java/org/apache/calcite/rel/convert/package-info.java b/core/src/main/java/org/apache/calcite/rel/convert/package-info.java index 7b2fe7fd68f4..e38719c3f5df 100644 --- a/core/src/main/java/org/apache/calcite/rel/convert/package-info.java +++ b/core/src/main/java/org/apache/calcite/rel/convert/package-info.java @@ -19,4 +19,7 @@ * Defines relational expressions and rules for converting between calling * conventions. */ +@NullMarked package org.apache.calcite.rel.convert; + +import org.jspecify.annotations.NullMarked; diff --git a/core/src/main/java/org/apache/calcite/rel/core/package-info.java b/core/src/main/java/org/apache/calcite/rel/core/package-info.java index db717abc9baa..fe9122c45baa 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/package-info.java +++ b/core/src/main/java/org/apache/calcite/rel/core/package-info.java @@ -31,4 +31,7 @@ * * */ +@NullMarked package org.apache.calcite.rel.core; + +import org.jspecify.annotations.NullMarked; diff --git a/core/src/main/java/org/apache/calcite/rel/externalize/package-info.java b/core/src/main/java/org/apache/calcite/rel/externalize/package-info.java index f7f78eaa4ad1..514f911687f2 100644 --- a/core/src/main/java/org/apache/calcite/rel/externalize/package-info.java +++ b/core/src/main/java/org/apache/calcite/rel/externalize/package-info.java @@ -19,4 +19,7 @@ * Facilities to externalize {@link org.apache.calcite.rel.RelNode}s to and from * XML and JSON format. */ +@NullMarked package org.apache.calcite.rel.externalize; + +import org.jspecify.annotations.NullMarked; diff --git a/core/src/main/java/org/apache/calcite/rel/hint/package-info.java b/core/src/main/java/org/apache/calcite/rel/hint/package-info.java index 2c38fb5535bd..0aa5ee78288d 100644 --- a/core/src/main/java/org/apache/calcite/rel/hint/package-info.java +++ b/core/src/main/java/org/apache/calcite/rel/hint/package-info.java @@ -84,4 +84,7 @@ *

      Design Doc

      * Calcite SQL and Planner Hints Design. */ +@NullMarked package org.apache.calcite.rel.hint; + +import org.jspecify.annotations.NullMarked; diff --git a/core/src/main/java/org/apache/calcite/rel/logical/package-info.java b/core/src/main/java/org/apache/calcite/rel/logical/package-info.java index 26797ece0064..c5baf3bada6f 100644 --- a/core/src/main/java/org/apache/calcite/rel/logical/package-info.java +++ b/core/src/main/java/org/apache/calcite/rel/logical/package-info.java @@ -31,4 +31,7 @@ * * */ +@NullMarked package org.apache.calcite.rel.logical; + +import org.jspecify.annotations.NullMarked; diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/janino/package-info.java b/core/src/main/java/org/apache/calcite/rel/metadata/janino/package-info.java index 2d5274195c21..eef8695fb46b 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/janino/package-info.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/janino/package-info.java @@ -15,12 +15,10 @@ * limitations under the License. */ -/** - * Defines metadata interfaces and utilities for relational - * expressions. - */ - /** * Code for generating metadata handlers. */ +@NullMarked package org.apache.calcite.rel.metadata.janino; + +import org.jspecify.annotations.NullMarked; diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/package-info.java b/core/src/main/java/org/apache/calcite/rel/metadata/package-info.java index d61c78209c68..2267ce00ac64 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/package-info.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/package-info.java @@ -19,4 +19,7 @@ * Defines metadata interfaces and utilities for relational * expressions. */ +@NullMarked package org.apache.calcite.rel.metadata; + +import org.jspecify.annotations.NullMarked; diff --git a/core/src/main/java/org/apache/calcite/rel/mutable/package-info.java b/core/src/main/java/org/apache/calcite/rel/mutable/package-info.java index 1d62994efa1c..5fccfe9aa9fb 100644 --- a/core/src/main/java/org/apache/calcite/rel/mutable/package-info.java +++ b/core/src/main/java/org/apache/calcite/rel/mutable/package-info.java @@ -31,4 +31,7 @@ * * */ +@NullMarked package org.apache.calcite.rel.mutable; + +import org.jspecify.annotations.NullMarked; diff --git a/core/src/main/java/org/apache/calcite/rel/package-info.java b/core/src/main/java/org/apache/calcite/rel/package-info.java index 6612057eaacd..7c5a52cc04c9 100644 --- a/core/src/main/java/org/apache/calcite/rel/package-info.java +++ b/core/src/main/java/org/apache/calcite/rel/package-info.java @@ -35,11 +35,7 @@ * * */ -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.FIELD) -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.PARAMETER) -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.RETURN) +@NullMarked package org.apache.calcite.rel; -import org.checkerframework.framework.qual.DefaultQualifier; -import org.checkerframework.framework.qual.TypeUseLocation; -import org.jspecify.annotations.NonNull; +import org.jspecify.annotations.NullMarked; diff --git a/core/src/main/java/org/apache/calcite/rel/rel2sql/package-info.java b/core/src/main/java/org/apache/calcite/rel/rel2sql/package-info.java index 573f4076a93e..8074fe54338c 100644 --- a/core/src/main/java/org/apache/calcite/rel/rel2sql/package-info.java +++ b/core/src/main/java/org/apache/calcite/rel/rel2sql/package-info.java @@ -18,4 +18,7 @@ /** * Translates a relational expression to SQL parse tree. */ +@NullMarked package org.apache.calcite.rel.rel2sql; + +import org.jspecify.annotations.NullMarked; diff --git a/core/src/main/java/org/apache/calcite/rel/rules/materialize/package-info.java b/core/src/main/java/org/apache/calcite/rel/rules/materialize/package-info.java index 5432e29b2ae0..f6bc7572600d 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/materialize/package-info.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/materialize/package-info.java @@ -18,4 +18,7 @@ /** * Provides a materialized rewriting algorithm encapsulated within a planner rule. */ +@NullMarked package org.apache.calcite.rel.rules.materialize; + +import org.jspecify.annotations.NullMarked; diff --git a/core/src/main/java/org/apache/calcite/rel/rules/package-info.java b/core/src/main/java/org/apache/calcite/rel/rules/package-info.java index df52edb2a45a..363f757692c7 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/package-info.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/package-info.java @@ -47,4 +47,7 @@ * provides an optimizer interface. * */ +@NullMarked package org.apache.calcite.rel.rules; + +import org.jspecify.annotations.NullMarked; diff --git a/core/src/main/java/org/apache/calcite/rel/stream/package-info.java b/core/src/main/java/org/apache/calcite/rel/stream/package-info.java index abc9584fce51..2c438b1e68ba 100644 --- a/core/src/main/java/org/apache/calcite/rel/stream/package-info.java +++ b/core/src/main/java/org/apache/calcite/rel/stream/package-info.java @@ -27,4 +27,7 @@ * * */ +@NullMarked package org.apache.calcite.rel.stream; + +import org.jspecify.annotations.NullMarked; diff --git a/core/src/main/java/org/apache/calcite/rel/type/package-info.java b/core/src/main/java/org/apache/calcite/rel/type/package-info.java index 6da568d4edc9..8b85c98f7399 100644 --- a/core/src/main/java/org/apache/calcite/rel/type/package-info.java +++ b/core/src/main/java/org/apache/calcite/rel/type/package-info.java @@ -18,4 +18,7 @@ /** * Defines a type system for relational expressions. */ +@NullMarked package org.apache.calcite.rel.type; + +import org.jspecify.annotations.NullMarked; diff --git a/core/src/main/java/org/apache/calcite/rex/package-info.java b/core/src/main/java/org/apache/calcite/rex/package-info.java index be7479293d29..9b2257bd905a 100644 --- a/core/src/main/java/org/apache/calcite/rex/package-info.java +++ b/core/src/main/java/org/apache/calcite/rex/package-info.java @@ -78,11 +78,7 @@ * * */ -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.FIELD) -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.PARAMETER) -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.RETURN) +@NullMarked package org.apache.calcite.rex; -import org.checkerframework.framework.qual.DefaultQualifier; -import org.checkerframework.framework.qual.TypeUseLocation; -import org.jspecify.annotations.NonNull; +import org.jspecify.annotations.NullMarked; diff --git a/core/src/main/java/org/apache/calcite/runtime/package-info.java b/core/src/main/java/org/apache/calcite/runtime/package-info.java index b645be030955..ef7ccadfb0f3 100644 --- a/core/src/main/java/org/apache/calcite/runtime/package-info.java +++ b/core/src/main/java/org/apache/calcite/runtime/package-info.java @@ -18,11 +18,7 @@ /** * Utilities required at runtime. */ -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.FIELD) -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.PARAMETER) -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.RETURN) +@NullMarked package org.apache.calcite.runtime; -import org.checkerframework.framework.qual.DefaultQualifier; -import org.checkerframework.framework.qual.TypeUseLocation; -import org.jspecify.annotations.NonNull; +import org.jspecify.annotations.NullMarked; diff --git a/core/src/main/java/org/apache/calcite/runtime/rtti/package-info.java b/core/src/main/java/org/apache/calcite/runtime/rtti/package-info.java index 548b4ba62452..b15646d8d6f7 100644 --- a/core/src/main/java/org/apache/calcite/runtime/rtti/package-info.java +++ b/core/src/main/java/org/apache/calcite/runtime/rtti/package-info.java @@ -18,4 +18,7 @@ /** * Support for runtime type information. */ +@NullMarked package org.apache.calcite.runtime.rtti; + +import org.jspecify.annotations.NullMarked; diff --git a/core/src/main/java/org/apache/calcite/runtime/variant/package-info.java b/core/src/main/java/org/apache/calcite/runtime/variant/package-info.java index f8910457bf72..ccf09f4dd9e2 100644 --- a/core/src/main/java/org/apache/calcite/runtime/variant/package-info.java +++ b/core/src/main/java/org/apache/calcite/runtime/variant/package-info.java @@ -18,4 +18,7 @@ /** * Runtime support for values of the VARIANT data type. */ +@NullMarked package org.apache.calcite.runtime.variant; + +import org.jspecify.annotations.NullMarked; diff --git a/core/src/main/java/org/apache/calcite/schema/impl/package-info.java b/core/src/main/java/org/apache/calcite/schema/impl/package-info.java index d8bf0bd9e59c..a819e808a138 100644 --- a/core/src/main/java/org/apache/calcite/schema/impl/package-info.java +++ b/core/src/main/java/org/apache/calcite/schema/impl/package-info.java @@ -18,4 +18,7 @@ /** * Utilities to help implement Calcite's SPIs. */ +@NullMarked package org.apache.calcite.schema.impl; + +import org.jspecify.annotations.NullMarked; diff --git a/core/src/main/java/org/apache/calcite/schema/lookup/package-info.java b/core/src/main/java/org/apache/calcite/schema/lookup/package-info.java index fd3821cc6587..e440267e0572 100644 --- a/core/src/main/java/org/apache/calcite/schema/lookup/package-info.java +++ b/core/src/main/java/org/apache/calcite/schema/lookup/package-info.java @@ -21,11 +21,7 @@ *

      The interfaces and classes in this package are used to lookup * tables and subschemas within a schema. */ -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.FIELD) -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.PARAMETER) -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.RETURN) +@NullMarked package org.apache.calcite.schema.lookup; -import org.checkerframework.framework.qual.DefaultQualifier; -import org.checkerframework.framework.qual.TypeUseLocation; -import org.jspecify.annotations.NonNull; +import org.jspecify.annotations.NullMarked; diff --git a/core/src/main/java/org/apache/calcite/schema/package-info.java b/core/src/main/java/org/apache/calcite/schema/package-info.java index cda5149c6249..5d34c0cef87e 100644 --- a/core/src/main/java/org/apache/calcite/schema/package-info.java +++ b/core/src/main/java/org/apache/calcite/schema/package-info.java @@ -22,11 +22,7 @@ * SQL validator to validate SQL abstract syntax trees and resolve * identifiers to objects. */ -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.FIELD) -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.PARAMETER) -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.RETURN) +@NullMarked package org.apache.calcite.schema; -import org.checkerframework.framework.qual.DefaultQualifier; -import org.checkerframework.framework.qual.TypeUseLocation; -import org.jspecify.annotations.NonNull; +import org.jspecify.annotations.NullMarked; diff --git a/core/src/main/java/org/apache/calcite/server/package-info.java b/core/src/main/java/org/apache/calcite/server/package-info.java index 84cfd70fa2c3..3b89aa4e14aa 100644 --- a/core/src/main/java/org/apache/calcite/server/package-info.java +++ b/core/src/main/java/org/apache/calcite/server/package-info.java @@ -18,11 +18,7 @@ /** * Provides a server for hosting Calcite connections. */ -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.FIELD) -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.PARAMETER) -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.RETURN) +@NullMarked package org.apache.calcite.server; -import org.checkerframework.framework.qual.DefaultQualifier; -import org.checkerframework.framework.qual.TypeUseLocation; -import org.jspecify.annotations.NonNull; +import org.jspecify.annotations.NullMarked; diff --git a/core/src/main/java/org/apache/calcite/sql/advise/package-info.java b/core/src/main/java/org/apache/calcite/sql/advise/package-info.java index 2b2726e43053..8b8b0d86c482 100644 --- a/core/src/main/java/org/apache/calcite/sql/advise/package-info.java +++ b/core/src/main/java/org/apache/calcite/sql/advise/package-info.java @@ -24,4 +24,7 @@ *

      The advisor uses the validation and parser framework set up in * org.apache.calcite.sql.validate package. */ +@NullMarked package org.apache.calcite.sql.advise; + +import org.jspecify.annotations.NullMarked; diff --git a/core/src/main/java/org/apache/calcite/sql/ddl/package-info.java b/core/src/main/java/org/apache/calcite/sql/ddl/package-info.java index 669d4c9491c7..fefba9b62768 100644 --- a/core/src/main/java/org/apache/calcite/sql/ddl/package-info.java +++ b/core/src/main/java/org/apache/calcite/sql/ddl/package-info.java @@ -27,4 +27,7 @@ * the parser and its supporting classes into your own module, rather than try * to extend this one. */ +@NullMarked package org.apache.calcite.sql.ddl; + +import org.jspecify.annotations.NullMarked; diff --git a/core/src/main/java/org/apache/calcite/sql/dialect/package-info.java b/core/src/main/java/org/apache/calcite/sql/dialect/package-info.java index 058a471a914b..7b3504f7e566 100644 --- a/core/src/main/java/org/apache/calcite/sql/dialect/package-info.java +++ b/core/src/main/java/org/apache/calcite/sql/dialect/package-info.java @@ -18,4 +18,7 @@ /** * SQL unparsers for JDBC dialects. */ +@NullMarked package org.apache.calcite.sql.dialect; + +import org.jspecify.annotations.NullMarked; diff --git a/core/src/main/java/org/apache/calcite/sql/fun/package-info.java b/core/src/main/java/org/apache/calcite/sql/fun/package-info.java index 0bf59f39637c..9df53ce0cda0 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/package-info.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/package-info.java @@ -27,4 +27,7 @@ * which are not row-level (e.g. select and join) should be defined in package * {@link org.apache.calcite.sql} instead. */ +@NullMarked package org.apache.calcite.sql.fun; + +import org.jspecify.annotations.NullMarked; diff --git a/core/src/main/java/org/apache/calcite/sql/package-info.java b/core/src/main/java/org/apache/calcite/sql/package-info.java index c71b95dfb1ec..dd61d0df01bf 100644 --- a/core/src/main/java/org/apache/calcite/sql/package-info.java +++ b/core/src/main/java/org/apache/calcite/sql/package-info.java @@ -92,11 +92,7 @@ * {@link org.apache.calcite.sql.SqlNode}s into a SQL string. A * {@link org.apache.calcite.sql.SqlDialect} defines how this happens. */ -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.FIELD) -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.PARAMETER) -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.RETURN) +@NullMarked package org.apache.calcite.sql; -import org.checkerframework.framework.qual.DefaultQualifier; -import org.checkerframework.framework.qual.TypeUseLocation; -import org.jspecify.annotations.NonNull; +import org.jspecify.annotations.NullMarked; diff --git a/core/src/main/java/org/apache/calcite/sql/parser/impl/package-info.java b/core/src/main/java/org/apache/calcite/sql/parser/impl/package-info.java index ed141a67c449..4532b3ee50d2 100644 --- a/core/src/main/java/org/apache/calcite/sql/parser/impl/package-info.java +++ b/core/src/main/java/org/apache/calcite/sql/parser/impl/package-info.java @@ -19,4 +19,7 @@ * Contains generated code for the * {@link org.apache.calcite.sql.parser Calcite SQL parser}. */ +@NullMarked package org.apache.calcite.sql.parser.impl; + +import org.jspecify.annotations.NullMarked; diff --git a/core/src/main/java/org/apache/calcite/sql/parser/package-info.java b/core/src/main/java/org/apache/calcite/sql/parser/package-info.java index 047c2e8e238f..385148825f70 100644 --- a/core/src/main/java/org/apache/calcite/sql/parser/package-info.java +++ b/core/src/main/java/org/apache/calcite/sql/parser/package-info.java @@ -18,4 +18,7 @@ /** * Provides a SQL parser. */ +@NullMarked package org.apache.calcite.sql.parser; + +import org.jspecify.annotations.NullMarked; diff --git a/core/src/main/java/org/apache/calcite/sql/pretty/package-info.java b/core/src/main/java/org/apache/calcite/sql/pretty/package-info.java index b6bf65b03303..c56d9568a3b9 100644 --- a/core/src/main/java/org/apache/calcite/sql/pretty/package-info.java +++ b/core/src/main/java/org/apache/calcite/sql/pretty/package-info.java @@ -18,4 +18,7 @@ /** * Provides a pretty-printer for SQL statements. */ +@NullMarked package org.apache.calcite.sql.pretty; + +import org.jspecify.annotations.NullMarked; diff --git a/core/src/main/java/org/apache/calcite/sql/type/package-info.java b/core/src/main/java/org/apache/calcite/sql/type/package-info.java index f5b16a21e412..42937228f183 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/package-info.java +++ b/core/src/main/java/org/apache/calcite/sql/type/package-info.java @@ -18,4 +18,7 @@ /** * SQL type system. */ +@NullMarked package org.apache.calcite.sql.type; + +import org.jspecify.annotations.NullMarked; diff --git a/core/src/main/java/org/apache/calcite/sql/util/package-info.java b/core/src/main/java/org/apache/calcite/sql/util/package-info.java index 064661cad30f..167c88c5397c 100644 --- a/core/src/main/java/org/apache/calcite/sql/util/package-info.java +++ b/core/src/main/java/org/apache/calcite/sql/util/package-info.java @@ -18,4 +18,7 @@ /** * Utility classes for the SQL object model, parsing, and validation. */ +@NullMarked package org.apache.calcite.sql.util; + +import org.jspecify.annotations.NullMarked; diff --git a/core/src/main/java/org/apache/calcite/sql/validate/implicit/package-info.java b/core/src/main/java/org/apache/calcite/sql/validate/implicit/package-info.java index 7e7c87c37698..fa062cde05c9 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/implicit/package-info.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/implicit/package-info.java @@ -79,4 +79,7 @@ * *

      See CalciteImplicitCasts. */ +@NullMarked package org.apache.calcite.sql.validate.implicit; + +import org.jspecify.annotations.NullMarked; diff --git a/core/src/main/java/org/apache/calcite/sql/validate/package-info.java b/core/src/main/java/org/apache/calcite/sql/validate/package-info.java index 20d5f0dc2900..9066d21ac730 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/package-info.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/package-info.java @@ -18,4 +18,7 @@ /** * SQL validation. */ +@NullMarked package org.apache.calcite.sql.validate; + +import org.jspecify.annotations.NullMarked; diff --git a/core/src/main/java/org/apache/calcite/sql2rel/package-info.java b/core/src/main/java/org/apache/calcite/sql2rel/package-info.java index 93b9d97d4a22..dacbb9ca2c68 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/package-info.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/package-info.java @@ -18,11 +18,7 @@ /** * Translates a SQL parse tree to relational expression. */ -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.FIELD) -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.PARAMETER) -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.RETURN) +@NullMarked package org.apache.calcite.sql2rel; -import org.checkerframework.framework.qual.DefaultQualifier; -import org.checkerframework.framework.qual.TypeUseLocation; -import org.jspecify.annotations.NonNull; +import org.jspecify.annotations.NullMarked; diff --git a/core/src/main/java/org/apache/calcite/statistic/package-info.java b/core/src/main/java/org/apache/calcite/statistic/package-info.java index 67012cd687b7..2ebb222f3842 100644 --- a/core/src/main/java/org/apache/calcite/statistic/package-info.java +++ b/core/src/main/java/org/apache/calcite/statistic/package-info.java @@ -20,11 +20,7 @@ * * @see org.apache.calcite.materialize.SqlStatisticProvider */ -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.FIELD) -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.PARAMETER) -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.RETURN) +@NullMarked package org.apache.calcite.statistic; -import org.checkerframework.framework.qual.DefaultQualifier; -import org.checkerframework.framework.qual.TypeUseLocation; -import org.jspecify.annotations.NonNull; +import org.jspecify.annotations.NullMarked; diff --git a/core/src/main/java/org/apache/calcite/tools/package-info.java b/core/src/main/java/org/apache/calcite/tools/package-info.java index 5b37f6e36cfb..f6d43c3e63b7 100644 --- a/core/src/main/java/org/apache/calcite/tools/package-info.java +++ b/core/src/main/java/org/apache/calcite/tools/package-info.java @@ -18,11 +18,7 @@ /** * Provides utility classes. */ -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.FIELD) -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.PARAMETER) -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.RETURN) +@NullMarked package org.apache.calcite.tools; -import org.checkerframework.framework.qual.DefaultQualifier; -import org.checkerframework.framework.qual.TypeUseLocation; -import org.jspecify.annotations.NonNull; +import org.jspecify.annotations.NullMarked; diff --git a/core/src/main/java/org/apache/calcite/util/format/package-info.java b/core/src/main/java/org/apache/calcite/util/format/package-info.java index 88ed6b551c56..437f16a31eff 100644 --- a/core/src/main/java/org/apache/calcite/util/format/package-info.java +++ b/core/src/main/java/org/apache/calcite/util/format/package-info.java @@ -18,4 +18,7 @@ /** * Utility classes for handling format strings. */ +@NullMarked package org.apache.calcite.util.format; + +import org.jspecify.annotations.NullMarked; diff --git a/core/src/main/java/org/apache/calcite/util/format/postgresql/format/compiled/package-info.java b/core/src/main/java/org/apache/calcite/util/format/postgresql/format/compiled/package-info.java index c0b9a1518fc5..ca9ac33bbdcb 100644 --- a/core/src/main/java/org/apache/calcite/util/format/postgresql/format/compiled/package-info.java +++ b/core/src/main/java/org/apache/calcite/util/format/postgresql/format/compiled/package-info.java @@ -18,4 +18,7 @@ /** * Classes for that represent components of a parsed date/time format. */ +@NullMarked package org.apache.calcite.util.format.postgresql.format.compiled; + +import org.jspecify.annotations.NullMarked; diff --git a/core/src/main/java/org/apache/calcite/util/format/postgresql/format/package-info.java b/core/src/main/java/org/apache/calcite/util/format/postgresql/format/package-info.java index e6da6fdf895a..f46917e029e4 100644 --- a/core/src/main/java/org/apache/calcite/util/format/postgresql/format/package-info.java +++ b/core/src/main/java/org/apache/calcite/util/format/postgresql/format/package-info.java @@ -18,4 +18,7 @@ /** * Classes used to build up a list of supported date/time format components. */ +@NullMarked package org.apache.calcite.util.format.postgresql.format; + +import org.jspecify.annotations.NullMarked; diff --git a/core/src/main/java/org/apache/calcite/util/format/postgresql/package-info.java b/core/src/main/java/org/apache/calcite/util/format/postgresql/package-info.java index 0337e8d91b17..f987d64dcae8 100644 --- a/core/src/main/java/org/apache/calcite/util/format/postgresql/package-info.java +++ b/core/src/main/java/org/apache/calcite/util/format/postgresql/package-info.java @@ -18,4 +18,7 @@ /** * Classes for handling date/time format strings for PostgreSQL. */ +@NullMarked package org.apache.calcite.util.format.postgresql; + +import org.jspecify.annotations.NullMarked; diff --git a/core/src/main/java/org/apache/calcite/util/graph/package-info.java b/core/src/main/java/org/apache/calcite/util/graph/package-info.java index 3e272a795827..2d24a3a13f40 100644 --- a/core/src/main/java/org/apache/calcite/util/graph/package-info.java +++ b/core/src/main/java/org/apache/calcite/util/graph/package-info.java @@ -18,4 +18,7 @@ /** * Graph-theoretic algorithms and data structures. */ +@NullMarked package org.apache.calcite.util.graph; + +import org.jspecify.annotations.NullMarked; diff --git a/core/src/main/java/org/apache/calcite/util/javac/package-info.java b/core/src/main/java/org/apache/calcite/util/javac/package-info.java index 4fc956fdfe5e..93899b55ab50 100644 --- a/core/src/main/java/org/apache/calcite/util/javac/package-info.java +++ b/core/src/main/java/org/apache/calcite/util/javac/package-info.java @@ -18,4 +18,7 @@ /** * Provides compilers for Java code. */ +@NullMarked package org.apache.calcite.util.javac; + +import org.jspecify.annotations.NullMarked; diff --git a/core/src/main/java/org/apache/calcite/util/mapping/package-info.java b/core/src/main/java/org/apache/calcite/util/mapping/package-info.java index 81ea4ed0e80f..9537e61e2025 100644 --- a/core/src/main/java/org/apache/calcite/util/mapping/package-info.java +++ b/core/src/main/java/org/apache/calcite/util/mapping/package-info.java @@ -18,4 +18,7 @@ /** * Support for algebraic maps. */ +@NullMarked package org.apache.calcite.util.mapping; + +import org.jspecify.annotations.NullMarked; diff --git a/core/src/main/java/org/apache/calcite/util/package-info.java b/core/src/main/java/org/apache/calcite/util/package-info.java index 23eb05e65e2e..612efa27d8e1 100644 --- a/core/src/main/java/org/apache/calcite/util/package-info.java +++ b/core/src/main/java/org/apache/calcite/util/package-info.java @@ -18,11 +18,7 @@ /** * Provides utility classes. */ -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.FIELD) -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.PARAMETER) -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.RETURN) +@NullMarked package org.apache.calcite.util; -import org.checkerframework.framework.qual.DefaultQualifier; -import org.checkerframework.framework.qual.TypeUseLocation; -import org.jspecify.annotations.NonNull; +import org.jspecify.annotations.NullMarked; diff --git a/core/src/main/java/org/apache/calcite/util/trace/package-info.java b/core/src/main/java/org/apache/calcite/util/trace/package-info.java index 425046e2114b..353480d49bd5 100644 --- a/core/src/main/java/org/apache/calcite/util/trace/package-info.java +++ b/core/src/main/java/org/apache/calcite/util/trace/package-info.java @@ -18,4 +18,7 @@ /** * Tracing services. */ +@NullMarked package org.apache.calcite.util.trace; + +import org.jspecify.annotations.NullMarked; diff --git a/core/src/test/java/org/apache/calcite/test/LintTest.java b/core/src/test/java/org/apache/calcite/test/LintTest.java index 0714beb1a71b..97060ffbad33 100644 --- a/core/src/test/java/org/apache/calcite/test/LintTest.java +++ b/core/src/test/java/org/apache/calcite/test/LintTest.java @@ -31,8 +31,11 @@ import org.junit.jupiter.api.Test; import java.io.File; +import java.io.IOException; import java.io.PrintWriter; import java.io.StringWriter; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; import java.util.ArrayList; import java.util.Comparator; import java.util.HashSet; @@ -66,6 +69,12 @@ class LintTest { compile("^(\\[CALCITE-[0-9]{1,4}][ ]).*"); private static final Pattern PATTERN = compile("^ *(// )?"); + private static final String PACKAGE_INFO = "package-info.java"; + /** Source roots that NullAway verifies; see {@code nullawayProjects} in the root + * {@code build.gradle.kts}. */ + private static final List NULL_MARKED_ROOTS = + ImmutableList.of("linq4j/src/main/java/", "core/src/main/java/"); + private static final Pattern COMMONS_LANG3_IMPORT_PATTERN = compile("^\\s*import\\s+(static\\s+)?" + "org\\.apache\\.commons\\.lang3\\..*;\\s*$"); @@ -364,6 +373,65 @@ private static boolean isJava(String filename) { assertThat(g.messages, empty()); } + /** Fails when a main-source package is not declared {@code @NullMarked}. + * + *

      NullAway analyzes {@code @NullMarked} code only, so a package that forgets the + * annotation is silently skipped rather than reported. Add a {@code package-info.java} + * to the new package, copying the {@code @NullMarked} declaration from a sibling + * package. + * + *

      Only the main sources of the modules that NullAway verifies are checked. Marking a + * package that nobody verifies would claim a guarantee that nothing backs. Widen + * {@link #NULL_MARKED_ROOTS} together with {@code nullawayProjects}. + * + *

      A package that spans both modules needs only one {@code package-info.java}, in + * either of them, because a second one would put a duplicate class on the classpath. */ + @Test void testLintNullMarked() throws IOException { + assumeTrue(TestUnsafe.haveGit(), "Invalid git environment"); + + final List messages = new ArrayList<>(); + final Set mainPackages = new HashSet<>(); + final Set markedPackages = new HashSet<>(); + for (File file : TestUnsafe.getJavaFiles()) { + final String path = file.getPath().replace(File.separatorChar, '/'); + final String root = + NULL_MARKED_ROOTS.stream() + .filter(path::contains) + .findFirst() + .orElse(null); + if (root == null) { + continue; + } + final int i = path.indexOf(root) + root.length(); + final String packageName = + path.substring(i, path.lastIndexOf('/')).replace('/', '.'); + mainPackages.add(packageName); + if (file.getName().equals(PACKAGE_INFO)) { + if (isNullMarked(file)) { + markedPackages.add(packageName); + } else { + messages.add(file + ": " + PACKAGE_INFO + " is not annotated @NullMarked"); + } + } + } + mainPackages.stream() + .filter(packageName -> !markedPackages.contains(packageName)) + .map(packageName -> + packageName + ": package has no " + PACKAGE_INFO + " declaring @NullMarked") + .sorted() + .forEach(messages::add); + + messages.forEach(System.out::println); + assertThat(messages, empty()); + } + + private static boolean isNullMarked(File file) throws IOException { + try (Stream lines = + Files.lines(file.toPath(), StandardCharsets.UTF_8)) { + return lines.anyMatch(line -> line.startsWith("@NullMarked")); + } + } + /** Tests that the most recent N commit messages are good. * *

      N needs to be large enough to verify multi-commit PRs, but not so large diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/function/package-info.java b/linq4j/src/main/java/org/apache/calcite/linq4j/function/package-info.java index eff40384c80b..4736e21bb0d8 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/function/package-info.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/function/package-info.java @@ -18,4 +18,7 @@ /** * Contains definitions of functions and predicates. */ +@NullMarked package org.apache.calcite.linq4j.function; + +import org.jspecify.annotations.NullMarked; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/package-info.java b/linq4j/src/main/java/org/apache/calcite/linq4j/package-info.java index 6a13f8f258d1..7c62e2bc2468 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/package-info.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/package-info.java @@ -18,11 +18,7 @@ /** * Language-integrated query for Java (linq4j) main package. */ -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.FIELD) -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.PARAMETER) -@DefaultQualifier(value = NonNull.class, locations = TypeUseLocation.RETURN) +@NullMarked package org.apache.calcite.linq4j; -import org.checkerframework.framework.qual.DefaultQualifier; -import org.checkerframework.framework.qual.TypeUseLocation; -import org.jspecify.annotations.NonNull; +import org.jspecify.annotations.NullMarked; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/package-info.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/package-info.java index 9c1eeda04dc5..6d1eb6fa9d6d 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/package-info.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/package-info.java @@ -25,4 +25,7 @@ * efficiency; for example, it may attempt to push down filters to the * source SQL system. */ +@NullMarked package org.apache.calcite.linq4j.tree; + +import org.jspecify.annotations.NullMarked; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/util/package-info.java b/linq4j/src/main/java/org/apache/calcite/linq4j/util/package-info.java index d1f427e9adcb..7636d26ab5aa 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/util/package-info.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/util/package-info.java @@ -18,4 +18,7 @@ /** * Provides utility classes. */ +@NullMarked package org.apache.calcite.linq4j.util; + +import org.jspecify.annotations.NullMarked; From 8a142d3145fdb49a616a535250de2e74b0d6b24b Mon Sep 17 00:00:00 2001 From: Vladimir Sitnikov Date: Sun, 23 Aug 2026 22:11:06 +0300 Subject: [PATCH 497/562] [CALCITE-7736] Migrate the Checker Framework annotations that JSpecify does not define JSpecify defines @Nullable, @NonNull and @NullMarked, and nothing else. The remaining Checker Framework annotations either move to a Calcite-owned equivalent or go away. Adds `org.apache.calcite.linq4j.annotations` with @Contract, @MonotonicNonNull, @RequiresNonNull, @EnsuresNonNull and @EnsuresNonNullIf. NullAway matches these by the last component of their name rather than by their package, so Calcite declares its own and takes no dependency on the checker: * `ContractUtils.hasSimpleNameContract` compares the simple name * `Nullness.isMonotonicNonNullAnnotation` tests `endsWith(".MonotonicNonNull")` * the field-contract handlers pass `exactMatch=false` to `NullabilityUtil.findAnnotation`, which also compares the suffix Moved to that package: @MonotonicNonNull (32), @RequiresNonNull (21), @EnsuresNonNull (23) and @EnsuresNonNullIf (14). The two @EnsuresNonNull that named a parameter rather than a field are dropped, along with the four @EnsuresNonNullIf whose expression was a method call: NullAway supports fields only. @PolyNull (300) becomes @Nullable. It says the result is null exactly when the argument is null, which @Nullable weakens to "may be null"; the next commit restores the other half with @Contract. Dropped, having no equivalent and no NullAway counterpart: @Pure (81), the initialization annotations (80), @KeyFor and @UnknownKeyFor (18), @Covariant (10), @HasQualifierParameter and @MinLen. NullAway checks initialization on its own, so the receiver parameters that existed only to carry @UnderInitialization are removed with them. `<@Nullable R>` becomes ``. The Checker Framework reads an annotation on a type parameter declaration as a bound on the lower bound, so `<@Nullable R>` there means R *must* be nullable; JSpecify tracks upper bounds only and can say no more than "may be". 215 @SuppressWarnings that named Checker Framework message keys such as `argument.type.incompatible` become "NullAway". `Nullness.castNonNull` keeps working as NullAway's `CastToNonNullMethod`; it loses @Pure and its parameter-naming @EnsuresNonNull, and its blanket suppression narrows to the one method that needs it. Co-Authored-By: Claude Opus 5 --- .../sql/babel/SqlBabelCreateTable.java | 2 +- .../calcite/adapter/clone/ArrayTable.java | 2 +- .../calcite/adapter/clone/ColumnLoader.java | 6 +- .../adapter/enumerable/AggImpState.java | 3 +- .../enumerable/EnumerableRelImplementor.java | 2 +- .../adapter/enumerable/EnumerableWindow.java | 2 +- .../enumerable/NestedBlockBuilderImpl.java | 2 +- .../adapter/enumerable/RexImpTable.java | 2 +- .../adapter/java/ReflectiveSchema.java | 2 +- .../adapter/jdbc/JdbcCatalogSchema.java | 4 +- .../calcite/adapter/jdbc/JdbcSchema.java | 2 +- .../calcite/adapter/jdbc/JdbcTable.java | 4 +- .../config/CalciteConnectionConfig.java | 25 ++- .../config/CalciteConnectionConfigImpl.java | 25 ++- .../calcite/interpreter/AggregateNode.java | 2 +- .../apache/calcite/interpreter/Context.java | 4 +- .../calcite/interpreter/Interpreter.java | 7 +- .../org/apache/calcite/interpreter/Nodes.java | 4 +- .../calcite/jdbc/CachingCalciteSchema.java | 2 +- .../apache/calcite/jdbc/CalciteSchema.java | 2 +- .../apache/calcite/jdbc/JavaCollation.java | 2 - .../apache/calcite/materialize/Lattice.java | 8 +- .../calcite/materialize/LatticeNode.java | 5 +- .../calcite/materialize/LatticeRootNode.java | 2 +- .../calcite/materialize/LatticeSpace.java | 6 +- .../calcite/materialize/LatticeSuggester.java | 2 +- .../calcite/materialize/MutableNode.java | 2 +- .../org/apache/calcite/materialize/Step.java | 8 +- .../apache/calcite/model/ModelHandler.java | 2 +- .../calcite/plan/AbstractRelOptPlanner.java | 6 +- .../org/apache/calcite/plan/Contexts.java | 6 +- .../calcite/plan/ConventionTraitDef.java | 4 +- .../calcite/plan/RelOptAbstractTable.java | 2 +- .../apache/calcite/plan/RelOptCluster.java | 5 +- .../org/apache/calcite/plan/RelOptRule.java | 3 - .../calcite/plan/RelOptRuleOperand.java | 15 +- .../org/apache/calcite/plan/RelOptUtil.java | 12 +- .../java/org/apache/calcite/plan/RelRule.java | 2 +- .../org/apache/calcite/plan/RelTraitSet.java | 3 +- .../calcite/plan/RexImplicationChecker.java | 5 +- .../calcite/plan/SubstitutionVisitor.java | 2 +- .../apache/calcite/plan/TableAccessMap.java | 2 +- .../calcite/plan/hep/HepInstruction.java | 2 +- .../plan/hep/HepRelMetadataProvider.java | 2 +- .../apache/calcite/plan/volcano/RelSet.java | 2 +- .../calcite/plan/volcano/RelSubset.java | 6 +- .../calcite/plan/volcano/VolcanoPlanner.java | 13 +- .../volcano/VolcanoRelMetadataProvider.java | 2 +- .../plan/volcano/VolcanoRuleMatch.java | 2 +- .../calcite/prepare/CalciteCatalogReader.java | 2 +- .../apache/calcite/prepare/PlannerImpl.java | 4 +- .../org/apache/calcite/prepare/Prepare.java | 2 +- .../calcite/prepare/QueryableRelBuilder.java | 5 +- .../calcite/prepare/RelOptTableImpl.java | 6 +- .../calcite/profile/SimpleProfiler.java | 4 +- .../apache/calcite/rel/AbstractRelNode.java | 10 +- .../java/org/apache/calcite/rel/RelNode.java | 6 +- .../apache/calcite/rel/core/Aggregate.java | 2 +- .../org/apache/calcite/rel/core/Calc.java | 2 +- .../org/apache/calcite/rel/core/Collect.java | 4 +- .../apache/calcite/rel/core/Correlate.java | 2 +- .../org/apache/calcite/rel/core/Filter.java | 4 +- .../org/apache/calcite/rel/core/Join.java | 2 - .../org/apache/calcite/rel/core/Project.java | 4 +- .../org/apache/calcite/rel/core/Snapshot.java | 4 +- .../apache/calcite/rel/core/TableModify.java | 2 +- .../apache/calcite/rel/core/Uncollect.java | 2 +- .../org/apache/calcite/rel/core/Values.java | 4 +- .../org/apache/calcite/rel/core/Window.java | 5 +- .../calcite/rel/externalize/RelJson.java | 5 +- .../metadata/CachingRelMetadataProvider.java | 2 +- .../metadata/ChainedRelMetadataProvider.java | 4 +- .../metadata/JaninoRelMetadataProvider.java | 2 +- .../calcite/rel/metadata/MetadataFactory.java | 2 +- .../rel/metadata/MetadataFactoryImpl.java | 2 +- .../ReflectiveRelMetadataProvider.java | 4 +- .../rel/metadata/RelMdColumnOrigins.java | 5 +- .../rel/metadata/RelMdExpressionLineage.java | 3 +- .../metadata/RelMdPercentageOriginalRows.java | 7 +- .../calcite/rel/metadata/RelMdUtil.java | 13 +- .../rel/metadata/RelMetadataProvider.java | 2 +- .../calcite/rel/mutable/MutableBiRel.java | 2 +- .../calcite/rel/mutable/MutableMultiRel.java | 2 +- .../calcite/rel/mutable/MutableSingleRel.java | 2 +- .../rel/rel2sql/RelToSqlConverter.java | 2 +- .../calcite/rel/rel2sql/SqlImplementor.java | 10 +- ...AggregateExpandDistinctAggregatesRule.java | 2 +- .../calcite/rel/rules/CalcRelSplitter.java | 2 +- .../calcite/rel/rules/LoptJoinTree.java | 13 +- .../calcite/rel/rules/LoptMultiJoin.java | 16 +- .../rel/rules/LoptOptimizeJoinRule.java | 7 +- .../calcite/rel/rules/ReduceDecimalsRule.java | 2 +- .../rel/type/DynamicRecordTypeImpl.java | 2 +- .../apache/calcite/rel/type/RelCrossType.java | 2 +- .../apache/calcite/rel/type/RelDataType.java | 8 - .../rel/type/RelDataTypeFactoryImpl.java | 2 +- .../calcite/rel/type/RelDataTypeField.java | 4 +- .../calcite/rel/type/RelDataTypeImpl.java | 5 +- .../apache/calcite/rex/RexBiVisitorImpl.java | 2 +- .../org/apache/calcite/rex/RexBuilder.java | 5 +- .../java/org/apache/calcite/rex/RexCall.java | 16 +- .../org/apache/calcite/rex/RexLiteral.java | 14 +- .../java/org/apache/calcite/rex/RexNode.java | 2 +- .../org/apache/calcite/rex/RexProgram.java | 8 +- .../apache/calcite/rex/RexProgramBuilder.java | 4 +- .../org/apache/calcite/rex/RexShuttle.java | 5 +- .../org/apache/calcite/rex/RexSimplify.java | 2 +- .../rex/RexSqlStandardConvertletTable.java | 2 +- .../apache/calcite/rex/RexUnaryBiVisitor.java | 2 +- .../apache/calcite/rex/RexVisitorImpl.java | 2 +- .../org/apache/calcite/rex/RexWindow.java | 2 +- .../apache/calcite/rex/RexWindowBound.java | 11 +- .../calcite/runtime/AutomatonBuilder.java | 4 +- .../runtime/CalciteContextException.java | 2 - .../calcite/runtime/CalciteException.java | 2 +- .../org/apache/calcite/runtime/ConsList.java | 3 +- .../runtime/DeterministicAutomaton.java | 2 +- .../org/apache/calcite/runtime/FlatLists.java | 19 +- .../calcite/runtime/ImmutablePairList.java | 4 +- .../apache/calcite/runtime/JsonFunctions.java | 6 +- .../org/apache/calcite/runtime/PairList.java | 5 +- .../calcite/runtime/RandomFunction.java | 3 +- .../org/apache/calcite/runtime/Resources.java | 22 +-- .../calcite/runtime/SpaceFillingCurve2D.java | 2 +- .../apache/calcite/runtime/SqlFunctions.java | 180 +++++++++--------- .../org/apache/calcite/schema/SchemaPlus.java | 2 +- .../org/apache/calcite/schema/Wrapper.java | 6 +- .../calcite/schema/impl/AbstractTable.java | 2 +- .../schema/impl/ModifiableViewTable.java | 2 +- .../apache/calcite/schema/impl/StarTable.java | 2 +- .../calcite/server/DdlExecutorImpl.java | 2 +- .../apache/calcite/sql/SqlAggFunction.java | 2 +- .../org/apache/calcite/sql/SqlAsofJoin.java | 6 +- .../org/apache/calcite/sql/SqlBasicCall.java | 2 +- .../java/org/apache/calcite/sql/SqlCall.java | 3 - .../apache/calcite/sql/SqlCallBinding.java | 4 +- .../org/apache/calcite/sql/SqlCollation.java | 4 - .../org/apache/calcite/sql/SqlDelete.java | 6 +- .../apache/calcite/sql/SqlDescribeSchema.java | 4 +- .../apache/calcite/sql/SqlDescribeTable.java | 6 +- .../org/apache/calcite/sql/SqlDialect.java | 4 +- .../org/apache/calcite/sql/SqlExplain.java | 12 +- .../org/apache/calcite/sql/SqlFunction.java | 2 - .../org/apache/calcite/sql/SqlIdentifier.java | 2 - .../org/apache/calcite/sql/SqlInsert.java | 8 +- .../calcite/sql/SqlJdbcFunctionCall.java | 2 +- .../java/org/apache/calcite/sql/SqlJoin.java | 6 +- .../org/apache/calcite/sql/SqlLiteral.java | 2 +- .../apache/calcite/sql/SqlMatchRecognize.java | 6 +- .../java/org/apache/calcite/sql/SqlMerge.java | 6 +- .../org/apache/calcite/sql/SqlNodeList.java | 14 +- .../apache/calcite/sql/SqlNumericLiteral.java | 2 - .../org/apache/calcite/sql/SqlOperator.java | 4 - .../calcite/sql/SqlOperatorBinding.java | 2 +- .../org/apache/calcite/sql/SqlOrderBy.java | 4 +- .../java/org/apache/calcite/sql/SqlPivot.java | 2 +- .../org/apache/calcite/sql/SqlSelect.java | 17 +- .../org/apache/calcite/sql/SqlSetOption.java | 4 +- .../org/apache/calcite/sql/SqlSnapshot.java | 2 +- .../apache/calcite/sql/SqlStarExclude.java | 2 +- .../apache/calcite/sql/SqlStarReplace.java | 2 +- .../org/apache/calcite/sql/SqlSyntax.java | 2 - .../org/apache/calcite/sql/SqlUnpivot.java | 2 +- .../org/apache/calcite/sql/SqlUpdate.java | 8 +- .../java/org/apache/calcite/sql/SqlUtil.java | 5 +- .../org/apache/calcite/sql/SqlWindow.java | 12 +- .../java/org/apache/calcite/sql/SqlWith.java | 4 +- .../org/apache/calcite/sql/SqlWithItem.java | 6 +- .../org/apache/calcite/sql/SqlWriter.java | 16 -- .../apache/calcite/sql/advise/SqlAdvisor.java | 2 +- .../sql/ddl/SqlAttributeDefinition.java | 2 +- .../calcite/sql/ddl/SqlCheckConstraint.java | 2 +- .../calcite/sql/ddl/SqlColumnDeclaration.java | 2 +- .../sql/ddl/SqlCreateForeignSchema.java | 2 +- .../sql/ddl/SqlCreateMaterializedView.java | 2 +- .../calcite/sql/ddl/SqlCreateTable.java | 2 +- .../apache/calcite/sql/ddl/SqlCreateType.java | 2 +- .../apache/calcite/sql/ddl/SqlCreateView.java | 2 +- .../calcite/sql/ddl/SqlKeyConstraint.java | 2 +- .../sql/fun/SqlAnyValueAggFunction.java | 2 +- .../calcite/sql/fun/SqlBasicAggFunction.java | 2 +- .../calcite/sql/fun/SqlBitOpAggFunction.java | 2 +- .../org/apache/calcite/sql/fun/SqlCase.java | 4 +- .../calcite/sql/fun/SqlCaseOperator.java | 2 +- .../calcite/sql/fun/SqlCountAggFunction.java | 2 +- .../calcite/sql/fun/SqlGroupingFunction.java | 2 +- .../calcite/sql/fun/SqlInternalOperators.java | 2 +- .../fun/SqlLibraryOperatorTableFactory.java | 2 +- .../calcite/sql/fun/SqlLibraryOperators.java | 6 +- .../sql/fun/SqlMapValueConstructor.java | 2 +- .../calcite/sql/fun/SqlMinMaxAggFunction.java | 2 +- .../sql/fun/SqlSingleValueAggFunction.java | 2 +- .../calcite/sql/fun/SqlSumAggFunction.java | 2 +- .../sql/fun/SqlSumEmptyIsZeroAggFunction.java | 2 +- .../sql/parser/SqlAbstractParserImpl.java | 8 +- .../calcite/sql/pretty/SqlPrettyWriter.java | 2 +- .../sql/type/CompositeOperandTypeChecker.java | 5 +- .../apache/calcite/sql/type/OperandTypes.java | 2 +- .../apache/calcite/sql/type/SqlTypeUtil.java | 8 +- .../TableFunctionReturnTypeInference.java | 4 +- .../calcite/sql/util/SqlBasicVisitor.java | 6 +- .../apache/calcite/sql/util/SqlString.java | 2 - .../calcite/sql/validate/DelegatingScope.java | 2 +- .../DelegatingSqlValidatorCatalogReader.java | 2 +- .../sql/validate/IdentifierNamespace.java | 2 +- .../calcite/sql/validate/SelectScope.java | 2 +- .../sql/validate/SqlNonNullableAccessors.java | 2 +- .../calcite/sql/validate/SqlValidator.java | 4 - .../sql/validate/SqlValidatorException.java | 2 +- .../sql/validate/SqlValidatorImpl.java | 23 +-- .../sql/validate/SqlValidatorNamespace.java | 2 - .../sql/validate/SqlValidatorUtil.java | 5 +- .../sql2rel/CorrelationReferenceFinder.java | 7 +- .../DeduplicateCorrelateVariables.java | 7 +- .../sql2rel/ReflectiveConvertletTable.java | 11 +- .../calcite/sql2rel/RelDecorrelator.java | 2 +- .../calcite/sql2rel/RelFieldTrimmer.java | 2 +- .../sql2rel/RelStructuredTypeFlattener.java | 3 +- .../calcite/sql2rel/SqlToRelConverter.java | 4 +- .../sql2rel/StandardConvertletTable.java | 11 -- .../sql2rel/TopDownGeneralDecorrelator.java | 4 +- .../java/org/apache/calcite/util/BitSets.java | 2 +- .../org/apache/calcite/util/BlackholeMap.java | 4 +- .../org/apache/calcite/util/ChunkList.java | 2 +- .../org/apache/calcite/util/CompositeMap.java | 11 +- .../org/apache/calcite/util/DateString.java | 2 +- .../org/apache/calcite/util/Filterator.java | 2 +- .../apache/calcite/util/ImmutableBitSet.java | 6 +- .../apache/calcite/util/ImmutableIntList.java | 2 +- .../calcite/util/ImmutableNullableSet.java | 2 +- .../java/org/apache/calcite/util/NameSet.java | 6 +- .../org/apache/calcite/util/NlsString.java | 5 - .../org/apache/calcite/util/NumberUtil.java | 15 +- .../java/org/apache/calcite/util/Pair.java | 5 +- .../calcite/util/PartiallyOrderedSet.java | 2 +- .../org/apache/calcite/util/Permutation.java | 7 +- .../org/apache/calcite/util/RangeSets.java | 4 +- .../org/apache/calcite/util/ReflectUtil.java | 4 +- .../util/ReflectiveVisitDispatcher.java | 2 +- .../java/org/apache/calcite/util/Sarg.java | 2 +- .../calcite/util/SerializableCharset.java | 4 +- .../calcite/util/SimpleNamespaceContext.java | 2 +- .../org/apache/calcite/util/TimeString.java | 2 +- .../java/org/apache/calcite/util/Util.java | 27 +-- .../org/apache/calcite/util/XmlOutput.java | 2 +- .../util/graph/AttributedDirectedGraph.java | 3 +- .../util/graph/DefaultDirectedGraph.java | 12 +- .../calcite/util/graph/DefaultEdge.java | 6 +- .../org/apache/calcite/util/graph/Graphs.java | 2 +- .../util/graph/TopologicalOrderIterator.java | 5 +- .../util/mapping/AbstractSourceMapping.java | 2 +- .../util/mapping/AbstractTargetMapping.java | 2 +- .../apache/calcite/util/mapping/Mappings.java | 6 +- .../org/apache/calcite/test/JdbcTest.java | 2 +- .../adapter/csv/CsvProjectTableScanRule.java | 2 +- .../calcite/linq4j/DefaultEnumerable.java | 7 +- .../org/apache/calcite/linq4j/Enumerable.java | 3 - .../calcite/linq4j/EnumerableDefaults.java | 48 ++--- .../org/apache/calcite/linq4j/Enumerator.java | 3 - .../calcite/linq4j/ExtendedEnumerable.java | 9 +- .../calcite/linq4j/ExtendedQueryable.java | 2 - .../org/apache/calcite/linq4j/Grouping.java | 3 - .../apache/calcite/linq4j/GroupingImpl.java | 4 +- .../org/apache/calcite/linq4j/Linq4j.java | 5 +- .../org/apache/calcite/linq4j/LookupImpl.java | 15 +- .../calcite/linq4j/MemoryEnumerator.java | 3 +- .../calcite/linq4j/MergeUnionEnumerator.java | 11 +- .../org/apache/calcite/linq4j/Nullness.java | 40 ++-- .../org/apache/calcite/linq4j/Queryable.java | 6 +- .../calcite/linq4j/QueryableFactory.java | 5 +- .../calcite/linq4j/QueryableRecorder.java | 9 +- .../apache/calcite/linq4j/RawEnumerable.java | 3 - .../apache/calcite/linq4j/RawQueryable.java | 2 - .../calcite/linq4j/annotations/Contract.java | 51 +++++ .../linq4j/annotations/EnsuresNonNull.java | 39 ++++ .../linq4j/annotations/EnsuresNonNullIf.java | 50 +++++ .../linq4j/annotations/MonotonicNonNull.java | 36 ++++ .../linq4j/annotations/RequiresNonNull.java | 40 ++++ .../linq4j/annotations/package-info.java | 34 ++++ .../calcite/linq4j/function/Function1.java | 3 +- .../calcite/linq4j/function/Function2.java | 3 +- .../calcite/linq4j/function/Functions.java | 10 +- .../calcite/linq4j/tree/BlockBuilder.java | 3 +- .../calcite/linq4j/tree/BlockStatement.java | 2 - .../linq4j/tree/ConstantExpression.java | 2 +- .../calcite/linq4j/tree/Expressions.java | 4 +- .../apache/calcite/linq4j/tree/Primitive.java | 2 +- .../org/apache/calcite/linq4j/tree/Types.java | 2 +- .../calcite/linq4j/tree/VisitorImpl.java | 2 +- .../calcite/server/MaterializedViewTable.java | 2 +- .../calcite/server/MutableArrayTable.java | 2 +- 291 files changed, 865 insertions(+), 945 deletions(-) create mode 100644 linq4j/src/main/java/org/apache/calcite/linq4j/annotations/Contract.java create mode 100644 linq4j/src/main/java/org/apache/calcite/linq4j/annotations/EnsuresNonNull.java create mode 100644 linq4j/src/main/java/org/apache/calcite/linq4j/annotations/EnsuresNonNullIf.java create mode 100644 linq4j/src/main/java/org/apache/calcite/linq4j/annotations/MonotonicNonNull.java create mode 100644 linq4j/src/main/java/org/apache/calcite/linq4j/annotations/RequiresNonNull.java create mode 100644 linq4j/src/main/java/org/apache/calcite/linq4j/annotations/package-info.java diff --git a/babel/src/main/java/org/apache/calcite/sql/babel/SqlBabelCreateTable.java b/babel/src/main/java/org/apache/calcite/sql/babel/SqlBabelCreateTable.java index 3a05fd9ed295..5a4653c2fd83 100644 --- a/babel/src/main/java/org/apache/calcite/sql/babel/SqlBabelCreateTable.java +++ b/babel/src/main/java/org/apache/calcite/sql/babel/SqlBabelCreateTable.java @@ -65,7 +65,7 @@ public SqlBabelCreateTable(SqlParserPos pos, boolean replace, this.volatile_ = volatile_; } - @SuppressWarnings("nullness") + @SuppressWarnings("NullAway") @Override public List getOperandList() { return ImmutableNullableList.of(SqlLiteral.createBoolean(getReplace(), pos), SqlLiteral.createSymbol(tableCollectionType, pos), diff --git a/core/src/main/java/org/apache/calcite/adapter/clone/ArrayTable.java b/core/src/main/java/org/apache/calcite/adapter/clone/ArrayTable.java index cfda64687a44..a6e5820787ab 100644 --- a/core/src/main/java/org/apache/calcite/adapter/clone/ArrayTable.java +++ b/core/src/main/java/org/apache/calcite/adapter/clone/ArrayTable.java @@ -433,7 +433,7 @@ public static class ObjectDictionary implements Representation { valueSet.map.keySet().toArray(new Comparable[n + extra]); // codeValues[0..n] is non-null since valueSet.map.keySet is non-null // There might be null at the very end, however, it won't participate in Arrays.sort - @SuppressWarnings("assignment.type.incompatible") + @SuppressWarnings("NullAway") Comparable[] nonNullCodeValues = codeValues; Arrays.sort(nonNullCodeValues, 0, n); ColumnLoader.ValueSet codeValueSet = diff --git a/core/src/main/java/org/apache/calcite/adapter/clone/ColumnLoader.java b/core/src/main/java/org/apache/calcite/adapter/clone/ColumnLoader.java index 8e7af9cec577..6cf1c3489d25 100644 --- a/core/src/main/java/org/apache/calcite/adapter/clone/ColumnLoader.java +++ b/core/src/main/java/org/apache/calcite/adapter/clone/ColumnLoader.java @@ -27,7 +27,6 @@ import org.apache.calcite.rel.type.RelProtoDataType; import org.apache.calcite.util.Util; -import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf; import org.jspecify.annotations.Nullable; import java.lang.reflect.Type; @@ -67,7 +66,7 @@ class ColumnLoader { * @param sourceTable Source data * @param protoRowType Logical row type * @param repList Physical row types, or null if not known */ - @SuppressWarnings("method.invocation.invalid") + @SuppressWarnings("NullAway") ColumnLoader(JavaTypeFactory typeFactory, Enumerable sourceTable, RelProtoDataType protoRowType, @@ -202,7 +201,7 @@ private void load(final RelDataType elementType, // We have discovered a the first unique key in the table. sort[0] = pair.i; // map.keySet().size() == list.size() above implies list contains only non-null elements - @SuppressWarnings("assignment.type.incompatible") + @SuppressWarnings("NullAway") final Comparable[] values = valueSet.values.toArray(new Comparable[0]); final Kev[] kevs = new Kev[list.size()]; @@ -385,7 +384,6 @@ private static long toLong(Object o) { } } - @EnsuresNonNullIf(result = true, expression = "#1") private static boolean canBeLong(@Nullable Object o) { return o instanceof Boolean || o instanceof Character diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/AggImpState.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/AggImpState.java index a22583778871..a4c983712231 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/AggImpState.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/AggImpState.java @@ -16,11 +16,10 @@ */ package org.apache.calcite.adapter.enumerable; +import org.apache.calcite.linq4j.annotations.MonotonicNonNull; import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.rel.core.AggregateCall; -import org.checkerframework.checker.nullness.qual.MonotonicNonNull; - import java.util.List; /** diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableRelImplementor.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableRelImplementor.java index ae380578c42b..3c764b42b5ef 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableRelImplementor.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableRelImplementor.java @@ -89,7 +89,7 @@ public class EnumerableRelImplementor extends JavaRelImplementor { private final Map, ParameterExpression> stashedParameters = new LinkedHashMap<>(); - @SuppressWarnings("methodref.receiver.bound.invalid") + @SuppressWarnings("NullAway") protected final Function1 allCorrelateVariables = this::getCorrelVariableGetter; diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java index b07ccd55b426..5a0ffc62afc3 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java @@ -128,7 +128,7 @@ private WindowRelInputGetter(Expression row, } } - @SuppressWarnings({"unused", "nullness"}) + @SuppressWarnings({"unused", "NullAway"}) private static void sampleOfTheGeneratedWindowedAggregate() { // Here's overview of the generated code // For each list of rows that have the same partitioning key, evaluate diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/NestedBlockBuilderImpl.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/NestedBlockBuilderImpl.java index 9b6bbd41614e..d5cdbecc594e 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/NestedBlockBuilderImpl.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/NestedBlockBuilderImpl.java @@ -34,7 +34,7 @@ public class NestedBlockBuilderImpl implements NestedBlockBuilder { * * @param block root code block */ - @SuppressWarnings("method.invocation.invalid") + @SuppressWarnings("NullAway") public NestedBlockBuilderImpl(BlockBuilder block) { nestBlock(block); } diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java index 65997cdaebde..d38b72f45a4e 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java @@ -2406,7 +2406,7 @@ static class RankImplementor extends StrictWinAggImplementor { int curentPosition; // position in for-win-agg-loop int startIndex; // index of start of window Comparable @Nullable [] rows; // accessed via WinAggAddContext.compareRows - @SuppressWarnings("nullness") + @SuppressWarnings("NullAway") void sample() { if (curentPosition > startIndex) { if (rows[curentPosition - 1].compareTo(rows[curentPosition]) diff --git a/core/src/main/java/org/apache/calcite/adapter/java/ReflectiveSchema.java b/core/src/main/java/org/apache/calcite/adapter/java/ReflectiveSchema.java index 190647b25a97..e551aa473e80 100644 --- a/core/src/main/java/org/apache/calcite/adapter/java/ReflectiveSchema.java +++ b/core/src/main/java/org/apache/calcite/adapter/java/ReflectiveSchema.java @@ -23,6 +23,7 @@ import org.apache.calcite.linq4j.Linq4j; import org.apache.calcite.linq4j.QueryProvider; import org.apache.calcite.linq4j.Queryable; +import org.apache.calcite.linq4j.annotations.MonotonicNonNull; import org.apache.calcite.linq4j.function.Function1; import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.linq4j.tree.Expressions; @@ -53,7 +54,6 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Multimap; -import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.jspecify.annotations.Nullable; import java.lang.annotation.ElementType; diff --git a/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcCatalogSchema.java b/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcCatalogSchema.java index c276e67145bb..64797f41f20a 100644 --- a/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcCatalogSchema.java +++ b/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcCatalogSchema.java @@ -68,7 +68,7 @@ public class JdbcCatalogSchema extends JdbcBaseSchema implements Wrapper { private final Lookup subSchemas; /** default schema name, lazily initialized. */ - @SuppressWarnings({"method.invocation.invalid", "Convert2MethodRef"}) + @SuppressWarnings({"NullAway", "Convert2MethodRef"}) private final Supplier> defaultSchemaName = Suppliers.memoize(() -> Optional.ofNullable(computeDefaultSchemaName())); @@ -170,7 +170,7 @@ public DataSource getDataSource() { } - @Override public @Nullable T unwrap(Class clazz) { + @Override public @Nullable T unwrap(Class clazz) { if (clazz.isInstance(this)) { return clazz.cast(this); } diff --git a/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcSchema.java b/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcSchema.java index 533741b00d6c..f88d5e1da465 100644 --- a/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcSchema.java +++ b/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcSchema.java @@ -499,7 +499,7 @@ private static RelDataType parseTypeString(RelDataTypeFactory typeFactory, } } - @Override public @Nullable T unwrap(Class clazz) { + @Override public @Nullable T unwrap(Class clazz) { if (clazz.isInstance(this)) { return clazz.cast(this); } diff --git a/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcTable.java b/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcTable.java index 0b3bff721ed2..13fffac750c0 100644 --- a/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcTable.java +++ b/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcTable.java @@ -78,7 +78,7 @@ */ public class JdbcTable extends AbstractQueryableTable implements TranslatableTable, ScannableTable, ModifiableTable { - @SuppressWarnings("methodref.receiver.bound.invalid") + @SuppressWarnings("NullAway") private final Supplier protoRowTypeSupplier = Suppliers.memoize(this::supplyProto); public final JdbcSchema jdbcSchema; @@ -106,7 +106,7 @@ public class JdbcTable extends AbstractQueryableTable return jdbcTableType; } - @Override public @Nullable C unwrap(Class aClass) { + @Override public @Nullable C unwrap(Class aClass) { if (aClass.isInstance(jdbcSchema.getDataSource())) { return aClass.cast(jdbcSchema.getDataSource()); } else if (aClass.isInstance(jdbcSchema.dialect)) { diff --git a/core/src/main/java/org/apache/calcite/config/CalciteConnectionConfig.java b/core/src/main/java/org/apache/calcite/config/CalciteConnectionConfig.java index 4f3770386554..84d52b62b7e2 100644 --- a/core/src/main/java/org/apache/calcite/config/CalciteConnectionConfig.java +++ b/core/src/main/java/org/apache/calcite/config/CalciteConnectionConfig.java @@ -22,7 +22,6 @@ import org.apache.calcite.model.JsonSchema; import org.apache.calcite.sql.validate.SqlConformance; -import org.checkerframework.checker.nullness.qual.PolyNull; import org.jspecify.annotations.Nullable; import java.util.Properties; @@ -61,8 +60,8 @@ public interface CalciteConnectionConfig extends ConnectionConfig { /** Returns the value of {@link CalciteConnectionProperty#FUN}, * or a default operator table if not set. If {@code defaultOperatorTable} * is not null, the result is never null. */ - @PolyNull T fun(Class operatorTableClass, - @PolyNull T defaultOperatorTable); + @Nullable T fun(Class operatorTableClass, + @Nullable T defaultOperatorTable); /** Returns the value of {@link CalciteConnectionProperty#MODEL}. */ @Nullable String model(); /** Returns the value of {@link CalciteConnectionProperty#LEX}. */ @@ -78,13 +77,13 @@ public interface CalciteConnectionConfig extends ConnectionConfig { /** Returns the value of {@link CalciteConnectionProperty#PARSER_FACTORY}, * or a default parser if not set. If {@code defaultParserFactory} * is not null, the result is never null. */ - @PolyNull T parserFactory(Class parserFactoryClass, - @PolyNull T defaultParserFactory); + @Nullable T parserFactory(Class parserFactoryClass, + @Nullable T defaultParserFactory); /** Returns the value of {@link CalciteConnectionProperty#SCHEMA_FACTORY}, * or a default schema factory if not set. If {@code defaultSchemaFactory} * is not null, the result is never null. */ - @PolyNull T schemaFactory(Class schemaFactoryClass, - @PolyNull T defaultSchemaFactory); + @Nullable T schemaFactory(Class schemaFactoryClass, + @Nullable T defaultSchemaFactory); /** Returns the value of {@link CalciteConnectionProperty#SCHEMA_TYPE}. */ JsonSchema.Type schemaType(); /** Returns the value of {@link CalciteConnectionProperty#SPARK}. */ @@ -95,8 +94,8 @@ public interface CalciteConnectionConfig extends ConnectionConfig { /** Returns the value of {@link CalciteConnectionProperty#TYPE_SYSTEM}, * or a default type system if not set. If {@code defaultTypeSystem} * is not null, the result is never null. */ - @PolyNull T typeSystem(Class typeSystemClass, - @PolyNull T defaultTypeSystem); + @Nullable T typeSystem(Class typeSystemClass, + @Nullable T defaultTypeSystem); /** Returns the value of {@link CalciteConnectionProperty#CONFORMANCE}. */ SqlConformance conformance(); /** Returns the value of {@link CalciteConnectionProperty#TIME_ZONE}. */ @@ -117,12 +116,12 @@ public interface CalciteConnectionConfig extends ConnectionConfig { /** Returns the value of {@link CalciteConnectionProperty#META_TABLE_FACTORY}, * or a default meta table factory if not set. If * {@code defaultMetaTableFactory} is not null, the result is never null. */ - @PolyNull T metaTableFactory(Class metaTableFactoryClass, - @PolyNull T defaultMetaTableFactory); + @Nullable T metaTableFactory(Class metaTableFactoryClass, + @Nullable T defaultMetaTableFactory); /** Returns the value of {@link CalciteConnectionProperty#META_COLUMN_FACTORY}, * or a default meta column factory if not set. If * {@code defaultMetaColumnFactory} is not null, the result is never null. */ - @PolyNull T metaColumnFactory(Class metaColumnFactoryClass, - @PolyNull T defaultMetaColumnFactory); + @Nullable T metaColumnFactory(Class metaColumnFactoryClass, + @Nullable T defaultMetaColumnFactory); } diff --git a/core/src/main/java/org/apache/calcite/config/CalciteConnectionConfigImpl.java b/core/src/main/java/org/apache/calcite/config/CalciteConnectionConfigImpl.java index 05b9329edbb1..e0b2cc632b05 100644 --- a/core/src/main/java/org/apache/calcite/config/CalciteConnectionConfigImpl.java +++ b/core/src/main/java/org/apache/calcite/config/CalciteConnectionConfigImpl.java @@ -27,7 +27,6 @@ import org.apache.calcite.sql.validate.SqlConformance; import org.apache.calcite.sql.validate.SqlConformanceEnum; -import org.checkerframework.checker.nullness.qual.PolyNull; import org.jspecify.annotations.Nullable; import java.util.List; @@ -106,8 +105,8 @@ public boolean isSet(CalciteConnectionProperty property) { .getEnum(NullCollation.class, NullCollation.HIGH); } - @Override public @PolyNull T fun(Class operatorTableClass, - @PolyNull T defaultOperatorTable) { + @Override public @Nullable T fun(Class operatorTableClass, + @Nullable T defaultOperatorTable) { final String fun = CalciteConnectionProperty.FUN.wrap(properties).getString(); if (fun == null || fun.equals("") || fun.equals("standard")) { @@ -153,14 +152,14 @@ public boolean isSet(CalciteConnectionProperty property) { .getBoolean(lex().caseSensitive); } - @Override public @PolyNull T parserFactory(Class parserFactoryClass, - @PolyNull T defaultParserFactory) { + @Override public @Nullable T parserFactory(Class parserFactoryClass, + @Nullable T defaultParserFactory) { return CalciteConnectionProperty.PARSER_FACTORY.wrap(properties) .getPlugin(parserFactoryClass, defaultParserFactory); } - @Override public @PolyNull T schemaFactory(Class schemaFactoryClass, - @PolyNull T defaultSchemaFactory) { + @Override public @Nullable T schemaFactory(Class schemaFactoryClass, + @Nullable T defaultSchemaFactory) { return CalciteConnectionProperty.SCHEMA_FACTORY.wrap(properties) .getPlugin(schemaFactoryClass, defaultSchemaFactory); } @@ -179,8 +178,8 @@ public boolean isSet(CalciteConnectionProperty property) { .getBoolean(); } - @Override public @PolyNull T typeSystem(Class typeSystemClass, - @PolyNull T defaultTypeSystem) { + @Override public @Nullable T typeSystem(Class typeSystemClass, + @Nullable T defaultTypeSystem) { return CalciteConnectionProperty.TYPE_SYSTEM.wrap(properties) .getPlugin(typeSystemClass, defaultTypeSystem); } @@ -220,16 +219,16 @@ public boolean isSet(CalciteConnectionProperty property) { .getBoolean(); } - @Override public @PolyNull T metaTableFactory( + @Override public @Nullable T metaTableFactory( Class metaTableFactoryClass, - @PolyNull T defaultMetaTableFactory) { + @Nullable T defaultMetaTableFactory) { return CalciteConnectionProperty.META_TABLE_FACTORY.wrap(properties) .getPlugin(metaTableFactoryClass, defaultMetaTableFactory); } - @Override public @PolyNull T metaColumnFactory( + @Override public @Nullable T metaColumnFactory( Class metaColumnFactoryClass, - @PolyNull T defaultMetaColumnFactory) { + @Nullable T defaultMetaColumnFactory) { return CalciteConnectionProperty.META_COLUMN_FACTORY.wrap(properties) .getPlugin(metaColumnFactoryClass, defaultMetaColumnFactory); } diff --git a/core/src/main/java/org/apache/calcite/interpreter/AggregateNode.java b/core/src/main/java/org/apache/calcite/interpreter/AggregateNode.java index 593c853a011c..a856d93e97d2 100644 --- a/core/src/main/java/org/apache/calcite/interpreter/AggregateNode.java +++ b/core/src/main/java/org/apache/calcite/interpreter/AggregateNode.java @@ -93,7 +93,7 @@ public AggregateNode(Compiler compiler, Aggregate rel) { ImmutableList.Builder builder = ImmutableList.builder(); for (AggregateCall aggregateCall : rel.getAggCallList()) { - @SuppressWarnings("method.invocation.invalid") + @SuppressWarnings("NullAway") AccumulatorFactory accumulator = getAccumulator(compiler, aggregateCall, false); builder.add(accumulator); diff --git a/core/src/main/java/org/apache/calcite/interpreter/Context.java b/core/src/main/java/org/apache/calcite/interpreter/Context.java index 0225cd6bcf48..1411e282d540 100644 --- a/core/src/main/java/org/apache/calcite/interpreter/Context.java +++ b/core/src/main/java/org/apache/calcite/interpreter/Context.java @@ -17,8 +17,8 @@ package org.apache.calcite.interpreter; import org.apache.calcite.DataContext; +import org.apache.calcite.linq4j.annotations.MonotonicNonNull; -import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.jspecify.annotations.Nullable; /** @@ -28,7 +28,7 @@ public class Context { public final DataContext root; /** Values of incoming columns from all inputs. */ - public @Nullable Object @MonotonicNonNull [] values; + @MonotonicNonNull public @Nullable Object[] values; Context(DataContext root) { this.root = root; diff --git a/core/src/main/java/org/apache/calcite/interpreter/Interpreter.java b/core/src/main/java/org/apache/calcite/interpreter/Interpreter.java index 3fb48ef62fe8..19de7be6e940 100644 --- a/core/src/main/java/org/apache/calcite/interpreter/Interpreter.java +++ b/core/src/main/java/org/apache/calcite/interpreter/Interpreter.java @@ -49,8 +49,6 @@ import com.google.common.collect.Lists; import com.google.common.collect.Multimap; -import org.checkerframework.checker.initialization.qual.NotOnlyInitialized; -import org.checkerframework.checker.initialization.qual.UnknownInitialization; import org.jspecify.annotations.Nullable; import java.util.ArrayDeque; @@ -84,7 +82,7 @@ public Interpreter(DataContext dataContext, RelNode rootRel) { final RelNode rel = optimize(rootRel); final CompilerImpl compiler = new Nodes.CoreCompiler(this, rootRel.getCluster()); - @SuppressWarnings("method.invocation.invalid") + @SuppressWarnings("NullAway") Pair> pair = compiler.visitRoot(rel); this.rootRel = pair.left; this.nodes = ImmutableMap.copyOf(pair.right); @@ -299,7 +297,6 @@ static class CompilerImpl extends RelVisitor final ScalarCompiler scalarCompiler; private final ReflectiveVisitDispatcher dispatcher = ReflectUtil.createDispatcher(CompilerImpl.class, RelNode.class); - @NotOnlyInitialized protected final Interpreter interpreter; protected @Nullable RelNode rootRel; protected @Nullable RelNode rel; @@ -311,7 +308,7 @@ static class CompilerImpl extends RelVisitor private static final String REWRITE_METHOD_NAME = "rewrite"; private static final String VISIT_METHOD_NAME = "visit"; - CompilerImpl(@UnknownInitialization Interpreter interpreter, RelOptCluster cluster) { + CompilerImpl(Interpreter interpreter, RelOptCluster cluster) { this.interpreter = interpreter; this.scalarCompiler = new JaninoRexCompiler(cluster.getRexBuilder()); } diff --git a/core/src/main/java/org/apache/calcite/interpreter/Nodes.java b/core/src/main/java/org/apache/calcite/interpreter/Nodes.java index 511eeaadf70b..c997a57b1881 100644 --- a/core/src/main/java/org/apache/calcite/interpreter/Nodes.java +++ b/core/src/main/java/org/apache/calcite/interpreter/Nodes.java @@ -34,8 +34,6 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.initialization.qual.UnknownInitialization; - /** * Helper methods for {@link Node} and implementations for core relational * expressions. @@ -46,7 +44,7 @@ public class Nodes { * that knows how to handle the core logical * {@link org.apache.calcite.rel.RelNode}s. */ public static class CoreCompiler extends Interpreter.CompilerImpl { - CoreCompiler(@UnknownInitialization Interpreter interpreter, RelOptCluster cluster) { + CoreCompiler(Interpreter interpreter, RelOptCluster cluster) { super(interpreter, cluster); } diff --git a/core/src/main/java/org/apache/calcite/jdbc/CachingCalciteSchema.java b/core/src/main/java/org/apache/calcite/jdbc/CachingCalciteSchema.java index bec2ff118294..a53051d66a6e 100644 --- a/core/src/main/java/org/apache/calcite/jdbc/CachingCalciteSchema.java +++ b/core/src/main/java/org/apache/calcite/jdbc/CachingCalciteSchema.java @@ -57,7 +57,7 @@ class CachingCalciteSchema extends CalciteSchema { this(parent, schema, name, null, null, null, null, null, null, null, null); } - @SuppressWarnings({"argument.type.incompatible", "return.type.incompatible"}) + @SuppressWarnings("NullAway") private CachingCalciteSchema(@Nullable CalciteSchema parent, Schema schema, String name, @Nullable NameMap subSchemaMap, diff --git a/core/src/main/java/org/apache/calcite/jdbc/CalciteSchema.java b/core/src/main/java/org/apache/calcite/jdbc/CalciteSchema.java index 0e69b4de05dd..92988239b608 100644 --- a/core/src/main/java/org/apache/calcite/jdbc/CalciteSchema.java +++ b/core/src/main/java/org/apache/calcite/jdbc/CalciteSchema.java @@ -694,7 +694,7 @@ CalciteSchema calciteSchema() { return calciteSchema.plus(); } - @Override public T unwrap(Class clazz) { + @Override public T unwrap(Class clazz) { if (clazz.isInstance(this)) { return clazz.cast(this); } diff --git a/core/src/main/java/org/apache/calcite/jdbc/JavaCollation.java b/core/src/main/java/org/apache/calcite/jdbc/JavaCollation.java index dede84d21c22..b6aaca733a67 100644 --- a/core/src/main/java/org/apache/calcite/jdbc/JavaCollation.java +++ b/core/src/main/java/org/apache/calcite/jdbc/JavaCollation.java @@ -18,7 +18,6 @@ import org.apache.calcite.sql.SqlCollation; -import org.checkerframework.checker.initialization.qual.UnderInitialization; import org.jspecify.annotations.Nullable; import java.nio.charset.Charset; @@ -59,7 +58,6 @@ private static String getStrengthString(int strengthValue) { } @Override protected String generateCollationName( - @UnderInitialization JavaCollation this, Charset charset) { return super.generateCollationName(charset) + "$JAVA_COLLATOR"; } diff --git a/core/src/main/java/org/apache/calcite/materialize/Lattice.java b/core/src/main/java/org/apache/calcite/materialize/Lattice.java index f6bc5f1032f5..b84d0c42d2fa 100644 --- a/core/src/main/java/org/apache/calcite/materialize/Lattice.java +++ b/core/src/main/java/org/apache/calcite/materialize/Lattice.java @@ -21,6 +21,8 @@ import org.apache.calcite.jdbc.CalcitePrepare; import org.apache.calcite.jdbc.CalciteSchema; import org.apache.calcite.linq4j.Ord; +import org.apache.calcite.linq4j.annotations.MonotonicNonNull; +import org.apache.calcite.linq4j.annotations.RequiresNonNull; import org.apache.calcite.linq4j.tree.Primitive; import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.rel.RelNode; @@ -65,9 +67,6 @@ import com.google.common.collect.Multimap; import com.google.common.collect.Ordering; -import org.checkerframework.checker.initialization.qual.UnknownInitialization; -import org.checkerframework.checker.nullness.qual.MonotonicNonNull; -import org.checkerframework.checker.nullness.qual.RequiresNonNull; import org.jspecify.annotations.Nullable; import java.util.ArrayList; @@ -135,7 +134,7 @@ private Lattice(CalciteSchema rootSchema, LatticeRootNode rootNode, } checkArgument(rowCountEstimate > 0d); this.rowCountEstimate = rowCountEstimate; - @SuppressWarnings("argument.type.incompatible") + @SuppressWarnings("NullAway") LatticeStatisticProvider statisticProvider = requireNonNull(statisticProviderFactory.apply(this)); this.statisticProvider = statisticProvider; @@ -148,7 +147,6 @@ public static Lattice create(CalciteSchema schema, String sql, boolean auto) { @RequiresNonNull({"rootNode", "defaultMeasures", "columns"}) private boolean isValid( - @UnknownInitialization Lattice this, Litmus litmus) { if (!rootNode.isValid(litmus)) { return false; diff --git a/core/src/main/java/org/apache/calcite/materialize/LatticeNode.java b/core/src/main/java/org/apache/calcite/materialize/LatticeNode.java index 87c2fe816065..b47d33856aa2 100644 --- a/core/src/main/java/org/apache/calcite/materialize/LatticeNode.java +++ b/core/src/main/java/org/apache/calcite/materialize/LatticeNode.java @@ -21,7 +21,6 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.initialization.qual.Initialized; import org.jspecify.annotations.Nullable; import java.util.List; @@ -77,8 +76,8 @@ public abstract class LatticeNode { if (i++ > 0) { sb.append(' '); } - @SuppressWarnings({"argument.type.incompatible", "assignment.type.incompatible"}) - final @Initialized LatticeChildNode node = + @SuppressWarnings("NullAway") + final LatticeChildNode node = new LatticeChildNode(space, this, mutableChild); sb.append(node.digest); b.add(node); diff --git a/core/src/main/java/org/apache/calcite/materialize/LatticeRootNode.java b/core/src/main/java/org/apache/calcite/materialize/LatticeRootNode.java index ecf244ad2903..f9dc05ff783a 100644 --- a/core/src/main/java/org/apache/calcite/materialize/LatticeRootNode.java +++ b/core/src/main/java/org/apache/calcite/materialize/LatticeRootNode.java @@ -29,7 +29,7 @@ public class LatticeRootNode extends LatticeNode { public final ImmutableList descendants; final ImmutableList paths; - @SuppressWarnings("method.invocation.invalid") + @SuppressWarnings("NullAway") LatticeRootNode(LatticeSpace space, MutableNode mutableNode) { super(space, null, mutableNode); diff --git a/core/src/main/java/org/apache/calcite/materialize/LatticeSpace.java b/core/src/main/java/org/apache/calcite/materialize/LatticeSpace.java index 17c1c4d4100a..138e769af27b 100644 --- a/core/src/main/java/org/apache/calcite/materialize/LatticeSpace.java +++ b/core/src/main/java/org/apache/calcite/materialize/LatticeSpace.java @@ -25,8 +25,6 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.initialization.qual.NotOnlyInitialized; - import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -41,8 +39,8 @@ class LatticeSpace { final SqlStatisticProvider statisticProvider; private final Map, LatticeTable> tableMap = new HashMap<>(); - @SuppressWarnings("assignment.type.incompatible") - final @NotOnlyInitialized AttributedDirectedGraph g = + @SuppressWarnings("NullAway") + final AttributedDirectedGraph g = new AttributedDirectedGraph<>(new Step.Factory(this)); private final Map, String> simpleTableNames = new HashMap<>(); private final Set simpleNames = new HashSet<>(); diff --git a/core/src/main/java/org/apache/calcite/materialize/LatticeSuggester.java b/core/src/main/java/org/apache/calcite/materialize/LatticeSuggester.java index 3520772baa2f..2d8fcd15af99 100644 --- a/core/src/main/java/org/apache/calcite/materialize/LatticeSuggester.java +++ b/core/src/main/java/org/apache/calcite/materialize/LatticeSuggester.java @@ -467,7 +467,7 @@ private static void frames(List frames, final Query q, RelNode r) { final ImmutableNullableList.Builder<@Nullable ColRef> columnBuilder = ImmutableNullableList.builder(); for (Pair p : project.getNamedProjects()) { - @SuppressWarnings("method.invocation.invalid") + @SuppressWarnings("NullAway") ColRef colRef = toColRef(p.left, p.right); columnBuilder.add(colRef); } diff --git a/core/src/main/java/org/apache/calcite/materialize/MutableNode.java b/core/src/main/java/org/apache/calcite/materialize/MutableNode.java index 67bdfb851cd2..ff91779e6beb 100644 --- a/core/src/main/java/org/apache/calcite/materialize/MutableNode.java +++ b/core/src/main/java/org/apache/calcite/materialize/MutableNode.java @@ -69,7 +69,7 @@ class MutableNode { } /** Creates a non-root node. */ - @SuppressWarnings("argument.type.incompatible") + @SuppressWarnings("NullAway") MutableNode(LatticeTable table, @Nullable MutableNode parent, @Nullable Step step) { this.table = requireNonNull(table, "table"); this.parent = parent; diff --git a/core/src/main/java/org/apache/calcite/materialize/Step.java b/core/src/main/java/org/apache/calcite/materialize/Step.java index 99c8105cd4a3..fcbab3477e88 100644 --- a/core/src/main/java/org/apache/calcite/materialize/Step.java +++ b/core/src/main/java/org/apache/calcite/materialize/Step.java @@ -24,8 +24,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Ordering; -import org.checkerframework.checker.initialization.qual.NotOnlyInitialized; -import org.checkerframework.checker.initialization.qual.UnderInitialization; import org.jspecify.annotations.Nullable; import java.util.List; @@ -145,10 +143,10 @@ private static double cardinality(SqlStatisticProvider statisticProvider, /** Creates {@link Step} instances. */ static class Factory implements AttributedDirectedGraph.AttributedEdgeFactory< LatticeTable, Step> { - private final @NotOnlyInitialized LatticeSpace space; + private final LatticeSpace space; - @SuppressWarnings("type.argument.type.incompatible") - Factory(@UnderInitialization LatticeSpace space) { + @SuppressWarnings("NullAway") + Factory(LatticeSpace space) { this.space = requireNonNull(space, "space"); } diff --git a/core/src/main/java/org/apache/calcite/model/ModelHandler.java b/core/src/main/java/org/apache/calcite/model/ModelHandler.java index 322cea082d4f..d54792573b45 100644 --- a/core/src/main/java/org/apache/calcite/model/ModelHandler.java +++ b/core/src/main/java/org/apache/calcite/model/ModelHandler.java @@ -96,7 +96,7 @@ public ModelHandler(SchemaPlus rootSchema, String uri) throws IOException { * by reflection from the model against {@code classNameFilter}. Use * this to apply a stricter (or more permissive) filter than the * standard one. */ - @SuppressWarnings("method.invocation.invalid") + @SuppressWarnings("NullAway") public ModelHandler(SchemaPlus rootSchema, String uri, ClassNameFilter classNameFilter) throws IOException { super(); diff --git a/core/src/main/java/org/apache/calcite/plan/AbstractRelOptPlanner.java b/core/src/main/java/org/apache/calcite/plan/AbstractRelOptPlanner.java index d37233963bd9..4c8680e0a085 100644 --- a/core/src/main/java/org/apache/calcite/plan/AbstractRelOptPlanner.java +++ b/core/src/main/java/org/apache/calcite/plan/AbstractRelOptPlanner.java @@ -16,6 +16,7 @@ */ package org.apache.calcite.plan; +import org.apache.calcite.linq4j.annotations.MonotonicNonNull; import org.apache.calcite.plan.volcano.RelSubset; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.metadata.RelMetadataProvider; @@ -30,9 +31,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import org.checkerframework.checker.initialization.qual.UnknownInitialization; -import org.checkerframework.checker.nullness.qual.MonotonicNonNull; -import org.checkerframework.dataflow.qual.Pure; import org.jspecify.annotations.Nullable; import org.slf4j.Logger; @@ -275,7 +273,6 @@ protected void onNewClass(RelNode node) { } @Override public void addListener( - @UnknownInitialization AbstractRelOptPlanner this, RelOptListener newListener) { if (listener == null) { listener = new MulticastRelOptListener(); @@ -467,7 +464,6 @@ protected void notifyDiscard(RelNode rel) { } } - @Pure public @Nullable RelOptListener getListener() { return listener; } diff --git a/core/src/main/java/org/apache/calcite/plan/Contexts.java b/core/src/main/java/org/apache/calcite/plan/Contexts.java index 732e291ec6a6..1e37d5570edf 100644 --- a/core/src/main/java/org/apache/calcite/plan/Contexts.java +++ b/core/src/main/java/org/apache/calcite/plan/Contexts.java @@ -121,7 +121,7 @@ private static class WrapContext implements Context { this.target = requireNonNull(target, "target"); } - @Override public @Nullable T unwrap(Class clazz) { + @Override public @Nullable T unwrap(Class clazz) { if (clazz.isInstance(target)) { return clazz.cast(target); } @@ -131,7 +131,7 @@ private static class WrapContext implements Context { /** Empty context. */ static class EmptyContext implements Context { - @Override public @Nullable T unwrap(Class clazz) { + @Override public @Nullable T unwrap(Class clazz) { return null; } } @@ -147,7 +147,7 @@ private static final class ChainContext implements Context { } } - @Override public @Nullable T unwrap(Class clazz) { + @Override public @Nullable T unwrap(Class clazz) { for (Context context : contexts) { final T t = context.unwrap(clazz); if (t != null) { diff --git a/core/src/main/java/org/apache/calcite/plan/ConventionTraitDef.java b/core/src/main/java/org/apache/calcite/plan/ConventionTraitDef.java index 5c14ba8aaa4d..048fb3ce501c 100644 --- a/core/src/main/java/org/apache/calcite/plan/ConventionTraitDef.java +++ b/core/src/main/java/org/apache/calcite/plan/ConventionTraitDef.java @@ -16,6 +16,7 @@ */ package org.apache.calcite.plan; +import org.apache.calcite.linq4j.annotations.MonotonicNonNull; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.convert.ConverterRule; import org.apache.calcite.rel.metadata.RelMetadataQuery; @@ -31,7 +32,6 @@ import com.google.common.collect.HashMultimap; import com.google.common.collect.Multimap; -import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.jspecify.annotations.Nullable; import java.util.List; @@ -227,7 +227,7 @@ private static final class ConversionData { final Multimap, ConverterRule> mapArcToConverterRule = HashMultimap.create(); - private Graphs.@MonotonicNonNull FrozenGraph pathMap; + @MonotonicNonNull private Graphs.FrozenGraph pathMap; public List> getPaths( Convention fromConvention, diff --git a/core/src/main/java/org/apache/calcite/plan/RelOptAbstractTable.java b/core/src/main/java/org/apache/calcite/plan/RelOptAbstractTable.java index f434246c4b2e..962f3ad886e4 100644 --- a/core/src/main/java/org/apache/calcite/plan/RelOptAbstractTable.java +++ b/core/src/main/java/org/apache/calcite/plan/RelOptAbstractTable.java @@ -88,7 +88,7 @@ public String getName() { return RelDistributions.BROADCAST_DISTRIBUTED; } - @Override public @Nullable T unwrap(Class clazz) { + @Override public @Nullable T unwrap(Class clazz) { return clazz.isInstance(this) ? clazz.cast(this) : null; diff --git a/core/src/main/java/org/apache/calcite/plan/RelOptCluster.java b/core/src/main/java/org/apache/calcite/plan/RelOptCluster.java index aacdf8aa6a9a..6cc88b8c0b99 100644 --- a/core/src/main/java/org/apache/calcite/plan/RelOptCluster.java +++ b/core/src/main/java/org/apache/calcite/plan/RelOptCluster.java @@ -16,6 +16,7 @@ */ package org.apache.calcite.plan; +import org.apache.calcite.linq4j.annotations.EnsuresNonNull; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.CorrelationId; import org.apache.calcite.rel.hint.HintStrategyTable; @@ -29,8 +30,6 @@ import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexNode; -import org.checkerframework.checker.initialization.qual.UnknownInitialization; -import org.checkerframework.checker.nullness.qual.EnsuresNonNull; import org.jspecify.annotations.Nullable; import java.util.HashMap; @@ -149,7 +148,6 @@ public RexBuilder getRexBuilder() { @EnsuresNonNull({"this.metadataProvider", "this.metadataFactory"}) @SuppressWarnings("deprecation") public void setMetadataProvider( - @UnknownInitialization RelOptCluster this, RelMetadataProvider metadataProvider) { this.metadataProvider = metadataProvider; this.metadataFactory = @@ -182,7 +180,6 @@ public MetadataFactory getMetadataFactory() { */ @EnsuresNonNull("this.mqSupplier") public void setMetadataQuerySupplier( - @UnknownInitialization RelOptCluster this, Supplier mqSupplier) { this.mqSupplier = mqSupplier; } diff --git a/core/src/main/java/org/apache/calcite/plan/RelOptRule.java b/core/src/main/java/org/apache/calcite/plan/RelOptRule.java index 5e6f72b8597c..d63a8ced5cf0 100644 --- a/core/src/main/java/org/apache/calcite/plan/RelOptRule.java +++ b/core/src/main/java/org/apache/calcite/plan/RelOptRule.java @@ -26,7 +26,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; -import org.checkerframework.checker.initialization.qual.UnderInitialization; import org.jspecify.annotations.Nullable; import java.util.ArrayList; @@ -379,7 +378,6 @@ public static RelOptRuleOperandChildren any() { * @return Flattened list of operands */ private List flattenOperands( - @UnderInitialization RelOptRule this, RelOptRuleOperand rootOperand) { final List operandList = new ArrayList<>(); @@ -400,7 +398,6 @@ private List flattenOperands( * @param parentOperand Parent of this operand */ private void flattenRecurse( - @UnderInitialization RelOptRule this, List operandList, RelOptRuleOperand parentOperand) { int k = 0; diff --git a/core/src/main/java/org/apache/calcite/plan/RelOptRuleOperand.java b/core/src/main/java/org/apache/calcite/plan/RelOptRuleOperand.java index a11b4b51f0a9..c665f78cb3f5 100644 --- a/core/src/main/java/org/apache/calcite/plan/RelOptRuleOperand.java +++ b/core/src/main/java/org/apache/calcite/plan/RelOptRuleOperand.java @@ -16,13 +16,11 @@ */ package org.apache.calcite.plan; +import org.apache.calcite.linq4j.annotations.MonotonicNonNull; import org.apache.calcite.rel.RelNode; import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.initialization.qual.NotOnlyInitialized; -import org.checkerframework.checker.initialization.qual.UnknownInitialization; -import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.jspecify.annotations.Nullable; import java.util.List; @@ -48,12 +46,12 @@ public class RelOptRuleOperand { //~ Instance fields -------------------------------------------------------- private @Nullable RelOptRuleOperand parent; - private @NotOnlyInitialized RelOptRule rule; + private RelOptRule rule; private final Predicate predicate; // REVIEW jvs 29-Aug-2004: some of these are Volcano-specific and should be // factored out - public int @MonotonicNonNull [] solveOrder; + @MonotonicNonNull public int[] solveOrder; public int ordinalInParent; public int ordinalInRule; public final @Nullable RelTrait trait; @@ -106,8 +104,7 @@ protected RelOptRuleOperand( * and add constructor parameters for them. See * [CALCITE-1166] * Disallow sub-classes of RelOptRuleOperand. */ - @SuppressWarnings({"initialization.fields.uninitialized", - "initialization.invalid.field.write.initialized", "unchecked"}) + @SuppressWarnings({"NullAway", "unchecked"}) RelOptRuleOperand( Class clazz, @Nullable RelTrait trait, @@ -171,8 +168,8 @@ public RelOptRule getRule() { * * @param rule containing rule */ - @SuppressWarnings("initialization.invalid.field.write.initialized") - public void setRule(@UnknownInitialization RelOptRule rule) { + @SuppressWarnings("NullAway") + public void setRule(RelOptRule rule) { this.rule = rule; } diff --git a/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java b/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java index e3b7e32784e6..166ffc81f3fa 100644 --- a/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java +++ b/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java @@ -114,9 +114,6 @@ import com.google.common.collect.Lists; import com.google.common.collect.Multimap; -import org.checkerframework.checker.initialization.qual.NotOnlyInitialized; -import org.checkerframework.checker.initialization.qual.UnknownInitialization; -import org.checkerframework.checker.nullness.qual.PolyNull; import org.jspecify.annotations.Nullable; import java.io.PrintWriter; @@ -2482,8 +2479,8 @@ public static String toString( * returns null if and only if {@code rel} is null, * returns expanded detail info for {@code rel} if {@code expand} is true. */ - public static @PolyNull String toString( - final @PolyNull RelNode rel, + public static @Nullable String toString( + final @Nullable RelNode rel, SqlExplainLevel detailLevel, boolean expand) { if (rel == null) { @@ -4556,10 +4553,9 @@ public static class VariableUsedVisitor extends RexShuttle { public final Set variables = new LinkedHashSet<>(); public final Multimap variableFields = LinkedHashMultimap.create(); - @NotOnlyInitialized private final @Nullable RelShuttle relShuttle; - public VariableUsedVisitor(@UnknownInitialization @Nullable RelShuttle relShuttle) { + public VariableUsedVisitor(@Nullable RelShuttle relShuttle) { this.relShuttle = relShuttle; } @@ -4988,7 +4984,7 @@ public boolean opposite(Side side) { * expression, including those that are inside * {@link RexSubQuery sub-queries}. */ private static class CorrelationCollector extends RelHomogeneousShuttle { - @SuppressWarnings("assignment.type.incompatible") + @SuppressWarnings("NullAway") private final VariableUsedVisitor vuv = new VariableUsedVisitor(this); @Override public RelNode visit(RelNode other) { diff --git a/core/src/main/java/org/apache/calcite/plan/RelRule.java b/core/src/main/java/org/apache/calcite/plan/RelRule.java index 4511c35d92ea..4c0aa11372b5 100644 --- a/core/src/main/java/org/apache/calcite/plan/RelRule.java +++ b/core/src/main/java/org/apache/calcite/plan/RelRule.java @@ -133,7 +133,7 @@ public interface Config { RelOptRule toRule(); /** Casts this configuration to another type, usually a sub-class. */ - default T as(Class class_) { + default T as(Class class_) { if (class_.isAssignableFrom(this.getClass())) { return class_.cast(this); } else { diff --git a/core/src/main/java/org/apache/calcite/plan/RelTraitSet.java b/core/src/main/java/org/apache/calcite/plan/RelTraitSet.java index 607535fc6194..dfd170915c8e 100644 --- a/core/src/main/java/org/apache/calcite/plan/RelTraitSet.java +++ b/core/src/main/java/org/apache/calcite/plan/RelTraitSet.java @@ -470,8 +470,7 @@ public List getCollations() { */ public T canonize(T trait) { if (trait == null) { - // Return "trait" makes the input type to be the same as the output type, - // so checkerframework is happy + // Returning "trait" keeps the output type identical to the input type return trait; } diff --git a/core/src/main/java/org/apache/calcite/plan/RexImplicationChecker.java b/core/src/main/java/org/apache/calcite/plan/RexImplicationChecker.java index d243a6cabb15..22ed8943ff85 100644 --- a/core/src/main/java/org/apache/calcite/plan/RexImplicationChecker.java +++ b/core/src/main/java/org/apache/calcite/plan/RexImplicationChecker.java @@ -343,9 +343,8 @@ private static boolean checkSupport(InputUsageFinder firstUsageFinder, final SqlKind sKind2 = secondLen == 2 ? secondUsageList.get(1).getKey().getKind() : null; - // Note: arguments to isEquivalentOp are never null, however checker-framework's - // dataflow is not strong enough, so the first parameter is marked as nullable - //noinspection ConstantConditions + // The first parameter of isEquivalentOp is declared @Nullable, so that a caller can + // pass a kind that was not found; here both kinds are known to be present if (firstLen == 2 && secondLen == 2 && fKind2 != null && sKind2 != null && !(isEquivalentOp(fKind, sKind) && isEquivalentOp(fKind2, sKind2)) diff --git a/core/src/main/java/org/apache/calcite/plan/SubstitutionVisitor.java b/core/src/main/java/org/apache/calcite/plan/SubstitutionVisitor.java index 537902de6fb5..402f872e3b98 100644 --- a/core/src/main/java/org/apache/calcite/plan/SubstitutionVisitor.java +++ b/core/src/main/java/org/apache/calcite/plan/SubstitutionVisitor.java @@ -1017,7 +1017,7 @@ assert equalType("query", call.query, "result", result, /** Abstract base class for implementing {@link UnifyRule}. */ public abstract static class AbstractUnifyRule extends UnifyRule { - @SuppressWarnings("method.invocation.invalid") + @SuppressWarnings("NullAway") protected AbstractUnifyRule(Operand queryOperand, Operand targetOperand, int slotCount) { super(slotCount, queryOperand, targetOperand); diff --git a/core/src/main/java/org/apache/calcite/plan/TableAccessMap.java b/core/src/main/java/org/apache/calcite/plan/TableAccessMap.java index 8075b890b659..f29a7dc97ca3 100644 --- a/core/src/main/java/org/apache/calcite/plan/TableAccessMap.java +++ b/core/src/main/java/org/apache/calcite/plan/TableAccessMap.java @@ -113,7 +113,7 @@ public TableAccessMap(List table, Mode mode) { /** * Returns a set of qualified names for all tables accessed. */ - @SuppressWarnings("return.type.incompatible") + @SuppressWarnings("NullAway") public Set> getTablesAccessed() { return accessMap.keySet(); } diff --git a/core/src/main/java/org/apache/calcite/plan/hep/HepInstruction.java b/core/src/main/java/org/apache/calcite/plan/hep/HepInstruction.java index 7bbfcfa48762..f2e8d46e3318 100644 --- a/core/src/main/java/org/apache/calcite/plan/hep/HepInstruction.java +++ b/core/src/main/java/org/apache/calcite/plan/hep/HepInstruction.java @@ -16,11 +16,11 @@ */ package org.apache.calcite.plan.hep; +import org.apache.calcite.linq4j.annotations.MonotonicNonNull; import org.apache.calcite.plan.RelOptRule; import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.jspecify.annotations.Nullable; import java.util.Collection; diff --git a/core/src/main/java/org/apache/calcite/plan/hep/HepRelMetadataProvider.java b/core/src/main/java/org/apache/calcite/plan/hep/HepRelMetadataProvider.java index 90520fab860e..67dd201631ab 100644 --- a/core/src/main/java/org/apache/calcite/plan/hep/HepRelMetadataProvider.java +++ b/core/src/main/java/org/apache/calcite/plan/hep/HepRelMetadataProvider.java @@ -51,7 +51,7 @@ class HepRelMetadataProvider implements RelMetadataProvider { } @Deprecated // to be removed before 2.0 - @Override public <@Nullable M extends @Nullable Metadata> UnboundMetadata apply( + @Override public UnboundMetadata apply( Class relClass, final Class metadataClass) { return (rel, mq) -> { diff --git a/core/src/main/java/org/apache/calcite/plan/volcano/RelSet.java b/core/src/main/java/org/apache/calcite/plan/volcano/RelSet.java index 58db11bbb9ba..e2b2440b8841 100644 --- a/core/src/main/java/org/apache/calcite/plan/volcano/RelSet.java +++ b/core/src/main/java/org/apache/calcite/plan/volcano/RelSet.java @@ -16,6 +16,7 @@ */ package org.apache.calcite.plan.volcano; +import org.apache.calcite.linq4j.annotations.MonotonicNonNull; import org.apache.calcite.plan.Convention; import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.plan.RelOptListener; @@ -32,7 +33,6 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.jspecify.annotations.Nullable; import org.slf4j.Logger; diff --git a/core/src/main/java/org/apache/calcite/plan/volcano/RelSubset.java b/core/src/main/java/org/apache/calcite/plan/volcano/RelSubset.java index 68609a5c7082..97cce4d58292 100644 --- a/core/src/main/java/org/apache/calcite/plan/volcano/RelSubset.java +++ b/core/src/main/java/org/apache/calcite/plan/volcano/RelSubset.java @@ -17,6 +17,7 @@ package org.apache.calcite.plan.volcano; import org.apache.calcite.linq4j.Linq4j; +import org.apache.calcite.linq4j.annotations.EnsuresNonNull; import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.plan.RelOptCost; import org.apache.calcite.plan.RelOptListener; @@ -41,8 +42,6 @@ import com.google.common.collect.Sets; import org.apiguardian.api.API; -import org.checkerframework.checker.initialization.qual.UnderInitialization; -import org.checkerframework.checker.nullness.qual.EnsuresNonNull; import org.jspecify.annotations.Nullable; import org.slf4j.Logger; @@ -171,12 +170,11 @@ public class RelSubset extends AbstractRelNode { */ @EnsuresNonNull("bestCost") private void computeBestCost( - @UnderInitialization RelSubset this, RelOptCluster cluster, RelOptPlanner planner) { bestCost = planner.getCostFactory().makeInfiniteCost(); final RelMetadataQuery mq = cluster.getMetadataQuery(); - @SuppressWarnings("method.invocation.invalid") + @SuppressWarnings("NullAway") Iterable rels = getRels(); for (RelNode rel : rels) { final RelOptCost cost = planner.getCost(rel, mq); diff --git a/core/src/main/java/org/apache/calcite/plan/volcano/VolcanoPlanner.java b/core/src/main/java/org/apache/calcite/plan/volcano/VolcanoPlanner.java index 54cc9c46f271..f1b27aae74dd 100644 --- a/core/src/main/java/org/apache/calcite/plan/volcano/VolcanoPlanner.java +++ b/core/src/main/java/org/apache/calcite/plan/volcano/VolcanoPlanner.java @@ -18,6 +18,9 @@ import org.apache.calcite.config.CalciteConnectionConfig; import org.apache.calcite.config.CalciteSystemProperty; +import org.apache.calcite.linq4j.annotations.EnsuresNonNull; +import org.apache.calcite.linq4j.annotations.MonotonicNonNull; +import org.apache.calcite.linq4j.annotations.RequiresNonNull; import org.apache.calcite.plan.AbstractRelOptPlanner; import org.apache.calcite.plan.Context; import org.apache.calcite.plan.Convention; @@ -61,11 +64,6 @@ import com.google.common.collect.Multimap; import org.apiguardian.api.API; -import org.checkerframework.checker.nullness.qual.EnsuresNonNull; -import org.checkerframework.checker.nullness.qual.MonotonicNonNull; -import org.checkerframework.checker.nullness.qual.PolyNull; -import org.checkerframework.checker.nullness.qual.RequiresNonNull; -import org.checkerframework.dataflow.qual.Pure; import org.jspecify.annotations.Nullable; import java.io.PrintWriter; @@ -228,7 +226,7 @@ public VolcanoPlanner(Context externalContext) { /** * Creates a {@code VolcanoPlanner} with a given cost factory. */ - @SuppressWarnings("method.invocation.invalid") + @SuppressWarnings("NullAway") public VolcanoPlanner(@Nullable RelOptCostFactory costFactory, @Nullable Context externalContext) { super(costFactory == null ? VolcanoCost.FACTORY : costFactory, @@ -282,7 +280,6 @@ public void setTopDownOpt(boolean value) { ensureRootConverters(); } - @Pure @Override public @Nullable RelNode getRoot() { return root; } @@ -1503,7 +1500,7 @@ private RelSubset registerSubset( * @param plan Plan * @return Normalized plan */ - public static @PolyNull String normalizePlan(@PolyNull String plan) { + public static @Nullable String normalizePlan(@Nullable String plan) { if (plan == null) { return null; } diff --git a/core/src/main/java/org/apache/calcite/plan/volcano/VolcanoRelMetadataProvider.java b/core/src/main/java/org/apache/calcite/plan/volcano/VolcanoRelMetadataProvider.java index 9ff9361c4412..359a06c7553f 100644 --- a/core/src/main/java/org/apache/calcite/plan/volcano/VolcanoRelMetadataProvider.java +++ b/core/src/main/java/org/apache/calcite/plan/volcano/VolcanoRelMetadataProvider.java @@ -51,7 +51,7 @@ public class VolcanoRelMetadataProvider implements RelMetadataProvider { } @Deprecated // to be removed before 2.0 - @Override public <@Nullable M extends @Nullable Metadata> @Nullable UnboundMetadata apply( + @Override public @Nullable UnboundMetadata apply( Class relClass, final Class metadataClass) { if (relClass != RelSubset.class) { diff --git a/core/src/main/java/org/apache/calcite/plan/volcano/VolcanoRuleMatch.java b/core/src/main/java/org/apache/calcite/plan/volcano/VolcanoRuleMatch.java index 8a24ecd9c139..c5a77a825094 100644 --- a/core/src/main/java/org/apache/calcite/plan/volcano/VolcanoRuleMatch.java +++ b/core/src/main/java/org/apache/calcite/plan/volcano/VolcanoRuleMatch.java @@ -42,7 +42,7 @@ class VolcanoRuleMatch extends VolcanoRuleCall { * can modify it later * @param nodeInputs Map from relational expressions to their inputs */ - @SuppressWarnings("method.invocation.invalid") + @SuppressWarnings("NullAway") VolcanoRuleMatch(VolcanoPlanner volcanoPlanner, RelOptRuleOperand operand0, RelNode[] rels, Map> nodeInputs) { super(volcanoPlanner, operand0, rels.clone(), nodeInputs); diff --git a/core/src/main/java/org/apache/calcite/prepare/CalciteCatalogReader.java b/core/src/main/java/org/apache/calcite/prepare/CalciteCatalogReader.java index 129a09985472..6932692982b7 100644 --- a/core/src/main/java/org/apache/calcite/prepare/CalciteCatalogReader.java +++ b/core/src/main/java/org/apache/calcite/prepare/CalciteCatalogReader.java @@ -515,7 +515,7 @@ private static RelDataType toSql(RelDataTypeFactory typeFactory, return nameMatcher; } - @Override public @Nullable C unwrap(Class aClass) { + @Override public @Nullable C unwrap(Class aClass) { if (aClass.isInstance(this)) { return aClass.cast(this); } diff --git a/core/src/main/java/org/apache/calcite/prepare/PlannerImpl.java b/core/src/main/java/org/apache/calcite/prepare/PlannerImpl.java index 046e77a4ee16..38b233da1ccf 100644 --- a/core/src/main/java/org/apache/calcite/prepare/PlannerImpl.java +++ b/core/src/main/java/org/apache/calcite/prepare/PlannerImpl.java @@ -23,6 +23,7 @@ import org.apache.calcite.config.CalciteSystemProperty; import org.apache.calcite.jdbc.CalciteSchema; import org.apache.calcite.jdbc.JavaTypeFactoryImpl; +import org.apache.calcite.linq4j.annotations.EnsuresNonNull; import org.apache.calcite.plan.Context; import org.apache.calcite.plan.ConventionTraitDef; import org.apache.calcite.plan.RelOptCluster; @@ -61,7 +62,6 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.EnsuresNonNull; import org.jspecify.annotations.Nullable; import java.io.Reader; @@ -104,7 +104,7 @@ public class PlannerImpl implements Planner, ViewExpander { /** Creates a planner. Not a public API; call * {@link org.apache.calcite.tools.Frameworks#getPlanner} instead. */ - @SuppressWarnings("method.invocation.invalid") + @SuppressWarnings("NullAway") public PlannerImpl(FrameworkConfig config) { this.costFactory = config.getCostFactory(); this.defaultSchema = config.getDefaultSchema(); diff --git a/core/src/main/java/org/apache/calcite/prepare/Prepare.java b/core/src/main/java/org/apache/calcite/prepare/Prepare.java index b3284a86f36a..89a516e6b4c5 100644 --- a/core/src/main/java/org/apache/calcite/prepare/Prepare.java +++ b/core/src/main/java/org/apache/calcite/prepare/Prepare.java @@ -22,6 +22,7 @@ import org.apache.calcite.jdbc.CalcitePrepare; import org.apache.calcite.jdbc.CalciteSchema; import org.apache.calcite.jdbc.CalciteSchema.LatticeEntry; +import org.apache.calcite.linq4j.annotations.MonotonicNonNull; import org.apache.calcite.plan.Convention; import org.apache.calcite.plan.RelOptLattice; import org.apache.calcite.plan.RelOptMaterialization; @@ -69,7 +70,6 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.jspecify.annotations.Nullable; import org.slf4j.Logger; diff --git a/core/src/main/java/org/apache/calcite/prepare/QueryableRelBuilder.java b/core/src/main/java/org/apache/calcite/prepare/QueryableRelBuilder.java index c2544c181b82..6ce1fbd1414f 100644 --- a/core/src/main/java/org/apache/calcite/prepare/QueryableRelBuilder.java +++ b/core/src/main/java/org/apache/calcite/prepare/QueryableRelBuilder.java @@ -51,7 +51,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import org.checkerframework.checker.nullness.qual.PolyNull; import org.jspecify.annotations.Nullable; import java.math.BigDecimal; @@ -254,8 +253,8 @@ private void setRel(RelNode rel) { throw new UnsupportedOperationException(); } - @Override public Queryable<@PolyNull T> defaultIfEmpty(Queryable source, - @PolyNull T value) { + @Override public Queryable<@Nullable T> defaultIfEmpty(Queryable source, + @Nullable T value) { throw new UnsupportedOperationException(); } diff --git a/core/src/main/java/org/apache/calcite/prepare/RelOptTableImpl.java b/core/src/main/java/org/apache/calcite/prepare/RelOptTableImpl.java index 3a5a63120432..62bb5482e957 100644 --- a/core/src/main/java/org/apache/calcite/prepare/RelOptTableImpl.java +++ b/core/src/main/java/org/apache/calcite/prepare/RelOptTableImpl.java @@ -187,7 +187,7 @@ public static RelOptTableImpl create(@Nullable RelOptSchema schema, return new RelOptTableImpl(schema, rowType, names, table, null, null); } - @Override public @Nullable T unwrap(Class clazz) { + @Override public @Nullable T unwrap(Class clazz) { if (clazz.isInstance(this)) { return clazz.cast(this); } @@ -273,7 +273,7 @@ public static RelOptTableImpl create(@Nullable RelOptSchema schema, final RelOptTable relOptTable = new RelOptTableImpl(this.schema, b.build(), this.names, this.table, this.tableExpressionFactory, this.rowCount) { - @Override public @Nullable T unwrap(Class clazz) { + @Override public @Nullable T unwrap(Class clazz) { if (clazz.isAssignableFrom(InitializerExpressionFactory.class)) { return clazz.cast(NullInitializerExpressionFactory.INSTANCE); } @@ -493,7 +493,7 @@ public static MySchemaPlus create(Path path) { return schema.isMutable(); } - @Override public @Nullable T unwrap(Class clazz) { + @Override public @Nullable T unwrap(Class clazz) { return null; } diff --git a/core/src/main/java/org/apache/calcite/profile/SimpleProfiler.java b/core/src/main/java/org/apache/calcite/profile/SimpleProfiler.java index 117f93489f83..0e38205f143c 100644 --- a/core/src/main/java/org/apache/calcite/profile/SimpleProfiler.java +++ b/core/src/main/java/org/apache/calcite/profile/SimpleProfiler.java @@ -17,6 +17,7 @@ package org.apache.calcite.profile; import org.apache.calcite.linq4j.Ord; +import org.apache.calcite.linq4j.annotations.RequiresNonNull; import org.apache.calcite.materialize.Lattice; import org.apache.calcite.rel.metadata.NullSentinel; import org.apache.calcite.runtime.FlatLists; @@ -27,8 +28,6 @@ import com.google.common.collect.ImmutableSortedSet; import com.google.common.collect.Iterables; -import org.checkerframework.checker.initialization.qual.UnknownInitialization; -import org.checkerframework.checker.nullness.qual.RequiresNonNull; import org.jspecify.annotations.Nullable; import java.util.ArrayList; @@ -295,7 +294,6 @@ private boolean hasNull(ImmutableBitSet columnOrdinals) { @RequiresNonNull("columns") private ImmutableSortedSet toColumns( - @UnknownInitialization Run this, Iterable ordinals) { //noinspection Convert2MethodRef return ImmutableSortedSet.copyOf( diff --git a/core/src/main/java/org/apache/calcite/rel/AbstractRelNode.java b/core/src/main/java/org/apache/calcite/rel/AbstractRelNode.java index 2dbc3ebf2dba..d248e01e1f9d 100644 --- a/core/src/main/java/org/apache/calcite/rel/AbstractRelNode.java +++ b/core/src/main/java/org/apache/calcite/rel/AbstractRelNode.java @@ -16,6 +16,7 @@ */ package org.apache.calcite.rel; +import org.apache.calcite.linq4j.annotations.MonotonicNonNull; import org.apache.calcite.plan.Convention; import org.apache.calcite.plan.ConventionTraitDef; import org.apache.calcite.plan.RelDigest; @@ -42,9 +43,6 @@ import com.google.common.collect.ImmutableSet; import org.apiguardian.api.API; -import org.checkerframework.checker.initialization.qual.UnknownInitialization; -import org.checkerframework.checker.nullness.qual.MonotonicNonNull; -import org.checkerframework.dataflow.qual.Pure; import org.jspecify.annotations.Nullable; import java.util.ArrayList; @@ -127,9 +125,7 @@ protected static T sole(List collection) { return cluster; } - @Pure - @Override public final @Nullable Convention getConvention( - @UnknownInitialization AbstractRelNode this) { + @Override public final @Nullable Convention getConvention() { return traitSet == null ? null : traitSet.getTrait(ConventionTraitDef.INSTANCE); } @@ -235,7 +231,7 @@ protected RelDataType deriveRowType() { } @Deprecated // to be removed before 2.0 - @Override public final <@Nullable M extends @Nullable Metadata> M metadata(Class metadataClass, + @Override public final M metadata(Class metadataClass, RelMetadataQuery mq) { final MetadataFactory factory = cluster.getMetadataFactory(); final M metadata = factory.query(this, mq, metadataClass); diff --git a/core/src/main/java/org/apache/calcite/rel/RelNode.java b/core/src/main/java/org/apache/calcite/rel/RelNode.java index f5d1bd672a63..000f4dd446eb 100644 --- a/core/src/main/java/org/apache/calcite/rel/RelNode.java +++ b/core/src/main/java/org/apache/calcite/rel/RelNode.java @@ -33,8 +33,6 @@ import org.apache.calcite.util.Litmus; import org.apiguardian.api.API; -import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf; -import org.checkerframework.dataflow.qual.Pure; import org.jspecify.annotations.Nullable; import java.util.List; @@ -90,7 +88,6 @@ public interface RelNode extends RelOptNode, Cloneable { * * @return this RelNode's CallingConvention */ - @Pure @Nullable Convention getConvention(); /** @@ -213,7 +210,7 @@ public interface RelNode extends RelOptNode, Cloneable { * return null from all methods) */ @Deprecated // to be removed before 2.0 - <@Nullable M extends @Nullable Metadata> M metadata(Class metadataClass, RelMetadataQuery mq); + M metadata(Class metadataClass, RelMetadataQuery mq); /** * Describes the inputs and attributes of this relational expression. @@ -297,7 +294,6 @@ default String explain() { * @return Whether the 2 RelNodes are equivalent or have the same digest. * @see #deepHashCode() */ - @EnsuresNonNullIf(expression = "#1", result = true) boolean deepEquals(@Nullable Object obj); /** diff --git a/core/src/main/java/org/apache/calcite/rel/core/Aggregate.java b/core/src/main/java/org/apache/calcite/rel/core/Aggregate.java index ef8538faa1db..f23c6128c489 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/Aggregate.java +++ b/core/src/main/java/org/apache/calcite/rel/core/Aggregate.java @@ -150,7 +150,7 @@ public static void checkIndicator(boolean indicator) { * @param groupSets List of all grouping sets; null for just {@code groupSet} * @param aggCalls Collection of calls to aggregate functions */ - @SuppressWarnings("method.invocation.invalid") + @SuppressWarnings("NullAway") protected Aggregate( RelOptCluster cluster, RelTraitSet traitSet, diff --git a/core/src/main/java/org/apache/calcite/rel/core/Calc.java b/core/src/main/java/org/apache/calcite/rel/core/Calc.java index 7b5725881d03..6610c14e500c 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/Calc.java +++ b/core/src/main/java/org/apache/calcite/rel/core/Calc.java @@ -68,7 +68,7 @@ public abstract class Calc extends SingleRel implements Hintable { * @param child Input relation * @param program Calc program */ - @SuppressWarnings("method.invocation.invalid") + @SuppressWarnings("NullAway") protected Calc( RelOptCluster cluster, RelTraitSet traits, diff --git a/core/src/main/java/org/apache/calcite/rel/core/Collect.java b/core/src/main/java/org/apache/calcite/rel/core/Collect.java index 48547f54301f..01a0c89761ff 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/Collect.java +++ b/core/src/main/java/org/apache/calcite/rel/core/Collect.java @@ -171,8 +171,8 @@ public static Collect create(RelNode input, } /** Returns the row type, guaranteed not null. - * (The row type is never null after initialization, but - * CheckerFramework can't deduce that references are safe.) */ + * (The field is nullable because it is populated lazily, but it is set by the time any + * caller can reach this method.) */ protected final RelDataType rowType() { return requireNonNull(rowType, "rowType"); } diff --git a/core/src/main/java/org/apache/calcite/rel/core/Correlate.java b/core/src/main/java/org/apache/calcite/rel/core/Correlate.java index 55351399c56f..06736eee2d12 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/Correlate.java +++ b/core/src/main/java/org/apache/calcite/rel/core/Correlate.java @@ -91,7 +91,7 @@ public abstract class Correlate extends BiRel implements Hintable { * @param requiredColumns Set of columns that are used by correlation * @param joinType Join type */ - @SuppressWarnings("method.invocation.invalid") + @SuppressWarnings("NullAway") protected Correlate( RelOptCluster cluster, RelTraitSet traitSet, diff --git a/core/src/main/java/org/apache/calcite/rel/core/Filter.java b/core/src/main/java/org/apache/calcite/rel/core/Filter.java index f1fd9a6b1195..b9440033dd13 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/Filter.java +++ b/core/src/main/java/org/apache/calcite/rel/core/Filter.java @@ -41,7 +41,6 @@ import com.google.common.collect.ImmutableList; import org.apiguardian.api.API; -import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf; import org.jspecify.annotations.Nullable; import java.util.List; @@ -78,7 +77,7 @@ public abstract class Filter extends SingleRel implements Hintable { * @param condition boolean expression which determines whether a row is * allowed to pass */ - @SuppressWarnings("method.invocation.invalid") + @SuppressWarnings("NullAway") protected Filter( RelOptCluster cluster, RelTraitSet traits, @@ -199,7 +198,6 @@ public static double estimateFilteredRows(RelNode child, RexNode condition) { } @API(since = "1.24", status = API.Status.INTERNAL) - @EnsuresNonNullIf(expression = "#1", result = true) protected boolean deepEquals0(@Nullable Object obj) { if (this == obj) { return true; diff --git a/core/src/main/java/org/apache/calcite/rel/core/Join.java b/core/src/main/java/org/apache/calcite/rel/core/Join.java index a5ecdaea4a53..f0b2d1d68cf9 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/Join.java +++ b/core/src/main/java/org/apache/calcite/rel/core/Join.java @@ -42,7 +42,6 @@ import com.google.common.collect.ImmutableSet; import org.apiguardian.api.API; -import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf; import org.jspecify.annotations.Nullable; import java.util.Collections; @@ -241,7 +240,6 @@ public static double estimateJoinedRows( } @API(since = "1.24", status = API.Status.INTERNAL) - @EnsuresNonNullIf(expression = "#1", result = true) protected boolean deepEquals0(@Nullable Object obj) { if (this == obj) { return true; diff --git a/core/src/main/java/org/apache/calcite/rel/core/Project.java b/core/src/main/java/org/apache/calcite/rel/core/Project.java index 2fd3ab7fdc11..567f07359f5e 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/Project.java +++ b/core/src/main/java/org/apache/calcite/rel/core/Project.java @@ -49,7 +49,6 @@ import com.google.common.collect.ImmutableSet; import org.apiguardian.api.API; -import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf; import org.jspecify.annotations.Nullable; import java.util.HashSet; @@ -89,7 +88,7 @@ public abstract class Project extends SingleRel implements Hintable { * @param variableSet Correlation variables set by this relational expression * to be used by nested expressions */ - @SuppressWarnings("method.invocation.invalid") + @SuppressWarnings("NullAway") protected Project( RelOptCluster cluster, RelTraitSet traits, @@ -344,7 +343,6 @@ private static int countTrivial(List refs) { } @API(since = "1.24", status = API.Status.INTERNAL) - @EnsuresNonNullIf(expression = "#1", result = true) protected boolean deepEquals0(@Nullable Object obj) { if (this == obj) { return true; diff --git a/core/src/main/java/org/apache/calcite/rel/core/Snapshot.java b/core/src/main/java/org/apache/calcite/rel/core/Snapshot.java index 89df44b05663..962b05a570d7 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/Snapshot.java +++ b/core/src/main/java/org/apache/calcite/rel/core/Snapshot.java @@ -80,7 +80,7 @@ public Snapshot(RelInput input) { * @param period Timestamp expression which as the table was at the given * time in the past */ - @SuppressWarnings("method.invocation.invalid") + @SuppressWarnings("NullAway") protected Snapshot(RelOptCluster cluster, RelTraitSet traitSet, List hints, RelNode input, RexNode period) { super(cluster, traitSet, input); @@ -98,7 +98,7 @@ protected Snapshot(RelOptCluster cluster, RelTraitSet traitSet, List hi * @param period Timestamp expression which as the table was at the given * time in the past */ - @SuppressWarnings("method.invocation.invalid") + @SuppressWarnings("NullAway") protected Snapshot( RelOptCluster cluster, RelTraitSet traitSet, RelNode input, RexNode period) { this(cluster, traitSet, ImmutableList.of(), input, period); diff --git a/core/src/main/java/org/apache/calcite/rel/core/TableModify.java b/core/src/main/java/org/apache/calcite/rel/core/TableModify.java index 6c222c7079e7..1c2ddae2e2f6 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/TableModify.java +++ b/core/src/main/java/org/apache/calcite/rel/core/TableModify.java @@ -16,6 +16,7 @@ */ package org.apache.calcite.rel.core; +import org.apache.calcite.linq4j.annotations.MonotonicNonNull; import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.plan.RelOptCost; import org.apache.calcite.plan.RelOptPlanner; @@ -38,7 +39,6 @@ import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.type.SqlTypeUtil; -import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/rel/core/Uncollect.java b/core/src/main/java/org/apache/calcite/rel/core/Uncollect.java index 039a17efbfcf..69125d76b16a 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/Uncollect.java +++ b/core/src/main/java/org/apache/calcite/rel/core/Uncollect.java @@ -115,7 +115,7 @@ public Uncollect(RelOptCluster cluster, RelTraitSet traitSet, RelNode input, * @param isOuter If true, an empty or NULL collection yields one row of * NULLs (LEFT JOIN); if false, it yields no rows (INNER) */ - @SuppressWarnings("method.invocation.invalid") + @SuppressWarnings("NullAway") public Uncollect(RelOptCluster cluster, RelTraitSet traitSet, RelNode input, boolean withOrdinality, List itemAliases, boolean expandStructFields, boolean isOuter) { diff --git a/core/src/main/java/org/apache/calcite/rel/core/Values.java b/core/src/main/java/org/apache/calcite/rel/core/Values.java index 2b3777a8c3b6..bbceb03d6ed3 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/Values.java +++ b/core/src/main/java/org/apache/calcite/rel/core/Values.java @@ -84,7 +84,7 @@ public abstract class Values extends AbstractRelNode implements Hintable { * list contains tuples; each inner list is one tuple; all * tuples must be of same length, conforming to rowType */ - @SuppressWarnings("method.invocation.invalid") + @SuppressWarnings("NullAway") protected Values( RelOptCluster cluster, List hints, @@ -111,7 +111,7 @@ protected Values( * list contains tuples; each inner list is one tuple; all * tuples must be of same length, conforming to rowType */ - @SuppressWarnings("method.invocation.invalid") + @SuppressWarnings("NullAway") protected Values( RelOptCluster cluster, RelDataType rowType, diff --git a/core/src/main/java/org/apache/calcite/rel/core/Window.java b/core/src/main/java/org/apache/calcite/rel/core/Window.java index ed28583a9aaf..c17b03441f81 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/Window.java +++ b/core/src/main/java/org/apache/calcite/rel/core/Window.java @@ -17,6 +17,7 @@ package org.apache.calcite.rel.core; import org.apache.calcite.linq4j.Ord; +import org.apache.calcite.linq4j.annotations.RequiresNonNull; import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.plan.RelOptCost; import org.apache.calcite.plan.RelOptPlanner; @@ -51,8 +52,6 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.initialization.qual.UnderInitialization; -import org.checkerframework.checker.nullness.qual.RequiresNonNull; import org.jspecify.annotations.Nullable; import java.util.AbstractList; @@ -295,7 +294,7 @@ public Group( } @RequiresNonNull({"keys", "orderKeys", "lowerBound", "upperBound", "aggCalls"}) - private String computeString(@UnderInitialization Group this) { + private String computeString() { final StringBuilder buf = new StringBuilder("window("); final int i = buf.length(); if (!keys.isEmpty()) { diff --git a/core/src/main/java/org/apache/calcite/rel/externalize/RelJson.java b/core/src/main/java/org/apache/calcite/rel/externalize/RelJson.java index 8029933a2a3a..8d87a87fc3a1 100644 --- a/core/src/main/java/org/apache/calcite/rel/externalize/RelJson.java +++ b/core/src/main/java/org/apache/calcite/rel/externalize/RelJson.java @@ -87,7 +87,6 @@ import com.google.common.collect.Range; import com.google.common.collect.RangeSet; -import org.checkerframework.checker.nullness.qual.PolyNull; import org.jspecify.annotations.NonNull; import org.jspecify.annotations.Nullable; @@ -195,7 +194,7 @@ private JsonBuilder jsonBuilder() { } @SuppressWarnings("unchecked") - private static T get(Map map, + private static T get(Map map, String key) { return (T) requireNonNull(map.get(key), () -> "entry for key " + key); } @@ -757,7 +756,7 @@ public RexNode toRex(RelOptCluster cluster, Object o) { } @SuppressWarnings({"rawtypes", "unchecked"}) - @PolyNull RexNode toRex(RelInput relInput, @PolyNull Object o) { + @Nullable RexNode toRex(RelInput relInput, @Nullable Object o) { final RelOptCluster cluster = relInput.getCluster(); final RexBuilder rexBuilder = cluster.getRexBuilder(); if (o == null) { diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/CachingRelMetadataProvider.java b/core/src/main/java/org/apache/calcite/rel/metadata/CachingRelMetadataProvider.java index c835a7118d18..a9c7d1f726fa 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/CachingRelMetadataProvider.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/CachingRelMetadataProvider.java @@ -61,7 +61,7 @@ public CachingRelMetadataProvider( //~ Methods ---------------------------------------------------------------- @Deprecated // to be removed before 2.0 - @Override public <@Nullable M extends @Nullable Metadata> @Nullable UnboundMetadata apply( + @Override public @Nullable UnboundMetadata apply( Class relClass, final Class metadataClass) { final UnboundMetadata function = diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/ChainedRelMetadataProvider.java b/core/src/main/java/org/apache/calcite/rel/metadata/ChainedRelMetadataProvider.java index 116e4d36f3cf..aac5f5b09daa 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/ChainedRelMetadataProvider.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/ChainedRelMetadataProvider.java @@ -51,7 +51,7 @@ public class ChainedRelMetadataProvider implements RelMetadataProvider { /** * Creates a chain. */ - @SuppressWarnings("argument.type.incompatible") + @SuppressWarnings("NullAway") protected ChainedRelMetadataProvider( ImmutableList providers) { this.providers = providers; @@ -71,7 +71,7 @@ protected ChainedRelMetadataProvider( } @Deprecated // to be removed before 2.0 - @Override public <@Nullable M extends @Nullable Metadata> @Nullable UnboundMetadata apply( + @Override public @Nullable UnboundMetadata apply( Class relClass, final Class metadataClass) { final List> functions = new ArrayList<>(); diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/JaninoRelMetadataProvider.java b/core/src/main/java/org/apache/calcite/rel/metadata/JaninoRelMetadataProvider.java index 199440d444a6..b7df1a707856 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/JaninoRelMetadataProvider.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/JaninoRelMetadataProvider.java @@ -106,7 +106,7 @@ private static CacheBuilder maxSize(CacheBuilder builder, } @Deprecated // to be removed before 2.0 - @Override public <@Nullable M extends @Nullable Metadata> UnboundMetadata apply( + @Override public UnboundMetadata apply( Class relClass, Class metadataClass) { throw new UnsupportedOperationException(); } diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/MetadataFactory.java b/core/src/main/java/org/apache/calcite/rel/metadata/MetadataFactory.java index 63c8cb2f1227..32f5f4ba9c74 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/MetadataFactory.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/MetadataFactory.java @@ -42,6 +42,6 @@ public interface MetadataFactory { * @param metadataClazz Metadata class * @return Metadata bound to {@code rel} and {@code query} */ - <@Nullable M extends @Nullable Metadata> M query(RelNode rel, RelMetadataQuery mq, + M query(RelNode rel, RelMetadataQuery mq, Class metadataClazz); } diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/MetadataFactoryImpl.java b/core/src/main/java/org/apache/calcite/rel/metadata/MetadataFactoryImpl.java index 854c3784f8f6..18444466ea06 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/MetadataFactoryImpl.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/MetadataFactoryImpl.java @@ -64,7 +64,7 @@ public MetadataFactoryImpl(RelMetadataProvider provider) { }); } - @Override public <@Nullable M extends @Nullable Metadata> M query( + @Override public M query( RelNode rel, RelMetadataQuery mq, Class metadataClazz) { try { diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/ReflectiveRelMetadataProvider.java b/core/src/main/java/org/apache/calcite/rel/metadata/ReflectiveRelMetadataProvider.java index e8129979627c..439be4e6c33f 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/ReflectiveRelMetadataProvider.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/ReflectiveRelMetadataProvider.java @@ -267,7 +267,7 @@ private static boolean couldImplement(Method handlerMethod, Method method) { //~ Methods ---------------------------------------------------------------- @Deprecated // to be removed before 2.0 - @Override public <@Nullable M extends @Nullable Metadata> @Nullable UnboundMetadata apply( + @Override public @Nullable UnboundMetadata apply( Class relClass, Class metadataClass) { if (metadataClass == metadataClass0) { return apply(relClass); @@ -278,7 +278,7 @@ private static boolean couldImplement(Method handlerMethod, Method method) { @SuppressWarnings({ "unchecked", "SuspiciousMethodCalls" }) @Deprecated // to be removed before 2.0 - public <@Nullable M extends @Nullable Metadata> @Nullable UnboundMetadata apply( + public @Nullable UnboundMetadata apply( Class relClass) { List> newSources = new ArrayList<>(); for (;;) { diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdColumnOrigins.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdColumnOrigins.java index 3d8fbc1c43bb..676263f70806 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdColumnOrigins.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdColumnOrigins.java @@ -42,7 +42,6 @@ import org.apache.calcite.rex.RexVisitorImpl; import org.apache.calcite.util.Util; -import org.checkerframework.checker.nullness.qual.PolyNull; import org.jspecify.annotations.Nullable; import java.util.ArrayList; @@ -277,8 +276,8 @@ private RelMdColumnOrigins() {} return set; } - private static @PolyNull Set createDerivedColumnOrigins( - @PolyNull Set inputSet) { + private static @Nullable Set createDerivedColumnOrigins( + @Nullable Set inputSet) { if (inputSet == null) { return null; } diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdExpressionLineage.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdExpressionLineage.java index bf5888615516..ba2f43940df1 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdExpressionLineage.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdExpressionLineage.java @@ -51,7 +51,6 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Multimap; -import org.checkerframework.checker.nullness.qual.KeyFor; import org.jspecify.annotations.Nullable; import java.util.ArrayList; @@ -501,7 +500,7 @@ protected RelMdExpressionLineage() {} private static Set createAllPossibleExpressions(RexBuilder rexBuilder, RexNode expr, ImmutableBitSet predFieldsUsed, Map> mapping, Map singleMapping) { - final @KeyFor("mapping") RexInputRef inputRef = mapping.keySet().iterator().next(); + final RexInputRef inputRef = mapping.keySet().iterator().next(); final Set replacements = requireNonNull(mapping.remove(inputRef), () -> "mapping.remove(inputRef) is null for " + inputRef); diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdPercentageOriginalRows.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdPercentageOriginalRows.java index 63e7123552ef..66b56cad6cac 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdPercentageOriginalRows.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdPercentageOriginalRows.java @@ -27,7 +27,6 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.PolyNull; import org.jspecify.annotations.Nullable; import java.util.List; @@ -235,9 +234,9 @@ public Double getPercentageOriginalRows(Union rel, RelMetadataQuery mq) { return rel.computeSelfCost(rel.getCluster().getPlanner(), mq); } - private static @PolyNull Double quotientForPercentage( - @PolyNull Double numerator, - @PolyNull Double denominator) { + private static @Nullable Double quotientForPercentage( + @Nullable Double numerator, + @Nullable Double denominator) { if ((numerator == null) || (denominator == null)) { return null; } diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdUtil.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdUtil.java index d8ad2f9a1fc1..baf9c6d5cdea 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdUtil.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdUtil.java @@ -49,7 +49,6 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.PolyNull; import org.jspecify.annotations.Nullable; import java.math.BigDecimal; @@ -315,9 +314,9 @@ public static void setLeftRightBitmaps( * @return the expected number of distinct values, or null if either argument * is null */ - public static @PolyNull Double numDistinctVals( - @PolyNull Double domainSize, - @PolyNull Double numSelected) { + public static @Nullable Double numDistinctVals( + @Nullable Double domainSize, + @Nullable Double numSelected) { if ((domainSize == null) || (numSelected == null)) { return domainSize; } @@ -911,7 +910,7 @@ public static double estimateFilteredRows(RelNode child, RexProgram program, public static double estimateFilteredRows(RelNode child, @Nullable RexNode condition, RelMetadataQuery mq) { - @SuppressWarnings("unboxing.of.nullable") + @SuppressWarnings("NullAway") double result = multiply(mq.getRowCount(child), mq.getSelectivity(child, condition)); return result; @@ -1081,7 +1080,7 @@ private static boolean alreadySmaller(RelMetadataQuery mq, RelNode input, *

      Throws if {@code result} is not null, not in range 0 to 1, * and assertions are enabled. */ - public static @PolyNull Double validatePercentage(@PolyNull Double result) { + public static @Nullable Double validatePercentage(@Nullable Double result) { assert isPercentage(result, true); return result; } @@ -1117,7 +1116,7 @@ private static boolean isPercentage(@Nullable Double result, boolean fail) { * @return the corrected value from the {@code result} * @throws AssertionError if the {@code result} is negative */ - public static @PolyNull Double validateResult(@PolyNull Double result) { + public static @Nullable Double validateResult(@Nullable Double result) { if (result == null) { return null; } diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMetadataProvider.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMetadataProvider.java index 562640ae78d7..a49b1f9c39cd 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMetadataProvider.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMetadataProvider.java @@ -68,7 +68,7 @@ public interface RelMetadataProvider { * provider cannot supply metadata of this type */ @Deprecated // to be removed before 2.0 - <@Nullable M extends @Nullable Metadata> @Nullable UnboundMetadata apply( + @Nullable UnboundMetadata apply( Class relClass, Class metadataClass); @Deprecated // to be removed before 2.0 diff --git a/core/src/main/java/org/apache/calcite/rel/mutable/MutableBiRel.java b/core/src/main/java/org/apache/calcite/rel/mutable/MutableBiRel.java index 30c6ff02188f..b807ff8430fd 100644 --- a/core/src/main/java/org/apache/calcite/rel/mutable/MutableBiRel.java +++ b/core/src/main/java/org/apache/calcite/rel/mutable/MutableBiRel.java @@ -28,7 +28,7 @@ abstract class MutableBiRel extends MutableRel { protected MutableRel left; protected MutableRel right; - @SuppressWarnings("initialization.invalid.field.write.initialized") + @SuppressWarnings("NullAway") protected MutableBiRel(MutableRelType type, RelOptCluster cluster, RelDataType rowType, MutableRel left, MutableRel right) { super(cluster, rowType, type); diff --git a/core/src/main/java/org/apache/calcite/rel/mutable/MutableMultiRel.java b/core/src/main/java/org/apache/calcite/rel/mutable/MutableMultiRel.java index 7921f192549c..2d366c26f838 100644 --- a/core/src/main/java/org/apache/calcite/rel/mutable/MutableMultiRel.java +++ b/core/src/main/java/org/apache/calcite/rel/mutable/MutableMultiRel.java @@ -28,7 +28,7 @@ abstract class MutableMultiRel extends MutableRel { protected final List inputs; - @SuppressWarnings("initialization.invalid.field.write.initialized") + @SuppressWarnings("NullAway") protected MutableMultiRel(RelOptCluster cluster, RelDataType rowType, MutableRelType type, List inputs) { super(cluster, rowType, type); diff --git a/core/src/main/java/org/apache/calcite/rel/mutable/MutableSingleRel.java b/core/src/main/java/org/apache/calcite/rel/mutable/MutableSingleRel.java index 995dc889c26e..5cb19c95b6f2 100644 --- a/core/src/main/java/org/apache/calcite/rel/mutable/MutableSingleRel.java +++ b/core/src/main/java/org/apache/calcite/rel/mutable/MutableSingleRel.java @@ -26,7 +26,7 @@ abstract class MutableSingleRel extends MutableRel { protected MutableRel input; - @SuppressWarnings("initialization.invalid.field.write.initialized") + @SuppressWarnings("NullAway") protected MutableSingleRel(MutableRelType type, RelDataType rowType, MutableRel input) { super(input.cluster, rowType, type); diff --git a/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java b/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java index 0b4270c0d793..2d0f8c56b231 100644 --- a/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java +++ b/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java @@ -146,7 +146,7 @@ public RelToSqlConverter(SqlDialect dialect) { /** Creates a RelToSqlConverter; if {@code preserveLiteralTypes}, literals * whose type is not implied by their SQL text are wrapped in CASTs; * see {@link SqlImplementor#toSql(RexProgram, RexLiteral, SqlDialect)}. */ - @SuppressWarnings("argument.type.incompatible") + @SuppressWarnings("NullAway") public RelToSqlConverter(SqlDialect dialect, boolean preserveLiteralTypes) { super(dialect, preserveLiteralTypes); dispatcher = diff --git a/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java b/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java index c9b82b3eecdb..794bf84e054e 100644 --- a/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java +++ b/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java @@ -124,7 +124,6 @@ import com.google.common.collect.Range; import com.google.common.collect.RangeSet; -import org.checkerframework.checker.initialization.qual.UnknownInitialization; import org.jspecify.annotations.Nullable; import java.math.BigDecimal; @@ -2305,7 +2304,6 @@ private void restoreOutputFieldNames(RelNode rel, SqlSelect select, /** Returns whether a new sub-query is required. */ private boolean needNewSubQuery( - @UnknownInitialization Result this, RelNode rel, List clauses, Set expectedClauses) { if (clauses.isEmpty()) { @@ -2415,8 +2413,7 @@ && hasGroupByLiteral(agg)) { * Returns whether any grouping key of {@code aggregate} is represented by * a literal expression in this result's {@code SELECT} list. */ - private boolean hasGroupByLiteral( - @UnknownInitialization Result this, Aggregate aggregate) { + private boolean hasGroupByLiteral(Aggregate aggregate) { if (!(node instanceof SqlSelect)) { return false; } @@ -2445,7 +2442,7 @@ private boolean hasGroupByLiteral( * * @param sqlNode SqlNode to check */ - private boolean hasSortByOrdinal(@UnknownInitialization Result this, + private boolean hasSortByOrdinal( @Nullable SqlNode sqlNode) { if (sqlNode == null) { return false; @@ -2473,7 +2470,7 @@ private boolean hasSortByOrdinal(@UnknownInitialization Result this, return false; } - private boolean containsOver(@UnknownInitialization Result this, + private boolean containsOver( @Nullable SqlNode node) { if (node == null) { return false; @@ -2506,7 +2503,6 @@ private boolean containsOver(@UnknownInitialization Result this, * @param operandPredicate Predicate for the nested operands * @return whether any nested operands matches the predicate */ private boolean hasNested( - @UnknownInitialization Result this, Aggregate aggregate, Predicate operandPredicate) { final boolean[] result = {false}; diff --git a/core/src/main/java/org/apache/calcite/rel/rules/AggregateExpandDistinctAggregatesRule.java b/core/src/main/java/org/apache/calcite/rel/rules/AggregateExpandDistinctAggregatesRule.java index b9abcc7a1b8b..6fc0915bdb29 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/AggregateExpandDistinctAggregatesRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/AggregateExpandDistinctAggregatesRule.java @@ -279,7 +279,7 @@ public AggregateExpandDistinctAggregatesRule( doRewrite(relBuilder, aggregate, n++, argList.left, argList.right, refs); } // It is assumed doRewrite above replaces nulls in refs - @SuppressWarnings("assignment.type.incompatible") + @SuppressWarnings("NullAway") List nonNullRefs = refs; relBuilder.project(nonNullRefs, fieldNames); call.transformTo(relBuilder.build()); diff --git a/core/src/main/java/org/apache/calcite/rel/rules/CalcRelSplitter.java b/core/src/main/java/org/apache/calcite/rel/rules/CalcRelSplitter.java index 88704f914c0b..f054c66b8ced 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/CalcRelSplitter.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/CalcRelSplitter.java @@ -1051,7 +1051,7 @@ private static class HighestUsageFinder extends RexVisitorImpl { continue; } currentLevel = exprLevels[i]; - @SuppressWarnings("argument.type.incompatible") + @SuppressWarnings("NullAway") final Void unused = exprs[i].accept(this); } } diff --git a/core/src/main/java/org/apache/calcite/rel/rules/LoptJoinTree.java b/core/src/main/java/org/apache/calcite/rel/rules/LoptJoinTree.java index dc991a8c6fc3..7229f534a857 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/LoptJoinTree.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/LoptJoinTree.java @@ -19,9 +19,6 @@ import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Join; -import org.checkerframework.checker.initialization.qual.NotOnlyInitialized; -import org.checkerframework.checker.initialization.qual.UnderInitialization; - import java.util.ArrayList; import java.util.List; @@ -40,7 +37,6 @@ public class LoptJoinTree { //~ Instance fields -------------------------------------------------------- - @NotOnlyInitialized private final BinaryTree factorTree; private final RelNode joinTree; private final boolean removableSelfJoin; @@ -53,7 +49,7 @@ public class LoptJoinTree { * @param joinTree RelNode corresponding to the single node * @param factorId factor id of the node */ - @SuppressWarnings("argument.type.incompatible") + @SuppressWarnings("NullAway") public LoptJoinTree(RelNode joinTree, int factorId) { this.joinTree = joinTree; this.factorTree = new Leaf(factorId, this); @@ -159,10 +155,9 @@ public boolean isRemovableSelfJoin() { * track of the parent LoptJoinTree object associated with the binary tree. */ protected abstract static class BinaryTree { - @NotOnlyInitialized private final LoptJoinTree parent; - protected BinaryTree(@UnderInitialization LoptJoinTree parent) { + protected BinaryTree(LoptJoinTree parent) { this.parent = parent; } @@ -177,7 +172,7 @@ public LoptJoinTree getParent() { protected static class Leaf extends BinaryTree { private final int id; - public Leaf(int rootId, @UnderInitialization LoptJoinTree parent) { + public Leaf(int rootId, LoptJoinTree parent) { super(parent); this.id = rootId; } @@ -197,7 +192,7 @@ protected static class Node extends BinaryTree { private final BinaryTree left; private final BinaryTree right; - public Node(BinaryTree left, BinaryTree right, @UnderInitialization LoptJoinTree parent) { + public Node(BinaryTree left, BinaryTree right, LoptJoinTree parent) { super(parent); this.left = requireNonNull(left, "left"); this.right = requireNonNull(right, "right"); diff --git a/core/src/main/java/org/apache/calcite/rel/rules/LoptMultiJoin.java b/core/src/main/java/org/apache/calcite/rel/rules/LoptMultiJoin.java index ae6a91d9827e..979dff18ffd4 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/LoptMultiJoin.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/LoptMultiJoin.java @@ -16,6 +16,8 @@ */ package org.apache.calcite.rel.rules; +import org.apache.calcite.linq4j.annotations.MonotonicNonNull; +import org.apache.calcite.linq4j.annotations.RequiresNonNull; import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.JoinRelType; @@ -35,10 +37,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; -import org.checkerframework.checker.initialization.qual.UnderInitialization; -import org.checkerframework.checker.initialization.qual.UnknownInitialization; -import org.checkerframework.checker.nullness.qual.MonotonicNonNull; -import org.checkerframework.checker.nullness.qual.RequiresNonNull; import org.jspecify.annotations.Nullable; import java.util.ArrayList; @@ -139,12 +137,12 @@ public class LoptMultiJoin { * Bitmap indicating which factors each factor references in join filters * that correspond to comparisons. */ - ImmutableBitSet @MonotonicNonNull [] factorsRefByFactor; + @MonotonicNonNull ImmutableBitSet[] factorsRefByFactor; /** * Weights of each factor combination. */ - int @MonotonicNonNull [][] factorWeights; + @MonotonicNonNull int[][] factorWeights; /** * Type factory. @@ -448,7 +446,6 @@ public void setJoinRemovalSemiJoin(int dimIdx, LogicalJoin semiJoin) { */ @RequiresNonNull({"joinStart", "nFieldsInJoinFactor"}) ImmutableBitSet getJoinFilterFactorBitmap( - @UnderInitialization LoptMultiJoin this, RexNode joinFilter, boolean setFields) { ImmutableBitSet fieldRefBitmap = fieldBitmap(joinFilter); @@ -470,8 +467,7 @@ private static ImmutableBitSet fieldBitmap(RexNode joinFilter) { * references. */ @RequiresNonNull({"allJoinFilters", "joinStart", "nFieldsInJoinFactor"}) - private void setJoinFilterRefs( - @UnderInitialization LoptMultiJoin this) { + private void setJoinFilterRefs() { ListIterator filterIter = allJoinFilters.listIterator(); while (filterIter.hasNext()) { RexNode joinFilter = filterIter.next(); @@ -497,7 +493,6 @@ private void setJoinFilterRefs( */ @RequiresNonNull({"joinStart", "nFieldsInJoinFactor"}) private ImmutableBitSet factorBitmap( - @UnknownInitialization LoptMultiJoin this, ImmutableBitSet fieldRefBitmap) { ImmutableBitSet.Builder factorRefBitmap = ImmutableBitSet.builder(); for (int field : fieldRefBitmap) { @@ -516,7 +511,6 @@ private ImmutableBitSet factorBitmap( */ @RequiresNonNull({"joinStart", "nFieldsInJoinFactor"}) public int findRef( - @UnknownInitialization LoptMultiJoin this, int rexInputRef) { for (int i = 0; i < nJoinFactors; i++) { if ((rexInputRef >= joinStart[i]) diff --git a/core/src/main/java/org/apache/calcite/rel/rules/LoptOptimizeJoinRule.java b/core/src/main/java/org/apache/calcite/rel/rules/LoptOptimizeJoinRule.java index 662ada2bf637..f3cd00bd2385 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/LoptOptimizeJoinRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/LoptOptimizeJoinRule.java @@ -47,7 +47,6 @@ import org.apache.calcite.util.Pair; import org.apache.calcite.util.mapping.IntPair; -import org.checkerframework.checker.nullness.qual.KeyFor; import org.immutables.value.Value; import org.jspecify.annotations.Nullable; @@ -308,15 +307,15 @@ private static void findRemovableSelfJoins(RelMetadataQuery mq, LoptMultiJoin mu // self-join. final List repeatedTables = new ArrayList<>(); final Map selfJoinPairs = new HashMap<>(); - @KeyFor("simpleFactors") Integer [] factors = + Integer [] factors = new TreeSet<>(simpleFactors.keySet()).toArray(new Integer[0]); for (int i = 0; i < factors.length; i++) { if (repeatedTables.contains(simpleFactors.get(factors[i]))) { continue; } for (int j = i + 1; j < factors.length; j++) { - @KeyFor("simpleFactors") int leftFactor = factors[i]; - @KeyFor("simpleFactors") int rightFactor = factors[j]; + int leftFactor = factors[i]; + int rightFactor = factors[j]; if (simpleFactors.get(leftFactor).getQualifiedName().equals( simpleFactors.get(rightFactor).getQualifiedName())) { selfJoinPairs.put(leftFactor, rightFactor); diff --git a/core/src/main/java/org/apache/calcite/rel/rules/ReduceDecimalsRule.java b/core/src/main/java/org/apache/calcite/rel/rules/ReduceDecimalsRule.java index 54aba9f91aba..07b69426be36 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/ReduceDecimalsRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/ReduceDecimalsRule.java @@ -17,6 +17,7 @@ package org.apache.calcite.rel.rules; import org.apache.calcite.linq4j.Ord; +import org.apache.calcite.linq4j.annotations.MonotonicNonNull; import org.apache.calcite.plan.Convention; import org.apache.calcite.plan.RelOptRuleCall; import org.apache.calcite.plan.RelRule; @@ -43,7 +44,6 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.immutables.value.Value; import org.jspecify.annotations.Nullable; diff --git a/core/src/main/java/org/apache/calcite/rel/type/DynamicRecordTypeImpl.java b/core/src/main/java/org/apache/calcite/rel/type/DynamicRecordTypeImpl.java index cd601e64c352..2d35b8021618 100644 --- a/core/src/main/java/org/apache/calcite/rel/type/DynamicRecordTypeImpl.java +++ b/core/src/main/java/org/apache/calcite/rel/type/DynamicRecordTypeImpl.java @@ -39,7 +39,7 @@ public class DynamicRecordTypeImpl extends DynamicRecordType { private final RelDataTypeHolder holder; /** Creates a DynamicRecordTypeImpl. */ - @SuppressWarnings("method.invocation.invalid") + @SuppressWarnings("NullAway") public DynamicRecordTypeImpl(RelDataTypeFactory typeFactory) { this.holder = new RelDataTypeHolder(typeFactory); computeDigest(); diff --git a/core/src/main/java/org/apache/calcite/rel/type/RelCrossType.java b/core/src/main/java/org/apache/calcite/rel/type/RelCrossType.java index 5423676b6492..f6b3cb67fefa 100644 --- a/core/src/main/java/org/apache/calcite/rel/type/RelCrossType.java +++ b/core/src/main/java/org/apache/calcite/rel/type/RelCrossType.java @@ -39,7 +39,7 @@ public class RelCrossType extends RelDataTypeImpl { * Creates a cartesian product type. This should only be called from a * factory method. */ - @SuppressWarnings("method.invocation.invalid") + @SuppressWarnings("NullAway") public RelCrossType( List types, List fields) { diff --git a/core/src/main/java/org/apache/calcite/rel/type/RelDataType.java b/core/src/main/java/org/apache/calcite/rel/type/RelDataType.java index 50c212d16c74..0f3190a0233f 100644 --- a/core/src/main/java/org/apache/calcite/rel/type/RelDataType.java +++ b/core/src/main/java/org/apache/calcite/rel/type/RelDataType.java @@ -24,7 +24,6 @@ import org.apache.calcite.sql.type.SqlTypeUtil; import org.apiguardian.api.API; -import org.checkerframework.dataflow.qual.Pure; import org.jspecify.annotations.Nullable; import java.nio.charset.Charset; @@ -50,7 +49,6 @@ public interface RelDataType { * @return whether this type has fields; examples include rows and * user-defined structured types in SQL, and classes in Java */ - @Pure boolean isStruct(); // NOTE jvs 17-Dec-2004: once we move to Java generics, getFieldList() @@ -116,7 +114,6 @@ public interface RelDataType { * * @return whether type allows null values */ - @Pure boolean isNullable(); /** @@ -124,7 +121,6 @@ public interface RelDataType { * * @return canonical type descriptor for components */ - @Pure @Nullable RelDataType getComponentType(); /** @@ -156,7 +152,6 @@ public interface RelDataType { * * @return charset of type */ - @Pure @Nullable Charset getCharset(); /** @@ -165,7 +160,6 @@ public interface RelDataType { * * @return collation of type */ - @Pure @Nullable SqlCollation getCollation(); /** @@ -174,7 +168,6 @@ public interface RelDataType { * * @return interval qualifier */ - @Pure @Nullable SqlIntervalQualifier getIntervalQualifier(); /** @@ -218,7 +211,6 @@ public interface RelDataType { * * @return SqlIdentifier, or null if this is not an SQL type */ - @Pure @Nullable SqlIdentifier getSqlIdentifier(); /** diff --git a/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeFactoryImpl.java b/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeFactoryImpl.java index 42d99c140a77..4b873b1ecaaf 100644 --- a/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeFactoryImpl.java +++ b/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeFactoryImpl.java @@ -656,7 +656,7 @@ public JavaType( this(clazz, nullable, null, null); } - @SuppressWarnings("argument.type.incompatible") + @SuppressWarnings("NullAway") public JavaType( Class clazz, boolean nullable, diff --git a/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeField.java b/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeField.java index da4e079a7aef..9683769786ad 100644 --- a/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeField.java +++ b/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeField.java @@ -36,7 +36,7 @@ public interface RelDataTypeField extends Map.Entry { * @deprecated Use {@code RelDataTypeField::getIndex} */ @Deprecated // to be removed before 2.0 - @SuppressWarnings("nullability") + @SuppressWarnings("NullAway") class ToFieldIndex implements com.google.common.base.Function { @Override public Integer apply(RelDataTypeField o) { @@ -51,7 +51,7 @@ class ToFieldIndex * @deprecated Use {@code RelDataTypeField::getName} */ @Deprecated // to be removed before 2.0 - @SuppressWarnings("nullability") + @SuppressWarnings("NullAway") class ToFieldName implements com.google.common.base.Function { @Override public String apply(RelDataTypeField o) { diff --git a/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeImpl.java b/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeImpl.java index b4e0c32345b9..52ef5f4d55d2 100644 --- a/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeImpl.java +++ b/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeImpl.java @@ -29,7 +29,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; -import org.checkerframework.checker.initialization.qual.UnknownInitialization; import org.jspecify.annotations.Nullable; import java.io.Serializable; @@ -353,8 +352,8 @@ protected abstract void generateTypeString( * This should be called in every non-abstract subclass * constructor once the type is fully defined. */ - @SuppressWarnings("method.invocation.invalid") - protected void computeDigest(@UnknownInitialization RelDataTypeImpl this) { + @SuppressWarnings("NullAway") + protected void computeDigest() { digest = null; innerDigest = new InnerRelDataTypeDigest(); if (!CalciteSystemProperty.DISABLE_GENERATE_REL_DATA_TYPE_DIGEST_STRING.value()) { diff --git a/core/src/main/java/org/apache/calcite/rex/RexBiVisitorImpl.java b/core/src/main/java/org/apache/calcite/rex/RexBiVisitorImpl.java index 527da65b52b7..1ae39484bfc9 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexBiVisitorImpl.java +++ b/core/src/main/java/org/apache/calcite/rex/RexBiVisitorImpl.java @@ -25,7 +25,7 @@ * @param Return type from each {@code visitXxx} method * @param

      Payload type */ -public class RexBiVisitorImpl<@Nullable R, P> implements RexBiVisitor { +public class RexBiVisitorImpl implements RexBiVisitor { //~ Instance fields -------------------------------------------------------- protected final boolean deep; diff --git a/core/src/main/java/org/apache/calcite/rex/RexBuilder.java b/core/src/main/java/org/apache/calcite/rex/RexBuilder.java index b146de2042cd..e002c84a7821 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexBuilder.java +++ b/core/src/main/java/org/apache/calcite/rex/RexBuilder.java @@ -67,7 +67,6 @@ import com.google.common.collect.RangeSet; import com.google.common.collect.TreeRangeSet; -import org.checkerframework.checker.nullness.qual.PolyNull; import org.jspecify.annotations.Nullable; import org.locationtech.jts.geom.Geometry; @@ -138,7 +137,7 @@ public class RexBuilder { * * @param typeFactory Type factory */ - @SuppressWarnings("method.invocation.invalid") + @SuppressWarnings("NullAway") public RexBuilder(RelDataTypeFactory typeFactory) { this.typeFactory = typeFactory; this.booleanTrue = @@ -2396,7 +2395,7 @@ public RexNode makeLambdaCall(RexNode expr, List parameters) { * {@link org.apache.calcite.rex.RexLiteral#valueMatchesType}. * *

      Returns null if and only if {@code o} is null. */ - private @PolyNull Object clean(@PolyNull Object o, RelDataType type) { + private @Nullable Object clean(@Nullable Object o, RelDataType type) { if (o == null) { return o; } diff --git a/core/src/main/java/org/apache/calcite/rex/RexCall.java b/core/src/main/java/org/apache/calcite/rex/RexCall.java index e2bfce84d65f..3af137e0a6cd 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexCall.java +++ b/core/src/main/java/org/apache/calcite/rex/RexCall.java @@ -138,12 +138,12 @@ protected final void appendOperands(StringBuilder sb) { if (SqlKind.SIMPLE_BINARY_OPS.contains(getKind())) { RexNode otherArg = operands.get(1 - i); if ((!(otherArg instanceof RexLiteral) - || digestSkipsType((RexLiteral) otherArg)) + || ((RexLiteral) otherArg).digestIncludesType() == RexDigestIncludeType.NO_TYPE) && SqlTypeUtil.equalSansNullability(operand.getType(), otherArg.getType())) { includeType = RexDigestIncludeType.NO_TYPE; } } - operandDigests.add(computeDigest((RexLiteral) operand, includeType)); + operandDigests.add(((RexLiteral) operand).computeDigest(includeType)); } int totalLength = (operandDigests.size() - 1) * 2; // commas for (String s : operandDigests) { @@ -159,18 +159,6 @@ protected final void appendOperands(StringBuilder sb) { } } - private static boolean digestSkipsType(RexLiteral literal) { - // This seems trivial, however, this method - // workarounds https://github.com/typetools/checker-framework/issues/3631 - return literal.digestIncludesType() == RexDigestIncludeType.NO_TYPE; - } - - private static String computeDigest(RexLiteral literal, RexDigestIncludeType includeType) { - // This seems trivial, however, this method - // workarounds https://github.com/typetools/checker-framework/issues/3631 - return literal.computeDigest(includeType); - } - protected String computeDigest(boolean withType) { final StringBuilder sb = new StringBuilder(op.getName()); if (operands.isEmpty() diff --git a/core/src/main/java/org/apache/calcite/rex/RexLiteral.java b/core/src/main/java/org/apache/calcite/rex/RexLiteral.java index 1dfb332fc03d..98a9179fefde 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexLiteral.java +++ b/core/src/main/java/org/apache/calcite/rex/RexLiteral.java @@ -20,6 +20,7 @@ import org.apache.calcite.avatica.util.DateTimeUtils; import org.apache.calcite.avatica.util.TimeUnit; import org.apache.calcite.config.CalciteSystemProperty; +import org.apache.calcite.linq4j.annotations.RequiresNonNull; import org.apache.calcite.linq4j.function.Functions; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.type.RelDataType; @@ -47,10 +48,6 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.initialization.qual.UnknownInitialization; -import org.checkerframework.checker.nullness.qual.PolyNull; -import org.checkerframework.checker.nullness.qual.RequiresNonNull; -import org.checkerframework.dataflow.qual.Pure; import org.jspecify.annotations.Nullable; import org.locationtech.jts.geom.Geometry; @@ -280,7 +277,6 @@ public class RexLiteral extends RexNode { */ @RequiresNonNull({"typeName", "type"}) public final String computeDigest( - @UnknownInitialization RexLiteral this, RexDigestIncludeType includeType) { if (includeType == RexDigestIncludeType.OPTIONAL) { if (digest != null) { @@ -307,8 +303,7 @@ public final String computeDigest( * @return whether {@link RexDigestIncludeType} digest would include data type */ @RequiresNonNull("type") - RexDigestIncludeType digestIncludesType( - @UnknownInitialization RexLiteral this) { + RexDigestIncludeType digestIncludesType() { return shouldIncludeType(value, type); } @@ -825,10 +820,10 @@ private static RexLiteral toLiteral(RelDataType type, Comparable value) { * by the Jdbc call to return a column as a string * @return a typed RexLiteral, or null */ - public static @PolyNull RexLiteral fromJdbcString( + public static @Nullable RexLiteral fromJdbcString( RelDataType type, SqlTypeName typeName, - @PolyNull String literal) { + @Nullable String literal) { if (literal == null) { return null; } @@ -967,7 +962,6 @@ public boolean isNull() { *

      For backwards compatibility, returns DATE. TIME and TIMESTAMP as a * {@link Calendar} value in UTC time zone. */ - @Pure public @Nullable Comparable getValue() { assert valueMatchesType(value, typeName, true) : value; if (value == null) { diff --git a/core/src/main/java/org/apache/calcite/rex/RexNode.java b/core/src/main/java/org/apache/calcite/rex/RexNode.java index 4440a6c31852..5d8e9b86fb85 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexNode.java +++ b/core/src/main/java/org/apache/calcite/rex/RexNode.java @@ -16,10 +16,10 @@ */ package org.apache.calcite.rex; +import org.apache.calcite.linq4j.annotations.MonotonicNonNull; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.sql.SqlKind; -import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.jspecify.annotations.Nullable; import java.util.Collection; diff --git a/core/src/main/java/org/apache/calcite/rex/RexProgram.java b/core/src/main/java/org/apache/calcite/rex/RexProgram.java index 53b3f846471b..f90ab5010583 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexProgram.java +++ b/core/src/main/java/org/apache/calcite/rex/RexProgram.java @@ -17,6 +17,7 @@ package org.apache.calcite.rex; import org.apache.calcite.linq4j.Ord; +import org.apache.calcite.linq4j.annotations.MonotonicNonNull; import org.apache.calcite.plan.RelOptPredicateList; import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.rel.RelCollation; @@ -41,9 +42,6 @@ import com.google.common.collect.Ordering; import com.google.errorprone.annotations.CheckReturnValue; -import org.checkerframework.checker.initialization.qual.UnknownInitialization; -import org.checkerframework.checker.nullness.qual.MonotonicNonNull; -import org.checkerframework.dataflow.qual.Pure; import org.jspecify.annotations.Nullable; import java.io.PrintWriter; @@ -100,7 +98,7 @@ public class RexProgram { /** * Reference counts for each expression, computed on demand. */ - private int @MonotonicNonNull[] refCounts; + @MonotonicNonNull private int[] refCounts; //~ Constructors ----------------------------------------------------------- @@ -178,7 +176,6 @@ public List> getNamedProjects() { * Returns the field reference of this program's filter condition, or null * if there is no condition. */ - @Pure public @Nullable RexLocalRef getCondition() { return condition; } @@ -443,7 +440,6 @@ public RelDataType getOutputRowType() { * @return Whether the program is valid */ public boolean isValid( - @UnknownInitialization RexProgram this, Litmus litmus, RelNode.@Nullable Context context) { if (inputRowType == null) { return litmus.fail(null); diff --git a/core/src/main/java/org/apache/calcite/rex/RexProgramBuilder.java b/core/src/main/java/org/apache/calcite/rex/RexProgramBuilder.java index 37a9c472decc..74b109cfeb05 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexProgramBuilder.java +++ b/core/src/main/java/org/apache/calcite/rex/RexProgramBuilder.java @@ -66,7 +66,7 @@ public RexProgramBuilder(RelDataType inputRowType, RexBuilder rexBuilder) { /** * Creates a program-builder. */ - @SuppressWarnings("method.invocation.invalid") + @SuppressWarnings("NullAway") private RexProgramBuilder(RelDataType inputRowType, RexBuilder rexBuilder, @Nullable RexSimplify unusedSimplify) { this.inputRowType = requireNonNull(inputRowType, "inputRowType"); @@ -94,7 +94,7 @@ private RexProgramBuilder(RelDataType inputRowType, RexBuilder rexBuilder, * @param normalize Whether to normalize * @param simplify Simplifier, or null to not simplify */ - @SuppressWarnings("method.invocation.invalid") + @SuppressWarnings("NullAway") private RexProgramBuilder( RexBuilder rexBuilder, final RelDataType inputRowType, diff --git a/core/src/main/java/org/apache/calcite/rex/RexShuttle.java b/core/src/main/java/org/apache/calcite/rex/RexShuttle.java index e28db54864ab..f1c3531c6bb8 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexShuttle.java +++ b/core/src/main/java/org/apache/calcite/rex/RexShuttle.java @@ -20,7 +20,6 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.PolyNull; import org.jspecify.annotations.Nullable; import java.util.ArrayList; @@ -277,7 +276,7 @@ protected List visitFieldCollations( * *

      Returns null if and only if {@code exprList} is null. */ - public final @PolyNull List apply(@PolyNull List exprList) { + public final @Nullable List apply(@Nullable List exprList) { if (exprList == null) { return exprList; } @@ -293,7 +292,7 @@ protected List visitFieldCollations( * Applies this shuttle to an expression, or returns null if the expression * is null. */ - public final @PolyNull RexNode apply(@PolyNull RexNode expr) { + public final @Nullable RexNode apply(@Nullable RexNode expr) { return (expr == null) ? expr : expr.accept(this); } } diff --git a/core/src/main/java/org/apache/calcite/rex/RexSimplify.java b/core/src/main/java/org/apache/calcite/rex/RexSimplify.java index 8696a57e7bb2..921b068c73ea 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexSimplify.java +++ b/core/src/main/java/org/apache/calcite/rex/RexSimplify.java @@ -3376,7 +3376,7 @@ private static boolean isLowerBound(final RexNode e) { *

      Returns whether the value was found. */ private static boolean replaceLast(List list, E oldVal, E newVal) { - @SuppressWarnings("argument.type.incompatible") + @SuppressWarnings("NullAway") final int index = list.lastIndexOf(oldVal); if (index < 0) { return false; diff --git a/core/src/main/java/org/apache/calcite/rex/RexSqlStandardConvertletTable.java b/core/src/main/java/org/apache/calcite/rex/RexSqlStandardConvertletTable.java index 5dec70be8417..b6fa31adcde1 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexSqlStandardConvertletTable.java +++ b/core/src/main/java/org/apache/calcite/rex/RexSqlStandardConvertletTable.java @@ -40,7 +40,7 @@ public class RexSqlStandardConvertletTable extends RexSqlReflectiveConvertletTable { //~ Constructors ----------------------------------------------------------- - @SuppressWarnings("method.invocation.invalid") + @SuppressWarnings("NullAway") public RexSqlStandardConvertletTable() { super(); diff --git a/core/src/main/java/org/apache/calcite/rex/RexUnaryBiVisitor.java b/core/src/main/java/org/apache/calcite/rex/RexUnaryBiVisitor.java index 4df853145e0a..7dce6e97bc01 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexUnaryBiVisitor.java +++ b/core/src/main/java/org/apache/calcite/rex/RexUnaryBiVisitor.java @@ -24,7 +24,7 @@ * * @param Return type from each {@code visitXxx} method */ -public class RexUnaryBiVisitor<@Nullable R> extends RexBiVisitorImpl { +public class RexUnaryBiVisitor extends RexBiVisitorImpl { /** Creates a RexUnaryBiVisitor. */ protected RexUnaryBiVisitor(boolean deep) { super(deep); diff --git a/core/src/main/java/org/apache/calcite/rex/RexVisitorImpl.java b/core/src/main/java/org/apache/calcite/rex/RexVisitorImpl.java index d1c3a911a542..2bfb5972b66a 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexVisitorImpl.java +++ b/core/src/main/java/org/apache/calcite/rex/RexVisitorImpl.java @@ -26,7 +26,7 @@ * * @param Return type from each {@code visitXxx} method. */ -public class RexVisitorImpl<@Nullable R> implements RexVisitor { +public class RexVisitorImpl implements RexVisitor { //~ Instance fields -------------------------------------------------------- protected final boolean deep; diff --git a/core/src/main/java/org/apache/calcite/rex/RexWindow.java b/core/src/main/java/org/apache/calcite/rex/RexWindow.java index 11b6df2e7d57..1a74d7f5788b 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexWindow.java +++ b/core/src/main/java/org/apache/calcite/rex/RexWindow.java @@ -72,7 +72,7 @@ public class RexWindow { * "ROWS BETWEEN 5 PRECEDING AND CURRENT ROW" is printed as * "ROWS 5 PRECEDING". */ - @SuppressWarnings("method.invocation.invalid") + @SuppressWarnings("NullAway") RexWindow( List partitionKeys, List orderKeys, diff --git a/core/src/main/java/org/apache/calcite/rex/RexWindowBound.java b/core/src/main/java/org/apache/calcite/rex/RexWindowBound.java index 58f0c159c89d..9489dc5ecde3 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexWindowBound.java +++ b/core/src/main/java/org/apache/calcite/rex/RexWindowBound.java @@ -18,8 +18,6 @@ import org.apache.calcite.sql.SqlNode; -import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf; -import org.checkerframework.dataflow.qual.Pure; import org.jspecify.annotations.Nullable; /** @@ -41,9 +39,7 @@ public static RexWindowBound create(SqlNode node, RexNode rexNode) { * * @return if the bound is unbounded */ - @Pure - @EnsuresNonNullIf(expression = "getOffset()", result = false) - @SuppressWarnings("contracts.conditional.postcondition.not.satisfied") + @SuppressWarnings("NullAway") public boolean isUnbounded() { return false; } @@ -81,9 +77,7 @@ public boolean isFollowing() { * * @return if the bound is CURRENT ROW */ - @Pure - @EnsuresNonNullIf(expression = "getOffset()", result = false) - @SuppressWarnings("contracts.conditional.postcondition.not.satisfied") + @SuppressWarnings("NullAway") public boolean isCurrentRow() { return false; } @@ -93,7 +87,6 @@ public boolean isCurrentRow() { * * @return offset from XX PRECEDING/FOLLOWING */ - @Pure public @Nullable RexNode getOffset() { return null; } diff --git a/core/src/main/java/org/apache/calcite/runtime/AutomatonBuilder.java b/core/src/main/java/org/apache/calcite/runtime/AutomatonBuilder.java index 683d50078cc6..049c230c375f 100644 --- a/core/src/main/java/org/apache/calcite/runtime/AutomatonBuilder.java +++ b/core/src/main/java/org/apache/calcite/runtime/AutomatonBuilder.java @@ -39,9 +39,9 @@ public class AutomatonBuilder { private final Map symbolIds = new HashMap<>(); private final List stateList = new ArrayList<>(); private final List transitionList = new ArrayList<>(); - @SuppressWarnings("method.invocation.invalid") + @SuppressWarnings("NullAway") private final State startState = createState(); - @SuppressWarnings("method.invocation.invalid") + @SuppressWarnings("NullAway") private final State endState = createState(); /** Adds a pattern as a start-to-end transition. */ diff --git a/core/src/main/java/org/apache/calcite/runtime/CalciteContextException.java b/core/src/main/java/org/apache/calcite/runtime/CalciteContextException.java index 53b4f363e101..32e1093730bd 100644 --- a/core/src/main/java/org/apache/calcite/runtime/CalciteContextException.java +++ b/core/src/main/java/org/apache/calcite/runtime/CalciteContextException.java @@ -20,7 +20,6 @@ // resource generation can use reflection. That means it must have no // dependencies on other Calcite code. -import org.checkerframework.checker.initialization.qual.UnknownInitialization; import org.jspecify.annotations.Nullable; import static java.util.Objects.requireNonNull; @@ -125,7 +124,6 @@ public void setPosition(int posLine, int posColumn) { * @param endPosColumn 1-based end column number */ public void setPosition( - @UnknownInitialization CalciteContextException this, int posLine, int posColumn, int endPosLine, diff --git a/core/src/main/java/org/apache/calcite/runtime/CalciteException.java b/core/src/main/java/org/apache/calcite/runtime/CalciteException.java index f417a413b975..ad2132abe9a7 100644 --- a/core/src/main/java/org/apache/calcite/runtime/CalciteException.java +++ b/core/src/main/java/org/apache/calcite/runtime/CalciteException.java @@ -53,7 +53,7 @@ public class CalciteException extends RuntimeException { * @param message error message * @param cause underlying cause */ - @SuppressWarnings({"argument.type.incompatible", "method.invocation.invalid"}) + @SuppressWarnings("NullAway") public CalciteException( String message, @Nullable Throwable cause) { diff --git a/core/src/main/java/org/apache/calcite/runtime/ConsList.java b/core/src/main/java/org/apache/calcite/runtime/ConsList.java index 1954a6526b81..5d4914c49d83 100644 --- a/core/src/main/java/org/apache/calcite/runtime/ConsList.java +++ b/core/src/main/java/org/apache/calcite/runtime/ConsList.java @@ -18,7 +18,6 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.PolyNull; import org.jspecify.annotations.Nullable; import java.util.ArrayList; @@ -115,7 +114,7 @@ private ConsList(E first, List rest) { return toList().listIterator(index); } - @Override public @PolyNull Object[] toArray(ConsList<@PolyNull E> this) { + @Override public @Nullable Object[] toArray(ConsList<@Nullable E> this) { return toList().toArray(); } diff --git a/core/src/main/java/org/apache/calcite/runtime/DeterministicAutomaton.java b/core/src/main/java/org/apache/calcite/runtime/DeterministicAutomaton.java index 0f2363234935..67acd8bd7118 100644 --- a/core/src/main/java/org/apache/calcite/runtime/DeterministicAutomaton.java +++ b/core/src/main/java/org/apache/calcite/runtime/DeterministicAutomaton.java @@ -42,7 +42,7 @@ public class DeterministicAutomaton { private final ImmutableList transitions; /** Constructs the DFA from an epsilon-NFA. */ - @SuppressWarnings("method.invocation.invalid") + @SuppressWarnings("NullAway") DeterministicAutomaton(Automaton automaton) { this.automaton = requireNonNull(automaton, "automaton"); // Calculate eps closure of start state diff --git a/core/src/main/java/org/apache/calcite/runtime/FlatLists.java b/core/src/main/java/org/apache/calcite/runtime/FlatLists.java index 68d1453322bb..a552342071cd 100644 --- a/core/src/main/java/org/apache/calcite/runtime/FlatLists.java +++ b/core/src/main/java/org/apache/calcite/runtime/FlatLists.java @@ -21,7 +21,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import org.checkerframework.checker.nullness.qual.PolyNull; import org.jspecify.annotations.NonNull; import org.jspecify.annotations.Nullable; @@ -263,7 +262,7 @@ private static ComparableList of_(List t) { } /** Returns a list that consists of a given list plus an element. */ - public static List append(List list, E e) { + public static List append(List list, E e) { if (list instanceof AbstractFlatList) { //noinspection unchecked return ((AbstractFlatList) list).append(e); @@ -275,13 +274,13 @@ public static List append(List list, E e) { /** Returns a list that consists of a given list plus an element, guaranteed * to be an {@link ImmutableList}. */ - public static ImmutableList append(ImmutableList list, E e) { + public static ImmutableList append(ImmutableList list, E e) { return ImmutableList.builder().addAll(list).add(e).build(); } /** Returns a map that consists of a given map plus an (key, value), * guaranteed to be an {@link ImmutableMap}. */ - public static ImmutableMap append( + public static ImmutableMap append( Map map, K k, V v) { final ImmutableMap.Builder builder = ImmutableMap.builder(); builder.put(k, v); @@ -407,7 +406,7 @@ protected static class Flat1List return a; } - @Override public @PolyNull Object[] toArray(Flat1List<@PolyNull T> this) { + @Override public @Nullable Object[] toArray(Flat1List<@Nullable T> this) { return new Object[] {castNonNull(t0)}; } @@ -540,7 +539,7 @@ protected static class Flat2List return a; } - @Override public @PolyNull Object[] toArray(Flat2List<@PolyNull T> this) { + @Override public @Nullable Object[] toArray(Flat2List<@Nullable T> this) { return new Object[] {castNonNull(t0), castNonNull(t1)}; } @@ -690,7 +689,7 @@ protected static class Flat3List return a; } - @Override public @PolyNull Object[] toArray(Flat3List<@PolyNull T> this) { + @Override public @Nullable Object[] toArray(Flat3List<@Nullable T> this) { return new Object[] {castNonNull(t0), castNonNull(t1), castNonNull(t2)}; } @@ -859,7 +858,7 @@ protected static class Flat4List return a; } - @Override public @PolyNull Object[] toArray(Flat4List<@PolyNull T> this) { + @Override public @Nullable Object[] toArray(Flat4List<@Nullable T> this) { return new Object[] {castNonNull(t0), castNonNull(t1), castNonNull(t2), castNonNull(t3)}; } @@ -1048,7 +1047,7 @@ protected static class Flat5List return a; } - @Override public @PolyNull Object[] toArray(Flat5List<@PolyNull T> this) { + @Override public @Nullable Object[] toArray(Flat5List<@Nullable T> this) { return new Object[] {castNonNull(t0), castNonNull(t1), castNonNull(t2), castNonNull(t3), castNonNull(t4)}; } @@ -1257,7 +1256,7 @@ protected static class Flat6List return a; } - @Override public @PolyNull Object[] toArray(Flat6List<@PolyNull T> this) { + @Override public @Nullable Object[] toArray(Flat6List<@Nullable T> this) { return new Object[] {castNonNull(t0), castNonNull(t1), castNonNull(t2), castNonNull(t3), castNonNull(t4), castNonNull(t5)}; } diff --git a/core/src/main/java/org/apache/calcite/runtime/ImmutablePairList.java b/core/src/main/java/org/apache/calcite/runtime/ImmutablePairList.java index 2ea23448d834..97c44108bc78 100644 --- a/core/src/main/java/org/apache/calcite/runtime/ImmutablePairList.java +++ b/core/src/main/java/org/apache/calcite/runtime/ImmutablePairList.java @@ -16,8 +16,6 @@ */ package org.apache.calcite.runtime; -import org.jspecify.annotations.NonNull; - import java.util.ArrayList; import java.util.Collection; import java.util.List; @@ -63,7 +61,7 @@ static PairList copyOf(T t, U u, Object... rest) { /** Creates an ImmutablePairList whose contents are a copy of a given * collection. */ @SuppressWarnings("unchecked") - static <@NonNull T, @NonNull U> ImmutablePairList copyOf( + static ImmutablePairList copyOf( Iterable> iterable) { // Every PairList - mutable and immutable - knows how to quickly make // itself immutable. diff --git a/core/src/main/java/org/apache/calcite/runtime/JsonFunctions.java b/core/src/main/java/org/apache/calcite/runtime/JsonFunctions.java index c6eb7868693c..5bb16adf62fa 100644 --- a/core/src/main/java/org/apache/calcite/runtime/JsonFunctions.java +++ b/core/src/main/java/org/apache/calcite/runtime/JsonFunctions.java @@ -16,6 +16,7 @@ */ package org.apache.calcite.runtime; +import org.apache.calcite.linq4j.annotations.EnsuresNonNullIf; import org.apache.calcite.linq4j.function.Deterministic; import org.apache.calcite.sql.SqlJsonConstructorNullClause; import org.apache.calcite.sql.SqlJsonExistsErrorBehavior; @@ -40,7 +41,6 @@ import com.jayway.jsonpath.spi.mapper.JacksonMappingProvider; import com.jayway.jsonpath.spi.mapper.MappingProvider; -import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf; import org.jspecify.annotations.Nullable; import java.util.ArrayList; @@ -901,7 +901,7 @@ private JsonPathContext(PathMode mode, @Nullable Object obj, @Nullable Exception this.exc = exc; } - @EnsuresNonNullIf(expression = "exc", result = true) + @EnsuresNonNullIf(value = "exc", result = true) public boolean hasException() { return exc != null; } @@ -966,7 +966,7 @@ Object obj() { return requireNonNull(obj, "json object must not be null"); } - @EnsuresNonNullIf(expression = "exc", result = true) + @EnsuresNonNullIf(value = "exc", result = true) public boolean hasException() { return exc != null; } diff --git a/core/src/main/java/org/apache/calcite/runtime/PairList.java b/core/src/main/java/org/apache/calcite/runtime/PairList.java index b7be90f6173a..a1d0275e3421 100644 --- a/core/src/main/java/org/apache/calcite/runtime/PairList.java +++ b/core/src/main/java/org/apache/calcite/runtime/PairList.java @@ -39,7 +39,8 @@ * @param First type * @param Second type */ -public interface PairList extends List> { +public interface PairList + extends List> { /** Creates an empty PairList. */ static PairList of() { return new PairLists.MutablePairList<>(new ArrayList<>()); @@ -187,6 +188,8 @@ default void reverse() { * reversed. * *

      Throws {@link NullPointerException} if any keys or values are null. */ + // On JDK 21 and later this overrides List.reversed(), but Calcite still compiles on JDK 8 + @SuppressWarnings("MissingOverride") ImmutablePairList reversed(); /** Action to be taken each step of an indexed iteration over a PairList. diff --git a/core/src/main/java/org/apache/calcite/runtime/RandomFunction.java b/core/src/main/java/org/apache/calcite/runtime/RandomFunction.java index 71936c383bd2..c4843fe8aeb9 100644 --- a/core/src/main/java/org/apache/calcite/runtime/RandomFunction.java +++ b/core/src/main/java/org/apache/calcite/runtime/RandomFunction.java @@ -16,11 +16,10 @@ */ package org.apache.calcite.runtime; +import org.apache.calcite.linq4j.annotations.MonotonicNonNull; import org.apache.calcite.linq4j.function.Deterministic; import org.apache.calcite.linq4j.function.Parameter; -import org.checkerframework.checker.nullness.qual.MonotonicNonNull; - import java.util.Random; /** diff --git a/core/src/main/java/org/apache/calcite/runtime/Resources.java b/core/src/main/java/org/apache/calcite/runtime/Resources.java index 01c88e1e9426..637debfe8ab6 100644 --- a/core/src/main/java/org/apache/calcite/runtime/Resources.java +++ b/core/src/main/java/org/apache/calcite/runtime/Resources.java @@ -16,10 +16,8 @@ */ package org.apache.calcite.runtime; -import org.checkerframework.checker.initialization.qual.UnderInitialization; import org.jspecify.annotations.Nullable; -import org.checkerframework.checker.nullness.qual.PolyNull; -import org.checkerframework.checker.nullness.qual.RequiresNonNull; +import org.apache.calcite.linq4j.annotations.RequiresNonNull; import java.io.IOException; import java.io.InputStream; @@ -287,7 +285,7 @@ public static class Element { protected final Method method; protected final String key; - @SuppressWarnings("method.invocation.invalid") + @SuppressWarnings("NullAway") public Element(Method method) { this.method = method; this.key = deriveKey(); @@ -626,9 +624,7 @@ protected Prop(PropertyAccessor accessor, Method method) { } @RequiresNonNull("method") - protected final @Nullable Default getDefault( - @UnderInitialization Prop this - ) { + protected final @Nullable Default getDefault() { if (hasDefault) { return castNonNull(method.getAnnotation(Default.class)); } else { @@ -775,7 +771,7 @@ public StringProp(PropertyAccessor accessor, Method method) { * value if the property is not set. * *

      If {@code defaultValue} is not null, never returns null. */ - public @PolyNull String get(@PolyNull String defaultValue) { + public @Nullable String get(@Nullable String defaultValue) { return accessor.stringValue(this, defaultValue); } @@ -800,7 +796,7 @@ public interface PropertyAccessor { int intValue(IntProp p); int intValue(IntProp p, int defaultValue); @Nullable String stringValue(StringProp p); - @PolyNull String stringValue(StringProp p, @PolyNull String defaultValue); + @Nullable String stringValue(StringProp p, @Nullable String defaultValue); boolean booleanValue(BooleanProp p); boolean booleanValue(BooleanProp p, boolean defaultValue); double doubleValue(DoubleProp p); @@ -829,8 +825,8 @@ public int intValue(IntProp p, int defaultValue) { return p.defaultValue(); } - @Override public @PolyNull String stringValue(StringProp p, - @PolyNull String defaultValue) { + @Override public @Nullable String stringValue(StringProp p, + @Nullable String defaultValue) { return defaultValue; } @@ -1143,8 +1139,8 @@ public int intValue(IntProp p, int defaultValue) { return p.defaultValue; } - @Override public @PolyNull String stringValue(StringProp p, - @PolyNull String defaultValue) { + @Override public @Nullable String stringValue(StringProp p, + @Nullable String defaultValue) { final String s = properties.getProperty(p.key); return s == null ? defaultValue : s; } diff --git a/core/src/main/java/org/apache/calcite/runtime/SpaceFillingCurve2D.java b/core/src/main/java/org/apache/calcite/runtime/SpaceFillingCurve2D.java index d66cbf9dc05d..3330307a7016 100644 --- a/core/src/main/java/org/apache/calcite/runtime/SpaceFillingCurve2D.java +++ b/core/src/main/java/org/apache/calcite/runtime/SpaceFillingCurve2D.java @@ -115,7 +115,7 @@ class OverlappingRange extends AbstractRange { /** Lexicographic ordering for {@link IndexRange}. */ class IndexRangeOrdering extends Ordering { - @SuppressWarnings("override.param.invalid") + @SuppressWarnings("NullAway") @Override public int compare(IndexRange x, IndexRange y) { final int c1 = Long.compare(x.lower(), y.lower()); if (c1 != 0) { diff --git a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java index eaba8ad81d9e..db7f0189e9b6 100644 --- a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java +++ b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java @@ -73,7 +73,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.UncheckedExecutionException; -import org.checkerframework.checker.nullness.qual.PolyNull; import org.joou.UByte; import org.joou.UInteger; import org.joou.ULong; @@ -251,7 +250,8 @@ public class SqlFunctions { /** * WARNING: keep this logic as a static method. JDK 8 and 11 produce invalid bytecode when - * checkerframework annotations are used on static lambdas. See CALCITE-6393. + * type-use annotations such as {@code @Nullable} are used on static lambdas. + * See CALCITE-6393. */ private static Enumerable<@Nullable Object[]> arrayCartesianProduct(Object[] lists) { final List> enumerators = new ArrayList<>(); @@ -2793,45 +2793,45 @@ public static int plus(int b0, int b1) { /** SQL + operator applied to int values; left side may be * null. */ - public static @PolyNull Integer plus(@PolyNull Integer b0, int b1) { + public static @Nullable Integer plus(@Nullable Integer b0, int b1) { return b0 == null ? castNonNull(null) : (b0 + b1); } /** SQL + operator applied to int values; right side may be * null. */ - public static @PolyNull Integer plus(int b0, @PolyNull Integer b1) { + public static @Nullable Integer plus(int b0, @Nullable Integer b1) { return b1 == null ? castNonNull(null) : (b0 + b1); } /** SQL + operator applied to nullable int values. */ - public static @PolyNull Integer plus(@PolyNull Integer b0, @PolyNull Integer b1) { + public static @Nullable Integer plus(@Nullable Integer b0, @Nullable Integer b1) { return (b0 == null || b1 == null) ? castNonNull(null) : (b0 + b1); } /** SQL + operator applied to nullable long and int values. */ - public static @PolyNull Long plus(@PolyNull Long b0, @PolyNull Integer b1) { + public static @Nullable Long plus(@Nullable Long b0, @Nullable Integer b1) { return (b0 == null || b1 == null) ? castNonNull(null) : (b0.longValue() + b1.longValue()); } /** SQL + operator applied to nullable int and long values. */ - public static @PolyNull Long plus(@PolyNull Integer b0, @PolyNull Long b1) { + public static @Nullable Long plus(@Nullable Integer b0, @Nullable Long b1) { return (b0 == null || b1 == null) ? castNonNull(null) : (b0.longValue() + b1.longValue()); } /** SQL + operator applied to BigDecimal values. */ - public static @PolyNull BigDecimal plus(@PolyNull BigDecimal b0, - @PolyNull BigDecimal b1) { + public static @Nullable BigDecimal plus(@Nullable BigDecimal b0, + @Nullable BigDecimal b1) { return (b0 == null || b1 == null) ? castNonNull(null) : b0.add(b1); } /** SQL + operator applied to Object values (at least one operand * has ANY type; either may be null). */ - public static @PolyNull Object plusAny(@PolyNull Object b0, - @PolyNull Object b1) { + public static @Nullable Object plusAny(@Nullable Object b0, + @Nullable Object b1) { if (b0 == null || b1 == null) { return castNonNull(null); } @@ -2843,25 +2843,25 @@ public static int plus(int b0, int b1) { throw notArithmetic("+", b0, b1); } - public static @PolyNull UByte plus(@PolyNull UByte b0, @PolyNull UByte b1) { + public static @Nullable UByte plus(@Nullable UByte b0, @Nullable UByte b1) { return (b0 == null || b1 == null) ? castNonNull(null) : b0.add(b1); } - public static @PolyNull UShort plus(@PolyNull UShort b0, @PolyNull UShort b1) { + public static @Nullable UShort plus(@Nullable UShort b0, @Nullable UShort b1) { return (b0 == null || b1 == null) ? castNonNull(null) : b0.add(b1); } - public static @PolyNull UInteger plus(@PolyNull UInteger b0, @PolyNull UInteger b1) { + public static @Nullable UInteger plus(@Nullable UInteger b0, @Nullable UInteger b1) { return (b0 == null || b1 == null) ? castNonNull(null) : b0.add(b1); } - public static @PolyNull ULong plus(@PolyNull ULong b0, @PolyNull ULong b1) { + public static @Nullable ULong plus(@Nullable ULong b0, @Nullable ULong b1) { return (b0 == null || b1 == null) ? castNonNull(null) : b0.add(b1); @@ -2926,51 +2926,51 @@ public static int minus(int b0, int b1) { /** SQL - operator applied to int values; left side may be * null. */ - public static @PolyNull Integer minus(@PolyNull Integer b0, int b1) { + public static @Nullable Integer minus(@Nullable Integer b0, int b1) { return b0 == null ? castNonNull(null) : (b0 - b1); } /** SQL - operator applied to int values; right side may be * null. */ - public static @PolyNull Integer minus(int b0, @PolyNull Integer b1) { + public static @Nullable Integer minus(int b0, @Nullable Integer b1) { return b1 == null ? castNonNull(null) : (b0 - b1); } /** SQL - operator applied to nullable int values. */ - public static @PolyNull Integer minus(@PolyNull Integer b0, @PolyNull Integer b1) { + public static @Nullable Integer minus(@Nullable Integer b0, @Nullable Integer b1) { return (b0 == null || b1 == null) ? castNonNull(null) : (b0 - b1); } /** SQL - operator applied to nullable long and int values. */ - public static @PolyNull Long minus(@PolyNull Long b0, @PolyNull Integer b1) { + public static @Nullable Long minus(@Nullable Long b0, @Nullable Integer b1) { return (b0 == null || b1 == null) ? castNonNull(null) : (b0.longValue() - b1.longValue()); } /** SQL - operator applied to nullable int and long values. */ - public static @PolyNull Long minus(@PolyNull Integer b0, @PolyNull Long b1) { + public static @Nullable Long minus(@Nullable Integer b0, @Nullable Long b1) { return (b0 == null || b1 == null) ? castNonNull(null) : (b0.longValue() - b1.longValue()); } /** SQL - operator applied to nullable long and long values. */ - public static @PolyNull Long minus(@PolyNull Long b0, @PolyNull Long b1) { + public static @Nullable Long minus(@Nullable Long b0, @Nullable Long b1) { return (b0 == null || b1 == null) ? castNonNull(null) : b0.longValue() - b1.longValue(); } /** SQL - operator applied to nullable BigDecimal values. */ - public static @PolyNull BigDecimal minus(@PolyNull BigDecimal b0, - @PolyNull BigDecimal b1) { + public static @Nullable BigDecimal minus(@Nullable BigDecimal b0, + @Nullable BigDecimal b1) { return (b0 == null || b1 == null) ? castNonNull(null) : b0.subtract(b1); } /** SQL - operator applied to Object values (at least one operand * has ANY type; either may be null). */ - public static @PolyNull Object minusAny(@PolyNull Object b0, @PolyNull Object b1) { + public static @Nullable Object minusAny(@Nullable Object b0, @Nullable Object b1) { if (b0 == null || b1 == null) { return castNonNull(null); } @@ -2982,20 +2982,20 @@ public static int minus(int b0, int b1) { throw notArithmetic("-", b0, b1); } - public static @PolyNull UByte minus(@PolyNull UByte b0, @PolyNull UByte b1) { + public static @Nullable UByte minus(@Nullable UByte b0, @Nullable UByte b1) { return (b0 == null || b1 == null) ? castNonNull(null) : b0.subtract(b1); } - public static @PolyNull UShort minus(@PolyNull UShort b0, @PolyNull UShort b1) { + public static @Nullable UShort minus(@Nullable UShort b0, @Nullable UShort b1) { return (b0 == null || b1 == null) ? castNonNull(null) : b0.subtract(b1); } - public static @PolyNull UInteger minus(@PolyNull UInteger b0, @PolyNull UInteger b1) { + public static @Nullable UInteger minus(@Nullable UInteger b0, @Nullable UInteger b1) { return (b0 == null || b1 == null) ? castNonNull(null) : b0.subtract(b1); } /** SQL - operator applied to nullable unsigned long and long values. */ - public static @PolyNull ULong minus(@PolyNull ULong b0, @PolyNull ULong b1) { + public static @Nullable ULong minus(@Nullable ULong b0, @Nullable ULong b1) { return (b0 == null || b1 == null) ? castNonNull(null) : b0.subtract(b1); @@ -3076,39 +3076,39 @@ public static int divide(int b0, int b1) { /** SQL / operator applied to int values; left side may be * null. */ - public static @PolyNull Integer divide(@PolyNull Integer b0, int b1) { + public static @Nullable Integer divide(@Nullable Integer b0, int b1) { return b0 == null ? castNonNull(null) : (b0 / b1); } /** SQL / operator applied to int values; right side may be * null. */ - public static @PolyNull Integer divide(int b0, @PolyNull Integer b1) { + public static @Nullable Integer divide(int b0, @Nullable Integer b1) { return b1 == null ? castNonNull(null) : (b0 / b1); } /** SQL / operator applied to nullable int values. */ - public static @PolyNull Integer divide(@PolyNull Integer b0, - @PolyNull Integer b1) { + public static @Nullable Integer divide(@Nullable Integer b0, + @Nullable Integer b1) { return (b0 == null || b1 == null) ? castNonNull(null) : (b0 / b1); } /** SQL / operator applied to nullable long and int values. */ - public static @PolyNull Long divide(Long b0, @PolyNull Integer b1) { + public static @Nullable Long divide(Long b0, @Nullable Integer b1) { return (b0 == null || b1 == null) ? castNonNull(null) : (b0.longValue() / b1.longValue()); } /** SQL / operator applied to nullable int and long values. */ - public static @PolyNull Long divide(@PolyNull Integer b0, @PolyNull Long b1) { + public static @Nullable Long divide(@Nullable Integer b0, @Nullable Long b1) { return (b0 == null || b1 == null) ? castNonNull(null) : (b0.longValue() / b1.longValue()); } /** SQL / operator applied to BigDecimal values. */ - public static @PolyNull BigDecimal divide(@PolyNull BigDecimal b0, - @PolyNull BigDecimal b1) { + public static @Nullable BigDecimal divide(@Nullable BigDecimal b0, + @Nullable BigDecimal b1) { return (b0 == null || b1 == null) ? castNonNull(null) : b0.divide(b1, MathContext.DECIMAL64); @@ -3116,8 +3116,8 @@ public static int divide(int b0, int b1) { /** SQL / operator applied to Object values (at least one operand * has ANY type; either may be null). */ - public static @PolyNull Object divideAny(@PolyNull Object b0, - @PolyNull Object b1) { + public static @Nullable Object divideAny(@Nullable Object b0, + @Nullable Object b1) { if (b0 == null || b1 == null) { return castNonNull(null); } @@ -3139,26 +3139,26 @@ public static long divide(long b0, BigDecimal b1) { .divide(b1, RoundingMode.HALF_DOWN).longValue(); } - public static @PolyNull UByte divide(@PolyNull UByte b0, - @PolyNull UByte b1) { + public static @Nullable UByte divide(@Nullable UByte b0, + @Nullable UByte b1) { return (b0 == null || b1 == null) ? castNonNull(null) : UByte.valueOf(b0.intValue() / b1.intValue()); } - public static @PolyNull UShort divide(@PolyNull UShort b0, - @PolyNull UShort b1) { + public static @Nullable UShort divide(@Nullable UShort b0, + @Nullable UShort b1) { return (b0 == null || b1 == null) ? castNonNull(null) : UShort.valueOf(b0.intValue() / b1.intValue()); } - public static @PolyNull UInteger divide(@PolyNull UInteger b0, - @PolyNull UInteger b1) { + public static @Nullable UInteger divide(@Nullable UInteger b0, + @Nullable UInteger b1) { return (b0 == null || b1 == null) ? castNonNull(null) : UInteger.valueOf(b0.longValue() / b1.longValue()); } - public static @PolyNull ULong divide(@PolyNull ULong b0, - @PolyNull ULong b1) { + public static @Nullable ULong divide(@Nullable ULong b0, + @Nullable ULong b1) { return (b0 == null || b1 == null) ? castNonNull(null) : ULong.valueOf(UnsignedType.toBigInteger(b0).divide(UnsignedType.toBigInteger(b1))); } @@ -3230,42 +3230,42 @@ public static int multiply(int b0, int b1) { /** SQL * operator applied to int values; left side may be * null. */ - public static @PolyNull Integer multiply(@PolyNull Integer b0, int b1) { + public static @Nullable Integer multiply(@Nullable Integer b0, int b1) { return b0 == null ? castNonNull(null) : (b0 * b1); } /** SQL * operator applied to int values; right side may be * null. */ - public static @PolyNull Integer multiply(int b0, @PolyNull Integer b1) { + public static @Nullable Integer multiply(int b0, @Nullable Integer b1) { return b1 == null ? castNonNull(null) : (b0 * b1); } /** SQL * operator applied to nullable int values. */ - public static @PolyNull Integer multiply(@PolyNull Integer b0, - @PolyNull Integer b1) { + public static @Nullable Integer multiply(@Nullable Integer b0, + @Nullable Integer b1) { return (b0 == null || b1 == null) ? castNonNull(null) : (b0 * b1); } - public static @PolyNull UByte multiply(@PolyNull UByte b0, - @PolyNull UByte b1) { + public static @Nullable UByte multiply(@Nullable UByte b0, + @Nullable UByte b1) { return (b0 == null || b1 == null) ? castNonNull(null) : UByte.valueOf(b0.longValue() * b1.longValue()); } - public static @PolyNull UShort multiply(@PolyNull UShort b0, - @PolyNull UShort b1) { + public static @Nullable UShort multiply(@Nullable UShort b0, + @Nullable UShort b1) { return (b0 == null || b1 == null) ? castNonNull(null) : UShort.valueOf(b0.intValue() * b1.intValue()); } - public static @PolyNull UInteger multiply(@PolyNull UInteger b0, - @PolyNull UInteger b1) { + public static @Nullable UInteger multiply(@Nullable UInteger b0, + @Nullable UInteger b1) { return (b0 == null || b1 == null) ? castNonNull(null) : UInteger.valueOf(b0.longValue() * b1.longValue()); } - public static @PolyNull ULong multiply(@PolyNull ULong b0, - @PolyNull ULong b1) { + public static @Nullable ULong multiply(@Nullable ULong b0, + @Nullable ULong b1) { if (b0 == null || b1 == null) { return castNonNull(null); } @@ -3274,29 +3274,29 @@ public static int multiply(int b0, int b1) { } /** SQL * operator applied to nullable long and int values. */ - public static @PolyNull Long multiply(@PolyNull Long b0, @PolyNull Integer b1) { + public static @Nullable Long multiply(@Nullable Long b0, @Nullable Integer b1) { return (b0 == null || b1 == null) ? castNonNull(null) : (b0.longValue() * b1.longValue()); } /** SQL * operator applied to nullable int and long values. */ - public static @PolyNull Long multiply(@PolyNull Integer b0, @PolyNull Long b1) { + public static @Nullable Long multiply(@Nullable Integer b0, @Nullable Long b1) { return (b0 == null || b1 == null) ? castNonNull(null) : (b0.longValue() * b1.longValue()); } /** SQL * operator applied to nullable BigDecimal values. */ - public static @PolyNull BigDecimal multiply(@PolyNull BigDecimal b0, - @PolyNull BigDecimal b1) { + public static @Nullable BigDecimal multiply(@Nullable BigDecimal b0, + @Nullable BigDecimal b1) { return (b0 == null || b1 == null) ? castNonNull(null) : b0.multiply(b1); } /** SQL * operator applied to Object values (at least one operand * has ANY type; either may be null). */ - public static @PolyNull Object multiplyAny(@PolyNull Object b0, - @PolyNull Object b1) { + public static @Nullable Object multiplyAny(@Nullable Object b0, + @Nullable Object b1) { if (b0 == null || b1 == null) { return castNonNull(null); } @@ -5381,7 +5381,7 @@ public static int toInt(java.sql.Date v, TimeZone timeZone) { * @see #toInt(java.sql.Date, TimeZone) * @see #internalToDate(Integer) converse method */ - public static @PolyNull Integer toIntOptional(java.sql.@PolyNull Date v) { + public static @Nullable Integer toIntOptional(java.sql.@Nullable Date v) { return v == null ? castNonNull(null) : toInt(v); @@ -5394,7 +5394,7 @@ public static int toInt(java.sql.Date v, TimeZone timeZone) { * * @see #toInt(java.sql.Date, TimeZone) */ - public static @PolyNull Integer toIntOptional(java.sql.@PolyNull Date v, + public static @Nullable Integer toIntOptional(java.sql.@Nullable Date v, TimeZone timeZone) { return v == null ? castNonNull(null) @@ -5422,7 +5422,7 @@ public static int toInt(java.sql.Time v) { * @see #toInt(java.sql.Time) * @see #internalToTime(Integer) converse method */ - public static @PolyNull Integer toIntOptional(java.sql.@PolyNull Time v) { + public static @Nullable Integer toIntOptional(java.sql.@Nullable Time v) { return v == null ? castNonNull(null) : toInt(v); } @@ -5443,7 +5443,7 @@ public static int toInt(Object o) { : (Integer) cannotConvert(o, int.class); } - public static @PolyNull Integer toIntOptional(@PolyNull Object o) { + public static @Nullable Integer toIntOptional(@Nullable Object o) { return o == null ? castNonNull(null) : toInt(o); } @@ -5508,7 +5508,7 @@ public static long toLong(Timestamp v, TimeZone timeZone) { * @see #toLong(Timestamp, TimeZone) * @see #internalToTimestamp(Long) converse method */ - public static @PolyNull Long toLongOptional(@PolyNull Timestamp v) { + public static @Nullable Long toLongOptional(@Nullable Timestamp v) { return v == null ? castNonNull(null) : toLong(v, LOCAL_TZ); } @@ -5519,7 +5519,7 @@ public static long toLong(Timestamp v, TimeZone timeZone) { * * @see #toLong(Timestamp, TimeZone) */ - public static @PolyNull Long toLongOptional(@PolyNull Timestamp v, + public static @Nullable Long toLongOptional(@Nullable Timestamp v, TimeZone timeZone) { if (v == null) { return castNonNull(null); @@ -5549,7 +5549,7 @@ public static long toLong(Object o) { : (Long) cannotConvert(o, long.class); } - public static @PolyNull Long toLongOptional(@PolyNull Object o) { + public static @Nullable Long toLongOptional(@Nullable Object o) { return o == null ? castNonNull(null) : toLong(o); } @@ -5646,7 +5646,7 @@ public static java.sql.Date internalToDate(int v) { * @see #internalToDate(int) * @see #toIntOptional(java.sql.Date) converse method */ - public static java.sql.@PolyNull Date internalToDate(@PolyNull Integer v) { + public static java.sql.@Nullable Date internalToDate(@Nullable Integer v) { return v == null ? castNonNull(null) : internalToDate(v.intValue()); } @@ -5670,11 +5670,11 @@ public static java.sql.Time internalToTime(int v) { * @see #internalToTime(Integer) * @see #toIntOptional(java.sql.Time) converse method */ - public static java.sql.@PolyNull Time internalToTime(@PolyNull Integer v) { + public static java.sql.@Nullable Time internalToTime(@Nullable Integer v) { return v == null ? castNonNull(null) : internalToTime(v.intValue()); } - public static @PolyNull Integer toTimeWithLocalTimeZone(@PolyNull String v) { + public static @Nullable Integer toTimeWithLocalTimeZone(@Nullable String v) { if (v == null) { return castNonNull(null); } @@ -5684,7 +5684,7 @@ public static java.sql.Time internalToTime(int v) { .getMillisOfDay(); } - public static @PolyNull Integer toTimeWithLocalTimeZone(@PolyNull String v, + public static @Nullable Integer toTimeWithLocalTimeZone(@Nullable String v, TimeZone timeZone) { if (v == null) { return castNonNull(null); @@ -6049,7 +6049,7 @@ public static java.sql.Timestamp internalToTimestamp(long v) { * @see #toLongOptional(Timestamp, TimeZone) * @see #toLongOptional(Timestamp) converse method */ - public static java.sql.@PolyNull Timestamp internalToTimestamp(@PolyNull Long v) { + public static java.sql.@Nullable Timestamp internalToTimestamp(@Nullable Long v) { return v == null ? castNonNull(null) : internalToTimestamp(v.longValue()); } @@ -6332,7 +6332,7 @@ public static int time(long timestampMillis, String timeZone) { / (1000L * 1000L)); // milli > micro > nano } - public static @PolyNull Long toTimestampWithLocalTimeZone(@PolyNull String v) { + public static @Nullable Long toTimestampWithLocalTimeZone(@Nullable String v) { if (v == null) { return castNonNull(null); } @@ -6342,7 +6342,7 @@ public static int time(long timestampMillis, String timeZone) { .getMillisSinceEpoch(); } - public static @PolyNull Long toTimestampWithLocalTimeZone(@PolyNull String v, + public static @Nullable Long toTimestampWithLocalTimeZone(@Nullable String v, TimeZone timeZone) { if (v == null) { return castNonNull(null); @@ -6356,7 +6356,7 @@ public static int time(long timestampMillis, String timeZone) { // Don't need shortValueOf etc. - Short.valueOf is sufficient. /** Helper for CAST(... AS VARCHAR(maxLength)). */ - public static @PolyNull String truncate(@PolyNull String s, int maxLength) { + public static @Nullable String truncate(@Nullable String s, int maxLength) { if (s == null) { return s; } else if (s.length() > maxLength) { @@ -6367,7 +6367,7 @@ public static int time(long timestampMillis, String timeZone) { } /** Helper for CAST(... AS CHAR(maxLength)). */ - public static @PolyNull String truncateOrPad(@PolyNull String s, int maxLength) { + public static @Nullable String truncateOrPad(@Nullable String s, int maxLength) { if (s == null) { return s; } else { @@ -6380,7 +6380,7 @@ public static int time(long timestampMillis, String timeZone) { } } - public static @PolyNull ByteString stringToBinary(@PolyNull String s, Charset charset) { + public static @Nullable ByteString stringToBinary(@Nullable String s, Charset charset) { if (s == null) { return null; } else { @@ -6388,7 +6388,7 @@ public static int time(long timestampMillis, String timeZone) { } } - public static @PolyNull ByteString byteArrayToByteString(byte @PolyNull [] bytes) { + public static @Nullable ByteString byteArrayToByteString(byte @Nullable [] bytes) { if (bytes == null) { return null; } else { @@ -6396,7 +6396,7 @@ public static int time(long timestampMillis, String timeZone) { } } - public static byte @PolyNull [] byteStringToByteArray(@PolyNull ByteString s) { + public static byte @Nullable [] byteStringToByteArray(@Nullable ByteString s) { if (s == null) { return null; } else { @@ -6405,7 +6405,7 @@ public static int time(long timestampMillis, String timeZone) { } /** Helper for CAST(... AS VARBINARY(maxLength)). */ - public static @PolyNull ByteString truncate(@PolyNull ByteString s, int maxLength) { + public static @Nullable ByteString truncate(@Nullable ByteString s, int maxLength) { if (s == null) { return s; } else if (s.length() > maxLength) { @@ -6416,7 +6416,7 @@ public static int time(long timestampMillis, String timeZone) { } /** Helper for CAST(... AS BINARY(maxLength)). */ - public static @PolyNull ByteString truncateOrPad(@PolyNull ByteString s, int maxLength) { + public static @Nullable ByteString truncateOrPad(@Nullable ByteString s, int maxLength) { if (s == null) { return s; } else { @@ -7030,12 +7030,12 @@ public static boolean isNotFalse(@Nullable Boolean b) { } /** NULL → NULL, FALSE → TRUE, TRUE → FALSE. */ - public static @PolyNull Boolean not(@PolyNull Boolean b) { + public static @Nullable Boolean not(@Nullable Boolean b) { return b == null ? castNonNull(null) : !b; } /** Converts a JDBC array to a list. */ - public static @PolyNull List arrayToList(final java.sql.@PolyNull Array a) { + public static @Nullable List arrayToList(final java.sql.@Nullable Array a) { if (a == null) { return castNonNull(null); } @@ -7090,7 +7090,7 @@ private static AtomicLong getAtomicLong(String key) { } /** Support the ARRAYS_ZIP function. */ - @SuppressWarnings("argument.type.incompatible") + @SuppressWarnings("NullAway") public static List arraysZip(List... lists) { final int biggestCardinality = lists.length == 0 ? 0 @@ -7142,7 +7142,7 @@ public static List distinct(List list) { } /** Support the ARRAY_MAX function. */ - public static @Nullable > T arrayMax( + public static @Nullable > T arrayMax( List list) { T max = null; @@ -7156,7 +7156,7 @@ public static List distinct(List list) { } /** Support the ARRAY_MIN function. */ - public static @Nullable > T arrayMin( + public static @Nullable > T arrayMin( List list) { T min = null; diff --git a/core/src/main/java/org/apache/calcite/schema/SchemaPlus.java b/core/src/main/java/org/apache/calcite/schema/SchemaPlus.java index bf8fab970b55..df4bb66fa800 100644 --- a/core/src/main/java/org/apache/calcite/schema/SchemaPlus.java +++ b/core/src/main/java/org/apache/calcite/schema/SchemaPlus.java @@ -93,7 +93,7 @@ default boolean removeTable(String name) { @Override boolean isMutable(); /** Returns an underlying object. */ - @Nullable T unwrap(Class clazz); + @Nullable T unwrap(Class clazz); void setPath(ImmutableList> path); diff --git a/core/src/main/java/org/apache/calcite/schema/Wrapper.java b/core/src/main/java/org/apache/calcite/schema/Wrapper.java index 6265c860f3da..29cf90ba2997 100644 --- a/core/src/main/java/org/apache/calcite/schema/Wrapper.java +++ b/core/src/main/java/org/apache/calcite/schema/Wrapper.java @@ -29,13 +29,13 @@ public interface Wrapper { /** Finds an instance of an interface implemented by this object, * or returns null if this object does not support that interface. */ - @Nullable C unwrap(Class aClass); + @Nullable C unwrap(Class aClass); /** Finds an instance of an interface implemented by this object, * or throws NullPointerException if this object does not support * that interface. */ @API(since = "1.27", status = API.Status.INTERNAL) - default C unwrapOrThrow(Class aClass) { + default C unwrapOrThrow(Class aClass) { return requireNonNull(unwrap(aClass), () -> "Can't unwrap " + aClass + " from " + this); } @@ -44,7 +44,7 @@ default C unwrapOrThrow(Class aClass) { * or returns {@link Optional#empty()} if this object does not support * that interface. */ @API(since = "1.27", status = API.Status.INTERNAL) - default Optional maybeUnwrap(Class aClass) { + default Optional maybeUnwrap(Class aClass) { return Optional.ofNullable(unwrap(aClass)); } } diff --git a/core/src/main/java/org/apache/calcite/schema/impl/AbstractTable.java b/core/src/main/java/org/apache/calcite/schema/impl/AbstractTable.java index 52f0806b6027..493afc10e4ed 100644 --- a/core/src/main/java/org/apache/calcite/schema/impl/AbstractTable.java +++ b/core/src/main/java/org/apache/calcite/schema/impl/AbstractTable.java @@ -48,7 +48,7 @@ protected AbstractTable() { return Schema.TableType.TABLE; } - @Override public @Nullable C unwrap(Class aClass) { + @Override public @Nullable C unwrap(Class aClass) { if (aClass.isInstance(this)) { return aClass.cast(this); } diff --git a/core/src/main/java/org/apache/calcite/schema/impl/ModifiableViewTable.java b/core/src/main/java/org/apache/calcite/schema/impl/ModifiableViewTable.java index 5b74e4285612..d72d45c18ac8 100644 --- a/core/src/main/java/org/apache/calcite/schema/impl/ModifiableViewTable.java +++ b/core/src/main/java/org/apache/calcite/schema/impl/ModifiableViewTable.java @@ -91,7 +91,7 @@ public ModifiableViewTable(Type elementType, RelProtoDataType rowType, return tablePath; } - @Override public @Nullable C unwrap(Class aClass) { + @Override public @Nullable C unwrap(Class aClass) { if (aClass.isInstance(initializerExpressionFactory)) { return aClass.cast(initializerExpressionFactory); } else if (aClass.isInstance(table)) { diff --git a/core/src/main/java/org/apache/calcite/schema/impl/StarTable.java b/core/src/main/java/org/apache/calcite/schema/impl/StarTable.java index 04c0fa5b5c0d..0e7592278474 100644 --- a/core/src/main/java/org/apache/calcite/schema/impl/StarTable.java +++ b/core/src/main/java/org/apache/calcite/schema/impl/StarTable.java @@ -16,6 +16,7 @@ */ package org.apache.calcite.schema.impl; +import org.apache.calcite.linq4j.annotations.MonotonicNonNull; import org.apache.calcite.materialize.Lattice; import org.apache.calcite.plan.Convention; import org.apache.calcite.plan.RelOptCluster; @@ -36,7 +37,6 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.jspecify.annotations.Nullable; import java.util.ArrayList; diff --git a/core/src/main/java/org/apache/calcite/server/DdlExecutorImpl.java b/core/src/main/java/org/apache/calcite/server/DdlExecutorImpl.java index 914b98f27b2a..0b35d45b219e 100644 --- a/core/src/main/java/org/apache/calcite/server/DdlExecutorImpl.java +++ b/core/src/main/java/org/apache/calcite/server/DdlExecutorImpl.java @@ -31,7 +31,7 @@ protected DdlExecutorImpl() { /** Dispatches calls to the appropriate method based on the type of the * first argument. */ - @SuppressWarnings({"method.invocation.invalid", "argument.type.incompatible"}) + @SuppressWarnings("NullAway") private final ReflectUtil.MethodDispatcher dispatcher = ReflectUtil.createMethodDispatcher(void.class, this, "execute", SqlNode.class, CalcitePrepare.Context.class); diff --git a/core/src/main/java/org/apache/calcite/sql/SqlAggFunction.java b/core/src/main/java/org/apache/calcite/sql/SqlAggFunction.java index 849f0968840e..913eb69497f5 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlAggFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlAggFunction.java @@ -119,7 +119,7 @@ protected SqlAggFunction( //~ Methods ---------------------------------------------------------------- - @Override public @Nullable T unwrap(Class clazz) { + @Override public @Nullable T unwrap(Class clazz) { return clazz.isInstance(this) ? clazz.cast(this) : null; } diff --git a/core/src/main/java/org/apache/calcite/sql/SqlAsofJoin.java b/core/src/main/java/org/apache/calcite/sql/SqlAsofJoin.java index 286e2df1df62..5321026824e0 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlAsofJoin.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlAsofJoin.java @@ -43,7 +43,7 @@ public SqlAsofJoin(SqlParserPos pos, SqlNode left, SqlLiteral natural, this.matchCondition = matchCondition; } - @SuppressWarnings("nullness") + @SuppressWarnings("NullAway") @Override public List getOperandList() { return ImmutableNullableList.of(left, natural, joinType, right, conditionType, condition, matchCondition); @@ -53,7 +53,7 @@ public SqlAsofJoin(SqlParserPos pos, SqlNode left, SqlLiteral natural, return ASOF_OPERATOR; } - @SuppressWarnings("assignment.type.incompatible") + @SuppressWarnings("NullAway") @Override public void setOperand(int i, @Nullable SqlNode operand) { switch (i) { case 0: @@ -107,7 +107,7 @@ private SqlAsofJoinOperator(String name, int prec) { return SqlSyntax.SPECIAL; } - @SuppressWarnings("argument.type.incompatible") + @SuppressWarnings("NullAway") @Override public SqlCall createCall( @Nullable SqlLiteral functionQualifier, SqlParserPos pos, diff --git a/core/src/main/java/org/apache/calcite/sql/SqlBasicCall.java b/core/src/main/java/org/apache/calcite/sql/SqlBasicCall.java index a49c278404d5..01af35095ee4 100755 --- a/core/src/main/java/org/apache/calcite/sql/SqlBasicCall.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlBasicCall.java @@ -110,7 +110,7 @@ public void setOperator(SqlOperator operator) { return operator; } - @SuppressWarnings("nullness") + @SuppressWarnings("NullAway") @Override public List getOperandList() { return operandList; } diff --git a/core/src/main/java/org/apache/calcite/sql/SqlCall.java b/core/src/main/java/org/apache/calcite/sql/SqlCall.java index 31aec4fa3559..7ad2ce061c49 100755 --- a/core/src/main/java/org/apache/calcite/sql/SqlCall.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlCall.java @@ -27,7 +27,6 @@ import org.apache.calcite.sql.validate.SqlValidatorScope; import org.apache.calcite.util.Litmus; -import org.checkerframework.dataflow.qual.Pure; import org.jspecify.annotations.Nullable; import java.util.ArrayList; @@ -76,7 +75,6 @@ public void setOperand(int i, @Nullable SqlNode operand) { return getOperator().getKind(); } - @Pure public abstract SqlOperator getOperator(); /** @@ -245,7 +243,6 @@ && operandCount() == 1) { return false; } - @Pure public @Nullable SqlLiteral getFunctionQuantifier() { return null; } diff --git a/core/src/main/java/org/apache/calcite/sql/SqlCallBinding.java b/core/src/main/java/org/apache/calcite/sql/SqlCallBinding.java index 8f1d1f020fab..1346ba0e46bb 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlCallBinding.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlCallBinding.java @@ -272,13 +272,13 @@ public SqlCall permutedCall() { throw new AssertionError(); } - @Override public @Nullable T getOperandLiteralValue(int ordinal, + @Override public @Nullable T getOperandLiteralValue(int ordinal, Class clazz) { final SqlNode node = operand(ordinal); return valueAs(node, clazz); } - private @Nullable T valueAs(SqlNode node, Class clazz) { + private @Nullable T valueAs(SqlNode node, Class clazz) { final SqlLiteral literal; switch (node.getKind()) { case ARRAY_VALUE_CONSTRUCTOR: diff --git a/core/src/main/java/org/apache/calcite/sql/SqlCollation.java b/core/src/main/java/org/apache/calcite/sql/SqlCollation.java index 7282ff431039..26e864f28739 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlCollation.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlCollation.java @@ -28,8 +28,6 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; -import org.checkerframework.checker.initialization.qual.UnderInitialization; -import org.checkerframework.dataflow.qual.Pure; import org.jspecify.annotations.Nullable; import java.io.Serializable; @@ -165,7 +163,6 @@ public SqlCollation( } protected String generateCollationName( - @UnderInitialization SqlCollation this, Charset charset) { return charset.name().toUpperCase(Locale.ROOT) + "$" + String.valueOf(locale) + "$" + strength; } @@ -340,7 +337,6 @@ public final Locale getLocale() { * collation, or {@code null} if no specific {@link Collator} is needed, in * which case {@link String#compareTo} will be used. */ - @Pure @JsonIgnore public @Nullable Collator getCollator() { return null; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlDelete.java b/core/src/main/java/org/apache/calcite/sql/SqlDelete.java index 4d085dcca565..edec0bae303c 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlDelete.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlDelete.java @@ -33,7 +33,7 @@ public class SqlDelete extends SqlCall { public static final SqlSpecialOperator OPERATOR = new SqlSpecialOperator("DELETE", SqlKind.DELETE) { - @SuppressWarnings("argument.type.incompatible") + @SuppressWarnings("NullAway") @Override public SqlCall createCall( @Nullable SqlLiteral functionQualifier, SqlParserPos pos, @@ -77,12 +77,12 @@ public SqlDelete( return OPERATOR; } - @SuppressWarnings("nullness") + @SuppressWarnings("NullAway") @Override public List getOperandList() { return ImmutableNullableList.of(targetTable, condition, sourceSelect, alias); } - @SuppressWarnings("assignment.type.incompatible") + @SuppressWarnings("NullAway") @Override public void setOperand(int i, @Nullable SqlNode operand) { switch (i) { case 0: diff --git a/core/src/main/java/org/apache/calcite/sql/SqlDescribeSchema.java b/core/src/main/java/org/apache/calcite/sql/SqlDescribeSchema.java index 34fa69f97e4d..80a5930ddf65 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlDescribeSchema.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlDescribeSchema.java @@ -31,7 +31,7 @@ public class SqlDescribeSchema extends SqlCall { public static final SqlSpecialOperator OPERATOR = new SqlSpecialOperator("DESCRIBE_SCHEMA", SqlKind.DESCRIBE_SCHEMA) { - @SuppressWarnings("argument.type.incompatible") + @SuppressWarnings("NullAway") @Override public SqlCall createCall(@Nullable SqlLiteral functionQualifier, SqlParserPos pos, @Nullable SqlNode... operands) { return new SqlDescribeSchema(pos, (SqlIdentifier) operands[0]); @@ -52,7 +52,7 @@ public SqlDescribeSchema(SqlParserPos pos, SqlIdentifier schema) { schema.unparse(writer, leftPrec, rightPrec); } - @SuppressWarnings("assignment.type.incompatible") + @SuppressWarnings("NullAway") @Override public void setOperand(int i, @Nullable SqlNode operand) { switch (i) { case 0: diff --git a/core/src/main/java/org/apache/calcite/sql/SqlDescribeTable.java b/core/src/main/java/org/apache/calcite/sql/SqlDescribeTable.java index 68150c251224..07ae6b204b09 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlDescribeTable.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlDescribeTable.java @@ -33,7 +33,7 @@ public class SqlDescribeTable extends SqlCall { public static final SqlSpecialOperator OPERATOR = new SqlSpecialOperator("DESCRIBE_TABLE", SqlKind.DESCRIBE_TABLE) { - @SuppressWarnings("argument.type.incompatible") + @SuppressWarnings("NullAway") @Override public SqlCall createCall(@Nullable SqlLiteral functionQualifier, SqlParserPos pos, @Nullable SqlNode... operands) { return new SqlDescribeTable(pos, (SqlIdentifier) operands[0], @@ -62,7 +62,7 @@ public SqlDescribeTable(SqlParserPos pos, } } - @SuppressWarnings("assignment.type.incompatible") + @SuppressWarnings("NullAway") @Override public void setOperand(int i, @Nullable SqlNode operand) { switch (i) { case 0: @@ -80,7 +80,7 @@ public SqlDescribeTable(SqlParserPos pos, return OPERATOR; } - @SuppressWarnings("nullness") + @SuppressWarnings("NullAway") @Override public List getOperandList() { return ImmutableNullableList.of(table, column); } diff --git a/core/src/main/java/org/apache/calcite/sql/SqlDialect.java b/core/src/main/java/org/apache/calcite/sql/SqlDialect.java index 7deaba883411..f2334f97c436 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlDialect.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlDialect.java @@ -47,7 +47,6 @@ import com.google.common.base.Suppliers; import com.google.common.collect.ImmutableSet; -import org.checkerframework.dataflow.qual.Pure; import org.jspecify.annotations.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -737,7 +736,6 @@ public DatabaseProduct getDatabaseProduct() { * Returns whether the dialect supports character set names as part of a * data type, for instance {@code VARCHAR(30) CHARACTER SET `ISO-8859-1`}. */ - @Pure public boolean supportsCharSet() { return true; } @@ -1548,7 +1546,7 @@ public enum DatabaseProduct { @SuppressWarnings("ImmutableEnumChecker") private final Supplier dialect; - @SuppressWarnings("argument.type.incompatible") + @SuppressWarnings("NullAway") DatabaseProduct(String databaseProductName, @Nullable String quoteString, NullCollation nullCollation) { requireNonNull(databaseProductName, "databaseProductName"); diff --git a/core/src/main/java/org/apache/calcite/sql/SqlExplain.java b/core/src/main/java/org/apache/calcite/sql/SqlExplain.java index ab9718fd42bc..62a6bf79ec9a 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlExplain.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlExplain.java @@ -19,7 +19,6 @@ import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.util.ImmutableNullableList; -import org.checkerframework.dataflow.qual.Pure; import org.jspecify.annotations.Nullable; import java.util.List; @@ -31,7 +30,7 @@ public class SqlExplain extends SqlCall { public static final SqlSpecialOperator OPERATOR = new SqlSpecialOperator("EXPLAIN", SqlKind.EXPLAIN) { - @SuppressWarnings("argument.type.incompatible") + @SuppressWarnings("NullAway") @Override public SqlCall createCall(@Nullable SqlLiteral functionQualifier, SqlParserPos pos, @Nullable SqlNode... operands) { return new SqlExplain(pos, operands[0], (SqlLiteral) operands[1], @@ -86,7 +85,7 @@ public SqlExplain(SqlParserPos pos, return ImmutableNullableList.of(explicandum, detailLevel, depth, format); } - @SuppressWarnings("assignment.type.incompatible") + @SuppressWarnings("NullAway") @Override public void setOperand(int i, @Nullable SqlNode operand) { switch (i) { case 0: @@ -109,7 +108,6 @@ public SqlExplain(SqlParserPos pos, /** * Returns the underlying SQL statement to be explained. */ - @Pure public SqlNode getExplicandum() { return explicandum; } @@ -117,7 +115,6 @@ public SqlNode getExplicandum() { /** * Return the detail level to be generated. */ - @Pure public SqlExplainLevel getDetailLevel() { return detailLevel.getValueAs(SqlExplainLevel.class); } @@ -125,7 +122,6 @@ public SqlExplainLevel getDetailLevel() { /** * Returns the level of abstraction at which this plan should be displayed. */ - @Pure public Depth getDepth() { return depth.getValueAs(Depth.class); } @@ -133,7 +129,6 @@ public Depth getDepth() { /** * Returns the number of dynamic parameters in the statement. */ - @Pure public int getDynamicParamCount() { return dynamicParameterCount; } @@ -141,7 +136,6 @@ public int getDynamicParamCount() { /** * Returns whether physical plan implementation should be returned. */ - @Pure public boolean withImplementation() { return getDepth() == Depth.PHYSICAL; } @@ -149,7 +143,6 @@ public boolean withImplementation() { /** * Returns whether type should be returned. */ - @Pure public boolean withType() { return getDepth() == Depth.TYPE; } @@ -157,7 +150,6 @@ public boolean withType() { /** * Returns the desired output format. */ - @Pure public SqlExplainFormat getFormat() { return format.getValueAs(SqlExplainFormat.class); } diff --git a/core/src/main/java/org/apache/calcite/sql/SqlFunction.java b/core/src/main/java/org/apache/calcite/sql/SqlFunction.java index afb7d10f06c5..d6ed2b9d39d8 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlFunction.java @@ -30,7 +30,6 @@ import org.apache.calcite.sql.validate.implicit.TypeCoercion; import org.apache.calcite.util.Util; -import org.checkerframework.dataflow.qual.Pure; import org.jspecify.annotations.Nullable; import java.util.List; @@ -191,7 +190,6 @@ public SqlFunctionCategory getFunctionType() { * ALL quantifier. The default is false; some aggregate * functions return true. */ - @Pure public boolean isQuantifierAllowed() { return false; } diff --git a/core/src/main/java/org/apache/calcite/sql/SqlIdentifier.java b/core/src/main/java/org/apache/calcite/sql/SqlIdentifier.java index 11c7fcf8e881..a55b133a4e84 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlIdentifier.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlIdentifier.java @@ -28,7 +28,6 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.dataflow.qual.Pure; import org.jspecify.annotations.Nullable; import java.util.ArrayList; @@ -324,7 +323,6 @@ public SqlIdentifier skipLast(int n) { return visitor.visit(this); } - @Pure public @Nullable SqlCollation getCollation() { return collation; } diff --git a/core/src/main/java/org/apache/calcite/sql/SqlInsert.java b/core/src/main/java/org/apache/calcite/sql/SqlInsert.java index c37f7726c510..29de324211f3 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlInsert.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlInsert.java @@ -21,7 +21,6 @@ import org.apache.calcite.sql.validate.SqlValidatorScope; import org.apache.calcite.util.ImmutableNullableList; -import org.checkerframework.dataflow.qual.Pure; import org.jspecify.annotations.Nullable; import java.util.List; @@ -35,7 +34,7 @@ public class SqlInsert extends SqlCall { public static final SqlSpecialOperator OPERATOR = new SqlSpecialOperator("INSERT", SqlKind.INSERT) { - @SuppressWarnings("argument.type.incompatible") + @SuppressWarnings("NullAway") @Override public SqlCall createCall(@Nullable SqlLiteral functionQualifier, SqlParserPos pos, @Nullable SqlNode... operands) { @@ -77,7 +76,7 @@ public SqlInsert(SqlParserPos pos, return OPERATOR; } - @SuppressWarnings("nullness") + @SuppressWarnings("NullAway") @Override public List getOperandList() { return ImmutableNullableList.of(keywords, targetTable, source, columnList); } @@ -91,7 +90,7 @@ public final boolean isUpsert() { return getModifierNode(SqlInsertKeyword.UPSERT) != null; } - @SuppressWarnings("assignment.type.incompatible") + @SuppressWarnings("NullAway") @Override public void setOperand(int i, @Nullable SqlNode operand) { switch (i) { case 0: @@ -134,7 +133,6 @@ public void setSource(SqlSelect source) { * Returns the list of target column names, or null for all columns in the * target table. */ - @Pure public @Nullable SqlNodeList getTargetColumnList() { return columnList; } diff --git a/core/src/main/java/org/apache/calcite/sql/SqlJdbcFunctionCall.java b/core/src/main/java/org/apache/calcite/sql/SqlJdbcFunctionCall.java index 73a20faf642a..2d06e78240e1 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlJdbcFunctionCall.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlJdbcFunctionCall.java @@ -674,7 +674,7 @@ private static class JdbcToInternalLookupTable { private final Map map; - @SuppressWarnings("method.invocation.invalid") + @SuppressWarnings("NullAway") private JdbcToInternalLookupTable() { // A table of all functions can be found at // http://java.sun.com/products/jdbc/driverdevs.html diff --git a/core/src/main/java/org/apache/calcite/sql/SqlJoin.java b/core/src/main/java/org/apache/calcite/sql/SqlJoin.java index a67774450971..e3d7ca13bef4 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlJoin.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlJoin.java @@ -96,13 +96,13 @@ public SqlJoin(SqlParserPos pos, SqlNode left, SqlLiteral natural, return SqlKind.JOIN; } - @SuppressWarnings("nullness") + @SuppressWarnings("NullAway") @Override public List getOperandList() { return ImmutableNullableList.of(left, natural, joinType, right, conditionType, condition); } - @SuppressWarnings("assignment.type.incompatible") + @SuppressWarnings("NullAway") @Override public void setOperand(int i, @Nullable SqlNode operand) { switch (i) { case 0: @@ -194,7 +194,7 @@ private SqlJoinOperator(String name, int prec) { return SqlSyntax.SPECIAL; } - @SuppressWarnings("argument.type.incompatible") + @SuppressWarnings("NullAway") @Override public SqlCall createCall( @Nullable SqlLiteral functionQualifier, SqlParserPos pos, diff --git a/core/src/main/java/org/apache/calcite/sql/SqlLiteral.java b/core/src/main/java/org/apache/calcite/sql/SqlLiteral.java index a5160d3abc40..666bb8055899 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlLiteral.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlLiteral.java @@ -297,7 +297,7 @@ public static boolean valueMatchesType( * * @throws AssertionError if the value type is not supported */ - public T getValueAs(Class clazz) { + public T getValueAs(Class clazz) { Object value = this.value; if (clazz.isInstance(value)) { return clazz.cast(value); diff --git a/core/src/main/java/org/apache/calcite/sql/SqlMatchRecognize.java b/core/src/main/java/org/apache/calcite/sql/SqlMatchRecognize.java index d45b4721e6a4..338652fe4faf 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlMatchRecognize.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlMatchRecognize.java @@ -105,7 +105,7 @@ public SqlMatchRecognize(SqlParserPos pos, SqlNode tableRef, SqlNode pattern, return SqlKind.MATCH_RECOGNIZE; } - @SuppressWarnings("nullness") + @SuppressWarnings("NullAway") @Override public List getOperandList() { return ImmutableNullableList.of(tableRef, pattern, strictStart, strictEnd, patternDefList, measureList, after, subsetList, rowsPerMatch, partitionList, orderList, @@ -121,7 +121,7 @@ public SqlMatchRecognize(SqlParserPos pos, SqlNode tableRef, SqlNode pattern, validator.validateMatchRecognize(this); } - @SuppressWarnings("assignment.type.incompatible") + @SuppressWarnings("NullAway") @Override public void setOperand(int i, @Nullable SqlNode operand) { switch (i) { case OPERAND_TABLE_REF: @@ -271,7 +271,7 @@ private SqlMatchRecognizeOperator() { return SqlSyntax.SPECIAL; } - @SuppressWarnings("argument.type.incompatible") + @SuppressWarnings("NullAway") @Override public SqlCall createCall( @Nullable SqlLiteral functionQualifier, SqlParserPos pos, diff --git a/core/src/main/java/org/apache/calcite/sql/SqlMerge.java b/core/src/main/java/org/apache/calcite/sql/SqlMerge.java index f533041d99eb..75eaf32ef135 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlMerge.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlMerge.java @@ -23,7 +23,6 @@ import org.apache.calcite.util.ImmutableNullableList; import org.apache.calcite.util.Pair; -import org.checkerframework.dataflow.qual.Pure; import org.jspecify.annotations.Nullable; import java.util.List; @@ -85,13 +84,13 @@ public SqlMerge(SqlParserPos pos, return SqlKind.MERGE; } - @SuppressWarnings("nullness") + @SuppressWarnings("NullAway") @Override public List<@Nullable SqlNode> getOperandList() { return ImmutableNullableList.of(targetTable, condition, source, updateCall, insertCall, sourceSelect, alias); } - @SuppressWarnings("assignment.type.incompatible") + @SuppressWarnings("NullAway") @Override public void setOperand(int i, @Nullable SqlNode operand) { switch (i) { case 0: @@ -127,7 +126,6 @@ public SqlNode getTargetTable() { } /** Returns the alias for the target table of this MERGE. */ - @Pure public @Nullable SqlIdentifier getAlias() { return alias; } diff --git a/core/src/main/java/org/apache/calcite/sql/SqlNodeList.java b/core/src/main/java/org/apache/calcite/sql/SqlNodeList.java index 06ff47ad3879..443393a14ec2 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlNodeList.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlNodeList.java @@ -136,27 +136,27 @@ public static SqlNodeList of(SqlParserPos pos, List<@Nullable SqlNode> list) { ((List) list).forEach(action); } - @SuppressWarnings("return.type.incompatible") + @SuppressWarnings("NullAway") @Override public Iterator iterator() { return list.iterator(); } - @SuppressWarnings("return.type.incompatible") + @SuppressWarnings("NullAway") @Override public ListIterator listIterator() { return list.listIterator(); } - @SuppressWarnings("return.type.incompatible") + @SuppressWarnings("NullAway") @Override public ListIterator listIterator(int index) { return list.listIterator(index); } - @SuppressWarnings("return.type.incompatible") + @SuppressWarnings("NullAway") @Override public List subList(int fromIndex, int toIndex) { return list.subList(fromIndex, toIndex); } - @SuppressWarnings("return.type.incompatible") + @SuppressWarnings("NullAway") @Override public /*Nullable*/ SqlNode get(int n) { return list.get(n); } @@ -181,14 +181,14 @@ public static SqlNodeList of(SqlParserPos pos, List<@Nullable SqlNode> list) { return list.lastIndexOf(o); } - @SuppressWarnings("return.type.incompatible") + @SuppressWarnings("NullAway") @Override public Object[] toArray() { // Per JDK specification, must return an Object[] not SqlNode[]; see e.g. // https://bugs.java.com/bugdatabase/view_bug.do?bug_id=6260652 return list.toArray(); } - @SuppressWarnings("return.type.incompatible") + @SuppressWarnings("NullAway") @Override public @Nullable T[] toArray(T @Nullable [] a) { return list.toArray(a); } diff --git a/core/src/main/java/org/apache/calcite/sql/SqlNumericLiteral.java b/core/src/main/java/org/apache/calcite/sql/SqlNumericLiteral.java index 949d68cc654f..ea7148e67584 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlNumericLiteral.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlNumericLiteral.java @@ -22,7 +22,6 @@ import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.util.Util; -import org.checkerframework.dataflow.qual.Pure; import org.jspecify.annotations.Nullable; import java.math.BigDecimal; @@ -66,7 +65,6 @@ private BigDecimal getValueNonNull() { return prec; } - @Pure public @Nullable Integer getScale() { return scale; } diff --git a/core/src/main/java/org/apache/calcite/sql/SqlOperator.java b/core/src/main/java/org/apache/calcite/sql/SqlOperator.java index b9a240bcb43c..b1edc4e57f54 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlOperator.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlOperator.java @@ -41,7 +41,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; -import org.checkerframework.dataflow.qual.Pure; import org.jspecify.annotations.Nullable; import java.util.List; @@ -247,7 +246,6 @@ public SqlIdentifier getNameAsId() { return new SqlIdentifier(getName(), SqlParserPos.ZERO); } - @Pure public SqlKind getKind() { return kind; } @@ -875,7 +873,6 @@ public String getAllowedSignatures(String opNameToUse) { * @return whether this operator is an analytic function (aggregate function * or window function) */ - @Pure public boolean isAggregator() { return false; } @@ -1032,7 +1029,6 @@ public void acceptCall( * * @see Strong */ - @Pure public @Nullable Supplier getStrongPolicyInference() { return null; } diff --git a/core/src/main/java/org/apache/calcite/sql/SqlOperatorBinding.java b/core/src/main/java/org/apache/calcite/sql/SqlOperatorBinding.java index cf2241265382..2102d242715b 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlOperatorBinding.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlOperatorBinding.java @@ -154,7 +154,7 @@ public int getIntLiteralOperand(int ordinal) { * * @return value of operand */ - public @Nullable T getOperandLiteralValue(int ordinal, Class clazz) { + public @Nullable T getOperandLiteralValue(int ordinal, Class clazz) { throw new UnsupportedOperationException(); } diff --git a/core/src/main/java/org/apache/calcite/sql/SqlOrderBy.java b/core/src/main/java/org/apache/calcite/sql/SqlOrderBy.java index 6ba5ee872af3..674732a2a64f 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlOrderBy.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlOrderBy.java @@ -33,7 +33,7 @@ */ public class SqlOrderBy extends SqlCall { public static final SqlSpecialOperator OPERATOR = new Operator() { - @SuppressWarnings("argument.type.incompatible") + @SuppressWarnings("NullAway") @Override public SqlCall createCall(@Nullable SqlLiteral functionQualifier, SqlParserPos pos, @Nullable SqlNode... operands) { return new SqlOrderBy(pos, operands[0], (SqlNodeList) operands[1], @@ -67,7 +67,7 @@ public SqlOrderBy(SqlParserPos pos, SqlNode query, SqlNodeList orderList, return OPERATOR; } - @SuppressWarnings("nullness") + @SuppressWarnings("NullAway") @Override public List getOperandList() { return ImmutableNullableList.of(query, orderList, offset, fetch); } diff --git a/core/src/main/java/org/apache/calcite/sql/SqlPivot.java b/core/src/main/java/org/apache/calcite/sql/SqlPivot.java index c7bc8c586a77..00c50f883aef 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlPivot.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlPivot.java @@ -76,7 +76,7 @@ public SqlPivot(SqlParserPos pos, SqlNode query, SqlNodeList aggList, return ImmutableNullableList.of(query, aggList, axisList, inList); } - @SuppressWarnings("nullness") + @SuppressWarnings("NullAway") @Override public void setOperand(int i, @Nullable SqlNode operand) { // Only 'query' is mutable. (It is required for validation.) switch (i) { diff --git a/core/src/main/java/org/apache/calcite/sql/SqlSelect.java b/core/src/main/java/org/apache/calcite/sql/SqlSelect.java index 4d8f09975831..7a8b67abef85 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlSelect.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlSelect.java @@ -16,14 +16,13 @@ */ package org.apache.calcite.sql; +import org.apache.calcite.linq4j.annotations.EnsuresNonNullIf; import org.apache.calcite.sql.fun.SqlInternalOperators; import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.sql.validate.SqlValidator; import org.apache.calcite.sql.validate.SqlValidatorScope; import org.apache.calcite.util.ImmutableNullableList; -import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf; -import org.checkerframework.dataflow.qual.Pure; import org.jspecify.annotations.Nullable; import java.util.List; @@ -141,7 +140,7 @@ public SqlSelect(SqlParserPos pos, return SqlKind.SELECT; } - @SuppressWarnings("nullness") + @SuppressWarnings("NullAway") @Override public List getOperandList() { return ImmutableNullableList.of(keywordList, selectList, from, where, groupBy, having, windowDecls, qualify, orderBy, offset, fetch, hints, @@ -214,7 +213,6 @@ public final boolean isDistinct() { return null; } - @Pure public final @Nullable SqlNode getFrom() { return from; } @@ -223,7 +221,6 @@ public void setFrom(@Nullable SqlNode from) { this.from = from; } - @Pure public final @Nullable SqlNodeList getGroup() { return groupBy; } @@ -232,7 +229,6 @@ public void setGroupBy(@Nullable SqlNodeList groupBy) { this.groupBy = groupBy; } - @Pure public final @Nullable SqlNode getHaving() { return having; } @@ -241,7 +237,6 @@ public void setHaving(@Nullable SqlNode having) { this.having = having; } - @Pure public final SqlNodeList getSelectList() { return selectList; } @@ -250,7 +245,6 @@ public void setSelectList(SqlNodeList selectList) { this.selectList = selectList; } - @Pure public final @Nullable SqlNode getWhere() { return where; } @@ -263,7 +257,6 @@ public final SqlNodeList getWindowList() { return windowDecls; } - @Pure public final @Nullable SqlNode getQualify() { return qualify; } @@ -272,7 +265,6 @@ public void setQualify(@Nullable SqlNode qualify) { this.qualify = qualify; } - @Pure public final @Nullable SqlNodeList getOrderList() { return orderBy; } @@ -289,7 +281,6 @@ public boolean hasByClause() { return hasByClause; } - @Pure public final @Nullable SqlNode getOffset() { return offset; } @@ -298,7 +289,6 @@ public void setOffset(@Nullable SqlNode offset) { this.offset = offset; } - @Pure public final @Nullable SqlNode getFetch() { return fetch; } @@ -307,12 +297,11 @@ public void setFetch(@Nullable SqlNode fetch) { this.fetch = fetch; } - @Pure public @Nullable SqlNodeList getHints() { return this.hints; } - @EnsuresNonNullIf(expression = "hints", result = true) + @EnsuresNonNullIf(value = "hints", result = true) public boolean hasHints() { // The hints may be passed as null explicitly. return this.hints != null && !this.hints.isEmpty(); diff --git a/core/src/main/java/org/apache/calcite/sql/SqlSetOption.java b/core/src/main/java/org/apache/calcite/sql/SqlSetOption.java index a1030afdcec1..9af4a9fb897c 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlSetOption.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlSetOption.java @@ -64,7 +64,7 @@ public class SqlSetOption extends SqlAlter { public static final SqlSpecialOperator OPERATOR = new SqlSpecialOperator("SET_OPTION", SqlKind.SET_OPTION) { - @SuppressWarnings("argument.type.incompatible") + @SuppressWarnings("NullAway") @Override public SqlCall createCall(@Nullable SqlLiteral functionQualifier, SqlParserPos pos, @Nullable SqlNode... operands) { final SqlNode scopeNode = operands[0]; @@ -135,7 +135,7 @@ public SqlSetOption(SqlParserPos pos, @Nullable String scope, SqlIdentifier name return OPERATOR; } - @SuppressWarnings("nullness") + @SuppressWarnings("NullAway") @Override public List getOperandList() { final List<@Nullable SqlNode> operandList = new ArrayList<>(); if (scope == null) { diff --git a/core/src/main/java/org/apache/calcite/sql/SqlSnapshot.java b/core/src/main/java/org/apache/calcite/sql/SqlSnapshot.java index d7b6ad835c4e..09de8dd059af 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlSnapshot.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlSnapshot.java @@ -96,7 +96,7 @@ private SqlSnapshotOperator() { return SqlSyntax.SPECIAL; } - @SuppressWarnings("argument.type.incompatible") + @SuppressWarnings("NullAway") @Override public SqlCall createCall( @Nullable SqlLiteral functionQualifier, SqlParserPos pos, diff --git a/core/src/main/java/org/apache/calcite/sql/SqlStarExclude.java b/core/src/main/java/org/apache/calcite/sql/SqlStarExclude.java index 17fa21947064..ec76666226b4 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlStarExclude.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlStarExclude.java @@ -33,7 +33,7 @@ public class SqlStarExclude extends SqlCall { public static final SqlOperator OPERATOR = new SqlSpecialOperator("STAR_EXCLUDE", SqlKind.OTHER) { - @SuppressWarnings("argument.type.incompatible") + @SuppressWarnings("NullAway") @Override public SqlCall createCall( @Nullable SqlLiteral functionQualifier, SqlParserPos pos, diff --git a/core/src/main/java/org/apache/calcite/sql/SqlStarReplace.java b/core/src/main/java/org/apache/calcite/sql/SqlStarReplace.java index f7775af0e3b6..bab7827f9063 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlStarReplace.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlStarReplace.java @@ -32,7 +32,7 @@ public class SqlStarReplace extends SqlCall { public static final SqlOperator OPERATOR = new SqlSpecialOperator("SELECT_STAR_REPLACE", SqlKind.OTHER) { - @SuppressWarnings("argument.type.incompatible") + @SuppressWarnings("NullAway") @Override public SqlCall createCall( @Nullable SqlLiteral functionQualifier, SqlParserPos pos, diff --git a/core/src/main/java/org/apache/calcite/sql/SqlSyntax.java b/core/src/main/java/org/apache/calcite/sql/SqlSyntax.java index d7157b2a31cf..59f2aa9fbf9d 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlSyntax.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlSyntax.java @@ -19,7 +19,6 @@ import org.apache.calcite.sql.validate.SqlConformance; import org.apache.calcite.util.Util; -import org.checkerframework.checker.initialization.qual.NotOnlyInitialized; import org.jspecify.annotations.Nullable; /** @@ -180,7 +179,6 @@ public enum SqlSyntax { }; /** Syntax to treat this syntax as equivalent to when resolving operators. */ - @NotOnlyInitialized public final SqlSyntax family; SqlSyntax() { diff --git a/core/src/main/java/org/apache/calcite/sql/SqlUnpivot.java b/core/src/main/java/org/apache/calcite/sql/SqlUnpivot.java index 0087fe3cd6c5..d99815a0c62e 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlUnpivot.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlUnpivot.java @@ -84,7 +84,7 @@ public SqlUnpivot(SqlParserPos pos, SqlNode query, boolean includeNulls, SqlLiteral.createBoolean(includeNulls, SqlParserPos.ZERO)); } - @SuppressWarnings("nullness") + @SuppressWarnings("NullAway") @Override public void setOperand(int i, @Nullable SqlNode operand) { // Only 'query' is mutable. (It is required for validation.) switch (i) { diff --git a/core/src/main/java/org/apache/calcite/sql/SqlUpdate.java b/core/src/main/java/org/apache/calcite/sql/SqlUpdate.java index 60c0e2463cf2..6047b0dee3b1 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlUpdate.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlUpdate.java @@ -23,7 +23,6 @@ import org.apache.calcite.util.ImmutableNullableList; import org.apache.calcite.util.Pair; -import org.checkerframework.dataflow.qual.Pure; import org.jspecify.annotations.Nullable; import java.util.List; @@ -37,7 +36,7 @@ public class SqlUpdate extends SqlCall { public static final SqlSpecialOperator OPERATOR = new SqlSpecialOperator("UPDATE", SqlKind.UPDATE) { - @SuppressWarnings("argument.type.incompatible") + @SuppressWarnings("NullAway") @Override public SqlCall createCall(@Nullable SqlLiteral functionQualifier, SqlParserPos pos, @Nullable SqlNode... operands) { @@ -90,13 +89,13 @@ public SqlUpdate(SqlParserPos pos, return OPERATOR; } - @SuppressWarnings("nullness") + @SuppressWarnings("NullAway") @Override public List<@Nullable SqlNode> getOperandList() { return ImmutableNullableList.of(targetTable, targetColumnList, sourceExpressionList, condition, sourceSelect, alias); } - @SuppressWarnings("assignment.type.incompatible") + @SuppressWarnings("NullAway") @Override public void setOperand(int i, @Nullable SqlNode operand) { switch (i) { case 0: @@ -129,7 +128,6 @@ public SqlNode getTargetTable() { } /** Returns the alias for the target table of this UPDATE. */ - @Pure public @Nullable SqlIdentifier getAlias() { return alias; } diff --git a/core/src/main/java/org/apache/calcite/sql/SqlUtil.java b/core/src/main/java/org/apache/calcite/sql/SqlUtil.java index c14171fab120..48049342ace6 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlUtil.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlUtil.java @@ -56,7 +56,6 @@ import com.google.common.collect.Iterators; import com.google.common.collect.Lists; -import org.checkerframework.checker.nullness.qual.PolyNull; import org.jspecify.annotations.Nullable; import java.math.BigDecimal; @@ -715,7 +714,7 @@ private static Iterator filterRoutinesByParameterTypeAndName( return true; } final SqlOperandMetadata operandMetadata = (SqlOperandMetadata) operandTypeChecker; - @SuppressWarnings("assignment.type.incompatible") + @SuppressWarnings("NullAway") final List<@Nullable RelDataType> paramTypes = operandMetadata.paramTypes(typeFactory); final List<@Nullable RelDataType> permutedArgTypes; @@ -1131,7 +1130,7 @@ public static void validateCharset(ByteString value, Charset charset) { /** If a node is "AS", returns the underlying expression; otherwise returns * the node. Returns null if and only if the node is null. */ - public static @PolyNull SqlNode stripAs(@PolyNull SqlNode node) { + public static @Nullable SqlNode stripAs(@Nullable SqlNode node) { if (node != null && node.getKind() == SqlKind.AS) { return ((SqlCall) node).operand(0); } diff --git a/core/src/main/java/org/apache/calcite/sql/SqlWindow.java b/core/src/main/java/org/apache/calcite/sql/SqlWindow.java index 01b561a8af71..f960fe856300 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlWindow.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlWindow.java @@ -17,6 +17,7 @@ package org.apache.calcite.sql; import org.apache.calcite.linq4j.Ord; +import org.apache.calcite.linq4j.annotations.EnsuresNonNullIf; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexWindowBound; import org.apache.calcite.rex.RexWindowBounds; @@ -35,8 +36,6 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf; -import org.checkerframework.dataflow.qual.Pure; import org.jspecify.annotations.Nullable; import java.math.BigDecimal; @@ -179,13 +178,13 @@ public static SqlWindow create(@Nullable SqlIdentifier declName, @Nullable SqlId return SqlKind.WINDOW; } - @SuppressWarnings("nullness") + @SuppressWarnings("NullAway") @Override public List getOperandList() { return ImmutableNullableList.of(declName, refName, partitionList, orderList, isRows, lowerBound, upperBound, allowPartial, exclude); } - @SuppressWarnings("assignment.type.incompatible") + @SuppressWarnings("NullAway") @Override public void setOperand(int i, @Nullable SqlNode operand) { switch (i) { case 0: @@ -305,7 +304,6 @@ public void setRows(SqlLiteral isRows) { this.isRows = isRows; } - @Pure public boolean isRows() { return isRows.booleanValue(); } @@ -597,7 +595,7 @@ private static boolean setOperand(@Nullable SqlNode clonedOperand, @Nullable Sql * (for example, a window of size 1 hour which has only 45 minutes of data * in it) will appear to windowed aggregate functions to be empty. */ - @EnsuresNonNullIf(expression = "allowPartial", result = false) + @EnsuresNonNullIf(value = "allowPartial", result = false) public boolean isAllowPartial() { // Default (and standard behavior) is to allow partial windows. return allowPartial == null @@ -915,7 +913,7 @@ private SqlWindowOperator() { return SqlSyntax.SPECIAL; } - @SuppressWarnings("argument.type.incompatible") + @SuppressWarnings("NullAway") @Override public SqlCall createCall( @Nullable SqlLiteral functionQualifier, SqlParserPos pos, diff --git a/core/src/main/java/org/apache/calcite/sql/SqlWith.java b/core/src/main/java/org/apache/calcite/sql/SqlWith.java index 89e036ba3d52..317fbc35b985 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlWith.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlWith.java @@ -55,7 +55,7 @@ public SqlWith(SqlParserPos pos, SqlNodeList withList, SqlNode body) { return ImmutableList.of(withList, body); } - @SuppressWarnings("assignment.type.incompatible") + @SuppressWarnings("NullAway") @Override public void setOperand(int i, @Nullable SqlNode operand) { switch (i) { case 0: @@ -115,7 +115,7 @@ private SqlWithOperator() { } - @SuppressWarnings("argument.type.incompatible") + @SuppressWarnings("NullAway") @Override public SqlCall createCall(@Nullable SqlLiteral functionQualifier, SqlParserPos pos, @Nullable SqlNode... operands) { return new SqlWith(pos, (SqlNodeList) operands[0], operands[1]); diff --git a/core/src/main/java/org/apache/calcite/sql/SqlWithItem.java b/core/src/main/java/org/apache/calcite/sql/SqlWithItem.java index 01f5059a3f1d..fb1e105292ff 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlWithItem.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlWithItem.java @@ -56,12 +56,12 @@ public SqlWithItem(SqlParserPos pos, SqlIdentifier name, return SqlKind.WITH_ITEM; } - @SuppressWarnings("nullness") + @SuppressWarnings("NullAway") @Override public List getOperandList() { return ImmutableNullableList.of(name, columnList, query, recursive); } - @SuppressWarnings("assignment.type.incompatible") + @SuppressWarnings("NullAway") @Override public void setOperand(int i, @Nullable SqlNode operand) { switch (i) { case 0: @@ -113,7 +113,7 @@ private static class SqlWithItemOperator extends SqlSpecialOperator { withItem.query.unparse(writer, MDX_PRECEDENCE, MDX_PRECEDENCE); } - @SuppressWarnings("argument.type.incompatible") + @SuppressWarnings("NullAway") @Override public SqlCall createCall(@Nullable SqlLiteral functionQualifier, SqlParserPos pos, @Nullable SqlNode... operands) { assert functionQualifier == null; diff --git a/core/src/main/java/org/apache/calcite/sql/SqlWriter.java b/core/src/main/java/org/apache/calcite/sql/SqlWriter.java index cbc987980015..1764a8f16170 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlWriter.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlWriter.java @@ -19,7 +19,6 @@ import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.util.SqlString; -import org.checkerframework.dataflow.qual.Pure; import org.jspecify.annotations.Nullable; import java.util.function.Consumer; @@ -342,7 +341,6 @@ public static FrameType create(final String name) { * convert to upper or lower case. Does not add quotation marks. Adds * preceding whitespace if necessary. */ - @Pure void literal(String s); /** @@ -350,13 +348,11 @@ public static FrameType create(final String name) { * contain a space. For example, keyword("SELECT"), * keyword("CHARACTER SET"). */ - @Pure void keyword(String s); /** * Prints a string, preceded by whitespace if necessary. */ - @Pure void print(String s); /** @@ -364,7 +360,6 @@ public static FrameType create(final String name) { * * @param x Integer */ - @Pure void print(int x); /** @@ -436,7 +431,6 @@ public static FrameType create(final String name) { * * @see #endFunCall(Frame) */ - @Pure Frame startFunCall(String funName); /** @@ -445,13 +439,11 @@ public static FrameType create(final String name) { * @param frame Frame * @see #startFunCall(String) */ - @Pure void endFunCall(Frame frame); /** * Starts a list. */ - @Pure Frame startList(String open, String close); /** @@ -460,7 +452,6 @@ public static FrameType create(final String name) { * @param frameType Type of list. For example, a SELECT list will be * governed according to SELECT-list formatting preferences. */ - @Pure Frame startList(FrameTypeEnum frameType); /** @@ -472,7 +463,6 @@ public static FrameType create(final String name) { * string. * @param close String to close the list */ - @Pure Frame startList(FrameType frameType, String open, String close); /** @@ -480,13 +470,11 @@ public static FrameType create(final String name) { * * @param frame The frame which was created by {@link #startList}. */ - @Pure void endList(@Nullable Frame frame); /** * Writes a list. */ - @Pure SqlWriter list(FrameTypeEnum frameType, Consumer action); /** @@ -495,7 +483,6 @@ public static FrameType create(final String name) { * {@link SqlStdOperatorTable#OR OR}, or * {@link #COMMA COMMA}). */ - @Pure SqlWriter list(FrameTypeEnum frameType, SqlBinaryOperator sepOp, SqlNodeList list); @@ -505,7 +492,6 @@ SqlWriter list(FrameTypeEnum frameType, SqlBinaryOperator sepOp, * * @param sep List separator, typically ",". */ - @Pure void sep(String sep); /** @@ -514,13 +500,11 @@ SqlWriter list(FrameTypeEnum frameType, SqlBinaryOperator sepOp, * @param sep List separator, typically "," * @param printFirst Whether to print the first occurrence of the separator */ - @Pure void sep(String sep, boolean printFirst); /** * Sets whether whitespace is needed before the next token. */ - @Pure void setNeedWhitespace(boolean needWhitespace); /** diff --git a/core/src/main/java/org/apache/calcite/sql/advise/SqlAdvisor.java b/core/src/main/java/org/apache/calcite/sql/advise/SqlAdvisor.java index 0d773ae67b71..0da2f56901af 100644 --- a/core/src/main/java/org/apache/calcite/sql/advise/SqlAdvisor.java +++ b/core/src/main/java/org/apache/calcite/sql/advise/SqlAdvisor.java @@ -17,6 +17,7 @@ package org.apache.calcite.sql.advise; import org.apache.calcite.avatica.util.Casing; +import org.apache.calcite.linq4j.annotations.EnsuresNonNull; import org.apache.calcite.runtime.CalciteContextException; import org.apache.calcite.runtime.CalciteException; import org.apache.calcite.sql.SqlIdentifier; @@ -38,7 +39,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; -import org.checkerframework.checker.nullness.qual.EnsuresNonNull; import org.jspecify.annotations.Nullable; import org.slf4j.Logger; diff --git a/core/src/main/java/org/apache/calcite/sql/ddl/SqlAttributeDefinition.java b/core/src/main/java/org/apache/calcite/sql/ddl/SqlAttributeDefinition.java index f1a796f3bda6..9ef3fbcd2ff8 100644 --- a/core/src/main/java/org/apache/calcite/sql/ddl/SqlAttributeDefinition.java +++ b/core/src/main/java/org/apache/calcite/sql/ddl/SqlAttributeDefinition.java @@ -72,7 +72,7 @@ public class SqlAttributeDefinition extends SqlCall { return OPERATOR; } - @SuppressWarnings("nullness") + @SuppressWarnings("NullAway") @Override public List getOperandList() { return ImmutableNullableList.of(name, dataType, expression, collation != null ? collation.asList() : null); diff --git a/core/src/main/java/org/apache/calcite/sql/ddl/SqlCheckConstraint.java b/core/src/main/java/org/apache/calcite/sql/ddl/SqlCheckConstraint.java index 0bc6ba734a67..69427c1fdc49 100644 --- a/core/src/main/java/org/apache/calcite/sql/ddl/SqlCheckConstraint.java +++ b/core/src/main/java/org/apache/calcite/sql/ddl/SqlCheckConstraint.java @@ -64,7 +64,7 @@ public class SqlCheckConstraint extends SqlCall { return OPERATOR; } - @SuppressWarnings("nullness") + @SuppressWarnings("NullAway") @Override public List getOperandList() { return ImmutableNullableList.of(name, expression); } diff --git a/core/src/main/java/org/apache/calcite/sql/ddl/SqlColumnDeclaration.java b/core/src/main/java/org/apache/calcite/sql/ddl/SqlColumnDeclaration.java index aa54f671a7e8..6e39b1fb6f81 100644 --- a/core/src/main/java/org/apache/calcite/sql/ddl/SqlColumnDeclaration.java +++ b/core/src/main/java/org/apache/calcite/sql/ddl/SqlColumnDeclaration.java @@ -76,7 +76,7 @@ public class SqlColumnDeclaration extends SqlCall { return OPERATOR; } - @SuppressWarnings("nullness") + @SuppressWarnings("NullAway") @Override public List getOperandList() { return ImmutableNullableList.of(name, dataType, expression, strategy != null ? new SqlIdentifier(strategy.name(), SqlParserPos.ZERO) : null); diff --git a/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateForeignSchema.java b/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateForeignSchema.java index eabd59fd1775..5e945d2066aa 100644 --- a/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateForeignSchema.java +++ b/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateForeignSchema.java @@ -79,7 +79,7 @@ public class SqlCreateForeignSchema extends SqlCreate { this.optionList = optionList; // may be null } - @SuppressWarnings("nullness") + @SuppressWarnings("NullAway") @Override public List getOperandList() { return ImmutableNullableList.of( SqlLiteral.createBoolean(getReplace(), SqlParserPos.ZERO), diff --git a/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateMaterializedView.java b/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateMaterializedView.java index 3a28c8aeb8ac..0ffbfbbd6bb8 100644 --- a/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateMaterializedView.java +++ b/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateMaterializedView.java @@ -66,7 +66,7 @@ public class SqlCreateMaterializedView extends SqlCreate { this.query = requireNonNull(query, "query"); } - @SuppressWarnings("nullness") + @SuppressWarnings("NullAway") @Override public List getOperandList() { return ImmutableNullableList.of( SqlLiteral.createBoolean(getReplace(), SqlParserPos.ZERO), diff --git a/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateTable.java b/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateTable.java index 76c296c965ad..84b05b0a9b50 100644 --- a/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateTable.java +++ b/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateTable.java @@ -71,7 +71,7 @@ protected SqlCreateTable(SqlParserPos pos, boolean replace, boolean ifNotExists, this(OPERATOR, pos, replace, ifNotExists, name, columnList, query); } - @SuppressWarnings("nullness") + @SuppressWarnings("NullAway") @Override public List getOperandList() { return ImmutableNullableList.of( SqlLiteral.createBoolean(getReplace(), SqlParserPos.ZERO), diff --git a/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateType.java b/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateType.java index 500456b6bae5..5a3c363b9e7a 100644 --- a/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateType.java +++ b/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateType.java @@ -65,7 +65,7 @@ public class SqlCreateType extends SqlCreate { this.dataType = dataType; // may be null } - @SuppressWarnings("nullness") + @SuppressWarnings("NullAway") @Override public List getOperandList() { return ImmutableNullableList.of( SqlLiteral.createBoolean(getReplace(), SqlParserPos.ZERO), diff --git a/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateView.java b/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateView.java index 131f11d86984..821d4f778fcb 100644 --- a/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateView.java +++ b/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateView.java @@ -64,7 +64,7 @@ public class SqlCreateView extends SqlCreate { this.query = requireNonNull(query, "query"); } - @SuppressWarnings("nullness") + @SuppressWarnings("NullAway") @Override public List getOperandList() { return ImmutableNullableList.of( SqlLiteral.createBoolean(getReplace(), SqlParserPos.ZERO), diff --git a/core/src/main/java/org/apache/calcite/sql/ddl/SqlKeyConstraint.java b/core/src/main/java/org/apache/calcite/sql/ddl/SqlKeyConstraint.java index a7423cc67fdd..7ed8b6eb326f 100644 --- a/core/src/main/java/org/apache/calcite/sql/ddl/SqlKeyConstraint.java +++ b/core/src/main/java/org/apache/calcite/sql/ddl/SqlKeyConstraint.java @@ -91,7 +91,7 @@ public static SqlKeyConstraint primary(SqlParserPos pos, SqlIdentifier name, return UNIQUE; } - @SuppressWarnings("nullness") + @SuppressWarnings("NullAway") @Override public List getOperandList() { return ImmutableNullableList.of(name, columnList); } diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlAnyValueAggFunction.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlAnyValueAggFunction.java index 284223fa767d..6ffb90af45a9 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlAnyValueAggFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlAnyValueAggFunction.java @@ -55,7 +55,7 @@ public SqlAnyValueAggFunction(SqlKind kind) { //~ Methods ---------------------------------------------------------------- - @Override public @Nullable T unwrap(Class clazz) { + @Override public @Nullable T unwrap(Class clazz) { if (clazz.isInstance(SqlSplittableAggFunction.SelfSplitter.INSTANCE)) { return clazz.cast(SqlSplittableAggFunction.SelfSplitter.INSTANCE); } diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlBasicAggFunction.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlBasicAggFunction.java index 6576553d0cc6..7a74b3e5f500 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlBasicAggFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlBasicAggFunction.java @@ -103,7 +103,7 @@ public static SqlBasicAggFunction create(String name, SqlKind kind, //~ Methods ---------------------------------------------------------------- - @Override public @Nullable T unwrap(Class clazz) { + @Override public @Nullable T unwrap(Class clazz) { if (clazz.isInstance(staticFun)) { return clazz.cast(staticFun); } diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlBitOpAggFunction.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlBitOpAggFunction.java index b908da9923a4..adabecabae88 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlBitOpAggFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlBitOpAggFunction.java @@ -73,7 +73,7 @@ public SqlBitOpAggFunction(String name, SqlKind kind) { || kind == SqlKind.BIT_XOR); } - @Override public @Nullable T unwrap(Class clazz) { + @Override public @Nullable T unwrap(Class clazz) { if (clazz.isInstance(SqlSplittableAggFunction.SelfSplitter.INSTANCE)) { return clazz.cast(SqlSplittableAggFunction.SelfSplitter.INSTANCE); } diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlCase.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlCase.java index 6cc2277438f2..eb9c246f826a 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlCase.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlCase.java @@ -102,12 +102,12 @@ public static SqlCase createSwitched(SqlParserPos pos, @Nullable SqlNode value, return SqlStdOperatorTable.CASE; } - @SuppressWarnings("nullness") + @SuppressWarnings("NullAway") @Override public List getOperandList() { return UnmodifiableArrayList.of(value, whenList, thenList, elseExpr); } - @SuppressWarnings("assignment.type.incompatible") + @SuppressWarnings("NullAway") @Override public void setOperand(int i, @Nullable SqlNode operand) { switch (i) { case 0: diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlCaseOperator.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlCaseOperator.java index 403200956012..23b68cd39823 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlCaseOperator.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlCaseOperator.java @@ -330,7 +330,7 @@ private static RelDataType inferTypeFromOperands(SqlOperatorBinding opBinding) { return SqlSyntax.SPECIAL; } - @SuppressWarnings("argument.type.incompatible") + @SuppressWarnings("NullAway") @Override public SqlCall createCall( @Nullable SqlLiteral functionQualifier, SqlParserPos pos, diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlCountAggFunction.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlCountAggFunction.java index 7429365f6559..80b6186fdf2b 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlCountAggFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlCountAggFunction.java @@ -91,7 +91,7 @@ public SqlCountAggFunction(String name, return super.deriveType(validator, scope, call); } - @Override public @Nullable T unwrap(Class clazz) { + @Override public @Nullable T unwrap(Class clazz) { if (clazz.isInstance(SqlSplittableAggFunction.CountSplitter.INSTANCE)) { return clazz.cast(SqlSplittableAggFunction.CountSplitter.INSTANCE); } diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlGroupingFunction.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlGroupingFunction.java index 499611ef60f4..c6dd1b4977c2 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlGroupingFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlGroupingFunction.java @@ -84,7 +84,7 @@ class SqlGroupingFunction extends SqlAbstractGroupFunction { return null; } - @Override public @Nullable T unwrap(Class clazz) { + @Override public @Nullable T unwrap(Class clazz) { if (clazz.isInstance(STATIC)) { return clazz.cast(STATIC); } diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlInternalOperators.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlInternalOperators.java index bb15edb98493..e86830b156fd 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlInternalOperators.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlInternalOperators.java @@ -57,7 +57,7 @@ private SqlInternalOperators() { new SqlRowOperator("$ANONYMOUS_ROW") { @Override public void unparse(SqlWriter writer, SqlCall call, int leftPrec, int rightPrec) { - @SuppressWarnings("assignment.type.incompatible") + @SuppressWarnings("NullAway") List<@Nullable SqlNode> operandList = call.getOperandList(); writer.list(SqlWriter.FrameTypeEnum.PARENTHESES, SqlWriter.COMMA, SqlNodeList.of(call.getParserPosition(), operandList)); diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperatorTableFactory.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperatorTableFactory.java index 7cea1c9bde0f..645e30bdc2ce 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperatorTableFactory.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperatorTableFactory.java @@ -70,7 +70,7 @@ private SqlLibraryOperatorTableFactory(Class... classes) { /** A cache that returns an operator table for a given library (or set of * libraries). */ - @SuppressWarnings("methodref.receiver.bound.invalid") + @SuppressWarnings("NullAway") private final LoadingCache, SqlOperatorTable> cache = CacheBuilder.newBuilder().build(CacheLoader.from(this::create)); diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java index 50495094fc16..d58dc0ba751c 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java @@ -1441,7 +1441,7 @@ private static RelDataType mapReturnType(SqlOperatorBinding opBinding) { OperandTypes.MAP_FUNCTION, SqlFunctionCategory.SYSTEM); - @SuppressWarnings("argument.type.incompatible") + @SuppressWarnings("NullAway") private static RelDataType arrayAppendPrependReturnType(SqlOperatorBinding opBinding) { final RelDataType arrayType = opBinding.collectOperandTypes().get(0); final RelDataType componentType = arrayType.getComponentType(); @@ -1493,7 +1493,7 @@ private static RelDataType arrayAppendPrependReturnType(SqlOperatorBinding opBin ReturnTypes.BOOLEAN_NULLABLE, OperandTypes.EXISTS); - @SuppressWarnings("argument.type.incompatible") + @SuppressWarnings("NullAway") private static RelDataType arrayCompactReturnType(SqlOperatorBinding opBinding) { final RelDataType arrayType = opBinding.collectOperandTypes().get(0); if (arrayType.getSqlTypeName() == SqlTypeName.NULL) { @@ -1573,7 +1573,7 @@ private static RelDataType arrayCompactReturnType(SqlOperatorBinding opBinding) OperandTypes.SAME_SAME, OperandTypes.family(SqlTypeFamily.ARRAY, SqlTypeFamily.ARRAY))); - @SuppressWarnings("argument.type.incompatible") + @SuppressWarnings("NullAway") private static RelDataType arrayInsertReturnType(SqlOperatorBinding opBinding) { final List operandTypes = opBinding.collectOperandTypes(); final RelDataType arrayType = operandTypes.get(0); diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlMapValueConstructor.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlMapValueConstructor.java index 9ad150bf7129..52d56ce6f900 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlMapValueConstructor.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlMapValueConstructor.java @@ -46,7 +46,7 @@ public SqlMapValueConstructor() { super("MAP", SqlKind.MAP_VALUE_CONSTRUCTOR, null); } - @SuppressWarnings("argument.type.incompatible") + @SuppressWarnings("NullAway") @Override public RelDataType inferReturnType(SqlOperatorBinding opBinding) { Pair<@Nullable RelDataType, @Nullable RelDataType> type = getComponentTypes( diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlMinMaxAggFunction.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlMinMaxAggFunction.java index a14db7292076..5bde15271ab5 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlMinMaxAggFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlMinMaxAggFunction.java @@ -145,7 +145,7 @@ public int getMinMaxKind() { } } - @Override public @Nullable T unwrap(Class clazz) { + @Override public @Nullable T unwrap(Class clazz) { if (clazz.isInstance(SqlSplittableAggFunction.SelfSplitter.INSTANCE)) { return clazz.cast(SqlSplittableAggFunction.SelfSplitter.INSTANCE); } diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlSingleValueAggFunction.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlSingleValueAggFunction.java index bdf2f0484e3a..910e3630c26f 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlSingleValueAggFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlSingleValueAggFunction.java @@ -81,7 +81,7 @@ public RelDataType getType() { return type; } - @Override public @Nullable T unwrap(Class clazz) { + @Override public @Nullable T unwrap(Class clazz) { if (clazz.isInstance(SqlSplittableAggFunction.SelfSplitter.INSTANCE)) { return clazz.cast(SqlSplittableAggFunction.SelfSplitter.INSTANCE); } diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlSumAggFunction.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlSumAggFunction.java index 3dbd165265f5..5ff0d4184667 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlSumAggFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlSumAggFunction.java @@ -79,7 +79,7 @@ public RelDataType getType() { return type; } - @Override public @Nullable T unwrap(Class clazz) { + @Override public @Nullable T unwrap(Class clazz) { if (clazz.isInstance(SqlSplittableAggFunction.SumSplitter.INSTANCE)) { return clazz.cast(SqlSplittableAggFunction.SumSplitter.INSTANCE); } diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlSumEmptyIsZeroAggFunction.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlSumEmptyIsZeroAggFunction.java index dea9b4f9ac32..b32ee67606ad 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlSumEmptyIsZeroAggFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlSumEmptyIsZeroAggFunction.java @@ -70,7 +70,7 @@ public SqlSumEmptyIsZeroAggFunction() { typeFactory.createSqlType(SqlTypeName.ANY), true); } - @Override public @Nullable T unwrap(Class clazz) { + @Override public @Nullable T unwrap(Class clazz) { if (clazz.isInstance(SqlSplittableAggFunction.Sum0Splitter.INSTANCE)) { return clazz.cast(SqlSplittableAggFunction.Sum0Splitter.INSTANCE); } diff --git a/core/src/main/java/org/apache/calcite/sql/parser/SqlAbstractParserImpl.java b/core/src/main/java/org/apache/calcite/sql/parser/SqlAbstractParserImpl.java index 9977e504f6e8..5b8a4bb9e54c 100644 --- a/core/src/main/java/org/apache/calcite/sql/parser/SqlAbstractParserImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/parser/SqlAbstractParserImpl.java @@ -37,7 +37,6 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; -import org.checkerframework.checker.initialization.qual.UnderInitialization; import org.jspecify.annotations.Nullable; import java.io.Reader; @@ -432,7 +431,7 @@ public static Set getSql92ReservedWords() { * @param operands Operands to call * @return Call */ - @SuppressWarnings("argument.type.incompatible") + @SuppressWarnings("NullAway") protected SqlCall createCall( SqlIdentifier funName, SqlParserPos pos, @@ -741,7 +740,6 @@ public MetadataImpl(SqlAbstractParserImpl sqlParser) { * Initializes lists of keywords. */ private void initList( - @UnderInitialization MetadataImpl this, SqlAbstractParserImpl parserImpl, Set keywords, String name) { @@ -788,7 +786,6 @@ private void initList( * @return Result of calling method */ private @Nullable Object virtualCall( - @UnderInitialization MetadataImpl this, SqlAbstractParserImpl parserImpl, String name) throws Throwable { Class clazz = parserImpl.getClass(); @@ -804,8 +801,7 @@ private void initList( /** * Builds a comma-separated list of JDBC reserved words. */ - private String constructSql92ReservedWordList( - @UnderInitialization MetadataImpl this) { + private String constructSql92ReservedWordList() { StringBuilder sb = new StringBuilder(); TreeSet jdbcReservedSet = new TreeSet<>(); jdbcReservedSet.addAll(tokenSet); diff --git a/core/src/main/java/org/apache/calcite/sql/pretty/SqlPrettyWriter.java b/core/src/main/java/org/apache/calcite/sql/pretty/SqlPrettyWriter.java index 3216aeaaf34b..e4c987f20994 100644 --- a/core/src/main/java/org/apache/calcite/sql/pretty/SqlPrettyWriter.java +++ b/core/src/main/java/org/apache/calcite/sql/pretty/SqlPrettyWriter.java @@ -287,7 +287,7 @@ public class SqlPrettyWriter implements SqlWriter { //~ Constructors ----------------------------------------------------------- - @SuppressWarnings("method.invocation.invalid") + @SuppressWarnings("NullAway") private SqlPrettyWriter(SqlWriterConfig config, StringBuilder buf, @SuppressWarnings("unused") boolean ignore) { this.buf = requireNonNull(buf, "buf"); diff --git a/core/src/main/java/org/apache/calcite/sql/type/CompositeOperandTypeChecker.java b/core/src/main/java/org/apache/calcite/sql/type/CompositeOperandTypeChecker.java index 5eb719cc6869..28983291e650 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/CompositeOperandTypeChecker.java +++ b/core/src/main/java/org/apache/calcite/sql/type/CompositeOperandTypeChecker.java @@ -27,7 +27,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; -import org.checkerframework.checker.nullness.qual.UnknownKeyFor; import org.jspecify.annotations.Nullable; import java.util.AbstractList; @@ -88,9 +87,7 @@ public enum Composition { //~ Instance fields -------------------------------------------------------- - // It is not clear if @UnknownKeyFor is needed here or not, however, checkerframework inference - // fails otherwise, see https://github.com/typetools/checker-framework/issues/4048 - protected final ImmutableList<@UnknownKeyFor ? extends SqlOperandTypeChecker> allowedRules; + protected final ImmutableList allowedRules; protected final Composition composition; private final @Nullable String allowedSignatures; private final @Nullable BiFunction signatureGenerator; diff --git a/core/src/main/java/org/apache/calcite/sql/type/OperandTypes.java b/core/src/main/java/org/apache/calcite/sql/type/OperandTypes.java index fb55e214e2b2..c4912279d27a 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/OperandTypes.java +++ b/core/src/main/java/org/apache/calcite/sql/type/OperandTypes.java @@ -650,7 +650,7 @@ public static SqlOperandTypeChecker variadic( new FamilyOperandTypeChecker( ImmutableList.of(SqlTypeFamily.ARRAY, SqlTypeFamily.CHARACTER, SqlTypeFamily.CHARACTER), i -> i == 2) { - @SuppressWarnings("argument.type.incompatible") + @SuppressWarnings("NullAway") @Override public boolean checkOperandTypes( SqlCallBinding callBinding, boolean throwOnFailure) { diff --git a/core/src/main/java/org/apache/calcite/sql/type/SqlTypeUtil.java b/core/src/main/java/org/apache/calcite/sql/type/SqlTypeUtil.java index e15885cc0e89..596b07965ada 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/SqlTypeUtil.java +++ b/core/src/main/java/org/apache/calcite/sql/type/SqlTypeUtil.java @@ -48,7 +48,6 @@ import com.google.common.collect.Sets; import org.apiguardian.api.API; -import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf; import org.jspecify.annotations.Nullable; import java.math.BigDecimal; @@ -486,16 +485,13 @@ public static boolean isTimestamp(RelDataType type) { } /** Returns whether a type is some kind of INTERVAL. */ - @SuppressWarnings("contracts.conditional.postcondition.not.satisfied") - @EnsuresNonNullIf(expression = "#1.getIntervalQualifier()", result = true) + @SuppressWarnings("NullAway") public static boolean isInterval(RelDataType type) { return SqlTypeFamily.DATETIME_INTERVAL.contains(type); } /** Returns whether a type is in SqlTypeFamily.Character. */ - @SuppressWarnings("contracts.conditional.postcondition.not.satisfied") - @EnsuresNonNullIf(expression = "#1.getCharset()", result = true) - @EnsuresNonNullIf(expression = "#1.getCollation()", result = true) + @SuppressWarnings("NullAway") public static boolean inCharFamily(RelDataType type) { return type.getFamily() == SqlTypeFamily.CHARACTER; } diff --git a/core/src/main/java/org/apache/calcite/sql/type/TableFunctionReturnTypeInference.java b/core/src/main/java/org/apache/calcite/sql/type/TableFunctionReturnTypeInference.java index 9220c2efc43c..c0328e87a68d 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/TableFunctionReturnTypeInference.java +++ b/core/src/main/java/org/apache/calcite/sql/type/TableFunctionReturnTypeInference.java @@ -16,6 +16,8 @@ */ package org.apache.calcite.sql.type; +import org.apache.calcite.linq4j.annotations.MonotonicNonNull; +import org.apache.calcite.linq4j.annotations.RequiresNonNull; import org.apache.calcite.rel.metadata.RelColumnMapping; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeField; @@ -23,8 +25,6 @@ import org.apache.calcite.sql.SqlCallBinding; import org.apache.calcite.sql.SqlOperatorBinding; -import org.checkerframework.checker.nullness.qual.MonotonicNonNull; -import org.checkerframework.checker.nullness.qual.RequiresNonNull; import org.jspecify.annotations.Nullable; import java.util.ArrayList; diff --git a/core/src/main/java/org/apache/calcite/sql/util/SqlBasicVisitor.java b/core/src/main/java/org/apache/calcite/sql/util/SqlBasicVisitor.java index effc9edcb564..8156b286e2e8 100644 --- a/core/src/main/java/org/apache/calcite/sql/util/SqlBasicVisitor.java +++ b/core/src/main/java/org/apache/calcite/sql/util/SqlBasicVisitor.java @@ -36,7 +36,7 @@ * * @param Return type */ -public class SqlBasicVisitor<@Nullable R> implements SqlVisitor { +public class SqlBasicVisitor implements SqlVisitor { //~ Methods ---------------------------------------------------------------- @Override public R visit(SqlLiteral literal) { @@ -77,7 +77,7 @@ public class SqlBasicVisitor<@Nullable R> implements SqlVisitor { /** Argument handler. * * @param result type */ - public interface ArgHandler { + public interface ArgHandler { /** Returns the result of visiting all children of a call to an operator, * then the call itself. * @@ -102,7 +102,7 @@ R visitChild( * * @param result type */ - public static class ArgHandlerImpl<@Nullable R> implements ArgHandler { + public static class ArgHandlerImpl implements ArgHandler { private static final ArgHandler INSTANCE = new ArgHandlerImpl<>(); @SuppressWarnings("unchecked") diff --git a/core/src/main/java/org/apache/calcite/sql/util/SqlString.java b/core/src/main/java/org/apache/calcite/sql/util/SqlString.java index 3848f4c78b2d..316338413e79 100644 --- a/core/src/main/java/org/apache/calcite/sql/util/SqlString.java +++ b/core/src/main/java/org/apache/calcite/sql/util/SqlString.java @@ -20,7 +20,6 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.dataflow.qual.Pure; import org.jspecify.annotations.Nullable; import static java.util.Objects.requireNonNull; @@ -96,7 +95,6 @@ public String getSql() { * * @return indices of dynamic parameters */ - @Pure public @Nullable ImmutableList getDynamicParameters() { return dynamicParameters; } diff --git a/core/src/main/java/org/apache/calcite/sql/validate/DelegatingScope.java b/core/src/main/java/org/apache/calcite/sql/validate/DelegatingScope.java index fd10d5c4b418..8cbbe69db3cb 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/DelegatingScope.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/DelegatingScope.java @@ -74,7 +74,7 @@ public abstract class DelegatingScope implements SqlValidatorScope { /** Computes and stores information that cannot be computed on construction, * but only after sub-queries have been validated. */ - @SuppressWarnings({"methodref.receiver.bound.invalid"}) + @SuppressWarnings("NullAway") public final Supplier resolved = Suppliers.memoize(this::resolve); diff --git a/core/src/main/java/org/apache/calcite/sql/validate/DelegatingSqlValidatorCatalogReader.java b/core/src/main/java/org/apache/calcite/sql/validate/DelegatingSqlValidatorCatalogReader.java index c996133d766d..09bee79a7cf4 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/DelegatingSqlValidatorCatalogReader.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/DelegatingSqlValidatorCatalogReader.java @@ -58,7 +58,7 @@ protected DelegatingSqlValidatorCatalogReader( return catalogReader.getSchemaPaths(); } - @Override public @Nullable C unwrap(Class aClass) { + @Override public @Nullable C unwrap(Class aClass) { return catalogReader.unwrap(aClass); } } diff --git a/core/src/main/java/org/apache/calcite/sql/validate/IdentifierNamespace.java b/core/src/main/java/org/apache/calcite/sql/validate/IdentifierNamespace.java index 17c6fe5c4468..afecff602d87 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/IdentifierNamespace.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/IdentifierNamespace.java @@ -16,6 +16,7 @@ */ package org.apache.calcite.sql.validate; +import org.apache.calcite.linq4j.annotations.MonotonicNonNull; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.sql.SqlCall; @@ -28,7 +29,6 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.jspecify.annotations.Nullable; import java.util.List; diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SelectScope.java b/core/src/main/java/org/apache/calcite/sql/validate/SelectScope.java index b61212b627ad..bc03aa188ead 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SelectScope.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SelectScope.java @@ -16,6 +16,7 @@ */ package org.apache.calcite.sql.validate; +import org.apache.calcite.linq4j.annotations.MonotonicNonNull; import org.apache.calcite.sql.SqlCall; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlNode; @@ -27,7 +28,6 @@ import org.apache.calcite.util.Litmus; import org.apache.calcite.util.Pair; -import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.jspecify.annotations.Nullable; import java.util.ArrayList; diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlNonNullableAccessors.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlNonNullableAccessors.java index 7fbca5f888fc..0a18f5a28b45 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlNonNullableAccessors.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlNonNullableAccessors.java @@ -109,7 +109,7 @@ public static SqlValidatorNamespace getNamespace(SqlCallBinding callBinding) { } @API(since = "1.27", status = API.Status.EXPERIMENTAL) - public static T getOperandLiteralValueOrThrow(SqlOperatorBinding opBinding, + public static T getOperandLiteralValueOrThrow(SqlOperatorBinding opBinding, int ordinal, Class clazz) { return requireNonNull(opBinding.getOperandLiteralValue(ordinal, clazz), () -> "expected non-null operand " + ordinal + " in " + safeToString(opBinding)); diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidator.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidator.java index 6750235c2651..d6018330f219 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidator.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidator.java @@ -53,7 +53,6 @@ import org.apache.calcite.sql.validate.implicit.TypeCoercions; import org.apiguardian.api.API; -import org.checkerframework.dataflow.qual.Pure; import org.immutables.value.Value; import org.jspecify.annotations.Nullable; @@ -125,7 +124,6 @@ public interface SqlValidator { * * @return catalog reader */ - @Pure SqlValidatorCatalogReader getCatalogReader(); /** @@ -133,7 +131,6 @@ public interface SqlValidator { * * @return operator table */ - @Pure SqlOperatorTable getOperatorTable(); /** @@ -511,7 +508,6 @@ SqlNodeList expandStar(SqlNodeList selectList, SqlSelect query, * * @return type factory */ - @Pure RelDataTypeFactory getTypeFactory(); /** diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorException.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorException.java index 12fbdbb3190d..a681e43a358b 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorException.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorException.java @@ -50,7 +50,7 @@ public class SqlValidatorException extends Exception * @param message error message * @param cause underlying cause */ - @SuppressWarnings({"argument.type.incompatible", "method.invocation.invalid"}) + @SuppressWarnings("NullAway") public SqlValidatorException( String message, Throwable cause) { diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java index 7932caf0f7d7..94fd2f0de89a 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java @@ -17,6 +17,7 @@ package org.apache.calcite.sql.validate; import org.apache.calcite.linq4j.Ord; +import org.apache.calcite.linq4j.annotations.RequiresNonNull; import org.apache.calcite.linq4j.function.Functions; import org.apache.calcite.plan.RelOptTable; import org.apache.calcite.plan.RelOptUtil; @@ -132,11 +133,6 @@ import com.google.common.collect.Sets; import org.apiguardian.api.API; -import org.checkerframework.checker.initialization.qual.UnknownInitialization; -import org.checkerframework.checker.nullness.qual.KeyFor; -import org.checkerframework.checker.nullness.qual.PolyNull; -import org.checkerframework.checker.nullness.qual.RequiresNonNull; -import org.checkerframework.dataflow.qual.Pure; import org.jspecify.annotations.NonNull; import org.jspecify.annotations.Nullable; import org.slf4j.Logger; @@ -344,7 +340,7 @@ protected SqlValidatorImpl( groupFinder = new AggFinder(opTab, false, false, true, null, nameMatcher); aggOrOverOrGroupFinder = new AggFinder(opTab, true, true, true, null, nameMatcher); - @SuppressWarnings("argument.type.incompatible") + @SuppressWarnings("NullAway") TypeCoercion typeCoercion = config.typeCoercionFactory().create(typeFactory, this); this.typeCoercion = typeCoercion; @@ -378,17 +374,14 @@ public SqlConformance getConformance() { return config.conformance(); } - @Pure @Override public SqlValidatorCatalogReader getCatalogReader() { return catalogReader; } - @Pure @Override public SqlOperatorTable getOperatorTable() { return opTab; } - @Pure @Override public RelDataTypeFactory getTypeFactory() { return typeFactory; } @@ -1820,8 +1813,8 @@ && requireNonNull(((SqlNumericLiteral) node).bigDecimalValue()) * @param underFrom whether node appears directly under a FROM clause * @return rewritten expression, or null if the original expression is null */ - protected @PolyNull SqlNode performUnconditionalRewrites( - @PolyNull SqlNode node, + protected @Nullable SqlNode performUnconditionalRewrites( + @Nullable SqlNode node, boolean underFrom) { if (node == null) { return null; @@ -6355,8 +6348,8 @@ private void checkConstraint( @SuppressWarnings("RedundantCast") final ImmutableBitSet constrainedColumns = ImmutableBitSet.of((Iterable) projectMap.keySet()); - @SuppressWarnings("assignment.type.incompatible") - List<@KeyFor({"tableIndexToTargetField", "projectMap"}) Integer> constrainedTargetColumns = + @SuppressWarnings("NullAway") + List constrainedTargetColumns = targetColumns.intersect(constrainedColumns).asList(); // Validate insert values against the view constraint. @@ -8875,7 +8868,7 @@ static class ExtendedExpander extends Expander { * Add all possible expandable 'group by' ordinals to {@link aliasOrdinalExpandSet}. */ @RequiresNonNull({"root"}) - private void addExpandableOrdinals(@UnknownInitialization ExtendedExpander this) { + private void addExpandableOrdinals() { switch (root.getKind()) { case LITERAL: aliasOrdinalExpandSet.add(root); @@ -8900,7 +8893,7 @@ private void addExpandableOrdinals(@UnknownInitialization ExtendedExpander this) * * @param sqlNode expression within grouping sets, rollup, cube */ - private void addOrdinal2ExpandSet(@UnknownInitialization ExtendedExpander this, + private void addOrdinal2ExpandSet( SqlNode sqlNode) { if (sqlNode.getKind() == SqlKind.ROW) { List rowOperandList = ((SqlCall) sqlNode).getOperandList(); diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorNamespace.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorNamespace.java index 23a0f6ceba44..8e4efe31b850 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorNamespace.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorNamespace.java @@ -21,7 +21,6 @@ import org.apache.calcite.sql.SqlNode; import org.apache.calcite.util.Pair; -import org.checkerframework.dataflow.qual.Pure; import org.jspecify.annotations.Nullable; import java.util.List; @@ -127,7 +126,6 @@ public interface SqlValidatorNamespace { * includes all decorations. If there are no decorations, returns the same * as {@link #getNode()}. */ - @Pure @Nullable SqlNode getEnclosingNode(); /** diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorUtil.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorUtil.java index ace6909646d3..cd5ff191a194 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorUtil.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorUtil.java @@ -246,10 +246,7 @@ public static ImmutableBitSet getOrdinalBitSet( ImmutableBitSet.of( Util.transform(sourceRowType.getFieldList(), RelDataTypeField::getIndex)); - // checkerframework: found : Set<@KeyFor("indexToField") Integer> - //noinspection RedundantCast - ImmutableBitSet target = - ImmutableBitSet.of((Iterable) indexToField.keySet()); + ImmutableBitSet target = ImmutableBitSet.of(indexToField.keySet()); return source.intersect(target); } diff --git a/core/src/main/java/org/apache/calcite/sql2rel/CorrelationReferenceFinder.java b/core/src/main/java/org/apache/calcite/sql2rel/CorrelationReferenceFinder.java index 0c18c357c89b..f6d2b3ecae4e 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/CorrelationReferenceFinder.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/CorrelationReferenceFinder.java @@ -25,15 +25,11 @@ import org.apache.calcite.rex.RexShuttle; import org.apache.calcite.rex.RexSubQuery; -import org.checkerframework.checker.initialization.qual.NotOnlyInitialized; -import org.checkerframework.checker.initialization.qual.UnderInitialization; - /** * Shuttle that finds references to a given {@link CorrelationId} within a tree * of {@link RelNode}s. */ public abstract class CorrelationReferenceFinder extends RelHomogeneousShuttle { - @NotOnlyInitialized private final MyRexVisitor rexVisitor; /** Creates CorrelationReferenceFinder. */ @@ -52,10 +48,9 @@ protected CorrelationReferenceFinder() { * Replaces alternative names of correlation variable to its canonical name. */ private static class MyRexVisitor extends RexShuttle { - @NotOnlyInitialized private final CorrelationReferenceFinder finder; - private MyRexVisitor(@UnderInitialization CorrelationReferenceFinder finder) { + private MyRexVisitor(CorrelationReferenceFinder finder) { this.finder = finder; } diff --git a/core/src/main/java/org/apache/calcite/sql2rel/DeduplicateCorrelateVariables.java b/core/src/main/java/org/apache/calcite/sql2rel/DeduplicateCorrelateVariables.java index b965b48f6443..e9c69d5126a7 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/DeduplicateCorrelateVariables.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/DeduplicateCorrelateVariables.java @@ -27,15 +27,11 @@ import com.google.common.collect.ImmutableSet; -import org.checkerframework.checker.initialization.qual.NotOnlyInitialized; -import org.checkerframework.checker.initialization.qual.UnderInitialization; - /** * Rewrites relations to ensure the same correlation is referenced by the same * correlation variable. */ public class DeduplicateCorrelateVariables extends RelHomogeneousShuttle { - @NotOnlyInitialized private final RexShuttle dedupRex; /** Creates a DeduplicateCorrelateVariables. */ @@ -69,12 +65,11 @@ private static class DeduplicateCorrelateVariablesShuttle extends RexShuttle { private final RexBuilder builder; private final CorrelationId canonicalId; private final ImmutableSet alternateIds; - @NotOnlyInitialized private final DeduplicateCorrelateVariables shuttle; private DeduplicateCorrelateVariablesShuttle(RexBuilder builder, CorrelationId canonicalId, ImmutableSet alternateIds, - @UnderInitialization DeduplicateCorrelateVariables shuttle) { + DeduplicateCorrelateVariables shuttle) { this.builder = builder; this.canonicalId = canonicalId; this.alternateIds = alternateIds; diff --git a/core/src/main/java/org/apache/calcite/sql2rel/ReflectiveConvertletTable.java b/core/src/main/java/org/apache/calcite/sql2rel/ReflectiveConvertletTable.java index ed1028b463eb..1e2e2cc90316 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/ReflectiveConvertletTable.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/ReflectiveConvertletTable.java @@ -16,14 +16,13 @@ */ package org.apache.calcite.sql2rel; +import org.apache.calcite.linq4j.annotations.RequiresNonNull; import org.apache.calcite.rex.RexNode; import org.apache.calcite.sql.SqlCall; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.parser.SqlParserPos; -import org.checkerframework.checker.initialization.qual.UnderInitialization; -import org.checkerframework.checker.nullness.qual.RequiresNonNull; import org.jspecify.annotations.Nullable; import java.lang.reflect.InvocationTargetException; @@ -66,7 +65,6 @@ public ReflectiveConvertletTable() { */ @RequiresNonNull("map") private void registerNodeTypeMethod( - @UnderInitialization ReflectiveConvertletTable this, final Method method) { if (!isPublic(method)) { return; @@ -90,7 +88,7 @@ private void registerNodeTypeMethod( } map.put(parameterType, (SqlRexConvertlet) (cx, call) -> { try { - @SuppressWarnings("argument.type.incompatible") + @SuppressWarnings("NullAway") RexNode result = (RexNode) method.invoke(ReflectiveConvertletTable.this, cx, call); return requireNonNull(result, @@ -109,7 +107,6 @@ private void registerNodeTypeMethod( */ @RequiresNonNull("map") private void registerOpTypeMethod( - @UnderInitialization ReflectiveConvertletTable this, final Method method) { if (!isPublic(method)) { return; @@ -137,7 +134,7 @@ private void registerOpTypeMethod( } map.put(opClass, (SqlRexConvertlet) (cx, call) -> { try { - @SuppressWarnings("argument.type.incompatible") + @SuppressWarnings("NullAway") RexNode result = (RexNode) method.invoke(ReflectiveConvertletTable.this, cx, call.getOperator(), call); @@ -192,7 +189,6 @@ private void registerOpTypeMethod( * @param convertlet Convertlet */ protected void registerOp( - @UnderInitialization ReflectiveConvertletTable this, SqlOperator op, SqlRexConvertlet convertlet) { map.put(op, convertlet); } @@ -204,7 +200,6 @@ protected void registerOp( * @param target Operator to translate calls to */ protected void addAlias( - @UnderInitialization ReflectiveConvertletTable this, final SqlOperator alias, final SqlOperator target) { map.put( alias, (SqlRexConvertlet) (cx, call) -> { diff --git a/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java b/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java index 7edfc2a4fd3e..68f14f2701e7 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java @@ -169,7 +169,7 @@ public class RelDecorrelator implements ReflectiveVisitor { * Each entry maps a CorrelationId to the Frame where its correlated variables originate. */ protected final Deque> frameStack = new ArrayDeque<>(); - @SuppressWarnings("method.invocation.invalid") + @SuppressWarnings("NullAway") protected final ReflectUtil.MethodDispatcher<@Nullable Frame> dispatcher = ReflectUtil.createMethodDispatcher( Frame.class, getVisitor(), "decorrelateRel", diff --git a/core/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java b/core/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java index 7210d4f397e8..c2bf100e4891 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java @@ -132,7 +132,7 @@ public class RelFieldTrimmer implements ReflectiveVisitor { public RelFieldTrimmer(@Nullable SqlValidator validator, RelBuilder relBuilder) { Util.discard(validator); // may be useful one day this.relBuilder = relBuilder; - @SuppressWarnings("argument.type.incompatible") + @SuppressWarnings("NullAway") ReflectUtil.MethodDispatcher dispatcher = ReflectUtil.createMethodDispatcher( TrimResult.class, diff --git a/core/src/main/java/org/apache/calcite/sql2rel/RelStructuredTypeFlattener.java b/core/src/main/java/org/apache/calcite/sql2rel/RelStructuredTypeFlattener.java index da142b295923..cd14be3a7e69 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/RelStructuredTypeFlattener.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/RelStructuredTypeFlattener.java @@ -86,7 +86,6 @@ import com.google.common.collect.Lists; import com.google.common.collect.SortedSetMultimap; -import org.checkerframework.common.value.qual.MinLen; import org.jspecify.annotations.Nullable; import java.util.ArrayDeque; @@ -1044,7 +1043,7 @@ private RelDataType removeDistinct(RelDataType type) { private RexNode flattenComparison( RexBuilder rexBuilder, SqlOperator op, - @MinLen(1) List exprs) { + List exprs) { final PairList flattenedExps = PairList.of(); flattenProjections(this, exprs, null, "", flattenedExps); int n = flattenedExps.size() / 2; diff --git a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java index 3e73ccdf52b2..d1e2054b1028 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java @@ -4636,7 +4636,7 @@ protected RelNode convertColumnList(final SqlInsert call, RelNode source) { } // sourceExps should not contain nulls (see the loop above) - @SuppressWarnings("assignment.type.incompatible") + @SuppressWarnings("NullAway") List nonNullExprs = sourceExps; return relBuilder.push(source) @@ -4682,7 +4682,7 @@ private static InitializerExpressionFactory getInitializerFactory( return NullInitializerExpressionFactory.INSTANCE; } - private static @Nullable T unwrap(@Nullable Object o, Class clazz) { + private static @Nullable T unwrap(@Nullable Object o, Class clazz) { if (o instanceof Wrapper) { return ((Wrapper) o).unwrap(clazz); } diff --git a/core/src/main/java/org/apache/calcite/sql2rel/StandardConvertletTable.java b/core/src/main/java/org/apache/calcite/sql2rel/StandardConvertletTable.java index fa98c332c945..bfee3b056a82 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/StandardConvertletTable.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/StandardConvertletTable.java @@ -86,7 +86,6 @@ import com.google.common.collect.ImmutableList; -import org.checkerframework.checker.initialization.qual.UnknownInitialization; import org.jspecify.annotations.Nullable; import java.math.BigDecimal; @@ -740,7 +739,6 @@ public RexNode convertJdbc( } protected RexNode convertCast( - @UnknownInitialization StandardConvertletTable this, SqlRexContext cx, final SqlCall call) { RelDataTypeFactory typeFactory = cx.getTypeFactory(); @@ -875,7 +873,6 @@ protected RexNode convertFloorCeil(SqlRexContext cx, SqlCall call) { } protected RexNode convertCharset( - @UnknownInitialization StandardConvertletTable this, SqlRexContext cx, SqlCall call) { final RexBuilder rexBuilder = cx.getRexBuilder(); final SqlParserPos pos = call.getParserPosition(); @@ -912,7 +909,6 @@ protected RexNode convertCharset( } protected RexNode translateCharset( - @UnknownInitialization StandardConvertletTable this, SqlRexContext cx, SqlCall call) { final SqlParserPos pos = call.getParserPosition(); final SqlNode expr = call.operand(0); @@ -968,7 +964,6 @@ private static RexNode divide(SqlParserPos pos, RexBuilder rexBuilder, RexNode r } public RexNode convertDatetimeMinus( - @UnknownInitialization StandardConvertletTable this, SqlRexContext cx, SqlDatetimeSubtractionOperator op, SqlCall call) { @@ -1137,7 +1132,6 @@ private static RexNode makeConstructorCall( } private RexNode convertItem( - @UnknownInitialization StandardConvertletTable this, SqlRexContext cx, SqlCall call) { final RexBuilder rexBuilder = cx.getRexBuilder(); @@ -1173,7 +1167,6 @@ private RexNode convertItem( } private RexNode convertColon( - @UnknownInitialization StandardConvertletTable this, SqlRexContext cx, SqlCall call) { final RexBuilder rexBuilder = cx.getRexBuilder(); @@ -1199,7 +1192,6 @@ private RexNode convertColon( * @return Rex call */ public RexNode convertCall( - @UnknownInitialization StandardConvertletTable this, SqlRexContext cx, SqlCall call) { final SqlOperator op = call.getOperator(); @@ -1381,7 +1373,6 @@ private static List convertOperands(SqlRexContext cx, } private RexNode convertPlus( - @UnknownInitialization StandardConvertletTable this, SqlRexContext cx, SqlCall call) { final RexNode rex = convertCall(cx, call); switch (rex.getType().getSqlTypeName()) { @@ -1422,7 +1413,6 @@ private RexNode convertPlus( } private RexNode convertIsDistinctFrom( - @UnknownInitialization StandardConvertletTable this, SqlRexContext cx, SqlCall call, boolean neg) { @@ -1640,7 +1630,6 @@ private static Pair convertOverlapsOperand(SqlRexContext cx, @Deprecated // to be removed before 2.0 public RexNode castToValidatedType( - @UnknownInitialization StandardConvertletTable this, SqlRexContext cx, SqlCall call, RexNode value) { diff --git a/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java b/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java index 3df3e9671004..41559298fcdd 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/TopDownGeneralDecorrelator.java @@ -137,7 +137,7 @@ public class TopDownGeneralDecorrelator implements ReflectiveVisitor { // invokes using reflection a method named unnestInternal based on the // runtime type of the argument. - @SuppressWarnings("method.invocation.invalid") + @SuppressWarnings("NullAway") private final ReflectUtil.MethodDispatcher dispatcher = ReflectUtil.createMethodDispatcher( RelNode.class, getVisitor(), "unnestInternal", RelNode.class, boolean.class); @@ -153,7 +153,7 @@ public class TopDownGeneralDecorrelator implements ReflectiveVisitor { * expressions * @param parentMapRelToUnnestedQuery a map from RelNode to its UnnestedQuery */ - @SuppressWarnings("initialization.fields.uninitialized") + @SuppressWarnings("NullAway") private TopDownGeneralDecorrelator( RelBuilder builder, boolean hasParent, diff --git a/core/src/main/java/org/apache/calcite/util/BitSets.java b/core/src/main/java/org/apache/calcite/util/BitSets.java index d4b941eebb3e..31cf8884fd78 100644 --- a/core/src/main/java/org/apache/calcite/util/BitSets.java +++ b/core/src/main/java/org/apache/calcite/util/BitSets.java @@ -327,7 +327,7 @@ private static class Closure { private final SortedMap equivalence; private final NavigableMap closure = new TreeMap<>(); - @SuppressWarnings({"JdkObsolete", "method.invocation.invalid"}) + @SuppressWarnings({"JdkObsolete", "NullAway"}) Closure(SortedMap equivalence) { this.equivalence = equivalence; final ImmutableIntList keys = diff --git a/core/src/main/java/org/apache/calcite/util/BlackholeMap.java b/core/src/main/java/org/apache/calcite/util/BlackholeMap.java index 430278bf9461..517a9180aa0a 100644 --- a/core/src/main/java/org/apache/calcite/util/BlackholeMap.java +++ b/core/src/main/java/org/apache/calcite/util/BlackholeMap.java @@ -94,12 +94,12 @@ public static Set of() { private BlackholeMap() {} - @SuppressWarnings("contracts.postcondition.not.satisfied") + @SuppressWarnings("NullAway") @Override public @Nullable V put(K key, V value) { return null; } - @SuppressWarnings("override.return.invalid") + @SuppressWarnings("NullAway") @Override public Set> entrySet() { return BHSet.of(); } diff --git a/core/src/main/java/org/apache/calcite/util/ChunkList.java b/core/src/main/java/org/apache/calcite/util/ChunkList.java index 46aa606e326b..148eae85f575 100644 --- a/core/src/main/java/org/apache/calcite/util/ChunkList.java +++ b/core/src/main/java/org/apache/calcite/util/ChunkList.java @@ -63,7 +63,7 @@ public ChunkList() { * Creates a ChunkList whose contents are a given Collection. */ public ChunkList(Collection collection) { - @SuppressWarnings({"method.invocation.invalid", "unused"}) + @SuppressWarnings({"NullAway", "unused"}) boolean ignore = addAll(collection); } diff --git a/core/src/main/java/org/apache/calcite/util/CompositeMap.java b/core/src/main/java/org/apache/calcite/util/CompositeMap.java index aca080f57267..2a2f8252298f 100644 --- a/core/src/main/java/org/apache/calcite/util/CompositeMap.java +++ b/core/src/main/java/org/apache/calcite/util/CompositeMap.java @@ -19,7 +19,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import org.checkerframework.checker.nullness.qual.KeyFor; import org.jspecify.annotations.Nullable; import java.util.Collection; @@ -71,7 +70,7 @@ private static ImmutableList list(E e, E[] es) { return true; } - @SuppressWarnings("contracts.conditional.postcondition.not.satisfied") + @SuppressWarnings("NullAway") @Override public boolean containsKey(@Nullable Object key) { for (Map map : maps) { if (map.containsKey(key)) { @@ -120,8 +119,8 @@ private static ImmutableList list(E e, E[] es) { throw new UnsupportedOperationException(); } - @SuppressWarnings("return.type.incompatible") - @Override public Set<@KeyFor("this") K> keySet() { + @SuppressWarnings("NullAway") + @Override public Set keySet() { final Set keys = new LinkedHashSet<>(); for (Map map : maps) { keys.addAll(map.keySet()); @@ -146,8 +145,8 @@ private Map combinedMap() { return combinedMap().values(); } - @SuppressWarnings("return.type.incompatible") - @Override public Set> entrySet() { + @SuppressWarnings("NullAway") + @Override public Set> entrySet() { return combinedMap().entrySet(); } } diff --git a/core/src/main/java/org/apache/calcite/util/DateString.java b/core/src/main/java/org/apache/calcite/util/DateString.java index 41c6672301be..1c178b7f47be 100644 --- a/core/src/main/java/org/apache/calcite/util/DateString.java +++ b/core/src/main/java/org/apache/calcite/util/DateString.java @@ -48,7 +48,7 @@ private DateString(String v, @SuppressWarnings("unused") boolean ignore) { } /** Creates a DateString. */ - @SuppressWarnings("method.invocation.invalid") + @SuppressWarnings("NullAway") public DateString(String v) { this(v, false); checkArgument(PATTERN.matcher(v).matches(), diff --git a/core/src/main/java/org/apache/calcite/util/Filterator.java b/core/src/main/java/org/apache/calcite/util/Filterator.java index adda2f08ae36..6aacbe69915c 100644 --- a/core/src/main/java/org/apache/calcite/util/Filterator.java +++ b/core/src/main/java/org/apache/calcite/util/Filterator.java @@ -34,7 +34,7 @@ * * @param Element type */ -public class Filterator implements Iterator { +public class Filterator implements Iterator { //~ Instance fields -------------------------------------------------------- final Class includeFilter; diff --git a/core/src/main/java/org/apache/calcite/util/ImmutableBitSet.java b/core/src/main/java/org/apache/calcite/util/ImmutableBitSet.java index 521e60c5b7a2..c2574a354d25 100644 --- a/core/src/main/java/org/apache/calcite/util/ImmutableBitSet.java +++ b/core/src/main/java/org/apache/calcite/util/ImmutableBitSet.java @@ -17,6 +17,7 @@ package org.apache.calcite.util; import org.apache.calcite.linq4j.Linq4j; +import org.apache.calcite.linq4j.annotations.RequiresNonNull; import org.apache.calcite.runtime.Utilities; import org.apache.calcite.util.mapping.Mappings; @@ -24,9 +25,6 @@ import com.google.common.collect.ImmutableSortedMap; import com.google.common.collect.Ordering; -import org.checkerframework.checker.initialization.qual.UnderInitialization; -import org.checkerframework.checker.nullness.qual.RequiresNonNull; -import org.checkerframework.dataflow.qual.Pure; import org.jspecify.annotations.Nullable; import java.io.Serializable; @@ -286,7 +284,6 @@ public static ImmutableBitSet range(int toIndex) { /** * Given a bit index, return word index containing it. */ - @Pure private static int wordIndex(int bitIndex) { return bitIndex >> ADDRESS_BITS_PER_WORD; } @@ -1044,7 +1041,6 @@ private static class Closure { @RequiresNonNull("equivalence") private ImmutableBitSet computeClosure( - @UnderInitialization Closure this, int pos) { ImmutableBitSet o = closure.get(pos); if (o != null) { diff --git a/core/src/main/java/org/apache/calcite/util/ImmutableIntList.java b/core/src/main/java/org/apache/calcite/util/ImmutableIntList.java index ba5cddd4a8bd..49427ff66d3b 100644 --- a/core/src/main/java/org/apache/calcite/util/ImmutableIntList.java +++ b/core/src/main/java/org/apache/calcite/util/ImmutableIntList.java @@ -135,7 +135,7 @@ private static ImmutableIntList copyFromCollection( return Arrays.hashCode(ints); } - @SuppressWarnings("contracts.conditional.postcondition.not.satisfied") + @SuppressWarnings("NullAway") @Override public boolean equals(@Nullable Object obj) { return ((this == obj) || (obj instanceof ImmutableIntList)) diff --git a/core/src/main/java/org/apache/calcite/util/ImmutableNullableSet.java b/core/src/main/java/org/apache/calcite/util/ImmutableNullableSet.java index c9a21af250e8..08c02d88876a 100644 --- a/core/src/main/java/org/apache/calcite/util/ImmutableNullableSet.java +++ b/core/src/main/java/org/apache/calcite/util/ImmutableNullableSet.java @@ -151,7 +151,7 @@ private static Set copyOf(E[] elements, boolean needCopy) { objects[i] = NullSentinel.INSTANCE; } } - @SuppressWarnings({"nullness", "NullableProblems"}) + @SuppressWarnings({"NullAway", "NullableProblems"}) @NonNull Object[] nonNullObjects = objects; return new ImmutableNullableSet(ImmutableSet.copyOf(nonNullObjects)); } diff --git a/core/src/main/java/org/apache/calcite/util/NameSet.java b/core/src/main/java/org/apache/calcite/util/NameSet.java index b024b2b5f0cd..3e73367217b0 100644 --- a/core/src/main/java/org/apache/calcite/util/NameSet.java +++ b/core/src/main/java/org/apache/calcite/util/NameSet.java @@ -70,11 +70,7 @@ public void add(String name) { * name. If case-sensitive, that iterable will have 0 or 1 elements; if * case-insensitive, it may have 0 or more. */ public Collection range(String name, boolean caseSensitive) { - // This produces checkerframework false-positive - // type of expression: Set<@KeyFor("this.names.range(name, caseSensitive)") String> - // method return type: Collection - //noinspection RedundantCast - return (Collection) names.range(name, caseSensitive).keySet(); + return names.range(name, caseSensitive).keySet(); } /** Returns whether this set contains the given name, with a given diff --git a/core/src/main/java/org/apache/calcite/util/NlsString.java b/core/src/main/java/org/apache/calcite/util/NlsString.java index 9dc578389bfc..42a39a0e0742 100644 --- a/core/src/main/java/org/apache/calcite/util/NlsString.java +++ b/core/src/main/java/org/apache/calcite/util/NlsString.java @@ -29,7 +29,6 @@ import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; -import org.checkerframework.dataflow.qual.Pure; import org.jspecify.annotations.Nullable; import java.nio.ByteBuffer; @@ -216,17 +215,14 @@ private NlsString(@Nullable String stringValue, @Nullable ByteString bytesValue, Comparator.nullsFirst(Comparator.naturalOrder())); } - @Pure public @Nullable String getCharsetName() { return charsetName; } - @Pure public @Nullable Charset getCharset() { return charset; } - @Pure public @Nullable SqlCollation getCollation() { return collation; } @@ -396,7 +392,6 @@ public NlsString copy(String value) { } /** Returns the value as a {@link ByteString}. */ - @Pure public @Nullable ByteString getValueBytes() { return bytesValue; } diff --git a/core/src/main/java/org/apache/calcite/util/NumberUtil.java b/core/src/main/java/org/apache/calcite/util/NumberUtil.java index fb9bd1c8d833..5885bfd5c24e 100644 --- a/core/src/main/java/org/apache/calcite/util/NumberUtil.java +++ b/core/src/main/java/org/apache/calcite/util/NumberUtil.java @@ -16,7 +16,6 @@ */ package org.apache.calcite.util; -import org.checkerframework.checker.nullness.qual.PolyNull; import org.jspecify.annotations.Nullable; import java.math.BigDecimal; @@ -93,7 +92,7 @@ public static BigInteger getMinUnscaled(int precision) { /** Sets the scale of a BigDecimal {@code bd} if it is not null; * always returns {@code bd}. */ - public static @PolyNull BigDecimal rescaleBigDecimal(@PolyNull BigDecimal bd, + public static @Nullable BigDecimal rescaleBigDecimal(@Nullable BigDecimal bd, int scale) { if (bd != null) { bd = bd.setScale(scale, RoundingMode.HALF_UP); @@ -108,7 +107,7 @@ public static BigDecimal toBigDecimal(Number number, int scale) { /** Converts a number to a BigDecimal with the same value; * returns null if and only if the number is null. */ - public static @PolyNull BigDecimal toBigDecimal(@PolyNull Number number) { + public static @Nullable BigDecimal toBigDecimal(@Nullable Number number) { if (number == null) { return castNonNull(null); } @@ -146,7 +145,7 @@ public static long round(double d) { } /** Returns the sum of two numbers, or null if either is null. */ - public static @PolyNull Double add(@PolyNull Double a, @PolyNull Double b) { + public static @Nullable Double add(@Nullable Double a, @Nullable Double b) { if (a == null || b == null) { return null; } @@ -156,7 +155,7 @@ public static long round(double d) { /** Returns the difference of two numbers, * or null if either is null. */ - public static @PolyNull Double subtract(@PolyNull Double a, @PolyNull Double b) { + public static @Nullable Double subtract(@Nullable Double a, @Nullable Double b) { if (a == null || b == null) { return castNonNull(null); } @@ -176,7 +175,7 @@ public static long round(double d) { /** Returns the product of two numbers, * or null if either is null. */ - public static @PolyNull Double multiply(@PolyNull Double a, @PolyNull Double b) { + public static @Nullable Double multiply(@Nullable Double a, @Nullable Double b) { if (a == null || b == null) { return castNonNull(null); } @@ -188,7 +187,7 @@ public static long round(double d) { * returns the lesser of two numbers, * ignoring numbers that are null, * or null if both are null. */ - public static @PolyNull Double min(@PolyNull Double a, @PolyNull Double b) { + public static @Nullable Double min(@Nullable Double a, @Nullable Double b) { if (a == null) { return b; } else if (b == null) { @@ -201,7 +200,7 @@ public static long round(double d) { /** Like {@link Math#max} but null safe; * returns the greater of two numbers, * or null if either is null. */ - public static @PolyNull Double max(@PolyNull Double a, @PolyNull Double b) { + public static @Nullable Double max(@Nullable Double a, @Nullable Double b) { if (a == null || b == null) { return castNonNull(null); } diff --git a/core/src/main/java/org/apache/calcite/util/Pair.java b/core/src/main/java/org/apache/calcite/util/Pair.java index 88c27ff21808..c640c52d5e0c 100644 --- a/core/src/main/java/org/apache/calcite/util/Pair.java +++ b/core/src/main/java/org/apache/calcite/util/Pair.java @@ -43,7 +43,7 @@ * @param Left-hand type * @param Right-hand type */ -@SuppressWarnings("type.argument.type.incompatible") +@SuppressWarnings("NullAway") public class Pair implements Comparable>, Map.Entry, Serializable { @@ -82,7 +82,8 @@ public Pair(T1 left, T2 right) { * @param right right value * @return A Pair */ - public static Pair of(T1 left, T2 right) { + public static Pair of( + T1 left, T2 right) { return new Pair<>(left, right); } diff --git a/core/src/main/java/org/apache/calcite/util/PartiallyOrderedSet.java b/core/src/main/java/org/apache/calcite/util/PartiallyOrderedSet.java index bf7caab180a9..aa09108f068b 100644 --- a/core/src/main/java/org/apache/calcite/util/PartiallyOrderedSet.java +++ b/core/src/main/java/org/apache/calcite/util/PartiallyOrderedSet.java @@ -134,7 +134,7 @@ public PartiallyOrderedSet(Ordering ordering, * @param ordering Ordering relation * @param collection Initial contents of partially-ordered set */ - @SuppressWarnings("method.invocation.invalid") + @SuppressWarnings("NullAway") public PartiallyOrderedSet(Ordering ordering, Collection collection) { this(ordering, new HashMap<>(collection.size() * 3 / 2), null, null); addAll(collection); diff --git a/core/src/main/java/org/apache/calcite/util/Permutation.java b/core/src/main/java/org/apache/calcite/util/Permutation.java index a592b318d6c9..f93c97b273ec 100644 --- a/core/src/main/java/org/apache/calcite/util/Permutation.java +++ b/core/src/main/java/org/apache/calcite/util/Permutation.java @@ -16,13 +16,12 @@ */ package org.apache.calcite.util; +import org.apache.calcite.linq4j.annotations.RequiresNonNull; import org.apache.calcite.util.mapping.IntPair; import org.apache.calcite.util.mapping.Mapping; import org.apache.calcite.util.mapping.MappingType; import org.apache.calcite.util.mapping.Mappings; -import org.checkerframework.checker.initialization.qual.UnknownInitialization; -import org.checkerframework.checker.nullness.qual.RequiresNonNull; import org.jspecify.annotations.Nullable; import java.util.Arrays; @@ -46,7 +45,7 @@ public class Permutation implements Mapping, Mappings.TargetMapping { * * @param size Number of elements in the permutation */ - @SuppressWarnings("method.invocation.invalid") + @SuppressWarnings("NullAway") public Permutation(int size) { targets = new int[size]; sources = new int[size]; @@ -437,7 +436,7 @@ private void setInternal(int source, int target) { * @return Whether valid */ @RequiresNonNull({"sources", "targets"}) - private boolean isValid(@UnknownInitialization Permutation this, boolean fail) { + private boolean isValid(boolean fail) { final int size = targets.length; if (sources.length != size) { assert !fail : "different lengths"; diff --git a/core/src/main/java/org/apache/calcite/util/RangeSets.java b/core/src/main/java/org/apache/calcite/util/RangeSets.java index 54d0924ab4dd..0395fe49681c 100644 --- a/core/src/main/java/org/apache/calcite/util/RangeSets.java +++ b/core/src/main/java/org/apache/calcite/util/RangeSets.java @@ -22,8 +22,6 @@ import com.google.common.collect.RangeSet; import com.google.common.collect.TreeRangeSet; -import org.jspecify.annotations.NonNull; - import java.util.Iterator; import java.util.Set; import java.util.function.BiConsumer; @@ -374,7 +372,7 @@ private static class SinkConsumer implements Consumer { * @param Value type * * @see Handler */ - public interface Consumer<@NonNull V> { + public interface Consumer { void all(); void atLeast(V lower); void atMost(V upper); diff --git a/core/src/main/java/org/apache/calcite/util/ReflectUtil.java b/core/src/main/java/org/apache/calcite/util/ReflectUtil.java index 1ad3789098f6..7bb3476137dc 100644 --- a/core/src/main/java/org/apache/calcite/util/ReflectUtil.java +++ b/core/src/main/java/org/apache/calcite/util/ReflectUtil.java @@ -417,7 +417,7 @@ private static boolean invokeVisitorInternal( * @return cache of methods */ public static ReflectiveVisitDispatcher createDispatcher( + E> ReflectiveVisitDispatcher createDispatcher( final Class visitorBaseClazz, final Class visiteeBaseClazz) { assert ReflectiveVisitor.class.isAssignableFrom(visitorBaseClazz); @@ -511,7 +511,7 @@ E extends Object> ReflectiveVisitDispatcher createDispatcher( * @param arg0Clazz Base type of argument zero * @param otherArgClasses Types of remaining arguments */ - public static MethodDispatcher createMethodDispatcher( + public static MethodDispatcher createMethodDispatcher( final Class returnClazz, final ReflectiveVisitor visitor, final String methodName, diff --git a/core/src/main/java/org/apache/calcite/util/ReflectiveVisitDispatcher.java b/core/src/main/java/org/apache/calcite/util/ReflectiveVisitDispatcher.java index 99ed1bae116f..defc9b8348b9 100644 --- a/core/src/main/java/org/apache/calcite/util/ReflectiveVisitDispatcher.java +++ b/core/src/main/java/org/apache/calcite/util/ReflectiveVisitDispatcher.java @@ -35,7 +35,7 @@ * @param Return type */ public interface ReflectiveVisitDispatcher { + E> { //~ Methods ---------------------------------------------------------------- /** diff --git a/core/src/main/java/org/apache/calcite/util/Sarg.java b/core/src/main/java/org/apache/calcite/util/Sarg.java index b25762c6de4f..bf05ac17e9ab 100644 --- a/core/src/main/java/org/apache/calcite/util/Sarg.java +++ b/core/src/main/java/org/apache/calcite/util/Sarg.java @@ -65,7 +65,7 @@ * * @see SqlStdOperatorTable#SEARCH */ -@SuppressWarnings("type.argument.type.incompatible") +@SuppressWarnings("NullAway") public class Sarg> implements Comparable> { public final RangeSet rangeSet; public final RexUnknownAs nullAs; diff --git a/core/src/main/java/org/apache/calcite/util/SerializableCharset.java b/core/src/main/java/org/apache/calcite/util/SerializableCharset.java index e55f14875ebc..000c20b8922f 100644 --- a/core/src/main/java/org/apache/calcite/util/SerializableCharset.java +++ b/core/src/main/java/org/apache/calcite/util/SerializableCharset.java @@ -16,7 +16,7 @@ */ package org.apache.calcite.util; -import org.checkerframework.checker.nullness.qual.PolyNull; +import org.jspecify.annotations.Nullable; import java.io.IOException; import java.io.ObjectInputStream; @@ -91,7 +91,7 @@ public Charset getCharset() { * @param charset Character set to wrap, or null * @return Wrapped charset */ - public static @PolyNull SerializableCharset forCharset(@PolyNull Charset charset) { + public static @Nullable SerializableCharset forCharset(@Nullable Charset charset) { if (charset == null) { return null; } diff --git a/core/src/main/java/org/apache/calcite/util/SimpleNamespaceContext.java b/core/src/main/java/org/apache/calcite/util/SimpleNamespaceContext.java index 53446f1f71a0..3a2a4775c227 100644 --- a/core/src/main/java/org/apache/calcite/util/SimpleNamespaceContext.java +++ b/core/src/main/java/org/apache/calcite/util/SimpleNamespaceContext.java @@ -36,7 +36,7 @@ public class SimpleNamespaceContext implements NamespaceContext { private final Map prefixToNamespaceUri = new HashMap<>(); private final Map> namespaceUriToPrefixes = new HashMap<>(); - @SuppressWarnings({"method.invocation.invalid", "methodref.receiver.bound.invalid"}) + @SuppressWarnings("NullAway") public SimpleNamespaceContext(Map bindings) { bindNamespaceUri(XMLConstants.XML_NS_PREFIX, XMLConstants.XML_NS_URI); bindNamespaceUri(XMLConstants.XMLNS_ATTRIBUTE, XMLConstants.XMLNS_ATTRIBUTE_NS_URI); diff --git a/core/src/main/java/org/apache/calcite/util/TimeString.java b/core/src/main/java/org/apache/calcite/util/TimeString.java index 6f16e93854e9..172d90ab427c 100644 --- a/core/src/main/java/org/apache/calcite/util/TimeString.java +++ b/core/src/main/java/org/apache/calcite/util/TimeString.java @@ -53,7 +53,7 @@ private TimeString(String v, @SuppressWarnings("unused") boolean ignore) { } /** Creates a TimeString. */ - @SuppressWarnings("method.invocation.invalid") + @SuppressWarnings("NullAway") public TimeString(String v) { this(v, false); checkArgument(INPUT_PATTERN.matcher(v).matches(), diff --git a/core/src/main/java/org/apache/calcite/util/Util.java b/core/src/main/java/org/apache/calcite/util/Util.java index 63a1bd48b576..9446c8d2fd25 100644 --- a/core/src/main/java/org/apache/calcite/util/Util.java +++ b/core/src/main/java/org/apache/calcite/util/Util.java @@ -43,8 +43,6 @@ import com.google.common.collect.Sets; import org.apiguardian.api.API; -import org.checkerframework.checker.nullness.qual.PolyNull; -import org.checkerframework.dataflow.qual.Pure; import org.jspecify.annotations.Nullable; import org.slf4j.Logger; @@ -684,7 +682,7 @@ public static boolean isWindows() { * characters found in {@code search} are replaced by the character in the same position in * {@code replacement}; if {@code replacement} is shorter, remaining matches are removed. */ - public static @PolyNull String replaceChars(@PolyNull String s, @Nullable String search, + public static @Nullable String replaceChars(@Nullable String s, @Nullable String search, @Nullable String replacement) { if (s == null || s.isEmpty() || search == null || search.isEmpty()) { return s; @@ -1865,8 +1863,8 @@ public static List cast(List list, Class clazz) { * @param clazz Class to cast to * @return An iterator whose members are of the desired type. */ - public static Iterator cast( - final Iterator iter, + public static Iterator cast( + final Iterator iter, final Class clazz) { return transform(iter, x -> clazz.cast(castNonNull(x))); } @@ -2127,7 +2125,7 @@ public static List> pairs(final List list) { * *

      Equivalent to the Elvis operator ({@code ?:}) of languages such as * Groovy or PHP. */ - public static @PolyNull T first(@Nullable T v0, @PolyNull T v1) { + public static @Nullable T first(@Nullable T v0, @Nullable T v1) { return v0 != null ? v0 : v1; } @@ -2576,7 +2574,7 @@ public static Map asIndexMapJ( } }; return new AbstractMap() { - @SuppressWarnings("override.return.invalid") + @SuppressWarnings("NullAway") @Override public Set> entrySet() { return entrySet; } @@ -2746,7 +2744,8 @@ public static UnaryOperator andThen(UnaryOperator op1, } /** Transforms a list, applying a function to each element. */ - public static List transform(List list, + public static + List transform(List list, java.util.function.Function function) { if (list.isEmpty() && list instanceof ImmutableList) { return ImmutableList.of(); // save ourselves some effort @@ -2759,7 +2758,8 @@ public static List transform(List list, /** Transforms a list, applying a function to each element, also passing in * the element's index in the list. */ - public static List transformIndexed(List list, + public static + List transformIndexed(List list, BiFunction function) { if (list.isEmpty() && list instanceof ImmutableList) { return ImmutableList.of(); // save ourselves some effort @@ -2772,7 +2772,8 @@ public static List transformIndexed(List list, /** Transforms an iterable, applying a function to each element. */ @API(since = "1.27", status = API.Status.EXPERIMENTAL) - public static Iterable transform(Iterable iterable, + public static + Iterable transform(Iterable iterable, java.util.function.Function function) { // FluentIterable provides toString return new FluentIterable() { @@ -2784,7 +2785,8 @@ public static Iterable transform(Iterable iterable, /** Transforms an iterator. */ @API(since = "1.27", status = API.Status.EXPERIMENTAL) - public static Iterator transform(Iterator iterator, + public static + Iterator transform(Iterator iterator, java.util.function.Function function) { return new TransformingIterator<>(iterator, function); } @@ -2883,7 +2885,6 @@ public FoundOne(@Nullable Object node) { this.node = node; } - @Pure public @Nullable Object getNode() { return node; } @@ -3003,7 +3004,7 @@ private static class FilteringIterator implements Iterator { Predicate predicate) { this.iterator = iterator; this.predicate = predicate; - @SuppressWarnings("method.invocation.invalid") + @SuppressWarnings("NullAway") T current = moveNext(); this.current = current; } diff --git a/core/src/main/java/org/apache/calcite/util/XmlOutput.java b/core/src/main/java/org/apache/calcite/util/XmlOutput.java index 94b9136a9c92..f7d08ca4f6b8 100644 --- a/core/src/main/java/org/apache/calcite/util/XmlOutput.java +++ b/core/src/main/java/org/apache/calcite/util/XmlOutput.java @@ -581,7 +581,7 @@ public void defineEscape(char from, String to) { * Call this before attempting to escape strings; after this, * defineEscape may not be called again. */ - @SuppressWarnings("assignment.type.incompatible") + @SuppressWarnings("NullAway") public void makeImmutable() { translationTable = requireNonNull(translationVector, "translationVector").toArray(new String[0]); diff --git a/core/src/main/java/org/apache/calcite/util/graph/AttributedDirectedGraph.java b/core/src/main/java/org/apache/calcite/util/graph/AttributedDirectedGraph.java index a8d759e7942e..387853c73cee 100644 --- a/core/src/main/java/org/apache/calcite/util/graph/AttributedDirectedGraph.java +++ b/core/src/main/java/org/apache/calcite/util/graph/AttributedDirectedGraph.java @@ -18,7 +18,6 @@ import org.apache.calcite.util.Util; -import org.checkerframework.checker.initialization.qual.UnknownInitialization; import org.jspecify.annotations.Nullable; import java.util.List; @@ -33,7 +32,7 @@ public class AttributedDirectedGraph extends DefaultDirectedGraph { /** Creates an attributed graph. */ - public AttributedDirectedGraph(@UnknownInitialization AttributedEdgeFactory edgeFactory) { + public AttributedDirectedGraph(AttributedEdgeFactory edgeFactory) { super(edgeFactory); } diff --git a/core/src/main/java/org/apache/calcite/util/graph/DefaultDirectedGraph.java b/core/src/main/java/org/apache/calcite/util/graph/DefaultDirectedGraph.java index bc1aa18d95d8..d55298a8fc2c 100644 --- a/core/src/main/java/org/apache/calcite/util/graph/DefaultDirectedGraph.java +++ b/core/src/main/java/org/apache/calcite/util/graph/DefaultDirectedGraph.java @@ -19,8 +19,6 @@ import com.google.common.collect.Ordering; import org.apiguardian.api.API; -import org.checkerframework.checker.initialization.qual.NotOnlyInitialized; -import org.checkerframework.checker.initialization.qual.UnknownInitialization; import org.jspecify.annotations.Nullable; import java.util.ArrayList; @@ -45,10 +43,10 @@ public class DefaultDirectedGraph implements DirectedGraph { final Set edges = new LinkedHashSet<>(); final Map> vertexMap = new LinkedHashMap<>(); - final @NotOnlyInitialized EdgeFactory edgeFactory; + final EdgeFactory edgeFactory; /** Creates a graph. */ - public DefaultDirectedGraph(@UnknownInitialization EdgeFactory edgeFactory) { + public DefaultDirectedGraph(EdgeFactory edgeFactory) { this.edgeFactory = edgeFactory; } @@ -97,7 +95,7 @@ private String toString(Ordering vertexOrdering, @API(since = "1.26", status = API.Status.EXPERIMENTAL) protected final VertexInfo getVertex(V vertex) { - @SuppressWarnings("argument.type.incompatible") + @SuppressWarnings("NullAway") final VertexInfo info = vertexMap.get(vertex); if (info == null) { throw new IllegalArgumentException("no vertex " + vertex); @@ -162,9 +160,7 @@ protected final VertexInfo getVertex(V vertex) { return outRemoved; } - @SuppressWarnings("return.type.incompatible") @Override public Set vertexSet() { - // Set -> Set return vertexMap.keySet(); } @@ -194,7 +190,7 @@ protected final VertexInfo getVertex(V vertex) { * if {@code collection} is a small fraction of the set of vertices. */ private void removeMinorityVertices(Collection collection) { for (V v : collection) { - @SuppressWarnings("argument.type.incompatible") // nullable keys are supported by .get + @SuppressWarnings("NullAway") // nullable keys are supported by .get final VertexInfo info = vertexMap.get(v); if (info == null) { continue; diff --git a/core/src/main/java/org/apache/calcite/util/graph/DefaultEdge.java b/core/src/main/java/org/apache/calcite/util/graph/DefaultEdge.java index bc007b1a2bc0..260fd9b830ba 100644 --- a/core/src/main/java/org/apache/calcite/util/graph/DefaultEdge.java +++ b/core/src/main/java/org/apache/calcite/util/graph/DefaultEdge.java @@ -47,9 +47,7 @@ public DefaultEdge(Object source, Object target) { return source + " -> " + target; } - public static DirectedGraph.EdgeFactory factory() { - // see https://github.com/typetools/checker-framework/issues/3637 - //noinspection Convert2MethodRef - return (source1, target1) -> new DefaultEdge(source1, target1); + public static DirectedGraph.EdgeFactory factory() { + return DefaultEdge::new; } } diff --git a/core/src/main/java/org/apache/calcite/util/graph/Graphs.java b/core/src/main/java/org/apache/calcite/util/graph/Graphs.java index de230db779fd..e7b84e0b83a8 100644 --- a/core/src/main/java/org/apache/calcite/util/graph/Graphs.java +++ b/core/src/main/java/org/apache/calcite/util/graph/Graphs.java @@ -101,7 +101,7 @@ public static FrozenGraph makeImmutable( * @param Vertex type * @param Edge type */ - public static class FrozenGraph { + public static class FrozenGraph { private final DefaultDirectedGraph graph; private final Map, int[]> shortestDistances; diff --git a/core/src/main/java/org/apache/calcite/util/graph/TopologicalOrderIterator.java b/core/src/main/java/org/apache/calcite/util/graph/TopologicalOrderIterator.java index e880d72e4f38..789db7154850 100644 --- a/core/src/main/java/org/apache/calcite/util/graph/TopologicalOrderIterator.java +++ b/core/src/main/java/org/apache/calcite/util/graph/TopologicalOrderIterator.java @@ -16,11 +16,9 @@ */ package org.apache.calcite.util.graph; +import org.apache.calcite.linq4j.annotations.RequiresNonNull; import org.apache.calcite.plan.hep.HepMatchOrder; -import org.checkerframework.checker.initialization.qual.UnderInitialization; -import org.checkerframework.checker.nullness.qual.RequiresNonNull; - import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; @@ -66,7 +64,6 @@ public static Iterable of( @RequiresNonNull("graph") private void populate( - @UnderInitialization TopologicalOrderIterator this, Map countMap, List empties) { for (V v : graph.vertexMap.keySet()) { countMap.put(v, new int[] {0}); diff --git a/core/src/main/java/org/apache/calcite/util/mapping/AbstractSourceMapping.java b/core/src/main/java/org/apache/calcite/util/mapping/AbstractSourceMapping.java index 805bda3c725a..caf22543736f 100644 --- a/core/src/main/java/org/apache/calcite/util/mapping/AbstractSourceMapping.java +++ b/core/src/main/java/org/apache/calcite/util/mapping/AbstractSourceMapping.java @@ -59,7 +59,7 @@ protected AbstractSourceMapping(int sourceCount, int targetCount) { return MappingType.INVERSE_PARTIAL_FUNCTION; } - @SuppressWarnings("method.invocation.invalid") + @SuppressWarnings("NullAway") @Override public Iterator iterator() { return new Iterator() { int source; diff --git a/core/src/main/java/org/apache/calcite/util/mapping/AbstractTargetMapping.java b/core/src/main/java/org/apache/calcite/util/mapping/AbstractTargetMapping.java index c9b7b9103c5c..375bf4081046 100644 --- a/core/src/main/java/org/apache/calcite/util/mapping/AbstractTargetMapping.java +++ b/core/src/main/java/org/apache/calcite/util/mapping/AbstractTargetMapping.java @@ -59,7 +59,7 @@ protected AbstractTargetMapping(int sourceCount, int targetCount) { return MappingType.PARTIAL_FUNCTION; } - @SuppressWarnings("method.invocation.invalid") + @SuppressWarnings("NullAway") @Override public Iterator iterator() { return new Iterator() { int source = -1; diff --git a/core/src/main/java/org/apache/calcite/util/mapping/Mappings.java b/core/src/main/java/org/apache/calcite/util/mapping/Mappings.java index b24d766d88f8..9592ee095290 100644 --- a/core/src/main/java/org/apache/calcite/util/mapping/Mappings.java +++ b/core/src/main/java/org/apache/calcite/util/mapping/Mappings.java @@ -25,7 +25,6 @@ import com.google.common.primitives.Ints; import com.google.errorprone.annotations.CheckReturnValue; -import org.checkerframework.checker.initialization.qual.UnknownInitialization; import org.jspecify.annotations.Nullable; import java.util.AbstractList; @@ -1358,8 +1357,7 @@ private class MappingItr implements Iterator { return i < targets.length; } - private void advance( - @UnknownInitialization MappingItr this) { + private void advance() { do { ++i; } while (i < targets.length && targets[i] == -1); @@ -1718,7 +1716,7 @@ private static class PartialFunctionImpl extends AbstractMapping return size; } - @SuppressWarnings("method.invocation.invalid") + @SuppressWarnings("NullAway") @Override public Iterator iterator() { return new Iterator() { int i = -1; diff --git a/core/src/test/java/org/apache/calcite/test/JdbcTest.java b/core/src/test/java/org/apache/calcite/test/JdbcTest.java index 7d5154a90a8b..3e1d316f3450 100644 --- a/core/src/test/java/org/apache/calcite/test/JdbcTest.java +++ b/core/src/test/java/org/apache/calcite/test/JdbcTest.java @@ -281,7 +281,7 @@ private static DataSource getDataSource() { return this; } - @Override public @Nullable T unwrap(Class clazz) { + @Override public @Nullable T unwrap(Class clazz) { if (schema instanceof Wrapper) { return ((Wrapper) schema).unwrap(clazz); } diff --git a/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvProjectTableScanRule.java b/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvProjectTableScanRule.java index 2649d1f5473a..a3fa3932966a 100644 --- a/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvProjectTableScanRule.java +++ b/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvProjectTableScanRule.java @@ -46,7 +46,7 @@ protected CsvProjectTableScanRule(Config config) { @Override public void onMatch(RelOptRuleCall call) { final LogicalProject project = call.rel(0); final CsvTableScan scan = call.rel(1); - @Nullable int[] fields = getProjectFields(project.getProjects()); + int @Nullable [] fields = getProjectFields(project.getProjects()); if (fields == null) { // Project contains expressions more complex than just field references. return; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/DefaultEnumerable.java b/linq4j/src/main/java/org/apache/calcite/linq4j/DefaultEnumerable.java index 23ce85ef3b0d..eccd6ab38e96 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/DefaultEnumerable.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/DefaultEnumerable.java @@ -34,7 +34,6 @@ import org.apache.calcite.linq4j.function.Predicate1; import org.apache.calcite.linq4j.function.Predicate2; -import org.checkerframework.checker.nullness.qual.PolyNull; import org.jspecify.annotations.Nullable; import java.math.BigDecimal; @@ -97,8 +96,8 @@ protected OrderedQueryable asOrderedQueryable() { return EnumerableDefaults.aggregate(getThis(), func); } - @Override public @PolyNull TAccumulate aggregate(@PolyNull TAccumulate seed, - Function2<@PolyNull TAccumulate, T, @PolyNull TAccumulate> func) { + @Override public @Nullable TAccumulate aggregate(@Nullable TAccumulate seed, + Function2<@Nullable TAccumulate, T, @Nullable TAccumulate> func) { return EnumerableDefaults.aggregate(getThis(), seed, func); } @@ -199,7 +198,7 @@ protected OrderedQueryable asOrderedQueryable() { return EnumerableDefaults.defaultIfEmpty(getThis()); } - @Override public Enumerable<@PolyNull T> defaultIfEmpty(@PolyNull T value) { + @Override public Enumerable<@Nullable T> defaultIfEmpty(@Nullable T value) { return EnumerableDefaults.defaultIfEmpty(getThis(), value); } diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/Enumerable.java b/linq4j/src/main/java/org/apache/calcite/linq4j/Enumerable.java index 4af96280dd20..b8afb58cb21e 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/Enumerable.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/Enumerable.java @@ -16,8 +16,6 @@ */ package org.apache.calcite.linq4j; -import org.checkerframework.framework.qual.Covariant; - /** * Exposes the enumerator, which supports a simple iteration over a collection. * @@ -28,7 +26,6 @@ * * @param Element type */ -@Covariant(0) public interface Enumerable extends RawEnumerable, Iterable, ExtendedEnumerable { /** diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java b/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java index 326989182111..05c9c40e2601 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java @@ -41,10 +41,6 @@ import com.google.common.collect.Sets; import org.apiguardian.api.API; -import org.checkerframework.checker.nullness.qual.KeyFor; -import org.checkerframework.checker.nullness.qual.PolyNull; -import org.checkerframework.dataflow.qual.Pure; -import org.checkerframework.framework.qual.HasQualifierParameter; import org.jspecify.annotations.Nullable; import java.math.BigDecimal; @@ -417,10 +413,10 @@ public static int count(Enumerable enumerable, * *

      If {@code value} is not null, the result is never null. */ - @SuppressWarnings("return.type.incompatible") - public static Enumerable<@PolyNull TSource> defaultIfEmpty( + @SuppressWarnings("NullAway") + public static Enumerable<@Nullable TSource> defaultIfEmpty( Enumerable enumerable, - @PolyNull TSource value) { + @Nullable TSource value) { try (Enumerator os = enumerable.enumerator()) { if (os.moveNext()) { return Linq4j.asEnumerable(() -> new Iterator() { @@ -569,7 +565,7 @@ public static Enumerable except( try (Enumerator os = source1.enumerator()) { while (os.moveNext()) { TSource o = os.current(); - @SuppressWarnings("argument.type.incompatible") + @SuppressWarnings("NullAway") boolean unused = collection.remove(o); } return Linq4j.asEnumerable(collection); @@ -1168,7 +1164,7 @@ private static Enumerable groupBy while (os.moveNext()) { TSource o = os.current(); TKey key = keySelector.apply(o); - @SuppressWarnings("argument.type.incompatible") + @SuppressWarnings("NullAway") TAccumulate accumulator = map.get(key); if (accumulator == null) { accumulator = accumulatorInitializer.apply(); @@ -1197,7 +1193,7 @@ private static Enumerable groupBy for (Function1 keySelector : keySelectors) { TSource o = os.current(); TKey key = keySelector.apply(o); - @SuppressWarnings("argument.type.incompatible") + @SuppressWarnings("NullAway") TAccumulate accumulator = map.get(key); if (accumulator == null) { accumulator = accumulatorInitializer.apply(); @@ -1251,7 +1247,7 @@ public static Enumerable groupJoin( return new Enumerator() { @Override public TResult current() { final Map.Entry entry = entries.current(); - @SuppressWarnings("argument.type.incompatible") + @SuppressWarnings("NullAway") final Enumerable inners = innerLookup.get(entry.getKey()); return resultSelector.apply(entry.getValue(), inners == null ? Linq4j.emptyEnumerable() : inners); @@ -1293,7 +1289,7 @@ public static Enumerable groupJoin( return new Enumerator() { @Override public TResult current() { final Map.Entry entry = entries.current(); - @SuppressWarnings("argument.type.incompatible") + @SuppressWarnings("NullAway") final Enumerable inners = innerLookup.get(entry.getKey()); return resultSelector.apply(entry.getValue(), inners == null ? Linq4j.emptyEnumerable() : inners); @@ -1338,7 +1334,7 @@ public static Enumerable intersect( try (Enumerator os = source0.enumerator()) { while (os.moveNext()) { TSource o = os.current(); - @SuppressWarnings("argument.type.incompatible") + @SuppressWarnings("NullAway") boolean removed = set1.remove(o); if (removed) { resultCollection.add(o); @@ -1529,7 +1525,7 @@ private static Enumerable hashEquiJoin // not the left. List list = new ArrayList<>(); for (TKey key : unmatchedKeys) { - @SuppressWarnings("argument.type.incompatible") + @SuppressWarnings("NullAway") Enumerable innerValues = requireNonNull(innerLookup.get(key)); for (TInner tInner : innerValues) { list.add(tInner); @@ -2236,7 +2232,7 @@ public static Enumerable correlateBatchJoin( int i = -1; // outer position int j = -1; // inner position - @SuppressWarnings("argument.type.incompatible") + @SuppressWarnings("NullAway") @Override public TResult current() { return resultSelector.apply(outerValue, innerValue); } @@ -2482,7 +2478,7 @@ private static Enumerable semiJoinWithPredicate final Predicate1 predicate = v0 -> { TKey key = outerKeySelector.apply(v0); - @SuppressWarnings("argument.type.incompatible") + @SuppressWarnings("NullAway") Enumerable innersOfKey = key == null ? null : innerLookup.get().get(key); if (innersOfKey == null) { return anti; @@ -3283,12 +3279,12 @@ public static Enumerable orderBy( TKey key = keySelector.apply(o); if (needed.signum() >= 0 && size.compareTo(needed) >= 0) { // the current row will never appear in the output, so just skip it - @KeyFor("map") TKey lastKey = map.lastKey(); + TKey lastKey = map.lastKey(); if (comparator.compare(key, lastKey) >= 0) { continue; } // remove last entry from tree map, so that we keep at most 'needed' rows - @SuppressWarnings("argument.type.incompatible") + @SuppressWarnings("NullAway") List l = map.get(lastKey); if (l.size() == 1) { map.remove(lastKey); @@ -4806,8 +4802,7 @@ static class SkipWhileBigDecimalEnumerator implements Enumerator source element type * @param element type */ - @HasQualifierParameter(Nullable.class) - static class CastingEnumerator + static class CastingEnumerator implements Enumerator { private final Enumerator enumerator; private final Class clazz; @@ -4878,13 +4873,13 @@ protected WrapMap(Function0, V>> mapProvider, EqualityComparer this.comparer = comparer; } - @Override public Set> entrySet() { + @Override public Set> entrySet() { return new WrapMapEntrySet(); } /** EntrySet for {@link WrapMap}. */ - private class WrapMapEntrySet extends AbstractSet> { - @SuppressWarnings("override.return.invalid") + private class WrapMapEntrySet extends AbstractSet> { + @SuppressWarnings("NullAway") @Override public Iterator> iterator() { final Iterator, V>> iterator = map.entrySet().iterator(); @@ -4910,12 +4905,11 @@ private class WrapMapEntrySet extends AbstractSet> { } } - @SuppressWarnings("contracts.conditional.postcondition.not.satisfied") + @SuppressWarnings("NullAway") @Override public boolean containsKey(@Nullable Object key) { return map.containsKey(wrap((K) key)); } - @Pure private Wrapped wrap(K key) { return Wrapped.upAs(comparer, key); } @@ -4924,7 +4918,7 @@ private Wrapped wrap(K key) { return map.get(wrap((K) key)); } - @SuppressWarnings("contracts.postcondition.not.satisfied") + @SuppressWarnings("NullAway") @Override public @Nullable V put(K key, V value) { return map.put(wrap(key), value); } @@ -5010,7 +5004,7 @@ private static class MergeJoinEnumerator leftEnumerable, Enumerable rightEnumerable, Function1 outerKeySelector, diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/Enumerator.java b/linq4j/src/main/java/org/apache/calcite/linq4j/Enumerator.java index e026ed33022d..bd9e28be31fc 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/Enumerator.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/Enumerator.java @@ -16,8 +16,6 @@ */ package org.apache.calcite.linq4j; -import org.checkerframework.framework.qual.Covariant; - /** * Supports a simple iteration over a collection. * @@ -28,7 +26,6 @@ * * @param Element type */ -@Covariant(0) public interface Enumerator extends AutoCloseable { /** * Gets the current element in the collection. diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedEnumerable.java b/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedEnumerable.java index 578d30ef0ffd..c4dde33c2fc0 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedEnumerable.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedEnumerable.java @@ -34,8 +34,6 @@ import org.apache.calcite.linq4j.function.Predicate1; import org.apache.calcite.linq4j.function.Predicate2; -import org.checkerframework.checker.nullness.qual.PolyNull; -import org.checkerframework.framework.qual.Covariant; import org.jspecify.annotations.Nullable; import java.math.BigDecimal; @@ -49,7 +47,6 @@ * * @param Element type */ -@Covariant(0) public interface ExtendedEnumerable { /** @@ -76,8 +73,8 @@ public interface ExtendedEnumerable { * *

      If {@code seed} is not null, the result is never null. */ - @PolyNull TAccumulate aggregate(@PolyNull TAccumulate seed, - Function2<@PolyNull TAccumulate, TSource, @PolyNull TAccumulate> func); + @Nullable TAccumulate aggregate(@Nullable TAccumulate seed, + Function2<@Nullable TAccumulate, TSource, @Nullable TAccumulate> func); /** * Applies an accumulator function over a @@ -280,7 +277,7 @@ TResult aggregate(TAccumulate seed, * *

      If {@code value} is not null, the result is never null. */ - Enumerable<@PolyNull TSource> defaultIfEmpty(@PolyNull TSource value); + Enumerable<@Nullable TSource> defaultIfEmpty(@Nullable TSource value); /** * Returns distinct elements from a sequence by using diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedQueryable.java b/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedQueryable.java index 0e16304eec0c..d9716d8d76ed 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedQueryable.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedQueryable.java @@ -33,7 +33,6 @@ import org.apache.calcite.linq4j.function.Predicate2; import org.apache.calcite.linq4j.tree.FunctionExpression; -import org.checkerframework.framework.qual.Covariant; import org.jspecify.annotations.Nullable; import java.math.BigDecimal; @@ -44,7 +43,6 @@ * * @param Element type */ -@Covariant(0) interface ExtendedQueryable extends ExtendedEnumerable { /** diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/Grouping.java b/linq4j/src/main/java/org/apache/calcite/linq4j/Grouping.java index aede1d0254f9..f3091afc27dc 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/Grouping.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/Grouping.java @@ -16,15 +16,12 @@ */ package org.apache.calcite.linq4j; -import org.checkerframework.framework.qual.Covariant; - /** * Represents a collection of objects that have a common key. * * @param Key type * @param Element type */ -@Covariant(0) public interface Grouping extends Enumerable { /** * Gets the key of this Grouping. diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/GroupingImpl.java b/linq4j/src/main/java/org/apache/calcite/linq4j/GroupingImpl.java index 0e7a23572791..1a869883ba2c 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/GroupingImpl.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/GroupingImpl.java @@ -29,8 +29,8 @@ * @param Key type * @param Value type */ -@SuppressWarnings("type.argument.type.incompatible") -class GroupingImpl extends AbstractEnumerable +@SuppressWarnings("NullAway") +class GroupingImpl extends AbstractEnumerable implements Grouping, Map.Entry> { private final K key; private final List values; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/Linq4j.java b/linq4j/src/main/java/org/apache/calcite/linq4j/Linq4j.java index 7e0f377bc627..b531f73fde04 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/Linq4j.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/Linq4j.java @@ -555,7 +555,7 @@ protected Collection getCollection() { return getCollection().size(); } - @SuppressWarnings("argument.type.incompatible") + @SuppressWarnings("NullAway") @Override public boolean contains(T element) { return getCollection().contains(element); } @@ -664,7 +664,8 @@ private static class SingletonEnumerator implements Enumerator { /** Enumerator that returns one null element. * * @param element type */ - private static class SingletonNullEnumerator<@Nullable E> implements Enumerator { + private static class SingletonNullEnumerator + implements Enumerator { int i = 0; @Override public E current() { diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/LookupImpl.java b/linq4j/src/main/java/org/apache/calcite/linq4j/LookupImpl.java index 9411ecbaa83d..277daa1fbc37 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/LookupImpl.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/LookupImpl.java @@ -18,7 +18,6 @@ import org.apache.calcite.linq4j.function.Function2; -import org.checkerframework.checker.nullness.qual.KeyFor; import org.jspecify.annotations.Nullable; import java.util.AbstractCollection; @@ -84,7 +83,7 @@ class LookupImpl extends AbstractEnumerable> return map.isEmpty(); } - @SuppressWarnings("contracts.conditional.postcondition.not.satisfied") + @SuppressWarnings("NullAway") @Override public boolean containsKey(@Nullable Object key) { return map.containsKey(key); } @@ -100,7 +99,7 @@ class LookupImpl extends AbstractEnumerable> return list == null ? null : Linq4j.asEnumerable(list); } - @SuppressWarnings("contracts.postcondition.not.satisfied") + @SuppressWarnings("NullAway") @Override public @Nullable Enumerable put(K key, Enumerable value) { final List list = map.put(key, value.toList()); return list == null ? null : Linq4j.asEnumerable(list); @@ -121,8 +120,8 @@ class LookupImpl extends AbstractEnumerable> map.clear(); } - @SuppressWarnings("return.type.incompatible") - @Override public Set<@KeyFor("this") K> keySet() { + @SuppressWarnings("NullAway") + @Override public Set keySet() { return map.keySet(); } @@ -153,9 +152,9 @@ class LookupImpl extends AbstractEnumerable> }; } - @SuppressWarnings("return.type.incompatible") - @Override public Set>> entrySet() { - final Set>> entries = map.entrySet(); + @SuppressWarnings("NullAway") + @Override public Set>> entrySet() { + final Set>> entries = map.entrySet(); return new AbstractSet>>() { @Override public Iterator>> iterator() { final Iterator>> iterator = entries.iterator(); diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/MemoryEnumerator.java b/linq4j/src/main/java/org/apache/calcite/linq4j/MemoryEnumerator.java index 41250f5b5413..feaafd5371e4 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/MemoryEnumerator.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/MemoryEnumerator.java @@ -25,7 +25,8 @@ * * @param Row value */ -public class MemoryEnumerator<@Nullable E> implements Enumerator> { +public class MemoryEnumerator + implements Enumerator> { private final Enumerator enumerator; private final MemoryFactory memoryFactory; private final AtomicInteger prevCounter; diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/MergeUnionEnumerator.java b/linq4j/src/main/java/org/apache/calcite/linq4j/MergeUnionEnumerator.java index 86a30ea1bb30..03640662d075 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/MergeUnionEnumerator.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/MergeUnionEnumerator.java @@ -16,11 +16,10 @@ */ package org.apache.calcite.linq4j; +import org.apache.calcite.linq4j.annotations.RequiresNonNull; import org.apache.calcite.linq4j.function.EqualityComparer; import org.apache.calcite.linq4j.function.Function1; -import org.checkerframework.checker.initialization.qual.UnknownInitialization; -import org.checkerframework.checker.nullness.qual.RequiresNonNull; import org.jspecify.annotations.Nullable; import java.util.Comparator; @@ -88,8 +87,8 @@ final class MergeUnionEnumerator implements Enumerator { } @RequiresNonNull("inputs") - @SuppressWarnings("method.invocation.invalid") - private void initEnumerators(@UnknownInitialization MergeUnionEnumerator this) { + @SuppressWarnings("NullAway") + private void initEnumerators() { for (int i = 0; i < inputs.length; i++) { moveEnumerator(i); } @@ -114,7 +113,7 @@ private boolean checkNotDuplicated(TSource value) { } // check duplicates - @SuppressWarnings("dereference.of.nullable") + @SuppressWarnings("NullAway") final EnumerableDefaults.Wrapped wrapped = wrapper.apply(value); if (!processed.contains(wrapped)) { final TKey key = sortKeySelector.apply(value); @@ -122,7 +121,7 @@ private boolean checkNotDuplicated(TSource value) { // Since inputs are sorted, we do not need to keep in the set all the items that we // have previously returned, just the ones with the same key, as soon as we see a new // key, we can clear the set containing the items belonging to the previous key - @SuppressWarnings("argument.type.incompatible") + @SuppressWarnings("NullAway") final int sortComparison = sortComparator.compare(key, currentKeyInProcessedSet); if (sortComparison != 0) { processed.clear(); diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/Nullness.java b/linq4j/src/main/java/org/apache/calcite/linq4j/Nullness.java index 6ad855ea33d9..b3f7f6ee07d8 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/Nullness.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/Nullness.java @@ -16,9 +16,6 @@ */ package org.apache.calcite.linq4j; -import org.checkerframework.checker.initialization.qual.UnderInitialization; -import org.checkerframework.checker.nullness.qual.EnsuresNonNull; -import org.checkerframework.dataflow.qual.Pure; import org.jspecify.annotations.NonNull; import org.jspecify.annotations.Nullable; @@ -28,10 +25,11 @@ * The methods in this class allow to cast nullable reference to a non-nullable one. * This is an internal class, and it is not meant to be used as a public API. * - *

      The class enables to remove checker-qual runtime dependency, and helps IDEs to see - * the resulting types of {@code castNonNull} better. + *

      The class keeps the nullness annotations out of the runtime dependencies, and helps IDEs + * to see the resulting types of {@code castNonNull} better. NullAway is configured to treat + * {@code castNonNull} as its {@code CastToNonNullMethod}, so it reports a call whose argument is + * already non-null. */ -@SuppressWarnings({"cast.unsafe", "RedundantCast", "contracts.postcondition.not.satisfied"}) public class Nullness { private Nullness() { } @@ -56,8 +54,8 @@ private Nullness() { * T get() { return value; } * * - *

      The issue is checkerframework does not permit that because {@code T} - * has unknown nullability, so the following needs to be used: + *

      The issue is that {@code T} has unknown nullability, so the following needs to be + * used: * * 

      
    * T get() { return castNonNull(value); }
@@ -68,10 +66,8 @@ private Nullness() {
    *
    * @return the argument, cast to have the type qualifier @NonNull
    */
-  @Pure
-  public static @EnsuresNonNull("#1")
-   @NonNull T castNonNull(
-      @Nullable T ref) {
+  @SuppressWarnings({"NullAway", "RedundantCast"})
+  public static  @NonNull T castNonNull(@Nullable T ref) {
     //noinspection ConstantConditions
     return (@NonNull T) ref;
   }
@@ -85,8 +81,7 @@ private Nullness() {
    * @return the argument, cast so that elements are @NonNull
    */
   @SuppressWarnings({"unchecked", "ConstantConditions"})
-  @Pure
-  public static  @NonNull T[] castNonNullArray(
+  public static  @NonNull T[] castNonNullArray(
       @Nullable T[] ts) {
     return (@NonNull T []) (Object) ts;
   }
@@ -100,26 +95,23 @@ private Nullness() {
    * @return the argument, cast so that elements are @NonNull
    */
   @SuppressWarnings({"unchecked", "rawtypes"})
-  @Pure
-  public static  List<@NonNull T> castNonNullList(
+  public static  List<@NonNull T> castNonNullList(
       List ts) {
     return (List) (Object) ts;
   }
 
   /**
-   * Allows you to treat an uninitialized or under-initialization object as
-   * initialized with no assertions.
+   * Allows you to pass a partly constructed object where a fully constructed one is expected.
    *
    * @param      The type of the reference
-   * @param ref     A reference that was @Uninitialized at some point but is
-   *                now fully initialized
+   * @param ref     A reference that is still under construction but is fully initialized by the
+   *                time the callee uses it
    *
-   * @return the argument, cast to have type qualifier @Initialized
+   * @return the argument
    */
   @SuppressWarnings({"unchecked"})
-  @Pure
-  public static  T castToInitialized(@UnderInitialization T ref) {
-    // To throw CheckerFramework off the scent, we put the object into an array,
+  public static  T castToInitialized(T ref) {
+    // To throw the nullness checker off the scent, we put the object into an array,
     // cast the array to an Object, and cast back to an array.
     Object src = new Object[] {ref};
     Object[] dest = (Object[]) src;
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/Queryable.java b/linq4j/src/main/java/org/apache/calcite/linq4j/Queryable.java
index a1eb8f4c16b4..ea54b9ba7189 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/Queryable.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/Queryable.java
@@ -16,8 +16,6 @@
  */
 package org.apache.calcite.linq4j;
 
-import org.checkerframework.framework.qual.Covariant;
-
 /**
  * Provides functionality to evaluate queries against a specific data source
  * wherein the type of the data is known.
@@ -26,6 +24,6 @@
  *
  * @param  Element type
  */
-@Covariant(0)
-public interface Queryable extends RawQueryable, ExtendedQueryable {
+public interface Queryable
+    extends RawQueryable, ExtendedQueryable {
 }
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/QueryableFactory.java b/linq4j/src/main/java/org/apache/calcite/linq4j/QueryableFactory.java
index c7a322ce1329..b2a6750c4458 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/QueryableFactory.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/QueryableFactory.java
@@ -33,8 +33,6 @@
 import org.apache.calcite.linq4j.function.Predicate2;
 import org.apache.calcite.linq4j.tree.FunctionExpression;
 
-import org.checkerframework.checker.nullness.qual.PolyNull;
-import org.checkerframework.framework.qual.Covariant;
 import org.jspecify.annotations.Nullable;
 
 import java.math.BigDecimal;
@@ -45,7 +43,6 @@
  *
  * @param  Element type
  */
-@Covariant(0)
 public interface QueryableFactory {
 
   /**
@@ -215,7 +212,7 @@ boolean contains(Queryable source, T element,
    *
    * 
      If {@code value} is not null, the result is never null.
    */
-  Queryable<@PolyNull T> defaultIfEmpty(Queryable source, @PolyNull T value);
+  Queryable<@Nullable T> defaultIfEmpty(Queryable source, @Nullable T value);
 
   /**
    * Returns distinct elements from a sequence by using
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/QueryableRecorder.java b/linq4j/src/main/java/org/apache/calcite/linq4j/QueryableRecorder.java
index fff571251240..1b164543bd38 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/QueryableRecorder.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/QueryableRecorder.java
@@ -33,8 +33,6 @@
 import org.apache.calcite.linq4j.function.Predicate2;
 import org.apache.calcite.linq4j.tree.FunctionExpression;
 
-import org.checkerframework.checker.nullness.qual.PolyNull;
-import org.checkerframework.framework.qual.Covariant;
 import org.jspecify.annotations.Nullable;
 
 import java.lang.reflect.Type;
@@ -53,7 +51,6 @@
  *
  * @param  Element type
  */
-@Covariant(0)
 public class QueryableRecorder implements QueryableFactory {
   private static final QueryableRecorder INSTANCE = new QueryableRecorder();
 
@@ -268,9 +265,9 @@ public static  QueryableRecorder instance() {
     };
   }
 
-  @SuppressWarnings("return.type.incompatible")
-  @Override public Queryable<@PolyNull T> defaultIfEmpty(final Queryable source,
-      final @PolyNull T value) {
+  @SuppressWarnings("NullAway")
+  @Override public Queryable<@Nullable T> defaultIfEmpty(final Queryable source,
+      final @Nullable T value) {
     return new NonLeafReplayableQueryable(source) {
       @Override public void replay(QueryableFactory factory) {
         factory.defaultIfEmpty(source, value);
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/RawEnumerable.java b/linq4j/src/main/java/org/apache/calcite/linq4j/RawEnumerable.java
index b77a0f179838..18d6c701cd7b 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/RawEnumerable.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/RawEnumerable.java
@@ -16,8 +16,6 @@
  */
 package org.apache.calcite.linq4j;
 
-import org.checkerframework.framework.qual.Covariant;
-
 /**
  * Exposes the enumerator, which supports a simple iteration over a collection,
  * without the extension methods.
@@ -31,7 +29,6 @@
  * @param  Element type
  * @see Enumerable
  */
-@Covariant(0)
 public interface RawEnumerable {
   /**
    * Returns an enumerator that iterates through a collection.
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/RawQueryable.java b/linq4j/src/main/java/org/apache/calcite/linq4j/RawQueryable.java
index 69398463ba1f..107c85529196 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/RawQueryable.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/RawQueryable.java
@@ -18,7 +18,6 @@
 
 import org.apache.calcite.linq4j.tree.Expression;
 
-import org.checkerframework.framework.qual.Covariant;
 import org.jspecify.annotations.Nullable;
 
 import java.lang.reflect.Type;
@@ -32,7 +31,6 @@
  *
  * @param  Element type
  */
-@Covariant(0)
 public interface RawQueryable extends Enumerable {
   /**
    * Gets the type of the element(s) that are returned when the expression
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/annotations/Contract.java b/linq4j/src/main/java/org/apache/calcite/linq4j/annotations/Contract.java
new file mode 100644
index 000000000000..6dcf274db197
--- /dev/null
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/annotations/Contract.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.linq4j.annotations;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * States how the nullness of a method's result follows from the nullness of its arguments.
+ *
+ * 
      The value holds one or more clauses separated by {@code ;}. A clause lists one constraint per
+ * parameter, then {@code ->}, then what the method guarantees:
+ *
+ * 
      + * @Contract("!null, _ -> !null")
+ * public static @Nullable Integer plus(@Nullable Integer b0, int b1) { ... }
+ * 
      
+ *
+ * 
      A constraint is {@code null}, {@code !null}, {@code true}, {@code false} or {@code _} for
+ * "anything". The guarantee is one of those, or {@code fail} for a method that always throws.
+ *
+ * 
      This replaces the {@code @PolyNull} qualifier of the Checker Framework, which has no JSpecify
+ * equivalent. It cannot describe a receiver parameter, a varargs method, or a type argument such
+ * as {@code Enumerable<@Nullable T>}.
+ */
+@Documented
+@Retention(RetentionPolicy.CLASS)
+@Target(ElementType.METHOD)
+public @interface Contract {
+  /** Returns the contract clauses.
+   *
+   * @return contract clauses, separated by {@code ;} */
+  String value();
+}
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/annotations/EnsuresNonNull.java b/linq4j/src/main/java/org/apache/calcite/linq4j/annotations/EnsuresNonNull.java
new file mode 100644
index 000000000000..2a59fa83a2bc
--- /dev/null
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/annotations/EnsuresNonNull.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.linq4j.annotations;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * States which nullable fields the method has assigned by the time it returns.
+ *
+ * 
      Callers may read those fields without a null check afterwards. This is what an
+ * {@code init}-style method needs in order to satisfy the constructor's initialization check.
+ */
+@Documented
+@Retention(RetentionPolicy.CLASS)
+@Target(ElementType.METHOD)
+public @interface EnsuresNonNull {
+  /** Returns the fields that are non-null once the method returns.
+   *
+   * @return field names, optionally qualified with {@code this.} */
+  String[] value();
+}
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/annotations/EnsuresNonNullIf.java b/linq4j/src/main/java/org/apache/calcite/linq4j/annotations/EnsuresNonNullIf.java
new file mode 100644
index 000000000000..024432d532ed
--- /dev/null
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/annotations/EnsuresNonNullIf.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.linq4j.annotations;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * States which nullable fields are non-null when the method returns a given boolean.
+ *
+ * 
      It turns a {@code hasFoo()} predicate into something a caller can rely on:
+ *
+ * 
      + * @EnsuresNonNullIf(value = "hints", result = true)
+ * public boolean hasHints() { return hints != null && !hints.isEmpty(); }
+ * 
      
+ *
+ * 
      The fields are guaranteed only on the branch that matches {@link #result()}.
+ */
+@Documented
+@Retention(RetentionPolicy.CLASS)
+@Target(ElementType.METHOD)
+public @interface EnsuresNonNullIf {
+  /** Returns the fields that are non-null when the method returns {@link #result()}.
+   *
+   * @return field names, optionally qualified with {@code this.} */
+  String[] value();
+
+  /** Returns the result for which the guarantee holds.
+   *
+   * @return the boolean result that makes the fields non-null */
+  boolean result();
+}
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/annotations/MonotonicNonNull.java b/linq4j/src/main/java/org/apache/calcite/linq4j/annotations/MonotonicNonNull.java
new file mode 100644
index 000000000000..63452b3fe7ac
--- /dev/null
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/annotations/MonotonicNonNull.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.linq4j.annotations;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * States that a field starts as null and is never assigned null again once it holds a value.
+ *
+ * 
      The field is exempt from the constructor's initialization check, and a null check on it holds
+ * across an intervening method call. Every read still needs that check: the field is null until
+ * something assigns it.
+ */
+@Documented
+@Retention(RetentionPolicy.CLASS)
+@Target({ElementType.FIELD, ElementType.TYPE_USE})
+public @interface MonotonicNonNull {
+}
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/annotations/RequiresNonNull.java b/linq4j/src/main/java/org/apache/calcite/linq4j/annotations/RequiresNonNull.java
new file mode 100644
index 000000000000..22e33f2e0ca7
--- /dev/null
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/annotations/RequiresNonNull.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.linq4j.annotations;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * States which nullable fields a caller must have checked before calling the method.
+ *
+ * 
      Inside the method those fields count as non-null. The caller has to establish that, so this
+ * moves the check to the one place that can do it rather than repeating it in every method that
+ * reads the field.
+ */
+@Documented
+@Retention(RetentionPolicy.CLASS)
+@Target(ElementType.METHOD)
+public @interface RequiresNonNull {
+  /** Returns the fields that must be non-null on entry.
+   *
+   * @return field names, optionally qualified with {@code this.} */
+  String[] value();
+}
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/annotations/package-info.java b/linq4j/src/main/java/org/apache/calcite/linq4j/annotations/package-info.java
new file mode 100644
index 000000000000..09597c86bed9
--- /dev/null
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/annotations/package-info.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Nullness annotations that JSpecify does not define.
+ *
+ * 
      JSpecify covers {@code @Nullable}, {@code @NonNull} and {@code @NullMarked} only. The
+ * annotations here cover what Calcite additionally needs to express: a method whose result is
+ * null exactly when an argument is null, a field that starts as null and stays non-null once
+ * assigned, and pre- and postconditions on fields.
+ *
+ * 
      A nullness checker recognises them by the last component of their name rather than by their
+ * package, so they need no dependency on the checker itself. NullAway is the checker Calcite
+ * runs; see {@code NullabilityUtil.findAnnotation} and
+ * {@code Nullness.isMonotonicNonNullAnnotation} in NullAway for the matching rules.
+ */
+@NullMarked
+package org.apache.calcite.linq4j.annotations;
+
+import org.jspecify.annotations.NullMarked;
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/function/Function1.java b/linq4j/src/main/java/org/apache/calcite/linq4j/function/Function1.java
index e76d430be3e4..5fff55e81779 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/function/Function1.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/function/Function1.java
@@ -22,7 +22,8 @@
  * @param  Result type
  * @param  Type of parameter 0
  */
-public interface Function1 extends Function {
+public interface Function1
+    extends Function {
   /**
    * The identity function.
    *
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/function/Function2.java b/linq4j/src/main/java/org/apache/calcite/linq4j/function/Function2.java
index a2ab26ac2f4b..05345343445e 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/function/Function2.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/function/Function2.java
@@ -23,6 +23,7 @@
  * @param  Type of argument #0
  * @param  Type of argument #1
  */
-public interface Function2 extends Function {
+public interface Function2 extends Function {
   R apply(T0 v0, T1 v1);
 }
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/function/Functions.java b/linq4j/src/main/java/org/apache/calcite/linq4j/function/Functions.java
index 63eab93e8904..6f8ffd84cc6e 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/function/Functions.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/function/Functions.java
@@ -16,8 +16,6 @@
  */
 package org.apache.calcite.linq4j.function;
 
-import org.checkerframework.framework.qual.DefaultQualifier;
-import org.checkerframework.framework.qual.TypeUseLocation;
 import org.jspecify.annotations.Nullable;
 
 import java.io.Serializable;
@@ -859,7 +857,7 @@ private static class NullsLastReverseComparator
    * @param  result type
    * @param  first argument type
    * @param  second argument type */
-  private static final class Ignore<@Nullable R, T0, T1>
+  private static final class Ignore
       implements Function0, Function1, Function2 {
     @Override public R apply() {
       return null;
@@ -873,12 +871,6 @@ private static final class Ignore<@Nullable R, T0, T1>
       return null;
     }
 
-    @DefaultQualifier(
-        value = Nullable.class,
-        locations = {
-        TypeUseLocation.LOWER_BOUND,
-        TypeUseLocation.UPPER_BOUND,
-    })
     static final Ignore INSTANCE = new Ignore<>();
   }
 
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/BlockBuilder.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/BlockBuilder.java
index ed9a562d0685..525922e8a2b2 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/BlockBuilder.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/BlockBuilder.java
@@ -16,7 +16,6 @@
  */
 package org.apache.calcite.linq4j.tree;
 
-import org.checkerframework.checker.nullness.qual.PolyNull;
 import org.jspecify.annotations.Nullable;
 
 import java.lang.reflect.Modifier;
@@ -201,7 +200,7 @@ public Expression append(String name, Expression expression) {
    * Appends an expression to a list of statements if it is not null,
    * and returns the expression.
    */
-  public @PolyNull Expression appendIfNotNull(String name, @PolyNull Expression expression) {
+  public @Nullable Expression appendIfNotNull(String name, @Nullable Expression expression) {
     if (expression == null) {
       return null;
     }
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/BlockStatement.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/BlockStatement.java
index 11d0372c2798..24a843d96438 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/BlockStatement.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/BlockStatement.java
@@ -16,7 +16,6 @@
  */
 package org.apache.calcite.linq4j.tree;
 
-import org.checkerframework.checker.initialization.qual.UnderInitialization;
 import org.jspecify.annotations.Nullable;
 
 import java.lang.reflect.Type;
@@ -43,7 +42,6 @@ public class BlockStatement extends Statement {
   }
 
   private boolean distinctVariables(
-      @UnderInitialization(BlockStatement.class) BlockStatement this,
       boolean fail) {
     Set names = new HashSet<>();
     for (Statement statement : statements) {
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ConstantExpression.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ConstantExpression.java
index ddc377a00709..6c27b11137b1 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ConstantExpression.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ConstantExpression.java
@@ -227,7 +227,7 @@ private static ExpressionWriter write(ExpressionWriter writer,
       writer.append("new ").append(value.getClass());
       list(writer,
           Arrays.stream(classFields)
-              // <@Nullable Object> is needed for CheckerFramework
+              // the witness keeps the element type nullable; getFieldValue may return null
               .<@Nullable Object>map(field -> getFieldValue(value, field))
               .collect(Collectors.toList()),
           "(\n", ",\n", ")");
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/Expressions.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/Expressions.java
index fd4dd28af7c5..46598c980727 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/Expressions.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/Expressions.java
@@ -2754,7 +2754,7 @@ public static BinaryExpression subtractChecked(Expression left,
    * Creates a SwitchExpression that represents a switch statement
    * without a default case.
    */
-  @SuppressWarnings("nullness")
+  @SuppressWarnings("NullAway")
   public static SwitchStatement switch_(Expression switchValue,
       SwitchCase... cases) {
     return switch_(switchValue, null, null, toList(cases));
@@ -2764,7 +2764,7 @@ public static SwitchStatement switch_(Expression switchValue,
    * Creates a SwitchExpression that represents a switch statement
    * that has a default case.
    */
-  @SuppressWarnings("nullness")
+  @SuppressWarnings("NullAway")
   public static SwitchStatement switch_(Expression switchValue,
       Expression defaultBody, SwitchCase... cases) {
     return switch_(switchValue, defaultBody, null, toList(cases));
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/Primitive.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/Primitive.java
index a52183f555bd..41c8725f0dab 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/Primitive.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/Primitive.java
@@ -1010,7 +1010,7 @@ public void send(Field field, Object o, Sink sink)
   /**
    * Reads value from a source into an array.
    */
-  @SuppressWarnings("argument.type.incompatible")
+  @SuppressWarnings("NullAway")
   public void arrayItem(Source source, Object dataSet, int ordinal) {
     switch (this) {
     case DOUBLE:
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/Types.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/Types.java
index ea9548086a24..aaddfde1fe73 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/Types.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/Types.java
@@ -276,7 +276,7 @@ public static boolean allAssignable(boolean varArgs,
    *
    * @return Whether parameter can be assigned from argument
    */
-  @SuppressWarnings("nullness")
+  @SuppressWarnings("NullAway")
   private static boolean assignableFrom(Class parameter, Class argument) {
     return parameter.isAssignableFrom(argument)
            || parameter.isPrimitive()
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/VisitorImpl.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/VisitorImpl.java
index 43264c13810f..61c3cbd67d8b 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/VisitorImpl.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/VisitorImpl.java
@@ -27,7 +27,7 @@
  * @param  Return type
  */
 @SuppressWarnings("unused")
-public class VisitorImpl<@Nullable R> implements Visitor {
+public class VisitorImpl implements Visitor {
   public VisitorImpl() {
     super();
   }
diff --git a/server/src/main/java/org/apache/calcite/server/MaterializedViewTable.java b/server/src/main/java/org/apache/calcite/server/MaterializedViewTable.java
index 5915514ccc5d..d17398127155 100644
--- a/server/src/main/java/org/apache/calcite/server/MaterializedViewTable.java
+++ b/server/src/main/java/org/apache/calcite/server/MaterializedViewTable.java
@@ -39,7 +39,7 @@ class MaterializedViewTable
     return Schema.TableType.MATERIALIZED_VIEW;
   }
 
-  @Override public  @Nullable C unwrap(Class aClass) {
+  @Override public  @Nullable C unwrap(Class aClass) {
     if (MaterializationKey.class.isAssignableFrom(aClass)
         && aClass.isInstance(key)) {
       return aClass.cast(key);
diff --git a/server/src/main/java/org/apache/calcite/server/MutableArrayTable.java b/server/src/main/java/org/apache/calcite/server/MutableArrayTable.java
index 95456bc4948c..4bb876bfb9e6 100644
--- a/server/src/main/java/org/apache/calcite/server/MutableArrayTable.java
+++ b/server/src/main/java/org/apache/calcite/server/MutableArrayTable.java
@@ -94,7 +94,7 @@ class MutableArrayTable extends AbstractModifiableTable
     return protoRowType.apply(typeFactory);
   }
 
-  @Override public  @Nullable C unwrap(Class aClass) {
+  @Override public  @Nullable C unwrap(Class aClass) {
     if (aClass.isInstance(initializerExpressionFactory)) {
       return aClass.cast(initializerExpressionFactory);
     }

From 9d32656587b092cd8822d837abdcf4b0193b4d11 Mon Sep 17 00:00:00 2001
From: Vladimir Sitnikov 
Date: Sun, 23 Aug 2026 22:11:06 +0300
Subject: [PATCH 498/562] [CALCITE-7736] Replace @PolyNull with @Contract

@PolyNull said the result is null exactly when the argument is null. The
previous commit weakened it to @Nullable, which makes every caller treat the
result as nullable even when it passed a non-null argument. @Contract states
the direction that matters at a call site:

    @Contract("!null, _ -> !null")
    public static @Nullable Integer plus(@Nullable Integer b0, int b1)

NullAway is configured with `CheckContracts=true`, so it verifies each clause
against the method body rather than trusting it, and none of the 108 clauses
here is rejected.

Not every @PolyNull converts. A clause describes arguments, so it cannot
describe a receiver parameter, and it cannot describe a varargs method, whose
call sites pass a different number of arguments. Nor does it reach a type
argument such as `Enumerable<@Nullable T>`, where the element rather than the
result is polymorphic. Those keep the plain @Nullable.

Passing a clause whose length does not match the call crashes NullAway with an
IndexOutOfBoundsException from `ContractHandler.onDataflowVisitMethodInvocation`,
which reads arguments by the antecedent's length without checking the arity it
was given. It validates that on declarations but not at call sites; worth
reporting upstream.

Co-Authored-By: Claude Opus 5 
---
 .../config/CalciteConnectionConfig.java       |  7 ++
 .../config/CalciteConnectionConfigImpl.java   |  7 ++
 .../org/apache/calcite/plan/RelOptUtil.java   |  2 +
 .../calcite/plan/volcano/VolcanoPlanner.java  |  2 +
 .../calcite/rel/externalize/RelJson.java      |  2 +
 .../rel/metadata/RelMdColumnOrigins.java      |  2 +
 .../metadata/RelMdPercentageOriginalRows.java |  2 +
 .../calcite/rel/metadata/RelMdUtil.java       |  4 ++
 .../org/apache/calcite/rex/RexBuilder.java    |  2 +
 .../org/apache/calcite/rex/RexLiteral.java    |  2 +
 .../org/apache/calcite/rex/RexShuttle.java    |  3 +
 .../org/apache/calcite/runtime/Resources.java |  4 ++
 .../apache/calcite/runtime/SqlFunctions.java  | 68 +++++++++++++++++++
 .../java/org/apache/calcite/sql/SqlUtil.java  |  2 +
 .../sql/validate/SqlValidatorImpl.java        |  2 +
 .../org/apache/calcite/util/NumberUtil.java   |  9 +++
 .../calcite/util/SerializableCharset.java     |  3 +
 .../java/org/apache/calcite/util/Util.java    |  2 +
 .../calcite/linq4j/DefaultEnumerable.java     |  2 +
 .../calcite/linq4j/ExtendedEnumerable.java    |  2 +
 .../calcite/linq4j/tree/BlockBuilder.java     |  3 +
 21 files changed, 132 insertions(+)

diff --git a/core/src/main/java/org/apache/calcite/config/CalciteConnectionConfig.java b/core/src/main/java/org/apache/calcite/config/CalciteConnectionConfig.java
index 84d52b62b7e2..15612b57a2cd 100644
--- a/core/src/main/java/org/apache/calcite/config/CalciteConnectionConfig.java
+++ b/core/src/main/java/org/apache/calcite/config/CalciteConnectionConfig.java
@@ -19,6 +19,7 @@
 import org.apache.calcite.avatica.ConnectionConfig;
 import org.apache.calcite.avatica.util.Casing;
 import org.apache.calcite.avatica.util.Quoting;
+import org.apache.calcite.linq4j.annotations.Contract;
 import org.apache.calcite.model.JsonSchema;
 import org.apache.calcite.sql.validate.SqlConformance;
 
@@ -60,6 +61,7 @@ public interface CalciteConnectionConfig extends ConnectionConfig {
   /** Returns the value of {@link CalciteConnectionProperty#FUN},
    * or a default operator table if not set. If {@code defaultOperatorTable}
    * is not null, the result is never null. */
+  @Contract("_, !null -> !null")
    @Nullable T fun(Class operatorTableClass,
       @Nullable T defaultOperatorTable);
   /** Returns the value of {@link CalciteConnectionProperty#MODEL}. */
@@ -77,11 +79,13 @@ public interface CalciteConnectionConfig extends ConnectionConfig {
   /** Returns the value of {@link CalciteConnectionProperty#PARSER_FACTORY},
    * or a default parser if not set. If {@code defaultParserFactory}
    * is not null, the result is never null. */
+  @Contract("_, !null -> !null")
    @Nullable T parserFactory(Class parserFactoryClass,
       @Nullable T defaultParserFactory);
   /** Returns the value of {@link CalciteConnectionProperty#SCHEMA_FACTORY},
    * or a default schema factory if not set. If {@code defaultSchemaFactory}
    * is not null, the result is never null. */
+  @Contract("_, !null -> !null")
    @Nullable T schemaFactory(Class schemaFactoryClass,
       @Nullable T defaultSchemaFactory);
   /** Returns the value of {@link CalciteConnectionProperty#SCHEMA_TYPE}. */
@@ -94,6 +98,7 @@ public interface CalciteConnectionConfig extends ConnectionConfig {
   /** Returns the value of {@link CalciteConnectionProperty#TYPE_SYSTEM},
    * or a default type system if not set. If {@code defaultTypeSystem}
    * is not null, the result is never null. */
+  @Contract("_, !null -> !null")
    @Nullable T typeSystem(Class typeSystemClass,
       @Nullable T defaultTypeSystem);
   /** Returns the value of {@link CalciteConnectionProperty#CONFORMANCE}. */
@@ -116,12 +121,14 @@ public interface CalciteConnectionConfig extends ConnectionConfig {
   /** Returns the value of {@link CalciteConnectionProperty#META_TABLE_FACTORY},
    * or a default meta table factory if not set. If
    * {@code defaultMetaTableFactory} is not null, the result is never null. */
+  @Contract("_, !null -> !null")
    @Nullable T metaTableFactory(Class metaTableFactoryClass,
       @Nullable T defaultMetaTableFactory);
 
   /** Returns the value of {@link CalciteConnectionProperty#META_COLUMN_FACTORY},
    * or a default meta column factory if not set. If
    * {@code defaultMetaColumnFactory} is not null, the result is never null. */
+  @Contract("_, !null -> !null")
    @Nullable T metaColumnFactory(Class metaColumnFactoryClass,
       @Nullable T defaultMetaColumnFactory);
 }
diff --git a/core/src/main/java/org/apache/calcite/config/CalciteConnectionConfigImpl.java b/core/src/main/java/org/apache/calcite/config/CalciteConnectionConfigImpl.java
index e0b2cc632b05..9410d0e33488 100644
--- a/core/src/main/java/org/apache/calcite/config/CalciteConnectionConfigImpl.java
+++ b/core/src/main/java/org/apache/calcite/config/CalciteConnectionConfigImpl.java
@@ -19,6 +19,7 @@
 import org.apache.calcite.avatica.ConnectionConfigImpl;
 import org.apache.calcite.avatica.util.Casing;
 import org.apache.calcite.avatica.util.Quoting;
+import org.apache.calcite.linq4j.annotations.Contract;
 import org.apache.calcite.model.JsonSchema;
 import org.apache.calcite.runtime.ConsList;
 import org.apache.calcite.sql.SqlOperatorTable;
@@ -105,6 +106,7 @@ public boolean isSet(CalciteConnectionProperty property) {
         .getEnum(NullCollation.class, NullCollation.HIGH);
   }
 
+  @Contract("_, !null -> !null")
   @Override public  @Nullable T fun(Class operatorTableClass,
       @Nullable T defaultOperatorTable) {
     final String fun =
@@ -152,12 +154,14 @@ public boolean isSet(CalciteConnectionProperty property) {
         .getBoolean(lex().caseSensitive);
   }
 
+  @Contract("_, !null -> !null")
   @Override public  @Nullable T parserFactory(Class parserFactoryClass,
       @Nullable T defaultParserFactory) {
     return CalciteConnectionProperty.PARSER_FACTORY.wrap(properties)
         .getPlugin(parserFactoryClass, defaultParserFactory);
   }
 
+  @Contract("_, !null -> !null")
   @Override public  @Nullable T schemaFactory(Class schemaFactoryClass,
       @Nullable T defaultSchemaFactory) {
     return CalciteConnectionProperty.SCHEMA_FACTORY.wrap(properties)
@@ -178,6 +182,7 @@ public boolean isSet(CalciteConnectionProperty property) {
         .getBoolean();
   }
 
+  @Contract("_, !null -> !null")
   @Override public  @Nullable T typeSystem(Class typeSystemClass,
       @Nullable T defaultTypeSystem) {
     return CalciteConnectionProperty.TYPE_SYSTEM.wrap(properties)
@@ -219,6 +224,7 @@ public boolean isSet(CalciteConnectionProperty property) {
         .getBoolean();
   }
 
+  @Contract("_, !null -> !null")
   @Override public  @Nullable T metaTableFactory(
       Class metaTableFactoryClass,
       @Nullable T defaultMetaTableFactory) {
@@ -226,6 +232,7 @@ public boolean isSet(CalciteConnectionProperty property) {
         .getPlugin(metaTableFactoryClass, defaultMetaTableFactory);
   }
 
+  @Contract("_, !null -> !null")
   @Override public  @Nullable T metaColumnFactory(
       Class metaColumnFactoryClass,
       @Nullable T defaultMetaColumnFactory) {
diff --git a/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java b/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java
index 166ffc81f3fa..815aae5a31fe 100644
--- a/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java
+++ b/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java
@@ -21,6 +21,7 @@
 import org.apache.calcite.config.CalciteSystemProperty;
 import org.apache.calcite.interpreter.Bindables;
 import org.apache.calcite.linq4j.Ord;
+import org.apache.calcite.linq4j.annotations.Contract;
 import org.apache.calcite.linq4j.function.Experimental;
 import org.apache.calcite.rel.RelCollations;
 import org.apache.calcite.rel.RelHomogeneousShuttle;
@@ -2479,6 +2480,7 @@ public static String toString(
    * returns null if and only if {@code rel} is null,
    * returns expanded detail info for {@code rel} if {@code expand} is true.
    */
+  @Contract("!null, _, _ -> !null")
   public static @Nullable String toString(
       final @Nullable RelNode rel,
       SqlExplainLevel detailLevel,
diff --git a/core/src/main/java/org/apache/calcite/plan/volcano/VolcanoPlanner.java b/core/src/main/java/org/apache/calcite/plan/volcano/VolcanoPlanner.java
index f1b27aae74dd..6f2b73379df9 100644
--- a/core/src/main/java/org/apache/calcite/plan/volcano/VolcanoPlanner.java
+++ b/core/src/main/java/org/apache/calcite/plan/volcano/VolcanoPlanner.java
@@ -18,6 +18,7 @@
 
 import org.apache.calcite.config.CalciteConnectionConfig;
 import org.apache.calcite.config.CalciteSystemProperty;
+import org.apache.calcite.linq4j.annotations.Contract;
 import org.apache.calcite.linq4j.annotations.EnsuresNonNull;
 import org.apache.calcite.linq4j.annotations.MonotonicNonNull;
 import org.apache.calcite.linq4j.annotations.RequiresNonNull;
@@ -1500,6 +1501,7 @@ private RelSubset registerSubset(
    * @param plan Plan
    * @return Normalized plan
    */
+  @Contract("!null -> !null")
   public static @Nullable String normalizePlan(@Nullable String plan) {
     if (plan == null) {
       return null;
diff --git a/core/src/main/java/org/apache/calcite/rel/externalize/RelJson.java b/core/src/main/java/org/apache/calcite/rel/externalize/RelJson.java
index 8d87a87fc3a1..1e6aae9c43bb 100644
--- a/core/src/main/java/org/apache/calcite/rel/externalize/RelJson.java
+++ b/core/src/main/java/org/apache/calcite/rel/externalize/RelJson.java
@@ -19,6 +19,7 @@
 import org.apache.calcite.avatica.AvaticaUtils;
 import org.apache.calcite.avatica.util.ByteString;
 import org.apache.calcite.avatica.util.TimeUnit;
+import org.apache.calcite.linq4j.annotations.Contract;
 import org.apache.calcite.plan.RelOptCluster;
 import org.apache.calcite.plan.RelOptTable;
 import org.apache.calcite.plan.RelTraitSet;
@@ -756,6 +757,7 @@ public RexNode toRex(RelOptCluster cluster, Object o) {
   }
 
   @SuppressWarnings({"rawtypes", "unchecked"})
+  @Contract("_, !null -> !null")
   @Nullable RexNode toRex(RelInput relInput, @Nullable Object o) {
     final RelOptCluster cluster = relInput.getCluster();
     final RexBuilder rexBuilder = cluster.getRexBuilder();
diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdColumnOrigins.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdColumnOrigins.java
index 676263f70806..efe93d1e039c 100644
--- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdColumnOrigins.java
+++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdColumnOrigins.java
@@ -16,6 +16,7 @@
  */
 package org.apache.calcite.rel.metadata;
 
+import org.apache.calcite.linq4j.annotations.Contract;
 import org.apache.calcite.plan.RelOptTable;
 import org.apache.calcite.rel.RelNode;
 import org.apache.calcite.rel.core.Aggregate;
@@ -276,6 +277,7 @@ private RelMdColumnOrigins() {}
     return set;
   }
 
+  @Contract("!null -> !null")
   private static @Nullable Set createDerivedColumnOrigins(
       @Nullable Set inputSet) {
     if (inputSet == null) {
diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdPercentageOriginalRows.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdPercentageOriginalRows.java
index 66b56cad6cac..f6f3b587bcff 100644
--- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdPercentageOriginalRows.java
+++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdPercentageOriginalRows.java
@@ -17,6 +17,7 @@
 package org.apache.calcite.rel.metadata;
 
 import org.apache.calcite.adapter.enumerable.EnumerableInterpreter;
+import org.apache.calcite.linq4j.annotations.Contract;
 import org.apache.calcite.plan.RelOptCost;
 import org.apache.calcite.rel.RelNode;
 import org.apache.calcite.rel.core.Aggregate;
@@ -234,6 +235,7 @@ public Double getPercentageOriginalRows(Union rel, RelMetadataQuery mq) {
     return rel.computeSelfCost(rel.getCluster().getPlanner(), mq);
   }
 
+  @Contract("!null, !null -> !null")
   private static @Nullable Double quotientForPercentage(
       @Nullable Double numerator,
       @Nullable Double denominator) {
diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdUtil.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdUtil.java
index baf9c6d5cdea..96da5a18b3db 100644
--- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdUtil.java
+++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdUtil.java
@@ -16,6 +16,7 @@
  */
 package org.apache.calcite.rel.metadata;
 
+import org.apache.calcite.linq4j.annotations.Contract;
 import org.apache.calcite.plan.RelOptUtil;
 import org.apache.calcite.rel.RelCollation;
 import org.apache.calcite.rel.RelNode;
@@ -314,6 +315,7 @@ public static void setLeftRightBitmaps(
    * @return the expected number of distinct values, or null if either argument
    * is null
    */
+  @Contract("!null, !null -> !null")
   public static @Nullable Double numDistinctVals(
       @Nullable Double domainSize,
       @Nullable Double numSelected) {
@@ -1080,6 +1082,7 @@ private static boolean alreadySmaller(RelMetadataQuery mq, RelNode input,
    * 
      Throws if {@code result} is not null, not in range 0 to 1,
    * and assertions are enabled.
    */
+  @Contract("!null -> !null")
   public static @Nullable Double validatePercentage(@Nullable Double result) {
     assert isPercentage(result, true);
     return result;
@@ -1116,6 +1119,7 @@ private static boolean isPercentage(@Nullable Double result, boolean fail) {
    * @return the corrected value from the {@code result}
    * @throws AssertionError if the {@code result} is negative
    */
+  @Contract("!null -> !null")
   public static @Nullable Double validateResult(@Nullable Double result) {
     if (result == null) {
       return null;
diff --git a/core/src/main/java/org/apache/calcite/rex/RexBuilder.java b/core/src/main/java/org/apache/calcite/rex/RexBuilder.java
index e002c84a7821..6a5108706519 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexBuilder.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexBuilder.java
@@ -20,6 +20,7 @@
 import org.apache.calcite.avatica.util.DateTimeUtils;
 import org.apache.calcite.avatica.util.Spaces;
 import org.apache.calcite.avatica.util.TimeUnit;
+import org.apache.calcite.linq4j.annotations.Contract;
 import org.apache.calcite.rel.RelNode;
 import org.apache.calcite.rel.core.AggregateCall;
 import org.apache.calcite.rel.core.CorrelationId;
@@ -2395,6 +2396,7 @@ public RexNode makeLambdaCall(RexNode expr, List parameters) {
    * {@link org.apache.calcite.rex.RexLiteral#valueMatchesType}.
    *
    * 
      Returns null if and only if {@code o} is null. */
+  @Contract("!null, _ -> !null")
   private @Nullable Object clean(@Nullable Object o, RelDataType type) {
     if (o == null) {
       return o;
diff --git a/core/src/main/java/org/apache/calcite/rex/RexLiteral.java b/core/src/main/java/org/apache/calcite/rex/RexLiteral.java
index 98a9179fefde..2c7b73bae74b 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexLiteral.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexLiteral.java
@@ -20,6 +20,7 @@
 import org.apache.calcite.avatica.util.DateTimeUtils;
 import org.apache.calcite.avatica.util.TimeUnit;
 import org.apache.calcite.config.CalciteSystemProperty;
+import org.apache.calcite.linq4j.annotations.Contract;
 import org.apache.calcite.linq4j.annotations.RequiresNonNull;
 import org.apache.calcite.linq4j.function.Functions;
 import org.apache.calcite.rel.RelNode;
@@ -820,6 +821,7 @@ private static RexLiteral toLiteral(RelDataType type, Comparable value) {
    *                 by the Jdbc call to return a column as a string
    * @return a typed RexLiteral, or null
    */
+  @Contract("_, _, !null -> !null")
   public static @Nullable RexLiteral fromJdbcString(
       RelDataType type,
       SqlTypeName typeName,
diff --git a/core/src/main/java/org/apache/calcite/rex/RexShuttle.java b/core/src/main/java/org/apache/calcite/rex/RexShuttle.java
index f1c3531c6bb8..7e013e27bbd9 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexShuttle.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexShuttle.java
@@ -16,6 +16,7 @@
  */
 package org.apache.calcite.rex;
 
+import org.apache.calcite.linq4j.annotations.Contract;
 import org.apache.calcite.sql.SqlAggFunction;
 
 import com.google.common.collect.ImmutableList;
@@ -276,6 +277,7 @@ protected List visitFieldCollations(
    *
    * 
      Returns null if and only if {@code exprList} is null.
    */
+  @Contract("!null -> !null")
   public final  @Nullable List apply(@Nullable List exprList) {
     if (exprList == null) {
       return exprList;
@@ -292,6 +294,7 @@ protected List visitFieldCollations(
    * Applies this shuttle to an expression, or returns null if the expression
    * is null.
    */
+  @Contract("!null -> !null")
   public final @Nullable RexNode apply(@Nullable RexNode expr) {
     return (expr == null) ? expr : expr.accept(this);
   }
diff --git a/core/src/main/java/org/apache/calcite/runtime/Resources.java b/core/src/main/java/org/apache/calcite/runtime/Resources.java
index 637debfe8ab6..3ff86dcd42f0 100644
--- a/core/src/main/java/org/apache/calcite/runtime/Resources.java
+++ b/core/src/main/java/org/apache/calcite/runtime/Resources.java
@@ -16,6 +16,7 @@
  */
 package org.apache.calcite.runtime;
 
+import org.apache.calcite.linq4j.annotations.Contract;
 import org.jspecify.annotations.Nullable;
 import org.apache.calcite.linq4j.annotations.RequiresNonNull;
 
@@ -771,6 +772,7 @@ public StringProp(PropertyAccessor accessor, Method method) {
      * value if the property is not set.
      *
      * 
      If {@code defaultValue} is not null, never returns null. */
+    @Contract("!null -> !null")
     public @Nullable String get(@Nullable String defaultValue) {
       return accessor.stringValue(this, defaultValue);
     }
@@ -796,6 +798,7 @@ public interface PropertyAccessor {
     int intValue(IntProp p);
     int intValue(IntProp p, int defaultValue);
     @Nullable String stringValue(StringProp p);
+    @Contract("_, !null -> !null")
     @Nullable String stringValue(StringProp p, @Nullable String defaultValue);
     boolean booleanValue(BooleanProp p);
     boolean booleanValue(BooleanProp p, boolean defaultValue);
@@ -825,6 +828,7 @@ public int intValue(IntProp p, int defaultValue) {
       return p.defaultValue();
     }
 
+    @Contract("_, !null -> !null")
     @Override public @Nullable String stringValue(StringProp p,
         @Nullable String defaultValue) {
       return defaultValue;
diff --git a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java
index db7f0189e9b6..59b93888772b 100644
--- a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java
+++ b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java
@@ -28,6 +28,7 @@
 import org.apache.calcite.linq4j.Enumerator;
 import org.apache.calcite.linq4j.Linq4j;
 import org.apache.calcite.linq4j.Ord;
+import org.apache.calcite.linq4j.annotations.Contract;
 import org.apache.calcite.linq4j.function.Deterministic;
 import org.apache.calcite.linq4j.function.Experimental;
 import org.apache.calcite.linq4j.function.Function1;
@@ -2793,22 +2794,26 @@ public static int plus(int b0, int b1) {
 
   /** SQL + operator applied to int values; left side may be
    * null. */
+  @Contract("!null, _ -> !null")
   public static @Nullable Integer plus(@Nullable Integer b0, int b1) {
     return b0 == null ? castNonNull(null) : (b0 + b1);
   }
 
   /** SQL + operator applied to int values; right side may be
    * null. */
+  @Contract("_, !null -> !null")
   public static @Nullable Integer plus(int b0, @Nullable Integer b1) {
     return b1 == null ? castNonNull(null) : (b0 + b1);
   }
 
   /** SQL + operator applied to nullable int values. */
+  @Contract("!null, !null -> !null")
   public static @Nullable Integer plus(@Nullable Integer b0, @Nullable Integer b1) {
     return (b0 == null || b1 == null) ? castNonNull(null) : (b0 + b1);
   }
 
   /** SQL + operator applied to nullable long and int values. */
+  @Contract("!null, !null -> !null")
   public static @Nullable Long plus(@Nullable Long b0, @Nullable Integer b1) {
     return (b0 == null || b1 == null)
         ? castNonNull(null)
@@ -2816,6 +2821,7 @@ public static int plus(int b0, int b1) {
   }
 
   /** SQL + operator applied to nullable int and long values. */
+  @Contract("!null, !null -> !null")
   public static @Nullable Long plus(@Nullable Integer b0, @Nullable Long b1) {
     return (b0 == null || b1 == null)
         ? castNonNull(null)
@@ -2823,6 +2829,7 @@ public static int plus(int b0, int b1) {
   }
 
   /** SQL + operator applied to BigDecimal values. */
+  @Contract("!null, !null -> !null")
   public static @Nullable BigDecimal plus(@Nullable BigDecimal b0,
       @Nullable BigDecimal b1) {
     return (b0 == null || b1 == null) ? castNonNull(null) : b0.add(b1);
@@ -2830,6 +2837,7 @@ public static int plus(int b0, int b1) {
 
   /** SQL + operator applied to Object values (at least one operand
    * has ANY type; either may be null). */
+  @Contract("!null, !null -> !null")
   public static @Nullable Object plusAny(@Nullable Object b0,
       @Nullable Object b1) {
     if (b0 == null || b1 == null) {
@@ -2843,24 +2851,28 @@ public static int plus(int b0, int b1) {
     throw notArithmetic("+", b0, b1);
   }
 
+  @Contract("!null, !null -> !null")
   public static @Nullable UByte plus(@Nullable UByte b0, @Nullable UByte b1) {
     return (b0 == null || b1 == null)
         ? castNonNull(null)
         : b0.add(b1);
   }
 
+  @Contract("!null, !null -> !null")
   public static @Nullable UShort plus(@Nullable UShort b0, @Nullable UShort b1) {
     return (b0 == null || b1 == null)
         ? castNonNull(null)
         : b0.add(b1);
   }
 
+  @Contract("!null, !null -> !null")
   public static @Nullable UInteger plus(@Nullable UInteger b0, @Nullable UInteger b1) {
     return (b0 == null || b1 == null)
         ? castNonNull(null)
         : b0.add(b1);
   }
 
+  @Contract("!null, !null -> !null")
   public static @Nullable ULong plus(@Nullable ULong b0, @Nullable ULong b1) {
     return (b0 == null || b1 == null)
         ? castNonNull(null)
@@ -2926,22 +2938,26 @@ public static int minus(int b0, int b1) {
 
   /** SQL - operator applied to int values; left side may be
    * null. */
+  @Contract("!null, _ -> !null")
   public static @Nullable Integer minus(@Nullable Integer b0, int b1) {
     return b0 == null ? castNonNull(null) : (b0 - b1);
   }
 
   /** SQL - operator applied to int values; right side may be
    * null. */
+  @Contract("_, !null -> !null")
   public static @Nullable Integer minus(int b0, @Nullable Integer b1) {
     return b1 == null ? castNonNull(null) : (b0 - b1);
   }
 
   /** SQL - operator applied to nullable int values. */
+  @Contract("!null, !null -> !null")
   public static @Nullable Integer minus(@Nullable Integer b0, @Nullable Integer b1) {
     return (b0 == null || b1 == null) ? castNonNull(null) : (b0 - b1);
   }
 
   /** SQL - operator applied to nullable long and int values. */
+  @Contract("!null, !null -> !null")
   public static @Nullable Long minus(@Nullable Long b0, @Nullable Integer b1) {
     return (b0 == null || b1 == null)
         ? castNonNull(null)
@@ -2949,6 +2965,7 @@ public static int minus(int b0, int b1) {
   }
 
   /** SQL - operator applied to nullable int and long values. */
+  @Contract("!null, !null -> !null")
   public static @Nullable Long minus(@Nullable Integer b0, @Nullable Long b1) {
     return (b0 == null || b1 == null)
         ? castNonNull(null)
@@ -2956,6 +2973,7 @@ public static int minus(int b0, int b1) {
   }
 
   /** SQL - operator applied to nullable long and long values. */
+  @Contract("!null, !null -> !null")
   public static @Nullable Long minus(@Nullable Long b0, @Nullable Long b1) {
     return (b0 == null || b1 == null)
         ? castNonNull(null)
@@ -2963,6 +2981,7 @@ public static int minus(int b0, int b1) {
   }
 
   /** SQL - operator applied to nullable BigDecimal values. */
+  @Contract("!null, !null -> !null")
   public static @Nullable BigDecimal minus(@Nullable BigDecimal b0,
       @Nullable BigDecimal b1) {
     return (b0 == null || b1 == null) ? castNonNull(null) : b0.subtract(b1);
@@ -2970,6 +2989,7 @@ public static int minus(int b0, int b1) {
 
   /** SQL - operator applied to Object values (at least one operand
    * has ANY type; either may be null). */
+  @Contract("!null, !null -> !null")
   public static @Nullable Object minusAny(@Nullable Object b0, @Nullable Object b1) {
     if (b0 == null || b1 == null) {
       return castNonNull(null);
@@ -2982,19 +3002,23 @@ public static int minus(int b0, int b1) {
     throw notArithmetic("-", b0, b1);
   }
 
+  @Contract("!null, !null -> !null")
   public static @Nullable UByte minus(@Nullable UByte b0, @Nullable UByte b1) {
     return (b0 == null || b1 == null) ? castNonNull(null) : b0.subtract(b1);
   }
 
+  @Contract("!null, !null -> !null")
   public static @Nullable UShort minus(@Nullable UShort b0, @Nullable UShort b1) {
     return (b0 == null || b1 == null) ? castNonNull(null) : b0.subtract(b1);
   }
 
+  @Contract("!null, !null -> !null")
   public static @Nullable UInteger minus(@Nullable UInteger b0, @Nullable UInteger b1) {
     return (b0 == null || b1 == null) ? castNonNull(null) : b0.subtract(b1);
   }
 
   /** SQL - operator applied to nullable unsigned long and long values. */
+  @Contract("!null, !null -> !null")
   public static @Nullable ULong minus(@Nullable ULong b0, @Nullable ULong b1) {
     return (b0 == null || b1 == null)
         ? castNonNull(null)
@@ -3076,23 +3100,27 @@ public static int divide(int b0, int b1) {
 
   /** SQL / operator applied to int values; left side may be
    * null. */
+  @Contract("!null, _ -> !null")
   public static @Nullable Integer divide(@Nullable Integer b0, int b1) {
     return b0 == null ? castNonNull(null) : (b0 / b1);
   }
 
   /** SQL / operator applied to int values; right side may be
    * null. */
+  @Contract("_, !null -> !null")
   public static @Nullable Integer divide(int b0, @Nullable Integer b1) {
     return b1 == null ? castNonNull(null) : (b0 / b1);
   }
 
   /** SQL / operator applied to nullable int values. */
+  @Contract("!null, !null -> !null")
   public static @Nullable Integer divide(@Nullable Integer b0,
       @Nullable Integer b1) {
     return (b0 == null || b1 == null) ? castNonNull(null) : (b0 / b1);
   }
 
   /** SQL / operator applied to nullable long and int values. */
+  @Contract("_, !null -> !null")
   public static @Nullable Long divide(Long b0, @Nullable Integer b1) {
     return (b0 == null || b1 == null)
         ? castNonNull(null)
@@ -3100,6 +3128,7 @@ public static int divide(int b0, int b1) {
   }
 
   /** SQL / operator applied to nullable int and long values. */
+  @Contract("!null, !null -> !null")
   public static @Nullable Long divide(@Nullable Integer b0, @Nullable Long b1) {
     return (b0 == null || b1 == null)
         ? castNonNull(null)
@@ -3107,6 +3136,7 @@ public static int divide(int b0, int b1) {
   }
 
   /** SQL / operator applied to BigDecimal values. */
+  @Contract("!null, !null -> !null")
   public static @Nullable BigDecimal divide(@Nullable BigDecimal b0,
       @Nullable BigDecimal b1) {
     return (b0 == null || b1 == null)
@@ -3116,6 +3146,7 @@ public static int divide(int b0, int b1) {
 
   /** SQL / operator applied to Object values (at least one operand
    * has ANY type; either may be null). */
+  @Contract("!null, !null -> !null")
   public static @Nullable Object divideAny(@Nullable Object b0,
       @Nullable Object b1) {
     if (b0 == null || b1 == null) {
@@ -3139,24 +3170,28 @@ public static long divide(long b0, BigDecimal b1) {
         .divide(b1, RoundingMode.HALF_DOWN).longValue();
   }
 
+  @Contract("!null, !null -> !null")
   public static @Nullable UByte divide(@Nullable UByte b0,
       @Nullable UByte b1) {
     return (b0 == null || b1 == null) ? castNonNull(null)
         : UByte.valueOf(b0.intValue() / b1.intValue());
   }
 
+  @Contract("!null, !null -> !null")
   public static @Nullable UShort divide(@Nullable UShort b0,
       @Nullable UShort b1) {
     return (b0 == null || b1 == null) ? castNonNull(null)
         : UShort.valueOf(b0.intValue() / b1.intValue());
   }
 
+  @Contract("!null, !null -> !null")
   public static @Nullable UInteger divide(@Nullable UInteger b0,
       @Nullable UInteger b1) {
     return (b0 == null || b1 == null) ? castNonNull(null)
         : UInteger.valueOf(b0.longValue() / b1.longValue());
   }
 
+  @Contract("!null, !null -> !null")
   public static @Nullable ULong divide(@Nullable ULong b0,
       @Nullable ULong b1) {
     return (b0 == null || b1 == null) ? castNonNull(null)
@@ -3230,40 +3265,47 @@ public static int multiply(int b0, int b1) {
 
   /** SQL * operator applied to int values; left side may be
    * null. */
+  @Contract("!null, _ -> !null")
   public static @Nullable Integer multiply(@Nullable Integer b0, int b1) {
     return b0 == null ? castNonNull(null) : (b0 * b1);
   }
 
   /** SQL * operator applied to int values; right side may be
    * null. */
+  @Contract("_, !null -> !null")
   public static @Nullable Integer multiply(int b0, @Nullable Integer b1) {
     return b1 == null ? castNonNull(null) : (b0 * b1);
   }
 
   /** SQL * operator applied to nullable int values. */
+  @Contract("!null, !null -> !null")
   public static @Nullable Integer multiply(@Nullable Integer b0,
       @Nullable Integer b1) {
     return (b0 == null || b1 == null) ? castNonNull(null) : (b0 * b1);
   }
 
+  @Contract("!null, !null -> !null")
   public static @Nullable UByte multiply(@Nullable UByte b0,
       @Nullable UByte b1) {
     return (b0 == null || b1 == null) ? castNonNull(null)
         : UByte.valueOf(b0.longValue() * b1.longValue());
   }
 
+  @Contract("!null, !null -> !null")
   public static @Nullable UShort multiply(@Nullable UShort b0,
       @Nullable UShort b1) {
     return (b0 == null || b1 == null) ? castNonNull(null)
         : UShort.valueOf(b0.intValue() * b1.intValue());
   }
 
+  @Contract("!null, !null -> !null")
   public static @Nullable UInteger multiply(@Nullable UInteger b0,
       @Nullable UInteger b1) {
     return (b0 == null || b1 == null) ? castNonNull(null)
         : UInteger.valueOf(b0.longValue() * b1.longValue());
   }
 
+  @Contract("!null, !null -> !null")
   public static @Nullable ULong multiply(@Nullable ULong b0,
       @Nullable ULong b1) {
     if (b0 == null || b1 == null) {
@@ -3274,6 +3316,7 @@ public static int multiply(int b0, int b1) {
   }
 
   /** SQL * operator applied to nullable long and int values. */
+  @Contract("!null, !null -> !null")
   public static @Nullable Long multiply(@Nullable Long b0, @Nullable Integer b1) {
     return (b0 == null || b1 == null)
         ? castNonNull(null)
@@ -3281,6 +3324,7 @@ public static int multiply(int b0, int b1) {
   }
 
   /** SQL * operator applied to nullable int and long values. */
+  @Contract("!null, !null -> !null")
   public static @Nullable Long multiply(@Nullable Integer b0, @Nullable Long b1) {
     return (b0 == null || b1 == null)
         ? castNonNull(null)
@@ -3288,6 +3332,7 @@ public static int multiply(int b0, int b1) {
   }
 
   /** SQL * operator applied to nullable BigDecimal values. */
+  @Contract("!null, !null -> !null")
   public static @Nullable BigDecimal multiply(@Nullable BigDecimal b0,
       @Nullable BigDecimal b1) {
     return (b0 == null || b1 == null) ? castNonNull(null) : b0.multiply(b1);
@@ -3295,6 +3340,7 @@ public static int multiply(int b0, int b1) {
 
   /** SQL * operator applied to Object values (at least one operand
    * has ANY type; either may be null). */
+  @Contract("!null, !null -> !null")
   public static @Nullable Object multiplyAny(@Nullable Object b0,
       @Nullable Object b1) {
     if (b0 == null || b1 == null) {
@@ -5381,6 +5427,7 @@ public static int toInt(java.sql.Date v, TimeZone timeZone) {
    * @see #toInt(java.sql.Date, TimeZone)
    * @see #internalToDate(Integer) converse method
    */
+  @Contract("!null -> !null")
   public static @Nullable Integer toIntOptional(java.sql.@Nullable Date v) {
     return v == null
         ? castNonNull(null)
@@ -5394,6 +5441,7 @@ public static int toInt(java.sql.Date v, TimeZone timeZone) {
    *
    * @see #toInt(java.sql.Date, TimeZone)
    */
+  @Contract("!null, _ -> !null")
   public static @Nullable Integer toIntOptional(java.sql.@Nullable Date v,
       TimeZone timeZone) {
     return v == null
@@ -5422,6 +5470,7 @@ public static int toInt(java.sql.Time v) {
    * @see #toInt(java.sql.Time)
    * @see #internalToTime(Integer) converse method
    */
+  @Contract("!null -> !null")
   public static @Nullable Integer toIntOptional(java.sql.@Nullable Time v) {
     return v == null ? castNonNull(null) : toInt(v);
   }
@@ -5443,6 +5492,7 @@ public static int toInt(Object o) {
         : (Integer) cannotConvert(o, int.class);
   }
 
+  @Contract("!null -> !null")
   public static @Nullable Integer toIntOptional(@Nullable Object o) {
     return o == null ? castNonNull(null) : toInt(o);
   }
@@ -5508,6 +5558,7 @@ public static long toLong(Timestamp v, TimeZone timeZone) {
    * @see #toLong(Timestamp, TimeZone)
    * @see #internalToTimestamp(Long) converse method
    */
+  @Contract("!null -> !null")
   public static @Nullable Long toLongOptional(@Nullable Timestamp v) {
     return v == null ? castNonNull(null) : toLong(v, LOCAL_TZ);
   }
@@ -5519,6 +5570,7 @@ public static long toLong(Timestamp v, TimeZone timeZone) {
    *
    * @see #toLong(Timestamp, TimeZone)
    */
+  @Contract("!null, _ -> !null")
   public static @Nullable Long toLongOptional(@Nullable Timestamp v,
       TimeZone timeZone) {
     if (v == null) {
@@ -5549,6 +5601,7 @@ public static long toLong(Object o) {
         : (Long) cannotConvert(o, long.class);
   }
 
+  @Contract("!null -> !null")
   public static @Nullable Long toLongOptional(@Nullable Object o) {
     return o == null ? castNonNull(null) : toLong(o);
   }
@@ -5646,6 +5699,7 @@ public static java.sql.Date internalToDate(int v) {
    * @see #internalToDate(int)
    * @see #toIntOptional(java.sql.Date) converse method
    */
+  @Contract("!null -> !null")
   public static java.sql.@Nullable Date internalToDate(@Nullable Integer v) {
     return v == null ? castNonNull(null) : internalToDate(v.intValue());
   }
@@ -5670,10 +5724,12 @@ public static java.sql.Time internalToTime(int v) {
    * @see #internalToTime(Integer)
    * @see #toIntOptional(java.sql.Time) converse method
    */
+  @Contract("!null -> !null")
   public static java.sql.@Nullable Time internalToTime(@Nullable Integer v) {
     return v == null ? castNonNull(null) : internalToTime(v.intValue());
   }
 
+  @Contract("!null -> !null")
   public static @Nullable Integer toTimeWithLocalTimeZone(@Nullable String v) {
     if (v == null) {
       return castNonNull(null);
@@ -5684,6 +5740,7 @@ public static java.sql.Time internalToTime(int v) {
         .getMillisOfDay();
   }
 
+  @Contract("!null, _ -> !null")
   public static @Nullable Integer toTimeWithLocalTimeZone(@Nullable String v,
       TimeZone timeZone) {
     if (v == null) {
@@ -6049,6 +6106,7 @@ public static java.sql.Timestamp internalToTimestamp(long v) {
    * @see #toLongOptional(Timestamp, TimeZone)
    * @see #toLongOptional(Timestamp) converse method
    */
+  @Contract("!null -> !null")
   public static java.sql.@Nullable Timestamp internalToTimestamp(@Nullable Long v) {
     return v == null ? castNonNull(null) : internalToTimestamp(v.longValue());
   }
@@ -6332,6 +6390,7 @@ public static int time(long timestampMillis, String timeZone) {
         / (1000L * 1000L)); // milli > micro > nano
   }
 
+  @Contract("!null -> !null")
   public static @Nullable Long toTimestampWithLocalTimeZone(@Nullable String v) {
     if (v == null) {
       return castNonNull(null);
@@ -6342,6 +6401,7 @@ public static int time(long timestampMillis, String timeZone) {
         .getMillisSinceEpoch();
   }
 
+  @Contract("!null, _ -> !null")
   public static @Nullable Long toTimestampWithLocalTimeZone(@Nullable String v,
       TimeZone timeZone) {
     if (v == null) {
@@ -6356,6 +6416,7 @@ public static int time(long timestampMillis, String timeZone) {
   // Don't need shortValueOf etc. - Short.valueOf is sufficient.
 
   /** Helper for CAST(... AS VARCHAR(maxLength)). */
+  @Contract("!null, _ -> !null")
   public static @Nullable String truncate(@Nullable String s, int maxLength) {
     if (s == null) {
       return s;
@@ -6367,6 +6428,7 @@ public static int time(long timestampMillis, String timeZone) {
   }
 
   /** Helper for CAST(... AS CHAR(maxLength)). */
+  @Contract("!null, _ -> !null")
   public static @Nullable String truncateOrPad(@Nullable String s, int maxLength) {
     if (s == null) {
       return s;
@@ -6380,6 +6442,7 @@ public static int time(long timestampMillis, String timeZone) {
     }
   }
 
+  @Contract("!null, _ -> !null")
   public static @Nullable ByteString stringToBinary(@Nullable String s, Charset charset) {
     if (s == null) {
       return null;
@@ -6388,6 +6451,7 @@ public static int time(long timestampMillis, String timeZone) {
     }
   }
 
+  @Contract("!null -> !null")
   public static @Nullable ByteString byteArrayToByteString(byte @Nullable [] bytes) {
     if (bytes == null) {
       return null;
@@ -6405,6 +6469,7 @@ public static int time(long timestampMillis, String timeZone) {
   }
 
   /** Helper for CAST(... AS VARBINARY(maxLength)). */
+  @Contract("!null, _ -> !null")
   public static @Nullable ByteString truncate(@Nullable ByteString s, int maxLength) {
     if (s == null) {
       return s;
@@ -6416,6 +6481,7 @@ public static int time(long timestampMillis, String timeZone) {
   }
 
   /** Helper for CAST(... AS BINARY(maxLength)). */
+  @Contract("!null, _ -> !null")
   public static @Nullable ByteString truncateOrPad(@Nullable ByteString s, int maxLength) {
     if (s == null) {
       return s;
@@ -7030,11 +7096,13 @@ public static boolean isNotFalse(@Nullable Boolean b) {
   }
 
   /** NULL → NULL, FALSE → TRUE, TRUE → FALSE. */
+  @Contract("!null -> !null")
   public static @Nullable Boolean not(@Nullable Boolean b) {
     return b == null ? castNonNull(null) : !b;
   }
 
   /** Converts a JDBC array to a list. */
+  @Contract("!null -> !null")
   public static @Nullable List arrayToList(final java.sql.@Nullable Array a) {
     if (a == null) {
       return castNonNull(null);
diff --git a/core/src/main/java/org/apache/calcite/sql/SqlUtil.java b/core/src/main/java/org/apache/calcite/sql/SqlUtil.java
index 48049342ace6..74daa93de52a 100644
--- a/core/src/main/java/org/apache/calcite/sql/SqlUtil.java
+++ b/core/src/main/java/org/apache/calcite/sql/SqlUtil.java
@@ -19,6 +19,7 @@
 import org.apache.calcite.avatica.util.ByteString;
 import org.apache.calcite.config.CalciteSystemProperty;
 import org.apache.calcite.linq4j.Ord;
+import org.apache.calcite.linq4j.annotations.Contract;
 import org.apache.calcite.linq4j.function.Functions;
 import org.apache.calcite.rel.RelNode;
 import org.apache.calcite.rel.hint.HintStrategyTable;
@@ -1130,6 +1131,7 @@ public static void validateCharset(ByteString value, Charset charset) {
 
   /** If a node is "AS", returns the underlying expression; otherwise returns
    * the node. Returns null if and only if the node is null. */
+  @Contract("!null -> !null")
   public static @Nullable SqlNode stripAs(@Nullable SqlNode node) {
     if (node != null && node.getKind() == SqlKind.AS) {
       return ((SqlCall) node).operand(0);
diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java
index 94fd2f0de89a..efa22c798539 100644
--- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java
+++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java
@@ -17,6 +17,7 @@
 package org.apache.calcite.sql.validate;
 
 import org.apache.calcite.linq4j.Ord;
+import org.apache.calcite.linq4j.annotations.Contract;
 import org.apache.calcite.linq4j.annotations.RequiresNonNull;
 import org.apache.calcite.linq4j.function.Functions;
 import org.apache.calcite.plan.RelOptTable;
@@ -1813,6 +1814,7 @@ && requireNonNull(((SqlNumericLiteral) node).bigDecimalValue())
    * @param underFrom whether node appears directly under a FROM clause
    * @return rewritten expression, or null if the original expression is null
    */
+  @Contract("!null, _ -> !null")
   protected @Nullable SqlNode performUnconditionalRewrites(
       @Nullable SqlNode node,
       boolean underFrom) {
diff --git a/core/src/main/java/org/apache/calcite/util/NumberUtil.java b/core/src/main/java/org/apache/calcite/util/NumberUtil.java
index 5885bfd5c24e..4ced1cbf04bd 100644
--- a/core/src/main/java/org/apache/calcite/util/NumberUtil.java
+++ b/core/src/main/java/org/apache/calcite/util/NumberUtil.java
@@ -16,6 +16,8 @@
  */
 package org.apache.calcite.util;
 
+import org.apache.calcite.linq4j.annotations.Contract;
+
 import org.jspecify.annotations.Nullable;
 
 import java.math.BigDecimal;
@@ -92,6 +94,7 @@ public static BigInteger getMinUnscaled(int precision) {
 
   /** Sets the scale of a BigDecimal {@code bd} if it is not null;
    * always returns {@code bd}. */
+  @Contract("!null, _ -> !null")
   public static @Nullable BigDecimal rescaleBigDecimal(@Nullable BigDecimal bd,
       int scale) {
     if (bd != null) {
@@ -107,6 +110,7 @@ public static BigDecimal toBigDecimal(Number number, int scale) {
 
   /** Converts a number to a BigDecimal with the same value;
    * returns null if and only if the number is null. */
+  @Contract("!null -> !null")
   public static @Nullable BigDecimal toBigDecimal(@Nullable Number number) {
     if (number == null) {
       return castNonNull(null);
@@ -145,6 +149,7 @@ public static long round(double d) {
   }
 
   /** Returns the sum of two numbers, or null if either is null. */
+  @Contract("!null, !null -> !null")
   public static @Nullable Double add(@Nullable Double a, @Nullable Double b) {
     if (a == null || b == null) {
       return null;
@@ -155,6 +160,7 @@ public static long round(double d) {
 
   /** Returns the difference of two numbers,
    * or null if either is null. */
+  @Contract("!null, !null -> !null")
   public static @Nullable Double subtract(@Nullable Double a, @Nullable Double b) {
     if (a == null || b == null) {
       return castNonNull(null);
@@ -175,6 +181,7 @@ public static long round(double d) {
 
   /** Returns the product of two numbers,
    * or null if either is null. */
+  @Contract("!null, !null -> !null")
   public static @Nullable Double multiply(@Nullable Double a, @Nullable Double b) {
     if (a == null || b == null) {
       return castNonNull(null);
@@ -187,6 +194,7 @@ public static long round(double d) {
    * returns the lesser of two numbers,
    * ignoring numbers that are null,
    * or null if both are null. */
+  @Contract("!null, !null -> !null")
   public static @Nullable Double min(@Nullable Double a, @Nullable Double b) {
     if (a == null) {
       return b;
@@ -200,6 +208,7 @@ public static long round(double d) {
   /** Like {@link Math#max} but null safe;
    * returns the greater of two numbers,
    * or null if either is null. */
+  @Contract("!null, !null -> !null")
   public static @Nullable Double max(@Nullable Double a, @Nullable Double b) {
     if (a == null || b == null) {
       return castNonNull(null);
diff --git a/core/src/main/java/org/apache/calcite/util/SerializableCharset.java b/core/src/main/java/org/apache/calcite/util/SerializableCharset.java
index 000c20b8922f..262c43d7822a 100644
--- a/core/src/main/java/org/apache/calcite/util/SerializableCharset.java
+++ b/core/src/main/java/org/apache/calcite/util/SerializableCharset.java
@@ -16,6 +16,8 @@
  */
 package org.apache.calcite.util;
 
+import org.apache.calcite.linq4j.annotations.Contract;
+
 import org.jspecify.annotations.Nullable;
 
 import java.io.IOException;
@@ -91,6 +93,7 @@ public Charset getCharset() {
    * @param charset Character set to wrap, or null
    * @return Wrapped charset
    */
+  @Contract("!null -> !null")
   public static @Nullable SerializableCharset forCharset(@Nullable Charset charset) {
     if (charset == null) {
       return null;
diff --git a/core/src/main/java/org/apache/calcite/util/Util.java b/core/src/main/java/org/apache/calcite/util/Util.java
index 9446c8d2fd25..bd8735b0a299 100644
--- a/core/src/main/java/org/apache/calcite/util/Util.java
+++ b/core/src/main/java/org/apache/calcite/util/Util.java
@@ -20,6 +20,7 @@
 import org.apache.calcite.avatica.util.Spaces;
 import org.apache.calcite.config.CalciteSystemProperty;
 import org.apache.calcite.linq4j.Ord;
+import org.apache.calcite.linq4j.annotations.Contract;
 import org.apache.calcite.runtime.CalciteException;
 import org.apache.calcite.sql.SqlAggFunction;
 import org.apache.calcite.sql.SqlCall;
@@ -682,6 +683,7 @@ public static boolean isWindows() {
    * characters found in {@code search} are replaced by the character in the same position in
    * {@code replacement}; if {@code replacement} is shorter, remaining matches are removed.
    */
+  @Contract("!null, _, _ -> !null")
   public static @Nullable String replaceChars(@Nullable String s, @Nullable String search,
       @Nullable String replacement) {
     if (s == null || s.isEmpty() || search == null || search.isEmpty()) {
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/DefaultEnumerable.java b/linq4j/src/main/java/org/apache/calcite/linq4j/DefaultEnumerable.java
index eccd6ab38e96..6854641ea159 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/DefaultEnumerable.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/DefaultEnumerable.java
@@ -16,6 +16,7 @@
  */
 package org.apache.calcite.linq4j;
 
+import org.apache.calcite.linq4j.annotations.Contract;
 import org.apache.calcite.linq4j.function.BigDecimalFunction1;
 import org.apache.calcite.linq4j.function.DoubleFunction1;
 import org.apache.calcite.linq4j.function.EqualityComparer;
@@ -96,6 +97,7 @@ protected OrderedQueryable asOrderedQueryable() {
     return EnumerableDefaults.aggregate(getThis(), func);
   }
 
+  @Contract("!null, !null -> !null")
   @Override public  @Nullable TAccumulate aggregate(@Nullable TAccumulate seed,
       Function2<@Nullable TAccumulate, T, @Nullable TAccumulate> func) {
     return EnumerableDefaults.aggregate(getThis(), seed, func);
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedEnumerable.java b/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedEnumerable.java
index c4dde33c2fc0..8136f645ec7a 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedEnumerable.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedEnumerable.java
@@ -16,6 +16,7 @@
  */
 package org.apache.calcite.linq4j;
 
+import org.apache.calcite.linq4j.annotations.Contract;
 import org.apache.calcite.linq4j.function.BigDecimalFunction1;
 import org.apache.calcite.linq4j.function.DoubleFunction1;
 import org.apache.calcite.linq4j.function.EqualityComparer;
@@ -73,6 +74,7 @@ public interface ExtendedEnumerable {
    *
    * 
      If {@code seed} is not null, the result is never null.
    */
+  @Contract("!null, !null -> !null")
    @Nullable TAccumulate aggregate(@Nullable TAccumulate seed,
       Function2<@Nullable TAccumulate, TSource, @Nullable TAccumulate> func);
 
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/BlockBuilder.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/BlockBuilder.java
index 525922e8a2b2..11370c31892e 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/BlockBuilder.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/BlockBuilder.java
@@ -16,6 +16,8 @@
  */
 package org.apache.calcite.linq4j.tree;
 
+import org.apache.calcite.linq4j.annotations.Contract;
+
 import org.jspecify.annotations.Nullable;
 
 import java.lang.reflect.Modifier;
@@ -200,6 +202,7 @@ public Expression append(String name, Expression expression) {
    * Appends an expression to a list of statements if it is not null,
    * and returns the expression.
    */
+  @Contract("_, !null -> !null")
   public @Nullable Expression appendIfNotNull(String name, @Nullable Expression expression) {
     if (expression == null) {
       return null;

From a29ea129848f182e1b47a79a4df78332b046b35c Mon Sep 17 00:00:00 2001
From: Vladimir Sitnikov 
Date: Sun, 23 Aug 2026 22:11:07 +0300
Subject: [PATCH 499/562] [CALCITE-7736] Give type parameters the nullable
 bounds the Checker Framework inferred

The two tools default an unwritten type parameter bound in opposite
directions. The Checker Framework's CLIMB-to-top rule gives implicit bounds the
top qualifier, so `` there means ``. JSpecify
fills in `Object`, which under `@NullMarked` is non-null, and its user guide
says as much: "`` means `` and that means it is not
`@Nullable`".

So every unbounded type parameter silently changed meaning. Calcite relied on
the Checker Framework reading, for instance here:

    public interface SqlVisitor { ... }
    public class SqlShuttle extends SqlBasicVisitor<@Nullable SqlNode> { ... }
    public abstract  R accept(SqlVisitor visitor);

`SqlShuttle` is a `SqlVisitor<@Nullable SqlNode>` and was passed to `accept`
with no suppression, which only typechecks if R admits a nullable argument.

Writes the bound out at 61 declarations: the Rex and Sql visitors and the 23
`accept` overrides, `Pair` and its factories, `PairList`, `ConsList`,
`FlatLists`, `Holder`, `ImmutableNullableList`, `TryThreadLocal`,
`Util.transform`, and the linq4j types `Enumerable`, `Enumerator`, `Queryable`,
`Function0`, `Function1`, `Function2` and `Ord`.

The erasure is unchanged, so this is binary compatible.

This accounts for most of what NullAway reported: 1126 errors down to 576 in
`calcite-core`. `Pair.of` alone was worth 132 -- its class already had the
bounds, but a static factory declares type parameters of its own.

Co-Authored-By: Claude Opus 5 
---
 .../org/apache/calcite/adapter/clone/ArrayTable.java |  2 +-
 .../apache/calcite/config/CalciteSystemProperty.java |  2 +-
 .../apache/calcite/plan/RexImplicationChecker.java   |  2 +-
 .../java/org/apache/calcite/rex/RexBiVisitor.java    |  4 +++-
 .../main/java/org/apache/calcite/rex/RexCall.java    |  2 +-
 .../org/apache/calcite/rex/RexCorrelVariable.java    |  2 +-
 .../java/org/apache/calcite/rex/RexDynamicParam.java |  2 +-
 .../java/org/apache/calcite/rex/RexFieldAccess.java  |  2 +-
 .../java/org/apache/calcite/rex/RexInputRef.java     |  2 +-
 .../main/java/org/apache/calcite/rex/RexLambda.java  |  2 +-
 .../java/org/apache/calcite/rex/RexLambdaRef.java    |  2 +-
 .../main/java/org/apache/calcite/rex/RexLiteral.java |  2 +-
 .../java/org/apache/calcite/rex/RexLocalRef.java     |  2 +-
 .../main/java/org/apache/calcite/rex/RexNode.java    |  2 +-
 .../org/apache/calcite/rex/RexNodeAndFieldIndex.java |  2 +-
 .../main/java/org/apache/calcite/rex/RexOver.java    |  2 +-
 .../org/apache/calcite/rex/RexPatternFieldRef.java   |  4 +++-
 .../java/org/apache/calcite/rex/RexRangeRef.java     |  2 +-
 .../java/org/apache/calcite/rex/RexSimplify.java     |  2 +-
 .../java/org/apache/calcite/rex/RexSubQuery.java     |  2 +-
 .../org/apache/calcite/rex/RexTableInputRef.java     |  2 +-
 .../main/java/org/apache/calcite/rex/RexVisitor.java |  4 +++-
 .../java/org/apache/calcite/runtime/ConsList.java    |  2 +-
 .../java/org/apache/calcite/runtime/FlatLists.java   | 12 ++++++------
 .../java/org/apache/calcite/runtime/PairList.java    |  2 +-
 .../apache/calcite/runtime/ResultSetEnumerable.java  |  2 +-
 .../main/java/org/apache/calcite/sql/SqlCall.java    |  2 +-
 .../java/org/apache/calcite/sql/SqlDataTypeSpec.java |  2 +-
 .../java/org/apache/calcite/sql/SqlDynamicParam.java |  2 +-
 .../java/org/apache/calcite/sql/SqlIdentifier.java   |  2 +-
 .../org/apache/calcite/sql/SqlIntervalQualifier.java |  2 +-
 .../main/java/org/apache/calcite/sql/SqlLiteral.java |  2 +-
 .../main/java/org/apache/calcite/sql/SqlNode.java    |  2 +-
 .../java/org/apache/calcite/sql/SqlNodeList.java     |  2 +-
 .../java/org/apache/calcite/sql/util/SqlVisitor.java |  4 +++-
 .../main/java/org/apache/calcite/util/Holder.java    |  2 +-
 .../apache/calcite/util/ImmutableNullableList.java   |  2 +-
 core/src/main/java/org/apache/calcite/util/Pair.java | 12 ++++++------
 .../java/org/apache/calcite/util/TryThreadLocal.java |  2 +-
 core/src/main/java/org/apache/calcite/util/Util.java | 10 +++++-----
 .../java/org/apache/calcite/linq4j/Enumerable.java   |  4 +++-
 .../apache/calcite/linq4j/EnumerableDefaults.java    |  2 +-
 .../java/org/apache/calcite/linq4j/Enumerator.java   |  4 +++-
 .../src/main/java/org/apache/calcite/linq4j/Ord.java |  2 +-
 .../java/org/apache/calcite/linq4j/Queryable.java    |  4 +++-
 .../apache/calcite/linq4j/function/Function0.java    |  4 +++-
 .../apache/calcite/linq4j/function/Function1.java    |  4 +++-
 .../apache/calcite/linq4j/function/Function2.java    |  6 ++++--
 48 files changed, 83 insertions(+), 63 deletions(-)

diff --git a/core/src/main/java/org/apache/calcite/adapter/clone/ArrayTable.java b/core/src/main/java/org/apache/calcite/adapter/clone/ArrayTable.java
index a6e5820787ab..5b36539fd028 100644
--- a/core/src/main/java/org/apache/calcite/adapter/clone/ArrayTable.java
+++ b/core/src/main/java/org/apache/calcite/adapter/clone/ArrayTable.java
@@ -798,7 +798,7 @@ public static void orLong(
     }
   }
 
-  private static  List permuteList(
+  private static  List permuteList(
       final List list, final int @Nullable [] sources) {
     if (sources == null) {
       return list;
diff --git a/core/src/main/java/org/apache/calcite/config/CalciteSystemProperty.java b/core/src/main/java/org/apache/calcite/config/CalciteSystemProperty.java
index a5ef2b7b4f2f..978379ab0d71 100644
--- a/core/src/main/java/org/apache/calcite/config/CalciteSystemProperty.java
+++ b/core/src/main/java/org/apache/calcite/config/CalciteSystemProperty.java
@@ -43,7 +43,7 @@
  *
  * @param  the type of the property value
  */
-public final class CalciteSystemProperty {
+public final class CalciteSystemProperty {
   /**
    * Holds all system properties related with the Calcite.
    *
diff --git a/core/src/main/java/org/apache/calcite/plan/RexImplicationChecker.java b/core/src/main/java/org/apache/calcite/plan/RexImplicationChecker.java
index 22ed8943ff85..36dd368ec01e 100644
--- a/core/src/main/java/org/apache/calcite/plan/RexImplicationChecker.java
+++ b/core/src/main/java/org/apache/calcite/plan/RexImplicationChecker.java
@@ -523,7 +523,7 @@ private void updateUsage(SqlOperator op, RexInputRef inputRef,
    *
    * @param  left type
    * @param  right type */
-  private static class InputRefUsage {
+  private static class InputRefUsage {
     private final PairList usageList = PairList.of();
     private int usageCount = 0;
   }
diff --git a/core/src/main/java/org/apache/calcite/rex/RexBiVisitor.java b/core/src/main/java/org/apache/calcite/rex/RexBiVisitor.java
index 995b2e97271f..37dd7745c73b 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexBiVisitor.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexBiVisitor.java
@@ -18,6 +18,8 @@
 
 import com.google.common.collect.ImmutableList;
 
+import org.jspecify.annotations.Nullable;
+
 import java.util.ArrayList;
 import java.util.List;
 
@@ -30,7 +32,7 @@
  * @param  Return type
  * @param 
       Payload type
  */
-public interface RexBiVisitor {
+public interface RexBiVisitor {
   //~ Methods ----------------------------------------------------------------
 
   R visitInputRef(RexInputRef inputRef, P arg);
diff --git a/core/src/main/java/org/apache/calcite/rex/RexCall.java b/core/src/main/java/org/apache/calcite/rex/RexCall.java
index 3af137e0a6cd..bba28f275361 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexCall.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexCall.java
@@ -192,7 +192,7 @@ private boolean digestWithType() {
     return isA(SqlKind.CAST) || isA(SqlKind.NEW_SPECIFICATION);
   }
 
-  @Override public  R accept(RexVisitor visitor) {
+  @Override public  R accept(RexVisitor visitor) {
     return visitor.visitCall(this);
   }
 
diff --git a/core/src/main/java/org/apache/calcite/rex/RexCorrelVariable.java b/core/src/main/java/org/apache/calcite/rex/RexCorrelVariable.java
index ee576085f73b..95b5ff1294c8 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexCorrelVariable.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexCorrelVariable.java
@@ -47,7 +47,7 @@ public class RexCorrelVariable extends RexVariable {
 
   //~ Methods ----------------------------------------------------------------
 
-  @Override public  R accept(RexVisitor visitor) {
+  @Override public  R accept(RexVisitor visitor) {
     return visitor.visitCorrelVariable(this);
   }
 
diff --git a/core/src/main/java/org/apache/calcite/rex/RexDynamicParam.java b/core/src/main/java/org/apache/calcite/rex/RexDynamicParam.java
index 10269b5729f2..aa011a46494b 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexDynamicParam.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexDynamicParam.java
@@ -56,7 +56,7 @@ public int getIndex() {
     return index;
   }
 
-  @Override public  R accept(RexVisitor visitor) {
+  @Override public  R accept(RexVisitor visitor) {
     return visitor.visitDynamicParam(this);
   }
 
diff --git a/core/src/main/java/org/apache/calcite/rex/RexFieldAccess.java b/core/src/main/java/org/apache/calcite/rex/RexFieldAccess.java
index da42d57aca43..7db5629ec2a0 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexFieldAccess.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexFieldAccess.java
@@ -99,7 +99,7 @@ public RelDataTypeField getField() {
     return SqlKind.FIELD_ACCESS;
   }
 
-  @Override public  R accept(RexVisitor visitor) {
+  @Override public  R accept(RexVisitor visitor) {
     return visitor.visitFieldAccess(this);
   }
 
diff --git a/core/src/main/java/org/apache/calcite/rex/RexInputRef.java b/core/src/main/java/org/apache/calcite/rex/RexInputRef.java
index 98f50214fe35..79492da8ce91 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexInputRef.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexInputRef.java
@@ -121,7 +121,7 @@ public static void add2(PairList list,
     return SqlKind.INPUT_REF;
   }
 
-  @Override public  R accept(RexVisitor visitor) {
+  @Override public  R accept(RexVisitor visitor) {
     return visitor.visitInputRef(this);
   }
 
diff --git a/core/src/main/java/org/apache/calcite/rex/RexLambda.java b/core/src/main/java/org/apache/calcite/rex/RexLambda.java
index fae0c4184b93..31a5d7c9d850 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexLambda.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexLambda.java
@@ -55,7 +55,7 @@ public class RexLambda extends RexNode {
     return SqlKind.LAMBDA;
   }
 
-  @Override public  R accept(RexVisitor visitor) {
+  @Override public  R accept(RexVisitor visitor) {
     return visitor.visitLambda(this);
   }
 
diff --git a/core/src/main/java/org/apache/calcite/rex/RexLambdaRef.java b/core/src/main/java/org/apache/calcite/rex/RexLambdaRef.java
index 092154f91b7f..edde8abc1405 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexLambdaRef.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexLambdaRef.java
@@ -36,7 +36,7 @@ public RexLambdaRef(int index, String name, RelDataType type) {
     return SqlKind.LAMBDA_REF;
   }
 
-  @Override public  R accept(RexVisitor visitor) {
+  @Override public  R accept(RexVisitor visitor) {
     return visitor.visitLambdaRef(this);
   }
 
diff --git a/core/src/main/java/org/apache/calcite/rex/RexLiteral.java b/core/src/main/java/org/apache/calcite/rex/RexLiteral.java
index 2c7b73bae74b..fb2987de49f5 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexLiteral.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexLiteral.java
@@ -1337,7 +1337,7 @@ public static boolean isNullLiteral(RexNode node) {
         && (((RexLiteral) node).value == null);
   }
 
-  @Override public  R accept(RexVisitor visitor) {
+  @Override public  R accept(RexVisitor visitor) {
     return visitor.visitLiteral(this);
   }
 
diff --git a/core/src/main/java/org/apache/calcite/rex/RexLocalRef.java b/core/src/main/java/org/apache/calcite/rex/RexLocalRef.java
index d6a337389435..bb5e1b6d3367 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexLocalRef.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexLocalRef.java
@@ -74,7 +74,7 @@ public RexLocalRef(int index, RelDataType type) {
     return Objects.hash(type, index);
   }
 
-  @Override public  R accept(RexVisitor visitor) {
+  @Override public  R accept(RexVisitor visitor) {
     return visitor.visitLocalRef(this);
   }
 
diff --git a/core/src/main/java/org/apache/calcite/rex/RexNode.java b/core/src/main/java/org/apache/calcite/rex/RexNode.java
index 5d8e9b86fb85..3112e03fee54 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexNode.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexNode.java
@@ -107,7 +107,7 @@ public int nodeCount() {
    * 
      Also see {@link RexUtil#apply(RexVisitor, java.util.List, RexNode)},
    * which applies a visitor to several expressions simultaneously.
    */
-  public abstract  R accept(RexVisitor visitor);
+  public abstract  R accept(RexVisitor visitor);
 
   /**
    * Accepts a visitor with a payload, dispatching to the right overloaded
diff --git a/core/src/main/java/org/apache/calcite/rex/RexNodeAndFieldIndex.java b/core/src/main/java/org/apache/calcite/rex/RexNodeAndFieldIndex.java
index f84f0d8dbe8b..be03733d45e1 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexNodeAndFieldIndex.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexNodeAndFieldIndex.java
@@ -74,7 +74,7 @@ public int getFieldIndex() {
     return fieldIndex;
   }
 
-  @Override public  R accept(RexVisitor visitor) {
+  @Override public  R accept(RexVisitor visitor) {
     return visitor.visitNodeAndFieldIndex(this);
   }
 
diff --git a/core/src/main/java/org/apache/calcite/rex/RexOver.java b/core/src/main/java/org/apache/calcite/rex/RexOver.java
index 4770ae40b460..89721bc84582 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexOver.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexOver.java
@@ -152,7 +152,7 @@ public boolean ignoreNulls() {
     return sb.toString();
   }
 
-  @Override public  R accept(RexVisitor visitor) {
+  @Override public  R accept(RexVisitor visitor) {
     return visitor.visitOver(this);
   }
 
diff --git a/core/src/main/java/org/apache/calcite/rex/RexPatternFieldRef.java b/core/src/main/java/org/apache/calcite/rex/RexPatternFieldRef.java
index 560f88f19e3e..7543b63a8bfe 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexPatternFieldRef.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexPatternFieldRef.java
@@ -19,6 +19,8 @@
 import org.apache.calcite.rel.type.RelDataType;
 import org.apache.calcite.sql.SqlKind;
 
+import org.jspecify.annotations.Nullable;
+
 /**
  * Variable that references a field of an input relational expression.
  */
@@ -43,7 +45,7 @@ public static RexPatternFieldRef of(String alpha, RexInputRef ref) {
     return new RexPatternFieldRef(alpha, ref.getIndex(), ref.getType());
   }
 
-  @Override public  R accept(RexVisitor visitor) {
+  @Override public  R accept(RexVisitor visitor) {
     return visitor.visitPatternFieldRef(this);
   }
 
diff --git a/core/src/main/java/org/apache/calcite/rex/RexRangeRef.java b/core/src/main/java/org/apache/calcite/rex/RexRangeRef.java
index 8d2a06bc3bd5..845cb071d9ec 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexRangeRef.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexRangeRef.java
@@ -71,7 +71,7 @@ public int getOffset() {
     return offset;
   }
 
-  @Override public  R accept(RexVisitor visitor) {
+  @Override public  R accept(RexVisitor visitor) {
     return visitor.visitRangeRef(this);
   }
 
diff --git a/core/src/main/java/org/apache/calcite/rex/RexSimplify.java b/core/src/main/java/org/apache/calcite/rex/RexSimplify.java
index 921b068c73ea..a9cc860a01a8 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexSimplify.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexSimplify.java
@@ -3685,7 +3685,7 @@ > Sarg build(boolean negate) {
           () -> "Can't find leastRestrictive type among " + distinctTypes);
     }
 
-    @Override public  R accept(RexVisitor visitor) {
+    @Override public  R accept(RexVisitor visitor) {
       throw new UnsupportedOperationException();
     }
 
diff --git a/core/src/main/java/org/apache/calcite/rex/RexSubQuery.java b/core/src/main/java/org/apache/calcite/rex/RexSubQuery.java
index 9bf31c32bffe..02a16ad0f02d 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexSubQuery.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexSubQuery.java
@@ -160,7 +160,7 @@ public static RexSubQuery map(RelNode rel) {
         ImmutableList.of(), rel);
   }
 
-  @Override public  R accept(RexVisitor visitor) {
+  @Override public  R accept(RexVisitor visitor) {
     return visitor.visitSubQuery(this);
   }
 
diff --git a/core/src/main/java/org/apache/calcite/rex/RexTableInputRef.java b/core/src/main/java/org/apache/calcite/rex/RexTableInputRef.java
index 51adef917eca..432c5c7fc612 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexTableInputRef.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexTableInputRef.java
@@ -85,7 +85,7 @@ public static RexTableInputRef of(RelTableRef tableRef, RexInputRef ref) {
     return new RexTableInputRef(tableRef, ref.getIndex(), ref.getType());
   }
 
-  @Override public  R accept(RexVisitor visitor) {
+  @Override public  R accept(RexVisitor visitor) {
     return visitor.visitTableInputRef(this);
   }
 
diff --git a/core/src/main/java/org/apache/calcite/rex/RexVisitor.java b/core/src/main/java/org/apache/calcite/rex/RexVisitor.java
index 01a3ff920fd4..041662884aa3 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexVisitor.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexVisitor.java
@@ -18,6 +18,8 @@
 
 import com.google.common.collect.ImmutableList;
 
+import org.jspecify.annotations.Nullable;
+
 import java.util.ArrayList;
 import java.util.List;
 
@@ -30,7 +32,7 @@
  *
  * @param  Return type
  */
-public interface RexVisitor {
+public interface RexVisitor {
   //~ Methods ----------------------------------------------------------------
 
   R visitInputRef(RexInputRef inputRef);
diff --git a/core/src/main/java/org/apache/calcite/runtime/ConsList.java b/core/src/main/java/org/apache/calcite/runtime/ConsList.java
index 5d4914c49d83..856a1b20e956 100644
--- a/core/src/main/java/org/apache/calcite/runtime/ConsList.java
+++ b/core/src/main/java/org/apache/calcite/runtime/ConsList.java
@@ -33,7 +33,7 @@
  *
  * @param  Element type
  */
-public class ConsList extends AbstractImmutableList {
+public class ConsList extends AbstractImmutableList {
   private final E first;
   private final List rest;
 
diff --git a/core/src/main/java/org/apache/calcite/runtime/FlatLists.java b/core/src/main/java/org/apache/calcite/runtime/FlatLists.java
index a552342071cd..9b3a0fbddf8e 100644
--- a/core/src/main/java/org/apache/calcite/runtime/FlatLists.java
+++ b/core/src/main/java/org/apache/calcite/runtime/FlatLists.java
@@ -321,7 +321,7 @@ public abstract static class AbstractFlatList
    *
    * @param  Element type
    */
-  protected static class Flat1List
+  protected static class Flat1List
       extends AbstractFlatList
       implements ComparableList {
     private final T t0;
@@ -433,7 +433,7 @@ protected static class Flat1List
    *
    * @param  Element type
    */
-  protected static class Flat2List
+  protected static class Flat2List
       extends AbstractFlatList
       implements ComparableList {
     private final T t0;
@@ -566,7 +566,7 @@ protected static class Flat2List
    *
    * @param  Element type
    */
-  protected static class Flat3List
+  protected static class Flat3List
       extends AbstractFlatList
       implements ComparableList {
     private final T t0;
@@ -716,7 +716,7 @@ protected static class Flat3List
    *
    * @param  Element type
    */
-  protected static class Flat4List
+  protected static class Flat4List
       extends AbstractFlatList
       implements ComparableList {
     private final T t0;
@@ -886,7 +886,7 @@ protected static class Flat4List
    *
    * @param  Element type
    */
-  protected static class Flat5List
+  protected static class Flat5List
       extends AbstractFlatList
       implements ComparableList {
     private final T t0;
@@ -1075,7 +1075,7 @@ protected static class Flat5List
    *
    * @param  Element type
    */
-  protected static class Flat6List
+  protected static class Flat6List
       extends AbstractFlatList
       implements ComparableList {
     private final T t0;
diff --git a/core/src/main/java/org/apache/calcite/runtime/PairList.java b/core/src/main/java/org/apache/calcite/runtime/PairList.java
index a1d0275e3421..5199d1cf0fb7 100644
--- a/core/src/main/java/org/apache/calcite/runtime/PairList.java
+++ b/core/src/main/java/org/apache/calcite/runtime/PairList.java
@@ -39,7 +39,7 @@
  * @param  First type
  * @param  Second type
  */
-public interface PairList
+public interface PairList
     extends List> {
   /** Creates an empty PairList. */
   static  PairList of() {
diff --git a/core/src/main/java/org/apache/calcite/runtime/ResultSetEnumerable.java b/core/src/main/java/org/apache/calcite/runtime/ResultSetEnumerable.java
index 8a0a08033b11..6db07d360740 100644
--- a/core/src/main/java/org/apache/calcite/runtime/ResultSetEnumerable.java
+++ b/core/src/main/java/org/apache/calcite/runtime/ResultSetEnumerable.java
@@ -61,7 +61,7 @@
  *
  * @param  Element type
  */
-public class ResultSetEnumerable extends AbstractEnumerable {
+public class ResultSetEnumerable extends AbstractEnumerable {
   private final DataSource dataSource;
   private final String sql;
   private final Function1> rowBuilderFactory;
diff --git a/core/src/main/java/org/apache/calcite/sql/SqlCall.java b/core/src/main/java/org/apache/calcite/sql/SqlCall.java
index 7ad2ce061c49..001cef2b48e6 100755
--- a/core/src/main/java/org/apache/calcite/sql/SqlCall.java
+++ b/core/src/main/java/org/apache/calcite/sql/SqlCall.java
@@ -170,7 +170,7 @@ private boolean needsParentheses(SqlWriter writer, int leftPrec, int rightPrec)
     // no valid options
   }
 
-  @Override public  R accept(SqlVisitor visitor) {
+  @Override public  R accept(SqlVisitor visitor) {
     return visitor.visit(this);
   }
 
diff --git a/core/src/main/java/org/apache/calcite/sql/SqlDataTypeSpec.java b/core/src/main/java/org/apache/calcite/sql/SqlDataTypeSpec.java
index db1e05122d16..d425c5e36749 100644
--- a/core/src/main/java/org/apache/calcite/sql/SqlDataTypeSpec.java
+++ b/core/src/main/java/org/apache/calcite/sql/SqlDataTypeSpec.java
@@ -194,7 +194,7 @@ public SqlDataTypeSpec getComponentTypeSpec() {
     validator.validateDataType(this);
   }
 
-  @Override public  R accept(SqlVisitor visitor) {
+  @Override public  R accept(SqlVisitor visitor) {
     return visitor.visit(this);
   }
 
diff --git a/core/src/main/java/org/apache/calcite/sql/SqlDynamicParam.java b/core/src/main/java/org/apache/calcite/sql/SqlDynamicParam.java
index eb149f2ba339..a44b3e5b9472 100644
--- a/core/src/main/java/org/apache/calcite/sql/SqlDynamicParam.java
+++ b/core/src/main/java/org/apache/calcite/sql/SqlDynamicParam.java
@@ -74,7 +74,7 @@ public int getIndex() {
     return SqlMonotonicity.CONSTANT;
   }
 
-  @Override public  R accept(SqlVisitor visitor) {
+  @Override public  R accept(SqlVisitor visitor) {
     return visitor.visit(this);
   }
 
diff --git a/core/src/main/java/org/apache/calcite/sql/SqlIdentifier.java b/core/src/main/java/org/apache/calcite/sql/SqlIdentifier.java
index a55b133a4e84..f3bb728683f9 100644
--- a/core/src/main/java/org/apache/calcite/sql/SqlIdentifier.java
+++ b/core/src/main/java/org/apache/calcite/sql/SqlIdentifier.java
@@ -319,7 +319,7 @@ public SqlIdentifier skipLast(int n) {
     return litmus.succeed();
   }
 
-  @Override public  R accept(SqlVisitor visitor) {
+  @Override public  R accept(SqlVisitor visitor) {
     return visitor.visit(this);
   }
 
diff --git a/core/src/main/java/org/apache/calcite/sql/SqlIntervalQualifier.java b/core/src/main/java/org/apache/calcite/sql/SqlIntervalQualifier.java
index c07ca28b47f9..4f5805639ce8 100644
--- a/core/src/main/java/org/apache/calcite/sql/SqlIntervalQualifier.java
+++ b/core/src/main/java/org/apache/calcite/sql/SqlIntervalQualifier.java
@@ -280,7 +280,7 @@ public boolean isWeek() {
     validator.validateIntervalQualifier(this);
   }
 
-  @Override public  R accept(SqlVisitor visitor) {
+  @Override public  R accept(SqlVisitor visitor) {
     return visitor.visit(this);
   }
 
diff --git a/core/src/main/java/org/apache/calcite/sql/SqlLiteral.java b/core/src/main/java/org/apache/calcite/sql/SqlLiteral.java
index 666bb8055899..8015961a4000 100644
--- a/core/src/main/java/org/apache/calcite/sql/SqlLiteral.java
+++ b/core/src/main/java/org/apache/calcite/sql/SqlLiteral.java
@@ -592,7 +592,7 @@ public static SqlLiteral unchain(SqlNode node) {
     validator.validateLiteral(this);
   }
 
-  @Override public  R accept(SqlVisitor visitor) {
+  @Override public  R accept(SqlVisitor visitor) {
     return visitor.visit(this);
   }
 
diff --git a/core/src/main/java/org/apache/calcite/sql/SqlNode.java b/core/src/main/java/org/apache/calcite/sql/SqlNode.java
index c35806ad67cd..c34e27515e93 100644
--- a/core/src/main/java/org/apache/calcite/sql/SqlNode.java
+++ b/core/src/main/java/org/apache/calcite/sql/SqlNode.java
@@ -287,7 +287,7 @@ public void validateExpr(
    * 
      The type parameter R must be consistent with the type
    * parameter of the visitor.
    */
-  public abstract  R accept(SqlVisitor visitor);
+  public abstract  R accept(SqlVisitor visitor);
 
   /**
    * Returns whether this node is structurally equivalent to another node.
diff --git a/core/src/main/java/org/apache/calcite/sql/SqlNodeList.java b/core/src/main/java/org/apache/calcite/sql/SqlNodeList.java
index 443393a14ec2..7283f3bdc652 100644
--- a/core/src/main/java/org/apache/calcite/sql/SqlNodeList.java
+++ b/core/src/main/java/org/apache/calcite/sql/SqlNodeList.java
@@ -269,7 +269,7 @@ void andOrList(SqlWriter writer, SqlBinaryOperator sepOp) {
     }
   }
 
-  @Override public  R accept(SqlVisitor visitor) {
+  @Override public  R accept(SqlVisitor visitor) {
     return visitor.visit(this);
   }
 
diff --git a/core/src/main/java/org/apache/calcite/sql/util/SqlVisitor.java b/core/src/main/java/org/apache/calcite/sql/util/SqlVisitor.java
index 635454d14103..b4c15eee72a6 100644
--- a/core/src/main/java/org/apache/calcite/sql/util/SqlVisitor.java
+++ b/core/src/main/java/org/apache/calcite/sql/util/SqlVisitor.java
@@ -26,6 +26,8 @@
 import org.apache.calcite.sql.SqlNodeList;
 import org.apache.calcite.sql.SqlOperator;
 
+import org.jspecify.annotations.Nullable;
+
 import java.util.List;
 
 /**
@@ -42,7 +44,7 @@
  *
  * @param  Return type
  */
-public interface SqlVisitor {
+public interface SqlVisitor {
   //~ Methods ----------------------------------------------------------------
 
   /**
diff --git a/core/src/main/java/org/apache/calcite/util/Holder.java b/core/src/main/java/org/apache/calcite/util/Holder.java
index 2124562bba5c..6b923e075265 100644
--- a/core/src/main/java/org/apache/calcite/util/Holder.java
+++ b/core/src/main/java/org/apache/calcite/util/Holder.java
@@ -29,7 +29,7 @@
  *
  * @param  Element type
  */
-public class Holder {
+public class Holder {
   private E e;
 
   /** Creates a Holder containing a given value.
diff --git a/core/src/main/java/org/apache/calcite/util/ImmutableNullableList.java b/core/src/main/java/org/apache/calcite/util/ImmutableNullableList.java
index 6fe64eeba65d..26408597114f 100644
--- a/core/src/main/java/org/apache/calcite/util/ImmutableNullableList.java
+++ b/core/src/main/java/org/apache/calcite/util/ImmutableNullableList.java
@@ -207,7 +207,7 @@ public static  Builder builder() {
    *
    * @param  element type
    */
-  public static final class Builder {
+  public static final class Builder {
     private final List contents = new ArrayList<>();
 
     /**
diff --git a/core/src/main/java/org/apache/calcite/util/Pair.java b/core/src/main/java/org/apache/calcite/util/Pair.java
index c640c52d5e0c..3f1099a8c376 100644
--- a/core/src/main/java/org/apache/calcite/util/Pair.java
+++ b/core/src/main/java/org/apache/calcite/util/Pair.java
@@ -44,7 +44,7 @@
  * @param  Right-hand type
  */
 @SuppressWarnings("NullAway")
-public class Pair
+public class Pair
     implements Comparable>, Map.Entry, Serializable {
 
   @SuppressWarnings({"rawtypes", "unchecked"})
@@ -82,7 +82,7 @@ public Pair(T1 left, T2 right) {
    * @param right right value
    * @return A Pair
    */
-  public static  Pair of(
+  public static  Pair of(
       T1 left, T2 right) {
     return new Pair<>(left, right);
   }
@@ -339,7 +339,7 @@ public static  void forEach(
    * @param       Right type
    * @return Iterable over the left elements
    */
-  public static  Iterable left(
+  public static  Iterable left(
       final Iterable> iterable) {
     return Util.transform(iterable, Map.Entry::getKey);
   }
@@ -352,13 +352,13 @@ public static  Iterable left(
    * @param       Right type
    * @return Iterable over the right elements
    */
-  public static  Iterable right(
+  public static  Iterable right(
       final Iterable> iterable) {
     return Util.transform(iterable, Map.Entry::getValue);
   }
 
   @SuppressWarnings("unchecked")
-  public static  List left(
+  public static  List left(
       final List> pairs) {
     if (pairs instanceof PairList) {
       return ((PairList) pairs).leftList();
@@ -367,7 +367,7 @@ public static  List left(
   }
 
   @SuppressWarnings("unchecked")
-  public static  List right(
+  public static  List right(
       final List> pairs) {
     if (pairs instanceof PairList) {
       return ((PairList) pairs).rightList();
diff --git a/core/src/main/java/org/apache/calcite/util/TryThreadLocal.java b/core/src/main/java/org/apache/calcite/util/TryThreadLocal.java
index 33a90e2e71d5..cc601dcdd14e 100644
--- a/core/src/main/java/org/apache/calcite/util/TryThreadLocal.java
+++ b/core/src/main/java/org/apache/calcite/util/TryThreadLocal.java
@@ -30,7 +30,7 @@
  *
  * @param  Value type
  */
-public abstract class TryThreadLocal extends ThreadLocal<@Nullable T> {
+public abstract class TryThreadLocal extends ThreadLocal<@Nullable T> {
   /** Creates a TryThreadLocal with a fixed initial value.
    *
    * @param initialValue Initial value
diff --git a/core/src/main/java/org/apache/calcite/util/Util.java b/core/src/main/java/org/apache/calcite/util/Util.java
index bd8735b0a299..e8aefee5c0ad 100644
--- a/core/src/main/java/org/apache/calcite/util/Util.java
+++ b/core/src/main/java/org/apache/calcite/util/Util.java
@@ -1865,7 +1865,7 @@ public static  List cast(List list, Class clazz) {
    * @param clazz Class to cast to
    * @return An iterator whose members are of the desired type.
    */
-  public static  Iterator cast(
+  public static  Iterator cast(
       final Iterator iter,
       final Class clazz) {
     return transform(iter, x -> clazz.cast(castNonNull(x)));
@@ -2746,7 +2746,7 @@ public static  UnaryOperator andThen(UnaryOperator op1,
   }
 
   /** Transforms a list, applying a function to each element. */
-  public static 
+  public static 
       List transform(List list,
       java.util.function.Function function) {
     if (list.isEmpty() && list instanceof ImmutableList) {
@@ -2760,7 +2760,7 @@ List transform(List list,
 
   /** Transforms a list, applying a function to each element, also passing in
    * the element's index in the list. */
-  public static 
+  public static 
       List transformIndexed(List list,
       BiFunction function) {
     if (list.isEmpty() && list instanceof ImmutableList) {
@@ -2774,7 +2774,7 @@ List transformIndexed(List list,
 
   /** Transforms an iterable, applying a function to each element. */
   @API(since = "1.27", status = API.Status.EXPERIMENTAL)
-  public static 
+  public static 
       Iterable transform(Iterable iterable,
       java.util.function.Function function) {
     // FluentIterable provides toString
@@ -2787,7 +2787,7 @@ Iterable transform(Iterable iterable,
 
   /** Transforms an iterator. */
   @API(since = "1.27", status = API.Status.EXPERIMENTAL)
-  public static 
+  public static 
       Iterator transform(Iterator iterator,
       java.util.function.Function function) {
     return new TransformingIterator<>(iterator, function);
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/Enumerable.java b/linq4j/src/main/java/org/apache/calcite/linq4j/Enumerable.java
index b8afb58cb21e..8c816e42f09a 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/Enumerable.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/Enumerable.java
@@ -16,6 +16,8 @@
  */
 package org.apache.calcite.linq4j;
 
+import org.jspecify.annotations.Nullable;
+
 /**
  * Exposes the enumerator, which supports a simple iteration over a collection.
  *
@@ -26,7 +28,7 @@
  *
  * @param  Element type
  */
-public interface Enumerable
+public interface Enumerable
     extends RawEnumerable, Iterable, ExtendedEnumerable {
   /**
    * Converts this Enumerable to a Queryable.
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java b/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java
index 05c9c40e2601..c4c012ddae44 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java
@@ -4802,7 +4802,7 @@ static class SkipWhileBigDecimalEnumerator implements Enumerator source element type
    * @param  element type
    */
-  static class CastingEnumerator
+  static class CastingEnumerator
       implements Enumerator {
     private final Enumerator enumerator;
     private final Class clazz;
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/Enumerator.java b/linq4j/src/main/java/org/apache/calcite/linq4j/Enumerator.java
index bd9e28be31fc..bb49d210b169 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/Enumerator.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/Enumerator.java
@@ -16,6 +16,8 @@
  */
 package org.apache.calcite.linq4j;
 
+import org.jspecify.annotations.Nullable;
+
 /**
  * Supports a simple iteration over a collection.
  *
@@ -26,7 +28,7 @@
  *
  * @param  Element type
  */
-public interface Enumerator extends AutoCloseable {
+public interface Enumerator extends AutoCloseable {
   /**
    * Gets the current element in the collection.
    *
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/Ord.java b/linq4j/src/main/java/org/apache/calcite/linq4j/Ord.java
index fbc95d5a61f0..55baf3fc2af4 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/Ord.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/Ord.java
@@ -33,7 +33,7 @@
  *
  * @param  Element type
  */
-public class Ord implements Map.Entry {
+public class Ord implements Map.Entry {
   public final int i;
   public final E e;
 
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/Queryable.java b/linq4j/src/main/java/org/apache/calcite/linq4j/Queryable.java
index ea54b9ba7189..b49833140ec8 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/Queryable.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/Queryable.java
@@ -16,6 +16,8 @@
  */
 package org.apache.calcite.linq4j;
 
+import org.jspecify.annotations.Nullable;
+
 /**
  * Provides functionality to evaluate queries against a specific data source
  * wherein the type of the data is known.
@@ -24,6 +26,6 @@
  *
  * @param  Element type
  */
-public interface Queryable
+public interface Queryable
     extends RawQueryable, ExtendedQueryable {
 }
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/function/Function0.java b/linq4j/src/main/java/org/apache/calcite/linq4j/function/Function0.java
index 2385a7637592..59c70c188d5f 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/function/Function0.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/function/Function0.java
@@ -16,11 +16,13 @@
  */
 package org.apache.calcite.linq4j.function;
 
+import org.jspecify.annotations.Nullable;
+
 /**
  * Function with no parameters.
  *
  * @param  Result type
  */
-public interface Function0 extends Function {
+public interface Function0 extends Function {
   R apply();
 }
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/function/Function1.java b/linq4j/src/main/java/org/apache/calcite/linq4j/function/Function1.java
index 5fff55e81779..641494fc7759 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/function/Function1.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/function/Function1.java
@@ -16,13 +16,15 @@
  */
 package org.apache.calcite.linq4j.function;
 
+import org.jspecify.annotations.Nullable;
+
 /**
  * Function with one parameter.
  *
  * @param  Result type
  * @param  Type of parameter 0
  */
-public interface Function1
+public interface Function1
     extends Function {
   /**
    * The identity function.
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/function/Function2.java b/linq4j/src/main/java/org/apache/calcite/linq4j/function/Function2.java
index 05345343445e..623a18c91f19 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/function/Function2.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/function/Function2.java
@@ -16,6 +16,8 @@
  */
 package org.apache.calcite.linq4j.function;
 
+import org.jspecify.annotations.Nullable;
+
 /**
  * Function with two parameters.
  *
@@ -23,7 +25,7 @@
  * @param  Type of argument #0
  * @param  Type of argument #1
  */
-public interface Function2 extends Function {
+public interface Function2 extends Function {
   R apply(T0 v0, T1 v1);
 }

From 877b9b45d11e82dd2797eb93e14fbd1d7a3382b2 Mon Sep 17 00:00:00 2001
From: Vladimir Sitnikov 
Date: Mon, 24 Aug 2026 10:43:57 +0300
Subject: [PATCH 500/562] [CALCITE-7736] Let the linq4j tree visitor return
 null

`Visitor` computes a value by walking a node tree, and there is nothing to
compute for an empty subtree: `Expressions.acceptNodes` returns the result of
the last node, or null when the list is empty, and `VisitorImpl` returns a bare
`null` for a `FunctionExpression` with no body and for a `GotoStatement` with no
expression.

`Visitor.visit` and `Node.accept(Visitor)` now say so, and the type variable
takes the nullable bound the Checker Framework used to infer for it. Under the
Checker Framework this was `VisitorImpl<@Nullable R>`, which forced R to be
nullable; JSpecify tracks upper bounds only, so the annotation moves to the
result.

`UseCounter` extended `VisitorImpl`, a type whose only value is null.
It now extends `VisitorImpl<@Nullable Void>`, like `MayThrowVisitor`.

Nothing outside `org.apache.calcite.linq4j.tree` implements `Visitor` or calls
the `accept` overload that takes one.

Co-Authored-By: Claude Opus 5 
---
 .../calcite/linq4j/tree/BinaryExpression.java |  2 +-
 .../calcite/linq4j/tree/BlockBuilder.java     |  2 +-
 .../calcite/linq4j/tree/BlockStatement.java   |  2 +-
 .../calcite/linq4j/tree/ClassDeclaration.java |  2 +-
 .../linq4j/tree/ConditionalExpression.java    |  2 +-
 .../linq4j/tree/ConditionalStatement.java     |  2 +-
 .../linq4j/tree/ConstantExpression.java       |  2 +-
 .../linq4j/tree/ConstructorDeclaration.java   |  2 +-
 .../linq4j/tree/DeclarationStatement.java     |  2 +-
 .../linq4j/tree/DefaultExpression.java        |  4 +-
 .../linq4j/tree/DynamicExpression.java        |  4 +-
 .../calcite/linq4j/tree/Expressions.java      |  2 +-
 .../calcite/linq4j/tree/FieldDeclaration.java |  2 +-
 .../calcite/linq4j/tree/ForEachStatement.java |  2 +-
 .../calcite/linq4j/tree/ForStatement.java     |  2 +-
 .../linq4j/tree/FunctionExpression.java       |  2 +-
 .../calcite/linq4j/tree/GotoStatement.java    |  2 +-
 .../calcite/linq4j/tree/IndexExpression.java  |  2 +-
 .../linq4j/tree/InvocationExpression.java     |  4 +-
 .../calcite/linq4j/tree/LabelStatement.java   |  2 +-
 .../calcite/linq4j/tree/LambdaExpression.java |  4 +-
 .../linq4j/tree/ListInitExpression.java       |  4 +-
 .../calcite/linq4j/tree/MemberExpression.java |  2 +-
 .../linq4j/tree/MemberInitExpression.java     |  4 +-
 .../linq4j/tree/MethodCallExpression.java     |  2 +-
 .../linq4j/tree/MethodDeclaration.java        |  2 +-
 .../linq4j/tree/NewArrayExpression.java       |  2 +-
 .../calcite/linq4j/tree/NewExpression.java    |  2 +-
 .../org/apache/calcite/linq4j/tree/Node.java  |  4 +-
 .../linq4j/tree/ParameterExpression.java      |  2 +-
 .../calcite/linq4j/tree/SwitchStatement.java  |  4 +-
 .../linq4j/tree/TernaryExpression.java        |  2 +-
 .../calcite/linq4j/tree/ThrowStatement.java   |  2 +-
 .../calcite/linq4j/tree/TryStatement.java     |  2 +-
 .../linq4j/tree/TypeBinaryExpression.java     |  2 +-
 .../calcite/linq4j/tree/UnaryExpression.java  |  2 +-
 .../apache/calcite/linq4j/tree/Visitor.java   | 72 ++++++++++---------
 .../calcite/linq4j/tree/VisitorImpl.java      | 68 +++++++++---------
 .../calcite/linq4j/tree/WhileStatement.java   |  2 +-
 39 files changed, 124 insertions(+), 106 deletions(-)

diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/BinaryExpression.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/BinaryExpression.java
index 0876ed0bc073..67c14f8202d5 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/BinaryExpression.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/BinaryExpression.java
@@ -46,7 +46,7 @@ public class BinaryExpression extends Expression {
     return visitor.visit(this, expression0, expression1);
   }
 
-  @Override public  R accept(Visitor visitor) {
+  @Override public  @Nullable R accept(Visitor visitor) {
     return visitor.visit(this);
   }
 
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/BlockBuilder.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/BlockBuilder.java
index 11370c31892e..0ed4846a3cd3 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/BlockBuilder.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/BlockBuilder.java
@@ -606,7 +606,7 @@ private static class InlineVariableVisitor extends SubstituteVariableVisitor {
   }
 
   /** Use counter. */
-  private static class UseCounter extends VisitorImpl {
+  private static class UseCounter extends VisitorImpl<@Nullable Void> {
     /** Map each parameter to information about how it is used. */
     private final IdentityHashMap map = new IdentityHashMap<>();
     /** Whether the node being visited is evaluated only if some other
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/BlockStatement.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/BlockStatement.java
index 24a843d96438..d8cf6abe7567 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/BlockStatement.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/BlockStatement.java
@@ -63,7 +63,7 @@ private boolean distinctVariables(
     return shuttle.visit(this, newStatements);
   }
 
-  @Override public  R accept(Visitor visitor) {
+  @Override public  @Nullable R accept(Visitor visitor) {
     return visitor.visit(this);
   }
 
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ClassDeclaration.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ClassDeclaration.java
index 2c34c99c995b..02ff51b05cd2 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ClassDeclaration.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ClassDeclaration.java
@@ -69,7 +69,7 @@ public ClassDeclaration(int modifier, String name, @Nullable Type extended,
     return shuttle.visit(this, members1);
   }
 
-  @Override public  R accept(Visitor visitor) {
+  @Override public  @Nullable R accept(Visitor visitor) {
     return visitor.visit(this);
   }
 
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ConditionalExpression.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ConditionalExpression.java
index 267c8da6d6eb..34993a8739d3 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ConditionalExpression.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ConditionalExpression.java
@@ -43,7 +43,7 @@ public ConditionalExpression(List expressionList, Type type) {
     this.expressionList = requireNonNull(expressionList, "expressionList");
   }
 
-  @Override public  R accept(Visitor visitor) {
+  @Override public  @Nullable R accept(Visitor visitor) {
     return visitor.visit(this);
   }
 
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ConditionalStatement.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ConditionalStatement.java
index db6b52c6149d..cb25ab1a65b9 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ConditionalStatement.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ConditionalStatement.java
@@ -48,7 +48,7 @@ public ConditionalStatement(List expressionList) {
     return shuttle.visit(this, list);
   }
 
-  @Override public  R accept(Visitor visitor) {
+  @Override public  @Nullable R accept(Visitor visitor) {
     return visitor.visit(this);
   }
 
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ConstantExpression.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ConstantExpression.java
index 6c27b11137b1..6bfe8677f4c4 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ConstantExpression.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ConstantExpression.java
@@ -69,7 +69,7 @@ public ConstantExpression(Type type, @Nullable Object value) {
     return shuttle.visit(this);
   }
 
-  @Override public  R accept(Visitor visitor) {
+  @Override public  @Nullable R accept(Visitor visitor) {
     return visitor.visit(this);
   }
 
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ConstructorDeclaration.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ConstructorDeclaration.java
index bae7ad7613c5..61298d1e61f4 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ConstructorDeclaration.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ConstructorDeclaration.java
@@ -52,7 +52,7 @@ public ConstructorDeclaration(int modifier, Type declaredAgainst,
     return shuttle.visit(this, body);
   }
 
-  @Override public  R accept(Visitor visitor) {
+  @Override public  @Nullable R accept(Visitor visitor) {
     return visitor.visit(this);
   }
 
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/DeclarationStatement.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/DeclarationStatement.java
index 787c8dad0d9e..82598c00ecfa 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/DeclarationStatement.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/DeclarationStatement.java
@@ -48,7 +48,7 @@ public DeclarationStatement(int modifiers, ParameterExpression parameter,
     return shuttle.visit(this, initializer);
   }
 
-  @Override public  R accept(Visitor visitor) {
+  @Override public  @Nullable R accept(Visitor visitor) {
     return visitor.visit(this);
   }
 
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/DefaultExpression.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/DefaultExpression.java
index 93fca6aad6dd..05c44100f48a 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/DefaultExpression.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/DefaultExpression.java
@@ -16,6 +16,8 @@
  */
 package org.apache.calcite.linq4j.tree;
 
+import org.jspecify.annotations.Nullable;
+
 /**
  * Represents the default value of a type or an empty expression.
  */
@@ -28,7 +30,7 @@ public DefaultExpression(Class type) {
     return shuttle.visit(this);
   }
 
-  @Override public  R accept(Visitor visitor) {
+  @Override public  @Nullable R accept(Visitor visitor) {
     return visitor.visit(this);
   }
 
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/DynamicExpression.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/DynamicExpression.java
index 1bba6b3a3978..4c1ef5f8b1c4 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/DynamicExpression.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/DynamicExpression.java
@@ -16,6 +16,8 @@
  */
 package org.apache.calcite.linq4j.tree;
 
+import org.jspecify.annotations.Nullable;
+
 /**
  * Represents a dynamic operation.
  */
@@ -28,7 +30,7 @@ public DynamicExpression(Class type) {
     return shuttle.visit(this);
   }
 
-  @Override public  R accept(Visitor visitor) {
+  @Override public  @Nullable R accept(Visitor visitor) {
     return visitor.visit(this);
   }
 
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/Expressions.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/Expressions.java
index 46598c980727..48916d4a7a8b 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/Expressions.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/Expressions.java
@@ -3262,7 +3262,7 @@ static List acceptExpressions(List expressions,
     return expressions1;
   }
 
-  static  @Nullable R acceptNodes(@Nullable List nodes,
+  static  @Nullable R acceptNodes(@Nullable List nodes,
       Visitor visitor) {
     R r = null;
     if (nodes != null) {
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/FieldDeclaration.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/FieldDeclaration.java
index 97633f2b73c4..4665560dc6f6 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/FieldDeclaration.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/FieldDeclaration.java
@@ -46,7 +46,7 @@ public FieldDeclaration(int modifier, ParameterExpression parameter,
     return shuttle.visit(this, initializer);
   }
 
-  @Override public  R accept(Visitor visitor) {
+  @Override public  @Nullable R accept(Visitor visitor) {
     return visitor.visit(this);
   }
 
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ForEachStatement.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ForEachStatement.java
index f2f3137ea9ae..82dcac7c264c 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ForEachStatement.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ForEachStatement.java
@@ -48,7 +48,7 @@ public ForEachStatement(ParameterExpression parameter, Expression iterable,
     return shuttle.visit(this, parameter, iterable1, body1);
   }
 
-  @Override public  R accept(Visitor visitor) {
+  @Override public  @Nullable R accept(Visitor visitor) {
     return visitor.visit(this);
   }
 
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ForStatement.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ForStatement.java
index 29e6105c79b1..81ec947078cc 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ForStatement.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ForStatement.java
@@ -58,7 +58,7 @@ public ForStatement(List declarations,
     return shuttle.visit(this, decls1, condition1, post1, body1);
   }
 
-  @Override public  R accept(Visitor visitor) {
+  @Override public  @Nullable R accept(Visitor visitor) {
     return visitor.visit(this);
   }
 
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/FunctionExpression.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/FunctionExpression.java
index 9dca4808f7fc..00a3500ffe9b 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/FunctionExpression.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/FunctionExpression.java
@@ -78,7 +78,7 @@ public FunctionExpression(Class type, BlockStatement body,
     return shuttle.visit(this, body);
   }
 
-  @Override public  R accept(Visitor visitor) {
+  @Override public  @Nullable R accept(Visitor visitor) {
     return visitor.visit(this);
   }
 
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/GotoStatement.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/GotoStatement.java
index 1bce13c7682c..1f87eee8a3f2 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/GotoStatement.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/GotoStatement.java
@@ -68,7 +68,7 @@ public class GotoStatement extends Statement {
     return shuttle.visit(this, expression1);
   }
 
-  @Override public  R accept(Visitor visitor) {
+  @Override public  @Nullable R accept(Visitor visitor) {
     return visitor.visit(this);
   }
 
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/IndexExpression.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/IndexExpression.java
index 04ddd91709ad..d945e729b912 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/IndexExpression.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/IndexExpression.java
@@ -51,7 +51,7 @@ public IndexExpression(Expression array, List indexExpressions) {
     return shuttle.visit(this, array, indexExpressions);
   }
 
-  @Override public  R accept(Visitor visitor) {
+  @Override public  @Nullable R accept(Visitor visitor) {
     return visitor.visit(this);
   }
 
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/InvocationExpression.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/InvocationExpression.java
index dc91a62072da..1b6e3ec63720 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/InvocationExpression.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/InvocationExpression.java
@@ -16,6 +16,8 @@
  */
 package org.apache.calcite.linq4j.tree;
 
+import org.jspecify.annotations.Nullable;
+
 /**
  * Represents an expression that applies a delegate or lambda expression to a
  * list of argument expressions.
@@ -29,7 +31,7 @@ public InvocationExpression(ExpressionType nodeType, Class type) {
     return shuttle.visit(this);
   }
 
-  @Override public  R accept(Visitor visitor) {
+  @Override public  @Nullable R accept(Visitor visitor) {
     return visitor.visit(this);
   }
 
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/LabelStatement.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/LabelStatement.java
index 62e6cf394d50..4a4bac7095a8 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/LabelStatement.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/LabelStatement.java
@@ -39,7 +39,7 @@ public LabelStatement(Expression defaultValue, ExpressionType nodeType) {
     return shuttle.visit(this);
   }
 
-  @Override public  R accept(Visitor visitor) {
+  @Override public  @Nullable R accept(Visitor visitor) {
     return visitor.visit(this);
   }
 
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/LambdaExpression.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/LambdaExpression.java
index dddafee40f44..4daf713b8791 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/LambdaExpression.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/LambdaExpression.java
@@ -16,6 +16,8 @@
  */
 package org.apache.calcite.linq4j.tree;
 
+import org.jspecify.annotations.Nullable;
+
 /**
  * Describes a lambda expression. This captures a block of code that is similar
  * to a Java method body.
@@ -29,7 +31,7 @@ public LambdaExpression(ExpressionType nodeType, Class type) {
     return shuttle.visit(this);
   }
 
-  @Override public  R accept(Visitor visitor) {
+  @Override public  @Nullable R accept(Visitor visitor) {
     return visitor.visit(this);
   }
 
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ListInitExpression.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ListInitExpression.java
index cf69b932d8e2..609b1f8162e6 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ListInitExpression.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ListInitExpression.java
@@ -16,6 +16,8 @@
  */
 package org.apache.calcite.linq4j.tree;
 
+import org.jspecify.annotations.Nullable;
+
 /**
  * Represents a constructor call that has a collection initializer.
  */
@@ -28,7 +30,7 @@ public ListInitExpression(ExpressionType nodeType, Class type) {
     return shuttle.visit(this);
   }
 
-  @Override public  R accept(Visitor visitor) {
+  @Override public  @Nullable R accept(Visitor visitor) {
     return visitor.visit(this);
   }
 
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/MemberExpression.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/MemberExpression.java
index 139eb45ca826..d7867b0f45dc 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/MemberExpression.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/MemberExpression.java
@@ -58,7 +58,7 @@ public MemberExpression(@Nullable Expression expression, PseudoField field) {
     return shuttle.visit(this, expression1);
   }
 
-  @Override public  R accept(Visitor visitor) {
+  @Override public  @Nullable R accept(Visitor visitor) {
     return visitor.visit(this);
   }
 
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/MemberInitExpression.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/MemberInitExpression.java
index 749a75ee1246..fe942da9c7af 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/MemberInitExpression.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/MemberInitExpression.java
@@ -16,6 +16,8 @@
  */
 package org.apache.calcite.linq4j.tree;
 
+import org.jspecify.annotations.Nullable;
+
 /**
  * Represents calling a constructor and initializing one or more members of the
  * new object.
@@ -29,7 +31,7 @@ public MemberInitExpression() {
     return shuttle.visit(this);
   }
 
-  @Override public  R accept(Visitor visitor) {
+  @Override public  @Nullable R accept(Visitor visitor) {
     return visitor.visit(this);
   }
 
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/MethodCallExpression.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/MethodCallExpression.java
index a3e76e2934c0..2f5a14607012 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/MethodCallExpression.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/MethodCallExpression.java
@@ -70,7 +70,7 @@ public class MethodCallExpression extends Expression {
     return shuttle.visit(this, targetExpression, expressions);
   }
 
-  @Override public  R accept(Visitor visitor) {
+  @Override public  @Nullable R accept(Visitor visitor) {
     return visitor.visit(this);
   }
 
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/MethodDeclaration.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/MethodDeclaration.java
index 3f97b34aafa2..4de5d91e870f 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/MethodDeclaration.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/MethodDeclaration.java
@@ -52,7 +52,7 @@ public MethodDeclaration(int modifier, String name, Type resultType,
     return shuttle.visit(this, body);
   }
 
-  @Override public  R accept(Visitor visitor) {
+  @Override public  @Nullable R accept(Visitor visitor) {
     return visitor.visit(this);
   }
 
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/NewArrayExpression.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/NewArrayExpression.java
index 86edb90f9972..be19ef79eede 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/NewArrayExpression.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/NewArrayExpression.java
@@ -54,7 +54,7 @@ public NewArrayExpression(Type type, int dimension, @Nullable Expression bound,
     return shuttle.visit(this, dimension, bound, expressions);
   }
 
-  @Override public  R accept(Visitor visitor) {
+  @Override public  @Nullable R accept(Visitor visitor) {
     return visitor.visit(this);
   }
 
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/NewExpression.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/NewExpression.java
index b51fd123a202..e09176b61059 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/NewExpression.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/NewExpression.java
@@ -56,7 +56,7 @@ public NewExpression(Type type, List arguments,
     return shuttle.visit(this, arguments, memberDeclarations);
   }
 
-  @Override public  R accept(Visitor visitor) {
+  @Override public  @Nullable R accept(Visitor visitor) {
     return visitor.visit(this);
   }
 
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/Node.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/Node.java
index 6cbe9e508597..940583fc4205 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/Node.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/Node.java
@@ -16,9 +16,11 @@
  */
 package org.apache.calcite.linq4j.tree;
 
+import org.jspecify.annotations.Nullable;
+
 /** Parse tree node. */
 public interface Node {
-   R accept(Visitor visitor);
+   @Nullable R accept(Visitor visitor);
 
   Node accept(Shuttle shuttle);
 
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ParameterExpression.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ParameterExpression.java
index ffe330979cab..9054cbbc4e0c 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ParameterExpression.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ParameterExpression.java
@@ -52,7 +52,7 @@ public ParameterExpression(int modifier, Type type, String name) {
     return shuttle.visit(this);
   }
 
-  @Override public  R accept(Visitor visitor) {
+  @Override public  @Nullable R accept(Visitor visitor) {
     return visitor.visit(this);
   }
 
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/SwitchStatement.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/SwitchStatement.java
index 62265ebb262b..84973096d2a9 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/SwitchStatement.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/SwitchStatement.java
@@ -16,6 +16,8 @@
  */
 package org.apache.calcite.linq4j.tree;
 
+import org.jspecify.annotations.Nullable;
+
 /**
  * Represents a control expression that handles multiple selections by passing
  * control to {@link SwitchCase}.
@@ -29,7 +31,7 @@ public SwitchStatement(ExpressionType nodeType) {
     return shuttle.visit(this);
   }
 
-  @Override public  R accept(Visitor visitor) {
+  @Override public  @Nullable R accept(Visitor visitor) {
     return visitor.visit(this);
   }
 
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/TernaryExpression.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/TernaryExpression.java
index f54cad02f0cb..b726e043c432 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/TernaryExpression.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/TernaryExpression.java
@@ -47,7 +47,7 @@ public class TernaryExpression extends Expression {
     return shuttle.visit(this, expression0, expression1, expression2);
   }
 
-  @Override public  R accept(Visitor visitor) {
+  @Override public  @Nullable R accept(Visitor visitor) {
     return visitor.visit(this);
   }
 
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ThrowStatement.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ThrowStatement.java
index 8d535043e7da..f503b42c35ed 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ThrowStatement.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/ThrowStatement.java
@@ -37,7 +37,7 @@ public ThrowStatement(Expression expression) {
     return shuttle.visit(this, expression);
   }
 
-  @Override public  R accept(Visitor visitor) {
+  @Override public  @Nullable R accept(Visitor visitor) {
     return visitor.visit(this);
   }
 
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/TryStatement.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/TryStatement.java
index 8a19ee1738d9..c937c8304b3d 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/TryStatement.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/TryStatement.java
@@ -54,7 +54,7 @@ public TryStatement(Statement body, List catchBlocks,
     return shuttle.visit(this, body1, catchBlocks1, fynally1);
   }
 
-  @Override public  R accept(Visitor visitor) {
+  @Override public  @Nullable R accept(Visitor visitor) {
     return visitor.visit(this);
   }
 
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/TypeBinaryExpression.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/TypeBinaryExpression.java
index 16b4bf689737..5fface4c4665 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/TypeBinaryExpression.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/TypeBinaryExpression.java
@@ -44,7 +44,7 @@ public TypeBinaryExpression(ExpressionType nodeType, Expression expression,
     return shuttle.visit(this, expression);
   }
 
-  @Override public  R accept(Visitor visitor) {
+  @Override public  @Nullable R accept(Visitor visitor) {
     return visitor.visit(this);
   }
 
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/UnaryExpression.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/UnaryExpression.java
index af9623490ff9..75fcdf920f28 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/UnaryExpression.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/UnaryExpression.java
@@ -40,7 +40,7 @@ public class UnaryExpression extends Expression {
     return shuttle.visit(this, expression);
   }
 
-  @Override public  R accept(Visitor visitor) {
+  @Override public  @Nullable R accept(Visitor visitor) {
     return visitor.visit(this);
   }
 
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/Visitor.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/Visitor.java
index 6063983227f4..f4c4103da8eb 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/Visitor.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/Visitor.java
@@ -16,44 +16,46 @@
  */
 package org.apache.calcite.linq4j.tree;
 
+import org.jspecify.annotations.Nullable;
+
 /**
  * Node visitor.
  *
  * @param  Return type
  */
-public interface Visitor {
-  R visit(BinaryExpression binaryExpression);
-  R visit(BlockStatement blockStatement);
-  R visit(ClassDeclaration classDeclaration);
-  R visit(ConditionalExpression conditionalExpression);
-  R visit(ConditionalStatement conditionalStatement);
-  R visit(ConstantExpression constantExpression);
-  R visit(ConstructorDeclaration constructorDeclaration);
-  R visit(DeclarationStatement declarationStatement);
-  R visit(DefaultExpression defaultExpression);
-  R visit(DynamicExpression dynamicExpression);
-  R visit(FieldDeclaration fieldDeclaration);
-  R visit(ForStatement forStatement);
-  R visit(ForEachStatement forEachStatement);
-  R visit(FunctionExpression functionExpression);
-  R visit(GotoStatement gotoStatement);
-  R visit(IndexExpression indexExpression);
-  R visit(InvocationExpression invocationExpression);
-  R visit(LabelStatement labelStatement);
-  R visit(LambdaExpression lambdaExpression);
-  R visit(ListInitExpression listInitExpression);
-  R visit(MemberExpression memberExpression);
-  R visit(MemberInitExpression memberInitExpression);
-  R visit(MethodCallExpression methodCallExpression);
-  R visit(MethodDeclaration methodDeclaration);
-  R visit(NewArrayExpression newArrayExpression);
-  R visit(NewExpression newExpression);
-  R visit(ParameterExpression parameterExpression);
-  R visit(SwitchStatement switchStatement);
-  R visit(TernaryExpression ternaryExpression);
-  R visit(ThrowStatement throwStatement);
-  R visit(TryStatement tryStatement);
-  R visit(TypeBinaryExpression typeBinaryExpression);
-  R visit(UnaryExpression unaryExpression);
-  R visit(WhileStatement whileStatement);
+public interface Visitor {
+  @Nullable R visit(BinaryExpression binaryExpression);
+  @Nullable R visit(BlockStatement blockStatement);
+  @Nullable R visit(ClassDeclaration classDeclaration);
+  @Nullable R visit(ConditionalExpression conditionalExpression);
+  @Nullable R visit(ConditionalStatement conditionalStatement);
+  @Nullable R visit(ConstantExpression constantExpression);
+  @Nullable R visit(ConstructorDeclaration constructorDeclaration);
+  @Nullable R visit(DeclarationStatement declarationStatement);
+  @Nullable R visit(DefaultExpression defaultExpression);
+  @Nullable R visit(DynamicExpression dynamicExpression);
+  @Nullable R visit(FieldDeclaration fieldDeclaration);
+  @Nullable R visit(ForStatement forStatement);
+  @Nullable R visit(ForEachStatement forEachStatement);
+  @Nullable R visit(FunctionExpression functionExpression);
+  @Nullable R visit(GotoStatement gotoStatement);
+  @Nullable R visit(IndexExpression indexExpression);
+  @Nullable R visit(InvocationExpression invocationExpression);
+  @Nullable R visit(LabelStatement labelStatement);
+  @Nullable R visit(LambdaExpression lambdaExpression);
+  @Nullable R visit(ListInitExpression listInitExpression);
+  @Nullable R visit(MemberExpression memberExpression);
+  @Nullable R visit(MemberInitExpression memberInitExpression);
+  @Nullable R visit(MethodCallExpression methodCallExpression);
+  @Nullable R visit(MethodDeclaration methodDeclaration);
+  @Nullable R visit(NewArrayExpression newArrayExpression);
+  @Nullable R visit(NewExpression newExpression);
+  @Nullable R visit(ParameterExpression parameterExpression);
+  @Nullable R visit(SwitchStatement switchStatement);
+  @Nullable R visit(TernaryExpression ternaryExpression);
+  @Nullable R visit(ThrowStatement throwStatement);
+  @Nullable R visit(TryStatement tryStatement);
+  @Nullable R visit(TypeBinaryExpression typeBinaryExpression);
+  @Nullable R visit(UnaryExpression unaryExpression);
+  @Nullable R visit(WhileStatement whileStatement);
 }
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/VisitorImpl.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/VisitorImpl.java
index 61c3cbd67d8b..0ac0a34ea6aa 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/VisitorImpl.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/VisitorImpl.java
@@ -32,38 +32,38 @@ public VisitorImpl() {
     super();
   }
 
-  @Override public R visit(BinaryExpression binaryExpression) {
+  @Override public @Nullable R visit(BinaryExpression binaryExpression) {
     R r0 = binaryExpression.expression0.accept(this);
     R r1 = binaryExpression.expression1.accept(this);
     return r1;
   }
 
-  @Override public R visit(BlockStatement blockStatement) {
+  @Override public @Nullable R visit(BlockStatement blockStatement) {
     return Expressions.acceptNodes(blockStatement.statements, this);
   }
 
-  @Override public R visit(ClassDeclaration classDeclaration) {
+  @Override public @Nullable R visit(ClassDeclaration classDeclaration) {
     return Expressions.acceptNodes(classDeclaration.memberDeclarations, this);
   }
 
-  @Override public R visit(ConditionalExpression conditionalExpression) {
+  @Override public @Nullable R visit(ConditionalExpression conditionalExpression) {
     return Expressions.acceptNodes(conditionalExpression.expressionList, this);
   }
 
-  @Override public R visit(ConditionalStatement conditionalStatement) {
+  @Override public @Nullable R visit(ConditionalStatement conditionalStatement) {
     return Expressions.acceptNodes(conditionalStatement.expressionList, this);
   }
 
-  @Override public R visit(ConstantExpression constantExpression) {
+  @Override public @Nullable R visit(ConstantExpression constantExpression) {
     return null;
   }
 
-  @Override public R visit(ConstructorDeclaration constructorDeclaration) {
+  @Override public @Nullable R visit(ConstructorDeclaration constructorDeclaration) {
     R r0 = Expressions.acceptNodes(constructorDeclaration.parameters, this);
     return constructorDeclaration.body.accept(this);
   }
 
-  @Override public R visit(DeclarationStatement declarationStatement) {
+  @Override public @Nullable R visit(DeclarationStatement declarationStatement) {
     R r = declarationStatement.parameter.accept(this);
     if (declarationStatement.initializer != null) {
       r = declarationStatement.initializer.accept(this);
@@ -71,67 +71,67 @@ public VisitorImpl() {
     return r;
   }
 
-  @Override public R visit(DefaultExpression defaultExpression) {
+  @Override public @Nullable R visit(DefaultExpression defaultExpression) {
     return null;
   }
 
-  @Override public R visit(DynamicExpression dynamicExpression) {
+  @Override public @Nullable R visit(DynamicExpression dynamicExpression) {
     return null;
   }
 
-  @Override public R visit(FieldDeclaration fieldDeclaration) {
+  @Override public @Nullable R visit(FieldDeclaration fieldDeclaration) {
     R r0 = fieldDeclaration.parameter.accept(this);
     return fieldDeclaration.initializer == null ? null
         : fieldDeclaration.initializer.accept(this);
   }
 
-  @Override public R visit(ForStatement forStatement) {
+  @Override public @Nullable R visit(ForStatement forStatement) {
     R r0 = Expressions.acceptNodes(forStatement.declarations, this);
     R r1 = forStatement.condition == null ? null : forStatement.condition.accept(this);
     R r2 = forStatement.post == null ? null : forStatement.post.accept(this);
     return forStatement.body.accept(this);
   }
 
-  @Override public R visit(ForEachStatement forEachStatement) {
+  @Override public @Nullable R visit(ForEachStatement forEachStatement) {
     R r0 = forEachStatement.parameter.accept(this);
     R r1 = forEachStatement.iterable.accept(this);
     return forEachStatement.body.accept(this);
   }
 
-  @Override public R visit(FunctionExpression functionExpression) {
+  @Override public @Nullable R visit(FunctionExpression functionExpression) {
     @SuppressWarnings("unchecked") final List parameterList =
         functionExpression.parameterList;
     R r0 = Expressions.acceptNodes(parameterList, this);
     return functionExpression.body == null ? null : functionExpression.body.accept(this);
   }
 
-  @Override public R visit(GotoStatement gotoStatement) {
+  @Override public @Nullable R visit(GotoStatement gotoStatement) {
     return gotoStatement.expression == null ? null
         : gotoStatement.expression.accept(this);
   }
 
-  @Override public R visit(IndexExpression indexExpression) {
+  @Override public @Nullable R visit(IndexExpression indexExpression) {
     R r0 = indexExpression.array.accept(this);
     return Expressions.acceptNodes(indexExpression.indexExpressions, this);
   }
 
-  @Override public R visit(InvocationExpression invocationExpression) {
+  @Override public @Nullable R visit(InvocationExpression invocationExpression) {
     return null;
   }
 
-  @Override public R visit(LabelStatement labelStatement) {
+  @Override public @Nullable R visit(LabelStatement labelStatement) {
     return labelStatement.defaultValue.accept(this);
   }
 
-  @Override public R visit(LambdaExpression lambdaExpression) {
+  @Override public @Nullable R visit(LambdaExpression lambdaExpression) {
     return null;
   }
 
-  @Override public R visit(ListInitExpression listInitExpression) {
+  @Override public @Nullable R visit(ListInitExpression listInitExpression) {
     return null;
   }
 
-  @Override public R visit(MemberExpression memberExpression) {
+  @Override public @Nullable R visit(MemberExpression memberExpression) {
     R r = null;
     if (memberExpression.expression != null) {
       r = memberExpression.expression.accept(this);
@@ -139,11 +139,11 @@ public VisitorImpl() {
     return r;
   }
 
-  @Override public R visit(MemberInitExpression memberInitExpression) {
+  @Override public @Nullable R visit(MemberInitExpression memberInitExpression) {
     return null;
   }
 
-  @Override public R visit(MethodCallExpression methodCallExpression) {
+  @Override public @Nullable R visit(MethodCallExpression methodCallExpression) {
     R r = null;
     if (methodCallExpression.targetExpression != null) {
       r = methodCallExpression.targetExpression.accept(this);
@@ -151,12 +151,12 @@ public VisitorImpl() {
     return Expressions.acceptNodes(methodCallExpression.expressions, this);
   }
 
-  @Override public R visit(MethodDeclaration methodDeclaration) {
+  @Override public @Nullable R visit(MethodDeclaration methodDeclaration) {
     R r0 = Expressions.acceptNodes(methodDeclaration.parameters, this);
     return methodDeclaration.body.accept(this);
   }
 
-  @Override public R visit(NewArrayExpression newArrayExpression) {
+  @Override public @Nullable R visit(NewArrayExpression newArrayExpression) {
     R r = null;
     if (newArrayExpression.bound != null) {
       r = newArrayExpression.bound.accept(this);
@@ -164,30 +164,30 @@ public VisitorImpl() {
     return Expressions.acceptNodes(newArrayExpression.expressions, this);
   }
 
-  @Override public R visit(NewExpression newExpression) {
+  @Override public @Nullable R visit(NewExpression newExpression) {
     R r0 = Expressions.acceptNodes(newExpression.arguments, this);
     return Expressions.acceptNodes(newExpression.memberDeclarations, this);
   }
 
-  @Override public R visit(ParameterExpression parameterExpression) {
+  @Override public @Nullable R visit(ParameterExpression parameterExpression) {
     return null;
   }
 
-  @Override public R visit(SwitchStatement switchStatement) {
+  @Override public @Nullable R visit(SwitchStatement switchStatement) {
     return null;
   }
 
-  @Override public R visit(TernaryExpression ternaryExpression) {
+  @Override public @Nullable R visit(TernaryExpression ternaryExpression) {
     R r0 = ternaryExpression.expression0.accept(this);
     R r1 = ternaryExpression.expression1.accept(this);
     return ternaryExpression.expression2.accept(this);
   }
 
-  @Override public R visit(ThrowStatement throwStatement) {
+  @Override public @Nullable R visit(ThrowStatement throwStatement) {
     return throwStatement.expression.accept(this);
   }
 
-  @Override public R visit(TryStatement tryStatement) {
+  @Override public @Nullable R visit(TryStatement tryStatement) {
     R r = tryStatement.body.accept(this);
     for (CatchBlock catchBlock : tryStatement.catchBlocks) {
       r = catchBlock.parameter.accept(this);
@@ -199,15 +199,15 @@ public VisitorImpl() {
     return r;
   }
 
-  @Override public R visit(TypeBinaryExpression typeBinaryExpression) {
+  @Override public @Nullable R visit(TypeBinaryExpression typeBinaryExpression) {
     return typeBinaryExpression.expression.accept(this);
   }
 
-  @Override public R visit(UnaryExpression unaryExpression) {
+  @Override public @Nullable R visit(UnaryExpression unaryExpression) {
     return unaryExpression.expression.accept(this);
   }
 
-  @Override public R visit(WhileStatement whileStatement) {
+  @Override public @Nullable R visit(WhileStatement whileStatement) {
     R r0 = whileStatement.condition.accept(this);
     return whileStatement.body.accept(this);
   }
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/WhileStatement.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/WhileStatement.java
index ddd73bff37f6..e968e3b98673 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/WhileStatement.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/WhileStatement.java
@@ -42,7 +42,7 @@ public WhileStatement(Expression condition, Statement body) {
     return shuttle.visit(this, condition1, body1);
   }
 
-  @Override public  R accept(Visitor visitor) {
+  @Override public  @Nullable R accept(Visitor visitor) {
     return visitor.visit(this);
   }
 

From 45975737e3b7fd9b57499b4fda1975e7050cfc0b Mon Sep 17 00:00:00 2001
From: Vladimir Sitnikov 
Date: Mon, 24 Aug 2026 10:51:31 +0300
Subject: [PATCH 501/562] [CALCITE-7736] Admit that min and max return null for
 an empty sequence

The seedless `aggregate` starts from the first element and returns null when
there is none. The `min` and `max` overloads that call it returned a non-null
type anyway, so an empty sequence produced a null the signature ruled out.
Eight of them now say `@Nullable`; the other overloads either seed the
accumulator or already wrap the call in `requireNonNull`, and are unchanged.
Four `min` and `max` overloads were already annotated, so this makes the family
consistent.

`aggregate` declares its accumulator `Function2<@Nullable TSource, TSource,
TSource>`, and the reducers it is called with genuinely handle a null
accumulator -- every `MIN` and `MAX` constant in `Extensions` opens with
`v1 == null`. Their declarations now match; the `SUM` constants do not test for
null and keep the non-null accumulator.

Co-Authored-By: Claude Opus 5 
---
 .../calcite/linq4j/EnumerableDefaults.java    | 28 +++++++++----------
 .../org/apache/calcite/linq4j/Extensions.java | 22 ++++++++-------
 2 files changed, 26 insertions(+), 24 deletions(-)

diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java b/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java
index c4c012ddae44..a11d72b6b305 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java
@@ -2942,7 +2942,7 @@ public static  long longCount(Enumerable enumerable,
    * Returns the maximum value in a generic
    * sequence.
    */
-  public static > TSource max(
+  public static @Nullable > TSource max(
       Enumerable source) {
     return aggregate(source, maxFunction());
   }
@@ -2951,7 +2951,7 @@ public static > TSource max(
    * Invokes a transform function on each element of a
    * sequence and returns the maximum Decimal value.
    */
-  public static  BigDecimal max(Enumerable source,
+  public static  @Nullable BigDecimal max(Enumerable source,
       BigDecimalFunction1 selector) {
     return aggregate(source.select(selector), maxFunction());
   }
@@ -2961,7 +2961,7 @@ public static  BigDecimal max(Enumerable source,
    * sequence and returns the maximum nullable Decimal
    * value.
    */
-  public static  BigDecimal max(Enumerable source,
+  public static  @Nullable BigDecimal max(Enumerable source,
       NullableBigDecimalFunction1 selector) {
     return aggregate(source.select(selector), maxFunction());
   }
@@ -2980,7 +2980,7 @@ public static  double max(Enumerable source,
    * sequence and returns the maximum nullable Double
    * value.
    */
-  public static  Double max(Enumerable source,
+  public static  @Nullable Double max(Enumerable source,
       NullableDoubleFunction1 selector) {
     return aggregate(source.select(selector), Extensions.DOUBLE_MAX);
   }
@@ -2999,7 +2999,7 @@ public static  int max(Enumerable source,
    * sequence and returns the maximum nullable int value. (Defined
    * by Enumerable.)
    */
-  public static  Integer max(Enumerable source,
+  public static  @Nullable Integer max(Enumerable source,
       NullableIntegerFunction1 selector) {
     return aggregate(source.select(selector), Extensions.INTEGER_MAX);
   }
@@ -3062,22 +3062,22 @@ public static  float max(Enumerable source,
   }
 
   @SuppressWarnings("unchecked")
-  private static > Function2
-      minFunction() {
-    return (Function2) (Function2) Extensions.COMPARABLE_MIN;
+  private static >
+      Function2<@Nullable TSource, TSource, TSource> minFunction() {
+    return (Function2<@Nullable TSource, TSource, TSource>) (Function2) Extensions.COMPARABLE_MIN;
   }
 
   @SuppressWarnings("unchecked")
-  private static > Function2
-      maxFunction() {
-    return (Function2) (Function2) Extensions.COMPARABLE_MAX;
+  private static >
+      Function2<@Nullable TSource, TSource, TSource> maxFunction() {
+    return (Function2<@Nullable TSource, TSource, TSource>) (Function2) Extensions.COMPARABLE_MAX;
   }
 
   /**
    * Invokes a transform function on each element of a
    * sequence and returns the minimum Decimal value.
    */
-  public static  BigDecimal min(Enumerable source,
+  public static  @Nullable BigDecimal min(Enumerable source,
       BigDecimalFunction1 selector) {
     Function2 min = minFunction();
     return aggregate(source.select(selector), null, min);
@@ -3088,7 +3088,7 @@ public static  BigDecimal min(Enumerable source,
    * sequence and returns the minimum nullable Decimal
    * value.
    */
-  public static  BigDecimal min(Enumerable source,
+  public static  @Nullable BigDecimal min(Enumerable source,
       NullableBigDecimalFunction1 selector) {
     return aggregate(source.select(selector), minFunction());
   }
@@ -3107,7 +3107,7 @@ public static  double min(Enumerable source,
    * sequence and returns the minimum nullable Double
    * value.
    */
-  public static  Double min(Enumerable source,
+  public static  @Nullable Double min(Enumerable source,
       NullableDoubleFunction1 selector) {
     return aggregate(source.select(selector), Extensions.DOUBLE_MIN);
   }
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/Extensions.java b/linq4j/src/main/java/org/apache/calcite/linq4j/Extensions.java
index e7434a252c75..184d52013758 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/Extensions.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/Extensions.java
@@ -18,6 +18,8 @@
 
 import org.apache.calcite.linq4j.function.Function2;
 
+import org.jspecify.annotations.Nullable;
+
 import java.math.BigDecimal;
 import java.util.Comparator;
 import java.util.Map;
@@ -126,35 +128,35 @@ private Extensions() {}
       (v1, v2) -> v1 + v2;
 
   @SuppressWarnings("unchecked")
-  static final Function2 COMPARABLE_MIN =
+  static final Function2<@Nullable Comparable, Comparable, Comparable> COMPARABLE_MIN =
       (v1, v2) -> v1 == null || v1.compareTo(v2) > 0 ? v2 : v1;
 
   @SuppressWarnings("unchecked")
-  static final Function2 COMPARABLE_MAX =
+  static final Function2<@Nullable Comparable, Comparable, Comparable> COMPARABLE_MAX =
       (v1, v2) -> v1 == null || v1.compareTo(v2) < 0 ? v2 : v1;
 
-  static final Function2 FLOAT_MIN =
+  static final Function2<@Nullable Float, Float, Float> FLOAT_MIN =
       (v1, v2) -> v1 == null || v1.compareTo(v2) > 0 ? v2 : v1;
 
-  static final Function2 FLOAT_MAX =
+  static final Function2<@Nullable Float, Float, Float> FLOAT_MAX =
       (v1, v2) -> v1 == null || v1.compareTo(v2) < 0 ? v2 : v1;
 
-  static final Function2 DOUBLE_MIN =
+  static final Function2<@Nullable Double, Double, Double> DOUBLE_MIN =
       (v1, v2) -> v1 == null || v1.compareTo(v2) > 0 ? v2 : v1;
 
-  static final Function2 DOUBLE_MAX =
+  static final Function2<@Nullable Double, Double, Double> DOUBLE_MAX =
       (v1, v2) -> v1 == null || v1.compareTo(v2) < 0 ? v2 : v1;
 
-  static final Function2 INTEGER_MIN =
+  static final Function2<@Nullable Integer, Integer, Integer> INTEGER_MIN =
       (v1, v2) -> v1 == null || v1.compareTo(v2) > 0 ? v2 : v1;
 
-  static final Function2 INTEGER_MAX =
+  static final Function2<@Nullable Integer, Integer, Integer> INTEGER_MAX =
       (v1, v2) -> v1 == null || v1.compareTo(v2) < 0 ? v2 : v1;
 
-  static final Function2 LONG_MIN =
+  static final Function2<@Nullable Long, Long, Long> LONG_MIN =
       (v1, v2) -> v1 == null || v1.compareTo(v2) > 0 ? v2 : v1;
 
-  static final Function2 LONG_MAX =
+  static final Function2<@Nullable Long, Long, Long> LONG_MAX =
       (v1, v2) -> v1 == null || v1.compareTo(v2) < 0 ? v2 : v1;
 
   // flags a piece of code we're yet to implement

From 9c1d74bc9607407fd627b9f19aa2f53a5629c1f3 Mon Sep 17 00:00:00 2001
From: Vladimir Sitnikov 
Date: Mon, 24 Aug 2026 11:00:20 +0300
Subject: [PATCH 502/562] [CALCITE-7736] Let a seeded aggregate start from null

`aggregate(source, seed, func)` is the reduce that takes a starting value, and
`min` and `max` start it at null. Its accumulator type variable was non-null, so
the seed, the reducer and the result all disagreed with the call.

`TAccumulate` and `TResult` take the nullable bound the Checker Framework used
to infer for them. The reducer parameter becomes
`Function2`: the constants in
`Extensions` accept a null accumulator but never produce one, and the wildcard
is what lets a reducer with a non-null result feed a nullable seed.

Three more `min` overloads return null for an empty sequence and now say so,
and `long min(Enumerable, LongFunction1)` wraps the call in `requireNonNull`,
which is what the other overloads that unbox the accumulator already do.

`EqualityComparer` takes the nullable bound as well; `Functions` builds
comparers over nullable elements.

The two `aggregate` bodies carry a NullAway suppression. Assigning the result of
a call returning `? extends TAccumulate` to a `TAccumulate` local makes NullAway
report the local as @Nullable, even on `return result;` where the local's type
is the return type. Spelling the type argument exactly reports nothing, so the
wildcard is what triggers it.

Co-Authored-By: Claude Opus 5 
---
 .../calcite/linq4j/EnumerableDefaults.java    | 26 +++++++++++++------
 .../linq4j/function/EqualityComparer.java     |  4 ++-
 2 files changed, 21 insertions(+), 9 deletions(-)

diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java b/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java
index a11d72b6b305..d17442ceec89 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java
@@ -102,9 +102,13 @@ public abstract class EnumerableDefaults {
    * sequence. The specified seed value is used as the initial
    * accumulator value.
    */
-  public static  TAccumulate aggregate(
+  // NullAway treats the result of a call returning `? extends TAccumulate` as @Nullable
+  // once it is assigned to a TAccumulate local, even though TAccumulate is the local's own
+  // type. The wildcard is what lets a reducer with a non-null result feed a nullable seed.
+  @SuppressWarnings("NullAway")
+  public static  TAccumulate aggregate(
       Enumerable source, TAccumulate seed,
-      Function2 func) {
+      Function2 func) {
     TAccumulate result = seed;
     try (Enumerator os = source.enumerator()) {
       while (os.moveNext()) {
@@ -121,9 +125,14 @@ public static  TAccumulate aggregate(
    * accumulator value, and the specified function is used to select
    * the result value.
    */
-  public static  TResult aggregate(
+  // NullAway treats the result of a call returning `? extends TAccumulate` as @Nullable
+  // once it is assigned to a TAccumulate local, even though TAccumulate is the local's own
+  // type. The wildcard is what lets a reducer with a non-null result feed a nullable seed.
+  @SuppressWarnings("NullAway")
+  public static  TResult aggregate(
       Enumerable source, TAccumulate seed,
-      Function2 func,
+      Function2 func,
       Function1 selector) {
     TAccumulate accumulate = seed;
     try (Enumerator os = source.enumerator()) {
@@ -3126,7 +3135,7 @@ public static  int min(Enumerable source,
    * sequence and returns the minimum nullable int value. (Defined
    * by Enumerable.)
    */
-  public static  Integer min(Enumerable source,
+  public static  @Nullable Integer min(Enumerable source,
       NullableIntegerFunction1 selector) {
     return aggregate(source.select(selector), null, Extensions.INTEGER_MIN);
   }
@@ -3137,7 +3146,8 @@ public static  Integer min(Enumerable source,
    */
   public static  long min(Enumerable source,
       LongFunction1 selector) {
-    return aggregate(source.select(adapt(selector)), null, Extensions.LONG_MIN);
+    return requireNonNull(
+        aggregate(source.select(adapt(selector)), null, Extensions.LONG_MIN));
   }
 
   /**
@@ -3145,7 +3155,7 @@ public static  long min(Enumerable source,
    * sequence and returns the minimum nullable long value. (Defined
    * by Enumerable.)
    */
-  public static  Long min(Enumerable source,
+  public static  @Nullable Long min(Enumerable source,
       NullableLongFunction1 selector) {
     return aggregate(source.select(selector), null, Extensions.LONG_MIN);
   }
@@ -3165,7 +3175,7 @@ public static  float min(Enumerable source,
    * sequence and returns the minimum nullable Float
    * value.
    */
-  public static  Float min(Enumerable source,
+  public static  @Nullable Float min(Enumerable source,
       NullableFloatFunction1 selector) {
     return aggregate(source.select(selector), null, Extensions.FLOAT_MIN);
   }
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/function/EqualityComparer.java b/linq4j/src/main/java/org/apache/calcite/linq4j/function/EqualityComparer.java
index 11a91672e05e..5581f25e7874 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/function/EqualityComparer.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/function/EqualityComparer.java
@@ -16,12 +16,14 @@
  */
 package org.apache.calcite.linq4j.function;
 
+import org.jspecify.annotations.Nullable;
+
 /**
  * Compares values for equality.
  *
  * @param  Value type
  */
-public interface EqualityComparer {
+public interface EqualityComparer {
   boolean equal(T v1, T v2);
 
   int hashCode(T t);

From a4f663bde204f8e5f8884475d32277a1a8480f50 Mon Sep 17 00:00:00 2001
From: Vladimir Sitnikov 
Date: Mon, 24 Aug 2026 11:18:51 +0300
Subject: [PATCH 503/562] [CALCITE-7736] Drop castNonNull calls whose argument
 is already non-null

NullAway reports a `castToNonNull` whose argument it can already prove non-null,
which is how it flags a cast that has stopped earning its place. Four of them in
`EnumerableDefaults`: `curAccumulator` is assigned from
`accumulatorInitializer.apply()` a few lines above each use, and `outerValue`
from `outerValues.get(i)`.

Co-Authored-By: Claude Opus 5 
---
 .../org/apache/calcite/linq4j/EnumerableDefaults.java     | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java b/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java
index d17442ceec89..350716f265cb 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java
@@ -1125,18 +1125,18 @@ private static class SortedAggregateEnumerator Enumerable correlateBatchJoin(
                 outerValue = outerValues.get(i); // get current outer value
                 nextInnerValue();
                 // Compare current block row to current inner value
-                if (predicate.apply(castNonNull(outerValue), castNonNull(innerValue))) {
+                if (predicate.apply(outerValue, castNonNull(innerValue))) {
                   atLeastOneResult = true;
                   // Skip the rest of inner values in case of
                   // ANTI and SEMI when a match is found

From 63e0c067eca1fa22127d32c44e4b38df8764eb11 Mon Sep 17 00:00:00 2001
From: Vladimir Sitnikov 
Date: Mon, 24 Aug 2026 11:22:59 +0300
Subject: [PATCH 504/562] [CALCITE-7736] Handle the nulls the JDK actually
 hands out

Four places read a value the JDK is entitled to leave null. The Checker
Framework knew about the first two from `InvocationHandler.astub`, which this
migration deleted; NullAway's own JDK models say the same thing.

* `InvocationHandler.invoke` receives a null `args` array when the proxied
  method declares no parameters. `Compatible` indexed it, and
  `FunctionExpression` forwarded it to a varargs call that requires an array.
* `Primitive.asList` adapts a primitive array, whose elements box to a non-null
  value, so the `Array.get` result is cast rather than checked.
* `EnumerableDefaults.takeTopN` looked up the key it had just read from
  `lastKey()`, and suppressed the finding on the declaration while dereferencing
  the value on the next line. It now uses `requireNonNull`.

Co-Authored-By: Claude Opus 5 
---
 .../org/apache/calcite/linq4j/EnumerableDefaults.java | 11 +++++------
 .../calcite/linq4j/tree/FunctionExpression.java       |  3 ++-
 .../org/apache/calcite/linq4j/tree/Primitive.java     |  4 +++-
 .../org/apache/calcite/linq4j/util/Compatible.java    |  2 +-
 4 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java b/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java
index 350716f265cb..71dc1e7315df 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java
@@ -3088,7 +3088,7 @@ public static  float max(Enumerable source,
    */
   public static  @Nullable BigDecimal min(Enumerable source,
       BigDecimalFunction1 selector) {
-    Function2 min = minFunction();
+    Function2<@Nullable BigDecimal, BigDecimal, BigDecimal> min = minFunction();
     return aggregate(source.select(selector), null, min);
   }
 
@@ -3166,8 +3166,8 @@ public static  long min(Enumerable source,
    */
   public static  float min(Enumerable source,
       FloatFunction1 selector) {
-    return aggregate(source.select(adapt(selector)), null,
-        Extensions.FLOAT_MIN);
+    return requireNonNull(
+        aggregate(source.select(adapt(selector)), null, Extensions.FLOAT_MIN));
   }
 
   /**
@@ -3187,7 +3187,7 @@ public static  float min(Enumerable source,
    */
   public static > @Nullable TResult min(
       Enumerable source, Function1 selector) {
-    Function2 min = minFunction();
+    Function2<@Nullable TResult, TResult, TResult> min = minFunction();
     return aggregate(source.select(selector), null, min);
   }
 
@@ -3294,8 +3294,7 @@ public static  Enumerable orderBy(
                 continue;
               }
               // remove last entry from tree map, so that we keep at most 'needed' rows
-              @SuppressWarnings("NullAway")
-              List l = map.get(lastKey);
+              List l = requireNonNull(map.get(lastKey), "map.get(lastKey)");
               if (l.size() == 1) {
                 map.remove(lastKey);
               } else {
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/FunctionExpression.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/FunctionExpression.java
index 00a3500ffe9b..f95f78041f59 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/FunctionExpression.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/FunctionExpression.java
@@ -104,7 +104,8 @@ public F getFunction() {
       dynamicFunction =
           (F) Proxy.newProxyInstance(classLoader,
               new Class[]{Types.toClass(type)},
-              (proxy, method, args) -> x.dynamicInvoke(args));
+              (proxy, method, args) ->
+                  x.dynamicInvoke(args == null ? new Object[0] : args));
     }
     return dynamicFunction;
   }
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/Primitive.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/Primitive.java
index 41c8725f0dab..c3b109bcfa98 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/Primitive.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/Primitive.java
@@ -37,6 +37,8 @@
 import java.util.List;
 import java.util.Map;
 
+import static org.apache.calcite.linq4j.Nullness.castNonNull;
+
 import static java.util.Objects.requireNonNull;
 
 /**
@@ -291,7 +293,7 @@ public static List asList(final Object array) {
     // REVIEW: A per-type list might be more efficient. (Or might not.)
     return new AbstractList() {
       @Override public Object get(int index) {
-        return Array.get(array, index);
+        return castNonNull(Array.get(array, index));
       }
 
       @Override public int size() {
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/util/Compatible.java b/linq4j/src/main/java/org/apache/calcite/linq4j/util/Compatible.java
index 9daeec2f9483..d84ea7cccf86 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/util/Compatible.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/util/Compatible.java
@@ -48,7 +48,7 @@ Compatible create() {
           new Class[]{Compatible.class},
           (proxy, method, args) -> {
             if ("isRecord".equals(method.getName())) {
-              return isRecord(requireNonNull(args[0], "args[0]"));
+              return isRecord(requireNonNull(requireNonNull(args, "args")[0], "args[0]"));
             }
             return null;
           });

From 3f789211ec09296f56d2f9e32f1457310e67d242 Mon Sep 17 00:00:00 2001
From: Vladimir Sitnikov 
Date: Mon, 24 Aug 2026 11:27:09 +0300
Subject: [PATCH 505/562] [CALCITE-7736] Remove two annotations the migration
 made untrue

`MergeUnionEnumerator.initEnumerators` carried `@RequiresNonNull("inputs")`.
`inputs` is a final field assigned in the constructor; the annotation existed
to get the Checker Framework's initialization checker past a call made from
that constructor. NullAway does its own initialization analysis, so the
annotation only made callers prove something already guaranteed, and the
suppression that silenced it goes with it.

`DefaultEnumerable.aggregate` carried `@Contract("!null, !null -> !null")`,
generated when `@PolyNull` was replaced. The original was polymorphic in the
accumulator function as well, so a non-null seed implied a non-null result. The
function may now return null, which makes the clause claim more than the method
delivers -- and NullAway, running with `CheckContracts`, said so.

Co-Authored-By: Claude Opus 5 
---
 .../main/java/org/apache/calcite/linq4j/DefaultEnumerable.java | 2 --
 .../java/org/apache/calcite/linq4j/MergeUnionEnumerator.java   | 3 ---
 2 files changed, 5 deletions(-)

diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/DefaultEnumerable.java b/linq4j/src/main/java/org/apache/calcite/linq4j/DefaultEnumerable.java
index 6854641ea159..eccd6ab38e96 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/DefaultEnumerable.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/DefaultEnumerable.java
@@ -16,7 +16,6 @@
  */
 package org.apache.calcite.linq4j;
 
-import org.apache.calcite.linq4j.annotations.Contract;
 import org.apache.calcite.linq4j.function.BigDecimalFunction1;
 import org.apache.calcite.linq4j.function.DoubleFunction1;
 import org.apache.calcite.linq4j.function.EqualityComparer;
@@ -97,7 +96,6 @@ protected OrderedQueryable asOrderedQueryable() {
     return EnumerableDefaults.aggregate(getThis(), func);
   }
 
-  @Contract("!null, !null -> !null")
   @Override public  @Nullable TAccumulate aggregate(@Nullable TAccumulate seed,
       Function2<@Nullable TAccumulate, T, @Nullable TAccumulate> func) {
     return EnumerableDefaults.aggregate(getThis(), seed, func);
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/MergeUnionEnumerator.java b/linq4j/src/main/java/org/apache/calcite/linq4j/MergeUnionEnumerator.java
index 03640662d075..fc320e96237d 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/MergeUnionEnumerator.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/MergeUnionEnumerator.java
@@ -16,7 +16,6 @@
  */
 package org.apache.calcite.linq4j;
 
-import org.apache.calcite.linq4j.annotations.RequiresNonNull;
 import org.apache.calcite.linq4j.function.EqualityComparer;
 import org.apache.calcite.linq4j.function.Function1;
 
@@ -86,8 +85,6 @@ final class MergeUnionEnumerator implements Enumerator {
     initEnumerators();
   }
 
-  @RequiresNonNull("inputs")
-  @SuppressWarnings("NullAway")
   private void initEnumerators() {
     for (int i = 0; i < inputs.length; i++) {
       moveEnumerator(i);

From 4d70b198f3805adf95f1347b962b8a91443c11b3 Mon Sep 17 00:00:00 2001
From: Vladimir Sitnikov 
Date: Mon, 24 Aug 2026 11:27:09 +0300
Subject: [PATCH 506/562] [CALCITE-7736] Suppress the two classes that only
 exist to produce null

`Linq4j.SingletonNullEnumerator` yields exactly one element and that element is
null. `Functions.Ignore` implements the function interfaces by returning null
from every `apply`. Both are meaningful only when their type argument is
instantiated nullable.

The Checker Framework could demand that: `<@Nullable E>` on a type parameter
declaration constrains the lower bound, so E had to be nullable. JSpecify tracks
upper bounds only, and `` says "may be" rather than
"must be". Neither class is public, and both carry a suppression naming the
reason.

Co-Authored-By: Claude Opus 5 
---
 linq4j/src/main/java/org/apache/calcite/linq4j/Linq4j.java       | 1 +
 .../main/java/org/apache/calcite/linq4j/function/Functions.java  | 1 +
 2 files changed, 2 insertions(+)

diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/Linq4j.java b/linq4j/src/main/java/org/apache/calcite/linq4j/Linq4j.java
index b531f73fde04..565f91803d41 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/Linq4j.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/Linq4j.java
@@ -664,6 +664,7 @@ private static class SingletonEnumerator implements Enumerator {
   /** Enumerator that returns one null element.
    *
    * @param  element type */
+  @SuppressWarnings("NullAway") // only meaningful when E is instantiated nullable
   private static class SingletonNullEnumerator
       implements Enumerator {
     int i = 0;
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/function/Functions.java b/linq4j/src/main/java/org/apache/calcite/linq4j/function/Functions.java
index 6f8ffd84cc6e..b7bfa48b9b2f 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/function/Functions.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/function/Functions.java
@@ -857,6 +857,7 @@ private static class NullsLastReverseComparator
    * @param  result type
    * @param  first argument type
    * @param  second argument type */
+  @SuppressWarnings("NullAway") // only meaningful when R is instantiated nullable
   private static final class Ignore
       implements Function0, Function1, Function2 {
     @Override public R apply() {

From 4f8ad16ba5991dea8bd7a4679873b1f6dac282bb Mon Sep 17 00:00:00 2001
From: Vladimir Sitnikov 
Date: Mon, 24 Aug 2026 11:27:09 +0300
Subject: [PATCH 507/562] [CALCITE-7736] Let the memory window hold the nulls
 it is padded with

`MemoryFactory` keeps a fixed-size window of rows for MATCH_RECOGNIZE, backed by
a `@Nullable Object[]`. `MemoryEnumerator` pads it with `add(null)` once the
input runs out, so that the last rows can still be read with their following
context, and a slot that has not been written yet reads back as null either way.

`add` takes `@Nullable E`, `Memory.get` returns it, and both type parameters
take the nullable bound.

Co-Authored-By: Claude Opus 5 
---
 .../java/org/apache/calcite/linq4j/MemoryFactory.java  | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/MemoryFactory.java b/linq4j/src/main/java/org/apache/calcite/linq4j/MemoryFactory.java
index 78b4e83229cf..f6e759e4d958 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/MemoryFactory.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/MemoryFactory.java
@@ -26,7 +26,7 @@
  *
  * @param  Type of the base Object
  */
-public class MemoryFactory {
+public class MemoryFactory {
 
   private final int history;
   private final int future;
@@ -42,7 +42,7 @@ public MemoryFactory(int history, int future) {
     this.offset = new ModularInteger(0, history + future + 1);
   }
 
-  public void add(E current) {
+  public void add(@Nullable E current) {
     values[offset.get()] = current;
     this.offset = offset.plus(1);
   }
@@ -61,7 +61,7 @@ public Memory create() {
    *
    * @param  Row type
    */
-  public static class Memory {
+  public static class Memory {
     private final int history;
     private final int future;
     private final ModularInteger offset;
@@ -79,11 +79,11 @@ public Memory(int history, int future,
       return Arrays.toString(this.values);
     }
 
-    public E get() {
+    public @Nullable E get() {
       return get(0);
     }
 
-    public E get(int position) {
+    public @Nullable E get(int position) {
       if (position < 0 && position < -1 * history) {
         throw new IllegalArgumentException("History can only go back " + history
             + " points in time, you wanted " + Math.abs(position));

From d87186fb583e3954c3823c54533cd147de503497 Mon Sep 17 00:00:00 2001
From: Vladimir Sitnikov 
Date: Mon, 24 Aug 2026 11:38:53 +0300
Subject: [PATCH 508/562] [CALCITE-7736] Let the join enumerators carry the
 null rows they emit

An outer join emits a row where one side is missing, and the merge and
correlate joins build that row from a null. The types said otherwise.

* `ExtendedEnumerable.correlateJoin` takes
  `Function2`. Its own javadoc
  already said "for semi/anti join inner argument is always null", and
  `EnumerableDefaults.correlateJoin` already declared the parameter that way.
* `CartesianProductJoinEnumerator` accepts
  `Enumerator`, which is what a left join hands it
  as the single null inner row; its result selector already took
  `@Nullable TInner`.
* `Linq4j.enumerator(Collection)` and `Linq4j.IterableEnumerator` take the
  nullable bound, so that null row can be enumerated at all.
* Merge join widens its result selector before handing it to `nestedLoopJoin`,
  which serves right and full joins too and so requires one that tolerates a
  null left row. Merge join rejects those join types up front.

`IterableEnumerator.moveNext` carries a NullAway suppression: assigning
`Iterator.next()` to a field of type `T` is reported as assigning
@Nullable to @NonNull, the same limitation the seeded `aggregate` hits.

`WrapMap.get` and `remove` take a nullable key, as `Map` requires, and wrapping
one throws. That is what the map did before it was annotated; `put` already
carried the same suppression.

With this, NullAway reports nothing in calcite-linq4j.

Co-Authored-By: Claude Opus 5 
---
 .../calcite/linq4j/DefaultEnumerable.java     |  2 +-
 .../calcite/linq4j/EnumerableDefaults.java    | 24 ++++++++++++++-----
 .../calcite/linq4j/ExtendedEnumerable.java    |  2 +-
 .../org/apache/calcite/linq4j/Linq4j.java     |  8 +++++--
 .../calcite/linq4j/QueryableRecorder.java     |  1 +
 5 files changed, 27 insertions(+), 10 deletions(-)

diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/DefaultEnumerable.java b/linq4j/src/main/java/org/apache/calcite/linq4j/DefaultEnumerable.java
index eccd6ab38e96..83b15374a997 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/DefaultEnumerable.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/DefaultEnumerable.java
@@ -472,7 +472,7 @@ protected OrderedQueryable asOrderedQueryable() {
 
   @Override public  Enumerable correlateJoin(
       JoinType joinType, Function1> inner,
-      Function2 resultSelector) {
+      Function2 resultSelector) {
     return EnumerableDefaults.correlateJoin(joinType, getThis(), inner,
         resultSelector);
   }
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java b/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java
index 71dc1e7315df..e7be128c5903 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java
@@ -1030,7 +1030,7 @@ public static  Enumerable asofJoin(
                   left = new Linq4j.IterableEnumerator<>(value);
                   List<@Nullable TInner> rightList =
                       requireNonNull(rightIndex.get(key));
-                  right = new Linq4j.IterableEnumerator<>(rightList);
+                  right = new Linq4j.IterableEnumerator<@Nullable TInner>(rightList);
                 } else {
                   // Done with the data, start emitting records with null keys
                   emittingNullKeys = true;
@@ -4923,6 +4923,9 @@ private Wrapped wrap(K key) {
       return Wrapped.upAs(comparer, key);
     }
 
+    // Map.get and Map.remove take a nullable key, and wrapping one throws, which is what
+    // this map did before it was annotated
+    @SuppressWarnings("NullAway")
     @Override public @Nullable V get(@Nullable Object key) {
       return map.get(wrap((K) key));
     }
@@ -4932,6 +4935,7 @@ private Wrapped wrap(K key) {
       return map.put(wrap(key), value);
     }
 
+    @SuppressWarnings("NullAway")
     @Override public @Nullable V remove(@Nullable Object key) {
       return map.remove(wrap((K) key));
     }
@@ -5156,7 +5160,8 @@ private boolean advance() {
               results =
                   new CartesianProductJoinEnumerator<>(resultSelector,
                       Linq4j.enumerator(lefts),
-                      Linq4j.enumerator(Collections.singletonList(null)));
+                      Linq4j.<@Nullable TInner>enumerator(
+                          Collections.singletonList(null)));
               return true;
             }
             if (!getLeftEnumerator().moveNext()) {
@@ -5211,10 +5216,17 @@ private boolean advance() {
               : new CartesianProductJoinEnumerator<>(resultSelector, Linq4j.enumerator(lefts),
                   Linq4j.enumerator(rights));
         } else {
-          // we must verify the non equi-join predicate, use nested loop join for that
+          // we must verify the non equi-join predicate, use nested loop join for that.
+          // nestedLoopJoin serves right and full joins too, so its result selector has to
+          // accept a null left row; merge join rejects those join types up front, so
+          // widening this one is sound
+          @SuppressWarnings("unchecked") final
+              Function2<@Nullable TSource, @Nullable TInner, TResult> nullTolerant =
+              (Function2<@Nullable TSource, @Nullable TInner, TResult>) resultSelector;
           results =
-              nestedLoopJoin(Linq4j.asEnumerable(lefts),
-                  Linq4j.asEnumerable(rights), extraPredicate, resultSelector,
+              EnumerableDefaults.nestedLoopJoin(
+                  Linq4j.asEnumerable(lefts),
+                  Linq4j.asEnumerable(rights), extraPredicate, nullTolerant,
                   joinType).enumerator();
         }
         return true;
@@ -5376,7 +5388,7 @@ private static class CartesianProductJoinEnumerator
 
     @SuppressWarnings("unchecked")
     CartesianProductJoinEnumerator(Function2 resultSelector,
-        Enumerator outer, Enumerator inner) {
+        Enumerator outer, Enumerator inner) {
       super(ImmutableList.of((Enumerator) outer, (Enumerator) inner));
       this.resultSelector = resultSelector;
     }
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedEnumerable.java b/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedEnumerable.java
index 8136f645ec7a..4f81e441b86a 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedEnumerable.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedEnumerable.java
@@ -748,7 +748,7 @@  Enumerable correlateLeftMarkJoin(
    */
    Enumerable correlateJoin(
       JoinType joinType, Function1> inner,
-      Function2 resultSelector);
+      Function2 resultSelector);
 
   /**
    * Returns the last element of a sequence. (Defined
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/Linq4j.java b/linq4j/src/main/java/org/apache/calcite/linq4j/Linq4j.java
index 565f91803d41..5c90204a37e6 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/Linq4j.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/Linq4j.java
@@ -189,7 +189,8 @@ public static  Enumerable asEnumerable(final T[] ts) {
    *
    * @return Enumerator over the collection
    */
-  public static  Enumerator enumerator(Collection values) {
+  public static  Enumerator enumerator(
+      Collection values) {
     if (values instanceof List && values instanceof RandomAccess) {
       //noinspection unchecked
       return listEnumerator((List) values);
@@ -430,7 +431,7 @@ private static  void closeIterator(@Nullable Iterator iterator)
    *
    * @param  element type */
   @SuppressWarnings("unchecked")
-  static class IterableEnumerator implements Enumerator {
+  static class IterableEnumerator implements Enumerator {
     private final Iterable iterable;
     @Nullable Iterator iterator;
     T current;
@@ -448,6 +449,9 @@ static class IterableEnumerator implements Enumerator {
       return current;
     }
 
+    // NullAway treats the result of `Iterator.next()` as @Nullable once it is
+    // assigned to a T field, even though T is the field's own type
+    @SuppressWarnings("NullAway")
     @Override public boolean moveNext() {
       if (requireNonNull(iterator, "iterator").hasNext()) {
         current = iterator.next();
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/QueryableRecorder.java b/linq4j/src/main/java/org/apache/calcite/linq4j/QueryableRecorder.java
index 1b164543bd38..aac25216f6c6 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/QueryableRecorder.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/QueryableRecorder.java
@@ -257,6 +257,7 @@ public static  QueryableRecorder instance() {
     }.castSingle(); // CHECKSTYLE: IGNORE 0
   }
 
+  @SuppressWarnings("NullAway")
   @Override public Queryable<@Nullable T> defaultIfEmpty(final Queryable source) {
     return new NonLeafReplayableQueryable(source) {
       @Override public void replay(QueryableFactory factory) {

From 2f6f2e46109d2def3933f4e2b303f9b72669626e Mon Sep 17 00:00:00 2001
From: Vladimir Sitnikov 
Date: Mon, 24 Aug 2026 12:06:17 +0300
Subject: [PATCH 509/562] [CALCITE-7736] Suppress the three do-nothing visitors

`RexVisitorImpl`, `RexBiVisitorImpl` and `SqlBasicVisitor` are the traversals
that subclasses extend and override where they care; every method they define
returns null. That is meaningful only when the result type is instantiated
nullable, which JSpecify cannot require: it tracks upper bounds, so
`` says "may be" rather than "must be". The Checker
Framework said it with `<@Nullable R>`, which constrains the lower bound.

Same treatment as `Linq4j.SingletonNullEnumerator` and `Functions.Ignore`.

Co-Authored-By: Claude Opus 5 
---
 .../main/java/org/apache/calcite/rex/RexBiVisitorImpl.java   | 5 +++++
 .../src/main/java/org/apache/calcite/rex/RexVisitorImpl.java | 5 +++++
 .../java/org/apache/calcite/sql/util/SqlBasicVisitor.java    | 5 +++++
 3 files changed, 15 insertions(+)

diff --git a/core/src/main/java/org/apache/calcite/rex/RexBiVisitorImpl.java b/core/src/main/java/org/apache/calcite/rex/RexBiVisitorImpl.java
index 1ae39484bfc9..fa8e3a2db94f 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexBiVisitorImpl.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexBiVisitorImpl.java
@@ -25,6 +25,11 @@
  * @param  Return type from each {@code visitXxx} method
  * @param 
       Payload type
  */
+
+// Every visitXxx returns null: this is the do-nothing traversal that subclasses override
+// where they care. It is meaningful only when R is instantiated nullable, and JSpecify
+// tracks upper bounds, so it cannot require that.
+@SuppressWarnings("NullAway")
 public class RexBiVisitorImpl implements RexBiVisitor {
   //~ Instance fields --------------------------------------------------------
 
diff --git a/core/src/main/java/org/apache/calcite/rex/RexVisitorImpl.java b/core/src/main/java/org/apache/calcite/rex/RexVisitorImpl.java
index 2bfb5972b66a..003b816c05a0 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexVisitorImpl.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexVisitorImpl.java
@@ -26,6 +26,11 @@
  *
  * @param  Return type from each {@code visitXxx} method.
  */
+
+// Every visitXxx returns null: this is the do-nothing traversal that subclasses override
+// where they care. It is meaningful only when R is instantiated nullable, and JSpecify
+// tracks upper bounds, so it cannot require that.
+@SuppressWarnings("NullAway")
 public class RexVisitorImpl implements RexVisitor {
   //~ Instance fields --------------------------------------------------------
 
diff --git a/core/src/main/java/org/apache/calcite/sql/util/SqlBasicVisitor.java b/core/src/main/java/org/apache/calcite/sql/util/SqlBasicVisitor.java
index 8156b286e2e8..356f0e254fdc 100644
--- a/core/src/main/java/org/apache/calcite/sql/util/SqlBasicVisitor.java
+++ b/core/src/main/java/org/apache/calcite/sql/util/SqlBasicVisitor.java
@@ -36,6 +36,11 @@
  *
  * @param  Return type
  */
+
+// Every visitXxx returns null: this is the do-nothing traversal that subclasses override
+// where they care. It is meaningful only when R is instantiated nullable, and JSpecify
+// tracks upper bounds, so it cannot require that.
+@SuppressWarnings("NullAway")
 public class SqlBasicVisitor implements SqlVisitor {
   //~ Methods ----------------------------------------------------------------
 

From 467bccb425bdc46adafd27d12c926111b76405b7 Mon Sep 17 00:00:00 2001
From: Vladimir Sitnikov 
Date: Mon, 24 Aug 2026 12:06:17 +0300
Subject: [PATCH 510/562] [CALCITE-7736] Instantiate the result-ignoring
 visitors with @Nullable Void

A visitor that computes nothing was written `RexVisitorImpl`. `Void` has
null as its only value, so a non-null `Void` is a type nothing inhabits, and the
traversal returns exactly the null it rules out.

68 sites across `RexVisitorImpl`, `RexVisitor`, `SqlBasicVisitor`, `SqlVisitor`
and `RexBiVisitorImpl` now say `@Nullable Void`, including the parameters and
fields that hold such a visitor.

Co-Authored-By: Claude Opus 5 
---
 .idea/vcs.xml                                 |  4 +-
 .../adapter/enumerable/EnumerableMatch.java   |  2 +-
 .../calcite/adapter/jdbc/JdbcRules.java       |  2 +-
 .../org/apache/calcite/plan/RelOptUtil.java   |  4 +-
 .../calcite/plan/RexImplicationChecker.java   |  2 +-
 .../java/org/apache/calcite/plan/Strong.java  |  4 +-
 .../calcite/plan/SubstitutionVisitor.java     |  4 +-
 .../org/apache/calcite/rel/core/Match.java    |  4 +-
 .../rel/metadata/RelMdColumnOrigins.java      |  4 +-
 .../calcite/rel/metadata/RelMdPredicates.java |  2 +-
 .../calcite/rel/rel2sql/SqlImplementor.java   |  2 +-
 .../calcite/rel/rules/CalcRelSplitter.java    |  8 ++--
 .../calcite/rel/rules/DateRangeRules.java     |  2 +-
 .../rel/rules/JoinExpandOrToUnionRule.java    |  3 +-
 .../rel/rules/JoinToMultiJoinRule.java        |  2 +-
 .../rel/rules/ProjectAggregateMergeRule.java  |  3 +-
 .../rel/rules/ProjectTableScanRule.java       |  3 +-
 .../rel/rules/ProjectToWindowRule.java        |  3 +-
 .../calcite/rel/rules/PushProjector.java      |  2 +-
 .../rel/rules/ReduceExpressionsRule.java      |  2 +-
 .../org/apache/calcite/rex/RexAnalyzer.java   |  4 +-
 .../apache/calcite/rex/RexMultisetUtil.java   |  2 +-
 .../java/org/apache/calcite/rex/RexOver.java  |  2 +-
 .../org/apache/calcite/rex/RexProgram.java    |  4 +-
 .../apache/calcite/rex/RexProgramBuilder.java |  4 +-
 .../org/apache/calcite/rex/RexSimplify.java   |  2 +-
 .../java/org/apache/calcite/rex/RexUtil.java  | 42 +++++++++----------
 .../java/org/apache/calcite/sql/SqlPivot.java |  2 +-
 .../org/apache/calcite/sql/SqlUnpivot.java    |  2 +-
 .../java/org/apache/calcite/sql/SqlUtil.java  |  6 +--
 .../calcite/sql/fun/SqlBetweenOperator.java   |  4 +-
 .../apache/calcite/sql/type/OperandTypes.java |  2 +-
 .../calcite/sql/validate/AggChecker.java      |  4 +-
 .../calcite/sql/validate/AggVisitor.java      |  2 +-
 .../sql/validate/SqlValidatorImpl.java        | 16 +++----
 .../apache/calcite/sql2rel/AggConverter.java  |  2 +-
 .../calcite/sql2rel/RelDecorrelator.java      |  6 +--
 .../calcite/sql2rel/SqlToRelConverter.java    |  2 +-
 .../java/org/apache/calcite/util/Util.java    |  2 +-
 .../file/CsvProjectFilterTableScanRule.java   |  5 ++-
 .../calcite/piglet/PigToSqlAggregateRule.java |  3 +-
 41 files changed, 97 insertions(+), 83 deletions(-)

diff --git a/.idea/vcs.xml b/.idea/vcs.xml
index 2f26af79ee0b..ed4157c5bc70 100644
--- a/.idea/vcs.xml
+++ b/.idea/vcs.xml
@@ -22,6 +22,6 @@
     
   
   
-    
+    
   
-
+
\ No newline at end of file
diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMatch.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMatch.java
index 1287d088d946..0989f486a2ec 100644
--- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMatch.java
+++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMatch.java
@@ -422,7 +422,7 @@ private static Expression implementPattern(Expression patternBuilder_,
   /**
    * Visitor that finds out how much "history" we need in the past and future.
    */
-  private static class MaxHistoryFutureVisitor extends RexVisitorImpl {
+  private static class MaxHistoryFutureVisitor extends RexVisitorImpl<@Nullable Void> {
     private int history;
     private int future;
 
diff --git a/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcRules.java b/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcRules.java
index a4f5a0d17e7b..a1b2cdf691bf 100644
--- a/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcRules.java
+++ b/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcRules.java
@@ -1104,7 +1104,7 @@ public static class JdbcValues extends Values implements JdbcRel {
   /** Visitor that checks whether part of a projection is a user-defined
    * function (UDF). */
   private static class CheckingUserDefinedFunctionVisitor
-      extends RexVisitorImpl {
+      extends RexVisitorImpl<@Nullable Void> {
 
     private boolean containsUsedDefinedFunction = false;
 
diff --git a/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java b/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java
index 815aae5a31fe..baec2a0c88c3 100644
--- a/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java
+++ b/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java
@@ -4056,7 +4056,7 @@ private static RelNode pushDownJoinConditions(Join originalJoin,
   private static boolean containsGet(RexNode node) {
     try {
       node.accept(
-          new RexVisitorImpl(true) {
+          new RexVisitorImpl<@Nullable Void>(true) {
             @Override public Void visitCall(RexCall call) {
               if (call.getOperator() == RexBuilder.GET_OPERATOR) {
                 throw Util.FoundOne.NULL;
@@ -4704,7 +4704,7 @@ public RexCorrelVariableMapShuttle(final CorrelationId correlationId,
   /**
    * Visitor which builds a bitmap of the inputs used by an expression.
    */
-  public static class InputFinder extends RexVisitorImpl {
+  public static class InputFinder extends RexVisitorImpl<@Nullable Void> {
     private final ImmutableBitSet.Builder bitBuilder;
     private final @Nullable Set extraFields;
     /** Correlation ids whose binder is the current scope. When non-null,
diff --git a/core/src/main/java/org/apache/calcite/plan/RexImplicationChecker.java b/core/src/main/java/org/apache/calcite/plan/RexImplicationChecker.java
index 36dd368ec01e..08ca75dfe503 100644
--- a/core/src/main/java/org/apache/calcite/plan/RexImplicationChecker.java
+++ b/core/src/main/java/org/apache/calcite/plan/RexImplicationChecker.java
@@ -441,7 +441,7 @@ private static boolean validate(RexNode first, RexNode second) {
    * 
    1. key: y value: {(>, 20), usageCount = 1}
    * 
    */
-  private static class InputUsageFinder extends RexVisitorImpl {
+  private static class InputUsageFinder extends RexVisitorImpl<@Nullable Void> {
     final Map> usageMap =
         new HashMap<>();
 
diff --git a/core/src/main/java/org/apache/calcite/plan/Strong.java b/core/src/main/java/org/apache/calcite/plan/Strong.java
index b6aa9678ed8d..ea9d7d718b91 100644
--- a/core/src/main/java/org/apache/calcite/plan/Strong.java
+++ b/core/src/main/java/org/apache/calcite/plan/Strong.java
@@ -35,6 +35,8 @@
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterables;
 
+import org.jspecify.annotations.Nullable;
+
 import java.util.ArrayList;
 import java.util.EnumMap;
 import java.util.List;
@@ -151,7 +153,7 @@ public static Policy policy(SqlOperator operator) {
   public static boolean isStrong(RexNode e) {
     final ImmutableBitSet.Builder nullColumns = ImmutableBitSet.builder();
     e.accept(
-        new RexVisitorImpl(true) {
+        new RexVisitorImpl<@Nullable Void>(true) {
           @Override public Void visitInputRef(RexInputRef inputRef) {
             nullColumns.set(inputRef.getIndex());
             return super.visitInputRef(inputRef);
diff --git a/core/src/main/java/org/apache/calcite/plan/SubstitutionVisitor.java b/core/src/main/java/org/apache/calcite/plan/SubstitutionVisitor.java
index 402f872e3b98..db746bc797a9 100644
--- a/core/src/main/java/org/apache/calcite/plan/SubstitutionVisitor.java
+++ b/core/src/main/java/org/apache/calcite/plan/SubstitutionVisitor.java
@@ -1483,7 +1483,7 @@ private AggregateOnCalcToAggregateUnifyRule() {
         try {
           // Fail the matching when filtering condition references
           // non-grouping columns in target.
-          qInputCond.accept(new RexVisitorImpl(true) {
+          qInputCond.accept(new RexVisitorImpl<@Nullable Void>(true) {
             @Override public Void visitInputRef(RexInputRef inputRef) {
               if (!target.groupSets.stream()
                   .allMatch(groupSet -> groupSet.get(inputRef.getIndex()))) {
@@ -1839,7 +1839,7 @@ private static boolean referenceByMapping(
     }
 
     try {
-      RexVisitor rexVisitor = new RexVisitorImpl(true) {
+      RexVisitor rexVisitor = new RexVisitorImpl<@Nullable Void>(true) {
         @Override public Void visitInputRef(RexInputRef inputRef) {
           if (!(projects.get(inputRef.getIndex()) instanceof RexInputRef)) {
             throw Util.FoundOne.NULL;
diff --git a/core/src/main/java/org/apache/calcite/rel/core/Match.java b/core/src/main/java/org/apache/calcite/rel/core/Match.java
index 09bfd9a1765e..27a6295eb9d7 100644
--- a/core/src/main/java/org/apache/calcite/rel/core/Match.java
+++ b/core/src/main/java/org/apache/calcite/rel/core/Match.java
@@ -214,7 +214,7 @@ public RelCollation getOrderKeys() {
   /**
    * Find aggregate functions in operands.
    */
-  private static class AggregateFinder extends RexVisitorImpl {
+  private static class AggregateFinder extends RexVisitorImpl<@Nullable Void> {
     final NavigableSet aggregateCalls = new TreeSet<>();
     final Map> aggregateCallsPerVar =
         new TreeMap<>();
@@ -291,7 +291,7 @@ public void go(RexCall call) {
    * Visits the operands of an aggregate call to retrieve relevant pattern
    * variables.
    */
-  private static class PatternVarFinder extends RexVisitorImpl {
+  private static class PatternVarFinder extends RexVisitorImpl<@Nullable Void> {
     final Set patternVars = new HashSet<>();
 
     PatternVarFinder() {
diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdColumnOrigins.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdColumnOrigins.java
index efe93d1e039c..6eda2bf7f742 100644
--- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdColumnOrigins.java
+++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdColumnOrigins.java
@@ -298,8 +298,8 @@ private RelMdColumnOrigins() {}
   private static @Nullable Set getMultipleColumns(RexNode rexNode, RelNode input,
       final RelMetadataQuery mq) {
     final Set set = new HashSet<>();
-    final RexVisitor visitor =
-        new RexVisitorImpl(true) {
+    final RexVisitor<@Nullable Void> visitor =
+        new RexVisitorImpl<@Nullable Void>(true) {
           @Override public Void visitInputRef(RexInputRef inputRef) {
             Set inputSet =
                 mq.getColumnOrigins(input, inputRef.getIndex());
diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdPredicates.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdPredicates.java
index b2330bff7718..125946ee6263 100644
--- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdPredicates.java
+++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdPredicates.java
@@ -971,7 +971,7 @@ private void markAsEquivalent(int p1, int p2) {
     /**
      * Find expressions of the form 'col_x = col_y'.
      */
-    class EquivalenceFinder extends RexVisitorImpl {
+    class EquivalenceFinder extends RexVisitorImpl<@Nullable Void> {
       protected EquivalenceFinder() {
         super(true);
       }
diff --git a/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java b/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java
index 794bf84e054e..d97e14a1fb89 100644
--- a/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java
+++ b/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java
@@ -2476,7 +2476,7 @@ private boolean containsOver(
         return false;
       }
       final boolean[] result = {false};
-      node.accept(new SqlBasicVisitor() {
+      node.accept(new SqlBasicVisitor<@Nullable Void>() {
         @Override public Void visit(SqlCall call) {
           if (result[0]) {
             return null;
diff --git a/core/src/main/java/org/apache/calcite/rel/rules/CalcRelSplitter.java b/core/src/main/java/org/apache/calcite/rel/rules/CalcRelSplitter.java
index f054c66b8ced..cfd843e0c530 100644
--- a/core/src/main/java/org/apache/calcite/rel/rules/CalcRelSplitter.java
+++ b/core/src/main/java/org/apache/calcite/rel/rules/CalcRelSplitter.java
@@ -467,7 +467,7 @@ private static List computeTopologicalOrdering(
         targets = cohort;
       }
       expr.accept(
-          new RexVisitorImpl(true) {
+          new RexVisitorImpl<@Nullable Void>(true) {
             @Override public Void visitLocalRef(RexLocalRef localRef) {
               for (Integer target : targets) {
                 graph.addEdge(localRef.getIndex(), target);
@@ -899,7 +899,7 @@ public boolean canImplement(RexProgram program) {
    * Visitor which returns whether an expression can be implemented in a given
    * type of relational expression.
    */
-  private static class ImplementTester extends RexVisitorImpl {
+  private static class ImplementTester extends RexVisitorImpl<@Nullable Void> {
     private final RelType relType;
 
     ImplementTester(RelType relType) {
@@ -1005,7 +1005,7 @@ private static class InputToCommonExprConverter extends RexShuttle {
   /**
    * Finds the highest level used by any of the inputs of a given expression.
    */
-  private static class MaxInputFinder extends RexVisitorImpl {
+  private static class MaxInputFinder extends RexVisitorImpl<@Nullable Void> {
     int level;
     private final int[] exprLevels;
 
@@ -1034,7 +1034,7 @@ public int maxInputFor(RexNode expr) {
    * Builds an array of the highest level which contains an expression which
    * uses each expression as an input.
    */
-  private static class HighestUsageFinder extends RexVisitorImpl {
+  private static class HighestUsageFinder extends RexVisitorImpl<@Nullable Void> {
     private final int[] maxUsingLevelOrdinals;
     private int currentLevel;
 
diff --git a/core/src/main/java/org/apache/calcite/rel/rules/DateRangeRules.java b/core/src/main/java/org/apache/calcite/rel/rules/DateRangeRules.java
index d25b579de7f0..4ce72c42756f 100644
--- a/core/src/main/java/org/apache/calcite/rel/rules/DateRangeRules.java
+++ b/core/src/main/java/org/apache/calcite/rel/rules/DateRangeRules.java
@@ -234,7 +234,7 @@ public interface FilterDateRangeRuleConfig extends RelRule.Config {
 
   /** Visitor that searches for calls to {@code EXTRACT}, {@code FLOOR} or
    * {@code CEIL}, building a list of distinct time units. */
-  private static class ExtractFinder extends RexVisitorImpl
+  private static class ExtractFinder extends RexVisitorImpl<@Nullable Void>
       implements AutoCloseable {
     private final Set timeUnits =
         EnumSet.noneOf(TimeUnitRange.class);
diff --git a/core/src/main/java/org/apache/calcite/rel/rules/JoinExpandOrToUnionRule.java b/core/src/main/java/org/apache/calcite/rel/rules/JoinExpandOrToUnionRule.java
index 67e061d5f7cb..3c3a5abda656 100644
--- a/core/src/main/java/org/apache/calcite/rel/rules/JoinExpandOrToUnionRule.java
+++ b/core/src/main/java/org/apache/calcite/rel/rules/JoinExpandOrToUnionRule.java
@@ -33,6 +33,7 @@
 import org.apache.calcite.tools.RelBuilder;
 
 import org.immutables.value.Value;
+import org.jspecify.annotations.Nullable;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -207,7 +208,7 @@ private static boolean doesNotReferToBothInputs(RexNode rex, int leftFieldCount)
 
   /**
    * Counts the number of InputRefs in a RexNode expression. */
-  private static class RexInputRefCounter extends RexVisitorImpl {
+  private static class RexInputRefCounter extends RexVisitorImpl<@Nullable Void> {
     private final int leftFieldCount;
     private int leftInputRefCount = 0;
     private int rightInputRefCount = 0;
diff --git a/core/src/main/java/org/apache/calcite/rel/rules/JoinToMultiJoinRule.java b/core/src/main/java/org/apache/calcite/rel/rules/JoinToMultiJoinRule.java
index ec34f6e619d1..826a924207d9 100644
--- a/core/src/main/java/org/apache/calcite/rel/rules/JoinToMultiJoinRule.java
+++ b/core/src/main/java/org/apache/calcite/rel/rules/JoinToMultiJoinRule.java
@@ -560,7 +560,7 @@ private static ImmutableMap addOnJoinFieldRefCounts(
   /**
    * Visitor that keeps a reference count of the inputs used by an expression.
    */
-  private static class InputReferenceCounter extends RexVisitorImpl {
+  private static class InputReferenceCounter extends RexVisitorImpl<@Nullable Void> {
     private final int[] refCounts;
 
     InputReferenceCounter(int[] refCounts) {
diff --git a/core/src/main/java/org/apache/calcite/rel/rules/ProjectAggregateMergeRule.java b/core/src/main/java/org/apache/calcite/rel/rules/ProjectAggregateMergeRule.java
index 815f275d1265..d2a7fe7326cc 100644
--- a/core/src/main/java/org/apache/calcite/rel/rules/ProjectAggregateMergeRule.java
+++ b/core/src/main/java/org/apache/calcite/rel/rules/ProjectAggregateMergeRule.java
@@ -39,6 +39,7 @@
 import org.apache.calcite.util.mapping.Mappings;
 
 import org.immutables.value.Value;
+import org.jspecify.annotations.Nullable;
 
 import java.math.BigDecimal;
 import java.util.ArrayList;
@@ -182,7 +183,7 @@ private static int findSum0(RelDataTypeFactory typeFactory, AggregateCall sum,
   private static int kindCount(Iterable nodes,
       final SqlKind kind) {
     final AtomicInteger kindCount = new AtomicInteger(0);
-    new RexVisitorImpl(true) {
+    new RexVisitorImpl<@Nullable Void>(true) {
       @Override public Void visitCall(RexCall call) {
         if (call.getKind() == kind) {
           kindCount.incrementAndGet();
diff --git a/core/src/main/java/org/apache/calcite/rel/rules/ProjectTableScanRule.java b/core/src/main/java/org/apache/calcite/rel/rules/ProjectTableScanRule.java
index 770860cc4a63..0f7d5063de0d 100644
--- a/core/src/main/java/org/apache/calcite/rel/rules/ProjectTableScanRule.java
+++ b/core/src/main/java/org/apache/calcite/rel/rules/ProjectTableScanRule.java
@@ -36,6 +36,7 @@
 import com.google.common.collect.ImmutableList;
 
 import org.immutables.value.Value;
+import org.jspecify.annotations.Nullable;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -107,7 +108,7 @@ protected void apply(RelOptRuleCall call, Project project, TableScan scan) {
     requireNonNull(table.unwrap(ProjectableFilterableTable.class));
 
     final List selectedColumns = new ArrayList<>();
-    final RexVisitorImpl visitor = new RexVisitorImpl(true) {
+    final RexVisitorImpl<@Nullable Void> visitor = new RexVisitorImpl<@Nullable Void>(true) {
       @Override public Void visitInputRef(RexInputRef inputRef) {
         if (!selectedColumns.contains(inputRef.getIndex())) {
           selectedColumns.add(inputRef.getIndex());
diff --git a/core/src/main/java/org/apache/calcite/rel/rules/ProjectToWindowRule.java b/core/src/main/java/org/apache/calcite/rel/rules/ProjectToWindowRule.java
index 14a333aace48..e1cb82703023 100644
--- a/core/src/main/java/org/apache/calcite/rel/rules/ProjectToWindowRule.java
+++ b/core/src/main/java/org/apache/calcite/rel/rules/ProjectToWindowRule.java
@@ -51,6 +51,7 @@
 import com.google.common.collect.ImmutableList;
 
 import org.immutables.value.Value;
+import org.jspecify.annotations.Nullable;
 
 import java.util.ArrayDeque;
 import java.util.ArrayList;
@@ -381,7 +382,7 @@ private static DirectedGraph createGraphFromExpression(
         graph.addVertex(i);
       }
 
-      new RexBiVisitorImpl(true) {
+      new RexBiVisitorImpl<@Nullable Void, Integer>(true) {
         @Override public Void visitLocalRef(RexLocalRef localRef, Integer i) {
           graph.addEdge(localRef.getIndex(), i);
           return null;
diff --git a/core/src/main/java/org/apache/calcite/rel/rules/PushProjector.java b/core/src/main/java/org/apache/calcite/rel/rules/PushProjector.java
index 2bbcb54235b5..004429836435 100644
--- a/core/src/main/java/org/apache/calcite/rel/rules/PushProjector.java
+++ b/core/src/main/java/org/apache/calcite/rel/rules/PushProjector.java
@@ -688,7 +688,7 @@ public RelNode createNewProject(RelNode projChild, int[] adjustments) {
    * Visitor which builds a bitmap of the inputs used by an expressions, as
    * well as locating expressions corresponding to special operators.
    */
-  private static class InputSpecialOpFinder extends RexVisitorImpl {
+  private static class InputSpecialOpFinder extends RexVisitorImpl<@Nullable Void> {
     private final BitSet rexRefs;
     private final ImmutableBitSet leftFields;
     private final @Nullable ImmutableBitSet rightFields;
diff --git a/core/src/main/java/org/apache/calcite/rel/rules/ReduceExpressionsRule.java b/core/src/main/java/org/apache/calcite/rel/rules/ReduceExpressionsRule.java
index 3fc1a6fcd614..444295e3128e 100644
--- a/core/src/main/java/org/apache/calcite/rel/rules/ReduceExpressionsRule.java
+++ b/core/src/main/java/org/apache/calcite/rel/rules/ReduceExpressionsRule.java
@@ -982,7 +982,7 @@ protected static class RexReplacer extends RexShuttle {
    * Helper class used to locate expressions that either can be reduced to
    * literals or contain redundant casts.
    */
-  protected static class ReducibleExprLocator extends RexVisitorImpl {
+  protected static class ReducibleExprLocator extends RexVisitorImpl<@Nullable Void> {
     /** Whether an expression is constant, and if so, whether it can be
      * reduced to a simpler constant. */
     enum Constancy {
diff --git a/core/src/main/java/org/apache/calcite/rex/RexAnalyzer.java b/core/src/main/java/org/apache/calcite/rex/RexAnalyzer.java
index 5ca25da26555..ce426e511b49 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexAnalyzer.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexAnalyzer.java
@@ -27,6 +27,8 @@
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 
+import org.jspecify.annotations.Nullable;
+
 import java.math.BigDecimal;
 import java.util.LinkedHashSet;
 import java.util.List;
@@ -114,7 +116,7 @@ private static List getComparables(RexNode variable) {
   /** Collects the variables (or other bindable sites) in an expression, and
    * counts features (such as CAST) that {@link RexInterpreter} cannot
    * handle. */
-  private static class VariableCollector extends RexVisitorImpl {
+  private static class VariableCollector extends RexVisitorImpl<@Nullable Void> {
     private final Set builder = new LinkedHashSet<>();
     private int unsupportedCount = 0;
 
diff --git a/core/src/main/java/org/apache/calcite/rex/RexMultisetUtil.java b/core/src/main/java/org/apache/calcite/rex/RexMultisetUtil.java
index 47252a1cde5a..c6d399fb6888 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexMultisetUtil.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexMultisetUtil.java
@@ -177,7 +177,7 @@ public static boolean isMultisetCast(RexCall call) {
    * 
      totalCount ≥ multisetCount always holds true.
    */
   private static class RexCallMultisetOperatorCounter
-      extends RexVisitorImpl {
+      extends RexVisitorImpl<@Nullable Void> {
     int totalCount = 0;
     int multisetCount = 0;
 
diff --git a/core/src/main/java/org/apache/calcite/rex/RexOver.java b/core/src/main/java/org/apache/calcite/rex/RexOver.java
index 89721bc84582..c8f2568ab0e9 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexOver.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexOver.java
@@ -222,7 +222,7 @@ private static class OverFound extends ControlFlowException {
    * 
      It is re-entrant (two threads can use an instance at the same time)
    * and it can be re-used for multiple visits.
    */
-  private static class Finder extends RexVisitorImpl {
+  private static class Finder extends RexVisitorImpl<@Nullable Void> {
     Finder() {
       super(true);
     }
diff --git a/core/src/main/java/org/apache/calcite/rex/RexProgram.java b/core/src/main/java/org/apache/calcite/rex/RexProgram.java
index f90ab5010583..0f952a1bcb8d 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexProgram.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexProgram.java
@@ -770,7 +770,7 @@ public boolean isPermutation() {
   public Set getCorrelVariableNames() {
     final Set paramIdSet = new HashSet<>();
     RexUtil.apply(
-        new RexVisitorImpl(true) {
+        new RexVisitorImpl<@Nullable Void>(true) {
           @Override public Void visitCorrelVariable(
               RexCorrelVariable correlVariable) {
             paramIdSet.add(correlVariable.getName());
@@ -995,7 +995,7 @@ private class Marshaller extends RexVisitorImpl<@Nullable RexNode> {
   /**
    * Visitor which marks which expressions are used.
    */
-  private static class ReferenceCounter extends RexVisitorImpl {
+  private static class ReferenceCounter extends RexVisitorImpl<@Nullable Void> {
     private final int[] refCounts;
 
     ReferenceCounter(int[] refCounts) {
diff --git a/core/src/main/java/org/apache/calcite/rex/RexProgramBuilder.java b/core/src/main/java/org/apache/calcite/rex/RexProgramBuilder.java
index 74b109cfeb05..98ce0d41cbfd 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexProgramBuilder.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexProgramBuilder.java
@@ -166,8 +166,8 @@ private static boolean assertionsAreEnabled() {
   }
 
   private void validate(final RexNode expr, final int fieldOrdinal) {
-    final RexVisitor validator =
-        new RexVisitorImpl(true) {
+    final RexVisitor<@Nullable Void> validator =
+        new RexVisitorImpl<@Nullable Void>(true) {
           @Override public Void visitInputRef(RexInputRef input) {
             final int index = input.getIndex();
             final List fields =
diff --git a/core/src/main/java/org/apache/calcite/rex/RexSimplify.java b/core/src/main/java/org/apache/calcite/rex/RexSimplify.java
index a9cc860a01a8..5c7fb733adb0 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexSimplify.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexSimplify.java
@@ -3130,7 +3130,7 @@ default boolean allowedInOr(RelOptPredicateList predicates) {
   /**
    * Visitor which finds all inputs used by an expressions.
    */
-  private static class VariableCollector extends RexVisitorImpl {
+  private static class VariableCollector extends RexVisitorImpl<@Nullable Void> {
     private final Set refs = new HashSet<>();
 
     VariableCollector() {
diff --git a/core/src/main/java/org/apache/calcite/rex/RexUtil.java b/core/src/main/java/org/apache/calcite/rex/RexUtil.java
index b262194ca624..ed6ba5ef9c5f 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexUtil.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexUtil.java
@@ -828,8 +828,8 @@ public static boolean isConstant(RexNode node) {
    */
   public static boolean isDeterministic(RexNode e) {
     try {
-      RexVisitor visitor =
-          new RexVisitorImpl(true) {
+      RexVisitor<@Nullable Void> visitor =
+          new RexVisitorImpl<@Nullable Void>(true) {
             @Override public Void visitCall(RexCall call) {
               if (!call.getOperator().isDeterministic()) {
                 throw Util.FoundOne.NULL;
@@ -849,7 +849,7 @@ public static boolean isDeterministic(RexNode e) {
   public static boolean containsDynamicFunction(RexNode e) {
     try {
       e.accept(
-          new RexVisitorImpl(true) {
+          new RexVisitorImpl<@Nullable Void>(true) {
             @Override public Void visitCall(RexCall call) {
               if (call.getOperator().isDynamicFunction()) {
                 throw Util.FoundOne.NULL;
@@ -868,7 +868,7 @@ public static boolean containsDynamicFunction(RexNode e) {
   public static boolean containsDynamicParam(RexNode e) {
     try {
       e.accept(
-          new RexVisitorImpl(true) {
+          new RexVisitorImpl<@Nullable Void>(true) {
             @Override public Void visitDynamicParam(RexDynamicParam dynamicParam) {
               throw Util.FoundOne.NULL;
             }
@@ -989,8 +989,8 @@ public static List retainDeterministic(List list) {
       final SqlOperator operator,
       RexNode node) {
     try {
-      RexVisitor visitor =
-          new RexVisitorImpl(true) {
+      RexVisitor<@Nullable Void> visitor =
+          new RexVisitorImpl<@Nullable Void>(true) {
             @Override public Void visitCall(RexCall call) {
               if (call.getOperator().equals(operator)) {
                 throw new Util.FoundOne(call);
@@ -1014,8 +1014,8 @@ public static List retainDeterministic(List list) {
   public static boolean containsInputRef(
       RexNode node) {
     try {
-      RexVisitor visitor =
-          new RexVisitorImpl(true) {
+      RexVisitor<@Nullable Void> visitor =
+          new RexVisitorImpl<@Nullable Void>(true) {
             @Override public Void visitInputRef(RexInputRef inputRef) {
               throw new Util.FoundOne(inputRef);
             }
@@ -1036,8 +1036,8 @@ public static boolean containsInputRef(
    */
   public static boolean containsFieldAccess(RexNode node) {
     try {
-      RexVisitor visitor =
-          new RexVisitorImpl(true) {
+      RexVisitor<@Nullable Void> visitor =
+          new RexVisitorImpl<@Nullable Void>(true) {
             @Override public Void visitFieldAccess(RexFieldAccess fieldAccess) {
               throw new Util.FoundOne(fieldAccess);
             }
@@ -1286,8 +1286,8 @@ public static boolean containsTableInputRef(List nodes) {
    */
   public static @Nullable RexTableInputRef containsTableInputRef(RexNode node) {
     try {
-      RexVisitor visitor =
-          new RexVisitorImpl(true) {
+      RexVisitor<@Nullable Void> visitor =
+          new RexVisitorImpl<@Nullable Void>(true) {
             @Override public Void visitTableInputRef(RexTableInputRef inputRef) {
               throw new Util.FoundOne(inputRef);
             }
@@ -1744,7 +1744,7 @@ public static  T[] apply(
    * @param expr    Single expression, may be null
    */
   public static void apply(
-      RexVisitor visitor,
+      RexVisitor<@Nullable Void> visitor,
       RexNode[] exprs,
       @Nullable RexNode expr) {
     for (RexNode e : exprs) {
@@ -1764,7 +1764,7 @@ public static void apply(
    * @param expr    Single expression, may be null
    */
   public static void apply(
-      RexVisitor visitor,
+      RexVisitor<@Nullable Void> visitor,
       List exprs,
       @Nullable RexNode expr) {
     for (RexNode e : exprs) {
@@ -2673,7 +2673,7 @@ public static RexNode swapColumnTableReferences(final RexBuilder rexBuilder,
    */
   public static Set gatherTableReferences(final List nodes) {
     final Set occurrences = new HashSet<>();
-    new RexVisitorImpl(true) {
+    new RexVisitorImpl<@Nullable Void>(true) {
       @Override public Void visitTableInputRef(RexTableInputRef ref) {
         occurrences.add(ref.getTableRef());
         return super.visitTableInputRef(ref);
@@ -2802,7 +2802,7 @@ private static class SubExprExistsException extends ControlFlowException {
    * input row type, or a {@link RexLocalRef} with ordinal greater than that set
    * using {@link #setLimit(int)}.
    */
-  private static class ForwardRefFinder extends RexVisitorImpl {
+  private static class ForwardRefFinder extends RexVisitorImpl<@Nullable Void> {
     private int limit = -1;
     private final RelDataType inputRowType;
 
@@ -2840,7 +2840,7 @@ static class IllegalForwardRefException extends ControlFlowException {
   /**
    * Visitor which builds a bitmap of the inputs used by an expression.
    */
-  public static class FieldAccessFinder extends RexVisitorImpl {
+  public static class FieldAccessFinder extends RexVisitorImpl<@Nullable Void> {
     private final List fieldAccessList;
 
     public FieldAccessFinder() {
@@ -3281,7 +3281,7 @@ private static class RexShiftShuttle extends RexShuttle {
 
   /** Visitor that throws {@link org.apache.calcite.util.Util.FoundOne} if
    * applied to an expression that contains a {@link RexCorrelVariable}. */
-  private static class CorrelationFinder extends RexVisitorImpl {
+  private static class CorrelationFinder extends RexVisitorImpl<@Nullable Void> {
     static final CorrelationFinder INSTANCE = new CorrelationFinder(null);
 
     /** Optional filter: when non-null, only correlation ids in this set
@@ -3360,7 +3360,7 @@ public FixNullabilityShuttle(RexBuilder rexBuilder,
 
   /** Visitor that collects all the top level SubQueries {@link RexSubQuery}
    *  in a projection list of a given {@link Project}.*/
-  public static class SubQueryCollector extends RexVisitorImpl {
+  public static class SubQueryCollector extends RexVisitorImpl<@Nullable Void> {
     private final List subQueries;
     private SubQueryCollector() {
       super(true);
@@ -3383,7 +3383,7 @@ public static List collect(Project project) {
 
   /** Visitor that throws {@link org.apache.calcite.util.Util.FoundOne} if
    * applied to an expression that contains a {@link RexSubQuery}. */
-  public static class SubQueryFinder extends RexVisitorImpl {
+  public static class SubQueryFinder extends RexVisitorImpl<@Nullable Void> {
     public static final SubQueryFinder INSTANCE = new SubQueryFinder();
     private final @Nullable SqlKind kind;
 
@@ -3543,7 +3543,7 @@ public ExprSimplifier(RexSimplify simplify, RexUnknownAs unknownAs,
 
   /** Visitor that tells whether a node matching a particular description exists
    * in a tree. */
-  public abstract static class RexFinder extends RexVisitorImpl {
+  public abstract static class RexFinder extends RexVisitorImpl<@Nullable Void> {
     RexFinder() {
       super(true);
     }
diff --git a/core/src/main/java/org/apache/calcite/sql/SqlPivot.java b/core/src/main/java/org/apache/calcite/sql/SqlPivot.java
index 00c50f883aef..89ecee5bdf3c 100644
--- a/core/src/main/java/org/apache/calcite/sql/SqlPivot.java
+++ b/core/src/main/java/org/apache/calcite/sql/SqlPivot.java
@@ -175,7 +175,7 @@ static SqlNodeList toNodes(SqlNode node) {
    * that are not used will become "GROUP BY" columns. */
   public Set usedColumnNames() {
     final Set columnNames = new HashSet<>();
-    final SqlVisitor nameCollector = new SqlBasicVisitor() {
+    final SqlVisitor<@Nullable Void> nameCollector = new SqlBasicVisitor<@Nullable Void>() {
       @Override public Void visit(SqlIdentifier id) {
         columnNames.add(Util.last(id.names));
         return super.visit(id);
diff --git a/core/src/main/java/org/apache/calcite/sql/SqlUnpivot.java b/core/src/main/java/org/apache/calcite/sql/SqlUnpivot.java
index d99815a0c62e..89aef8c420e2 100644
--- a/core/src/main/java/org/apache/calcite/sql/SqlUnpivot.java
+++ b/core/src/main/java/org/apache/calcite/sql/SqlUnpivot.java
@@ -144,7 +144,7 @@ public void forEachNameValues(
    * clause. All columns that are not used will be part of the returned row. */
   public Set usedColumnNames() {
     final Set columnNames = new HashSet<>();
-    final SqlVisitor nameCollector = new SqlBasicVisitor() {
+    final SqlVisitor<@Nullable Void> nameCollector = new SqlBasicVisitor<@Nullable Void>() {
       @Override public Void visit(SqlIdentifier id) {
         columnNames.add(Util.last(id.names));
         return super.visit(id);
diff --git a/core/src/main/java/org/apache/calcite/sql/SqlUtil.java b/core/src/main/java/org/apache/calcite/sql/SqlUtil.java
index 74daa93de52a..a2de0dfc58fe 100644
--- a/core/src/main/java/org/apache/calcite/sql/SqlUtil.java
+++ b/core/src/main/java/org/apache/calcite/sql/SqlUtil.java
@@ -1334,8 +1334,8 @@ public static boolean containsAgg(SqlNode node) {
   public static boolean containsCall(SqlNode node,
       Predicate callPredicate) {
     try {
-      SqlVisitor visitor =
-          new SqlBasicVisitor() {
+      SqlVisitor<@Nullable Void> visitor =
+          new SqlBasicVisitor<@Nullable Void>() {
             @Override public Void visit(SqlCall call) {
               if (callPredicate.test(call)) {
                 throw new Util.FoundOne(call);
@@ -1426,7 +1426,7 @@ public String getIdentifierQuoteString() {
 
   /** Walks over a {@link org.apache.calcite.sql.SqlNode} tree and returns the
    * ancestry stack when it finds a given node. */
-  private static class Genealogist extends SqlBasicVisitor {
+  private static class Genealogist extends SqlBasicVisitor<@Nullable Void> {
     private final List ancestors = new ArrayList<>();
     private final Predicate predicate;
     private final Predicate postPredicate;
diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlBetweenOperator.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlBetweenOperator.java
index 7d11e510a223..321b5c54d19f 100644
--- a/core/src/main/java/org/apache/calcite/sql/fun/SqlBetweenOperator.java
+++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlBetweenOperator.java
@@ -37,6 +37,8 @@
 import org.apache.calcite.util.Litmus;
 import org.apache.calcite.util.Util;
 
+import org.jspecify.annotations.Nullable;
+
 import static org.apache.calcite.util.Static.RESOURCE;
 
 /**
@@ -263,7 +265,7 @@ private static SqlBetweenOperator of(boolean negated, boolean symmetric) {
   /**
    * Finds an AND operator in an expression.
    */
-  private static class AndFinder extends SqlBasicVisitor {
+  private static class AndFinder extends SqlBasicVisitor<@Nullable Void> {
     @Override public Void visit(SqlCall call) {
       final SqlOperator operator = call.getOperator();
       if (operator == SqlStdOperatorTable.AND) {
diff --git a/core/src/main/java/org/apache/calcite/sql/type/OperandTypes.java b/core/src/main/java/org/apache/calcite/sql/type/OperandTypes.java
index c4912279d27a..58754538a125 100644
--- a/core/src/main/java/org/apache/calcite/sql/type/OperandTypes.java
+++ b/core/src/main/java/org/apache/calcite/sql/type/OperandTypes.java
@@ -2050,7 +2050,7 @@ protected boolean checkReturnType(
      * validated for the second time based on the given parameter type,
      * the type cached during the first validation must be cleared.
      */
-    protected static class TypeRemover extends SqlBasicVisitor {
+    protected static class TypeRemover extends SqlBasicVisitor<@Nullable Void> {
       private final SqlValidator validator;
 
       protected TypeRemover(SqlValidator validator) {
diff --git a/core/src/main/java/org/apache/calcite/sql/validate/AggChecker.java b/core/src/main/java/org/apache/calcite/sql/validate/AggChecker.java
index 23928b83ac69..6fa0e7cb872a 100644
--- a/core/src/main/java/org/apache/calcite/sql/validate/AggChecker.java
+++ b/core/src/main/java/org/apache/calcite/sql/validate/AggChecker.java
@@ -28,6 +28,8 @@
 
 import com.google.common.collect.Iterables;
 
+import org.jspecify.annotations.Nullable;
+
 import java.util.ArrayDeque;
 import java.util.Deque;
 import java.util.List;
@@ -40,7 +42,7 @@
  * Visitor which throws an exception if any component of the expression is not a
  * group expression.
  */
-class AggChecker extends SqlBasicVisitor {
+class AggChecker extends SqlBasicVisitor<@Nullable Void> {
   //~ Instance fields --------------------------------------------------------
 
   private final Deque scopes = new ArrayDeque<>();
diff --git a/core/src/main/java/org/apache/calcite/sql/validate/AggVisitor.java b/core/src/main/java/org/apache/calcite/sql/validate/AggVisitor.java
index 2b1f51e9598f..6b93767d4496 100644
--- a/core/src/main/java/org/apache/calcite/sql/validate/AggVisitor.java
+++ b/core/src/main/java/org/apache/calcite/sql/validate/AggVisitor.java
@@ -36,7 +36,7 @@
 /** Visitor that can find aggregate and windowed aggregate functions.
  *
  * @see AggFinder */
-abstract class AggVisitor extends SqlBasicVisitor {
+abstract class AggVisitor extends SqlBasicVisitor<@Nullable Void> {
   protected final SqlOperatorTable opTab;
   /** Whether to find windowed aggregates. */
   protected final boolean over;
diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java
index efa22c798539..e68a36848188 100644
--- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java
+++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java
@@ -1782,7 +1782,7 @@ && requireNonNull(((SqlNumericLiteral) node).bigDecimalValue())
           RESOURCE.offsetFetchValueMustNotBeNegative(kind));
     }
     validateNoAggs(aggOrOverFinder, node, kind);
-    node.accept(new SqlBasicVisitor() {
+    node.accept(new SqlBasicVisitor<@Nullable Void>() {
       @Override public Void visit(SqlIdentifier id) {
         if (makeNullaryCall(id) != null) {
           return null;
@@ -4573,7 +4573,7 @@ protected void validateSelect(
    */
   private static void forEachQualified(SqlNode node, SqlValidatorScope scope,
       Consumer consumer) {
-    node.accept(new SqlBasicVisitor() {
+    node.accept(new SqlBasicVisitor<@Nullable Void>() {
       @Override public Void visit(SqlIdentifier id) {
         final SqlQualified qualified = scope.fullyQualify(id);
         consumer.accept(qualified);
@@ -4587,7 +4587,7 @@ private static void forEachQualified(SqlNode node, SqlValidatorScope scope,
   private static void purgeForBypassFields(SqlNode node, SqlValidatorScope scope,
       Set qualifieds, Set bypassQualifieds,
       Set remnantMustFilterFields) {
-    node.accept(new SqlBasicVisitor() {
+    node.accept(new SqlBasicVisitor<@Nullable Void>() {
       @Override public Void visit(SqlIdentifier id) {
         final SqlQualified qualified = scope.fullyQualify(id);
         if (bypassQualifieds.contains(qualified)) {
@@ -6051,7 +6051,7 @@ private boolean referencesOnlyOuterColumns(SqlNode node,
     // ok[0] is cleared if any identifier is not an outer reference;
     // ok[1] is set once at least one outer column is found.
     final boolean[] ok = {true, false};
-    node.accept(new SqlBasicVisitor() {
+    node.accept(new SqlBasicVisitor<@Nullable Void>() {
       @Override public Void visit(SqlIdentifier id) {
         if (!isOuterReference(currentScope, id)) {
           ok[0] = false;
@@ -6076,7 +6076,7 @@ private boolean referencesOnlyOuterColumns(SqlNode node,
    */
   private static boolean containsSubQuery(SqlNode node) {
     final boolean[] found = {false};
-    node.accept(new SqlBasicVisitor() {
+    node.accept(new SqlBasicVisitor<@Nullable Void>() {
       @Override public Void visit(SqlCall call) {
         if (call.getKind().belongsTo(SqlKind.QUERY)) {
           found[0] = true;
@@ -7998,7 +7998,7 @@ private static class MergeNamespace extends DmlNamespace {
   }
 
   /** Visitor that retrieves pattern variables defined. */
-  private static class PatternVarVisitor implements SqlVisitor {
+  private static class PatternVarVisitor implements SqlVisitor<@Nullable Void> {
     private final MatchRecognizeScope scope;
 
     PatternVarVisitor(MatchRecognizeScope scope) {
@@ -8915,8 +8915,8 @@ private void addOrdinal2ExpandSet(
      */
     private boolean containsIdentifier(SqlNode sqlNode, SqlIdentifier target) {
       try {
-        SqlVisitor visitor =
-            new SqlBasicVisitor() {
+        SqlVisitor<@Nullable Void> visitor =
+            new SqlBasicVisitor<@Nullable Void>() {
               @Override public Void visit(SqlIdentifier identifier) {
                 if (identifier.equalsDeep(target, Litmus.IGNORE)) {
                   throw new Util.FoundOne(target);
diff --git a/core/src/main/java/org/apache/calcite/sql2rel/AggConverter.java b/core/src/main/java/org/apache/calcite/sql2rel/AggConverter.java
index 6d77076484d6..fbdec67744c1 100644
--- a/core/src/main/java/org/apache/calcite/sql2rel/AggConverter.java
+++ b/core/src/main/java/org/apache/calcite/sql2rel/AggConverter.java
@@ -85,7 +85,7 @@
  * 
    2. aggCalls = {AggCall(SUM, {1})}
      3. 
  * 
  */
-class AggConverter implements SqlVisitor {
+class AggConverter implements SqlVisitor<@Nullable Void> {
   private final SqlToRelConverter.Blackboard bb;
   private final Map nameMap;
 
diff --git a/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java b/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java
index 68f14f2701e7..72b66be88c68 100644
--- a/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java
+++ b/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java
@@ -2800,7 +2800,7 @@ private RexNode createCaseExpression(
       // expression. They need to be added to the window partition keys so that
       // decorrelation does not widen the window computation scope.
       final List correlationFields = new ArrayList<>();
-      over.accept(new RexVisitorImpl(true) {
+      over.accept(new RexVisitorImpl<@Nullable Void>(true) {
         @Override public Void visitFieldAccess(RexFieldAccess fieldAccess) {
           if (cm.mapFieldAccessToCorRef.containsKey(fieldAccess)
               && !correlationFields.contains(fieldAccess)) {
@@ -3992,8 +3992,8 @@ private RelNode visitJoin(BiRel join) {
       return join;
     }
 
-    private RexVisitorImpl rexVisitor(final RelNode rel) {
-      return new RexVisitorImpl(true) {
+    private RexVisitorImpl<@Nullable Void> rexVisitor(final RelNode rel) {
+      return new RexVisitorImpl<@Nullable Void>(true) {
         @Override public Void visitFieldAccess(RexFieldAccess fieldAccess) {
           final RexNode ref = fieldAccess.getReferenceExpr();
           if (ref instanceof RexCorrelVariable) {
diff --git a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
index d1e2054b1028..3b5876f8a046 100644
--- a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
+++ b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
@@ -6739,7 +6739,7 @@ public static class SqlIdentifierFinder implements SqlVisitor {
   /**
    * Visitor that collects all aggregate functions in a {@link SqlNode} tree.
    */
-  private static class AggregateFinder extends SqlBasicVisitor {
+  private static class AggregateFinder extends SqlBasicVisitor<@Nullable Void> {
     final List list = new ArrayList<>();
     final List filterList = new ArrayList<>();
     final List distinctList = new ArrayList<>();
diff --git a/core/src/main/java/org/apache/calcite/util/Util.java b/core/src/main/java/org/apache/calcite/util/Util.java
index e8aefee5c0ad..23faa50b3f32 100644
--- a/core/src/main/java/org/apache/calcite/util/Util.java
+++ b/core/src/main/java/org/apache/calcite/util/Util.java
@@ -2896,7 +2896,7 @@ public FoundOne(@Nullable Object node) {
    * Visitor which looks for an OVER clause inside a tree of
    * {@link SqlNode} objects.
    */
-  public static class OverFinder extends SqlBasicVisitor {
+  public static class OverFinder extends SqlBasicVisitor<@Nullable Void> {
     public static final OverFinder INSTANCE = new Util.OverFinder();
 
     @Override public Void visit(SqlCall call) {
diff --git a/file/src/main/java/org/apache/calcite/adapter/file/CsvProjectFilterTableScanRule.java b/file/src/main/java/org/apache/calcite/adapter/file/CsvProjectFilterTableScanRule.java
index 3bbcb5037ffb..76c7eb61c962 100644
--- a/file/src/main/java/org/apache/calcite/adapter/file/CsvProjectFilterTableScanRule.java
+++ b/file/src/main/java/org/apache/calcite/adapter/file/CsvProjectFilterTableScanRule.java
@@ -26,6 +26,7 @@
 import org.apache.calcite.rex.RexUtil;
 
 import org.immutables.value.Value;
+import org.jspecify.annotations.Nullable;
 
 import java.util.List;
 
@@ -52,7 +53,7 @@ protected CsvProjectFilterTableScanRule(Config config) {
     // Find all input fields referenced by the project expressions
     final java.util.Set projectInputFields = new java.util.HashSet<>();
     for (RexNode proj : project.getProjects()) {
-      proj.accept(new org.apache.calcite.rex.RexVisitorImpl(true) {
+      proj.accept(new org.apache.calcite.rex.RexVisitorImpl<@Nullable Void>(true) {
         @Override public Void visitInputRef(RexInputRef inputRef) {
           projectInputFields.add(inputRef.getIndex());
           return null;
@@ -62,7 +63,7 @@ protected CsvProjectFilterTableScanRule(Config config) {
 
     // Find all input fields referenced by the filter condition
     final java.util.Set filterInputFields = new java.util.HashSet<>();
-    filter.getCondition().accept(new org.apache.calcite.rex.RexVisitorImpl(true) {
+    filter.getCondition().accept(new org.apache.calcite.rex.RexVisitorImpl<@Nullable Void>(true) {
       @Override public Void visitInputRef(RexInputRef inputRef) {
         filterInputFields.add(inputRef.getIndex());
         return null;
diff --git a/piglet/src/main/java/org/apache/calcite/piglet/PigToSqlAggregateRule.java b/piglet/src/main/java/org/apache/calcite/piglet/PigToSqlAggregateRule.java
index 71f206d09afc..cf3d46b752de 100644
--- a/piglet/src/main/java/org/apache/calcite/piglet/PigToSqlAggregateRule.java
+++ b/piglet/src/main/java/org/apache/calcite/piglet/PigToSqlAggregateRule.java
@@ -36,6 +36,7 @@
 import org.apache.calcite.tools.RelBuilder;
 
 import org.immutables.value.Value;
+import org.jspecify.annotations.Nullable;
 
 import java.math.BigDecimal;
 import java.util.ArrayList;
@@ -80,7 +81,7 @@ protected PigToSqlAggregateRule(Config config) {
    * projection called in an expression and also whether a column is
    * referred in that expression.
    */
-  private static class PigAggUdfFinder extends RexVisitorImpl {
+  private static class PigAggUdfFinder extends RexVisitorImpl<@Nullable Void> {
     // Index of the column
     private final int projectCol;
     // List of all Pig aggregate UDFs found in the expression

From 6548f366727023bb0682623821f5e40c2882c273 Mon Sep 17 00:00:00 2001
From: Vladimir Sitnikov 
Date: Mon, 24 Aug 2026 12:11:23 +0300
Subject: [PATCH 511/562] [CALCITE-7736] Drop castNonNull calls whose argument
 is already non-null

NullAway reports a `castToNonNull` whose argument it can already prove non-null,
which is how it flags a cast that has stopped earning its place. 46 of them
across 20 files, most in the `FlatLists` and `PairLists` constructors, where the
elements arrive as non-null parameters.

Co-Authored-By: Claude Opus 5 
---
 .../calcite/adapter/enumerable/EnumUtils.java  |  4 ++--
 .../apache/calcite/jdbc/CalcitePrepare.java    |  2 +-
 .../apache/calcite/materialize/Lattice.java    |  2 +-
 .../org/apache/calcite/plan/RelOptCluster.java |  6 ++----
 .../apache/calcite/plan/volcano/RelSubset.java |  2 +-
 .../rel/metadata/RelMetadataQueryBase.java     |  4 +---
 .../calcite/rel/rel2sql/SqlImplementor.java    |  2 +-
 .../materialize/MaterializedViewRule.java      |  2 +-
 .../org/apache/calcite/rex/RexLiteral.java     |  2 +-
 .../org/apache/calcite/runtime/FlatLists.java  | 18 +++++++++---------
 .../calcite/runtime/ImmutablePairList.java     | 10 ++++------
 .../org/apache/calcite/runtime/PairLists.java  |  4 ++--
 .../java/org/apache/calcite/sql/SqlCall.java   |  4 +---
 .../org/apache/calcite/sql/SqlOperator.java    |  3 +--
 .../sql/ddl/SqlCreateForeignSchema.java        |  6 ++----
 .../calcite/sql2rel/SqlToRelConverter.java     |  2 +-
 .../org/apache/calcite/util/BuiltInMethod.java |  2 +-
 .../util/graph/DefaultDirectedGraph.java       |  6 ++----
 18 files changed, 34 insertions(+), 47 deletions(-)

diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java
index 4c0dfafb7575..c42e04950ecd 100644
--- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java
+++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java
@@ -294,7 +294,7 @@ static Expression joinSelectorCompact(JoinRelType joinType, PhysType physType,
       // Delegate copying the row values to JavaRowFormat
       final List copyStatements =
           Nullness.castNonNull(
-              inputPhysType.getFormat().copy(parameter, Nullness.castNonNull(compactOutputVar),
+              inputPhysType.getFormat().copy(parameter, compactOutputVar,
                   outputField, fieldCount));
       if (joinType.generatesNullsOn(ord.i)) {
         // [CALCITE-6593] NPE when outer joining tables with many fields and unmatching rows
@@ -309,7 +309,7 @@ static Expression joinSelectorCompact(JoinRelType joinType, PhysType physType,
       outputField += fieldCount;
     }
 
-    compactCode.add(Nullness.castNonNull(compactOutputVar));
+    compactCode.add(compactOutputVar);
     return Expressions.lambda(
         Function2.class,
         compactCode.toBlock(),
diff --git a/core/src/main/java/org/apache/calcite/jdbc/CalcitePrepare.java b/core/src/main/java/org/apache/calcite/jdbc/CalcitePrepare.java
index 3c051292a7e3..b8f406379a7b 100644
--- a/core/src/main/java/org/apache/calcite/jdbc/CalcitePrepare.java
+++ b/core/src/main/java/org/apache/calcite/jdbc/CalcitePrepare.java
@@ -214,7 +214,7 @@ public static Context peek() {
 
     public static void pop(Context context) {
       final Deque stack = THREAD_CONTEXT_STACK.get();
-      Context x = castNonNull(stack).pop();
+      Context x = stack.pop();
       assert x == context;
     }
 
diff --git a/core/src/main/java/org/apache/calcite/materialize/Lattice.java b/core/src/main/java/org/apache/calcite/materialize/Lattice.java
index b84d0c42d2fa..5f476c42cf13 100644
--- a/core/src/main/java/org/apache/calcite/materialize/Lattice.java
+++ b/core/src/main/java/org/apache/calcite/materialize/Lattice.java
@@ -1148,7 +1148,7 @@ void fixUp(MutableNode node) {
               SqlValidatorUtil.uniquify(name, columnAliases,
                   SqlValidatorUtil.ATTEMPT_SUGGESTER);
           final BaseColumn column =
-              new BaseColumn(c++, castNonNull(node.alias), name, alias);
+              new BaseColumn(c++, node.alias, name, alias);
           columnList.add(column);
           columnAliasList.put(name, column); // name before it is made unique
         }
diff --git a/core/src/main/java/org/apache/calcite/plan/RelOptCluster.java b/core/src/main/java/org/apache/calcite/plan/RelOptCluster.java
index 6cc88b8c0b99..c4141bceb1cf 100644
--- a/core/src/main/java/org/apache/calcite/plan/RelOptCluster.java
+++ b/core/src/main/java/org/apache/calcite/plan/RelOptCluster.java
@@ -37,8 +37,6 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Supplier;
 
-import static org.apache.calcite.linq4j.Nullness.castNonNull;
-
 import static java.util.Objects.requireNonNull;
 
 /**
@@ -111,7 +109,7 @@ public static RelOptCluster create(RelOptPlanner planner,
 
   @Deprecated // to be removed before 2.0
   public RelOptQuery getQuery() {
-    return new RelOptQuery(castNonNull(planner), nextCorrel, mapCorrelToRel);
+    return new RelOptQuery(planner, nextCorrel, mapCorrelToRel);
   }
 
   @Deprecated // to be removed before 2.0
@@ -193,7 +191,7 @@ public void setMetadataQuerySupplier(
    * method, then use {@link RelOptRuleCall#getMetadataQuery()} instead. */
   public RelMetadataQuery getMetadataQuery() {
     if (mq == null) {
-      mq = castNonNull(mqSupplier).get();
+      mq = mqSupplier.get();
     }
     return mq;
   }
diff --git a/core/src/main/java/org/apache/calcite/plan/volcano/RelSubset.java b/core/src/main/java/org/apache/calcite/plan/volcano/RelSubset.java
index 97cce4d58292..60b6855cff40 100644
--- a/core/src/main/java/org/apache/calcite/plan/volcano/RelSubset.java
+++ b/core/src/main/java/org/apache/calcite/plan/volcano/RelSubset.java
@@ -659,7 +659,7 @@ public RelNode visit(
                 finder.deadEnds.stream()
                     .filter(deadSubset -> deadSubset.getOriginal() != null)
                     .map(x -> {
-                      RelNode original = castNonNull(x.getOriginal());
+                      RelNode original = x.getOriginal();
                       return original.getClass().getSimpleName()
                           + traitDiff(original.getTraitSet(), x.getTraitSet());
                     })
diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMetadataQueryBase.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMetadataQueryBase.java
index f4cc78fea47b..cb8840b0011c 100644
--- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMetadataQueryBase.java
+++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMetadataQueryBase.java
@@ -27,8 +27,6 @@
 import java.util.Map;
 import java.util.function.Supplier;
 
-import static org.apache.calcite.linq4j.Nullness.castNonNull;
-
 import static java.util.Objects.requireNonNull;
 
 /**
@@ -120,7 +118,7 @@ protected > H revise(Class def) {
 
   private MetadataHandlerProvider getMetadataHandlerProvider() {
     requireNonNull(metadataHandlerProvider, "metadataHandlerProvider");
-    return castNonNull(metadataHandlerProvider);
+    return metadataHandlerProvider;
   }
 
   /**
diff --git a/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java b/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java
index d97e14a1fb89..7a3be7be3f48 100644
--- a/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java
+++ b/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java
@@ -1735,7 +1735,7 @@ public static SqlNode toSql(RexLiteral literal) {
         if (!defaultCharset.equals(charsetName)) {
           // Set the charset only if it is not the same as the default charset
           return SqlLiteral.createCharString(
-              castNonNull(value).getValue(), charsetName, POS);
+              value.getValue(), charsetName, POS);
         }
       }
       // Create a string without specifying a charset
diff --git a/core/src/main/java/org/apache/calcite/rel/rules/materialize/MaterializedViewRule.java b/core/src/main/java/org/apache/calcite/rel/rules/materialize/MaterializedViewRule.java
index 20215f7796f5..28da23359439 100644
--- a/core/src/main/java/org/apache/calcite/rel/rules/materialize/MaterializedViewRule.java
+++ b/core/src/main/java/org/apache/calcite/rel/rules/materialize/MaterializedViewRule.java
@@ -765,7 +765,7 @@ protected boolean compensatePartial(
                 RexTableInputRef.of(parentTRef, pair.target, uniqueKeyColumnType);
             if (!foreignKeyColumnType.isNullable()
                 && sourceEC.getEquivalenceClassesMap().containsKey(uniqueKeyColumnRef)
-                && castNonNull(sourceEC.getEquivalenceClassesMap().get(uniqueKeyColumnRef))
+                && sourceEC.getEquivalenceClassesMap().get(uniqueKeyColumnRef)
                     .contains(foreignKeyColumnRef)) {
               equiColumns.put(foreignKeyColumnRef, uniqueKeyColumnRef);
             } else {
diff --git a/core/src/main/java/org/apache/calcite/rex/RexLiteral.java b/core/src/main/java/org/apache/calcite/rex/RexLiteral.java
index fb2987de49f5..89394da3edce 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexLiteral.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexLiteral.java
@@ -755,7 +755,7 @@ private static void appendAsJava(@Nullable Comparable value, StringBuilder sb,
     case ARRAY:
       assert value instanceof List : "value must implement List: " + value;
       @SuppressWarnings("unchecked") final List list =
-          (List) castNonNull(value);
+          (List) value;
       Util.asStringBuilder(sb, sb2 ->
           Util.printList(sb, list.size(), (sb3, i) ->
               sb3.append(list.get(i).computeDigest(includeType))));
diff --git a/core/src/main/java/org/apache/calcite/runtime/FlatLists.java b/core/src/main/java/org/apache/calcite/runtime/FlatLists.java
index 9b3a0fbddf8e..9a7d2366fff4 100644
--- a/core/src/main/java/org/apache/calcite/runtime/FlatLists.java
+++ b/core/src/main/java/org/apache/calcite/runtime/FlatLists.java
@@ -407,7 +407,7 @@ protected static class Flat1List
     }
 
     @Override public @Nullable Object[] toArray(Flat1List<@Nullable T> this) {
-      return new Object[] {castNonNull(t0)};
+      return new Object[] {t0};
     }
 
     @Override public int compareTo(List o) {
@@ -540,7 +540,7 @@ protected static class Flat2List
     }
 
     @Override public @Nullable Object[] toArray(Flat2List<@Nullable T> this) {
-      return new Object[] {castNonNull(t0), castNonNull(t1)};
+      return new Object[] {t0, t1};
     }
 
     @Override public int compareTo(List o) {
@@ -690,7 +690,7 @@ protected static class Flat3List
     }
 
     @Override public @Nullable Object[] toArray(Flat3List<@Nullable T> this) {
-      return new Object[] {castNonNull(t0), castNonNull(t1), castNonNull(t2)};
+      return new Object[] {t0, t1, t2};
     }
 
     @Override public int compareTo(List o) {
@@ -859,8 +859,8 @@ protected static class Flat4List
     }
 
     @Override public @Nullable Object[] toArray(Flat4List<@Nullable T> this) {
-      return new Object[] {castNonNull(t0), castNonNull(t1), castNonNull(t2),
-          castNonNull(t3)};
+      return new Object[] {t0, t1, t2,
+          t3};
     }
 
     @Override public int compareTo(List o) {
@@ -1048,8 +1048,8 @@ protected static class Flat5List
     }
 
     @Override public @Nullable Object[] toArray(Flat5List<@Nullable T> this) {
-      return new Object[] {castNonNull(t0), castNonNull(t1), castNonNull(t2),
-          castNonNull(t3), castNonNull(t4)};
+      return new Object[] {t0, t1, t2,
+          t3, t4};
     }
 
     @Override public int compareTo(List o) {
@@ -1257,8 +1257,8 @@ protected static class Flat6List
     }
 
     @Override public @Nullable Object[] toArray(Flat6List<@Nullable T> this) {
-      return new Object[] {castNonNull(t0), castNonNull(t1), castNonNull(t2),
-          castNonNull(t3), castNonNull(t4), castNonNull(t5)};
+      return new Object[] {t0, t1, t2,
+          t3, t4, t5};
     }
 
     @Override public int compareTo(List o) {
diff --git a/core/src/main/java/org/apache/calcite/runtime/ImmutablePairList.java b/core/src/main/java/org/apache/calcite/runtime/ImmutablePairList.java
index 97c44108bc78..b77d6d9a5b17 100644
--- a/core/src/main/java/org/apache/calcite/runtime/ImmutablePairList.java
+++ b/core/src/main/java/org/apache/calcite/runtime/ImmutablePairList.java
@@ -23,8 +23,6 @@
 
 import static com.google.common.base.Preconditions.checkArgument;
 
-import static org.apache.calcite.linq4j.Nullness.castNonNull;
-
 import static java.util.Objects.requireNonNull;
 
 /** Immutable list of pairs.
@@ -88,8 +86,8 @@ static  ImmutablePairList copyOf(
         Object[] elements = new Object[2 * collection.size()];
         int i = 0;
         for (Map.Entry entry2 : iterable) {
-          elements[i++] = castNonNull(entry2.getKey());
-          elements[i++] = castNonNull(entry2.getValue());
+          elements[i++] = entry2.getKey();
+          elements[i++] = entry2.getValue();
         }
         return new PairLists.ArrayImmutablePairList<>(elements);
       }
@@ -98,8 +96,8 @@ static  ImmutablePairList copyOf(
     // Not a collection, so we don't know its size in advance.
     final List list = new ArrayList<>();
     iterable.forEach(entry -> {
-      list.add(castNonNull(entry.getKey()));
-      list.add(castNonNull(entry.getValue()));
+      list.add(entry.getKey());
+      list.add(entry.getValue());
     });
     return PairLists.immutableBackedBy(list);
   }
diff --git a/core/src/main/java/org/apache/calcite/runtime/PairLists.java b/core/src/main/java/org/apache/calcite/runtime/PairLists.java
index 1e5fb9196c97..2fd096f691fa 100644
--- a/core/src/main/java/org/apache/calcite/runtime/PairLists.java
+++ b/core/src/main/java/org/apache/calcite/runtime/PairLists.java
@@ -56,8 +56,8 @@ static  ImmutablePairList immutableBackedBy(
       return ImmutablePairList.of();
     case 2:
       return new SingletonImmutablePairList<>(
-          castNonNull((T) list.get(0)),
-          castNonNull((U) list.get(1)));
+          (T) list.get(0),
+          (U) list.get(1));
     default:
       return new ArrayImmutablePairList<>(list.toArray());
     }
diff --git a/core/src/main/java/org/apache/calcite/sql/SqlCall.java b/core/src/main/java/org/apache/calcite/sql/SqlCall.java
index 001cef2b48e6..701ad8761a87 100755
--- a/core/src/main/java/org/apache/calcite/sql/SqlCall.java
+++ b/core/src/main/java/org/apache/calcite/sql/SqlCall.java
@@ -33,8 +33,6 @@
 import java.util.Collection;
 import java.util.List;
 
-import static org.apache.calcite.linq4j.Nullness.castNonNull;
-
 import static java.util.Objects.requireNonNull;
 
 /**
@@ -103,7 +101,7 @@ public void setOperand(int i, @Nullable SqlNode operand) {
   public  S operand(int i) {
     // Note: in general, null elements exist in the list, however, the code
     // assumes operand(..) is non-nullable, so we add a cast here
-    return (S) castNonNull(getOperandList().get(i));
+    return (S) getOperandList().get(i);
   }
 
   public int operandCount() {
diff --git a/core/src/main/java/org/apache/calcite/sql/SqlOperator.java b/core/src/main/java/org/apache/calcite/sql/SqlOperator.java
index b1edc4e57f54..12dad78c287c 100644
--- a/core/src/main/java/org/apache/calcite/sql/SqlOperator.java
+++ b/core/src/main/java/org/apache/calcite/sql/SqlOperator.java
@@ -47,7 +47,6 @@
 import java.util.Objects;
 import java.util.function.Supplier;
 
-import static org.apache.calcite.linq4j.Nullness.castNonNull;
 import static org.apache.calcite.util.Static.RESOURCE;
 
 import static java.util.Objects.requireNonNull;
@@ -625,7 +624,7 @@ argTypes, null, null, getSyntax(), getKind(),
       throw validator.handleUnresolvedFunction(call, this, argTypes, null);
     }
 
-    ((SqlBasicCall) call).setOperator(castNonNull(sqlOperator));
+    ((SqlBasicCall) call).setOperator(sqlOperator);
     RelDataType type = call.getOperator().validateOperands(validator, scope, call);
 
     // Validate and determine coercibility and resulting collation
diff --git a/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateForeignSchema.java b/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateForeignSchema.java
index 5e945d2066aa..4e4fb30a1988 100644
--- a/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateForeignSchema.java
+++ b/core/src/main/java/org/apache/calcite/sql/ddl/SqlCreateForeignSchema.java
@@ -39,8 +39,6 @@
 
 import static com.google.common.base.Preconditions.checkArgument;
 
-import static org.apache.calcite.linq4j.Nullness.castNonNull;
-
 import static java.util.Objects.requireNonNull;
 
 /**
@@ -133,8 +131,8 @@ private static List> options(
     }
     return new AbstractList>() {
       @Override public Pair get(int index) {
-        return Pair.of((SqlIdentifier) castNonNull(optionList.get(index * 2)),
-            castNonNull(optionList.get(index * 2 + 1)));
+        return Pair.of((SqlIdentifier) optionList.get(index * 2),
+            optionList.get(index * 2 + 1));
       }
 
       @Override public int size() {
diff --git a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
index 3b5876f8a046..dc03fb280301 100644
--- a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
+++ b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
@@ -1044,7 +1044,7 @@ private void distinctify(
       if (idx >= 0) {
         topExprs.add(rexBuilder.makeInputRef(aggregate, idx));
       } else {
-        topExprs.add(castNonNull(project).getProjects().get(i).accept(shuttle));
+        topExprs.add(project.getProjects().get(i).accept(shuttle));
       }
     }
     bb.setRoot(
diff --git a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java
index 390efb81ee01..e4bd92f0527c 100644
--- a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java
+++ b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java
@@ -1077,6 +1077,6 @@ public enum BuiltInMethod {
   }
 
   public String getMethodName() {
-    return castNonNull(method).getName();
+    return method.getName();
   }
 }
diff --git a/core/src/main/java/org/apache/calcite/util/graph/DefaultDirectedGraph.java b/core/src/main/java/org/apache/calcite/util/graph/DefaultDirectedGraph.java
index d55298a8fc2c..f025c5df6374 100644
--- a/core/src/main/java/org/apache/calcite/util/graph/DefaultDirectedGraph.java
+++ b/core/src/main/java/org/apache/calcite/util/graph/DefaultDirectedGraph.java
@@ -31,8 +31,6 @@
 import java.util.Map;
 import java.util.Set;
 
-import static org.apache.calcite.linq4j.Nullness.castNonNull;
-
 /**
  * Default implementation of {@link DirectedGraph}.
  *
@@ -221,8 +219,8 @@ private void removeMinorityVertices(Collection collection) {
   private void removeMajorityVertices(Set vertexSet) {
     vertexMap.keySet().removeAll(vertexSet);
     for (VertexInfo info : vertexMap.values()) {
-      info.outEdges.removeIf(e -> vertexSet.contains(castNonNull((V) e.target)));
-      info.inEdges.removeIf(e -> vertexSet.contains(castNonNull((V) e.source)));
+      info.outEdges.removeIf(e -> vertexSet.contains((V) e.target));
+      info.inEdges.removeIf(e -> vertexSet.contains((V) e.source));
     }
   }
 

From 6846450bd2cc1d4c0d5964e187ee50eed2acd3f2 Mon Sep 17 00:00:00 2001
From: Vladimir Sitnikov 
Date: Mon, 24 Aug 2026 12:21:46 +0300
Subject: [PATCH 512/562] [CALCITE-7736] Match the JDK signature in the
 FlatLists toArray overrides

`java.util.List` declares ` T[] toArray(T[] a)`:
the type variable carries the nullability and the array argument is required.
The six `FlatNList` overrides declared ` @Nullable T2[] toArray(T2 @Nullable
[] a)`, which disagreed on all three counts, and then had to cast the argument
back to non-null before reading its length.

The six `Object[] toArray()` overrides carried a receiver parameter,
`Flat1List<@Nullable T> this`, which is how the Checker Framework said "only
when T is nullable". JSpecify has no such form, and the annotation is what
crashed NullAway when the migration generated a `@Contract` for these methods.

`ComparableListImpl.toArray` casts what it delegates to. NullAway's JDK model
reports `Collection.toArray()` as returning `@Nullable Object[]` at a call site
while requiring an override to return `Object[]`, so a class that both
implements `List` and calls `toArray` on another list has no signature that
satisfies both.

Co-Authored-By: Claude Opus 5 
---
 .../org/apache/calcite/runtime/FlatLists.java | 45 +++++++++----------
 1 file changed, 22 insertions(+), 23 deletions(-)

diff --git a/core/src/main/java/org/apache/calcite/runtime/FlatLists.java b/core/src/main/java/org/apache/calcite/runtime/FlatLists.java
index 9a7d2366fff4..0318a7a1414f 100644
--- a/core/src/main/java/org/apache/calcite/runtime/FlatLists.java
+++ b/core/src/main/java/org/apache/calcite/runtime/FlatLists.java
@@ -21,7 +21,6 @@
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 
-import org.jspecify.annotations.NonNull;
 import org.jspecify.annotations.Nullable;
 
 import java.util.AbstractList;
@@ -34,7 +33,7 @@
 import java.util.Objects;
 import java.util.RandomAccess;
 
-import static org.apache.calcite.linq4j.Nullness.castNonNull;
+import static org.apache.calcite.linq4j.Nullness.castNonNullArray;
 
 /**
  * Space-efficient, comparable, immutable lists.
@@ -257,7 +256,7 @@ private static  ComparableList of_(List t) {
       //   write our own implementation and reduce creation overhead a
       //   bit.
       //noinspection unchecked
-      return new ComparableListImpl(Arrays.asList(t.toArray()));
+      return new ComparableListImpl(Arrays.asList(castNonNullArray(t.toArray())));
     }
   }
 
@@ -397,8 +396,8 @@ protected static class Flat1List
     }
 
     @SuppressWarnings({"unchecked" })
-    @Override public  @Nullable T2[] toArray(T2 @Nullable [] a) {
-      if (castNonNull(a).length < 1) {
+    @Override public  T2[] toArray(T2[] a) {
+      if (a.length < 1) {
         // Make a new array of a's runtime type, but my contents:
         return (T2[]) Arrays.copyOf(toArray(), 1, a.getClass());
       }
@@ -406,7 +405,7 @@ protected static class Flat1List
       return a;
     }
 
-    @Override public @Nullable Object[] toArray(Flat1List<@Nullable T> this) {
+    @Override public Object[] toArray() {
       return new Object[] {t0};
     }
 
@@ -529,8 +528,8 @@ protected static class Flat2List
     }
 
     @SuppressWarnings({"unchecked" })
-    @Override public  @Nullable T2[] toArray(T2 @Nullable [] a) {
-      if (castNonNull(a).length < 2) {
+    @Override public  T2[] toArray(T2[] a) {
+      if (a.length < 2) {
         // Make a new array of a's runtime type, but my contents:
         return (T2[]) Arrays.copyOf(toArray(), 2, a.getClass());
       }
@@ -539,7 +538,7 @@ protected static class Flat2List
       return a;
     }
 
-    @Override public @Nullable Object[] toArray(Flat2List<@Nullable T> this) {
+    @Override public Object[] toArray() {
       return new Object[] {t0, t1};
     }
 
@@ -678,8 +677,8 @@ protected static class Flat3List
     }
 
     @SuppressWarnings({"unchecked" })
-    @Override public  @Nullable T2[] toArray(T2 @Nullable [] a) {
-      if (castNonNull(a).length < 3) {
+    @Override public  T2[] toArray(T2[] a) {
+      if (a.length < 3) {
         // Make a new array of a's runtime type, but my contents:
         return (T2[]) Arrays.copyOf(toArray(), 3, a.getClass());
       }
@@ -689,7 +688,7 @@ protected static class Flat3List
       return a;
     }
 
-    @Override public @Nullable Object[] toArray(Flat3List<@Nullable T> this) {
+    @Override public Object[] toArray() {
       return new Object[] {t0, t1, t2};
     }
 
@@ -846,8 +845,8 @@ protected static class Flat4List
     }
 
     @SuppressWarnings({"unchecked" })
-    @Override public  @Nullable T2[] toArray(T2 @Nullable [] a) {
-      if (castNonNull(a).length < 4) {
+    @Override public  T2[] toArray(T2[] a) {
+      if (a.length < 4) {
         // Make a new array of a's runtime type, but my contents:
         return (T2[]) Arrays.copyOf(toArray(), 4, a.getClass());
       }
@@ -858,7 +857,7 @@ protected static class Flat4List
       return a;
     }
 
-    @Override public @Nullable Object[] toArray(Flat4List<@Nullable T> this) {
+    @Override public Object[] toArray() {
       return new Object[] {t0, t1, t2,
           t3};
     }
@@ -1034,8 +1033,8 @@ protected static class Flat5List
     }
 
     @SuppressWarnings({"unchecked" })
-    @Override public  @Nullable T2[] toArray(T2 @Nullable [] a) {
-      if (castNonNull(a).length < 5) {
+    @Override public  T2[] toArray(T2[] a) {
+      if (a.length < 5) {
         // Make a new array of a's runtime type, but my contents:
         return (T2[]) Arrays.copyOf(toArray(), 5, a.getClass());
       }
@@ -1047,7 +1046,7 @@ protected static class Flat5List
       return a;
     }
 
-    @Override public @Nullable Object[] toArray(Flat5List<@Nullable T> this) {
+    @Override public Object[] toArray() {
       return new Object[] {t0, t1, t2,
           t3, t4};
     }
@@ -1242,8 +1241,8 @@ protected static class Flat6List
     }
 
     @SuppressWarnings({"unchecked" })
-    @Override public  @Nullable T2[] toArray(T2 @Nullable [] a) {
-      if (castNonNull(a).length < 6) {
+    @Override public  T2[] toArray(T2[] a) {
+      if (a.length < 6) {
         // Make a new array of a's runtime type, but my contents:
         return (T2[]) Arrays.copyOf(toArray(), 6, a.getClass());
       }
@@ -1256,7 +1255,7 @@ protected static class Flat6List
       return a;
     }
 
-    @Override public @Nullable Object[] toArray(Flat6List<@Nullable T> this) {
+    @Override public Object[] toArray() {
       return new Object[] {t0, t1, t2,
           t3, t4, t5};
     }
@@ -1336,8 +1335,8 @@ protected ComparableListImpl(List list) {
       return list.size();
     }
 
-    @Override @NonNull public Object[] toArray(@NonNull ComparableListImpl this) {
-      return this.list.toArray();
+    @Override public Object[] toArray() {
+      return castNonNullArray(this.list.toArray());
     }
 
     @Override public int compareTo(List o) {

From 54831461234e45c6c13b7489fb52a6b7127171c0 Mon Sep 17 00:00:00 2001
From: Vladimir Sitnikov 
Date: Mon, 24 Aug 2026 12:55:21 +0300
Subject: [PATCH 513/562] [CALCITE-7736] Let the PairList implementations hold
 nullable pairs

`PairList` is declared ``, but the classes in `PairLists` that implement it, and the `MapEntry`
they hand back, declared plain ``. A nullable type argument was therefore
rejected at every one of them.

`MutablePairList` keeps both halves of every pair in one
`List<@Nullable Object>`, so the element type cannot carry the nullability of T
and of U separately, and reading a slot back produced a `@Nullable` value at 26
call sites. They now go through one `element` helper that explains the packing
and carries the suppression, rather than each site arguing the point.

`backingList` answers with `Arrays.asList` and `Collections.emptyList` instead
of `ImmutableList`, which rejects a `@Nullable` element type. The array-backed
implementation already answered that way.

Co-Authored-By: Claude Opus 5 
---
 .../org/apache/calcite/runtime/MapEntry.java  |  3 +-
 .../org/apache/calcite/runtime/PairLists.java | 82 +++++++++++--------
 2 files changed, 52 insertions(+), 33 deletions(-)

diff --git a/core/src/main/java/org/apache/calcite/runtime/MapEntry.java b/core/src/main/java/org/apache/calcite/runtime/MapEntry.java
index 1021d76b5ac5..209d1c552758 100644
--- a/core/src/main/java/org/apache/calcite/runtime/MapEntry.java
+++ b/core/src/main/java/org/apache/calcite/runtime/MapEntry.java
@@ -31,7 +31,8 @@
  * @param  Key type
  * @param  Value type
  */
-public class MapEntry implements Map.Entry {
+public class MapEntry
+    implements Map.Entry {
   final T t;
   final U u;
 
diff --git a/core/src/main/java/org/apache/calcite/runtime/PairLists.java b/core/src/main/java/org/apache/calcite/runtime/PairLists.java
index 2fd096f691fa..81a05292fa95 100644
--- a/core/src/main/java/org/apache/calcite/runtime/PairLists.java
+++ b/core/src/main/java/org/apache/calcite/runtime/PairLists.java
@@ -27,6 +27,7 @@
 
 import java.util.AbstractList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.RandomAccess;
@@ -56,8 +57,8 @@ static  ImmutablePairList immutableBackedBy(
       return ImmutablePairList.of();
     case 2:
       return new SingletonImmutablePairList<>(
-          (T) list.get(0),
-          (U) list.get(1));
+          element(list.get(0)),
+          element(list.get(1)));
     default:
       return new ArrayImmutablePairList<>(list.toArray());
     }
@@ -78,12 +79,28 @@ static void checkElementNotNull(int i, @Nullable Object element) {
     }
   }
 
+  /** Casts an element of a packed list back to the type it was stored with.
+   *
+   * 
      A {@link PairList} keeps both halves of every pair in one
+   * {@code List<@Nullable Object>}, so the element type cannot carry the nullability of
+   * {@code T} and of {@code U} separately. A slot holds whatever was written into it: null
+   * only when the type argument it belongs to is itself nullable.
+   *
+   * @param o element of the packed list
+   * @param  type the element was stored with
+   * @return the element, typed
+   */
+  @SuppressWarnings({"unchecked", "NullAway"})
+  private static  E element(@Nullable Object o) {
+    return (E) o;
+  }
+
   /** Base class for all implementations of PairList.
    *
    * @param  First type
    * @param  Second type
    */
-  abstract static class AbstractPairList
+  abstract static class AbstractPairList
       extends AbstractList>
       implements PairList {
     /** Returns a list containing the alternating left and right elements
@@ -112,7 +129,8 @@ static void subListRangeCheck(int fromIndex, int toIndex, int size) {
    * @param  First type
    * @param  Second type
    */
-  static class MutablePairList extends AbstractPairList {
+  static class MutablePairList
+      extends AbstractPairList {
     final List<@Nullable Object> list;
 
     MutablePairList(List<@Nullable Object> list) {
@@ -138,19 +156,19 @@ static class MutablePairList extends AbstractPairList {
     @SuppressWarnings("unchecked")
     @Override public Map.Entry get(int index) {
       int x = index * 2;
-      return new MapEntry<>((T) list.get(x), (U) list.get(x + 1));
+      return new MapEntry<>(element(list.get(x)), element(list.get(x + 1)));
     }
 
     @SuppressWarnings("unchecked")
     @Override public T left(int index) {
       int x = index * 2;
-      return (T) list.get(x);
+      return element(list.get(x));
     }
 
     @SuppressWarnings("unchecked")
     @Override public U right(int index) {
       int x = index * 2;
-      return (U) list.get(x + 1);
+      return element(list.get(x + 1));
     }
 
     @Override public Map.Entry set(int index,
@@ -164,16 +182,16 @@ static class MutablePairList extends AbstractPairList {
     @SuppressWarnings("unchecked")
     @Override public Map.Entry set(int index, T t, U u) {
       int x = index * 2;
-      T t0 = (T) list.set(x, t);
-      U u0 = (U) list.set(x + 1, u);
+      T t0 = element(list.set(x, t));
+      U u0 = element(list.set(x + 1, u));
       return new MapEntry<>(t0, u0);
     }
 
     @SuppressWarnings("unchecked")
     @Override public Map.Entry remove(int index) {
       final int x = index * 2;
-      T t = (T) list.remove(x);
-      U u = (U) list.remove(x);
+      T t = element(list.remove(x));
+      U u = element(list.remove(x));
       return new MapEntry<>(t, u);
     }
 
@@ -222,7 +240,7 @@ static class MutablePairList extends AbstractPairList {
         }
 
         @Override public T get(int index) {
-          return (T) list.get(index * 2);
+          return element(list.get(index * 2));
         }
       };
     }
@@ -236,7 +254,7 @@ static class MutablePairList extends AbstractPairList {
         }
 
         @Override public U get(int index) {
-          return (U) list.get(index * 2 + 1);
+          return element(list.get(index * 2 + 1));
         }
       };
     }
@@ -245,8 +263,8 @@ static class MutablePairList extends AbstractPairList {
     @Override public void forEach(BiConsumer consumer) {
       requireNonNull(consumer, "consumer");
       for (int i = 0; i < list.size();) {
-        T t = (T) list.get(i++);
-        U u = (U) list.get(i++);
+        T t = element(list.get(i++));
+        U u = element(list.get(i++));
         consumer.accept(t, u);
       }
     }
@@ -255,8 +273,8 @@ static class MutablePairList extends AbstractPairList {
     @Override public void forEachIndexed(IndexedBiConsumer consumer) {
       requireNonNull(consumer, "consumer");
       for (int i = 0, j = 0; i < list.size();) {
-        T t = (T) list.get(i++);
-        U u = (U) list.get(i++);
+        T t = element(list.get(i++));
+        U u = element(list.get(i++));
         consumer.accept(j++, t, u);
       }
     }
@@ -275,8 +293,8 @@ static class MutablePairList extends AbstractPairList {
     @Override public  List transform(BiFunction function) {
       return Functions.generate(list.size() / 2, index -> {
         final int x = index * 2;
-        final T t = (T) list.get(x);
-        final U u = (U) list.get(x + 1);
+        final T t = element(list.get(x));
+        final U u = element(list.get(x + 1));
         return function.apply(t, u);
       });
     }
@@ -289,8 +307,8 @@ static class MutablePairList extends AbstractPairList {
       }
       final ImmutableList.Builder builder = ImmutableList.builder();
       for (int i = 0, n = list.size(); i < n;) {
-        final T t = (T) list.get(i++);
-        final U u = (U) list.get(i++);
+        final T t = element(list.get(i++));
+        final U u = element(list.get(i++));
         builder.add(function.apply(t, u));
       }
       return builder.build();
@@ -303,8 +321,8 @@ static class MutablePairList extends AbstractPairList {
     @SuppressWarnings("unchecked")
     @Override public boolean anyMatch(BiPredicate predicate) {
       for (int i = 0; i < list.size();) {
-        final T t = (T) list.get(i++);
-        final U u = (U) list.get(i++);
+        final T t = element(list.get(i++));
+        final U u = element(list.get(i++));
         if (predicate.test(t, u)) {
           return true;
         }
@@ -315,8 +333,8 @@ static class MutablePairList extends AbstractPairList {
     @SuppressWarnings("unchecked")
     @Override public boolean allMatch(BiPredicate predicate) {
       for (int i = 0; i < list.size();) {
-        final T t = (T) list.get(i++);
-        final U u = (U) list.get(i++);
+        final T t = element(list.get(i++));
+        final U u = element(list.get(i++));
         if (!predicate.test(t, u)) {
           return false;
         }
@@ -327,8 +345,8 @@ static class MutablePairList extends AbstractPairList {
     @SuppressWarnings("unchecked")
     @Override public boolean noMatch(BiPredicate predicate) {
       for (int i = 0; i < list.size();) {
-        final T t = (T) list.get(i++);
-        final U u = (U) list.get(i++);
+        final T t = element(list.get(i++));
+        final U u = element(list.get(i++));
         if (predicate.test(t, u)) {
           return false;
         }
@@ -369,11 +387,11 @@ static class MutablePairList extends AbstractPairList {
    * @param  First type
    * @param  Second type
    */
-  static class EmptyImmutablePairList
+  static class EmptyImmutablePairList
       extends AbstractPairList
       implements ImmutablePairList {
     @Override List<@Nullable Object> backingList() {
-      return ImmutableList.of();
+      return Collections.emptyList();
     }
 
     @Override public Map.Entry get(int index) {
@@ -443,7 +461,7 @@ static class EmptyImmutablePairList
    * @param  First type
    * @param  Second type
    */
-  static class SingletonImmutablePairList
+  static class SingletonImmutablePairList
       extends AbstractPairList
       implements ImmutablePairList {
     private final T t;
@@ -457,7 +475,7 @@ static class SingletonImmutablePairList
     }
 
     @Override List<@Nullable Object> backingList() {
-      return ImmutableList.of(t, u);
+      return Arrays.asList(t, u);
     }
 
     @Override public Map.Entry get(int index) {
@@ -547,7 +565,7 @@ abstract static class RandomAccessList
    * @param  First type
    * @param  Second type
    */
-  static class ArrayImmutablePairList
+  static class ArrayImmutablePairList
       extends AbstractPairList
       implements ImmutablePairList {
     private final Object[] elements;

From 4892c707e4983ec2b71415b73e3961dfc778ce8d Mon Sep 17 00:00:00 2001
From: Vladimir Sitnikov 
Date: Mon, 24 Aug 2026 13:37:16 +0300
Subject: [PATCH 514/562] [CALCITE-7736] Let select and its siblings produce a
 nullable result

`select` maps each element through a function, and the function is free to
return null: `SqlFunctions` does exactly that when it projects a column that may
be absent. The `` type parameter said otherwise on `ExtendedEnumerable`,
`ExtendedQueryable`, `QueryableFactory`, `EnumerableDefaults` and the classes
that implement them, so a nullable type argument was rejected at the call.

Adds the nullable bound to every `` and `` method in
linq4j, and to the two overrides of those methods in core.

Co-Authored-By: Claude Opus 5 
---
 .../calcite/adapter/enumerable/EnumUtils.java     |  2 +-
 .../calcite/prepare/QueryableRelBuilder.java      | 10 +++++-----
 .../apache/calcite/linq4j/DefaultEnumerable.java  | 12 +++++++-----
 .../apache/calcite/linq4j/DefaultQueryable.java   | 11 ++++++-----
 .../apache/calcite/linq4j/EnumerableDefaults.java | 10 +++++-----
 .../calcite/linq4j/EnumerableQueryable.java       | 11 ++++++-----
 .../apache/calcite/linq4j/ExtendedEnumerable.java | 11 ++++++-----
 .../apache/calcite/linq4j/ExtendedQueryable.java  | 10 +++++-----
 .../java/org/apache/calcite/linq4j/Linq4j.java    |  4 ++--
 .../java/org/apache/calcite/linq4j/Lookup.java    |  4 +++-
 .../org/apache/calcite/linq4j/LookupImpl.java     |  2 +-
 .../apache/calcite/linq4j/QueryableDefaults.java  |  2 +-
 .../apache/calcite/linq4j/QueryableFactory.java   | 10 +++++-----
 .../apache/calcite/linq4j/QueryableRecorder.java  | 15 ++++++++++-----
 14 files changed, 63 insertions(+), 51 deletions(-)

diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java
index c42e04950ecd..c09ce0d1751e 100644
--- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java
+++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java
@@ -1412,7 +1412,7 @@ private static PairList hopWindows(long tsMillis,
   /**
    * Apply tumbling per row from the enumerable input.
    */
-  public static  Enumerable tumbling(
+  public static  Enumerable tumbling(
       Enumerable inputEnumerable,
       Function1 outSelector) {
     return new AbstractEnumerable() {
diff --git a/core/src/main/java/org/apache/calcite/prepare/QueryableRelBuilder.java b/core/src/main/java/org/apache/calcite/prepare/QueryableRelBuilder.java
index 6ce1fbd1414f..99d24d98a873 100644
--- a/core/src/main/java/org/apache/calcite/prepare/QueryableRelBuilder.java
+++ b/core/src/main/java/org/apache/calcite/prepare/QueryableRelBuilder.java
@@ -501,7 +501,7 @@ private void setRel(RelNode rel) {
     throw new UnsupportedOperationException();
   }
 
-  @Override public  Queryable ofType(
+  @Override public  Queryable ofType(
       Queryable source, Class clazz) {
     throw new UnsupportedOperationException();
   }
@@ -543,7 +543,7 @@ private void setRel(RelNode rel) {
     throw new UnsupportedOperationException();
   }
 
-  @Override public  Queryable select(
+  @Override public  Queryable select(
       Queryable source,
       FunctionExpression> selector) {
     RelNode child = toRel(source);
@@ -554,19 +554,19 @@ private void setRel(RelNode rel) {
     return castNonNull(null);
   }
 
-  @Override public  Queryable selectN(
+  @Override public  Queryable selectN(
       Queryable source,
       FunctionExpression> selector) {
     throw new UnsupportedOperationException();
   }
 
-  @Override public  Queryable selectMany(
+  @Override public  Queryable selectMany(
       Queryable source,
       FunctionExpression>> selector) {
     throw new UnsupportedOperationException();
   }
 
-  @Override public  Queryable selectManyN(
+  @Override public  Queryable selectManyN(
       Queryable source,
       FunctionExpression>> selector) {
     throw new UnsupportedOperationException();
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/DefaultEnumerable.java b/linq4j/src/main/java/org/apache/calcite/linq4j/DefaultEnumerable.java
index 83b15374a997..f2d2741ea132 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/DefaultEnumerable.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/DefaultEnumerable.java
@@ -601,7 +601,8 @@ protected OrderedQueryable asOrderedQueryable() {
     return EnumerableDefaults.min(getThis(), selector);
   }
 
-  @Override public  Enumerable ofType(Class clazz) {
+  @Override public 
+      Enumerable ofType(Class clazz) {
     return EnumerableDefaults.ofType(getThis(), clazz);
   }
 
@@ -630,21 +631,22 @@ protected OrderedQueryable asOrderedQueryable() {
     return EnumerableDefaults.reverse(getThis());
   }
 
-  @Override public  Enumerable select(Function1 selector) {
+  @Override public 
+      Enumerable select(Function1 selector) {
     return EnumerableDefaults.select(getThis(), selector);
   }
 
-  @Override public  Enumerable select(
+  @Override public  Enumerable select(
       Function2 selector) {
     return EnumerableDefaults.select(getThis(), selector);
   }
 
-  @Override public  Enumerable selectMany(
+  @Override public  Enumerable selectMany(
       Function1> selector) {
     return EnumerableDefaults.selectMany(getThis(), selector);
   }
 
-  @Override public  Enumerable selectMany(
+  @Override public  Enumerable selectMany(
       Function2> selector) {
     return EnumerableDefaults.selectMany(getThis(), selector);
   }
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/DefaultQueryable.java b/linq4j/src/main/java/org/apache/calcite/linq4j/DefaultQueryable.java
index d60bba500bf1..24eadff4af92 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/DefaultQueryable.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/DefaultQueryable.java
@@ -146,7 +146,8 @@ protected OrderedQueryable getThisOrderedQueryable() {
     return factory.distinct(getThis(), comparer);
   }
 
-  @Override public  Queryable ofType(Class clazz) {
+  @Override public 
+      Queryable ofType(Class clazz) {
     return factory.ofType(getThis(), clazz);
   }
 
@@ -386,22 +387,22 @@ protected OrderedQueryable getThisOrderedQueryable() {
     return factory.orderByDescending(getThis(), keySelector, comparator);
   }
 
-  @Override public  Queryable select(
+  @Override public  Queryable select(
       FunctionExpression> selector) {
     return factory.select(getThis(), selector);
   }
 
-  @Override public  Queryable selectN(
+  @Override public  Queryable selectN(
       FunctionExpression> selector) {
     return factory.selectN(getThis(), selector);
   }
 
-  @Override public  Queryable selectMany(
+  @Override public  Queryable selectMany(
       FunctionExpression>> selector) {
     return factory.selectMany(getThis(), selector);
   }
 
-  @Override public  Queryable selectManyN(
+  @Override public  Queryable selectManyN(
       FunctionExpression>> selector) {
     return factory.selectManyN(getThis(), selector);
   }
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java b/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java
index e7be128c5903..425ea1f81589 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java
@@ -3202,7 +3202,7 @@ public static  float min(Enumerable source,
    *
    * @return Collection of T2
    */
-  public static  Enumerable ofType(
+  public static  Enumerable ofType(
       Enumerable enumerable, Class clazz) {
     //noinspection unchecked
     return (Enumerable) where(enumerable,
@@ -3399,7 +3399,7 @@ public static  Enumerable reverse(
   /**
    * Projects each element of a sequence into a new form.
    */
-  public static  Enumerable select(
+  public static  Enumerable select(
       final Enumerable source,
       final Function1 selector) {
     if (selector == Functions.identitySelector()) {
@@ -3435,7 +3435,7 @@ public static  Enumerable select(
    * Projects each element of a sequence into a new
    * form by incorporating the element's index.
    */
-  public static  Enumerable select(
+  public static  Enumerable select(
       final Enumerable source,
       final Function2 selector) {
     return new AbstractEnumerable() {
@@ -3474,7 +3474,7 @@ public static  Enumerable select(
    * {@code Enumerable} and flattens the resulting sequences into one
    * sequence.
    */
-  public static  Enumerable selectMany(
+  public static  Enumerable selectMany(
       final Enumerable source,
       final Function1> selector) {
     return new AbstractEnumerable() {
@@ -3520,7 +3520,7 @@ public static  Enumerable selectMany(
    * sequence. The index of each source element is used in the
    * projected form of that element.
    */
-  public static  Enumerable selectMany(
+  public static  Enumerable selectMany(
       final Enumerable source,
       final Function2> selector) {
     return new AbstractEnumerable() {
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableQueryable.java b/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableQueryable.java
index 0c777684556d..e0ec59f7c328 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableQueryable.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableQueryable.java
@@ -151,7 +151,8 @@ protected Queryable queryable() {
     return EnumerableDefaults.distinct(getThis(), comparer).asQueryable();
   }
 
-  @Override public  Queryable ofType(Class clazz) {
+  @Override public 
+      Queryable ofType(Class clazz) {
     return EnumerableDefaults.ofType(getThis(), clazz).asQueryable();
   }
 
@@ -423,25 +424,25 @@ protected Queryable queryable() {
             keySelector.getFunction(), comparator));
   }
 
-  @Override public  Queryable select(
+  @Override public  Queryable select(
       FunctionExpression> selector) {
     return EnumerableDefaults.select(getThis(), selector.getFunction())
         .asQueryable();
   }
 
-  @Override public  Queryable selectN(
+  @Override public  Queryable selectN(
       FunctionExpression> selector) {
     return EnumerableDefaults.select(getThis(), selector.getFunction())
         .asQueryable();
   }
 
-  @Override public  Queryable selectMany(
+  @Override public  Queryable selectMany(
       FunctionExpression>> selector) {
     return EnumerableDefaults.selectMany(getThis(), selector.getFunction())
         .asQueryable();
   }
 
-  @Override public  Queryable selectManyN(
+  @Override public  Queryable selectManyN(
       FunctionExpression>> selector) {
     return EnumerableDefaults.selectMany(getThis(), selector.getFunction())
         .asQueryable();
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedEnumerable.java b/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedEnumerable.java
index 4f81e441b86a..171b3e7d9303 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedEnumerable.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedEnumerable.java
@@ -956,7 +956,7 @@  Enumerable correlateJoin(
    *
    * @return Collection of T2
    */
-   Enumerable ofType(Class clazz);
+   Enumerable ofType(Class clazz);
 
   /**
    * Sorts the elements of a sequence in ascending
@@ -996,13 +996,14 @@  Enumerable orderByDescending(
    * Projects each element of a sequence into a new
    * form.
    */
-   Enumerable select(Function1 selector);
+  
+      Enumerable select(Function1 selector);
 
   /**
    * Projects each element of a sequence into a new
    * form by incorporating the element's index.
    */
-   Enumerable select(
+   Enumerable select(
       Function2 selector);
 
   /**
@@ -1010,7 +1011,7 @@  Enumerable select(
    * {@code Enumerable} and flattens the resulting sequences into one
    * sequence.
    */
-   Enumerable selectMany(
+   Enumerable selectMany(
       Function1> selector);
 
   /**
@@ -1019,7 +1020,7 @@  Enumerable selectMany(
    * sequence. The index of each source element is used in the
    * projected form of that element.
    */
-   Enumerable selectMany(
+   Enumerable selectMany(
       Function2> selector);
 
   /**
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedQueryable.java b/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedQueryable.java
index d9716d8d76ed..7fb59d19bda3 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedQueryable.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedQueryable.java
@@ -445,7 +445,7 @@  Queryable join(Enumerable inner,
    * 
      NOTE: clazz parameter not present in C# LINQ; necessary because of
    * Java type erasure.
    */
-  @Override  Queryable ofType(Class clazz);
+  @Override  Queryable ofType(Class clazz);
 
   @Override  Queryable cast(Class clazz);
 
@@ -488,7 +488,7 @@  OrderedQueryable orderByDescending(
   /**
    * Projects each element of a sequence into a new form.
    */
-   Queryable select(
+   Queryable select(
       FunctionExpression> selector);
 
   /**
@@ -498,7 +498,7 @@  Queryable select(
    * 
      NOTE: Renamed from {@code select} because had same erasure as
    * {@link #select(org.apache.calcite.linq4j.tree.FunctionExpression)}.
    */
-   Queryable selectN(
+   Queryable selectN(
       FunctionExpression> selector);
 
 
@@ -507,7 +507,7 @@  Queryable selectN(
    * {@code Enumerable} and combines the resulting sequences into one
    * sequence.
    */
-   Queryable selectMany(
+   Queryable selectMany(
       FunctionExpression>> selector);
 
   /**
@@ -519,7 +519,7 @@  Queryable selectMany(
    * 
      NOTE: Renamed from {@code selectMany} because had same erasure as
    * {@link #selectMany(org.apache.calcite.linq4j.tree.FunctionExpression)}.
    */
-   Queryable selectManyN(
+   Queryable selectManyN(
       FunctionExpression>>
         selector);
 
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/Linq4j.java b/linq4j/src/main/java/org/apache/calcite/linq4j/Linq4j.java
index 5c90204a37e6..4b7fed20bb0d 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/Linq4j.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/Linq4j.java
@@ -249,7 +249,7 @@ public static  Enumerator transform(Enumerator enumerator,
    * @see #ofType
    * @see #asEnumerable(Iterable)
    */
-  public static  Enumerable cast(
+  public static  Enumerable cast(
       Iterable source, Class clazz) {
     return asEnumerable(source).cast(clazz);
   }
@@ -285,7 +285,7 @@ public static  Enumerable cast(
    * @see Enumerable#cast(Class)
    * @see #cast
    */
-  public static  Enumerable ofType(
+  public static  Enumerable ofType(
       Iterable source, Class clazz) {
     return asEnumerable(source).ofType(clazz);
   }
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/Lookup.java b/linq4j/src/main/java/org/apache/calcite/linq4j/Lookup.java
index 1ab339a76731..47fcf1928c88 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/Lookup.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/Lookup.java
@@ -18,6 +18,8 @@
 
 import org.apache.calcite.linq4j.function.Function2;
 
+import org.jspecify.annotations.Nullable;
+
 import java.util.Map;
 
 /**
@@ -37,6 +39,6 @@ public interface Lookup
    *
    * @return Enumerable over results
    */
-   Enumerable applyResultSelector(
+   Enumerable applyResultSelector(
       Function2, TResult> resultSelector);
 }
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/LookupImpl.java b/linq4j/src/main/java/org/apache/calcite/linq4j/LookupImpl.java
index 277daa1fbc37..3e3c5a841bdd 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/LookupImpl.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/LookupImpl.java
@@ -181,7 +181,7 @@ class LookupImpl extends AbstractEnumerable>
     };
   }
 
-  @Override public  Enumerable applyResultSelector(
+  @Override public  Enumerable applyResultSelector(
       final Function2, TResult> resultSelector) {
     return new AbstractEnumerable() {
       @Override public Enumerator enumerator() {
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/QueryableDefaults.java b/linq4j/src/main/java/org/apache/calcite/linq4j/QueryableDefaults.java
index eab9a894aeef..d35de9c3b056 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/QueryableDefaults.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/QueryableDefaults.java
@@ -685,7 +685,7 @@ public static  TResult min(Queryable queryable,
    * 
      NOTE: clazz parameter not present in C# LINQ; necessary because of
    * Java type erasure.
    */
-  public static  Queryable ofType(Queryable queryable,
+  public static  Queryable ofType(Queryable queryable,
       Class clazz) {
     throw Extensions.todo();
   }
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/QueryableFactory.java b/linq4j/src/main/java/org/apache/calcite/linq4j/QueryableFactory.java
index b2a6750c4458..27357d88fded 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/QueryableFactory.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/QueryableFactory.java
@@ -523,7 +523,7 @@ > TResult min(Queryable source,
    * Filters the elements of an IQueryable based on a
    * specified type.
    */
-   Queryable ofType(Queryable source,
+   Queryable ofType(Queryable source,
       Class clazz);
 
    Queryable cast(Queryable source, Class clazz);
@@ -566,14 +566,14 @@  OrderedQueryable orderByDescending(Queryable source,
   /**
    * Projects each element of a sequence into a new form.
    */
-   Queryable select(Queryable source,
+   Queryable select(Queryable source,
       FunctionExpression> selector);
 
   /**
    * Projects each element of a sequence into a new
    * form by incorporating the element's index.
    */
-   Queryable selectN(Queryable source,
+   Queryable selectN(Queryable source,
       FunctionExpression> selector);
 
 
@@ -582,7 +582,7 @@  Queryable selectN(Queryable source,
    * {@code Enumerable} and combines the resulting sequences into one
    * sequence.
    */
-   Queryable selectMany(Queryable source,
+   Queryable selectMany(Queryable source,
       FunctionExpression>> selector);
 
   /**
@@ -591,7 +591,7 @@  Queryable selectMany(Queryable source,
    * sequence. The index of each source element is used in the
    * projected form of that element.
    */
-   Queryable selectManyN(Queryable source,
+   Queryable selectManyN(Queryable source,
       FunctionExpression>> selector);
 
   /**
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/QueryableRecorder.java b/linq4j/src/main/java/org/apache/calcite/linq4j/QueryableRecorder.java
index aac25216f6c6..c3b7cb615e97 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/QueryableRecorder.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/QueryableRecorder.java
@@ -630,7 +630,8 @@ public static  QueryableRecorder instance() {
     }.castSingle(); // CHECKSTYLE: IGNORE 0
   }
 
-  @Override public  Queryable ofType(final Queryable source,
+  @Override public 
+      Queryable ofType(final Queryable source,
       final Class clazz) {
     return new NonLeafReplayableQueryable(source) {
       @Override public void replay(QueryableFactory factory) {
@@ -687,7 +688,8 @@ public static  QueryableRecorder instance() {
     };
   }
 
-  @Override public  Queryable select(final Queryable source,
+  @Override public 
+      Queryable select(final Queryable source,
       final FunctionExpression> selector) {
     return new NonLeafReplayableQueryable(source) {
       @Override public void replay(QueryableFactory factory) {
@@ -700,7 +702,8 @@ public static  QueryableRecorder instance() {
     }.castQueryable(); // CHECKSTYLE: IGNORE 0
   }
 
-  @Override public  Queryable selectN(final Queryable source,
+  @Override public 
+      Queryable selectN(final Queryable source,
       final FunctionExpression> selector) {
     return new NonLeafReplayableQueryable(source) {
       @Override public void replay(QueryableFactory factory) {
@@ -709,7 +712,8 @@ public static  QueryableRecorder instance() {
     }.castQueryable(); // CHECKSTYLE: IGNORE 0
   }
 
-  @Override public  Queryable selectMany(final Queryable source,
+  @Override public 
+      Queryable selectMany(final Queryable source,
       final FunctionExpression>> selector) {
     return new NonLeafReplayableQueryable(source) {
       @Override public void replay(QueryableFactory factory) {
@@ -718,7 +722,8 @@ public static  QueryableRecorder instance() {
     }.castQueryable(); // CHECKSTYLE: IGNORE 0
   }
 
-  @Override public  Queryable selectManyN(final Queryable source,
+  @Override public 
+      Queryable selectManyN(final Queryable source,
       final FunctionExpression>>
         selector) {
     return new NonLeafReplayableQueryable(source) {

From 0fd3f8640d58cfbdf2541b9e1d71ac3f21ba1e6a Mon Sep 17 00:00:00 2001
From: Vladimir Sitnikov 
Date: Mon, 24 Aug 2026 13:53:07 +0300
Subject: [PATCH 515/562] [CALCITE-7736] Let the SQL collection helpers carry
 null elements

A SQL array or multiset may hold nulls, and an outer join hands the adapter a
null collection outright, so the types these helpers work with have to say so.

* `Functions.compareLists` and `compareMaps` accept `? extends @Nullable Object`
* `Linq4j.product` gives its element type a nullable bound
* the two outer-join adapters in `SqlFunctions` take `@Nullable List<@Nullable
  Object>` rather than `List`, which is what they were already handed
* `mapFromEntries` builds a `Map<@Nullable Object, @Nullable Object>` rather
  than a raw one
* `IS_JDK_8` reads `java.version`, which is always set, through `requireNonNull`

`ArrayCartesianProductEnumerable` calls `toArray` through a lambda rather than a
method reference, because the JDK model reports one signature at a call site and
requires another in an override.

Co-Authored-By: Claude Opus 5 
---
 .../apache/calcite/runtime/SqlFunctions.java  | 19 ++++++++++---------
 .../org/apache/calcite/linq4j/Linq4j.java     |  4 ++--
 .../calcite/linq4j/function/Functions.java    |  6 ++++--
 3 files changed, 16 insertions(+), 13 deletions(-)

diff --git a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java
index 59b93888772b..71f79aa6b3e8 100644
--- a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java
+++ b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java
@@ -224,14 +224,14 @@ public class SqlFunctions {
 
   /** Like {@link #LIST_AS_ENUMERABLE}, but for outer join mode: an empty or NULL
    * collection yields one NULL element rather than no elements. */
-  private static final Function1, Enumerable<@Nullable Object>>
-      OUTER_LIST_AS_ENUMERABLE =
+  private static final Function1<@Nullable List<@Nullable Object>,
+      Enumerable<@Nullable Object>> OUTER_LIST_AS_ENUMERABLE =
           a0 -> a0 == null || a0.isEmpty() ? Linq4j.asEnumerable(SINGLE_NULL)
               : Linq4j.asEnumerable(a0);
 
   /** Like {@link #STRUCT_LIST_AS_ENUMERABLE}, but for outer join mode. */
-  private static final Function1, Enumerable<@Nullable Object>>
-      OUTER_STRUCT_LIST_AS_ENUMERABLE =
+  private static final Function1<@Nullable List<@Nullable Object>,
+      Enumerable<@Nullable Object>> OUTER_STRUCT_LIST_AS_ENUMERABLE =
           a0 -> a0 == null || a0.isEmpty() ? Linq4j.asEnumerable(SINGLE_NULL)
               : Linq4j.asEnumerable(a0).<@Nullable Object>select(SqlFunctions::structValue);
 
@@ -273,7 +273,7 @@ private static class ArrayCartesianProductEnumerable
     }
 
     @Override public Enumerator<@Nullable Object[]> enumerator() {
-      return Linq4j.transform(product, List::toArray);
+      return Linq4j.transform(product, list -> list.toArray());
     }
   }
 
@@ -332,7 +332,8 @@ public static void resetThreadSequences() {
 
   /** Whether the current Java version is 8 (1.8). */
   private static final boolean IS_JDK_8 =
-      System.getProperty("java.version").startsWith("1.8");
+      requireNonNull(System.getProperty("java.version"), "java.version")
+          .startsWith("1.8");
 
   private SqlFunctions() {
   }
@@ -7499,7 +7500,7 @@ public static Map mapFromArrays(List keysArray, List valuesArray) {
     if (keysArray.size() != valuesArray.size()) {
       throw RESOURCE.illegalArgumentsInMapFromArraysFunc(keysArray.size(), valuesArray.size()).ex();
     }
-    final Map map = new LinkedHashMap<>();
+    final Map<@Nullable Object, @Nullable Object> map = new LinkedHashMap<>();
     for (int i = 0; i < keysArray.size(); i++) {
       map.put(keysArray.get(i), valuesArray.get(i));
     }
@@ -7508,7 +7509,7 @@ public static Map mapFromArrays(List keysArray, List valuesArray) {
 
   /** Support the MAP_FROM_ENTRIES function. */
   public static @Nullable Map mapFromEntries(List entries) {
-    final Map map = new LinkedHashMap<>();
+    final Map<@Nullable Object, @Nullable Object> map = new LinkedHashMap<>();
     for (Object entry : entries) {
       if (entry == null) {
         return null;
@@ -7523,7 +7524,7 @@ public static Map mapFromArrays(List keysArray, List valuesArray) {
    * 
      odd-indexed elements are keys and even-indexed elements are values.
    */
   public static Map map(Object... args) {
-    final Map map = new LinkedHashMap<>();
+    final Map<@Nullable Object, @Nullable Object> map = new LinkedHashMap<>();
     for (int i = 0; i < args.length; i += 2) {
       Object key = args[i];
       Object value = args[i + 1];
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/Linq4j.java b/linq4j/src/main/java/org/apache/calcite/linq4j/Linq4j.java
index 4b7fed20bb0d..1a8b1ad25cd1 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/Linq4j.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/Linq4j.java
@@ -386,13 +386,13 @@ public static  Enumerable concat(
    *
    * @return Enumerator over the cartesian product
    */
-  public static  Enumerator> product(
+  public static  Enumerator> product(
       List> enumerators) {
     return new CartesianProductListEnumerator<>(enumerators);
   }
 
   /** Returns the cartesian product of an iterable of iterables. */
-  public static  Iterable> product(
+  public static  Iterable> product(
       final Iterable> iterables) {
     return () -> {
       final List> enumerators = new ArrayList<>();
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/function/Functions.java b/linq4j/src/main/java/org/apache/calcite/linq4j/function/Functions.java
index b7bfa48b9b2f..4c1f8e5d5f28 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/function/Functions.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/function/Functions.java
@@ -714,7 +714,8 @@ private static class NullsFirstReverseComparator
     }
   }
 
-  public static int compareLists(List b0, List b1) {
+  public static int compareLists(List b0,
+      List b1) {
     if (b0 == b1) {
       return 0;
     }
@@ -735,7 +736,8 @@ public static int compareLists(List b0, List b1) {
    *
    * 
      Entries are compared in a canonical order, sorted by key and then by value.
    */
-  public static int compareMaps(Map b0, Map b1) {
+  public static int compareMaps(Map b0,
+      Map b1) {
     if (b0 == b1) {
       return 0;
     }

From 6317f10fea6b516982b7ecc140bc70c9c026451b Mon Sep 17 00:00:00 2001
From: Vladimir Sitnikov 
Date: Mon, 24 Aug 2026 14:04:04 +0300
Subject: [PATCH 516/562] [CALCITE-7736] Point the NullAway workarounds at the
 issues they came from

Three findings turned out to be NullAway defects rather than Calcite ones, and
are now filed upstream. The places that work around them say which:

* uber/NullAway#1726, a `@Contract` clause whose antecedent arity does not match
  the call crashes the analysis. The rule in the contributing guide says so.
* uber/NullAway#1727, a call returning `? extends T` reads as `@Nullable` when
  `T` has a nullable upper bound. Both `EnumerableDefaults.aggregate` overloads.
* uber/NullAway#1728, `Collection.toArray()` reports one signature at a call
  site and requires another in an override. `FlatLists` twice, and the lambda in
  `SqlFunctions` that replaced a `List::toArray` method reference.

Co-Authored-By: Claude Opus 5 
---
 core/src/main/java/org/apache/calcite/runtime/FlatLists.java | 5 +++++
 .../main/java/org/apache/calcite/runtime/SqlFunctions.java   | 3 +++
 .../java/org/apache/calcite/linq4j/EnumerableDefaults.java   | 2 ++
 site/develop/index.md                                        | 3 ++-
 4 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/core/src/main/java/org/apache/calcite/runtime/FlatLists.java b/core/src/main/java/org/apache/calcite/runtime/FlatLists.java
index 0318a7a1414f..c2926bb980e2 100644
--- a/core/src/main/java/org/apache/calcite/runtime/FlatLists.java
+++ b/core/src/main/java/org/apache/calcite/runtime/FlatLists.java
@@ -256,6 +256,8 @@ private static  ComparableList of_(List t) {
       //   write our own implementation and reduce creation overhead a
       //   bit.
       //noinspection unchecked
+      // toArray() yields @Nullable Object[] from the JDK model, and these elements are
+      // Comparable. https://github.com/uber/NullAway/issues/1728
       return new ComparableListImpl(Arrays.asList(castNonNullArray(t.toArray())));
     }
   }
@@ -1336,6 +1338,9 @@ protected ComparableListImpl(List list) {
     }
 
     @Override public Object[] toArray() {
+      // Object[] is the only return type the override check accepts, and the delegated
+      // call produces @Nullable Object[].
+      // https://github.com/uber/NullAway/issues/1728
       return castNonNullArray(this.list.toArray());
     }
 
diff --git a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java
index 71f79aa6b3e8..eb7631915f92 100644
--- a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java
+++ b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java
@@ -273,6 +273,9 @@ private static class ArrayCartesianProductEnumerable
     }
 
     @Override public Enumerator<@Nullable Object[]> enumerator() {
+      // A List::toArray method reference does not resolve: the JDK model reports one
+      // signature at a call site and requires another in an override.
+      // https://github.com/uber/NullAway/issues/1728
       return Linq4j.transform(product, list -> list.toArray());
     }
   }
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java b/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java
index 425ea1f81589..b42286a2d569 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java
@@ -105,6 +105,7 @@ public abstract class EnumerableDefaults {
   // NullAway treats the result of a call returning `? extends TAccumulate` as @Nullable
   // once it is assigned to a TAccumulate local, even though TAccumulate is the local's own
   // type. The wildcard is what lets a reducer with a non-null result feed a nullable seed.
+  // https://github.com/uber/NullAway/issues/1727
   @SuppressWarnings("NullAway")
   public static  TAccumulate aggregate(
       Enumerable source, TAccumulate seed,
@@ -128,6 +129,7 @@ public abstract class EnumerableDefaults {
   // NullAway treats the result of a call returning `? extends TAccumulate` as @Nullable
   // once it is assigned to a TAccumulate local, even though TAccumulate is the local's own
   // type. The wildcard is what lets a reducer with a non-null result feed a nullable seed.
+  // https://github.com/uber/NullAway/issues/1727
   @SuppressWarnings("NullAway")
   public static  TResult aggregate(
diff --git a/site/develop/index.md b/site/develop/index.md
index ccc55352ee85..bb01fd9e867f 100644
--- a/site/develop/index.md
+++ b/site/develop/index.md
@@ -265,7 +265,8 @@ so it is better to stick with `org.jspecify.annotations.Nullable`.
   per parameter, `!null` for the ones the result depends on and `_` for the rest. NullAway verifies
   the clause against the body, and a caller that passes a non-null argument gets a non-null result.
   The annotation does not apply to a receiver parameter, to a varargs method, or to a type argument
-  such as `Enumerable<@Nullable T>`.
+  such as `Enumerable<@Nullable T>`. A clause whose length does not match the call crashes
+  NullAway ([#1726](https://github.com/uber/NullAway/issues/1726)).
 
 * NullAway verifies code method by method. That means, it can't account for method execution order.
   That is why `@Nullable` fields should be verified in each method where they are used.

From 1bacbe74cdca8a41262837e3d699808e72492935 Mon Sep 17 00:00:00 2001
From: Vladimir Sitnikov 
Date: Mon, 24 Aug 2026 14:11:27 +0300
Subject: [PATCH 517/562] [CALCITE-7736] Read the HyperGraph edge maps through
 get rather than getOrDefault

`getOrDefault(key, new BitSet())` cannot return null: the maps hold non-null
`BitSet` values and the default is non-null. NullAway reported all 16 calls as
nullable anyway, because the fields are declared `HashMap`, which overrides
`getOrDefault`, and the model of `Map.getOrDefault` is not carried over to the
override. A field declared `Map` reports nothing, and so does `TreeMap`, which
inherits the method rather than overriding it.

One `edgesOf` helper reads the map with `get` and answers an empty set, which
depends on no model and says at one place what the 16 call sites were each
implying.

Reported as uber/NullAway#1729, see
nullaway-bugs/jdk-model-shadows-inherited-library-model.md.

Co-Authored-By: Claude Opus 5 
---
 .../apache/calcite/rel/rules/HyperGraph.java  | 48 ++++++++++++-------
 1 file changed, 32 insertions(+), 16 deletions(-)

diff --git a/core/src/main/java/org/apache/calcite/rel/rules/HyperGraph.java b/core/src/main/java/org/apache/calcite/rel/rules/HyperGraph.java
index 3f4e720be48b..a94e59b189c8 100644
--- a/core/src/main/java/org/apache/calcite/rel/rules/HyperGraph.java
+++ b/core/src/main/java/org/apache/calcite/rel/rules/HyperGraph.java
@@ -200,9 +200,25 @@ public long getNotProjectInputs() {
     return notProjectInputs;
   }
 
+  /** Returns the edges recorded for a vertex set, or an empty set.
+   *
+   * 
      Reads the map through {@link Map#get} rather than {@code getOrDefault}: the fields are
+   * declared {@link HashMap}, which overrides {@code getOrDefault}, and NullAway does not carry
+   * the model of the interface method over to the override.
+   * See NullAway#1729.
+   *
+   * @param map one of the edge maps
+   * @param key vertex set
+   * @return the edges, or an empty set
+   */
+  private static BitSet edgesOf(Map map, long key) {
+    final BitSet edges = map.get(key);
+    return edges == null ? new BitSet() : edges;
+  }
+
   public long getNeighborBitmap(long csg, long forbidden) {
     long neighbors = 0L;
-    List simpleEdges = simpleEdgesMap.getOrDefault(csg, new BitSet()).stream()
+    List simpleEdges = edgesOf(simpleEdgesMap, csg).stream()
         .mapToObj(edges::get)
         .collect(Collectors.toList());
     for (HyperEdge edge : simpleEdges) {
@@ -213,7 +229,7 @@ public long getNeighborBitmap(long csg, long forbidden) {
     neighbors = neighbors & ~forbidden;
     forbidden = forbidden | neighbors;
 
-    List complexEdges = complexEdgesMap.getOrDefault(csg, new BitSet()).stream()
+    List complexEdges = edgesOf(complexEdgesMap, csg).stream()
         .mapToObj(edges::get)
         .collect(Collectors.toList());
     for (HyperEdge edge : complexEdges) {
@@ -237,12 +253,12 @@ public List connectCsgCmp(long csg, long cmp) {
     checkArgument(simpleEdgesMap.containsKey(cmp));
     List connectedEdges = new ArrayList<>();
     BitSet connectedEdgesBitmap = new BitSet();
-    connectedEdgesBitmap.or(simpleEdgesMap.getOrDefault(csg, new BitSet()));
-    connectedEdgesBitmap.or(complexEdgesMap.getOrDefault(csg, new BitSet()));
+    connectedEdgesBitmap.or(edgesOf(simpleEdgesMap, csg));
+    connectedEdgesBitmap.or(edgesOf(complexEdgesMap, csg));
 
     BitSet cmpEdgesBitmap = new BitSet();
-    cmpEdgesBitmap.or(simpleEdgesMap.getOrDefault(cmp, new BitSet()));
-    cmpEdgesBitmap.or(complexEdgesMap.getOrDefault(cmp, new BitSet()));
+    cmpEdgesBitmap.or(edgesOf(simpleEdgesMap, cmp));
+    cmpEdgesBitmap.or(edgesOf(complexEdgesMap, cmp));
     connectedEdgesBitmap.and(cmpEdgesBitmap);
 
     // only consider the records related to csg and cmp in the simpleEdgesMap/complexEdgesMap,
@@ -250,8 +266,8 @@ public List connectCsgCmp(long csg, long cmp) {
     // csg = {t1, t3}, cmp = {t2}, will omit the edge (t1, t2)——(t3)
     BitSet mayMissedEdges = new BitSet();
     mayMissedEdges.or(complexEdgesBitmap.toBitSet());
-    mayMissedEdges.andNot(ccpUsedEdgesMap.getOrDefault(csg, new BitSet()));
-    mayMissedEdges.andNot(ccpUsedEdgesMap.getOrDefault(cmp, new BitSet()));
+    mayMissedEdges.andNot(edgesOf(ccpUsedEdgesMap, csg));
+    mayMissedEdges.andNot(edgesOf(ccpUsedEdgesMap, cmp));
     mayMissedEdges.andNot(connectedEdgesBitmap);
     mayMissedEdges.stream()
             .forEach(index -> {
@@ -264,8 +280,8 @@ public List connectCsgCmp(long csg, long cmp) {
     // record hyper edges are used by current csg ∪ cmp
     BitSet curUsedEdges = new BitSet();
     curUsedEdges.or(connectedEdgesBitmap);
-    curUsedEdges.or(ccpUsedEdgesMap.getOrDefault(csg, new BitSet()));
-    curUsedEdges.or(ccpUsedEdgesMap.getOrDefault(cmp, new BitSet()));
+    curUsedEdges.or(edgesOf(ccpUsedEdgesMap, csg));
+    curUsedEdges.or(edgesOf(ccpUsedEdgesMap, cmp));
     if (ccpUsedEdgesMap.containsKey(csg | cmp)) {
       checkArgument(
           curUsedEdges.equals(ccpUsedEdgesMap.get(csg | cmp)));
@@ -311,16 +327,16 @@ public void updateEdgesForUnion(long subset1, long subset2) {
     }
 
     BitSet unionSimpleBitSet = new BitSet();
-    unionSimpleBitSet.or(simpleEdgesMap.getOrDefault(subset1, new BitSet()));
-    unionSimpleBitSet.or(simpleEdgesMap.getOrDefault(subset2, new BitSet()));
+    unionSimpleBitSet.or(edgesOf(simpleEdgesMap, subset1));
+    unionSimpleBitSet.or(edgesOf(simpleEdgesMap, subset2));
 
     BitSet unionComplexBitSet = new BitSet();
-    unionComplexBitSet.or(complexEdgesMap.getOrDefault(subset1, new BitSet()));
-    unionComplexBitSet.or(complexEdgesMap.getOrDefault(subset2, new BitSet()));
+    unionComplexBitSet.or(edgesOf(complexEdgesMap, subset1));
+    unionComplexBitSet.or(edgesOf(complexEdgesMap, subset2));
 
     BitSet unionOverlapBitSet = new BitSet();
-    unionOverlapBitSet.or(overlapEdgesMap.getOrDefault(subset1, new BitSet()));
-    unionOverlapBitSet.or(overlapEdgesMap.getOrDefault(subset2, new BitSet()));
+    unionOverlapBitSet.or(edgesOf(overlapEdgesMap, subset1));
+    unionOverlapBitSet.or(edgesOf(overlapEdgesMap, subset2));
 
     // the overlaps edge that belongs to subset1/subset2
     // may be complex edge for subset1 union subset2

From 69e2fc51c61d2e19398b90f33a89a080df36cbc9 Mon Sep 17 00:00:00 2001
From: Vladimir Sitnikov 
Date: Mon, 24 Aug 2026 14:22:49 +0300
Subject: [PATCH 518/562] [CALCITE-7736] Let the null-tolerant collection
 helpers take nullable elements

`ImmutableNullableList` exists to hold nulls, and `Pair` is declared
``. Their static
factories said otherwise: a static method declares type parameters of its own,
and these had none, so a nullable element was rejected at every call.

* `ImmutableNullableList`, its three `copyOf` overloads and `builder`. The inner
  `Builder` already had the bound, which is why `builder()` returning
  `Builder` could not feed a `Builder<@Nullable Double>`.
* the 16 remaining statics on `Pair`: `zip`, `toMap`, `forEach`,
  `forEachIndexed`, `adjacents`, `firstAnd` and the rest.

Together these account for 62 of the findings, most of them in `RelMdSize`,
which computes a size per column and has no size for some of them.

Co-Authored-By: Claude Opus 5 
---
 .../calcite/util/ImmutableNullableList.java   | 10 +++----
 .../java/org/apache/calcite/util/Pair.java    | 30 +++++++++++--------
 2 files changed, 23 insertions(+), 17 deletions(-)

diff --git a/core/src/main/java/org/apache/calcite/util/ImmutableNullableList.java b/core/src/main/java/org/apache/calcite/util/ImmutableNullableList.java
index 26408597114f..d40f825b887b 100644
--- a/core/src/main/java/org/apache/calcite/util/ImmutableNullableList.java
+++ b/core/src/main/java/org/apache/calcite/util/ImmutableNullableList.java
@@ -37,7 +37,7 @@
  *
  * @param  Element type
  */
-public class ImmutableNullableList extends AbstractList {
+public class ImmutableNullableList extends AbstractList {
   private static final List SINGLETON_NULL = Collections.singletonList(null);
 
   private final E[] elements;
@@ -54,7 +54,7 @@ private ImmutableNullableList(E[] elements) {
    * {@link com.google.common.collect.ImmutableList#copyOf(java.util.Collection)}
    * except that this list allows nulls.
    */
-  public static  List copyOf(Collection elements) {
+  public static  List copyOf(Collection elements) {
     if (elements instanceof ImmutableNullableList
         || elements instanceof ImmutableList
         || elements == SINGLETON_NULL) {
@@ -82,7 +82,7 @@ public static  List copyOf(Collection elements) {
    * {@link com.google.common.collect.ImmutableList#copyOf(Iterable)}
    * except that this list allows nulls.
    */
-  public static  List copyOf(Iterable elements) {
+  public static  List copyOf(Iterable elements) {
     if (elements instanceof ImmutableNullableList
         || elements instanceof ImmutableList
         || elements == SINGLETON_NULL) {
@@ -109,7 +109,7 @@ public static  List copyOf(Iterable elements) {
    * {@link com.google.common.collect.ImmutableList#copyOf(Object[])}
    * except that this list allows nulls.
    */
-  public static  List copyOf(E[] elements) {
+  public static  List copyOf(E[] elements) {
     // Check for nulls.
     for (E object : elements) {
       if (object == null) {
@@ -198,7 +198,7 @@ public static  List of(E e1, E e2, E e3, E e4, E e5, E e6, E e7, E e8,
    * Returns a new builder. The generated builder is equivalent to the builder
    * created by the {@link Builder} constructor.
    */
-  public static  Builder builder() {
+  public static  Builder builder() {
     return new Builder<>();
   }
 
diff --git a/core/src/main/java/org/apache/calcite/util/Pair.java b/core/src/main/java/org/apache/calcite/util/Pair.java
index 3f1099a8c376..104d9c65eabe 100644
--- a/core/src/main/java/org/apache/calcite/util/Pair.java
+++ b/core/src/main/java/org/apache/calcite/util/Pair.java
@@ -88,7 +88,8 @@ public Pair(T1 left, T2 right) {
   }
 
   /** Creates a {@code Pair} from a {@link java.util.Map.Entry}. */
-  public static  Pair of(Map.Entry entry) {
+  public static 
+      Pair of(Map.Entry entry) {
     return of(entry.getKey(), entry.getValue());
   }
 
@@ -149,7 +150,8 @@ public static  Pair of(Map.Entry entry) {
    * @param pairs Collection of Pair objects
    * @return map with the same contents as the collection
    */
-  public static  Map toMap(Iterable> pairs) {
+  public static 
+      Map toMap(Iterable> pairs) {
     final Map map = new HashMap<>();
     for (Pair pair : pairs) {
       map.put(pair.left, pair.right);
@@ -167,7 +169,8 @@ public static  Map toMap(Iterable List> zip(List ks, List vs) {
+  public static 
+      List> zip(List ks, List vs) {
     return zip(ks, vs, false);
   }
 
@@ -183,7 +186,7 @@ public static  List> zip(List ks, List List> zip(
+  public static  List> zip(
       final List ks,
       final List vs,
       boolean strict) {
@@ -209,7 +212,7 @@ public static  List> zip(
    * @param vs Right iterable
    * @return Iterable over pairs
    */
-  public static  Iterable> zip(
+  public static  Iterable> zip(
       final Iterable ks,
       final Iterable vs) {
     return () -> {
@@ -231,7 +234,7 @@ public static  Iterable> zip(
    * @param vs Right array
    * @return List of pairs
    */
-  public static  List> zip(
+  public static  List> zip(
       final K[] ks,
       final V[] vs) {
     return new AbstractList>() {
@@ -252,7 +255,8 @@ public static  List> zip(
    *
    * @param  Key (left) value type
    * @param  Value (right) value type */
-  public static  List> zipMutable(
+  public static 
+      List> zipMutable(
       final List ks,
       final List vs) {
     return new MutableZipList<>(ks, vs);
@@ -273,7 +277,7 @@ public static  List> zipMutable(
    * @param  Left type
    * @param  Right type
    */
-  public static  void forEach(
+  public static  void forEach(
       final Iterable ks,
       final Iterable vs,
       BiConsumer consumer) {
@@ -286,7 +290,8 @@ public static  void forEach(
 
   /** Calls a consumer with an ordinal for each pair of items in two
    * iterables. */
-  public static  void forEachIndexed(Iterable ks, Iterable vs,
+  public static 
+      void forEachIndexed(Iterable ks, Iterable vs,
       PairWithOrdinalConsumer consumer) {
     int i = 0;
     final Iterator ki = ks.iterator();
@@ -298,7 +303,7 @@ public static  void forEachIndexed(Iterable ks, Iterable vs,
 
   /** Calls a consumer with an ordinal for each pair of items in an iterable
    * of pairs. */
-  public static  void forEachIndexed(
+  public static  void forEachIndexed(
       Iterable> pairs,
       PairWithOrdinalConsumer consumer) {
     int i = 0;
@@ -308,7 +313,8 @@ public static  void forEachIndexed(
   }
 
   /** Calls a consumer for each entry in a map. */
-  public static  void forEachIndexed(Map map,
+  public static 
+      void forEachIndexed(Map map,
       PairWithOrdinalConsumer consumer) {
     forEachIndexed(map.entrySet(), consumer);
   }
@@ -323,7 +329,7 @@ public static  void forEachIndexed(Map map,
    * @param  Left type
    * @param  Right type
    */
-  public static  void forEach(
+  public static  void forEach(
       final Iterable> entries,
       BiConsumer consumer) {
     for (Map.Entry entry : entries) {

From 41e3dd5bef97e2282a682025a53ad72e37dffadd Mon Sep 17 00:00:00 2001
From: Vladimir Sitnikov 
Date: Mon, 24 Aug 2026 14:28:42 +0300
Subject: [PATCH 519/562] [CALCITE-7736] Match the JDK signature in the
 ConsList toArray overrides

Same treatment `FlatLists` got. `java.util.List` declares
` T[] toArray(T[] a)`; the override declared
` @Nullable T[] toArray(T @Nullable [] a)` and then cast the argument back to
non-null to read its length. `Object[] toArray()` carried a
`ConsList<@Nullable E> this` receiver parameter, which is how the Checker
Framework said "only when E is nullable".

The two delegated `toArray()` calls are cast, and say why: the JDK model reports
`@Nullable Object[]` at a call site while the override check accepts only
`Object[]` (uber/NullAway#1728).

Co-Authored-By: Claude Opus 5 
---
 .../java/org/apache/calcite/runtime/ConsList.java | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/core/src/main/java/org/apache/calcite/runtime/ConsList.java b/core/src/main/java/org/apache/calcite/runtime/ConsList.java
index 856a1b20e956..5bb699a4339d 100644
--- a/core/src/main/java/org/apache/calcite/runtime/ConsList.java
+++ b/core/src/main/java/org/apache/calcite/runtime/ConsList.java
@@ -27,6 +27,7 @@
 import java.util.ListIterator;
 
 import static org.apache.calcite.linq4j.Nullness.castNonNull;
+import static org.apache.calcite.linq4j.Nullness.castNonNullArray;
 
 /**
  * List that consists of a head element and an immutable non-empty list.
@@ -114,13 +115,15 @@ private ConsList(E first, List rest) {
     return toList().listIterator(index);
   }
 
-  @Override public @Nullable Object[] toArray(ConsList<@Nullable E> this) {
-    return toList().toArray();
+  @Override public Object[] toArray() {
+    // toArray() yields @Nullable Object[] from the JDK model, and the override check
+    // accepts only Object[]. https://github.com/uber/NullAway/issues/1728
+    return castNonNullArray(toList().toArray());
   }
 
-  @Override public  @Nullable T[] toArray(T @Nullable [] a) {
+  @Override public  T[] toArray(T[] a) {
     final int s = size();
-    if (s > castNonNull(a).length) {
+    if (s > a.length) {
       a = (T[]) Arrays.copyOf(a, s, a.getClass());
     } else if (s < a.length) {
       a[s] = castNonNull(null);
@@ -130,7 +133,9 @@ private ConsList(E first, List rest) {
       //noinspection unchecked
       a[i++] = (T) c.first;
       if (!(c.rest instanceof ConsList)) {
-        Object[] a2 = c.rest.toArray();
+        // toArray() yields @Nullable Object[] from the JDK model.
+        // https://github.com/uber/NullAway/issues/1728
+        @Nullable Object[] a2 = c.rest.toArray();
         //noinspection SuspiciousSystemArraycopy
         System.arraycopy(a2, 0, a, i, a2.length);
         return a;

From 80ac2fec4ce002aea6227b4246a69fc257d403a1 Mon Sep 17 00:00:00 2001
From: Vladimir Sitnikov 
Date: Mon, 24 Aug 2026 14:38:22 +0300
Subject: [PATCH 520/562] [CALCITE-7736] Let SqlFunctions carry the nulls that
 SQL values have

The runtime works on SQL arrays, multisets and maps, whose elements and values
are nullable, and on rows whose fields may be absent. The types said otherwise.

* `Linq4j.asEnumerable` gives its element type a nullable bound, at all four
  overloads: viewing a collection as an `Enumerable` should not lose the fact
  that it holds nulls
* `Functions.compareMaps` accepts a nullable key type as well as a nullable
  value type, which is what a cast `(Map)` produces
* `FlatLists.ofSingle` and `of(List)` take nullable elements. They build a
  `Flat1List` and a flat list, both of which already admit them
* `flatListOuter` takes `@Nullable List<@Nullable Object>`, which is what an
  outer join hands it, and its body already tested for null
* `mapFromString` builds `Map`: a pair with no
  separator has no value. `slice` collects `structAccess` results, which are
  null for an absent field

Three locals that hold a `toArray()` result are declared `@Nullable Object[]`,
and say why (uber/NullAway#1728).

Co-Authored-By: Claude Opus 5 
---
 .../java/org/apache/calcite/runtime/FlatLists.java  |  4 ++--
 .../org/apache/calcite/runtime/SqlFunctions.java    | 13 ++++++++-----
 .../main/java/org/apache/calcite/linq4j/Linq4j.java | 10 ++++++----
 .../apache/calcite/linq4j/function/Functions.java   |  4 ++--
 4 files changed, 18 insertions(+), 13 deletions(-)

diff --git a/core/src/main/java/org/apache/calcite/runtime/FlatLists.java b/core/src/main/java/org/apache/calcite/runtime/FlatLists.java
index c2926bb980e2..d26b90ca31b7 100644
--- a/core/src/main/java/org/apache/calcite/runtime/FlatLists.java
+++ b/core/src/main/java/org/apache/calcite/runtime/FlatLists.java
@@ -65,7 +65,7 @@ public static  List of(T t0) {
    * @param  Element type
    * @return    List containing the given members
    */
-  public static  List ofSingle(T t0) {
+  public static  List ofSingle(T t0) {
     return new Flat1List<>(t0);
   }
 
@@ -224,7 +224,7 @@ private static  List flatListNotComparable(T[] t) {
    * @param  Element type
    * @return List containing the given members
    */
-  public static  List of(List t) {
+  public static  List of(List t) {
     return of_(t);
   }
 
diff --git a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java
index eb7631915f92..4c1e47c9832e 100644
--- a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java
+++ b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java
@@ -7294,7 +7294,7 @@ public static List arrayExcept(List list1, List list2) {
       return null;
     }
     int posInt = (int) pos;
-    Object[] baseArray = baselist.toArray();
+    @Nullable Object[] baseArray = baselist.toArray();
     if (posInt == 0 || posInt >= MAX_ARRAY_LENGTH || posInt <= -MAX_ARRAY_LENGTH) {
       throw new IllegalArgumentException("The index 0 is invalid. "
           + "An index shall be either < 0 or > 0 (the first element has index 1) "
@@ -7575,7 +7575,7 @@ public static Map map(Object... args) {
 
   /** Support the STR_TO_MAP function. */
   public static Map strToMap(String string, String stringDelimiter, String keyValueDelimiter) {
-    final Map map = new LinkedHashMap();
+    final Map map = new LinkedHashMap<>();
     final String[] keyValues = string.split(stringDelimiter, -1);
     for (String s : keyValues) {
       String[] keyValueArray = s.split(keyValueDelimiter, 2);
@@ -7639,7 +7639,7 @@ private static int rfind(String string, String delim, int start) {
 
   /** Support the SLICE function. */
   public static List slice(List list) {
-    List result = new ArrayList(list.size());
+    List<@Nullable Object> result = new ArrayList<>(list.size());
     for (Object e : list) {
       result.add(structAccess(e, 0, null));
     }
@@ -7860,7 +7860,8 @@ public static String arrayToString(List list, String delimiter, @Nullable String
    * Variant of {@link #flatList} for outer mode: an empty or {@code NULL}
    * collection yields one {@code NULL} element rather than no elements.
    */
-  public static Function1, Enumerable<@Nullable Object>> flatListOuter() {
+  public static Function1<@Nullable List<@Nullable Object>,
+      Enumerable<@Nullable Object>> flatListOuter() {
     return inputList -> inputList == null || inputList.isEmpty()
         ? Linq4j.asEnumerable(SINGLE_NULL)
         : Linq4j.asEnumerable(inputList)
@@ -8265,7 +8266,9 @@ private static class ProductComparableListEnumerator
     @Override public FlatLists.ComparableList current() {
       int i = 0;
       for (Object element : (Object[]) elements) {
-        Object[] a;
+        // toArray() yields @Nullable Object[] from the JDK model, and a SQL array may
+        // hold nulls. https://github.com/uber/NullAway/issues/1728
+        @Nullable Object[] a;
         if (element.getClass().isArray()) {
           a = (Object[]) element;
         } else {
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/Linq4j.java b/linq4j/src/main/java/org/apache/calcite/linq4j/Linq4j.java
index 1a8b1ad25cd1..67dbdc9ade9b 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/Linq4j.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/Linq4j.java
@@ -127,7 +127,7 @@ public static  Enumerator iterableEnumerator(
    *
    * @return enumerable
    */
-  public static  Enumerable asEnumerable(final List list) {
+  public static  Enumerable asEnumerable(final List list) {
     return new ListEnumerable<>(list);
   }
 
@@ -142,7 +142,8 @@ public static  Enumerable asEnumerable(final List list) {
    *
    * @return enumerable
    */
-  public static  Enumerable asEnumerable(final Collection collection) {
+  public static 
+      Enumerable asEnumerable(final Collection collection) {
     if (collection instanceof List) {
       //noinspection unchecked
       return asEnumerable((List) collection);
@@ -161,7 +162,8 @@ public static  Enumerable asEnumerable(final Collection collection) {
    *
    * @return enumerable
    */
-  public static  Enumerable asEnumerable(final Iterable iterable) {
+  public static 
+      Enumerable asEnumerable(final Iterable iterable) {
     if (iterable instanceof Collection) {
       //noinspection unchecked
       return asEnumerable((Collection) iterable);
@@ -177,7 +179,7 @@ public static  Enumerable asEnumerable(final Iterable iterable) {
    *
    * @return enumerable
    */
-  public static  Enumerable asEnumerable(final T[] ts) {
+  public static  Enumerable asEnumerable(final T[] ts) {
     return new ListEnumerable<>(Arrays.asList(ts));
   }
 
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/function/Functions.java b/linq4j/src/main/java/org/apache/calcite/linq4j/function/Functions.java
index 4c1f8e5d5f28..9721d8fa5dc8 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/function/Functions.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/function/Functions.java
@@ -736,8 +736,8 @@ public static int compareLists(List b0,
    *
    * 
      Entries are compared in a canonical order, sorted by key and then by value.
    */
-  public static int compareMaps(Map b0,
-      Map b1) {
+  public static int compareMaps(Map b0,
+      Map b1) {
     if (b0 == b1) {
       return 0;
     }

From 7e02d4464bed8d2b6f0cd85f1ff18a8aa503bb7c Mon Sep 17 00:00:00 2001
From: Vladimir Sitnikov 
Date: Mon, 24 Aug 2026 14:50:32 +0300
Subject: [PATCH 521/562] [CALCITE-7736] Let the shared utilities take nullable
 arguments

Six utilities that everything else calls declared type parameters without a
nullable bound, so a nullable argument was rejected at every call.

* `Ord` holds a nullable element type; its seven statics now say so
* `PairList`'s seven statics, matching the interface they build
* `Util.firstDuplicate`, which compares elements rather than dereferencing them
* `SqlOperator.acceptCall`, both overloads, and
  `SqlBasicVisitor.ArgHandler.instance`: a visitor result may be nullable, and
  `acceptCall` passes it through

`Util.first` is the Elvis operator: its result is non-null when the fallback
is, which `@Contract("_, !null -> !null")` states and NullAway verifies. Callers
in `RelBuilder`, `SqlValidatorImpl` and `ModelHandler` relied on that.

`Ord.zip(Iterator)` suppresses one finding: `next()` on an
`Iterator` reads as `@Nullable E` when `E` has a nullable upper
bound (uber/NullAway#1727).

91 findings.

Co-Authored-By: Claude Opus 5 
---
 .../org/apache/calcite/runtime/PairList.java  | 17 +++++++++-------
 .../org/apache/calcite/sql/SqlOperator.java   |  4 ++--
 .../calcite/sql/util/SqlBasicVisitor.java     |  2 +-
 .../java/org/apache/calcite/util/Util.java    |  5 +++--
 .../java/org/apache/calcite/linq4j/Ord.java   | 20 ++++++++++++-------
 5 files changed, 29 insertions(+), 19 deletions(-)

diff --git a/core/src/main/java/org/apache/calcite/runtime/PairList.java b/core/src/main/java/org/apache/calcite/runtime/PairList.java
index 5199d1cf0fb7..667d7cb76839 100644
--- a/core/src/main/java/org/apache/calcite/runtime/PairList.java
+++ b/core/src/main/java/org/apache/calcite/runtime/PairList.java
@@ -42,13 +42,13 @@
 public interface PairList
     extends List> {
   /** Creates an empty PairList. */
-  static  PairList of() {
+  static  PairList of() {
     return new PairLists.MutablePairList<>(new ArrayList<>());
   }
 
   /** Creates a singleton PairList. */
   @SuppressWarnings("RedundantCast")
-  static  PairList of(T t, U u) {
+  static  PairList of(T t, U u) {
     final List<@Nullable Object> list = new ArrayList<>();
     list.add((Object) t);
     list.add((Object) u);
@@ -56,14 +56,16 @@ static  PairList of(T t, U u) {
   }
 
   /** Creates a PairList with one or more entries. */
-  static  PairList copyOf(T t, U u, Object... rest) {
+  static 
+      PairList copyOf(T t, U u, Object... rest) {
     checkArgument(rest.length % 2 == 0, "even number");
     final List<@Nullable Object> list = Lists.asList(t, u, rest);
     return new PairLists.MutablePairList<>(new ArrayList<>(list));
   }
 
   /** Creates an empty PairList with a specified initial capacity. */
-  static  PairList withCapacity(int initialCapacity) {
+  static 
+      PairList withCapacity(int initialCapacity) {
     return backedBy(new ArrayList<>(initialCapacity));
   }
 
@@ -71,13 +73,14 @@ static  PairList withCapacity(int initialCapacity) {
    *
    * 
      Changes to the backing list will be reflected in the PairList.
    * If the backing list is immutable, this PairList will be also. */
-  static  PairList backedBy(List<@Nullable Object> list) {
+  static 
+      PairList backedBy(List<@Nullable Object> list) {
     return new PairLists.MutablePairList<>(list);
   }
 
   /** Creates a PairList from a Map. */
   @SuppressWarnings("RedundantCast")
-  static  PairList of(Map map) {
+  static  PairList of(Map map) {
     final List<@Nullable Object> list = new ArrayList<>(map.size() * 2);
     map.forEach((t, u) -> {
       list.add((Object) t);
@@ -87,7 +90,7 @@ static  PairList of(Map map) {
   }
 
   /** Creates a Builder. */
-  static  Builder builder() {
+  static  Builder builder() {
     return new Builder<>();
   }
 
diff --git a/core/src/main/java/org/apache/calcite/sql/SqlOperator.java b/core/src/main/java/org/apache/calcite/sql/SqlOperator.java
index 12dad78c287c..4ac84ac892b4 100644
--- a/core/src/main/java/org/apache/calcite/sql/SqlOperator.java
+++ b/core/src/main/java/org/apache/calcite/sql/SqlOperator.java
@@ -947,7 +947,7 @@ public boolean isGroupAuxiliary() {
    * @param visitor Visitor
    * @param call    Call to visit
    */
-  public  @Nullable R acceptCall(SqlVisitor visitor, SqlCall call) {
+  public  @Nullable R acceptCall(SqlVisitor visitor, SqlCall call) {
     for (SqlNode operand : call.getOperandList()) {
       if (operand == null) {
         continue;
@@ -972,7 +972,7 @@ public boolean isGroupAuxiliary() {
    *                        AS operator
    * @param argHandler      Called for each operand
    */
-  public  void acceptCall(
+  public  void acceptCall(
       SqlVisitor visitor,
       SqlCall call,
       boolean onlyExpressions,
diff --git a/core/src/main/java/org/apache/calcite/sql/util/SqlBasicVisitor.java b/core/src/main/java/org/apache/calcite/sql/util/SqlBasicVisitor.java
index 356f0e254fdc..73f110f94ae8 100644
--- a/core/src/main/java/org/apache/calcite/sql/util/SqlBasicVisitor.java
+++ b/core/src/main/java/org/apache/calcite/sql/util/SqlBasicVisitor.java
@@ -111,7 +111,7 @@ public static class ArgHandlerImpl implements ArgHan
     private static final ArgHandler INSTANCE = new ArgHandlerImpl<>();
 
     @SuppressWarnings("unchecked")
-    public static  ArgHandler instance() {
+    public static  ArgHandler instance() {
       return (ArgHandler) INSTANCE;
     }
 
diff --git a/core/src/main/java/org/apache/calcite/util/Util.java b/core/src/main/java/org/apache/calcite/util/Util.java
index 23faa50b3f32..c1fd0b0795de 100644
--- a/core/src/main/java/org/apache/calcite/util/Util.java
+++ b/core/src/main/java/org/apache/calcite/util/Util.java
@@ -2127,7 +2127,8 @@ public static  List> pairs(final List list) {
    *
    * 
      Equivalent to the Elvis operator ({@code ?:}) of languages such as
    * Groovy or PHP. */
-  public static  @Nullable T first(@Nullable T v0, @Nullable T v1) {
+  @Contract("_, !null -> !null")
+  public static  @Nullable T first(@Nullable T v0, @Nullable T v1) {
     return v0 != null ? v0 : v1;
   }
 
@@ -2271,7 +2272,7 @@ public static  boolean isDistinct(List list) {
    * @param list List
    * @return Ordinal of first duplicate, or -1 if not found
    */
-  public static  int firstDuplicate(List list) {
+  public static  int firstDuplicate(List list) {
     final int size = list.size();
     if (size < 2) {
       // Lists of size 0 and 1 are always distinct.
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/Ord.java b/linq4j/src/main/java/org/apache/calcite/linq4j/Ord.java
index 55baf3fc2af4..1428cc239965 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/Ord.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/Ord.java
@@ -48,7 +48,7 @@ public Ord(int i, E e) {
   /**
    * Creates an Ord.
    */
-  public static  Ord of(int n, E e) {
+  public static  Ord of(int n, E e) {
     return new Ord<>(n, e);
   }
 
@@ -66,14 +66,16 @@ public static  Ord of(int n, E e) {
   /**
    * Creates an iterable of {@code Ord}s over an iterable.
    */
-  public static  Iterable> zip(final Iterable iterable) {
+  public static 
+      Iterable> zip(final Iterable iterable) {
     return () -> zip(iterable.iterator());
   }
 
   /**
    * Creates an iterator of {@code Ord}s over an iterator.
    */
-  public static  Iterator> zip(final Iterator iterator) {
+  public static 
+      Iterator> zip(final Iterator iterator) {
     return new Iterator>() {
       int n = 0;
 
@@ -81,6 +83,9 @@ public static  Iterator> zip(final Iterator iterator) {
         return iterator.hasNext();
       }
 
+      // next() on an Iterator reads as @Nullable E when E has a nullable
+      // upper bound. https://github.com/uber/NullAway/issues/1727
+      @SuppressWarnings("NullAway")
       @Override public Ord next() {
         return Ord.of(n++, iterator.next());
       }
@@ -94,14 +99,14 @@ public static  Iterator> zip(final Iterator iterator) {
   /**
    * Returns a numbered list based on an array.
    */
-  public static  List> zip(final E[] elements) {
+  public static  List> zip(final E[] elements) {
     return new OrdArrayList<>(elements);
   }
 
   /**
    * Returns a numbered list.
    */
-  public static  List> zip(final List elements) {
+  public static  List> zip(final List elements) {
     return elements instanceof RandomAccess
         ? new OrdRandomAccessList<>(elements)
         : new OrdList<>(elements);
@@ -114,7 +119,7 @@ public static  List> zip(final List elements) {
    * (0, "a").
    */
   @SafeVarargs // heap pollution is not possible because we only read
-  public static  Iterable> reverse(E... elements) {
+  public static  Iterable> reverse(E... elements) {
     return reverse(ImmutableList.copyOf(elements));
   }
 
@@ -124,7 +129,8 @@ public static  Iterable> reverse(E... elements) {
    * 
      Given the list ["a", "b", "c"], returns (2, "c") then (1, "b") then
    * (0, "a").
    */
-  public static  Iterable> reverse(Iterable elements) {
+  public static 
+      Iterable> reverse(Iterable elements) {
     final ImmutableList elementList = ImmutableList.copyOf(elements);
     return () -> new Iterator>() {
       int i = elementList.size() - 1;

From 68b0bd99e3dd5e590791e3c5f091a96d947d0a82 Mon Sep 17 00:00:00 2001
From: Vladimir Sitnikov 
Date: Mon, 24 Aug 2026 14:59:49 +0300
Subject: [PATCH 522/562] [CALCITE-7736] Let the utility collections and
 visitors carry nulls

Eight files, all saying that a value cannot be null where it can.

* `ImmutableIntList.toArray`, both overrides, match the JDK signature
  ` T[] toArray(T[] a)` and stop casting the
  argument back to non-null to read its length. Same as `FlatLists` and
  `ConsList`.
* `RexNode.accept(RexBiVisitor, P)` and the `P` of `RexBiVisitor` and
  `RexBiVisitorImpl` take a nullable bound. `LogicVisitor` returns a nullable
  `Logic`, and could not be passed to its own `accept`.
* `Util.skip`, `SqlParserUtil.replaceSublist` and `ResultSetEnumerable.of` join
  the statics that carry the bound of the type they work on.
* `ArrayTable` reads a column whose values may be null into a list and an array
  that say so.

`Permutation.isValid` loses `@RequiresNonNull({"sources", "targets"})`: both are
`int[]`, and a primitive array is never null, so the precondition the Checker
Framework needed during construction states nothing NullAway does not know.

`JdbcTypeImpl` is suppressed and says why. Its `XXX_NULLABLE` constants return
null for a null column, and cannot say so: the enum implements `JdbcType` raw,
because a constant cannot carry its own type argument.

Co-Authored-By: Claude Opus 5 
---
 .../java/org/apache/calcite/adapter/clone/ArrayTable.java | 4 ++--
 .../main/java/org/apache/calcite/rex/RexBiVisitor.java    | 2 +-
 .../java/org/apache/calcite/rex/RexBiVisitorImpl.java     | 3 ++-
 core/src/main/java/org/apache/calcite/rex/RexNode.java    | 3 ++-
 .../org/apache/calcite/runtime/ResultSetEnumerable.java   | 4 ++--
 .../java/org/apache/calcite/sql/parser/SqlParserUtil.java | 2 +-
 .../java/org/apache/calcite/util/ImmutableIntList.java    | 8 ++++----
 .../main/java/org/apache/calcite/util/JdbcTypeImpl.java   | 5 ++++-
 .../main/java/org/apache/calcite/util/Permutation.java    | 2 --
 core/src/main/java/org/apache/calcite/util/Util.java      | 4 ++--
 10 files changed, 20 insertions(+), 17 deletions(-)

diff --git a/core/src/main/java/org/apache/calcite/adapter/clone/ArrayTable.java b/core/src/main/java/org/apache/calcite/adapter/clone/ArrayTable.java
index 5b36539fd028..bf77da18da5f 100644
--- a/core/src/main/java/org/apache/calcite/adapter/clone/ArrayTable.java
+++ b/core/src/main/java/org/apache/calcite/adapter/clone/ArrayTable.java
@@ -280,7 +280,7 @@ public static class ObjectArray implements Representation {
 
     @Override public Object freeze(ColumnLoader.ValueSet valueSet, int @Nullable [] sources) {
       // We assume the values have been canonized.
-      final List list = permuteList(valueSet.values, sources);
+      final List<@Nullable Comparable> list = permuteList(valueSet.values, sources);
       return list.toArray(new Comparable[0]);
     }
 
@@ -438,7 +438,7 @@ public static class ObjectDictionary implements Representation {
       Arrays.sort(nonNullCodeValues, 0, n);
       ColumnLoader.ValueSet codeValueSet =
           new ColumnLoader.ValueSet(int.class);
-      final List list = permuteList(valueSet.values, sources);
+      final List<@Nullable Comparable> list = permuteList(valueSet.values, sources);
       for (Comparable value : list) {
         int code;
         if (value == null) {
diff --git a/core/src/main/java/org/apache/calcite/rex/RexBiVisitor.java b/core/src/main/java/org/apache/calcite/rex/RexBiVisitor.java
index 37dd7745c73b..2babccb7b337 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexBiVisitor.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexBiVisitor.java
@@ -32,7 +32,7 @@
  * @param  Return type
  * @param 
       Payload type
  */
-public interface RexBiVisitor {
+public interface RexBiVisitor {
   //~ Methods ----------------------------------------------------------------
 
   R visitInputRef(RexInputRef inputRef, P arg);
diff --git a/core/src/main/java/org/apache/calcite/rex/RexBiVisitorImpl.java b/core/src/main/java/org/apache/calcite/rex/RexBiVisitorImpl.java
index fa8e3a2db94f..fe03069548f6 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexBiVisitorImpl.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexBiVisitorImpl.java
@@ -30,7 +30,8 @@
 // where they care. It is meaningful only when R is instantiated nullable, and JSpecify
 // tracks upper bounds, so it cannot require that.
 @SuppressWarnings("NullAway")
-public class RexBiVisitorImpl implements RexBiVisitor {
+public class RexBiVisitorImpl
+    implements RexBiVisitor {
   //~ Instance fields --------------------------------------------------------
 
   protected final boolean deep;
diff --git a/core/src/main/java/org/apache/calcite/rex/RexNode.java b/core/src/main/java/org/apache/calcite/rex/RexNode.java
index 3112e03fee54..11f5eb9f1c6c 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexNode.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexNode.java
@@ -113,7 +113,8 @@ public int nodeCount() {
    * Accepts a visitor with a payload, dispatching to the right overloaded
    * {@link RexBiVisitor#visitInputRef(RexInputRef, Object)} visitXxx} method.
    */
-  public abstract  R accept(RexBiVisitor visitor, P arg);
+  public abstract  R accept(
+      RexBiVisitor visitor, P arg);
 
   /** {@inheritDoc}
    *
diff --git a/core/src/main/java/org/apache/calcite/runtime/ResultSetEnumerable.java b/core/src/main/java/org/apache/calcite/runtime/ResultSetEnumerable.java
index 6db07d360740..c9370153a833 100644
--- a/core/src/main/java/org/apache/calcite/runtime/ResultSetEnumerable.java
+++ b/core/src/main/java/org/apache/calcite/runtime/ResultSetEnumerable.java
@@ -151,7 +151,7 @@ private ResultSetEnumerable(
 
   /** Executes a SQL query and returns the results as an enumerator, using a
    * row builder to convert JDBC column values into rows. */
-  public static  ResultSetEnumerable of(
+  public static  ResultSetEnumerable of(
       DataSource dataSource,
       String sql,
       Function1> rowBuilderFactory) {
@@ -163,7 +163,7 @@ public static  ResultSetEnumerable of(
    *
    * 
      It uses a {@link PreparedStatement} for computing the query result,
    * and that means that it can bind parameters. */
-  public static  ResultSetEnumerable of(
+  public static  ResultSetEnumerable of(
       DataSource dataSource,
       String sql,
       Function1> rowBuilderFactory,
diff --git a/core/src/main/java/org/apache/calcite/sql/parser/SqlParserUtil.java b/core/src/main/java/org/apache/calcite/sql/parser/SqlParserUtil.java
index ca47e878b6ff..486c586ed51d 100644
--- a/core/src/main/java/org/apache/calcite/sql/parser/SqlParserUtil.java
+++ b/core/src/main/java/org/apache/calcite/sql/parser/SqlParserUtil.java
@@ -902,7 +902,7 @@ public static String rightTrim(
    * example, if list contains {A, B, C, D, E} then 
    * replaceSublist(list, X, 1, 4) returns {A, X, E}.
    */
-  public static  void replaceSublist(
+  public static  void replaceSublist(
       List list,
       int start,
       int end,
diff --git a/core/src/main/java/org/apache/calcite/util/ImmutableIntList.java b/core/src/main/java/org/apache/calcite/util/ImmutableIntList.java
index 49427ff66d3b..0be7eee90bd5 100644
--- a/core/src/main/java/org/apache/calcite/util/ImmutableIntList.java
+++ b/core/src/main/java/org/apache/calcite/util/ImmutableIntList.java
@@ -179,9 +179,9 @@ public void forEachInt(IntConsumer action) {
     return objects;
   }
 
-  @Override public  @Nullable T[] toArray(T @Nullable [] a) {
+  @Override public  T[] toArray(T[] a) {
     final int size = ints.length;
-    if (castNonNull(a).length < size) {
+    if (a.length < size) {
       // Make a new array of a's runtime type, but my contents:
       a = a.getClass() == Object[].class
           ? (T[]) new Object[size]
@@ -335,8 +335,8 @@ private static class EmptyImmutableIntList extends ImmutableIntList {
       return EMPTY_ARRAY;
     }
 
-    @Override public  @Nullable T[] toArray(T @Nullable [] a) {
-      if (castNonNull(a).length > 0) {
+    @Override public  T[] toArray(T[] a) {
+      if (a.length > 0) {
         a[0] = castNonNull(null);
       }
       return a;
diff --git a/core/src/main/java/org/apache/calcite/util/JdbcTypeImpl.java b/core/src/main/java/org/apache/calcite/util/JdbcTypeImpl.java
index 3ae906b0cbc6..aafd9a5ddf53 100644
--- a/core/src/main/java/org/apache/calcite/util/JdbcTypeImpl.java
+++ b/core/src/main/java/org/apache/calcite/util/JdbcTypeImpl.java
@@ -32,7 +32,10 @@
  * 
      It is frustrating that we can't use an {@code enum} to implement an
  * interface with a type parameter. At times like this, we wish Java had
  * Generalized Algebraic Data Types (GADTs). */
-@SuppressWarnings("rawtypes")
+// The XXX_NULLABLE constants return null for a null column. They cannot say so in their
+// signature: the enum implements JdbcType raw, because a constant cannot carry its own
+// type argument, so every get() is checked against the erased T.
+@SuppressWarnings({"rawtypes", "NullAway"})
 enum JdbcTypeImpl implements JdbcType {
   BIG_DECIMAL(BigDecimal.class, false) {
     @Override public BigDecimal get(int column,
diff --git a/core/src/main/java/org/apache/calcite/util/Permutation.java b/core/src/main/java/org/apache/calcite/util/Permutation.java
index f93c97b273ec..58ba67b1d72e 100644
--- a/core/src/main/java/org/apache/calcite/util/Permutation.java
+++ b/core/src/main/java/org/apache/calcite/util/Permutation.java
@@ -16,7 +16,6 @@
  */
 package org.apache.calcite.util;
 
-import org.apache.calcite.linq4j.annotations.RequiresNonNull;
 import org.apache.calcite.util.mapping.IntPair;
 import org.apache.calcite.util.mapping.Mapping;
 import org.apache.calcite.util.mapping.MappingType;
@@ -435,7 +434,6 @@ private void setInternal(int source, int target) {
    * @param fail Whether to assert if invalid
    * @return Whether valid
    */
-  @RequiresNonNull({"sources", "targets"})
   private boolean isValid(boolean fail) {
     final int size = targets.length;
     if (sources.length != size) {
diff --git a/core/src/main/java/org/apache/calcite/util/Util.java b/core/src/main/java/org/apache/calcite/util/Util.java
index c1fd0b0795de..a8c5b4d9de9b 100644
--- a/core/src/main/java/org/apache/calcite/util/Util.java
+++ b/core/src/main/java/org/apache/calcite/util/Util.java
@@ -2221,12 +2221,12 @@ public static  List last(List list, int n) {
   }
 
   /** Returns all but the first element of a list. */
-  public static  List skip(List list) {
+  public static  List skip(List list) {
     return skip(list, 1);
   }
 
   /** Returns all but the first {@code n} elements of a list. */
-  public static  List skip(List list, int fromIndex) {
+  public static  List skip(List list, int fromIndex) {
     return fromIndex == 0 ? list : list.subList(fromIndex, list.size());
   }
 

From 8b82a02fca3573acfaa4d3066ade877c588b7c5d Mon Sep 17 00:00:00 2001
From: Vladimir Sitnikov 
Date: Mon, 24 Aug 2026 15:08:13 +0300
Subject: [PATCH 523/562] [CALCITE-7736] Drop the preconditions that only the
 Checker Framework needed

`@RequiresNonNull` states which nullable fields a caller must have checked.
`RexLiteral.computeDigest`, `RexLiteral.digestIncludesType` and
`Resources.Prop.getDefault` named fields that are `final` and non-null:
`RexLiteral.type` and `typeName`, and `Resources.Element.method`. The Checker
Framework needed them because the methods run while the object is still under
construction, which it tracked and NullAway does not. Under NullAway the
annotations state nothing new and reject every caller.

Same reason `Permutation.isValid` lost its own in the previous commit.

Also here: the 16 overrides of `RexNode.accept(RexBiVisitor, P)` take the
bounds the abstract method now has, and `ImmutableNullableList.of`, all eight
overloads, joins `copyOf` and `builder` in admitting the nulls the class exists
to hold. `JdbcUtils` builds its connection key from a url, user, password and
driver class, of which only the url is required.

42 findings.

Co-Authored-By: Claude Opus 5 
---
 .../apache/calcite/adapter/jdbc/JdbcUtils.java  |  3 ++-
 .../java/org/apache/calcite/rex/RexCall.java    |  3 ++-
 .../apache/calcite/rex/RexCorrelVariable.java   |  3 ++-
 .../org/apache/calcite/rex/RexDynamicParam.java |  3 ++-
 .../org/apache/calcite/rex/RexFieldAccess.java  |  3 ++-
 .../org/apache/calcite/rex/RexInputRef.java     |  3 ++-
 .../java/org/apache/calcite/rex/RexLambda.java  |  3 ++-
 .../org/apache/calcite/rex/RexLambdaRef.java    |  3 ++-
 .../java/org/apache/calcite/rex/RexLiteral.java |  6 ++----
 .../org/apache/calcite/rex/RexLocalRef.java     |  3 ++-
 .../calcite/rex/RexNodeAndFieldIndex.java       |  3 ++-
 .../java/org/apache/calcite/rex/RexOver.java    |  3 ++-
 .../apache/calcite/rex/RexPatternFieldRef.java  |  3 ++-
 .../org/apache/calcite/rex/RexRangeRef.java     |  3 ++-
 .../org/apache/calcite/rex/RexSimplify.java     |  3 ++-
 .../org/apache/calcite/rex/RexSubQuery.java     |  3 ++-
 .../apache/calcite/rex/RexTableInputRef.java    |  3 ++-
 .../org/apache/calcite/runtime/Resources.java   |  2 --
 .../calcite/util/ImmutableNullableList.java     | 17 +++++++++--------
 19 files changed, 43 insertions(+), 30 deletions(-)

diff --git a/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcUtils.java b/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcUtils.java
index 5243ca6da207..0d4ba2a09207 100644
--- a/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcUtils.java
+++ b/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcUtils.java
@@ -274,7 +274,8 @@ public DataSource get(String url, @Nullable String driverClassName,
       // Get data source objects from a cache, so that we don't have to sniff
       // out what kind of database they are quite as often.
       final List<@Nullable String> key =
-          ImmutableNullableList.of(url, username, password, driverClassName);
+          ImmutableNullableList.<@Nullable String>of(url, username, password,
+              driverClassName);
       return cache.getUnchecked(key);
     }
   }
diff --git a/core/src/main/java/org/apache/calcite/rex/RexCall.java b/core/src/main/java/org/apache/calcite/rex/RexCall.java
index bba28f275361..f2c17fc9df1a 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexCall.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexCall.java
@@ -196,7 +196,8 @@ private boolean digestWithType() {
     return visitor.visitCall(this);
   }
 
-  @Override public  R accept(RexBiVisitor visitor, P arg) {
+  @Override public  R accept(
+      RexBiVisitor visitor, P arg) {
     return visitor.visitCall(this, arg);
   }
 
diff --git a/core/src/main/java/org/apache/calcite/rex/RexCorrelVariable.java b/core/src/main/java/org/apache/calcite/rex/RexCorrelVariable.java
index 95b5ff1294c8..2148e85e9852 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexCorrelVariable.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexCorrelVariable.java
@@ -51,7 +51,8 @@ public class RexCorrelVariable extends RexVariable {
     return visitor.visitCorrelVariable(this);
   }
 
-  @Override public  R accept(RexBiVisitor visitor, P arg) {
+  @Override public  R accept(
+      RexBiVisitor visitor, P arg) {
     return visitor.visitCorrelVariable(this, arg);
   }
 
diff --git a/core/src/main/java/org/apache/calcite/rex/RexDynamicParam.java b/core/src/main/java/org/apache/calcite/rex/RexDynamicParam.java
index aa011a46494b..629fc4a9d89d 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexDynamicParam.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexDynamicParam.java
@@ -60,7 +60,8 @@ public int getIndex() {
     return visitor.visitDynamicParam(this);
   }
 
-  @Override public  R accept(RexBiVisitor visitor, P arg) {
+  @Override public  R accept(
+      RexBiVisitor visitor, P arg) {
     return visitor.visitDynamicParam(this, arg);
   }
 
diff --git a/core/src/main/java/org/apache/calcite/rex/RexFieldAccess.java b/core/src/main/java/org/apache/calcite/rex/RexFieldAccess.java
index 7db5629ec2a0..6182cec013db 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexFieldAccess.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexFieldAccess.java
@@ -103,7 +103,8 @@ public RelDataTypeField getField() {
     return visitor.visitFieldAccess(this);
   }
 
-  @Override public  R accept(RexBiVisitor visitor, P arg) {
+  @Override public  R accept(
+      RexBiVisitor visitor, P arg) {
     return visitor.visitFieldAccess(this, arg);
   }
 
diff --git a/core/src/main/java/org/apache/calcite/rex/RexInputRef.java b/core/src/main/java/org/apache/calcite/rex/RexInputRef.java
index 79492da8ce91..a86a3a4e3737 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexInputRef.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexInputRef.java
@@ -125,7 +125,8 @@ public static void add2(PairList list,
     return visitor.visitInputRef(this);
   }
 
-  @Override public  R accept(RexBiVisitor visitor, P arg) {
+  @Override public  R accept(
+      RexBiVisitor visitor, P arg) {
     return visitor.visitInputRef(this, arg);
   }
 
diff --git a/core/src/main/java/org/apache/calcite/rex/RexLambda.java b/core/src/main/java/org/apache/calcite/rex/RexLambda.java
index 31a5d7c9d850..3c59bb76725c 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexLambda.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexLambda.java
@@ -59,7 +59,8 @@ public class RexLambda extends RexNode {
     return visitor.visitLambda(this);
   }
 
-  @Override public  R accept(RexBiVisitor visitor, P arg) {
+  @Override public  R accept(
+      RexBiVisitor visitor, P arg) {
     return visitor.visitLambda(this, arg);
   }
 
diff --git a/core/src/main/java/org/apache/calcite/rex/RexLambdaRef.java b/core/src/main/java/org/apache/calcite/rex/RexLambdaRef.java
index edde8abc1405..4688405d00cb 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexLambdaRef.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexLambdaRef.java
@@ -40,7 +40,8 @@ public RexLambdaRef(int index, String name, RelDataType type) {
     return visitor.visitLambdaRef(this);
   }
 
-  @Override public  R accept(RexBiVisitor visitor, P arg) {
+  @Override public  R accept(
+      RexBiVisitor visitor, P arg) {
     return (R) null;
   }
 
diff --git a/core/src/main/java/org/apache/calcite/rex/RexLiteral.java b/core/src/main/java/org/apache/calcite/rex/RexLiteral.java
index 89394da3edce..95054884323b 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexLiteral.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexLiteral.java
@@ -21,7 +21,6 @@
 import org.apache.calcite.avatica.util.TimeUnit;
 import org.apache.calcite.config.CalciteSystemProperty;
 import org.apache.calcite.linq4j.annotations.Contract;
-import org.apache.calcite.linq4j.annotations.RequiresNonNull;
 import org.apache.calcite.linq4j.function.Functions;
 import org.apache.calcite.rel.RelNode;
 import org.apache.calcite.rel.type.RelDataType;
@@ -276,7 +275,6 @@ public class RexLiteral extends RexNode {
    * @param includeType whether the digest should include type or not
    * @return digest
    */
-  @RequiresNonNull({"typeName", "type"})
   public final String computeDigest(
       RexDigestIncludeType includeType) {
     if (includeType == RexDigestIncludeType.OPTIONAL) {
@@ -303,7 +301,6 @@ public final String computeDigest(
    * @see RexCall#computeDigest(boolean)
    * @return whether {@link RexDigestIncludeType} digest would include data type
    */
-  @RequiresNonNull("type")
   RexDigestIncludeType digestIncludesType() {
     return shouldIncludeType(value, type);
   }
@@ -1341,7 +1338,8 @@ public static boolean isNullLiteral(RexNode node) {
     return visitor.visitLiteral(this);
   }
 
-  @Override public  R accept(RexBiVisitor visitor, P arg) {
+  @Override public  R accept(
+      RexBiVisitor visitor, P arg) {
     return visitor.visitLiteral(this, arg);
   }
 }
diff --git a/core/src/main/java/org/apache/calcite/rex/RexLocalRef.java b/core/src/main/java/org/apache/calcite/rex/RexLocalRef.java
index bb5e1b6d3367..d9d3a9b07af6 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexLocalRef.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexLocalRef.java
@@ -78,7 +78,8 @@ public RexLocalRef(int index, RelDataType type) {
     return visitor.visitLocalRef(this);
   }
 
-  @Override public  R accept(RexBiVisitor visitor, P arg) {
+  @Override public  R accept(
+      RexBiVisitor visitor, P arg) {
     return visitor.visitLocalRef(this, arg);
   }
 
diff --git a/core/src/main/java/org/apache/calcite/rex/RexNodeAndFieldIndex.java b/core/src/main/java/org/apache/calcite/rex/RexNodeAndFieldIndex.java
index be03733d45e1..fc2cf355cda2 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexNodeAndFieldIndex.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexNodeAndFieldIndex.java
@@ -78,7 +78,8 @@ public int getFieldIndex() {
     return visitor.visitNodeAndFieldIndex(this);
   }
 
-  @Override public  R accept(RexBiVisitor visitor, P arg) {
+  @Override public  R accept(
+      RexBiVisitor visitor, P arg) {
     return visitor.visitNodeAndFieldIndex(this, arg);
   }
 
diff --git a/core/src/main/java/org/apache/calcite/rex/RexOver.java b/core/src/main/java/org/apache/calcite/rex/RexOver.java
index c8f2568ab0e9..4336ae9e42d3 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexOver.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexOver.java
@@ -156,7 +156,8 @@ public boolean ignoreNulls() {
     return visitor.visitOver(this);
   }
 
-  @Override public  R accept(RexBiVisitor visitor, P arg) {
+  @Override public  R accept(
+      RexBiVisitor visitor, P arg) {
     return visitor.visitOver(this, arg);
   }
 
diff --git a/core/src/main/java/org/apache/calcite/rex/RexPatternFieldRef.java b/core/src/main/java/org/apache/calcite/rex/RexPatternFieldRef.java
index 7543b63a8bfe..a991a6894fb4 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexPatternFieldRef.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexPatternFieldRef.java
@@ -49,7 +49,8 @@ public static RexPatternFieldRef of(String alpha, RexInputRef ref) {
     return visitor.visitPatternFieldRef(this);
   }
 
-  @Override public  R accept(RexBiVisitor visitor, P arg) {
+  @Override public  R accept(
+      RexBiVisitor visitor, P arg) {
     return visitor.visitPatternFieldRef(this, arg);
   }
 
diff --git a/core/src/main/java/org/apache/calcite/rex/RexRangeRef.java b/core/src/main/java/org/apache/calcite/rex/RexRangeRef.java
index 845cb071d9ec..c1a117774500 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexRangeRef.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexRangeRef.java
@@ -75,7 +75,8 @@ public int getOffset() {
     return visitor.visitRangeRef(this);
   }
 
-  @Override public  R accept(RexBiVisitor visitor, P arg) {
+  @Override public  R accept(
+      RexBiVisitor visitor, P arg) {
     return visitor.visitRangeRef(this, arg);
   }
 
diff --git a/core/src/main/java/org/apache/calcite/rex/RexSimplify.java b/core/src/main/java/org/apache/calcite/rex/RexSimplify.java
index 5c7fb733adb0..2170c8a146b3 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexSimplify.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexSimplify.java
@@ -3689,7 +3689,8 @@ > Sarg build(boolean negate) {
       throw new UnsupportedOperationException();
     }
 
-    @Override public  R accept(RexBiVisitor visitor, P arg) {
+    @Override public  R accept(
+      RexBiVisitor visitor, P arg) {
       throw new UnsupportedOperationException();
     }
 
diff --git a/core/src/main/java/org/apache/calcite/rex/RexSubQuery.java b/core/src/main/java/org/apache/calcite/rex/RexSubQuery.java
index 02a16ad0f02d..b2cbf9c0985c 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexSubQuery.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexSubQuery.java
@@ -164,7 +164,8 @@ public static RexSubQuery map(RelNode rel) {
     return visitor.visitSubQuery(this);
   }
 
-  @Override public  R accept(RexBiVisitor visitor, P arg) {
+  @Override public  R accept(
+      RexBiVisitor visitor, P arg) {
     return visitor.visitSubQuery(this, arg);
   }
 
diff --git a/core/src/main/java/org/apache/calcite/rex/RexTableInputRef.java b/core/src/main/java/org/apache/calcite/rex/RexTableInputRef.java
index 432c5c7fc612..7a62ce7fdffa 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexTableInputRef.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexTableInputRef.java
@@ -89,7 +89,8 @@ public static RexTableInputRef of(RelTableRef tableRef, RexInputRef ref) {
     return visitor.visitTableInputRef(this);
   }
 
-  @Override public  R accept(RexBiVisitor visitor, P arg) {
+  @Override public  R accept(
+      RexBiVisitor visitor, P arg) {
     return visitor.visitTableInputRef(this, arg);
   }
 
diff --git a/core/src/main/java/org/apache/calcite/runtime/Resources.java b/core/src/main/java/org/apache/calcite/runtime/Resources.java
index 3ff86dcd42f0..5770b5ee11b6 100644
--- a/core/src/main/java/org/apache/calcite/runtime/Resources.java
+++ b/core/src/main/java/org/apache/calcite/runtime/Resources.java
@@ -18,7 +18,6 @@
 
 import org.apache.calcite.linq4j.annotations.Contract;
 import org.jspecify.annotations.Nullable;
-import org.apache.calcite.linq4j.annotations.RequiresNonNull;
 
 import java.io.IOException;
 import java.io.InputStream;
@@ -624,7 +623,6 @@ protected Prop(PropertyAccessor accessor, Method method) {
       this.hasDefault = resource != null;
     }
 
-    @RequiresNonNull("method")
     protected final @Nullable Default getDefault() {
       if (hasDefault) {
         return castNonNull(method.getAnnotation(Default.class));
diff --git a/core/src/main/java/org/apache/calcite/util/ImmutableNullableList.java b/core/src/main/java/org/apache/calcite/util/ImmutableNullableList.java
index d40f825b887b..4ca730415f0a 100644
--- a/core/src/main/java/org/apache/calcite/util/ImmutableNullableList.java
+++ b/core/src/main/java/org/apache/calcite/util/ImmutableNullableList.java
@@ -122,55 +122,56 @@ private ImmutableNullableList(E[] elements) {
   }
 
   /** Creates an immutable list of 1 element. */
-  public static  List of(@Nullable E e1) {
+  public static  List of(@Nullable E e1) {
     //noinspection unchecked
     return e1 == null ? (List) SINGLETON_NULL : ImmutableList.of(e1);
   }
 
   /** Creates an immutable list of 2 elements. */
-  public static  List of(E e1, E e2) {
+  public static  List of(E e1, E e2) {
     // Only we can see the varargs array. Therefore the list is immutable.
     //noinspection unchecked
     return UnmodifiableArrayList.of(e1, e2);
   }
 
   /** Creates an immutable list of 3 elements. */
-  public static  List of(E e1, E e2, E e3) {
+  public static  List of(E e1, E e2, E e3) {
     // Only we can see the varargs array. Therefore the list is immutable.
     //noinspection unchecked
     return UnmodifiableArrayList.of(e1, e2, e3);
   }
 
   /** Creates an immutable list of 4 elements. */
-  public static  List of(E e1, E e2, E e3, E e4) {
+  public static  List of(E e1, E e2, E e3, E e4) {
     // Only we can see the varargs array. Therefore the list is immutable.
     //noinspection unchecked
     return UnmodifiableArrayList.of(e1, e2, e3, e4);
   }
 
   /** Creates an immutable list of 5 elements. */
-  public static  List of(E e1, E e2, E e3, E e4, E e5) {
+  public static  List of(E e1, E e2, E e3, E e4, E e5) {
     // Only we can see the varargs array. Therefore the list is immutable.
     //noinspection unchecked
     return UnmodifiableArrayList.of(e1, e2, e3, e4, e5);
   }
 
   /** Creates an immutable list of 6 elements. */
-  public static  List of(E e1, E e2, E e3, E e4, E e5, E e6) {
+  public static  List of(E e1, E e2, E e3, E e4, E e5, E e6) {
     // Only we can see the varargs array. Therefore the list is immutable.
     //noinspection unchecked
     return UnmodifiableArrayList.of(e1, e2, e3, e4, e5, e6);
   }
 
   /** Creates an immutable list of 7 elements. */
-  public static  List of(E e1, E e2, E e3, E e4, E e5, E e6, E e7) {
+  public static  List of(E e1, E e2, E e3, E e4, E e5, E e6, E e7) {
     // Only we can see the varargs array. Therefore the list is immutable.
     //noinspection unchecked
     return UnmodifiableArrayList.of(e1, e2, e3, e4, e5, e6, e7);
   }
 
   /** Creates an immutable list of 8 or more elements. */
-  public static  List of(E e1, E e2, E e3, E e4, E e5, E e6, E e7, E e8,
+  public static 
+      List of(E e1, E e2, E e3, E e4, E e5, E e6, E e7, E e8,
       E... others) {
     @SuppressWarnings("unchecked")
     E[] array = (E[]) new Object[8 + others.length];

From 0b9dbd6f8e2d110f4ac6edca9e3add2591c7569b Mon Sep 17 00:00:00 2001
From: Vladimir Sitnikov 
Date: Mon, 24 Aug 2026 15:19:19 +0300
Subject: [PATCH 524/562] [CALCITE-7736] Say what the guards already check, and
 drop two more dead preconditions

`canBeLong` answers `o instanceof Boolean || ...`, which is false for null.
`@Contract("null -> false")` states that, and NullAway narrows the caller in
`ColumnLoader` accordingly, where a dropped `@EnsuresNonNullIf` used to.

`LoptMultiJoin` loses `@RequiresNonNull` from six methods: `joinStart` and
`nFieldsInJoinFactor` are `int[]`, and a primitive array is never null. Same as
`Permutation`, `RexLiteral` and `Resources`.

`LoptOptimizeJoinRule` reads each table out of `simpleFactors` once, through
`requireNonNull`, rather than looking the same key up three times. The keys come
from that map, which a dropped `@KeyFor` used to say.

`ChunkList` packs the neighbouring chunks into slots 0 and 1 of the element
array, so those two writes are suppressed and say why. The two places that
narrow a chunk reference now assign the result of `castNonNull` instead of
casting the same nullable local at each use.

`CalciteConnectionConfigImpl` suppresses its six plugin accessors: `getPlugin`
comes from Avatica, which is not annotated, so NullAway cannot infer one `T`
that suits both the `Class` argument and a nullable default. The `@Contract` on
each stays, and NullAway verifies it.

Three lambdas in `ColumnLoader` drop their explicit parameter type, which was
non-null where the functional interface says nullable.

Co-Authored-By: Claude Opus 5 
---
 .../calcite/adapter/clone/ColumnLoader.java    |  8 +++++---
 .../config/CalciteConnectionConfigImpl.java    | 18 ++++++++++++++++++
 .../calcite/rel/rules/LoptMultiJoin.java       |  4 ----
 .../rel/rules/LoptOptimizeJoinRule.java        | 10 +++++++---
 .../org/apache/calcite/util/ChunkList.java     | 15 ++++++++-------
 5 files changed, 38 insertions(+), 17 deletions(-)

diff --git a/core/src/main/java/org/apache/calcite/adapter/clone/ColumnLoader.java b/core/src/main/java/org/apache/calcite/adapter/clone/ColumnLoader.java
index 6cf1c3489d25..181b78d6990a 100644
--- a/core/src/main/java/org/apache/calcite/adapter/clone/ColumnLoader.java
+++ b/core/src/main/java/org/apache/calcite/adapter/clone/ColumnLoader.java
@@ -21,6 +21,7 @@
 import org.apache.calcite.avatica.util.DateTimeUtils;
 import org.apache.calcite.linq4j.Enumerable;
 import org.apache.calcite.linq4j.Ord;
+import org.apache.calcite.linq4j.annotations.Contract;
 import org.apache.calcite.linq4j.tree.Primitive;
 import org.apache.calcite.rel.type.RelDataType;
 import org.apache.calcite.rel.type.RelDataTypeField;
@@ -250,7 +251,7 @@ private void load(final RelDataType elementType,
       case JAVA_SQL_TIMESTAMP:
         final List<@Nullable Long> longs =
             Util.transform((List<@Nullable Timestamp>) list,
-                (Timestamp t) -> t == null ? null : t.getTime());
+                t -> t == null ? null : t.getTime());
         return longs;
       default:
         break;
@@ -262,7 +263,7 @@ private void load(final RelDataType elementType,
       case JAVA_SQL_TIME:
         return Util.<@Nullable Time, @Nullable Integer>transform(
             (List<@Nullable Time>) list,
-            (Time t) -> t == null ? null
+            t -> t == null ? null
                 : (int) (t.getTime() % DateTimeUtils.MILLIS_PER_DAY));
       default:
         break;
@@ -273,7 +274,7 @@ private void load(final RelDataType elementType,
       case OBJECT:
       case JAVA_SQL_DATE:
         return Util.<@Nullable Date, @Nullable Integer>transform(
-            (List<@Nullable Date>) list, (Date d) -> d == null
+            (List<@Nullable Date>) list, d -> d == null
                 ? null
                 : (int) (d.getTime() / DateTimeUtils.MILLIS_PER_DAY));
       default:
@@ -384,6 +385,7 @@ private static long toLong(Object o) {
       }
     }
 
+    @Contract("null -> false")
     private static boolean canBeLong(@Nullable Object o) {
       return o instanceof Boolean
           || o instanceof Character
diff --git a/core/src/main/java/org/apache/calcite/config/CalciteConnectionConfigImpl.java b/core/src/main/java/org/apache/calcite/config/CalciteConnectionConfigImpl.java
index 9410d0e33488..baf46bce2da2 100644
--- a/core/src/main/java/org/apache/calcite/config/CalciteConnectionConfigImpl.java
+++ b/core/src/main/java/org/apache/calcite/config/CalciteConnectionConfigImpl.java
@@ -107,6 +107,9 @@ public boolean isSet(CalciteConnectionProperty property) {
   }
 
   @Contract("_, !null -> !null")
+  // getPlugin comes from Avatica, which is not annotated, so NullAway cannot infer a T
+  // that suits both the Class argument and a nullable default.
+  @SuppressWarnings("NullAway")
   @Override public  @Nullable T fun(Class operatorTableClass,
       @Nullable T defaultOperatorTable) {
     final String fun =
@@ -155,6 +158,9 @@ public boolean isSet(CalciteConnectionProperty property) {
   }
 
   @Contract("_, !null -> !null")
+  // getPlugin comes from Avatica, which is not annotated, so NullAway cannot infer a T
+  // that suits both the Class argument and a nullable default.
+  @SuppressWarnings("NullAway")
   @Override public  @Nullable T parserFactory(Class parserFactoryClass,
       @Nullable T defaultParserFactory) {
     return CalciteConnectionProperty.PARSER_FACTORY.wrap(properties)
@@ -162,6 +168,9 @@ public boolean isSet(CalciteConnectionProperty property) {
   }
 
   @Contract("_, !null -> !null")
+  // getPlugin comes from Avatica, which is not annotated, so NullAway cannot infer a T
+  // that suits both the Class argument and a nullable default.
+  @SuppressWarnings("NullAway")
   @Override public  @Nullable T schemaFactory(Class schemaFactoryClass,
       @Nullable T defaultSchemaFactory) {
     return CalciteConnectionProperty.SCHEMA_FACTORY.wrap(properties)
@@ -183,6 +192,9 @@ public boolean isSet(CalciteConnectionProperty property) {
   }
 
   @Contract("_, !null -> !null")
+  // getPlugin comes from Avatica, which is not annotated, so NullAway cannot infer a T
+  // that suits both the Class argument and a nullable default.
+  @SuppressWarnings("NullAway")
   @Override public  @Nullable T typeSystem(Class typeSystemClass,
       @Nullable T defaultTypeSystem) {
     return CalciteConnectionProperty.TYPE_SYSTEM.wrap(properties)
@@ -225,6 +237,9 @@ public boolean isSet(CalciteConnectionProperty property) {
   }
 
   @Contract("_, !null -> !null")
+  // getPlugin comes from Avatica, which is not annotated, so NullAway cannot infer a T
+  // that suits both the Class argument and a nullable default.
+  @SuppressWarnings("NullAway")
   @Override public  @Nullable T metaTableFactory(
       Class metaTableFactoryClass,
       @Nullable T defaultMetaTableFactory) {
@@ -233,6 +248,9 @@ public boolean isSet(CalciteConnectionProperty property) {
   }
 
   @Contract("_, !null -> !null")
+  // getPlugin comes from Avatica, which is not annotated, so NullAway cannot infer a T
+  // that suits both the Class argument and a nullable default.
+  @SuppressWarnings("NullAway")
   @Override public  @Nullable T metaColumnFactory(
       Class metaColumnFactoryClass,
       @Nullable T defaultMetaColumnFactory) {
diff --git a/core/src/main/java/org/apache/calcite/rel/rules/LoptMultiJoin.java b/core/src/main/java/org/apache/calcite/rel/rules/LoptMultiJoin.java
index 979dff18ffd4..a526d19f2588 100644
--- a/core/src/main/java/org/apache/calcite/rel/rules/LoptMultiJoin.java
+++ b/core/src/main/java/org/apache/calcite/rel/rules/LoptMultiJoin.java
@@ -444,7 +444,6 @@ public void setJoinRemovalSemiJoin(int dimIdx, LogicalJoin semiJoin) {
    *
    * @return the bitmap containing the factor references
    */
-  @RequiresNonNull({"joinStart", "nFieldsInJoinFactor"})
   ImmutableBitSet getJoinFilterFactorBitmap(
       RexNode joinFilter,
       boolean setFields) {
@@ -466,7 +465,6 @@ private static ImmutableBitSet fieldBitmap(RexNode joinFilter) {
    * Sets bitmaps indicating which factors and fields each join filter
    * references.
    */
-  @RequiresNonNull({"allJoinFilters", "joinStart", "nFieldsInJoinFactor"})
   private void setJoinFilterRefs() {
     ListIterator filterIter = allJoinFilters.listIterator();
     while (filterIter.hasNext()) {
@@ -491,7 +489,6 @@ private void setJoinFilterRefs() {
    * @return bitmap representing factors referenced that will
    * be set by this method
    */
-  @RequiresNonNull({"joinStart", "nFieldsInJoinFactor"})
   private ImmutableBitSet factorBitmap(
       ImmutableBitSet fieldRefBitmap) {
     ImmutableBitSet.Builder factorRefBitmap = ImmutableBitSet.builder();
@@ -509,7 +506,6 @@ private ImmutableBitSet factorBitmap(
    *
    * @return index corresponding to join factor
    */
-  @RequiresNonNull({"joinStart", "nFieldsInJoinFactor"})
   public int findRef(
       int rexInputRef) {
     for (int i = 0; i < nJoinFactors; i++) {
diff --git a/core/src/main/java/org/apache/calcite/rel/rules/LoptOptimizeJoinRule.java b/core/src/main/java/org/apache/calcite/rel/rules/LoptOptimizeJoinRule.java
index f3cd00bd2385..19ae6f7bc0eb 100644
--- a/core/src/main/java/org/apache/calcite/rel/rules/LoptOptimizeJoinRule.java
+++ b/core/src/main/java/org/apache/calcite/rel/rules/LoptOptimizeJoinRule.java
@@ -316,10 +316,14 @@ private static void findRemovableSelfJoins(RelMetadataQuery mq, LoptMultiJoin mu
       for (int j = i + 1; j < factors.length; j++) {
         int leftFactor = factors[i];
         int rightFactor = factors[j];
-        if (simpleFactors.get(leftFactor).getQualifiedName().equals(
-            simpleFactors.get(rightFactor).getQualifiedName())) {
+        final RelOptTable leftTable =
+            requireNonNull(simpleFactors.get(leftFactor), "leftFactor");
+        final RelOptTable rightTable =
+            requireNonNull(simpleFactors.get(rightFactor), "rightFactor");
+        if (leftTable.getQualifiedName().equals(
+            rightTable.getQualifiedName())) {
           selfJoinPairs.put(leftFactor, rightFactor);
-          repeatedTables.add(simpleFactors.get(leftFactor));
+          repeatedTables.add(leftTable);
           break;
         }
       }
diff --git a/core/src/main/java/org/apache/calcite/util/ChunkList.java b/core/src/main/java/org/apache/calcite/util/ChunkList.java
index 148eae85f575..47af033dcfc7 100644
--- a/core/src/main/java/org/apache/calcite/util/ChunkList.java
+++ b/core/src/main/java/org/apache/calcite/util/ChunkList.java
@@ -156,6 +156,9 @@ boolean isValid(boolean fail) {
     return (E @Nullable []) chunk[0];
   }
 
+  // Slots 0 and 1 hold the neighbouring chunks rather than elements, and a chunk at
+  // either end has no neighbour.
+  @SuppressWarnings("NullAway")
   private static  void setPrev(E[] chunk, E @Nullable [] prev) {
     //noinspection unchecked
     chunk[0] = (E) prev;
@@ -166,6 +169,7 @@ private static  void setPrev(E[] chunk, E @Nullable [] prev) {
     return (E @Nullable []) chunk[1];
   }
 
+  @SuppressWarnings("NullAway")
   private static  void setNext(E[] chunk, E @Nullable [] next) {
     assert chunk != next;
     //noinspection unchecked
@@ -331,11 +335,8 @@ private E[] currentChunk() {
       if (r < start) {
         // Element we wish to eliminate is the last element in the previous
         // block.
-        E[] c = chunk;
-        if (c == null) {
-          c = last;
-        }
-        int o = occupied(castNonNull(c));
+        final E[] c = castNonNull(chunk == null ? last : chunk);
+        int o = occupied(c);
         if (o == 1) {
           // Block is now empty; remove it
           final E[] prev = prev(c);
@@ -375,8 +376,8 @@ private E[] currentChunk() {
       int s = start;
       if (p < start) {
         // The element is at the end of the previous chunk
-        c = prev(c);
-        s -= occupied(castNonNull(c));
+        c = castNonNull(prev(c));
+        s -= occupied(c);
       }
       setElement(c, HEADER_SIZE + p - s, e);
     }

From 3c7440479a0eb8b49c3d611e21f74367ac283447 Mon Sep 17 00:00:00 2001
From: Vladimir Sitnikov 
Date: Mon, 24 Aug 2026 15:38:07 +0300
Subject: [PATCH 525/562] [CALCITE-7736] Take the nullable bound on the
 acceptCall overrides, and two more dead preconditions

Nine overrides of `SqlOperator.acceptCall` take the bound the base method got
two commits ago. A visitor result may be nullable, and `acceptCall` passes it
through.

Four `InvocationHandler.invoke` implementations move the annotation from the
elements to the array: `@Nullable Object @Nullable [] args`. The JDK passes null
for `args` when the proxied method declares no parameters, which is what the
deleted `InvocationHandler.astub` used to say.

`ReflectiveConvertletTable.map` and `SimpleProfiler.Run.columns` are `final` and
non-null, so the `@RequiresNonNull` naming them stated nothing and rejected
their callers.

Co-Authored-By: Claude Opus 5 
---
 .../main/java/org/apache/calcite/profile/SimpleProfiler.java  | 3 ---
 .../calcite/rel/metadata/CachingRelMetadataProvider.java      | 4 ++--
 .../calcite/rel/metadata/ChainedRelMetadataProvider.java      | 4 ++--
 core/src/main/java/org/apache/calcite/sql/SqlAsOperator.java  | 4 +++-
 .../main/java/org/apache/calcite/sql/SqlMatchRecognize.java   | 2 +-
 .../src/main/java/org/apache/calcite/sql/SqlOverOperator.java | 4 +++-
 .../main/java/org/apache/calcite/sql/SqlSelectOperator.java   | 2 +-
 core/src/main/java/org/apache/calcite/sql/SqlSnapshot.java    | 2 +-
 core/src/main/java/org/apache/calcite/sql/SqlWindow.java      | 2 +-
 .../java/org/apache/calcite/sql/fun/SqlColonOperator.java     | 4 +++-
 .../java/org/apache/calcite/sql/fun/SqlConvertFunction.java   | 2 +-
 .../main/java/org/apache/calcite/sql/fun/SqlDotOperator.java  | 4 +++-
 .../org/apache/calcite/sql2rel/ReflectiveConvertletTable.java | 3 ---
 .../org/apache/calcite/util/BarfingInvocationHandler.java     | 2 +-
 .../org/apache/calcite/util/DelegatingInvocationHandler.java  | 2 +-
 15 files changed, 23 insertions(+), 21 deletions(-)

diff --git a/core/src/main/java/org/apache/calcite/profile/SimpleProfiler.java b/core/src/main/java/org/apache/calcite/profile/SimpleProfiler.java
index 0e38205f143c..92baaca63233 100644
--- a/core/src/main/java/org/apache/calcite/profile/SimpleProfiler.java
+++ b/core/src/main/java/org/apache/calcite/profile/SimpleProfiler.java
@@ -17,7 +17,6 @@
 package org.apache.calcite.profile;
 
 import org.apache.calcite.linq4j.Ord;
-import org.apache.calcite.linq4j.annotations.RequiresNonNull;
 import org.apache.calcite.materialize.Lattice;
 import org.apache.calcite.rel.metadata.NullSentinel;
 import org.apache.calcite.runtime.FlatLists;
@@ -291,8 +290,6 @@ private boolean hasNull(ImmutableBitSet columnOrdinals) {
       }
       return false;
     }
-
-    @RequiresNonNull("columns")
     private ImmutableSortedSet toColumns(
         Iterable ordinals) {
       //noinspection Convert2MethodRef
diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/CachingRelMetadataProvider.java b/core/src/main/java/org/apache/calcite/rel/metadata/CachingRelMetadataProvider.java
index a9c7d1f726fa..37241cedc2de 100644
--- a/core/src/main/java/org/apache/calcite/rel/metadata/CachingRelMetadataProvider.java
+++ b/core/src/main/java/org/apache/calcite/rel/metadata/CachingRelMetadataProvider.java
@@ -118,8 +118,8 @@ private class CachingInvocationHandler implements InvocationHandler {
       this.metadata = requireNonNull(metadata, "metadata");
     }
 
-    @Override public @Nullable Object invoke(Object proxy, Method method, @Nullable Object[] args)
-        throws Throwable {
+    @Override public @Nullable Object invoke(Object proxy, Method method,
+        @Nullable Object @Nullable [] args) throws Throwable {
       // Compute hash key.
       final ImmutableList.Builder builder = ImmutableList.builder();
       builder.add(method);
diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/ChainedRelMetadataProvider.java b/core/src/main/java/org/apache/calcite/rel/metadata/ChainedRelMetadataProvider.java
index aac5f5b09daa..87e4b110a4c6 100644
--- a/core/src/main/java/org/apache/calcite/rel/metadata/ChainedRelMetadataProvider.java
+++ b/core/src/main/java/org/apache/calcite/rel/metadata/ChainedRelMetadataProvider.java
@@ -140,8 +140,8 @@ private static class ChainedInvocationHandler implements InvocationHandler {
       this.metadataList = ImmutableList.copyOf(metadataList);
     }
 
-    @Override public @Nullable Object invoke(Object proxy, Method method, @Nullable Object[] args)
-        throws Throwable {
+    @Override public @Nullable Object invoke(Object proxy, Method method,
+        @Nullable Object @Nullable [] args) throws Throwable {
       for (Metadata metadata : metadataList) {
         try {
           final Object o = method.invoke(metadata, args);
diff --git a/core/src/main/java/org/apache/calcite/sql/SqlAsOperator.java b/core/src/main/java/org/apache/calcite/sql/SqlAsOperator.java
index 0bf30013144c..6cf9fe60802d 100644
--- a/core/src/main/java/org/apache/calcite/sql/SqlAsOperator.java
+++ b/core/src/main/java/org/apache/calcite/sql/SqlAsOperator.java
@@ -30,6 +30,8 @@
 import org.apache.calcite.sql.validate.SqlValidatorScope;
 import org.apache.calcite.util.Util;
 
+import org.jspecify.annotations.Nullable;
+
 import java.util.List;
 
 import static org.apache.calcite.util.Static.RESOURCE;
@@ -119,7 +121,7 @@ protected SqlAsOperator(String name, SqlKind kind, int prec,
     }
   }
 
-  @Override public  void acceptCall(
+  @Override public  void acceptCall(
       SqlVisitor visitor,
       SqlCall call,
       boolean onlyExpressions,
diff --git a/core/src/main/java/org/apache/calcite/sql/SqlMatchRecognize.java b/core/src/main/java/org/apache/calcite/sql/SqlMatchRecognize.java
index 338652fe4faf..36cdab245311 100644
--- a/core/src/main/java/org/apache/calcite/sql/SqlMatchRecognize.java
+++ b/core/src/main/java/org/apache/calcite/sql/SqlMatchRecognize.java
@@ -286,7 +286,7 @@ private SqlMatchRecognizeOperator() {
           (SqlNodeList) operands[9], (SqlNodeList) operands[10], (SqlLiteral) operands[11]);
     }
 
-    @Override public  void acceptCall(
+    @Override public  void acceptCall(
         SqlVisitor visitor,
         SqlCall call,
         boolean onlyExpressions,
diff --git a/core/src/main/java/org/apache/calcite/sql/SqlOverOperator.java b/core/src/main/java/org/apache/calcite/sql/SqlOverOperator.java
index 1ec62f3309a2..d0ad2a2167d6 100644
--- a/core/src/main/java/org/apache/calcite/sql/SqlOverOperator.java
+++ b/core/src/main/java/org/apache/calcite/sql/SqlOverOperator.java
@@ -25,6 +25,8 @@
 import org.apache.calcite.sql.validate.SqlValidator;
 import org.apache.calcite.sql.validate.SqlValidatorScope;
 
+import org.jspecify.annotations.Nullable;
+
 import static org.apache.calcite.util.Static.RESOURCE;
 
 /**
@@ -164,7 +166,7 @@ public SqlOverOperator() {
    *
    * @param visitor Visitor
    */
-  @Override public  void acceptCall(
+  @Override public  void acceptCall(
       SqlVisitor visitor,
       SqlCall call,
       boolean onlyExpressions,
diff --git a/core/src/main/java/org/apache/calcite/sql/SqlSelectOperator.java b/core/src/main/java/org/apache/calcite/sql/SqlSelectOperator.java
index 4dbce066e598..36fb6ce2066d 100644
--- a/core/src/main/java/org/apache/calcite/sql/SqlSelectOperator.java
+++ b/core/src/main/java/org/apache/calcite/sql/SqlSelectOperator.java
@@ -119,7 +119,7 @@ public SqlSelect createCall(
         null);
   }
 
-  @Override public  void acceptCall(
+  @Override public  void acceptCall(
       SqlVisitor visitor,
       SqlCall call,
       boolean onlyExpressions,
diff --git a/core/src/main/java/org/apache/calcite/sql/SqlSnapshot.java b/core/src/main/java/org/apache/calcite/sql/SqlSnapshot.java
index 09de8dd059af..6ce1cf163729 100644
--- a/core/src/main/java/org/apache/calcite/sql/SqlSnapshot.java
+++ b/core/src/main/java/org/apache/calcite/sql/SqlSnapshot.java
@@ -106,7 +106,7 @@ private SqlSnapshotOperator() {
       return new SqlSnapshot(pos, operands[0], operands[1]);
     }
 
-    @Override public  void acceptCall(
+    @Override public  void acceptCall(
         SqlVisitor visitor,
         SqlCall call,
         boolean onlyExpressions,
diff --git a/core/src/main/java/org/apache/calcite/sql/SqlWindow.java b/core/src/main/java/org/apache/calcite/sql/SqlWindow.java
index f960fe856300..11215bd50760 100644
--- a/core/src/main/java/org/apache/calcite/sql/SqlWindow.java
+++ b/core/src/main/java/org/apache/calcite/sql/SqlWindow.java
@@ -933,7 +933,7 @@ private SqlWindowOperator() {
           pos);
     }
 
-    @Override public  void acceptCall(
+    @Override public  void acceptCall(
         SqlVisitor visitor,
         SqlCall call,
         boolean onlyExpressions,
diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlColonOperator.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlColonOperator.java
index a1ac8e5d417f..c1b7a4507167 100644
--- a/core/src/main/java/org/apache/calcite/sql/fun/SqlColonOperator.java
+++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlColonOperator.java
@@ -33,6 +33,8 @@
 import org.apache.calcite.sql.validate.SqlValidatorScope;
 import org.apache.calcite.util.Litmus;
 
+import org.jspecify.annotations.Nullable;
+
 import java.util.Arrays;
 
 import static java.util.Objects.requireNonNull;
@@ -50,7 +52,7 @@ public class SqlColonOperator extends SqlSpecialOperator {
 
   // Path segments are structural literals/identifiers, never column refs, so
   // they must not be routed through expression visitors such as AggChecker.
-  @Override public  void acceptCall(SqlVisitor visitor, SqlCall call,
+  @Override public  void acceptCall(SqlVisitor visitor, SqlCall call,
       boolean onlyExpressions, SqlBasicVisitor.ArgHandler argHandler) {
     if (onlyExpressions) {
       argHandler.visitChild(visitor, call, 0, call.operand(0));
diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlConvertFunction.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlConvertFunction.java
index c0a23611a23a..5e5e979fa51a 100644
--- a/core/src/main/java/org/apache/calcite/sql/fun/SqlConvertFunction.java
+++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlConvertFunction.java
@@ -101,7 +101,7 @@ protected SqlConvertFunction(String name, SqlKind kind,
     super.validateQuantifier(validator, call);
   }
 
-  @Override public  void acceptCall(SqlVisitor visitor, SqlCall call,
+  @Override public  void acceptCall(SqlVisitor visitor, SqlCall call,
       boolean onlyExpressions, SqlBasicVisitor.ArgHandler argHandler) {
     if (onlyExpressions) {
       // Both operand[1] and operand[2] are not an expression, but Charset
diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlDotOperator.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlDotOperator.java
index 9d9749c3e589..88c9c6bc0eb1 100644
--- a/core/src/main/java/org/apache/calcite/sql/fun/SqlDotOperator.java
+++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlDotOperator.java
@@ -43,6 +43,8 @@
 import org.apache.calcite.util.Litmus;
 import org.apache.calcite.util.Static;
 
+import org.jspecify.annotations.Nullable;
+
 import java.util.Arrays;
 
 import static org.apache.calcite.sql.validate.SqlNonNullableAccessors.getOperandLiteralValueOrThrow;
@@ -86,7 +88,7 @@ public class SqlDotOperator extends SqlSpecialOperator {
     return SqlOperandCountRanges.of(2);
   }
 
-  @Override public  void acceptCall(SqlVisitor visitor, SqlCall call,
+  @Override public  void acceptCall(SqlVisitor visitor, SqlCall call,
       boolean onlyExpressions, SqlBasicVisitor.ArgHandler argHandler) {
     if (onlyExpressions) {
       // Do not visit operands[1] -- it is not an expression.
diff --git a/core/src/main/java/org/apache/calcite/sql2rel/ReflectiveConvertletTable.java b/core/src/main/java/org/apache/calcite/sql2rel/ReflectiveConvertletTable.java
index 1e2e2cc90316..1d89431d0804 100644
--- a/core/src/main/java/org/apache/calcite/sql2rel/ReflectiveConvertletTable.java
+++ b/core/src/main/java/org/apache/calcite/sql2rel/ReflectiveConvertletTable.java
@@ -16,7 +16,6 @@
  */
 package org.apache.calcite.sql2rel;
 
-import org.apache.calcite.linq4j.annotations.RequiresNonNull;
 import org.apache.calcite.rex.RexNode;
 import org.apache.calcite.sql.SqlCall;
 import org.apache.calcite.sql.SqlNode;
@@ -63,7 +62,6 @@ public ReflectiveConvertletTable() {
    * c. has a return type of "RexNode" or a subtype d. has a 2 parameters with
    * types ConvertletContext and SqlNode (or a subtype) respectively.
    */
-  @RequiresNonNull("map")
   private void registerNodeTypeMethod(
       final Method method) {
     if (!isPublic(method)) {
@@ -105,7 +103,6 @@ private void registerNodeTypeMethod(
    * types: ConvertletContext; SqlOperator (or a subtype), SqlCall (or a
    * subtype).
    */
-  @RequiresNonNull("map")
   private void registerOpTypeMethod(
       final Method method) {
     if (!isPublic(method)) {
diff --git a/core/src/main/java/org/apache/calcite/util/BarfingInvocationHandler.java b/core/src/main/java/org/apache/calcite/util/BarfingInvocationHandler.java
index 1bec8d98139c..0f0e02cf0907 100644
--- a/core/src/main/java/org/apache/calcite/util/BarfingInvocationHandler.java
+++ b/core/src/main/java/org/apache/calcite/util/BarfingInvocationHandler.java
@@ -44,7 +44,7 @@ protected BarfingInvocationHandler() {
   @Override public @Nullable Object invoke(
       Object proxy,
       Method method,
-      @Nullable Object[] args) throws Throwable {
+      @Nullable Object @Nullable [] args) throws Throwable {
     Class clazz = getClass();
     Method matchingMethod;
     try {
diff --git a/core/src/main/java/org/apache/calcite/util/DelegatingInvocationHandler.java b/core/src/main/java/org/apache/calcite/util/DelegatingInvocationHandler.java
index bc305a62c287..5d237632150e 100644
--- a/core/src/main/java/org/apache/calcite/util/DelegatingInvocationHandler.java
+++ b/core/src/main/java/org/apache/calcite/util/DelegatingInvocationHandler.java
@@ -56,7 +56,7 @@ public abstract class DelegatingInvocationHandler implements InvocationHandler {
   @Override public @Nullable Object invoke(
       Object proxy,
       Method method,
-      @Nullable Object[] args) throws Throwable {
+      @Nullable Object @Nullable [] args) throws Throwable {
     Class clazz = getClass();
     Method matchingMethod;
     try {

From 7b794c6e3255d04b264bbbe06dc4521b9a2ef746 Mon Sep 17 00:00:00 2001
From: Vladimir Sitnikov 
Date: Mon, 24 Aug 2026 15:52:21 +0300
Subject: [PATCH 526/562] [CALCITE-7736] Add Util.firstNonNull, which states in
 its signature what @Contract cannot

`Util.first` is the Elvis operator of the codebase, and its
`@Contract("_, !null -> !null")` is honoured everywhere except where it matters
most. NullAway consults a contract in its dataflow but not in the JSpecify
generics check, which infers the type argument from the declared return type
alone and reports

    inference failure: type variable T constrained to be both @NonNull and @Nullable

at 20 call sites. The same contract on a non-generic method reports nothing, and
an explicit type witness silences the generic one, so the two halves of NullAway
disagree rather than the calls being wrong. Reported as uber/NullAway#1730, see
nullaway-bugs/nullable-return-type-breaks-generic-inference.md.

    static  T firstNonNull(@Nullable T v0, T v1)

says the same thing without a contract: `T` is inferred non-null from the
fallback, `@Nullable T` still accepts a nullable first argument, and the result
is non-null. `first` stays for the calls whose fallback is itself nullable.

`CalciteSystemProperty` already static-imports Guava's `MoreObjects.firstNonNull`
and calls the new one qualified.

Co-Authored-By: Claude Opus 5 
---
 .../adapter/enumerable/EnumerableHashJoin.java   |  2 +-
 .../enumerable/EnumerableRepeatUnion.java        |  2 +-
 .../calcite/config/CalciteSystemProperty.java    |  6 +++---
 .../apache/calcite/interpreter/Interpreter.java  |  2 +-
 .../org/apache/calcite/jdbc/CalciteMetaImpl.java |  2 +-
 .../org/apache/calcite/materialize/Lattice.java  |  2 +-
 .../materialize/MaterializationService.java      |  2 +-
 .../org/apache/calcite/model/ModelHandler.java   |  2 +-
 .../org/apache/calcite/plan/hep/HepPlanner.java  |  2 +-
 .../calcite/prepare/CalcitePrepareImpl.java      |  2 +-
 .../org/apache/calcite/rel/core/Aggregate.java   |  2 +-
 .../calcite/rel/metadata/RelMdPredicates.java    |  2 +-
 .../calcite/rel/rel2sql/SqlImplementor.java      |  2 +-
 .../apache/calcite/sql/SqlIntervalQualifier.java |  2 +-
 .../org/apache/calcite/sql/fun/SqlLibrary.java   |  4 ++--
 .../calcite/sql/validate/SetopNamespace.java     |  2 +-
 .../calcite/sql/validate/SqlValidatorImpl.java   |  3 ++-
 .../apache/calcite/sql2rel/RelDecorrelator.java  |  2 +-
 .../org/apache/calcite/tools/Frameworks.java     |  2 +-
 .../org/apache/calcite/tools/RelBuilder.java     |  5 +++--
 .../main/java/org/apache/calcite/util/Util.java  | 16 ++++++++++++++++
 21 files changed, 42 insertions(+), 24 deletions(-)

diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableHashJoin.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableHashJoin.java
index 3ed56bfcb72c..d186b2c15bd8 100644
--- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableHashJoin.java
+++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableHashJoin.java
@@ -375,7 +375,7 @@ private Result implementHashJoin(EnumerableRelImplementor implementor, Prefer pr
                         ImmutableList.of(
                             leftResult.physType, rightResult.physType)))
                     .append(
-                        Util.first(keyPhysType.comparer(),
+                        Util.firstNonNull(keyPhysType.comparer(),
                             Expressions.constant(null)))
                     .append(
                         Expressions.constant(joinType.generatesNullsOnLeft()))
diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableRepeatUnion.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableRepeatUnion.java
index 5d40346879e7..071f8200caf7 100644
--- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableRepeatUnion.java
+++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableRepeatUnion.java
@@ -113,7 +113,7 @@ public class EnumerableRepeatUnion extends RepeatUnion implements EnumerableRel
             iterativeExp,
             Expressions.constant(iterationLimit, int.class),
             Expressions.constant(all, boolean.class),
-            Util.first(physType.comparer(),
+            Util.firstNonNull(physType.comparer(),
                 Expressions.call(BuiltInMethod.IDENTITY_COMPARER.method)),
             cleanUpFunctionExp);
     builder.add(unionExp);
diff --git a/core/src/main/java/org/apache/calcite/config/CalciteSystemProperty.java b/core/src/main/java/org/apache/calcite/config/CalciteSystemProperty.java
index 978379ab0d71..1a6f860b0e6c 100644
--- a/core/src/main/java/org/apache/calcite/config/CalciteSystemProperty.java
+++ b/core/src/main/java/org/apache/calcite/config/CalciteSystemProperty.java
@@ -16,6 +16,8 @@
  */
 package org.apache.calcite.config;
 
+import org.apache.calcite.util.Util;
+
 import com.google.common.collect.ImmutableSet;
 
 import org.jspecify.annotations.Nullable;
@@ -30,8 +32,6 @@
 import java.util.function.IntPredicate;
 import java.util.stream.Stream;
 
-import static com.google.common.base.MoreObjects.firstNonNull;
-
 import static java.lang.Boolean.parseBoolean;
 import static java.lang.Integer.parseInt;
 import static java.util.Objects.requireNonNull;
@@ -566,7 +566,7 @@ private static CalciteSystemProperty stringProperty(
   private static Properties loadProperties() {
     Properties saffronProperties = new Properties();
     ClassLoader classLoader =
-        firstNonNull(Thread.currentThread().getContextClassLoader(),
+        Util.firstNonNull(Thread.currentThread().getContextClassLoader(),
             CalciteSystemProperty.class.getClassLoader());
     // Read properties from the file "saffron.properties", if it exists in classpath
     try (InputStream stream = requireNonNull(classLoader, "classLoader")
diff --git a/core/src/main/java/org/apache/calcite/interpreter/Interpreter.java b/core/src/main/java/org/apache/calcite/interpreter/Interpreter.java
index 19de7be6e940..9ae4c26a6400 100644
--- a/core/src/main/java/org/apache/calcite/interpreter/Interpreter.java
+++ b/core/src/main/java/org/apache/calcite/interpreter/Interpreter.java
@@ -350,7 +350,7 @@ Pair> visitRoot(RelNode p) {
       // rewrite children first (from left to right)
       final List inputs = relInputs.get(p);
       RelNode finalP = p;
-      Ord.forEach(Util.first(inputs, p.getInputs()),
+      Ord.forEach(Util.firstNonNull(inputs, p.getInputs()),
           (r, i) -> outEdges.put(r, new Edge(finalP, i)));
       if (inputs != null) {
         for (int i = 0; i < inputs.size(); i++) {
diff --git a/core/src/main/java/org/apache/calcite/jdbc/CalciteMetaImpl.java b/core/src/main/java/org/apache/calcite/jdbc/CalciteMetaImpl.java
index 0c4d0df1c198..5b847dea34c6 100644
--- a/core/src/main/java/org/apache/calcite/jdbc/CalciteMetaImpl.java
+++ b/core/src/main/java/org/apache/calcite/jdbc/CalciteMetaImpl.java
@@ -413,7 +413,7 @@ Enumerable schemas(final String catalog, final LikePattern pattern)
           return new CalciteMetaSchema(schema, catalog, schema.getName());
         })
         .orderBy((Function1) metaSchema ->
-            (Comparable) FlatLists.of(Util.first(metaSchema.tableCatalog, ""),
+            (Comparable) FlatLists.of(Util.firstNonNull(metaSchema.tableCatalog, ""),
                 metaSchema.tableSchem));
   }
 
diff --git a/core/src/main/java/org/apache/calcite/materialize/Lattice.java b/core/src/main/java/org/apache/calcite/materialize/Lattice.java
index 5f476c42cf13..904412e6e3b2 100644
--- a/core/src/main/java/org/apache/calcite/materialize/Lattice.java
+++ b/core/src/main/java/org/apache/calcite/materialize/Lattice.java
@@ -1106,7 +1106,7 @@ public Column expression(RexNode e, String alias,
         final int derivedOrdinal = derivedColumnsByName.size();
         final int ordinal = baseColumns.size() + derivedOrdinal;
         return new DerivedColumn(ordinal,
-            Util.first(alias, "e$" + derivedOrdinal), e, tableAliases);
+            Util.firstNonNull(alias, "e$" + derivedOrdinal), e, tableAliases);
       });
     }
 
diff --git a/core/src/main/java/org/apache/calcite/materialize/MaterializationService.java b/core/src/main/java/org/apache/calcite/materialize/MaterializationService.java
index 6ac184d177ee..0a3b59230b38 100644
--- a/core/src/main/java/org/apache/calcite/materialize/MaterializationService.java
+++ b/core/src/main/java/org/apache/calcite/materialize/MaterializationService.java
@@ -142,7 +142,7 @@ private MaterializationService() {
     if (tableEntry == null) {
       Table table = tableFactory.createTable(schema, viewSql, viewSchemaPath);
       final String tableName =
-          Schemas.uniqueTableName(schema, Util.first(suggestedTableName, "m"));
+          Schemas.uniqueTableName(schema, Util.firstNonNull(suggestedTableName, "m"));
       tableEntry = schema.add(tableName, table, ImmutableList.of(viewSql));
       Hook.CREATE_MATERIALIZATION.run(tableName);
       rowType = table.getRowType(connection.getTypeFactory());
diff --git a/core/src/main/java/org/apache/calcite/model/ModelHandler.java b/core/src/main/java/org/apache/calcite/model/ModelHandler.java
index d54792573b45..0f433161017d 100644
--- a/core/src/main/java/org/apache/calcite/model/ModelHandler.java
+++ b/core/src/main/java/org/apache/calcite/model/ModelHandler.java
@@ -192,7 +192,7 @@ public static void addFunctions(ClassNameFilter filter, SchemaPlus schema,
     final TableFunction tableFunction =
         TableFunctionImpl.create(clazz, methodNameOrDefault);
     if (tableFunction != null) {
-      schema.add(Util.first(functionName, methodNameOrDefault),
+      schema.add(Util.firstNonNull(functionName, methodNameOrDefault),
           tableFunction);
       return;
     }
diff --git a/core/src/main/java/org/apache/calcite/plan/hep/HepPlanner.java b/core/src/main/java/org/apache/calcite/plan/hep/HepPlanner.java
index df28ae14d36a..35ffdaf22598 100644
--- a/core/src/main/java/org/apache/calcite/plan/hep/HepPlanner.java
+++ b/core/src/main/java/org/apache/calcite/plan/hep/HepPlanner.java
@@ -189,7 +189,7 @@ public HepPlanner(
       RelOptCostFactory costFactory) {
     super(costFactory, context);
     this.mainProgram = requireNonNull(program, "program");
-    this.onCopyHook = Util.first(onCopyHook, Functions.ignore2());
+    this.onCopyHook = Util.firstNonNull(onCopyHook, Functions.ignore2());
     this.noDag = noDag;
     this.largePlanMode = CalciteSystemProperty.HEP_PLANNER_LARGE_PLAN_MODE.value();
   }
diff --git a/core/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java b/core/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java
index f44178c8a6c8..8df45081ab5d 100644
--- a/core/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java
+++ b/core/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java
@@ -845,7 +845,7 @@ private static ColumnMetaData.AvaticaType avaticaType(JavaTypeFactory typeFactor
         // fall through
       default:
         final Type clazz =
-            typeFactory.getJavaClass(Util.first(fieldType, type));
+            typeFactory.getJavaClass(Util.firstNonNull(fieldType, type));
         final ColumnMetaData.Rep rep =
             requireNonNull(ColumnMetaData.Rep.of(clazz));
         return ColumnMetaData.scalar(typeOrdinal, typeName, rep);
diff --git a/core/src/main/java/org/apache/calcite/rel/core/Aggregate.java b/core/src/main/java/org/apache/calcite/rel/core/Aggregate.java
index f23c6128c489..d79b522260b3 100644
--- a/core/src/main/java/org/apache/calcite/rel/core/Aggregate.java
+++ b/core/src/main/java/org/apache/calcite/rel/core/Aggregate.java
@@ -340,7 +340,7 @@ public ImmutableList getGroupSets() {
         .itemIf("aggs", aggCalls, pw.nest());
     if (!pw.nest()) {
       for (Ord ord : Ord.zip(aggCalls)) {
-        pw.item(Util.first(ord.e.name, "agg#" + ord.i), ord.e);
+        pw.item(Util.firstNonNull(ord.e.name, "agg#" + ord.i), ord.e);
       }
     }
     return pw;
diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdPredicates.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdPredicates.java
index 125946ee6263..d7bf38498647 100644
--- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdPredicates.java
+++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdPredicates.java
@@ -648,7 +648,7 @@ public RelOptPredicateList getPredicates(RelSubset r,
         list = list == null ? list2 : list.union(rexBuilder, list2);
       }
     }
-    return Util.first(list, RelOptPredicateList.EMPTY);
+    return Util.firstNonNull(list, RelOptPredicateList.EMPTY);
   }
 
   /**
diff --git a/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java b/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java
index 7a3be7be3f48..6cf68058e780 100644
--- a/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java
+++ b/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java
@@ -536,7 +536,7 @@ private static RelDataType adjustedRowType(RelNode rel, SqlNode node) {
           rowType.getFieldList(),
           (selectItem, field) ->
               builder.add(
-                  Util.first(SqlValidatorUtil.alias(selectItem),
+                  Util.firstNonNull(SqlValidatorUtil.alias(selectItem),
                       field.getName()),
                   field.getType()));
       return builder.build();
diff --git a/core/src/main/java/org/apache/calcite/sql/SqlIntervalQualifier.java b/core/src/main/java/org/apache/calcite/sql/SqlIntervalQualifier.java
index 4f5805639ce8..93d987beb2c6 100644
--- a/core/src/main/java/org/apache/calcite/sql/SqlIntervalQualifier.java
+++ b/core/src/main/java/org/apache/calcite/sql/SqlIntervalQualifier.java
@@ -401,7 +401,7 @@ public TimeUnit getEndUnit() {
   /** Returns {@code SECOND} for both {@code HOUR TO SECOND} and
    * {@code SECOND}. */
   public TimeUnit getUnit() {
-    return Util.first(timeUnitRange.endUnit, timeUnitRange.startUnit);
+    return Util.firstNonNull(timeUnitRange.endUnit, timeUnitRange.startUnit);
   }
 
   @Override public SqlNode clone(SqlParserPos pos) {
diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlLibrary.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlLibrary.java
index dc127ebc18c4..1615406bc559 100644
--- a/core/src/main/java/org/apache/calcite/sql/fun/SqlLibrary.java
+++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlLibrary.java
@@ -34,7 +34,7 @@
 import static com.google.common.base.Preconditions.checkArgument;
 
 import static org.apache.calcite.util.Util.filter;
-import static org.apache.calcite.util.Util.first;
+import static org.apache.calcite.util.Util.firstNonNull;
 
 import static java.util.Objects.requireNonNull;
 
@@ -135,7 +135,7 @@ public List children() {
    * 
      For example, {@link #REDSHIFT} inherits from {@link #POSTGRESQL}.
    * Never returns null. */
   public Set inheritors() {
-    return first(INHERITOR_MAP.get(this), ImmutableSet.of());
+    return firstNonNull(INHERITOR_MAP.get(this), ImmutableSet.of());
   }
 
   /** Looks up a value.
diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SetopNamespace.java b/core/src/main/java/org/apache/calcite/sql/validate/SetopNamespace.java
index 53c537bab8c5..9d5af3b18a37 100644
--- a/core/src/main/java/org/apache/calcite/sql/validate/SetopNamespace.java
+++ b/core/src/main/java/org/apache/calcite/sql/validate/SetopNamespace.java
@@ -74,7 +74,7 @@ protected SetopNamespace(
               namespace.getMonotonicity(
                   namespace.getRowType().getFieldNames().get(index)));
     }
-    return Util.first(monotonicity, SqlMonotonicity.NOT_MONOTONIC);
+    return Util.firstNonNull(monotonicity, SqlMonotonicity.NOT_MONOTONIC);
   }
 
   private static SqlMonotonicity combine(@Nullable SqlMonotonicity m0,
diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java
index e68a36848188..04515b18c3ab 100644
--- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java
+++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java
@@ -179,6 +179,7 @@
 import static org.apache.calcite.sql.validate.SqlNonNullableAccessors.getTable;
 import static org.apache.calcite.util.Static.RESOURCE;
 import static org.apache.calcite.util.Util.first;
+import static org.apache.calcite.util.Util.firstNonNull;
 
 import static java.util.Collections.emptyList;
 import static java.util.Objects.requireNonNull;
@@ -402,7 +403,7 @@ public SqlConformance getConformance() {
     final Map expansions = new HashMap<>();
     for (final SqlNode selectItem : selectList) {
       final RelDataType originalType = getValidatedNodeTypeIfKnown(selectItem);
-      expandSelectItem(selectItem, select, first(originalType, unknownType),
+      expandSelectItem(selectItem, select, firstNonNull(originalType, unknownType),
           list, catalogReader.nameMatcher().createSet(), types, expansions,
           includeSystemVars);
     }
diff --git a/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java b/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java
index 72b66be88c68..278978224b27 100644
--- a/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java
+++ b/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java
@@ -290,7 +290,7 @@ public static RelNode decorrelateQuery(RelNode rootRel,
   private void setCurrent(@Nullable RelNode root, @Nullable Correlate corRel) {
     currentRel = corRel;
     if (corRel != null) {
-      cm = new CorelMapBuilder().build(Util.first(root, corRel));
+      cm = new CorelMapBuilder().build(Util.firstNonNull(root, corRel));
     }
   }
 
diff --git a/core/src/main/java/org/apache/calcite/tools/Frameworks.java b/core/src/main/java/org/apache/calcite/tools/Frameworks.java
index fa0a8966122f..1df10860f658 100644
--- a/core/src/main/java/org/apache/calcite/tools/Frameworks.java
+++ b/core/src/main/java/org/apache/calcite/tools/Frameworks.java
@@ -137,7 +137,7 @@ public static  R withPlanner(final PlannerAction action,
         (cluster, relOptSchema, rootSchema, statement) -> {
           final CalciteSchema schema =
               CalciteSchema.from(
-                  Util.first(config.getDefaultSchema(), rootSchema));
+                  Util.firstNonNull(config.getDefaultSchema(), rootSchema));
           return action.apply(cluster, relOptSchema, schema.root().plus());
         });
   }
diff --git a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java
index 8a356136796f..c73b66d12f3e 100644
--- a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java
+++ b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java
@@ -172,6 +172,7 @@
 import static org.apache.calcite.sql.SqlKind.UNION;
 import static org.apache.calcite.util.Static.RESOURCE;
 import static org.apache.calcite.util.Util.first;
+import static org.apache.calcite.util.Util.firstNonNull;
 
 import static java.util.Objects.requireNonNull;
 
@@ -3959,7 +3960,7 @@ private static RelFieldCollation collation(RexNode node,
     switch (node.getKind()) {
     case INPUT_REF:
       return new RelFieldCollation(((RexInputRef) node).getIndex(), direction,
-          first(nullDirection, direction.defaultNullDirection()));
+          firstNonNull(nullDirection, direction.defaultNullDirection()));
     case DESCENDING:
       return collation(((RexCall) node).getOperands().get(0),
           RelFieldCollation.Direction.DESCENDING,
@@ -3974,7 +3975,7 @@ private static RelFieldCollation collation(RexNode node,
       final int fieldIndex = extraNodes.size();
       extraNodes.add(node);
       return new RelFieldCollation(fieldIndex, direction,
-          first(nullDirection, direction.defaultNullDirection()));
+          firstNonNull(nullDirection, direction.defaultNullDirection()));
     }
   }
 
diff --git a/core/src/main/java/org/apache/calcite/util/Util.java b/core/src/main/java/org/apache/calcite/util/Util.java
index a8c5b4d9de9b..8ed233f6e18d 100644
--- a/core/src/main/java/org/apache/calcite/util/Util.java
+++ b/core/src/main/java/org/apache/calcite/util/Util.java
@@ -2132,6 +2132,22 @@ public static  List> pairs(final List list) {
     return v0 != null ? v0 : v1;
   }
 
+  /** Returns the first argument if it is not null, otherwise the second.
+   *
+   * 
      Same as {@link #first(Object, Object)}, for the common case where the fallback is not
+   * null and so neither is the result. Stating that in the signature rather than in a
+   * {@code @Contract} keeps it visible to NullAway's generic type inference; see
+   * NullAway#1730.
+   *
+   * @param v0 value, may be null
+   * @param v1 fallback, used when {@code v0} is null
+   * @param  value type
+   * @return {@code v0} if it is not null, otherwise {@code v1}
+   */
+  public static  T firstNonNull(@Nullable T v0, T v1) {
+    return v0 != null ? v0 : v1;
+  }
+
   /** Unboxes a {@link Double} value,
    * using a given default value if it is null. */
   public static double first(@Nullable Double v0, double v1) {

From 3f67eb9d4d4e3867f26a1f672956919f097d0f69 Mon Sep 17 00:00:00 2001
From: Vladimir Sitnikov 
Date: Mon, 24 Aug 2026 16:00:20 +0300
Subject: [PATCH 527/562] [CALCITE-7736] Read the nulls the JDK hands out, and
 restore two narrowings

Four proxy invocation handlers index `args`, which the JDK leaves null when the
proxied method declares no parameters, so they read it through `requireNonNull`.
Three places declare a `toArray()` result `@Nullable Object[]`, which is what
the JDK model gives them.

`Util.LINE_SEPARATOR` and `FILE_SEPARATOR` read properties that are always set,
and `toUrl` reads `file.separator` a second time rather than the constant it
already has.

`deepEquals0` used to narrow its argument through `@EnsuresNonNullIf`, which
NullAway supports for fields only, so the three `deepEquals` in
`LogicalFilter`, `LogicalJoin` and `LogicalAsofJoin` say `requireNonNull`
instead.

`SqlBinaryOperator` and `SqlTypeUtil` ask for a charset after `inCharFamily`
has established there is one.

Co-Authored-By: Claude Opus 5 
---
 .../org/apache/calcite/rel/logical/LogicalAsofJoin.java     | 3 ++-
 .../java/org/apache/calcite/rel/logical/LogicalFilter.java  | 2 +-
 .../java/org/apache/calcite/rel/logical/LogicalJoin.java    | 2 +-
 .../calcite/rel/metadata/JaninoRelMetadataProvider.java     | 3 ++-
 .../rel/metadata/ProxyingMetadataHandlerProvider.java       | 2 +-
 .../apache/calcite/rel/metadata/RelMetadataQueryBase.java   | 3 ++-
 .../main/java/org/apache/calcite/sql/SqlBinaryOperator.java | 2 +-
 .../main/java/org/apache/calcite/sql/type/SqlTypeUtil.java  | 5 ++++-
 core/src/main/java/org/apache/calcite/util/Compatible.java  | 3 ++-
 .../java/org/apache/calcite/util/ImmutableNullableList.java | 2 +-
 core/src/main/java/org/apache/calcite/util/Util.java        | 6 +++---
 11 files changed, 20 insertions(+), 13 deletions(-)

diff --git a/core/src/main/java/org/apache/calcite/rel/logical/LogicalAsofJoin.java b/core/src/main/java/org/apache/calcite/rel/logical/LogicalAsofJoin.java
index d6935607e384..e00f442ec78f 100644
--- a/core/src/main/java/org/apache/calcite/rel/logical/LogicalAsofJoin.java
+++ b/core/src/main/java/org/apache/calcite/rel/logical/LogicalAsofJoin.java
@@ -125,7 +125,8 @@ public LogicalAsofJoin copy(
       return true;
     }
     return deepEquals0(obj)
-        && matchCondition.equals(((LogicalAsofJoin) obj).matchCondition)
+        && matchCondition.equals(
+            ((LogicalAsofJoin) requireNonNull(obj, "obj")).matchCondition)
         && systemFieldList.equals(((LogicalAsofJoin) obj).systemFieldList);
   }
 
diff --git a/core/src/main/java/org/apache/calcite/rel/logical/LogicalFilter.java b/core/src/main/java/org/apache/calcite/rel/logical/LogicalFilter.java
index 9703f76fbf74..7ac196f951ab 100644
--- a/core/src/main/java/org/apache/calcite/rel/logical/LogicalFilter.java
+++ b/core/src/main/java/org/apache/calcite/rel/logical/LogicalFilter.java
@@ -165,7 +165,7 @@ public static LogicalFilter create(final RelNode input, RexNode condition,
 
   @Override public boolean deepEquals(@Nullable Object obj) {
     return deepEquals0(obj)
-        && variablesSet.equals(((LogicalFilter) obj).variablesSet);
+        && variablesSet.equals(((LogicalFilter) requireNonNull(obj, "obj")).variablesSet);
   }
 
   @Override public int deepHashCode() {
diff --git a/core/src/main/java/org/apache/calcite/rel/logical/LogicalJoin.java b/core/src/main/java/org/apache/calcite/rel/logical/LogicalJoin.java
index ee24239828bf..361de3353f4b 100644
--- a/core/src/main/java/org/apache/calcite/rel/logical/LogicalJoin.java
+++ b/core/src/main/java/org/apache/calcite/rel/logical/LogicalJoin.java
@@ -197,7 +197,7 @@ public static LogicalJoin create(RelNode left, RelNode right, List hint
       return true;
     }
     return deepEquals0(obj)
-        && semiJoinDone == ((LogicalJoin) obj).semiJoinDone
+        && semiJoinDone == ((LogicalJoin) requireNonNull(obj, "obj")).semiJoinDone
         && systemFieldList.equals(((LogicalJoin) obj).systemFieldList);
   }
 
diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/JaninoRelMetadataProvider.java b/core/src/main/java/org/apache/calcite/rel/metadata/JaninoRelMetadataProvider.java
index b7df1a707856..661779fcb8a9 100644
--- a/core/src/main/java/org/apache/calcite/rel/metadata/JaninoRelMetadataProvider.java
+++ b/core/src/main/java/org/apache/calcite/rel/metadata/JaninoRelMetadataProvider.java
@@ -238,7 +238,8 @@ private Key(Class> handlerClass,
     return handlerClass.cast(
         Proxy.newProxyInstance(RelMetadataQuery.class.getClassLoader(),
             new Class[] {handlerClass}, (proxy, method, args) -> {
-              final RelNode r = requireNonNull((RelNode) args[0], "(RelNode) args[0]");
+              final RelNode r =
+                  requireNonNull((RelNode) requireNonNull(args, "args")[0], "args[0]");
               throw new NoHandler(r.getClass());
             }));
   }
diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/ProxyingMetadataHandlerProvider.java b/core/src/main/java/org/apache/calcite/rel/metadata/ProxyingMetadataHandlerProvider.java
index 92490cdd6de9..af3d2cee18f0 100644
--- a/core/src/main/java/org/apache/calcite/rel/metadata/ProxyingMetadataHandlerProvider.java
+++ b/core/src/main/java/org/apache/calcite/rel/metadata/ProxyingMetadataHandlerProvider.java
@@ -82,7 +82,7 @@ public ProxyingMetadataHandlerProvider(RelMetadataProvider provider) {
       Method metadataMethod =
           requireNonNull(methodMap.get(method.getName()),
               () -> "Not supported: " + method);
-      RelNode rel = requireNonNull((RelNode) args[0], "rel must be non null");
+      RelNode rel = requireNonNull((RelNode) requireNonNull(args, "args")[0], "args[0]");
       RelMetadataQuery mq =
           requireNonNull((RelMetadataQuery) args[1], "mq must be non null");
 
diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMetadataQueryBase.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMetadataQueryBase.java
index cb8840b0011c..f45e90b31fc8 100644
--- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMetadataQueryBase.java
+++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMetadataQueryBase.java
@@ -95,7 +95,8 @@ protected static  H initialHandler(Class handlerClass) {
     return handlerClass.cast(
         Proxy.newProxyInstance(RelMetadataQuery.class.getClassLoader(),
             new Class[] {handlerClass}, (proxy, method, args) -> {
-              final RelNode r = requireNonNull((RelNode) args[0], "(RelNode) args[0]");
+              final RelNode r =
+                  requireNonNull((RelNode) requireNonNull(args, "args")[0], "args[0]");
               throw new JaninoRelMetadataProvider.NoHandler(r.getClass());
             }));
   }
diff --git a/core/src/main/java/org/apache/calcite/sql/SqlBinaryOperator.java b/core/src/main/java/org/apache/calcite/sql/SqlBinaryOperator.java
index 3bdf92e31e01..34b1cb7cf928 100644
--- a/core/src/main/java/org/apache/calcite/sql/SqlBinaryOperator.java
+++ b/core/src/main/java/org/apache/calcite/sql/SqlBinaryOperator.java
@@ -166,7 +166,7 @@ private RelDataType convertType(SqlValidator validator, SqlCall call, RelDataTyp
             validator.getTypeFactory()
                 .createTypeWithCharsetAndCollation(
                     type,
-                    type.getCharset(),
+                    requireNonNull(type.getCharset(), "charset"),
                     requireNonNull(resultCol, "resultCol"));
       }
     }
diff --git a/core/src/main/java/org/apache/calcite/sql/type/SqlTypeUtil.java b/core/src/main/java/org/apache/calcite/sql/type/SqlTypeUtil.java
index 596b07965ada..71b73b4a4632 100644
--- a/core/src/main/java/org/apache/calcite/sql/type/SqlTypeUtil.java
+++ b/core/src/main/java/org/apache/calcite/sql/type/SqlTypeUtil.java
@@ -1418,7 +1418,10 @@ public static SqlDataTypeSpec convertTypeToSpec(RelDataType type,
    */
   public static SqlDataTypeSpec convertTypeToSpec(RelDataType type) {
     // TODO jvs 28-Dec-2004:  collation
-    String charSetName = inCharFamily(type) ? type.getCharset().name() : null;
+    String charSetName =
+        inCharFamily(type)
+            ? requireNonNull(type.getCharset(), "charset").name()
+            : null;
     return convertTypeToSpec(type, charSetName,
         RelDataType.PRECISION_NOT_SPECIFIED, RelDataType.SCALE_NOT_SPECIFIED);
   }
diff --git a/core/src/main/java/org/apache/calcite/util/Compatible.java b/core/src/main/java/org/apache/calcite/util/Compatible.java
index 924621d2f9a4..9af767f97b0b 100644
--- a/core/src/main/java/org/apache/calcite/util/Compatible.java
+++ b/core/src/main/java/org/apache/calcite/util/Compatible.java
@@ -47,7 +47,8 @@ Compatible create() {
               // Use MethodHandles.privateLookupIn if it is available (JDK 9
               // and above)
               @SuppressWarnings("rawtypes")
-              final Class clazz = (Class) requireNonNull(args[0], "args[0]");
+              final Class clazz =
+                  (Class) requireNonNull(requireNonNull(args, "args")[0], "args[0]");
               try {
                 final Method privateLookupMethod =
                     MethodHandles.class.getMethod("privateLookupIn",
diff --git a/core/src/main/java/org/apache/calcite/util/ImmutableNullableList.java b/core/src/main/java/org/apache/calcite/util/ImmutableNullableList.java
index 4ca730415f0a..1d186cbe0945 100644
--- a/core/src/main/java/org/apache/calcite/util/ImmutableNullableList.java
+++ b/core/src/main/java/org/apache/calcite/util/ImmutableNullableList.java
@@ -67,7 +67,7 @@ private ImmutableNullableList(E[] elements) {
     // If there are no nulls, ImmutableList is better.
     for (E object : elements) {
       if (object == null) {
-        final Object[] objects = elements.toArray();
+        final @Nullable Object[] objects = elements.toArray();
         //noinspection unchecked
         return new ImmutableNullableList<>((E[]) objects);
       }
diff --git a/core/src/main/java/org/apache/calcite/util/Util.java b/core/src/main/java/org/apache/calcite/util/Util.java
index 8ed233f6e18d..4c1344d26a9d 100644
--- a/core/src/main/java/org/apache/calcite/util/Util.java
+++ b/core/src/main/java/org/apache/calcite/util/Util.java
@@ -140,13 +140,13 @@ private Util() {}
    * necessary, to make them look like Linux actual.
    */
   public static final String LINE_SEPARATOR =
-      System.getProperty("line.separator");
+      requireNonNull(System.getProperty("line.separator"), "line.separator");
 
   /**
    * System-dependent file separator, for example, "/" or "\."
    */
   public static final String FILE_SEPARATOR =
-      System.getProperty("file.separator");
+      requireNonNull(System.getProperty("file.separator"), "file.separator");
 
   /**
    * Datetime format string for generating a timestamp string to be used as
@@ -753,7 +753,7 @@ public static URL toURL(File file) throws MalformedURLException {
     // This is a bunch of weird code that is required to
     // make a valid URL on the Windows platform, due
     // to inconsistencies in what getAbsolutePath returns.
-    String fs = System.getProperty("file.separator");
+    String fs = FILE_SEPARATOR;
     if (fs.length() == 1) {
       char sep = fs.charAt(0);
       if (sep != '/') {

From 44c1bb8d76c26b29d6ab68579f16d9c2fd3ed481 Mon Sep 17 00:00:00 2001
From: Vladimir Sitnikov 
Date: Mon, 24 Aug 2026 16:07:22 +0300
Subject: [PATCH 528/562] [CALCITE-7736] Let the values that are genuinely
 absent say so

`UnboundMetadata.bind` answers null when the provider has no metadata for a
node, which three implementations already did and none declared.

`LazyReference` holds nothing until its supplier has run, so its
`AtomicReference` is `<@Nullable T>` and the two `(T) null` casts go away.

`jsonObjectAggAdd` and `jsonArrayAggAdd` took a raw `Map` and `List` and put
null into them, which is what SQL/JSON asks for under NULL ON NULL. They now
take `Map` and `List<@Nullable Object>`.

`Utilities.compare` is suppressed: its `Comparator` parameter is raw, so
NullAway checks the call against the erased `compare(Object, Object)`, while a
SQL comparator is exactly the thing that orders nulls.

`Matcher` casts the row at offset 0 of the memory window, which is the one the
predicate has just accepted. The window itself is padded with nulls at either
end, which is why `Memory.get` is nullable.

Co-Authored-By: Claude Opus 5 
---
 .../org/apache/calcite/rel/metadata/UnboundMetadata.java  | 2 +-
 .../java/org/apache/calcite/runtime/JsonFunctions.java    | 5 +++--
 .../src/main/java/org/apache/calcite/runtime/Matcher.java | 7 +++++--
 .../main/java/org/apache/calcite/runtime/Utilities.java   | 1 +
 .../main/java/org/apache/calcite/util/LazyReference.java  | 8 +++++---
 5 files changed, 15 insertions(+), 8 deletions(-)

diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/UnboundMetadata.java b/core/src/main/java/org/apache/calcite/rel/metadata/UnboundMetadata.java
index a4ac28ccc047..7bd6a81c6ae1 100644
--- a/core/src/main/java/org/apache/calcite/rel/metadata/UnboundMetadata.java
+++ b/core/src/main/java/org/apache/calcite/rel/metadata/UnboundMetadata.java
@@ -28,5 +28,5 @@
  */
 @FunctionalInterface
 public interface UnboundMetadata {
-  M bind(RelNode rel, RelMetadataQuery mq);
+  @Nullable M bind(RelNode rel, RelMetadataQuery mq);
 }
diff --git a/core/src/main/java/org/apache/calcite/runtime/JsonFunctions.java b/core/src/main/java/org/apache/calcite/runtime/JsonFunctions.java
index 5bb16adf62fa..a3e6b0c273dd 100644
--- a/core/src/main/java/org/apache/calcite/runtime/JsonFunctions.java
+++ b/core/src/main/java/org/apache/calcite/runtime/JsonFunctions.java
@@ -447,7 +447,8 @@ public static String jsonObject(SqlJsonConstructorNullClause nullClause,
     return jsonize(map);
   }
 
-  public static void jsonObjectAggAdd(Map map, String k, @Nullable Object v,
+  public static void jsonObjectAggAdd(Map map, String k,
+      @Nullable Object v,
       SqlJsonConstructorNullClause nullClause) {
     if (k == null) {
       throw RESOURCE.nullKeyOfJsonObjectNotAllowed().ex();
@@ -476,7 +477,7 @@ public static String jsonArray(SqlJsonConstructorNullClause nullClause,
     return jsonize(list);
   }
 
-  public static void jsonArrayAggAdd(List list, @Nullable Object element,
+  public static void jsonArrayAggAdd(List<@Nullable Object> list, @Nullable Object element,
       SqlJsonConstructorNullClause nullClause) {
     if (element == null) {
       if (nullClause == SqlJsonConstructorNullClause.NULL_ON_NULL) {
diff --git a/core/src/main/java/org/apache/calcite/runtime/Matcher.java b/core/src/main/java/org/apache/calcite/runtime/Matcher.java
index 347a6887dc12..699db8eee7bf 100644
--- a/core/src/main/java/org/apache/calcite/runtime/Matcher.java
+++ b/core/src/main/java/org/apache/calcite/runtime/Matcher.java
@@ -38,6 +38,8 @@
 import java.util.function.Predicate;
 import java.util.stream.Collectors;
 
+import static org.apache.calcite.linq4j.Nullness.castNonNull;
+
 import static java.util.Objects.requireNonNull;
 
 /**
@@ -123,7 +125,8 @@ protected List> matchOneWithSymbols(MemoryFactory.Memory rows
           for (DeterministicAutomaton.Transition transition : transitions) {
             // System.out.println("Append new transition to ");
             final PartialMatch newMatch =
-                pm.append(transition.symbol, rows.get(), transition.toState);
+                pm.append(transition.symbol, castNonNull(rows.get()),
+                    transition.toState);
             newMatches.add(newMatch);
           }
         }
@@ -139,7 +142,7 @@ protected List> matchOneWithSymbols(MemoryFactory.Memory rows
         for (DeterministicAutomaton.Transition transition : transitions) {
           final PartialMatch newMatch =
               new PartialMatch<>(-1L, ImmutableList.of(transition.symbol),
-                  ImmutableList.of(rows.get()), transition.toState);
+                  ImmutableList.of(castNonNull(rows.get())), transition.toState);
           newMatches.add(newMatch);
         }
       }
diff --git a/core/src/main/java/org/apache/calcite/runtime/Utilities.java b/core/src/main/java/org/apache/calcite/runtime/Utilities.java
index 186630240add..148f2acca4f7 100644
--- a/core/src/main/java/org/apache/calcite/runtime/Utilities.java
+++ b/core/src/main/java/org/apache/calcite/runtime/Utilities.java
@@ -220,6 +220,7 @@ public static int compareNullsLast(@Nullable Comparable v0, @Nullable Comparable
                 : v0.compareTo(v1);
   }
 
+  @SuppressWarnings("NullAway") // a SQL comparator orders nulls; Comparator here is raw
   public static int compare(@Nullable Comparable v0, @Nullable Comparable v1,
       Comparator comparator) {
     //noinspection unchecked
diff --git a/core/src/main/java/org/apache/calcite/util/LazyReference.java b/core/src/main/java/org/apache/calcite/util/LazyReference.java
index 32f5c174f7a3..d3aa98899e1e 100644
--- a/core/src/main/java/org/apache/calcite/util/LazyReference.java
+++ b/core/src/main/java/org/apache/calcite/util/LazyReference.java
@@ -16,6 +16,8 @@
  */
 package org.apache.calcite.util;
 
+import org.jspecify.annotations.Nullable;
+
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Supplier;
 
@@ -32,7 +34,7 @@
  */
 public class LazyReference {
 
-  private final AtomicReference value = new AtomicReference<>();
+  private final AtomicReference<@Nullable T> value = new AtomicReference<>();
 
   /**
    * Atomically sets the value to {@code supplier.get()}
@@ -51,7 +53,7 @@ public T getOrCompute(Supplier supplier) {
         return result;
       }
       T computed = supplier.get();
-      if (value.compareAndSet((T) null, computed)) {
+      if (value.compareAndSet(null, computed)) {
         return computed;
       }
     }
@@ -61,6 +63,6 @@ public T getOrCompute(Supplier supplier) {
    * Resets the current value.
    */
   public void reset() {
-    value.set((T) null);
+    value.set(null);
   }
 }

From 21a43a514aed210537dc35e24f543d8d427090ec Mon Sep 17 00:00:00 2001
From: Vladimir Sitnikov 
Date: Mon, 24 Aug 2026 16:16:18 +0300
Subject: [PATCH 529/562] [CALCITE-7736] Type the collections and results that
 hold absent values

A run of one-off corrections, all of the same shape: a value that can be absent
was flowing into a type that said it could not.

* `Collections.nCopies(n, null)` builds a list of absent values, in
  `ProfilerImpl` twice and `AggregateReduceFunctionsRule`
* `CacheUtil` returns what `toArray()` gives it, `@Nullable Object[]`
* `EnumerableTableModify` keys an update by a source row, whose column values
  may be null
* `RexSimplify.evaluate` returns a value or an exception, exactly one of which
  is set, so both halves of the `Pair` are nullable
* `RelToSqlConverter` collects into a list that becomes a `SqlNodeList`
* `SerializableCharset.readObject` reads back what `writeObject` wrote, and
  `DelegatingScope` looks up a key it took from the map's own key set
* `EnumerableWindow` asks for an offset after ruling out CURRENT ROW, which is
  the case that has none

`SqlNode.toList` and `RelMdColumnUniqueness` replace a method reference with a
lambda. A reference resolves against the JDK model, which disagrees with the
functional interface it is being assigned to; a lambda is inferred from the
target type instead.

Co-Authored-By: Claude Opus 5 
---
 .../adapter/enumerable/EnumerableTableModify.java      |  6 ++++--
 .../calcite/adapter/enumerable/EnumerableWindow.java   |  4 ++--
 .../java/org/apache/calcite/profile/ProfilerImpl.java  | 10 +++++-----
 .../calcite/rel/metadata/RelMdColumnUniqueness.java    |  4 +---
 .../apache/calcite/rel/metadata/janino/CacheUtil.java  |  6 ++++--
 .../apache/calcite/rel/rel2sql/RelToSqlConverter.java  |  6 +++---
 .../rel/rules/AggregateReduceFunctionsRule.java        |  2 +-
 .../main/java/org/apache/calcite/rex/RexSimplify.java  |  9 ++++++---
 core/src/main/java/org/apache/calcite/sql/SqlNode.java |  2 +-
 .../apache/calcite/sql/validate/DelegatingScope.java   |  2 +-
 .../org/apache/calcite/util/SerializableCharset.java   |  2 +-
 11 files changed, 29 insertions(+), 24 deletions(-)

diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableTableModify.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableTableModify.java
index 49528837b52b..2c926b66092a 100644
--- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableTableModify.java
+++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableTableModify.java
@@ -227,7 +227,8 @@ public static long applyUpdateOneToOne(Enumerable source, List e = source.enumerator()) {
       while (e.moveNext()) {
         final Object[] sourceRow = e.current();
-        final List key = Arrays.asList(Arrays.copyOf(sourceRow, tableFieldCount));
+        final List<@Nullable Object> key =
+            Arrays.asList(Arrays.copyOf(sourceRow, tableFieldCount));
         final Object[] newRow = applyUpdate(sourceRow, tableFieldCount, updateColumnIndices);
         updatesByKey.computeIfAbsent(key, k -> new ArrayDeque<>()).addLast(newRow);
       }
@@ -516,7 +517,8 @@ public static void applyDeleteRowsByKey(Enumerable sourceKeys,
    * @return key for row
    */
   private static List keyOf(Object[] rowValues) {
-    return Arrays.asList(Arrays.copyOf(rowValues, rowValues.length));
+    return Arrays.<@Nullable Object>asList(
+        Arrays.copyOf(rowValues, rowValues.length));
   }
 
 }
diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java
index 5a0ffc62afc3..818f70a6bf86 100644
--- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java
+++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java
@@ -953,7 +953,7 @@ private static Expression translateBound(RexToLixTranslator translator,
       if (bound.isCurrentRow()) {
         return i_;
       }
-      RexNode node = bound.getOffset();
+      RexNode node = requireNonNull(bound.getOffset(), "offset");
       Expression offs = translator.translate(node);
       // Floating offset does not make sense since we refer to array index.
       // Nulls do not make sense as well.
@@ -1004,7 +1004,7 @@ private static Expression translateBound(RexToLixTranslator translator,
     Expression val =
         translator.translate(new RexInputRef(orderKey, keyType), desiredKeyType);
     if (!bound.isCurrentRow()) {
-      RexNode node = bound.getOffset();
+      RexNode node = requireNonNull(bound.getOffset(), "offset");
       Expression offs = translator.translate(node);
       // TODO: support date + interval somehow
       if (bound.isFollowing()) {
diff --git a/core/src/main/java/org/apache/calcite/profile/ProfilerImpl.java b/core/src/main/java/org/apache/calcite/profile/ProfilerImpl.java
index 6a9337c6af73..81448756bcaf 100644
--- a/core/src/main/java/org/apache/calcite/profile/ProfilerImpl.java
+++ b/core/src/main/java/org/apache/calcite/profile/ProfilerImpl.java
@@ -166,7 +166,8 @@ class Run {
         }
       }
       this.singletonSpaces =
-          new ArrayList<>(Collections.nCopies(columns.size(), (Space) null));
+          new ArrayList<>(
+              Collections.<@Nullable Space>nCopies(columns.size(), null));
       if (combinationsPerPass > Math.pow(2D, columns.size())) {
         // There are not many columns. We can compute all combinations in the
         // first pass.
@@ -627,11 +628,10 @@ static class CompositeCollector extends Collector {
         // Too many values. Switch to a sketch collector.
         final HllCompositeCollector collector =
             new HllCompositeCollector(space, columnOrdinals);
-        final List list =
+        final List<@Nullable Comparable> list =
             new ArrayList<>(
-                Collections.nCopies(columnOrdinals[columnOrdinals.length - 1]
-                        + 1,
-                    null));
+                Collections.<@Nullable Comparable>nCopies(
+                    columnOrdinals[columnOrdinals.length - 1] + 1, null));
         for (FlatLists.ComparableList value : this.values) {
           for (int i = 0; i < value.size(); i++) {
             Comparable c = (Comparable) value.get(i);
diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdColumnUniqueness.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdColumnUniqueness.java
index 8fb570305b8c..9fd69e13eadc 100644
--- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdColumnUniqueness.java
+++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdColumnUniqueness.java
@@ -46,7 +46,6 @@
 import org.apache.calcite.rex.RexLiteral;
 import org.apache.calcite.rex.RexNode;
 import org.apache.calcite.rex.RexProgram;
-import org.apache.calcite.rex.RexSlot;
 import org.apache.calcite.rex.RexSubQuery;
 import org.apache.calcite.rex.RexUtil;
 import org.apache.calcite.sql.SqlKind;
@@ -537,8 +536,7 @@ static ImmutableBitSet getConstantColumnSet(RelOptPredicateList relOptPredicateL
     relOptPredicateList.constantMap.keySet()
         .stream()
         .filter(RexInputRef.class::isInstance)
-        .map(RexInputRef.class::cast)
-        .map(RexSlot::getIndex)
+        .map(ref -> ((RexInputRef) ref).getIndex())
         .forEach(builder::set);
 
     relOptPredicateList.pulledUpPredicates.forEach(rex -> {
diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/janino/CacheUtil.java b/core/src/main/java/org/apache/calcite/rel/metadata/janino/CacheUtil.java
index 06aa79752afb..c474249904d8 100644
--- a/core/src/main/java/org/apache/calcite/rel/metadata/janino/CacheUtil.java
+++ b/core/src/main/java/org/apache/calcite/rel/metadata/janino/CacheUtil.java
@@ -17,6 +17,7 @@
 package org.apache.calcite.rel.metadata.janino;
 
 import org.apiguardian.api.API;
+import org.jspecify.annotations.Nullable;
 
 import java.util.stream.IntStream;
 
@@ -30,7 +31,7 @@ private CacheUtil() {
   }
 
   @API(status = API.Status.INTERNAL)
-  public static Object[] generateRange(String description, int min, int max) {
+  public static @Nullable Object[] generateRange(String description, int min, int max) {
     return IntStream.range(min, max)
         .mapToObj(i -> description + "(" + i + ")")
         .map(org.apache.calcite.rel.metadata.janino.DescriptiveCacheKey::new)
@@ -38,7 +39,8 @@ public static Object[] generateRange(String description, int min, int max) {
   }
 
   @API(status = API.Status.INTERNAL)
-  public static > Object[] generateEnum(String description, E[] values) {
+  public static > @Nullable Object[] generateEnum(String description,
+      E[] values) {
     return java.util.Arrays.stream(values)
         .map(e -> description + "(" + e + ")")
         .map(org.apache.calcite.rel.metadata.janino.DescriptiveCacheKey::new)
diff --git a/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java b/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java
index 2d0f8c56b231..af1e03a4e1a3 100644
--- a/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java
+++ b/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java
@@ -1078,7 +1078,7 @@ public Result visit(TableScan e) {
               SqlNodeList.of(pos,
                   hints.stream()
                       .map(h -> RelToSqlConverter.toSqlHint(h, pos))
-                      .collect(Collectors.toList())));
+                      .collect(Collectors.<@Nullable SqlNode>toList())));
     } else {
       node = identifier;
     }
@@ -1092,13 +1092,13 @@ private static SqlHint toSqlHint(RelHint hint, SqlParserPos pos) {
               .flatMap(
                   e -> Stream.of(new SqlIdentifier(e.getKey(), pos),
                   SqlLiteral.createCharString(e.getValue(), pos)))
-              .collect(Collectors.toList())),
+              .collect(Collectors.<@Nullable SqlNode>toList())),
           SqlHint.HintOptionFormat.KV_LIST);
     } else if (hint.listOptions != null) {
       return new SqlHint(pos, new SqlIdentifier(hint.hintName, pos),
           SqlNodeList.of(pos, hint.listOptions.stream()
               .map(e -> SqlLiteral.createCharString(e, pos))
-              .collect(Collectors.toList())),
+              .collect(Collectors.<@Nullable SqlNode>toList())),
           SqlHint.HintOptionFormat.LITERAL_LIST);
     }
     return new SqlHint(pos, new SqlIdentifier(hint.hintName, pos),
diff --git a/core/src/main/java/org/apache/calcite/rel/rules/AggregateReduceFunctionsRule.java b/core/src/main/java/org/apache/calcite/rel/rules/AggregateReduceFunctionsRule.java
index d0e1a86bab03..c8f145acede6 100644
--- a/core/src/main/java/org/apache/calcite/rel/rules/AggregateReduceFunctionsRule.java
+++ b/core/src/main/java/org/apache/calcite/rel/rules/AggregateReduceFunctionsRule.java
@@ -262,7 +262,7 @@ private void reduceAggs(
       relBuilder.project(inputExprs,
           CompositeList.of(
               relBuilder.peek().getRowType().getFieldNames(),
-              Collections.nCopies(extraArgCount, null)));
+              Collections.<@Nullable String>nCopies(extraArgCount, null)));
     }
     newAggregateRel(relBuilder, oldAggRel, newCalls);
     newCalcRel(relBuilder, oldAggRel.getRowType(), projList);
diff --git a/core/src/main/java/org/apache/calcite/rex/RexSimplify.java b/core/src/main/java/org/apache/calcite/rex/RexSimplify.java
index 2170c8a146b3..a0902d1577cf 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexSimplify.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexSimplify.java
@@ -2430,7 +2430,8 @@ private static void absorb(List terms, SqlKind compositeKind) {
     }
   }
 
-  private Pair evaluate(RexNode e, Map map) {
+  private Pair<@Nullable Comparable, @Nullable RuntimeException> evaluate(RexNode e,
+      Map map) {
     Comparable c = null;
     RuntimeException ex = null;
     try {
@@ -2474,8 +2475,10 @@ private void verify(RexNode before, RexNode simplified, RexUnknownAs unknownAs)
           continue assignment_loop;
         }
       }
-      Pair p0 = evaluate(foo0.e, map);
-      Pair p1 = evaluate(foo1.e, map);
+      Pair<@Nullable Comparable, @Nullable RuntimeException> p0 =
+          evaluate(foo0.e, map);
+      Pair<@Nullable Comparable, @Nullable RuntimeException> p1 =
+          evaluate(foo1.e, map);
       if (p0.right != null || p1.right != null) {
         if (p0.right == null || p1.right == null) {
           throw Util.first(p0.right, p1.right);
diff --git a/core/src/main/java/org/apache/calcite/sql/SqlNode.java b/core/src/main/java/org/apache/calcite/sql/SqlNode.java
index c34e27515e93..270d842825c0 100644
--- a/core/src/main/java/org/apache/calcite/sql/SqlNode.java
+++ b/core/src/main/java/org/apache/calcite/sql/SqlNode.java
@@ -384,7 +384,7 @@ public static boolean equalDeep(List operands0,
       ArrayList<@Nullable SqlNode>, SqlNodeList> toList(SqlParserPos pos) {
     //noinspection RedundantTypeArguments
     return Collector., SqlNodeList>of(
-        ArrayList::new, ArrayList::add, Util::combine,
+        ArrayList::new, (list, e) -> list.add(e), Util::combine,
         (ArrayList<@Nullable SqlNode> list) -> SqlNodeList.of(pos, list));
   }
 }
diff --git a/core/src/main/java/org/apache/calcite/sql/validate/DelegatingScope.java b/core/src/main/java/org/apache/calcite/sql/validate/DelegatingScope.java
index 8cbbe69db3cb..f9b58ad37670 100644
--- a/core/src/main/java/org/apache/calcite/sql/validate/DelegatingScope.java
+++ b/core/src/main/java/org/apache/calcite/sql/validate/DelegatingScope.java
@@ -294,7 +294,7 @@ protected void addColumnNames(
             RESOURCE.columnNotFound(columnName));
       case 1:
         tableName = map.keySet().iterator().next();
-        namespace = map.get(tableName).namespace;
+        namespace = requireNonNull(map.get(tableName), "tableName").namespace;
         break;
       default:
         throw validator.newValidationError(identifier,
diff --git a/core/src/main/java/org/apache/calcite/util/SerializableCharset.java b/core/src/main/java/org/apache/calcite/util/SerializableCharset.java
index 262c43d7822a..924830979eae 100644
--- a/core/src/main/java/org/apache/calcite/util/SerializableCharset.java
+++ b/core/src/main/java/org/apache/calcite/util/SerializableCharset.java
@@ -71,7 +71,7 @@ private void writeObject(ObjectOutputStream out) throws IOException {
   @SuppressWarnings("JdkObsolete")
   private void readObject(ObjectInputStream in)
       throws IOException, ClassNotFoundException {
-    charsetName = (String) in.readObject();
+    charsetName = requireNonNull((String) in.readObject(), "charsetName");
     charset =
         requireNonNull(Charset.availableCharsets().get(this.charsetName),
             () -> "charset is not found: " + charsetName);

From 8e9c6e8070afa8a645437657038bc5adcd8bf174 Mon Sep 17 00:00:00 2001
From: Vladimir Sitnikov 
Date: Mon, 24 Aug 2026 16:31:03 +0300
Subject: [PATCH 530/562] [CALCITE-7736] Let unwrap say it may find nothing,
 and split two contract guards

`SqlValidatorNamespace.unwrap` answers null when the namespace is not of the
requested type, which `AbstractNamespace` has always done and neither it nor
`DelegatingNamespace` declared. Six call sites in `SqlValidatorImpl` and
`SqlValidatorUtil` ask through `requireNonNull`; each has already established
the type it is asking for.

`NumberUtil.add` and `RelMdPercentageOriginalRows.quotientForPercentage` write
one `if` per argument rather than `if (a == null || b == null)`, so that no
control-flow merge precedes the `return null`. `CheckContracts` treats a return
reached by a merge as reachable even when every incoming edge is not, and
reports the `@Contract("!null, !null -> !null")` as violated. Reported as
uber/NullAway#1731, see nullaway-bugs/contract-check-misses-merged-guard.md. The
other 108 contracts in the codebase verify without complaint.

Also: `TryThreadLocal.of` and `Functions.generate` take the nullable bound of
the type they build, and `Prepare.THREAD_INSUBQUERY_THRESHOLD` holds a threshold
that may be unset.

Co-Authored-By: Claude Opus 5 
---
 core/src/main/java/org/apache/calcite/DataContext.java    | 4 +++-
 .../calcite/adapter/enumerable/EnumerableTableModify.java | 2 +-
 .../src/main/java/org/apache/calcite/prepare/Prepare.java | 2 +-
 .../apache/calcite/rel/metadata/MetadataFactoryImpl.java  | 4 +++-
 .../calcite/rel/metadata/RelMdPercentageOriginalRows.java | 8 +++++++-
 .../org/apache/calcite/rel/rel2sql/SqlImplementor.java    | 6 ++++--
 .../main/java/org/apache/calcite/rex/RexLambdaRef.java    | 5 ++++-
 core/src/main/java/org/apache/calcite/runtime/Hook.java   | 4 +++-
 .../apache/calcite/sql/validate/DelegatingNamespace.java  | 2 +-
 .../org/apache/calcite/sql/validate/SqlValidatorImpl.java | 8 +++++---
 .../calcite/sql/validate/SqlValidatorNamespace.java       | 2 +-
 .../org/apache/calcite/sql/validate/SqlValidatorUtil.java | 7 ++++---
 .../src/main/java/org/apache/calcite/util/NumberUtil.java | 8 +++++++-
 .../main/java/org/apache/calcite/util/TryThreadLocal.java | 2 +-
 .../org/apache/calcite/linq4j/function/Functions.java     | 2 +-
 15 files changed, 46 insertions(+), 20 deletions(-)

diff --git a/core/src/main/java/org/apache/calcite/DataContext.java b/core/src/main/java/org/apache/calcite/DataContext.java
index 82f0823ee424..43a025fc2b90 100644
--- a/core/src/main/java/org/apache/calcite/DataContext.java
+++ b/core/src/main/java/org/apache/calcite/DataContext.java
@@ -35,6 +35,8 @@
 import java.util.TimeZone;
 import java.util.concurrent.atomic.AtomicBoolean;
 
+import static org.apache.calcite.linq4j.Nullness.castNonNull;
+
 /**
  * Runtime context allowing access to the tables in a database.
  *
@@ -152,7 +154,7 @@ enum Variable {
     /** Returns the value of this variable in a given data context. */
     public  T get(DataContext dataContext) {
       //noinspection unchecked
-      return (T) clazz.cast(dataContext.get(camelName));
+      return (T) castNonNull(clazz.cast(dataContext.get(camelName)));
     }
   }
 }
diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableTableModify.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableTableModify.java
index 2c926b66092a..07eca16cb917 100644
--- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableTableModify.java
+++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableTableModify.java
@@ -223,7 +223,7 @@ private Result implementUpdate(
    */
   public static long applyUpdateOneToOne(Enumerable source, List sink,
       int tableFieldCount, int[] updateColumnIndices) {
-    final Map, Deque> updatesByKey = new HashMap<>();
+    final Map, Deque> updatesByKey = new HashMap<>();
     try (Enumerator e = source.enumerator()) {
       while (e.moveNext()) {
         final Object[] sourceRow = e.current();
diff --git a/core/src/main/java/org/apache/calcite/prepare/Prepare.java b/core/src/main/java/org/apache/calcite/prepare/Prepare.java
index 89a516e6b4c5..0e6be4cfe01f 100644
--- a/core/src/main/java/org/apache/calcite/prepare/Prepare.java
+++ b/core/src/main/java/org/apache/calcite/prepare/Prepare.java
@@ -116,7 +116,7 @@ public abstract class Prepare {
 
   // temporary. for testing.
   public static final TryThreadLocal<@Nullable Integer> THREAD_INSUBQUERY_THRESHOLD =
-      TryThreadLocal.of(DEFAULT_IN_SUB_QUERY_THRESHOLD);
+      TryThreadLocal.<@Nullable Integer>of(DEFAULT_IN_SUB_QUERY_THRESHOLD);
 
   protected Prepare(CalcitePrepare.Context context, CatalogReader catalogReader,
       Convention resultConvention) {
diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/MetadataFactoryImpl.java b/core/src/main/java/org/apache/calcite/rel/metadata/MetadataFactoryImpl.java
index 18444466ea06..e74d71f9e438 100644
--- a/core/src/main/java/org/apache/calcite/rel/metadata/MetadataFactoryImpl.java
+++ b/core/src/main/java/org/apache/calcite/rel/metadata/MetadataFactoryImpl.java
@@ -29,6 +29,8 @@
 
 import java.util.concurrent.ExecutionException;
 
+import static org.apache.calcite.linq4j.Nullness.castNonNull;
+
 /**
  * Implementation of {@link MetadataFactory} that gets providers from a
  * {@link RelMetadataProvider} and stores them in a cache.
@@ -72,7 +74,7 @@ public MetadataFactoryImpl(RelMetadataProvider provider) {
       final Pair, Class> key =
           Pair.of((Class) rel.getClass(), (Class) metadataClazz);
       final Metadata apply = cache.get(key).bind(rel, mq);
-      return metadataClazz.cast(apply);
+      return castNonNull(metadataClazz.cast(apply));
     } catch (UncheckedExecutionException | ExecutionException e) {
       throw Util.throwAsRuntime(Util.causeOrSelf(e));
     }
diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdPercentageOriginalRows.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdPercentageOriginalRows.java
index f6f3b587bcff..d76bf8daa20d 100644
--- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdPercentageOriginalRows.java
+++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdPercentageOriginalRows.java
@@ -239,7 +239,13 @@ public Double getPercentageOriginalRows(Union rel, RelMetadataQuery mq) {
   private static @Nullable Double quotientForPercentage(
       @Nullable Double numerator,
       @Nullable Double denominator) {
-    if ((numerator == null) || (denominator == null)) {
+    // One if per argument, so that no control-flow merge precedes the return: the
+    // contract check treats a return reached by a merge as reachable even when every
+    // incoming edge is not. https://github.com/uber/NullAway/issues/1731
+    if (numerator == null) {
+      return null;
+    }
+    if (denominator == null) {
       return null;
     }
 
diff --git a/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java b/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java
index 6cf68058e780..6d69014b01cb 100644
--- a/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java
+++ b/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java
@@ -1315,7 +1315,8 @@ private SqlNode createSqlWindowBound(RexWindowBound rexWindowBound) {
         if (rexWindowBound.isUnbounded()) {
           return SqlWindow.createUnboundedPreceding(POS);
         } else {
-          SqlNode literal = toSql(null, rexWindowBound.getOffset());
+          SqlNode literal =
+              toSql(null, requireNonNull(rexWindowBound.getOffset(), "offset"));
           return SqlWindow.createPreceding(literal, POS);
         }
       }
@@ -1323,7 +1324,8 @@ private SqlNode createSqlWindowBound(RexWindowBound rexWindowBound) {
         if (rexWindowBound.isUnbounded()) {
           return SqlWindow.createUnboundedFollowing(POS);
         } else {
-          SqlNode literal = toSql(null, rexWindowBound.getOffset());
+          SqlNode literal =
+              toSql(null, requireNonNull(rexWindowBound.getOffset(), "offset"));
           return SqlWindow.createFollowing(literal, POS);
         }
       }
diff --git a/core/src/main/java/org/apache/calcite/rex/RexLambdaRef.java b/core/src/main/java/org/apache/calcite/rex/RexLambdaRef.java
index 4688405d00cb..e68ee4ab329e 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexLambdaRef.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexLambdaRef.java
@@ -23,6 +23,8 @@
 
 import java.util.Objects;
 
+import static org.apache.calcite.linq4j.Nullness.castNonNull;
+
 /**
  * Variable that references a field of a lambda expression.
  */
@@ -42,7 +44,8 @@ public RexLambdaRef(int index, String name, RelDataType type) {
 
   @Override public  R accept(
       RexBiVisitor visitor, P arg) {
-    return (R) null;
+    // a lambda reference has no payload to hand a bi-visitor
+    return castNonNull(null);
   }
 
   @Override public boolean equals(final @Nullable Object obj) {
diff --git a/core/src/main/java/org/apache/calcite/runtime/Hook.java b/core/src/main/java/org/apache/calcite/runtime/Hook.java
index eced70ba2c56..3dbc8fe4038a 100644
--- a/core/src/main/java/org/apache/calcite/runtime/Hook.java
+++ b/core/src/main/java/org/apache/calcite/runtime/Hook.java
@@ -22,6 +22,7 @@
 import org.apache.calcite.util.Util;
 
 import org.apiguardian.api.API;
+import org.jspecify.annotations.Nullable;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -189,7 +190,8 @@ private boolean removeThread(Consumer handler) {
   /** @deprecated Use {@link #propertyJ}. */
   @SuppressWarnings("Guava")
   @Deprecated // return type will change in 2.0
-  public static  com.google.common.base.Function, Void> property(final V v) {
+  public static  com.google.common.base.Function, @Nullable Void> property(
+      final V v) {
     return holder -> {
       holder.set(v);
       return null;
diff --git a/core/src/main/java/org/apache/calcite/sql/validate/DelegatingNamespace.java b/core/src/main/java/org/apache/calcite/sql/validate/DelegatingNamespace.java
index f00939874d3b..6fc4d7d7f679 100644
--- a/core/src/main/java/org/apache/calcite/sql/validate/DelegatingNamespace.java
+++ b/core/src/main/java/org/apache/calcite/sql/validate/DelegatingNamespace.java
@@ -104,7 +104,7 @@ protected DelegatingNamespace(SqlValidatorNamespace namespace) {
   @Override public void makeNullable() {
   }
 
-  @Override public  T unwrap(Class clazz) {
+  @Override public  @Nullable T unwrap(Class clazz) {
     if (clazz.isInstance(this)) {
       return clazz.cast(this);
     } else {
diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java
index 04515b18c3ab..7984053df897 100644
--- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java
+++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java
@@ -1238,7 +1238,7 @@ private void lookupFromHints(
     }
     final SqlValidatorNamespace ns = getNamespaceOrThrow(node);
     if (ns.isWrapperFor(IdentifierNamespace.class)) {
-      IdentifierNamespace idNs = ns.unwrap(IdentifierNamespace.class);
+      IdentifierNamespace idNs = requireNonNull(ns.unwrap(IdentifierNamespace.class), "idNs");
       final SqlIdentifier id = idNs.getId();
       for (int i = 0; i < id.names.size(); i++) {
         if (pos.toString().equals(
@@ -7825,7 +7825,8 @@ public SqlNode extendedExpandGroupBy(SqlNode expr,
             requireNonNull(qualified.namespace,
                 () -> "namespace for " + qualified);
         if (namespace.isWrapperFor(AliasNamespace.class)) {
-          AliasNamespace aliasNs = namespace.unwrap(AliasNamespace.class);
+          AliasNamespace aliasNs =
+              requireNonNull(namespace.unwrap(AliasNamespace.class), "aliasNs");
           SqlNode aliased = requireNonNull(aliasNs.getNode(), () ->
               "sqlNode for aliasNs " + aliasNs);
           namespace = getNamespaceOrThrow(stripAs(aliased));
@@ -7840,7 +7841,8 @@ public SqlNode extendedExpandGroupBy(SqlNode expr,
         for (String name : qualified.suffix()) {
           if (namespace.isWrapperFor(UnnestNamespace.class)) {
             // If identifier is drawn from a repeated subrecord via unnest, add name of array field
-            UnnestNamespace unnestNamespace = namespace.unwrap(UnnestNamespace.class);
+            UnnestNamespace unnestNamespace =
+                requireNonNull(namespace.unwrap(UnnestNamespace.class), "unnestNamespace");
             final SqlQualified columnUnnestedFrom = unnestNamespace.getColumnUnnestedFrom(name);
             if (columnUnnestedFrom != null) {
               origin.addAll(columnUnnestedFrom.suffix());
diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorNamespace.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorNamespace.java
index 8e4efe31b850..73c3eb8046d1 100644
--- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorNamespace.java
+++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorNamespace.java
@@ -182,7 +182,7 @@ default boolean fieldExists(String name) {
    * @return This namespace cast to desired type
    * @throws ClassCastException if no such interface is available
    */
-   T unwrap(Class clazz);
+   @Nullable T unwrap(Class clazz);
 
   /**
    * Returns whether this namespace implements a given interface, or wraps a
diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorUtil.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorUtil.java
index cd5ff191a194..a48c2c7a52ce 100644
--- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorUtil.java
+++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorUtil.java
@@ -122,16 +122,17 @@ private SqlValidatorUtil() {}
       boolean @Nullable [] usedDataset) {
     if (namespace.isWrapperFor(TableNamespace.class)) {
       final TableNamespace tableNamespace =
-          namespace.unwrap(TableNamespace.class);
+          requireNonNull(namespace.unwrap(TableNamespace.class), "tableNamespace");
       return getRelOptTable(tableNamespace,
           requireNonNull(catalogReader, "catalogReader"), datasetName, usedDataset,
           tableNamespace.extendedFields);
     } else if (namespace.isWrapperFor(SqlValidatorImpl.DmlNamespace.class)) {
       final SqlValidatorImpl.DmlNamespace dmlNamespace =
-          namespace.unwrap(SqlValidatorImpl.DmlNamespace.class);
+          requireNonNull(namespace.unwrap(SqlValidatorImpl.DmlNamespace.class), "dmlNamespace");
       final SqlValidatorNamespace resolvedNamespace = dmlNamespace.resolve();
       if (resolvedNamespace.isWrapperFor(TableNamespace.class)) {
-        final TableNamespace tableNamespace = resolvedNamespace.unwrap(TableNamespace.class);
+        final TableNamespace tableNamespace =
+            requireNonNull(resolvedNamespace.unwrap(TableNamespace.class), "tableNamespace");
         final SqlValidatorTable validatorTable = tableNamespace.getTable();
         final List extendedFields = dmlNamespace.extendList == null
             ? ImmutableList.of()
diff --git a/core/src/main/java/org/apache/calcite/util/NumberUtil.java b/core/src/main/java/org/apache/calcite/util/NumberUtil.java
index 4ced1cbf04bd..7891d4b4bae7 100644
--- a/core/src/main/java/org/apache/calcite/util/NumberUtil.java
+++ b/core/src/main/java/org/apache/calcite/util/NumberUtil.java
@@ -151,7 +151,13 @@ public static long round(double d) {
   /** Returns the sum of two numbers, or null if either is null. */
   @Contract("!null, !null -> !null")
   public static @Nullable Double add(@Nullable Double a, @Nullable Double b) {
-    if (a == null || b == null) {
+    // One if per argument, so that no control-flow merge precedes the return: the
+    // contract check treats a return reached by a merge as reachable even when every
+    // incoming edge is not. https://github.com/uber/NullAway/issues/1731
+    if (a == null) {
+      return null;
+    }
+    if (b == null) {
       return null;
     }
 
diff --git a/core/src/main/java/org/apache/calcite/util/TryThreadLocal.java b/core/src/main/java/org/apache/calcite/util/TryThreadLocal.java
index cc601dcdd14e..47c3158314d6 100644
--- a/core/src/main/java/org/apache/calcite/util/TryThreadLocal.java
+++ b/core/src/main/java/org/apache/calcite/util/TryThreadLocal.java
@@ -35,7 +35,7 @@ public abstract class TryThreadLocal extends ThreadL
    *
    * @param initialValue Initial value
    */
-  public static  TryThreadLocal of(S initialValue) {
+  public static  TryThreadLocal of(S initialValue) {
     return new FixedTryThreadLocal<>(initialValue);
   }
 
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/function/Functions.java b/linq4j/src/main/java/org/apache/calcite/linq4j/function/Functions.java
index 9721d8fa5dc8..f48f468e16f6 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/function/Functions.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/function/Functions.java
@@ -374,7 +374,7 @@ public static  boolean all(List list,
 
   /** Returns a list generated by applying a function to each index between
    * 0 and {@code size} - 1. */
-  public static  List generate(final int size,
+  public static  List generate(final int size,
       final IntFunction fn) {
     if (size < 0) {
       throw new IllegalArgumentException();

From 78c0b34852a2b1e8f696c8d02e2c9baa81c849cb Mon Sep 17 00:00:00 2001
From: Vladimir Sitnikov 
Date: Mon, 24 Aug 2026 16:42:35 +0300
Subject: [PATCH 531/562] [CALCITE-7736] Ask for what is already established,
 and let the expression lists hold nulls

`getNamespaceOrThrow(node).unwrap(XNamespace.class)` asks for a namespace kind
the caller has just built, at five places in `SqlValidatorImpl`, and now says so
with `requireNonNull` rather than relying on the declaration `unwrap` no longer
makes.

`Expressions.list` takes a nullable element bound, at all three overloads: a
generated expression list holds a null comparer when the row type needs none.
Five `Util.first(physType.comparer(), ...)` calls move to `firstNonNull`, whose
fallback is a constant expression.

`ArrayTable` permutes a column into an array that keeps the column's nulls.

`SqlRowOperator` reads a field name once rather than testing one call and using
another, `FormatModels` checks the match it just found, and
`CalciteConnectionImpl` reads `user.name`, which is not guaranteed.

Co-Authored-By: Claude Opus 5 
---
 .../calcite/adapter/clone/ArrayTable.java     |  4 +--
 .../enumerable/EnumerableHashJoin.java        |  6 ++--
 .../enumerable/EnumerableMergeJoin.java       |  2 +-
 .../enumerable/EnumerableMergeUnion.java      |  2 +-
 .../calcite/jdbc/CalciteConnectionImpl.java   |  3 +-
 .../calcite/sql/fun/SqlRowOperator.java       |  5 ++--
 .../sql/validate/SqlValidatorImpl.java        | 29 +++++++++++++++----
 .../calcite/util/format/FormatModels.java     |  2 +-
 .../calcite/linq4j/tree/Expressions.java      |  6 ++--
 9 files changed, 39 insertions(+), 20 deletions(-)

diff --git a/core/src/main/java/org/apache/calcite/adapter/clone/ArrayTable.java b/core/src/main/java/org/apache/calcite/adapter/clone/ArrayTable.java
index bf77da18da5f..08872f02747f 100644
--- a/core/src/main/java/org/apache/calcite/adapter/clone/ArrayTable.java
+++ b/core/src/main/java/org/apache/calcite/adapter/clone/ArrayTable.java
@@ -281,13 +281,13 @@ public static class ObjectArray implements Representation {
     @Override public Object freeze(ColumnLoader.ValueSet valueSet, int @Nullable [] sources) {
       // We assume the values have been canonized.
       final List<@Nullable Comparable> list = permuteList(valueSet.values, sources);
-      return list.toArray(new Comparable[0]);
+      return list.toArray(new @Nullable Comparable[0]);
     }
 
     @Override public Object permute(Object dataSet, int[] sources) {
       @Nullable Comparable[] list = (@Nullable Comparable[]) dataSet;
       final int size = list.length;
-      final @Nullable Comparable[] comparables = new Comparable[size];
+      final @Nullable Comparable[] comparables = new @Nullable Comparable[size];
       for (int i = 0; i < size; i++) {
         comparables[i] = list[sources[i]];
       }
diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableHashJoin.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableHashJoin.java
index d186b2c15bd8..4b67f66f5512 100644
--- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableHashJoin.java
+++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableHashJoin.java
@@ -252,11 +252,11 @@ private Result implementHashMarkJoin(EnumerableRelImplementor implementor, Prefe
     final PhysType nullSafeKeyPhysType =
         leftResult.physType.project(leftNullSafeKeys, JavaRowFormat.LIST);
     final Expression nullSafeKeyComparator =
-        Util.first(nullSafeKeyPhysType.comparer(), Expressions.constant(null));
+        Util.firstNonNull(nullSafeKeyPhysType.comparer(), Expressions.constant(null));
     final PhysType keyPhysType =
         leftResult.physType.project(joinInfo.leftKeys, JavaRowFormat.LIST);
     final Expression keyComparator =
-        Util.first(keyPhysType.comparer(), Expressions.constant(null));
+        Util.firstNonNull(keyPhysType.comparer(), Expressions.constant(null));
 
     return implementor.result(physType,
         builder.append(
@@ -322,7 +322,7 @@ private Result implementHashSemiJoin(EnumerableRelImplementor implementor, Prefe
                         joinInfo.leftKeys, joinInfo.nullExclusionFlags),
                     rightResult.physType.generateNullAwareAccessor(
                         joinInfo.rightKeys, joinInfo.nullExclusionFlags),
-                    Util.first(keyPhysType.comparer(),
+                    Util.firstNonNull(keyPhysType.comparer(),
                         Expressions.constant(null)),
                     predicate)))
             .toBlock());
diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeJoin.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeJoin.java
index e4222961fbf6..022cd369e2bd 100644
--- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeJoin.java
+++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeJoin.java
@@ -545,7 +545,7 @@ public static EnumerableMergeJoin create(RelNode left, RelNode right,
                             leftResult.physType, rightResult.physType)),
                     Expressions.constant(EnumUtils.toLinq4jJoinType(joinType)),
                     comparator,
-                    Util.first(
+                    Util.firstNonNull(
                         leftKeyPhysType.comparer(),
                         Expressions.constant(null))))).toBlock());
   }
diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeUnion.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeUnion.java
index 09d5a9e221e8..c945ac2a0bc5 100644
--- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeUnion.java
+++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableMergeUnion.java
@@ -113,7 +113,7 @@ public static EnumerableMergeUnion create(RelCollation collation,
     final Expression sortComparator = pair.right;
 
     final Expression equalityComparator =
-        Util.first(physType.comparer(),
+        Util.firstNonNull(physType.comparer(),
             Expressions.call(BuiltInMethod.IDENTITY_COMPARER.method));
 
     final Expression unionExp =
diff --git a/core/src/main/java/org/apache/calcite/jdbc/CalciteConnectionImpl.java b/core/src/main/java/org/apache/calcite/jdbc/CalciteConnectionImpl.java
index b3d5a69a3f6c..244e6b1adb03 100644
--- a/core/src/main/java/org/apache/calcite/jdbc/CalciteConnectionImpl.java
+++ b/core/src/main/java/org/apache/calcite/jdbc/CalciteConnectionImpl.java
@@ -438,7 +438,8 @@ static class DataContextImpl implements DataContext {
       final long currentOffset = localOffset;
       final long sysOffset = TimeZone.getDefault().getOffset(time);
       final String user = "sa";
-      final String systemUser = System.getProperty("user.name");
+      final String systemUser =
+          requireNonNull(System.getProperty("user.name"), "user.name");
       final String localeName = connection.config().locale();
       final Locale locale = localeName != null
           ? Util.parseLocale(localeName) : Locale.ROOT;
diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlRowOperator.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlRowOperator.java
index c2abcdc30942..a854b1e6076c 100644
--- a/core/src/main/java/org/apache/calcite/sql/fun/SqlRowOperator.java
+++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlRowOperator.java
@@ -95,10 +95,9 @@ public SqlRowOperator(String name, @Nullable List fi
     final RelDataTypeFactory typeFactory = opBinding.getTypeFactory();
     final RelDataTypeFactory.Builder builder = typeFactory.builder();
     for (int i = 0; i < opBinding.getOperandCount(); i++) {
+      final String givenName = fieldNames == null ? null : fieldNames.get(i);
       final String fieldName =
-          fieldNames != null && fieldNames.get(i) != null
-              ? fieldNames.get(i)
-              : SqlUtil.deriveAliasFromOrdinal(i);
+          givenName != null ? givenName : SqlUtil.deriveAliasFromOrdinal(i);
       builder.add(fieldName, opBinding.getOperandType(i));
     }
     final RelDataType recordType = builder.build();
diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java
index 7984053df897..737cc7ad9fc7 100644
--- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java
+++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java
@@ -4474,7 +4474,9 @@ protected void validateSelect(
 
     // Namespace is either a select namespace or a wrapper around one.
     final SelectNamespace ns =
-        getNamespaceOrThrow(select).unwrap(SelectNamespace.class);
+        requireNonNull(
+            getNamespaceOrThrow(select).unwrap(SelectNamespace.class),
+            "ns");
 
     // Its rowtype is null, meaning it hasn't been validated yet.
     // This is important, because we need to take the targetRowType into
@@ -7072,7 +7074,9 @@ public void setOriginal(SqlNode expr, SqlNode original) {
     final SqlLambdaScope scope = (SqlLambdaScope) scopes.get(lambdaExpr);
     requireNonNull(scope, "scope");
     final LambdaNamespace ns =
-        getNamespaceOrThrow(lambdaExpr).unwrap(LambdaNamespace.class);
+        requireNonNull(
+            getNamespaceOrThrow(lambdaExpr).unwrap(LambdaNamespace.class),
+            "ns");
 
     // Check for duplicate lambda parameter names
     final SqlNameMatcher nameMatcher = catalogReader.nameMatcher();
@@ -7110,7 +7114,12 @@ public void setOriginal(SqlNode expr, SqlNode original) {
         (MatchRecognizeScope) getMatchRecognizeScope(matchRecognize);
 
     final MatchRecognizeNamespace ns =
-        getNamespaceOrThrow(call).unwrap(MatchRecognizeNamespace.class);
+
+        requireNonNull(
+
+            getNamespaceOrThrow(call).unwrap(MatchRecognizeNamespace.class),
+
+            "ns");
     assert ns.rowType == null;
 
     // rows per match
@@ -7363,7 +7372,12 @@ public void validatePivot(SqlPivot pivot) {
     final PivotScope scope = (PivotScope) getJoinScope(pivot);
 
     final PivotNamespace ns =
-        getNamespaceOrThrow(pivot).unwrap(PivotNamespace.class);
+
+        requireNonNull(
+
+            getNamespaceOrThrow(pivot).unwrap(PivotNamespace.class),
+
+            "ns");
     assert ns.rowType == null;
 
     // Given
@@ -7439,7 +7453,12 @@ public void validateUnpivot(SqlUnpivot unpivot) {
     final UnpivotScope scope = (UnpivotScope) getJoinScope(unpivot);
 
     final UnpivotNamespace ns =
-        getNamespaceOrThrow(unpivot).unwrap(UnpivotNamespace.class);
+
+        requireNonNull(
+
+            getNamespaceOrThrow(unpivot).unwrap(UnpivotNamespace.class),
+
+            "ns");
     assert ns.rowType == null;
 
     // Given
diff --git a/core/src/main/java/org/apache/calcite/util/format/FormatModels.java b/core/src/main/java/org/apache/calcite/util/format/FormatModels.java
index d9c68950f709..5188e56b396c 100644
--- a/core/src/main/java/org/apache/calcite/util/format/FormatModels.java
+++ b/core/src/main/java/org/apache/calcite/util/format/FormatModels.java
@@ -314,7 +314,7 @@ private static class FormatModelImpl implements FormatModel {
           elements.add(literalElement(literal));
         }
         // add the element match - use literal as default to be safe.
-        String key = matcher.group();
+        String key = requireNonNull(matcher.group(), "matcher.group()");
         elements.add(getElementMap().getOrDefault(key, literalElement(key)));
         i = matcher.end();
       }
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/Expressions.java b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/Expressions.java
index 48916d4a7a8b..6bdb4d30e8f5 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/tree/Expressions.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/tree/Expressions.java
@@ -3105,21 +3105,21 @@ public static Expression foldOr(List conditions) {
   /**
    * Creates an empty fluent list.
    */
-  public static  FluentList list() {
+  public static  FluentList list() {
     return new FluentArrayList<>();
   }
 
   /**
    * Creates a fluent list with given elements.
    */
-  @SafeVarargs public static  FluentList list(T... ts) {
+  @SafeVarargs public static  FluentList list(T... ts) {
     return new FluentArrayList<>(Arrays.asList(ts));
   }
 
   /**
    * Creates a fluent list with elements from the given collection.
    */
-  public static  FluentList list(Iterable ts) {
+  public static  FluentList list(Iterable ts) {
     return new FluentArrayList<>(toList(ts));
   }
 

From bed24c748efb60c6e8d5a98c7244f44e251d4c27 Mon Sep 17 00:00:00 2001
From: Vladimir Sitnikov 
Date: Mon, 24 Aug 2026 16:50:31 +0300
Subject: [PATCH 532/562] [CALCITE-7736] Keep Guava's firstNonNull in
 CalciteSystemProperty

Calling `Util.firstNonNull` here drags `Util`'s static initialiser into
`CalciteSystemProperty`'s, and `Util` reads
`CalciteSystemProperty.DEFAULT_CHARSET`, which is still null at that point. The
result is an `ExceptionInInitializerError` from a class-initialisation cycle
that takes every test using `CalciteAssert` with it.

The call was already `MoreObjects.firstNonNull` and stays that way. The previous
commit qualified it as `Util.firstNonNull` only to settle a name clash that the
static import had already settled.

Co-Authored-By: Claude Opus 5 
---
 .../org/apache/calcite/config/CalciteSystemProperty.java    | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/core/src/main/java/org/apache/calcite/config/CalciteSystemProperty.java b/core/src/main/java/org/apache/calcite/config/CalciteSystemProperty.java
index 1a6f860b0e6c..978379ab0d71 100644
--- a/core/src/main/java/org/apache/calcite/config/CalciteSystemProperty.java
+++ b/core/src/main/java/org/apache/calcite/config/CalciteSystemProperty.java
@@ -16,8 +16,6 @@
  */
 package org.apache.calcite.config;
 
-import org.apache.calcite.util.Util;
-
 import com.google.common.collect.ImmutableSet;
 
 import org.jspecify.annotations.Nullable;
@@ -32,6 +30,8 @@
 import java.util.function.IntPredicate;
 import java.util.stream.Stream;
 
+import static com.google.common.base.MoreObjects.firstNonNull;
+
 import static java.lang.Boolean.parseBoolean;
 import static java.lang.Integer.parseInt;
 import static java.util.Objects.requireNonNull;
@@ -566,7 +566,7 @@ private static CalciteSystemProperty stringProperty(
   private static Properties loadProperties() {
     Properties saffronProperties = new Properties();
     ClassLoader classLoader =
-        Util.firstNonNull(Thread.currentThread().getContextClassLoader(),
+        firstNonNull(Thread.currentThread().getContextClassLoader(),
             CalciteSystemProperty.class.getClassLoader());
     // Read properties from the file "saffron.properties", if it exists in classpath
     try (InputStream stream = requireNonNull(classLoader, "classLoader")

From 4ff71d4e480f43c263d357bac6c5a5e9312502f4 Mon Sep 17 00:00:00 2001
From: Vladimir Sitnikov 
Date: Mon, 24 Aug 2026 17:14:52 +0300
Subject: [PATCH 533/562] [CALCITE-7736] Say where a value is genuinely absent,
 and work around the bytecode wildcard

`VolcanoPlanner.normalizePlan` reads a group after `find()` has succeeded,
`MatchNode` names the row it reads at end of stream, `DataContexts.MapDataContext`
maps a name to a value that may be absent, and `SqlBasicCall.set` copies a list
whose elements may be null into an array that says so.

`SqlNodeList` and `SubstitutionVisitor` suppress `containsAll`, `removeAll` and
`retainAll`. An unbounded wildcard read from a bytecode signature behaves as
`? extends Object` and rejects a nullable type argument, while the same
`Collection` written in annotated source accepts one. Reported as
uber/NullAway#1732, see nullaway-bugs/bytecode-unbounded-wildcard-rejects-nullable.md.
`SubstitutionVisitor` puts the suppression on one `removeParents` helper rather
than on the constructor that calls it twice.

Co-Authored-By: Claude Opus 5 
---
 .../java/org/apache/calcite/DataContexts.java |  4 ++--
 .../apache/calcite/interpreter/MatchNode.java |  4 +++-
 .../calcite/plan/SubstitutionVisitor.java     | 22 +++++++++++++++++--
 .../calcite/plan/volcano/VolcanoPlanner.java  |  3 ++-
 .../org/apache/calcite/sql/SqlBasicCall.java  |  2 +-
 .../org/apache/calcite/sql/SqlNodeList.java   | 12 ++++++++++
 6 files changed, 40 insertions(+), 7 deletions(-)

diff --git a/core/src/main/java/org/apache/calcite/DataContexts.java b/core/src/main/java/org/apache/calcite/DataContexts.java
index 03d0316f711d..ada6afe8e177 100644
--- a/core/src/main/java/org/apache/calcite/DataContexts.java
+++ b/core/src/main/java/org/apache/calcite/DataContexts.java
@@ -83,9 +83,9 @@ private static class EmptyDataContext implements DataContext, Serializable {
    * value for a key, remove the key from the map; the effect will be the
    * same. */
   private static class MapDataContext extends EmptyDataContext {
-    private final ImmutableMap map;
+    private final ImmutableMap map;
 
-    MapDataContext(Map map) {
+    MapDataContext(Map map) {
       this.map = ImmutableMap.copyOf(map);
     }
 
diff --git a/core/src/main/java/org/apache/calcite/interpreter/MatchNode.java b/core/src/main/java/org/apache/calcite/interpreter/MatchNode.java
index f330a86038bb..b3cfb1f4ea18 100644
--- a/core/src/main/java/org/apache/calcite/interpreter/MatchNode.java
+++ b/core/src/main/java/org/apache/calcite/interpreter/MatchNode.java
@@ -18,6 +18,8 @@
 
 import org.apache.calcite.rel.core.Match;
 
+import org.jspecify.annotations.Nullable;
+
 /**
  * Interpreter node that implements a
  * {@link Match}.
@@ -28,7 +30,7 @@ public class MatchNode extends AbstractSingleNode {
   }
 
   @Override public void run() throws InterruptedException {
-    Row row;
+    @Nullable Row row;
     while ((row = source.receive()) != null) {
       sink.send(row);
     }
diff --git a/core/src/main/java/org/apache/calcite/plan/SubstitutionVisitor.java b/core/src/main/java/org/apache/calcite/plan/SubstitutionVisitor.java
index db746bc797a9..b82f73b12f5b 100644
--- a/core/src/main/java/org/apache/calcite/plan/SubstitutionVisitor.java
+++ b/core/src/main/java/org/apache/calcite/plan/SubstitutionVisitor.java
@@ -186,6 +186,22 @@ public class SubstitutionVisitor {
   protected final MutableRel[] slots = new MutableRel[2];
 
   /** Creates a SubstitutionVisitor with the default rule set. */
+  /** Removes from {@code nodes} every node that is a parent of another.
+   *
+   * 
      The unbounded wildcard of {@code removeAll(Collection)} is read from the
+   * bytecode signature, where it behaves as {@code ? extends Object} and rejects the
+   * nullable element type of {@code parents}, of which the root's is null.
+   * See NullAway#1732.
+   *
+   * @param nodes all nodes in the tree
+   * @param parents the parents, of which the root's is null
+   */
+  @SuppressWarnings("NullAway")
+  private static void removeParents(List nodes,
+      Set<@Nullable MutableRel> parents) {
+    nodes.removeAll(parents);
+  }
+
   public SubstitutionVisitor(RelNode target_, RelNode query_) {
     this(target_, query_, DEFAULT_RULES, RelFactories.LOGICAL_BUILDER);
   }
@@ -224,13 +240,15 @@ public SubstitutionVisitor(RelNode target_, RelNode query_,
     // Populate the list of leaves in the tree under "target".
     // Leaves are all nodes that are not parents.
     // For determinism, it is important that the list is in scan order.
-    allNodes.removeAll(parents);
+    // The root's parent is null; see removeParents.
+    removeParents(allNodes, parents);
     targetLeaves = ImmutableList.copyOf(allNodes);
 
     allNodes.clear();
     parents.clear();
     visitor.go(query);
-    allNodes.removeAll(parents);
+    // The root's parent is null; see removeParents.
+    removeParents(allNodes, parents);
     queryLeaves = ImmutableList.copyOf(allNodes);
   }
 
diff --git a/core/src/main/java/org/apache/calcite/plan/volcano/VolcanoPlanner.java b/core/src/main/java/org/apache/calcite/plan/volcano/VolcanoPlanner.java
index 6f2b73379df9..8c2c3c37c446 100644
--- a/core/src/main/java/org/apache/calcite/plan/volcano/VolcanoPlanner.java
+++ b/core/src/main/java/org/apache/calcite/plan/volcano/VolcanoPlanner.java
@@ -1513,7 +1513,8 @@ private RelSubset registerSubset(
       if (!matcher.find()) {
         return plan;
       }
-      final String token = matcher.group(); // e.g. "Subset#23."
+      // e.g. "Subset#23."; find() above has succeeded
+      final String token = requireNonNull(matcher.group(), "matcher.group()");
       plan = plan.replace(token, "Subset#{" + i++ + "}.");
     }
   }
diff --git a/core/src/main/java/org/apache/calcite/sql/SqlBasicCall.java b/core/src/main/java/org/apache/calcite/sql/SqlBasicCall.java
index 01af35095ee4..c98e69b2a76d 100755
--- a/core/src/main/java/org/apache/calcite/sql/SqlBasicCall.java
+++ b/core/src/main/java/org/apache/calcite/sql/SqlBasicCall.java
@@ -140,7 +140,7 @@ public void setOperator(SqlOperator operator) {
       return ImmutableNullableList.of(e);
     }
     //noinspection unchecked
-    @Nullable E[] objects = (E[]) list.toArray();
+    @Nullable E[] objects = (@Nullable E[]) list.toArray();
     objects[i] = e;
     return ImmutableNullableList.copyOf(objects);
   }
diff --git a/core/src/main/java/org/apache/calcite/sql/SqlNodeList.java b/core/src/main/java/org/apache/calcite/sql/SqlNodeList.java
index 7283f3bdc652..8820f16fd3fa 100644
--- a/core/src/main/java/org/apache/calcite/sql/SqlNodeList.java
+++ b/core/src/main/java/org/apache/calcite/sql/SqlNodeList.java
@@ -169,6 +169,10 @@ public static SqlNodeList of(SqlParserPos pos, List<@Nullable SqlNode> list) {
     return list.contains(o);
   }
 
+  // The unbounded wildcard of Collection is read from the bytecode signature, where it
+  // behaves as ? extends Object and rejects a nullable type argument.
+  // https://github.com/uber/NullAway/issues/1732
+  @SuppressWarnings("NullAway")
   @Override public boolean containsAll(Collection c) {
     return list.containsAll(c);
   }
@@ -221,10 +225,18 @@ public static SqlNodeList of(SqlParserPos pos, List<@Nullable SqlNode> list) {
     return castNonNull(list.remove(index));
   }
 
+  // The unbounded wildcard of Collection is read from the bytecode signature, where it
+  // behaves as ? extends Object and rejects a nullable type argument.
+  // https://github.com/uber/NullAway/issues/1732
+  @SuppressWarnings("NullAway")
   @Override public boolean removeAll(Collection c) {
     return list.removeAll(c);
   }
 
+  // The unbounded wildcard of Collection is read from the bytecode signature, where it
+  // behaves as ? extends Object and rejects a nullable type argument.
+  // https://github.com/uber/NullAway/issues/1732
+  @SuppressWarnings("NullAway")
   @Override public boolean retainAll(Collection c) {
     return list.retainAll(c);
   }

From a9ca9fbdb9c4336e624e923b13d29822760bc33e Mon Sep 17 00:00:00 2001
From: Vladimir Sitnikov 
Date: Mon, 24 Aug 2026 17:27:38 +0300
Subject: [PATCH 534/562] [CALCITE-7736] Let the profiled rows and composed
 lists hold absent values

A profiled row carries a value per column, and a column may have none, so
`Collector.add` and its three overrides take `List<@Nullable Comparable>` and
`CompositeCollector` keeps a `@Nullable Comparable[]`.

`FlatLists.of(T, T, T)`, the six statics of `CompositeList` and
`SqlBasicCall.set` take the nullable element bound of the lists they build.
`LatticeSuggester` keys a node by its parent, and a root has none;
`AggregateReduceFunctionsRule` names the extra columns it projects, of which the
new ones have no name yet.

Co-Authored-By: Claude Opus 5 
---
 .../calcite/materialize/LatticeSuggester.java      |  2 +-
 .../org/apache/calcite/profile/ProfilerImpl.java   | 14 +++++++-------
 .../rel/rules/AggregateReduceFunctionsRule.java    |  2 +-
 .../java/org/apache/calcite/runtime/FlatLists.java |  2 +-
 .../java/org/apache/calcite/sql/SqlBasicCall.java  |  2 +-
 .../org/apache/calcite/util/CompositeList.java     | 14 ++++++++------
 6 files changed, 19 insertions(+), 17 deletions(-)

diff --git a/core/src/main/java/org/apache/calcite/materialize/LatticeSuggester.java b/core/src/main/java/org/apache/calcite/materialize/LatticeSuggester.java
index 2d8fcd15af99..db07bbb85004 100644
--- a/core/src/main/java/org/apache/calcite/materialize/LatticeSuggester.java
+++ b/core/src/main/java/org/apache/calcite/materialize/LatticeSuggester.java
@@ -191,7 +191,7 @@ private void addFrame(Query q, Frame frame, List lattices) {
       case 1:
         final StepRef edge = edges.get(0);
         final MutableNode parent = nodes.get(edge.source());
-        final List key =
+        final List<@Nullable Object> key =
             FlatLists.of(parent, tableRef.table, edge.step.keys);
         final MutableNode existingNode = nodesByParent.get(key);
         if (existingNode == null) {
diff --git a/core/src/main/java/org/apache/calcite/profile/ProfilerImpl.java b/core/src/main/java/org/apache/calcite/profile/ProfilerImpl.java
index 81448756bcaf..6a2913708da4 100644
--- a/core/src/main/java/org/apache/calcite/profile/ProfilerImpl.java
+++ b/core/src/main/java/org/apache/calcite/profile/ProfilerImpl.java
@@ -539,7 +539,7 @@ abstract static class Collector {
       this.space = space;
     }
 
-    abstract void add(List row);
+    abstract void add(List<@Nullable Comparable> row);
     abstract void finish();
 
     /** Creates an initial collector of the appropriate kind. */
@@ -568,7 +568,7 @@ static class SingletonCollector extends Collector {
       this.sketchThreshold = sketchThreshold;
     }
 
-    @Override public void add(List row) {
+    @Override public void add(List<@Nullable Comparable> row) {
       final Comparable v = row.get(columnOrdinal);
       if (v == NullSentinel.INSTANCE) {
         nullCount++;
@@ -597,18 +597,18 @@ static class CompositeCollector extends Collector {
     protected static final ImmutableBitSet OF = ImmutableBitSet.of(2, 13);
     final Set values = new HashSet<>();
     final int[] columnOrdinals;
-    final Comparable[] columnValues;
+    final @Nullable Comparable[] columnValues;
     int nullCount = 0;
     private final int sketchThreshold;
 
     CompositeCollector(Space space, int[] columnOrdinals, int sketchThreshold) {
       super(space);
       this.columnOrdinals = columnOrdinals;
-      this.columnValues = new Comparable[columnOrdinals.length];
+      this.columnValues = new @Nullable Comparable[columnOrdinals.length];
       this.sketchThreshold = sketchThreshold;
     }
 
-    @Override public void add(List row) {
+    @Override public void add(List<@Nullable Comparable> row) {
       if (space.columnOrdinals.equals(OF)) {
         Util.discard(0);
       }
@@ -700,7 +700,7 @@ static class HllSingletonCollector extends HllCollector {
       this.columnOrdinal = columnOrdinal;
     }
 
-    @Override public void add(List row) {
+    @Override public void add(List<@Nullable Comparable> row) {
       final Comparable value = row.get(columnOrdinal);
       if (value == NullSentinel.INSTANCE) {
         nullCount++;
@@ -722,7 +722,7 @@ static class HllCompositeCollector extends HllCollector {
       this.columnOrdinals = columnOrdinals;
     }
 
-    @Override public void add(List row) {
+    @Override public void add(List<@Nullable Comparable> row) {
       if (space.columnOrdinals.equals(OF)) {
         Util.discard(0);
       }
diff --git a/core/src/main/java/org/apache/calcite/rel/rules/AggregateReduceFunctionsRule.java b/core/src/main/java/org/apache/calcite/rel/rules/AggregateReduceFunctionsRule.java
index c8f145acede6..efbcbf657965 100644
--- a/core/src/main/java/org/apache/calcite/rel/rules/AggregateReduceFunctionsRule.java
+++ b/core/src/main/java/org/apache/calcite/rel/rules/AggregateReduceFunctionsRule.java
@@ -260,7 +260,7 @@ private void reduceAggs(
         inputExprs.size() - relBuilder.peek().getRowType().getFieldCount();
     if (extraArgCount > 0) {
       relBuilder.project(inputExprs,
-          CompositeList.of(
+          CompositeList.<@Nullable String>of(
               relBuilder.peek().getRowType().getFieldNames(),
               Collections.<@Nullable String>nCopies(extraArgCount, null)));
     }
diff --git a/core/src/main/java/org/apache/calcite/runtime/FlatLists.java b/core/src/main/java/org/apache/calcite/runtime/FlatLists.java
index d26b90ca31b7..3887f649e9b1 100644
--- a/core/src/main/java/org/apache/calcite/runtime/FlatLists.java
+++ b/core/src/main/java/org/apache/calcite/runtime/FlatLists.java
@@ -75,7 +75,7 @@ public static  List of(T t0, T t1) {
   }
 
   /** Creates a flat list with 3 elements. */
-  public static  List of(T t0, T t1, T t2) {
+  public static  List of(T t0, T t1, T t2) {
     return new Flat3List<>(t0, t1, t2);
   }
 
diff --git a/core/src/main/java/org/apache/calcite/sql/SqlBasicCall.java b/core/src/main/java/org/apache/calcite/sql/SqlBasicCall.java
index c98e69b2a76d..8a7c6fc9ca61 100755
--- a/core/src/main/java/org/apache/calcite/sql/SqlBasicCall.java
+++ b/core/src/main/java/org/apache/calcite/sql/SqlBasicCall.java
@@ -134,7 +134,7 @@ public void setOperator(SqlOperator operator) {
 
   /** Sets the {@code i}th element of {@code list} to value {@code e}, creating
    * an immutable copy of the list. */
-  private static  List<@Nullable E> set(List list, int i, @Nullable E e) {
+  private static  List<@Nullable E> set(List<@Nullable E> list, int i, @Nullable E e) {
     if (i == 0 && list.size() == 1) {
       // short-cut case where the contents of the previous list can be ignored
       return ImmutableNullableList.of(e);
diff --git a/core/src/main/java/org/apache/calcite/util/CompositeList.java b/core/src/main/java/org/apache/calcite/util/CompositeList.java
index a79a90ff2b0e..96e498faec85 100644
--- a/core/src/main/java/org/apache/calcite/util/CompositeList.java
+++ b/core/src/main/java/org/apache/calcite/util/CompositeList.java
@@ -18,6 +18,8 @@
 
 import com.google.common.collect.ImmutableList;
 
+import org.jspecify.annotations.Nullable;
+
 import java.util.AbstractList;
 import java.util.List;
 
@@ -55,7 +57,7 @@ private CompositeList(ImmutableList> lists) {
    * @return List consisting of all lists
    */
   @SafeVarargs
-  public static  CompositeList of(List... lists) {
+  public static  CompositeList of(List... lists) {
     //noinspection unchecked
     return new CompositeList((ImmutableList) ImmutableList.copyOf(lists));
   }
@@ -67,7 +69,7 @@ public static  CompositeList of(List... lists) {
    * @param    Element type
    * @return List consisting of all lists
    */
-  public static  CompositeList ofCopy(Iterable> lists) {
+  public static  CompositeList ofCopy(Iterable> lists) {
     final ImmutableList> list = ImmutableList.copyOf(lists);
     return new CompositeList<>(list);
   }
@@ -78,7 +80,7 @@ public static  CompositeList ofCopy(Iterable> lists) {
    * @param    Element type
    * @return List consisting of all lists
    */
-  public static  List of() {
+  public static  List of() {
     return ImmutableList.of();
   }
 
@@ -89,7 +91,7 @@ public static  List of() {
    * @param    Element type
    * @return List consisting of all lists
    */
-  public static  List of(List list0) {
+  public static  List of(List list0) {
     return list0;
   }
 
@@ -101,7 +103,7 @@ public static  List of(List list0) {
    * @param    Element type
    * @return List consisting of all lists
    */
-  public static  CompositeList of(List list0,
+  public static  CompositeList of(List list0,
       List list1) {
     //noinspection unchecked
     return new CompositeList((ImmutableList) ImmutableList.of(list0, list1));
@@ -116,7 +118,7 @@ public static  CompositeList of(List list0,
    * @param    Element type
    * @return List consisting of all lists
    */
-  public static  CompositeList of(List list0,
+  public static  CompositeList of(List list0,
       List list1,
       List list2) {
     //noinspection unchecked

From ad9f5daaabec64f911c918e80ef6e215d6c5ee5e Mon Sep 17 00:00:00 2001
From: Vladimir Sitnikov 
Date: Mon, 24 Aug 2026 17:34:43 +0300
Subject: [PATCH 535/562] [CALCITE-7736] Read each value once, and drop a
 castNonNull that outlived its reason

`SqlToRelConverter` reads the project it has just cast rather than casting it at
each use, and asks for a `DmlNamespace` after `isWrapperFor` has established
there is one. `SqlValidatorImpl` collects aliases into a list that admits the
null a child of the FROM clause may have, and looks up a column by an index it
took from the map it is reading.

`JavaRowFormat.copy` returns a list of statements and never null, so the
`castNonNull` around it in `EnumUtils` goes away.

`FilterProjectTransposeRule` answers `replaceIfs` with null when the input has
no distribution, rather than a singleton list holding null. `replaceIfs` takes a
supplier that may answer null, and does the same thing with it.

`CalciteCatalogReader` falls back on a family constant, which is what
`firstNonNull` is for.

Co-Authored-By: Claude Opus 5 
---
 .../apache/calcite/adapter/enumerable/EnumUtils.java   |  6 ++----
 .../org/apache/calcite/plan/VisitorDataContext.java    |  2 +-
 .../apache/calcite/prepare/CalciteCatalogReader.java   |  2 +-
 .../calcite/rel/rules/FilterProjectTransposeRule.java  | 10 ++++++++--
 .../apache/calcite/sql/validate/SqlValidatorImpl.java  |  4 ++--
 .../org/apache/calcite/sql2rel/SqlToRelConverter.java  |  9 ++++++---
 6 files changed, 20 insertions(+), 13 deletions(-)

diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java
index c09ce0d1751e..0e94d780dd1c 100644
--- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java
+++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java
@@ -23,7 +23,6 @@
 import org.apache.calcite.linq4j.Enumerable;
 import org.apache.calcite.linq4j.Enumerator;
 import org.apache.calcite.linq4j.JoinType;
-import org.apache.calcite.linq4j.Nullness;
 import org.apache.calcite.linq4j.Ord;
 import org.apache.calcite.linq4j.function.Function1;
 import org.apache.calcite.linq4j.function.Function2;
@@ -293,9 +292,8 @@ static Expression joinSelectorCompact(JoinRelType joinType, PhysType physType,
       final int fieldCount = inputPhysType.getRowType().getFieldCount();
       // Delegate copying the row values to JavaRowFormat
       final List copyStatements =
-          Nullness.castNonNull(
-              inputPhysType.getFormat().copy(parameter, compactOutputVar,
-                  outputField, fieldCount));
+          inputPhysType.getFormat().copy(parameter, compactOutputVar,
+              outputField, fieldCount);
       if (joinType.generatesNullsOn(ord.i)) {
         // [CALCITE-6593] NPE when outer joining tables with many fields and unmatching rows
         compactCode.add(
diff --git a/core/src/main/java/org/apache/calcite/plan/VisitorDataContext.java b/core/src/main/java/org/apache/calcite/plan/VisitorDataContext.java
index a563218a2027..cb04e1519e4c 100644
--- a/core/src/main/java/org/apache/calcite/plan/VisitorDataContext.java
+++ b/core/src/main/java/org/apache/calcite/plan/VisitorDataContext.java
@@ -239,7 +239,7 @@ public VisitorDataContext(@Nullable Object[] values) {
         if (value instanceof NlsString) {
           return Pair.of(index, ((NlsString) value).getValue());
         } else {
-          return Pair.of(index, value);
+          return Pair.of(index, value);
         }
       }
     }
diff --git a/core/src/main/java/org/apache/calcite/prepare/CalciteCatalogReader.java b/core/src/main/java/org/apache/calcite/prepare/CalciteCatalogReader.java
index 6932692982b7..238a6acf579a 100644
--- a/core/src/main/java/org/apache/calcite/prepare/CalciteCatalogReader.java
+++ b/core/src/main/java/org/apache/calcite/prepare/CalciteCatalogReader.java
@@ -364,7 +364,7 @@ private static SqlOperator toOp(SqlIdentifier name,
         typeFactory -> argTypesFactory.apply(typeFactory)
             .stream()
             .map(type ->
-                Util.first(type.getSqlTypeName().getFamily(),
+                Util.firstNonNull(type.getSqlTypeName().getFamily(),
                     SqlTypeFamily.ANY))
             .collect(toImmutableList());
     final Function> paramTypesFactory =
diff --git a/core/src/main/java/org/apache/calcite/rel/rules/FilterProjectTransposeRule.java b/core/src/main/java/org/apache/calcite/rel/rules/FilterProjectTransposeRule.java
index 3c365222cd66..2efca881295d 100644
--- a/core/src/main/java/org/apache/calcite/rel/rules/FilterProjectTransposeRule.java
+++ b/core/src/main/java/org/apache/calcite/rel/rules/FilterProjectTransposeRule.java
@@ -22,6 +22,7 @@
 import org.apache.calcite.plan.RelRule;
 import org.apache.calcite.plan.RelTraitSet;
 import org.apache.calcite.rel.RelCollationTraitDef;
+import org.apache.calcite.rel.RelDistribution;
 import org.apache.calcite.rel.RelDistributionTraitDef;
 import org.apache.calcite.rel.RelNode;
 import org.apache.calcite.rel.core.Filter;
@@ -34,6 +35,7 @@
 import org.apache.calcite.tools.RelBuilderFactory;
 
 import org.immutables.value.Value;
+import org.jspecify.annotations.Nullable;
 
 import java.util.Collections;
 import java.util.List;
@@ -186,8 +188,12 @@ protected FilterProjectTransposeRule(
           .replaceIfs(RelCollationTraitDef.INSTANCE,
               () -> input.getTraitSet().getTraits(RelCollationTraitDef.INSTANCE))
           .replaceIfs(RelDistributionTraitDef.INSTANCE,
-              () -> Collections.singletonList(
-                  input.getTraitSet().getTrait(RelDistributionTraitDef.INSTANCE)));
+              () -> {
+                final @Nullable RelDistribution distribution =
+                    input.getTraitSet().getTrait(RelDistributionTraitDef.INSTANCE);
+                return distribution == null ? null
+                    : Collections.singletonList(distribution);
+              });
       newCondition = RexUtil.removeNullabilityCast(relBuilder.getTypeFactory(), newCondition);
       newFilterRel = filter.copy(traitSet, input, newCondition);
     } else {
diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java
index 737cc7ad9fc7..2eead5ea4a3c 100644
--- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java
+++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java
@@ -4514,7 +4514,7 @@ protected void validateSelect(
       //noinspection RedundantTypeArguments
       names = names.stream()
           .<@Nullable String>map(s -> s == null ? null : s.toUpperCase(Locale.ROOT))
-          .collect(Collectors.toList());
+          .collect(Collectors.<@Nullable String>toList());
     }
     final int duplicateAliasOrdinal = Util.firstDuplicate(names);
     if (duplicateAliasOrdinal >= 0) {
@@ -6371,7 +6371,7 @@ private void checkConstraint(
                   RESOURCE.viewConstraintNotSatisfied(colName,
                       Util.last(validatorTable.getQualifiedName())));
           RelOptUtil.validateValueAgainstConstraint(sourceValue,
-              projectMap.get(colIndex), validationError);
+              requireNonNull(projectMap.get(colIndex), "colIndex"), validationError);
         }
       }
     }
diff --git a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
index dc03fb280301..ae2987d882cc 100644
--- a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
+++ b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
@@ -1019,9 +1019,10 @@ private void distinctify(
       bottomNames.add(rel.getRowType().getFieldNames().get(i));
     }
 
+    final Project bottomProject = castNonNull(project);
     bb.setRoot(
-        LogicalProject.create(castNonNull(project).getInput(), project.getHints(),
-            bottomExprs, bottomNames, project.getVariablesSet()), false);
+        LogicalProject.create(bottomProject.getInput(), bottomProject.getHints(),
+            bottomExprs, bottomNames, bottomProject.getVariablesSet()), false);
 
     final ImmutableBitSet aggGroupSet = ImmutableBitSet.range(groupSet.cardinality());
     bb.setRoot(
@@ -4555,7 +4556,9 @@ protected RelOptTable getTargetTable(SqlNode call) {
     final SqlValidatorNamespace targetNs = getNamespace(call);
     SqlValidatorNamespace namespace;
     if (targetNs.isWrapperFor(SqlValidatorImpl.DmlNamespace.class)) {
-      namespace = targetNs.unwrap(SqlValidatorImpl.DmlNamespace.class);
+      namespace =
+          requireNonNull(targetNs.unwrap(SqlValidatorImpl.DmlNamespace.class),
+              "DmlNamespace");
     } else {
       namespace = targetNs.resolve();
     }

From aac1609e1a0b46cfa923ac9e105ea32e96ffc992 Mon Sep 17 00:00:00 2001
From: Vladimir Sitnikov 
Date: Mon, 24 Aug 2026 18:10:41 +0300
Subject: [PATCH 536/562] [CALCITE-7736] Close out the last of the core
 findings

`ProfilerImpl` separates the two kinds of row it works with: a scanned row uses
`NullSentinel` for a SQL null and so holds no Java null, while the sketch path
builds a sparse row filled only at the ordinals of the space it feeds.
`Collector.add` takes `List` so it accepts
both, and each collector casts the ordinals its own space owns.

Type parameters that carry the nullable bound of what they build:
`RexWindowBound.accept` and its override, `Functions.ignore2`, `Util.combine`,
`SqlNodeList.toArray`, `HepPlanner.onCopyHook`, `EnumerableTableModify.keyOf`
and the maps keyed by it, and `ArrayTable.asList`.

`SqlNode.toList` and `RelBuilder` replace a method reference and a Guava call
whose wildcard comes from bytecode with a lambda and a direct iterator check.
`TableFunctionScanNode` drops its raw `Enumerable` for a typed one.

`ArrayTable.permute` is suppressed: an array creation keeps a non-null component
type whatever it is assigned to, so writing a nullable element reports even
though both arrays are declared `@Nullable Comparable[]`.

Co-Authored-By: Claude Opus 5 
---
 .../java/org/apache/calcite/DataContexts.java |  8 +++++--
 .../calcite/adapter/clone/ArrayTable.java     | 10 ++++++---
 .../enumerable/EnumerableTableModify.java     |  8 +++----
 .../calcite/config/CalciteSystemProperty.java |  3 +++
 .../interpreter/AbstractSingleNode.java       |  1 +
 .../apache/calcite/interpreter/MatchNode.java |  1 +
 .../interpreter/TableFunctionScanNode.java    | 16 +++++++++-----
 .../apache/calcite/plan/hep/HepPlanner.java   |  8 ++++---
 .../apache/calcite/profile/ProfilerImpl.java  | 22 +++++++++----------
 .../apache/calcite/rex/RexWindowBound.java    |  2 +-
 .../apache/calcite/rex/RexWindowBounds.java   |  2 +-
 .../java/org/apache/calcite/sql/SqlNode.java  |  3 ++-
 .../org/apache/calcite/sql/SqlNodeList.java   |  2 +-
 .../sql/fun/SqlSpatialTypeFunctions.java      |  5 +++--
 .../calcite/sql2rel/SqlToRelConverter.java    |  2 +-
 .../org/apache/calcite/tools/RelBuilder.java  |  2 +-
 .../java/org/apache/calcite/util/Util.java    |  4 ++--
 .../calcite/linq4j/function/Functions.java    |  3 ++-
 18 files changed, 62 insertions(+), 40 deletions(-)

diff --git a/core/src/main/java/org/apache/calcite/DataContexts.java b/core/src/main/java/org/apache/calcite/DataContexts.java
index ada6afe8e177..a7f1e0f6337c 100644
--- a/core/src/main/java/org/apache/calcite/DataContexts.java
+++ b/core/src/main/java/org/apache/calcite/DataContexts.java
@@ -83,9 +83,13 @@ private static class EmptyDataContext implements DataContext, Serializable {
    * value for a key, remove the key from the map; the effect will be the
    * same. */
   private static class MapDataContext extends EmptyDataContext {
-    private final ImmutableMap map;
+    private final ImmutableMap map;
 
-    MapDataContext(Map map) {
+    // The unbounded wildcard of ImmutableMap.copyOf is read from the bytecode signature,
+    // where it rejects the capture of this one.
+    // https://github.com/uber/NullAway/issues/1732
+    @SuppressWarnings("NullAway")
+    MapDataContext(Map map) {
       this.map = ImmutableMap.copyOf(map);
     }
 
diff --git a/core/src/main/java/org/apache/calcite/adapter/clone/ArrayTable.java b/core/src/main/java/org/apache/calcite/adapter/clone/ArrayTable.java
index 08872f02747f..712f3231edb0 100644
--- a/core/src/main/java/org/apache/calcite/adapter/clone/ArrayTable.java
+++ b/core/src/main/java/org/apache/calcite/adapter/clone/ArrayTable.java
@@ -225,7 +225,7 @@ public static List asList(final Representation representation,
         final Object dataSet) {
       // Cache size. It might be expensive to compute.
       final int size = representation.size(dataSet);
-      return new AbstractList() {
+      return new AbstractList<@Nullable Object>() {
         @Override public @Nullable Object get(int index) {
           return representation.getObject(dataSet, index);
         }
@@ -284,10 +284,14 @@ public static class ObjectArray implements Representation {
       return list.toArray(new @Nullable Comparable[0]);
     }
 
+    // Both arrays hold the column's values, nulls included, but an array creation keeps a
+    // non-null component type whatever it is assigned to.
+    @SuppressWarnings("NullAway")
     @Override public Object permute(Object dataSet, int[] sources) {
-      @Nullable Comparable[] list = (@Nullable Comparable[]) dataSet;
+      final @Nullable Comparable[] list = (@Nullable Comparable[]) dataSet;
       final int size = list.length;
-      final @Nullable Comparable[] comparables = new @Nullable Comparable[size];
+      final @Nullable Comparable[] comparables =
+          (@Nullable Comparable[]) new Comparable[size];
       for (int i = 0; i < size; i++) {
         comparables[i] = list[sources[i]];
       }
diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableTableModify.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableTableModify.java
index 07eca16cb917..44756f77f150 100644
--- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableTableModify.java
+++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableTableModify.java
@@ -484,17 +484,17 @@ public static void applyDeleteRowsByKey(Enumerable sourceKeys,
       Collection sinkRows, Function1 sinkKeySelector) {
 
     // Build a map of source keys to the number of sink rows that must be removed for each.
-    final Map, Integer> pendingByKey = new HashMap<>();
+    final Map, Integer> pendingByKey = new HashMap<>();
     try (Enumerator e = sourceKeys.enumerator()) {
       while (e.moveNext()) {
-        final List key = keyOf(e.current());
+        final List<@Nullable Object> key = keyOf(e.current());
         pendingByKey.put(key, pendingByKey.getOrDefault(key, 0) + 1);
       }
     }
 
     // Iterate over sink rows and remove matching rows based on key.
     for (java.util.Iterator it = sinkRows.iterator(); it.hasNext();) {
-      final List key = keyOf(sinkKeySelector.apply(it.next()));
+      final List<@Nullable Object> key = keyOf(sinkKeySelector.apply(it.next()));
       final Integer pending = pendingByKey.get(key);
       if (pending == null || pending == 0) {
         continue;
@@ -516,7 +516,7 @@ public static void applyDeleteRowsByKey(Enumerable sourceKeys,
    * @param rowValues row values
    * @return key for row
    */
-  private static List keyOf(Object[] rowValues) {
+  private static List<@Nullable Object> keyOf(Object[] rowValues) {
     return Arrays.<@Nullable Object>asList(
         Arrays.copyOf(rowValues, rowValues.length));
   }
diff --git a/core/src/main/java/org/apache/calcite/config/CalciteSystemProperty.java b/core/src/main/java/org/apache/calcite/config/CalciteSystemProperty.java
index 978379ab0d71..1c24a11d9045 100644
--- a/core/src/main/java/org/apache/calcite/config/CalciteSystemProperty.java
+++ b/core/src/main/java/org/apache/calcite/config/CalciteSystemProperty.java
@@ -601,6 +601,9 @@ private static Properties loadProperties() {
 
   private final T value;
 
+  // apply() returns `? extends T`, which reads as @Nullable when T has a nullable bound.
+  // https://github.com/uber/NullAway/issues/1727
+  @SuppressWarnings("NullAway")
   private CalciteSystemProperty(String key,
       Function valueParser) {
     this.value = valueParser.apply(PROPERTIES.getProperty(key));
diff --git a/core/src/main/java/org/apache/calcite/interpreter/AbstractSingleNode.java b/core/src/main/java/org/apache/calcite/interpreter/AbstractSingleNode.java
index 9d328bb787fb..4a22c16b73bc 100644
--- a/core/src/main/java/org/apache/calcite/interpreter/AbstractSingleNode.java
+++ b/core/src/main/java/org/apache/calcite/interpreter/AbstractSingleNode.java
@@ -23,6 +23,7 @@
  *
  * @param  Type of relational expression
  */
+
 abstract class AbstractSingleNode implements Node {
   protected final Source source;
   protected final Sink sink;
diff --git a/core/src/main/java/org/apache/calcite/interpreter/MatchNode.java b/core/src/main/java/org/apache/calcite/interpreter/MatchNode.java
index b3cfb1f4ea18..5d94880c50bb 100644
--- a/core/src/main/java/org/apache/calcite/interpreter/MatchNode.java
+++ b/core/src/main/java/org/apache/calcite/interpreter/MatchNode.java
@@ -24,6 +24,7 @@
  * Interpreter node that implements a
  * {@link Match}.
  */
+
 public class MatchNode extends AbstractSingleNode {
   MatchNode(Compiler compiler, Match rel) {
     super(compiler, rel);
diff --git a/core/src/main/java/org/apache/calcite/interpreter/TableFunctionScanNode.java b/core/src/main/java/org/apache/calcite/interpreter/TableFunctionScanNode.java
index 255144a4c866..295fb39bd83f 100644
--- a/core/src/main/java/org/apache/calcite/interpreter/TableFunctionScanNode.java
+++ b/core/src/main/java/org/apache/calcite/interpreter/TableFunctionScanNode.java
@@ -32,15 +32,18 @@
 
 import org.jspecify.annotations.Nullable;
 
+import static org.apache.calcite.linq4j.Nullness.castNonNull;
+
 /**
  * Interpreter node that implements a
  * {@link TableFunctionScan}.
  */
+
 public class TableFunctionScanNode implements Node {
   private final Scalar scalar;
   private final Context context;
   private final Sink sink;
-  private final Function1 mapFn;
+  private final Function1<@Nullable Object, Row> mapFn;
 
   private TableFunctionScanNode(Compiler compiler, TableFunctionScan rel) {
     final RelDataType rowType = rel.getRowType();
@@ -49,18 +52,19 @@ private TableFunctionScanNode(Compiler compiler, TableFunctionScan rel) {
     this.sink = compiler.sink(rel);
     if (rowType.getFieldCount() == 1
         && rel.getElementType() != Object[].class) {
-      this.mapFn = (Function1) Row::of;
+      this.mapFn = Row::of;
     } else {
-      this.mapFn = (Function1<@Nullable Object[], Row>) Row::asCopy;
+      this.mapFn = o -> Row.asCopy(castNonNull((@Nullable Object[]) o));
     }
   }
 
   @Override public void run() throws InterruptedException {
     final Object o = scalar.execute(context);
     if (o instanceof Enumerable) {
-      for (@SuppressWarnings({"unchecked", "rawtypes"})
-           final Enumerator enumerator =
-           ((Enumerable) o).select(mapFn).enumerator();
+      @SuppressWarnings("unchecked") final Enumerable<@Nullable Object> enumerable =
+          (Enumerable<@Nullable Object>) o;
+      for (final Enumerator enumerator =
+               enumerable.select(mapFn).enumerator();
            enumerator.moveNext();) {
         sink.send(enumerator.current());
       }
diff --git a/core/src/main/java/org/apache/calcite/plan/hep/HepPlanner.java b/core/src/main/java/org/apache/calcite/plan/hep/HepPlanner.java
index 35ffdaf22598..030fb508b4f5 100644
--- a/core/src/main/java/org/apache/calcite/plan/hep/HepPlanner.java
+++ b/core/src/main/java/org/apache/calcite/plan/hep/HepPlanner.java
@@ -116,7 +116,7 @@ public class HepPlanner extends AbstractRelOptPlanner {
   private final DirectedGraph graph =
       DefaultDirectedGraph.create();
 
-  private final Function2 onCopyHook;
+  private final Function2 onCopyHook;
 
   private final List materializations =
       new ArrayList<>();
@@ -185,11 +185,13 @@ public HepPlanner(
       HepProgram program,
       @Nullable Context context,
       boolean noDag,
-      @Nullable Function2 onCopyHook,
+      @Nullable Function2 onCopyHook,
       RelOptCostFactory costFactory) {
     super(costFactory, context);
     this.mainProgram = requireNonNull(program, "program");
-    this.onCopyHook = Util.firstNonNull(onCopyHook, Functions.ignore2());
+    this.onCopyHook =
+        Util.firstNonNull(onCopyHook,
+            Functions.ignore2());
     this.noDag = noDag;
     this.largePlanMode = CalciteSystemProperty.HEP_PLANNER_LARGE_PLAN_MODE.value();
   }
diff --git a/core/src/main/java/org/apache/calcite/profile/ProfilerImpl.java b/core/src/main/java/org/apache/calcite/profile/ProfilerImpl.java
index 6a2913708da4..dd137cbd09a4 100644
--- a/core/src/main/java/org/apache/calcite/profile/ProfilerImpl.java
+++ b/core/src/main/java/org/apache/calcite/profile/ProfilerImpl.java
@@ -539,7 +539,7 @@ abstract static class Collector {
       this.space = space;
     }
 
-    abstract void add(List<@Nullable Comparable> row);
+    abstract void add(List row);
     abstract void finish();
 
     /** Creates an initial collector of the appropriate kind. */
@@ -568,8 +568,8 @@ static class SingletonCollector extends Collector {
       this.sketchThreshold = sketchThreshold;
     }
 
-    @Override public void add(List<@Nullable Comparable> row) {
-      final Comparable v = row.get(columnOrdinal);
+    @Override public void add(List row) {
+      final Comparable v = castNonNull(row.get(columnOrdinal));
       if (v == NullSentinel.INSTANCE) {
         nullCount++;
       } else {
@@ -597,24 +597,24 @@ static class CompositeCollector extends Collector {
     protected static final ImmutableBitSet OF = ImmutableBitSet.of(2, 13);
     final Set values = new HashSet<>();
     final int[] columnOrdinals;
-    final @Nullable Comparable[] columnValues;
+    final Comparable[] columnValues;
     int nullCount = 0;
     private final int sketchThreshold;
 
     CompositeCollector(Space space, int[] columnOrdinals, int sketchThreshold) {
       super(space);
       this.columnOrdinals = columnOrdinals;
-      this.columnValues = new @Nullable Comparable[columnOrdinals.length];
+      this.columnValues = new Comparable[columnOrdinals.length];
       this.sketchThreshold = sketchThreshold;
     }
 
-    @Override public void add(List<@Nullable Comparable> row) {
+    @Override public void add(List row) {
       if (space.columnOrdinals.equals(OF)) {
         Util.discard(0);
       }
       int nullCountThisRow = 0;
       for (int i = 0, length = columnOrdinals.length; i < length; i++) {
-        final Comparable value = row.get(columnOrdinals[i]);
+        final Comparable value = castNonNull(row.get(columnOrdinals[i]));
         if (value == NullSentinel.INSTANCE) {
           if (nullCountThisRow++ == 0) {
             nullCount++;
@@ -700,8 +700,8 @@ static class HllSingletonCollector extends HllCollector {
       this.columnOrdinal = columnOrdinal;
     }
 
-    @Override public void add(List<@Nullable Comparable> row) {
-      final Comparable value = row.get(columnOrdinal);
+    @Override public void add(List row) {
+      final Comparable value = castNonNull(row.get(columnOrdinal));
       if (value == NullSentinel.INSTANCE) {
         nullCount++;
         sketch.update(NULL_BITS);
@@ -722,14 +722,14 @@ static class HllCompositeCollector extends HllCollector {
       this.columnOrdinals = columnOrdinals;
     }
 
-    @Override public void add(List<@Nullable Comparable> row) {
+    @Override public void add(List row) {
       if (space.columnOrdinals.equals(OF)) {
         Util.discard(0);
       }
       int nullCountThisRow = 0;
       buf.clear();
       for (int columnOrdinal : columnOrdinals) {
-        final Comparable value = row.get(columnOrdinal);
+        final Comparable value = castNonNull(row.get(columnOrdinal));
         if (value == NullSentinel.INSTANCE) {
           if (nullCountThisRow++ == 0) {
             nullCount++;
diff --git a/core/src/main/java/org/apache/calcite/rex/RexWindowBound.java b/core/src/main/java/org/apache/calcite/rex/RexWindowBound.java
index 9489dc5ecde3..3b922c87b629 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexWindowBound.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexWindowBound.java
@@ -108,7 +108,7 @@ public int getOrderKey() {
    * @param  return type of the visitor
    * @return transformed bound
    */
-  public  RexWindowBound accept(RexVisitor visitor) {
+  public  RexWindowBound accept(RexVisitor visitor) {
     return this;
   }
 
diff --git a/core/src/main/java/org/apache/calcite/rex/RexWindowBounds.java b/core/src/main/java/org/apache/calcite/rex/RexWindowBounds.java
index 7ae667b73666..b2e1155ccff0 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexWindowBounds.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexWindowBounds.java
@@ -178,7 +178,7 @@ private RexBoundedWindowBound(SqlKind sqlKind, RexNode offset) {
       return super.nodeCount() + offset.nodeCount();
     }
 
-    @Override public  RexWindowBound accept(RexVisitor visitor) {
+    @Override public  RexWindowBound accept(RexVisitor visitor) {
       R r = offset.accept(visitor);
       if (r instanceof RexNode && r != offset) {
         return new RexBoundedWindowBound(sqlKind, (RexNode) r);
diff --git a/core/src/main/java/org/apache/calcite/sql/SqlNode.java b/core/src/main/java/org/apache/calcite/sql/SqlNode.java
index 270d842825c0..29b3ed6f2c7b 100644
--- a/core/src/main/java/org/apache/calcite/sql/SqlNode.java
+++ b/core/src/main/java/org/apache/calcite/sql/SqlNode.java
@@ -384,7 +384,8 @@ public static boolean equalDeep(List operands0,
       ArrayList<@Nullable SqlNode>, SqlNodeList> toList(SqlParserPos pos) {
     //noinspection RedundantTypeArguments
     return Collector., SqlNodeList>of(
-        ArrayList::new, (list, e) -> list.add(e), Util::combine,
+        () -> new ArrayList<@Nullable SqlNode>(), (list, e) -> list.add(e),
+        (list0, list1) -> Util.combine(list0, list1),
         (ArrayList<@Nullable SqlNode> list) -> SqlNodeList.of(pos, list));
   }
 }
diff --git a/core/src/main/java/org/apache/calcite/sql/SqlNodeList.java b/core/src/main/java/org/apache/calcite/sql/SqlNodeList.java
index 8820f16fd3fa..bacd3b08ff55 100644
--- a/core/src/main/java/org/apache/calcite/sql/SqlNodeList.java
+++ b/core/src/main/java/org/apache/calcite/sql/SqlNodeList.java
@@ -193,7 +193,7 @@ public static SqlNodeList of(SqlParserPos pos, List<@Nullable SqlNode> list) {
   }
 
   @SuppressWarnings("NullAway")
-  @Override public  @Nullable T[] toArray(T @Nullable [] a) {
+  @Override public  T[] toArray(T[] a) {
     return list.toArray(a);
   }
 
diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlSpatialTypeFunctions.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlSpatialTypeFunctions.java
index 61c994caff98..8008725c7759 100644
--- a/core/src/main/java/org/apache/calcite/sql/fun/SqlSpatialTypeFunctions.java
+++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlSpatialTypeFunctions.java
@@ -156,8 +156,9 @@ public static class GridTable implements ScannableTable {
       if (geom != null && deltaX != null && deltaY != null) {
         if (deltaX.compareTo(BigDecimal.ZERO) > 0
             && deltaY.compareTo(BigDecimal.ZERO) > 0) {
-          return new SpatialTypeFunctions.GridEnumerable(geom.getEnvelopeInternal(), deltaX, deltaY,
-              point);
+          return new SpatialTypeFunctions.GridEnumerable(geom.getEnvelopeInternal(),
+              deltaX, deltaY, point)
+              .select(row -> row);
         }
       }
       return Linq4j.emptyEnumerable();
diff --git a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
index ae2987d882cc..bbd850dbf228 100644
--- a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
+++ b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
@@ -1045,7 +1045,7 @@ private void distinctify(
       if (idx >= 0) {
         topExprs.add(rexBuilder.makeInputRef(aggregate, idx));
       } else {
-        topExprs.add(project.getProjects().get(i).accept(shuttle));
+        topExprs.add(bottomProject.getProjects().get(i).accept(shuttle));
       }
     }
     bb.setRoot(
diff --git a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java
index c73b66d12f3e..517d8684f808 100644
--- a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java
+++ b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java
@@ -2123,7 +2123,7 @@ private RelBuilder project_(
 
     // Perform a quick check for identity. We'll do a deeper check
     // later when we've derived column names.
-    if (!force && Iterables.isEmpty(fieldNames)
+    if (!force && !fieldNames.iterator().hasNext()
         && RexUtil.isIdentity(nodeList, inputRowType)) {
       return this;
     }
diff --git a/core/src/main/java/org/apache/calcite/util/Util.java b/core/src/main/java/org/apache/calcite/util/Util.java
index 4c1344d26a9d..7484f8d1b709 100644
--- a/core/src/main/java/org/apache/calcite/util/Util.java
+++ b/core/src/main/java/org/apache/calcite/util/Util.java
@@ -2741,14 +2741,14 @@ public static Calendar calendar(long millis, TimeZone timeZone) {
   }
 
   /** Combines a second immutable list builder into a first. */
-  public static  ImmutableList.Builder combine(
+  public static  ImmutableList.Builder combine(
       ImmutableList.Builder b0, ImmutableList.Builder b1) {
     b0.addAll(b1.build());
     return b0;
   }
 
   /** Combines a second array list into a first. */
-  public static  ArrayList combine(ArrayList list0,
+  public static  ArrayList combine(ArrayList list0,
       ArrayList list1) {
     list0.addAll(list1);
     return list0;
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/function/Functions.java b/linq4j/src/main/java/org/apache/calcite/linq4j/function/Functions.java
index f48f468e16f6..c1088f8b170e 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/function/Functions.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/function/Functions.java
@@ -413,7 +413,8 @@ public static  Function1 ignore1() {
    * @param  Type of parameter 1
    * @return Function that does nothing.
    */
-  public static  Function2 ignore2() {
+  public static  Function2 ignore2() {
     //noinspection unchecked
     return Ignore.INSTANCE;
   }

From e20462d3f48ee3397462e4be938c418a2aec495b Mon Sep 17 00:00:00 2001
From: Vladimir Sitnikov 
Date: Tue, 25 Aug 2026 17:21:40 +0300
Subject: [PATCH 537/562] [CALCITE-7736] Give the Void visitors the return type
 their visitor interface declares

A visitor over SqlNode declares itself SqlBasicVisitor<@Nullable Void> or
SqlVisitor<@Nullable Void>, so its visit methods return @Nullable Void. The
overrides narrowed that to Void, which for a type whose only value is null
promises nothing, and NullAway could not infer R for SqlNode.accept: the
argument constrained it to be both @NonNull and @Nullable.

See https://github.com/uber/NullAway/issues/1733

Co-Authored-By: Claude Opus 5 
---
 .../calcite/rel/rel2sql/SqlImplementor.java   |  2 +-
 .../java/org/apache/calcite/sql/SqlPivot.java |  2 +-
 .../org/apache/calcite/sql/SqlUnpivot.java    |  2 +-
 .../java/org/apache/calcite/sql/SqlUtil.java  | 16 ++++++------
 .../calcite/sql/fun/SqlBetweenOperator.java   |  2 +-
 .../apache/calcite/sql/type/OperandTypes.java |  4 +--
 .../calcite/sql/validate/AggChecker.java      |  4 +--
 .../calcite/sql/validate/AggVisitor.java      |  2 +-
 .../sql/validate/SqlValidatorImpl.java        | 26 +++++++++----------
 .../apache/calcite/sql2rel/AggConverter.java  | 14 +++++-----
 .../calcite/sql2rel/SqlToRelConverter.java    |  2 +-
 .../java/org/apache/calcite/util/Util.java    |  2 +-
 12 files changed, 39 insertions(+), 39 deletions(-)

diff --git a/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java b/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java
index 6d69014b01cb..3d1b0a6c888d 100644
--- a/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java
+++ b/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java
@@ -2479,7 +2479,7 @@ private boolean containsOver(
       }
       final boolean[] result = {false};
       node.accept(new SqlBasicVisitor<@Nullable Void>() {
-        @Override public Void visit(SqlCall call) {
+        @Override public @Nullable Void visit(SqlCall call) {
           if (result[0]) {
             return null;
           }
diff --git a/core/src/main/java/org/apache/calcite/sql/SqlPivot.java b/core/src/main/java/org/apache/calcite/sql/SqlPivot.java
index 89ecee5bdf3c..d22de8549de3 100644
--- a/core/src/main/java/org/apache/calcite/sql/SqlPivot.java
+++ b/core/src/main/java/org/apache/calcite/sql/SqlPivot.java
@@ -176,7 +176,7 @@ static SqlNodeList toNodes(SqlNode node) {
   public Set usedColumnNames() {
     final Set columnNames = new HashSet<>();
     final SqlVisitor<@Nullable Void> nameCollector = new SqlBasicVisitor<@Nullable Void>() {
-      @Override public Void visit(SqlIdentifier id) {
+      @Override public @Nullable Void visit(SqlIdentifier id) {
         columnNames.add(Util.last(id.names));
         return super.visit(id);
       }
diff --git a/core/src/main/java/org/apache/calcite/sql/SqlUnpivot.java b/core/src/main/java/org/apache/calcite/sql/SqlUnpivot.java
index 89aef8c420e2..bd46e038e62a 100644
--- a/core/src/main/java/org/apache/calcite/sql/SqlUnpivot.java
+++ b/core/src/main/java/org/apache/calcite/sql/SqlUnpivot.java
@@ -145,7 +145,7 @@ public void forEachNameValues(
   public Set usedColumnNames() {
     final Set columnNames = new HashSet<>();
     final SqlVisitor<@Nullable Void> nameCollector = new SqlBasicVisitor<@Nullable Void>() {
-      @Override public Void visit(SqlIdentifier id) {
+      @Override public @Nullable Void visit(SqlIdentifier id) {
         columnNames.add(Util.last(id.names));
         return super.visit(id);
       }
diff --git a/core/src/main/java/org/apache/calcite/sql/SqlUtil.java b/core/src/main/java/org/apache/calcite/sql/SqlUtil.java
index a2de0dfc58fe..6dcb1f20713f 100644
--- a/core/src/main/java/org/apache/calcite/sql/SqlUtil.java
+++ b/core/src/main/java/org/apache/calcite/sql/SqlUtil.java
@@ -1336,7 +1336,7 @@ public static boolean containsCall(SqlNode node,
     try {
       SqlVisitor<@Nullable Void> visitor =
           new SqlBasicVisitor<@Nullable Void>() {
-            @Override public Void visit(SqlCall call) {
+            @Override public @Nullable Void visit(SqlCall call) {
               if (callPredicate.test(call)) {
                 throw new Util.FoundOne(call);
               }
@@ -1466,11 +1466,11 @@ private void visitChild(@Nullable SqlNode node) {
       ancestors.remove(ancestors.size() - 1);
     }
 
-    @Override public Void visit(SqlIdentifier id) {
+    @Override public @Nullable Void visit(SqlIdentifier id) {
       return check(id);
     }
 
-    @Override public Void visit(SqlCall call) {
+    @Override public @Nullable Void visit(SqlCall call) {
       preCheck(call);
       for (SqlNode node : call.getOperandList()) {
         visitChild(node);
@@ -1478,15 +1478,15 @@ private void visitChild(@Nullable SqlNode node) {
       return postCheck(call);
     }
 
-    @Override public Void visit(SqlIntervalQualifier intervalQualifier) {
+    @Override public @Nullable Void visit(SqlIntervalQualifier intervalQualifier) {
       return check(intervalQualifier);
     }
 
-    @Override public Void visit(SqlLiteral literal) {
+    @Override public @Nullable Void visit(SqlLiteral literal) {
       return check(literal);
     }
 
-    @Override public Void visit(SqlNodeList nodeList) {
+    @Override public @Nullable Void visit(SqlNodeList nodeList) {
       preCheck(nodeList);
       for (SqlNode node : nodeList) {
         visitChild(node);
@@ -1494,11 +1494,11 @@ private void visitChild(@Nullable SqlNode node) {
       return postCheck(nodeList);
     }
 
-    @Override public Void visit(SqlDynamicParam param) {
+    @Override public @Nullable Void visit(SqlDynamicParam param) {
       return check(param);
     }
 
-    @Override public Void visit(SqlDataTypeSpec type) {
+    @Override public @Nullable Void visit(SqlDataTypeSpec type) {
       return check(type);
     }
   }
diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlBetweenOperator.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlBetweenOperator.java
index 321b5c54d19f..470031f2932a 100644
--- a/core/src/main/java/org/apache/calcite/sql/fun/SqlBetweenOperator.java
+++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlBetweenOperator.java
@@ -266,7 +266,7 @@ private static SqlBetweenOperator of(boolean negated, boolean symmetric) {
    * Finds an AND operator in an expression.
    */
   private static class AndFinder extends SqlBasicVisitor<@Nullable Void> {
-    @Override public Void visit(SqlCall call) {
+    @Override public @Nullable Void visit(SqlCall call) {
       final SqlOperator operator = call.getOperator();
       if (operator == SqlStdOperatorTable.AND) {
         throw Util.FoundOne.NULL;
diff --git a/core/src/main/java/org/apache/calcite/sql/type/OperandTypes.java b/core/src/main/java/org/apache/calcite/sql/type/OperandTypes.java
index 58754538a125..7bdeb0142545 100644
--- a/core/src/main/java/org/apache/calcite/sql/type/OperandTypes.java
+++ b/core/src/main/java/org/apache/calcite/sql/type/OperandTypes.java
@@ -2057,12 +2057,12 @@ protected TypeRemover(SqlValidator validator) {
         this.validator = validator;
       }
 
-      @Override public Void visit(SqlIdentifier id) {
+      @Override public @Nullable Void visit(SqlIdentifier id) {
         validator.removeValidatedNodeType(id);
         return super.visit(id);
       }
 
-      @Override public Void visit(SqlCall call) {
+      @Override public @Nullable Void visit(SqlCall call) {
         validator.removeValidatedNodeType(call);
         return super.visit(call);
       }
diff --git a/core/src/main/java/org/apache/calcite/sql/validate/AggChecker.java b/core/src/main/java/org/apache/calcite/sql/validate/AggChecker.java
index 6fa0e7cb872a..718fda743841 100644
--- a/core/src/main/java/org/apache/calcite/sql/validate/AggChecker.java
+++ b/core/src/main/java/org/apache/calcite/sql/validate/AggChecker.java
@@ -102,7 +102,7 @@ boolean isMeasureExp(SqlNode e) {
     return false;
   }
 
-  @Override public Void visit(SqlIdentifier id) {
+  @Override public @Nullable Void visit(SqlIdentifier id) {
     if (id.isStar()) {
       // Star may validly occur in "SELECT COUNT(*) OVER w"
       return null;
@@ -167,7 +167,7 @@ private boolean isOuterReference(SqlValidatorScope scope, SqlQualified fqId) {
     return resolved.count() == 1 && !resolved.only().scope.isWithin(currentSelectScope);
   }
 
-  @Override public Void visit(SqlCall call) {
+  @Override public @Nullable Void visit(SqlCall call) {
     final SqlValidatorScope scope =
         requireNonNull(scopes.peek(), () -> "scope for " + call);
     if (call.getOperator().isAggregator()) {
diff --git a/core/src/main/java/org/apache/calcite/sql/validate/AggVisitor.java b/core/src/main/java/org/apache/calcite/sql/validate/AggVisitor.java
index 6b93767d4496..9d7dde721854 100644
--- a/core/src/main/java/org/apache/calcite/sql/validate/AggVisitor.java
+++ b/core/src/main/java/org/apache/calcite/sql/validate/AggVisitor.java
@@ -69,7 +69,7 @@ abstract class AggVisitor extends SqlBasicVisitor<@Nullable Void> {
     this.nameMatcher = requireNonNull(nameMatcher, "nameMatcher");
   }
 
-  @Override public Void visit(SqlCall call) {
+  @Override public @Nullable Void visit(SqlCall call) {
     final SqlOperator operator = call.getOperator();
     // If nested aggregates disallowed or found an aggregate at invalid level
     if (operator.isAggregator()
diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java
index 2eead5ea4a3c..fc7238394450 100644
--- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java
+++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java
@@ -1784,7 +1784,7 @@ && requireNonNull(((SqlNumericLiteral) node).bigDecimalValue())
     }
     validateNoAggs(aggOrOverFinder, node, kind);
     node.accept(new SqlBasicVisitor<@Nullable Void>() {
-      @Override public Void visit(SqlIdentifier id) {
+      @Override public @Nullable Void visit(SqlIdentifier id) {
         if (makeNullaryCall(id) != null) {
           return null;
         }
@@ -4577,7 +4577,7 @@ protected void validateSelect(
   private static void forEachQualified(SqlNode node, SqlValidatorScope scope,
       Consumer consumer) {
     node.accept(new SqlBasicVisitor<@Nullable Void>() {
-      @Override public Void visit(SqlIdentifier id) {
+      @Override public @Nullable Void visit(SqlIdentifier id) {
         final SqlQualified qualified = scope.fullyQualify(id);
         consumer.accept(qualified);
         return null;
@@ -4591,7 +4591,7 @@ private static void purgeForBypassFields(SqlNode node, SqlValidatorScope scope,
       Set qualifieds, Set bypassQualifieds,
       Set remnantMustFilterFields) {
     node.accept(new SqlBasicVisitor<@Nullable Void>() {
-      @Override public Void visit(SqlIdentifier id) {
+      @Override public @Nullable Void visit(SqlIdentifier id) {
         final SqlQualified qualified = scope.fullyQualify(id);
         if (bypassQualifieds.contains(qualified)) {
           // Clear all the must-filter qualifieds from the same table identifier
@@ -6055,7 +6055,7 @@ private boolean referencesOnlyOuterColumns(SqlNode node,
     // ok[1] is set once at least one outer column is found.
     final boolean[] ok = {true, false};
     node.accept(new SqlBasicVisitor<@Nullable Void>() {
-      @Override public Void visit(SqlIdentifier id) {
+      @Override public @Nullable Void visit(SqlIdentifier id) {
         if (!isOuterReference(currentScope, id)) {
           ok[0] = false;
           return null;
@@ -6080,7 +6080,7 @@ private boolean referencesOnlyOuterColumns(SqlNode node,
   private static boolean containsSubQuery(SqlNode node) {
     final boolean[] found = {false};
     node.accept(new SqlBasicVisitor<@Nullable Void>() {
-      @Override public Void visit(SqlCall call) {
+      @Override public @Nullable Void visit(SqlCall call) {
         if (call.getKind().belongsTo(SqlKind.QUERY)) {
           found[0] = true;
           return null;
@@ -8027,36 +8027,36 @@ private static class PatternVarVisitor implements SqlVisitor<@Nullable Void> {
       this.scope = scope;
     }
 
-    @Override public Void visit(SqlLiteral literal) {
+    @Override public @Nullable Void visit(SqlLiteral literal) {
       return null;
     }
 
-    @Override public Void visit(SqlCall call) {
+    @Override public @Nullable Void visit(SqlCall call) {
       for (int i = 0; i < call.getOperandList().size(); i++) {
         call.getOperandList().get(i).accept(this);
       }
       return null;
     }
 
-    @Override public Void visit(SqlNodeList nodeList) {
+    @Override public @Nullable Void visit(SqlNodeList nodeList) {
       throw Util.needToImplement(nodeList);
     }
 
-    @Override public Void visit(SqlIdentifier id) {
+    @Override public @Nullable Void visit(SqlIdentifier id) {
       checkArgument(id.isSimple());
       scope.addPatternVar(id.getSimple());
       return null;
     }
 
-    @Override public Void visit(SqlDataTypeSpec type) {
+    @Override public @Nullable Void visit(SqlDataTypeSpec type) {
       throw Util.needToImplement(type);
     }
 
-    @Override public Void visit(SqlDynamicParam param) {
+    @Override public @Nullable Void visit(SqlDynamicParam param) {
       throw Util.needToImplement(param);
     }
 
-    @Override public Void visit(SqlIntervalQualifier intervalQualifier) {
+    @Override public @Nullable Void visit(SqlIntervalQualifier intervalQualifier) {
       throw Util.needToImplement(intervalQualifier);
     }
   }
@@ -8939,7 +8939,7 @@ private boolean containsIdentifier(SqlNode sqlNode, SqlIdentifier target) {
       try {
         SqlVisitor<@Nullable Void> visitor =
             new SqlBasicVisitor<@Nullable Void>() {
-              @Override public Void visit(SqlIdentifier identifier) {
+              @Override public @Nullable Void visit(SqlIdentifier identifier) {
                 if (identifier.equalsDeep(target, Litmus.IGNORE)) {
                   throw new Util.FoundOne(target);
                 }
diff --git a/core/src/main/java/org/apache/calcite/sql2rel/AggConverter.java b/core/src/main/java/org/apache/calcite/sql2rel/AggConverter.java
index fbdec67744c1..0e6eaf2fab00 100644
--- a/core/src/main/java/org/apache/calcite/sql2rel/AggConverter.java
+++ b/core/src/main/java/org/apache/calcite/sql2rel/AggConverter.java
@@ -242,7 +242,7 @@ private void addExpr(RexNode expr, @Nullable String name) {
     convertedInputExprs.add(expr, name);
   }
 
-  @Override public Void visit(SqlIdentifier id) {
+  @Override public @Nullable Void visit(SqlIdentifier id) {
     if (isMeasureExpr(id)) {
       final SqlCall call =
           SqlInternalOperators.AGG_M2V.createCall(SqlParserPos.ZERO, id);
@@ -256,28 +256,28 @@ private void addExpr(RexNode expr, @Nullable String name) {
     return null;
   }
 
-  @Override public Void visit(SqlNodeList nodeList) {
+  @Override public @Nullable Void visit(SqlNodeList nodeList) {
     nodeList.forEach(this::visitNode);
     return null;
   }
 
-  @Override public Void visit(SqlLiteral lit) {
+  @Override public @Nullable Void visit(SqlLiteral lit) {
     return null;
   }
 
-  @Override public Void visit(SqlDataTypeSpec type) {
+  @Override public @Nullable Void visit(SqlDataTypeSpec type) {
     return null;
   }
 
-  @Override public Void visit(SqlDynamicParam param) {
+  @Override public @Nullable Void visit(SqlDynamicParam param) {
     return null;
   }
 
-  @Override public Void visit(SqlIntervalQualifier intervalQualifier) {
+  @Override public @Nullable Void visit(SqlIntervalQualifier intervalQualifier) {
     return null;
   }
 
-  @Override public Void visit(SqlCall call) {
+  @Override public @Nullable Void visit(SqlCall call) {
     switch (call.getKind()) {
     case FILTER:
     case IGNORE_NULLS:
diff --git a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
index bbd850dbf228..e19b638ddc7b 100644
--- a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
+++ b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
@@ -6748,7 +6748,7 @@ private static class AggregateFinder extends SqlBasicVisitor<@Nullable Void> {
     final List distinctList = new ArrayList<>();
     final List orderList = new ArrayList<>();
 
-    @Override public Void visit(SqlCall call) {
+    @Override public @Nullable Void visit(SqlCall call) {
       // ignore window aggregates and ranking functions (associated with OVER operator)
       if (call.getOperator().getKind() == SqlKind.OVER) {
         return null;
diff --git a/core/src/main/java/org/apache/calcite/util/Util.java b/core/src/main/java/org/apache/calcite/util/Util.java
index 7484f8d1b709..122f47c41fc2 100644
--- a/core/src/main/java/org/apache/calcite/util/Util.java
+++ b/core/src/main/java/org/apache/calcite/util/Util.java
@@ -2916,7 +2916,7 @@ public FoundOne(@Nullable Object node) {
   public static class OverFinder extends SqlBasicVisitor<@Nullable Void> {
     public static final OverFinder INSTANCE = new Util.OverFinder();
 
-    @Override public Void visit(SqlCall call) {
+    @Override public @Nullable Void visit(SqlCall call) {
       if (call.getKind() == SqlKind.OVER) {
         throw FoundOne.NULL;
       }

From 872fe88ce67c8cf65139b8a52493383800d32050 Mon Sep 17 00:00:00 2001
From: Vladimir Sitnikov 
Date: Tue, 25 Aug 2026 19:45:46 +0300
Subject: [PATCH 538/562] [CALCITE-7736] Let NullAway itself require the
 explicit null-marking

NullAway 0.12.13 and later ship a RequireExplicitNullMarking Error Prone check
that fails a top-level class which is neither annotated nor covered by an
annotated package or module. It is what the OnlyNullMarked setting needs, and
it is stricter than the LintTest check it replaces: a class that sits in a
package with no package-info.java is reported by name, whether or not the
package holds a package-info.java at all.

Co-Authored-By: Claude Opus 5 
---
 build.gradle.kts                              |  3 +
 .../org/apache/calcite/test/LintTest.java     | 68 -------------------
 2 files changed, 3 insertions(+), 68 deletions(-)

diff --git a/build.gradle.kts b/build.gradle.kts
index 69291c9f3df1..ca138df5194c 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -846,6 +846,9 @@ allprojects {
                     if (nullawayEnabled && mainCode) {
                         // Nullness errors must fail the build, so the annotations stay trustworthy
                         error("NullAway")
+                        // OnlyNullMarked skips a package that forgets @NullMarked rather than
+                        // reporting it, so require every class to say which of the two it is
+                        error("RequireExplicitNullMarking")
                         // Only @NullMarked code is analyzed, so an unannotated package is skipped
                         // rather than assumed non-null
                         option("NullAway:OnlyNullMarked", "true")
diff --git a/core/src/test/java/org/apache/calcite/test/LintTest.java b/core/src/test/java/org/apache/calcite/test/LintTest.java
index 97060ffbad33..0714beb1a71b 100644
--- a/core/src/test/java/org/apache/calcite/test/LintTest.java
+++ b/core/src/test/java/org/apache/calcite/test/LintTest.java
@@ -31,11 +31,8 @@
 import org.junit.jupiter.api.Test;
 
 import java.io.File;
-import java.io.IOException;
 import java.io.PrintWriter;
 import java.io.StringWriter;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
 import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.HashSet;
@@ -69,12 +66,6 @@ class LintTest {
       compile("^(\\[CALCITE-[0-9]{1,4}][ ]).*");
   private static final Pattern PATTERN = compile("^ *(// )?");
 
-  private static final String PACKAGE_INFO = "package-info.java";
-  /** Source roots that NullAway verifies; see {@code nullawayProjects} in the root
-   * {@code build.gradle.kts}. */
-  private static final List NULL_MARKED_ROOTS =
-      ImmutableList.of("linq4j/src/main/java/", "core/src/main/java/");
-
   private static final Pattern COMMONS_LANG3_IMPORT_PATTERN =
       compile("^\\s*import\\s+(static\\s+)?"
           + "org\\.apache\\.commons\\.lang3\\..*;\\s*$");
@@ -373,65 +364,6 @@ private static boolean isJava(String filename) {
     assertThat(g.messages, empty());
   }
 
-  /** Fails when a main-source package is not declared {@code @NullMarked}.
-   *
-   * 
      NullAway analyzes {@code @NullMarked} code only, so a package that forgets the
-   * annotation is silently skipped rather than reported. Add a {@code package-info.java}
-   * to the new package, copying the {@code @NullMarked} declaration from a sibling
-   * package.
-   *
-   * 
      Only the main sources of the modules that NullAway verifies are checked. Marking a
-   * package that nobody verifies would claim a guarantee that nothing backs. Widen
-   * {@link #NULL_MARKED_ROOTS} together with {@code nullawayProjects}.
-   *
-   * 
      A package that spans both modules needs only one {@code package-info.java}, in
-   * either of them, because a second one would put a duplicate class on the classpath. */
-  @Test void testLintNullMarked() throws IOException {
-    assumeTrue(TestUnsafe.haveGit(), "Invalid git environment");
-
-    final List messages = new ArrayList<>();
-    final Set mainPackages = new HashSet<>();
-    final Set markedPackages = new HashSet<>();
-    for (File file : TestUnsafe.getJavaFiles()) {
-      final String path = file.getPath().replace(File.separatorChar, '/');
-      final String root =
-          NULL_MARKED_ROOTS.stream()
-              .filter(path::contains)
-              .findFirst()
-              .orElse(null);
-      if (root == null) {
-        continue;
-      }
-      final int i = path.indexOf(root) + root.length();
-      final String packageName =
-          path.substring(i, path.lastIndexOf('/')).replace('/', '.');
-      mainPackages.add(packageName);
-      if (file.getName().equals(PACKAGE_INFO)) {
-        if (isNullMarked(file)) {
-          markedPackages.add(packageName);
-        } else {
-          messages.add(file + ": " + PACKAGE_INFO + " is not annotated @NullMarked");
-        }
-      }
-    }
-    mainPackages.stream()
-        .filter(packageName -> !markedPackages.contains(packageName))
-        .map(packageName ->
-            packageName + ": package has no " + PACKAGE_INFO + " declaring @NullMarked")
-        .sorted()
-        .forEach(messages::add);
-
-    messages.forEach(System.out::println);
-    assertThat(messages, empty());
-  }
-
-  private static boolean isNullMarked(File file) throws IOException {
-    try (Stream lines =
-             Files.lines(file.toPath(), StandardCharsets.UTF_8)) {
-      return lines.anyMatch(line -> line.startsWith("@NullMarked"));
-    }
-  }
-
   /** Tests that the most recent N commit messages are good.
    *
    * 
      N needs to be large enough to verify multi-commit PRs, but not so large

From 1219ced2b2941533281961317d20fdb3185870f5 Mon Sep 17 00:00:00 2001
From: Vladimir Sitnikov 
Date: Tue, 25 Aug 2026 19:51:59 +0300
Subject: [PATCH 539/562] [CALCITE-7736] Put :server back under nullness
 verification

The Checker Framework covered :server as well, so NullAway takes it over. The
module needs no source changes: its four main classes live in
org.apache.calcite.server, a package that :core already declares @NullMarked,
and the generated DDL parser sits under a javacc directory, which the
XepExcludedPaths setting skips the way AskipDefs used to.

Co-Authored-By: Claude Opus 5 
---
 build.gradle.kts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build.gradle.kts b/build.gradle.kts
index ca138df5194c..af5256b0a175 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -94,7 +94,7 @@ val werror by props(true) // treat javac warnings as errors
 
 // Projects whose main code NullAway verifies. The other projects are not annotated well enough
 // yet, so NullAway would only produce noise there.
-val nullawayProjects = listOf(":linq4j", ":core")
+val nullawayProjects = listOf(":linq4j", ":core", ":server")
 
 val hepLargePlanModeTestIncludes = mapOf(
     ":core" to listOf(

From 2963f6f0e1b3cad7288d41320ff53b83579fb4fb Mon Sep 17 00:00:00 2001
From: Vladimir Sitnikov 
Date: Tue, 25 Aug 2026 20:14:17 +0300
Subject: [PATCH 540/562] [CALCITE-7736] Put :druid under nullness verification

The Druid adapter now declares its package @NullMarked, and NullAway verifies
it. Most of the change is stating in the signatures what the bodies already
did: a visitor over a Druid column returns a pair whose halves are both absent
when the column cannot be pushed down, an extraction function carries no
granularity and no locale, and the filters and plans that writeFieldIf skips
take an absent value.

Three places said something the code did not mean. Only the Checker Framework
needed the preconditions on DruidTable.create and DruidType.getTypeFromMetric,
whose callers are now the ones that check. DruidProjectRule named a field null
for an expression that is not an input reference, but splitProjects puts
nothing but input references there, so the branch was dead. And a rolled-up
column with no parent node dereferenced that parent, where the Table contract
has said it may be absent since the method was introduced.

The Jackson result classes keep non-null fields under a NullAway.Init
suppression, since Druid always populates them; the three that depend on the
analysis types the query asked for are @Nullable, which is what the reader of
aggregators already assumed.

Co-Authored-By: Claude Opus 5 
---
 build.gradle.kts                              |  2 +-
 .../adapter/druid/DruidConnectionImpl.java    | 40 +++++----
 .../adapter/druid/DruidJsonFilter.java        | 46 +++++------
 .../calcite/adapter/druid/DruidQuery.java     | 81 +++++++++++--------
 .../calcite/adapter/druid/DruidRules.java     | 30 +++----
 .../adapter/druid/DruidSqlCastConverter.java  |  4 +-
 .../calcite/adapter/druid/DruidTable.java     | 14 ++--
 .../adapter/druid/DruidTableFactory.java      |  7 +-
 .../calcite/adapter/druid/DruidType.java      |  1 -
 .../adapter/druid/TimeExtractionFunction.java | 12 +--
 .../calcite/adapter/druid/VirtualColumn.java  | 19 +++--
 .../calcite/adapter/druid/package-info.java   |  3 +
 12 files changed, 141 insertions(+), 118 deletions(-)

diff --git a/build.gradle.kts b/build.gradle.kts
index af5256b0a175..9219111450fb 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -94,7 +94,7 @@ val werror by props(true) // treat javac warnings as errors
 
 // Projects whose main code NullAway verifies. The other projects are not annotated well enough
 // yet, so NullAway would only produce noise there.
-val nullawayProjects = listOf(":linq4j", ":core", ":server")
+val nullawayProjects = listOf(":linq4j", ":core", ":server", ":druid")
 
 val hepLargePlanModeTestIncludes = mapOf(
     ":core" to listOf(
diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidConnectionImpl.java b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidConnectionImpl.java
index dedadefd60ab..8f3f287cc4b5 100644
--- a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidConnectionImpl.java
+++ b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidConnectionImpl.java
@@ -97,7 +97,7 @@ class DruidConnectionImpl implements DruidConnection {
    * @param page Page definition (in/out)
    */
   public void request(QueryType queryType, String data, Sink sink,
-      List fieldNames, List fieldTypes,
+      List fieldNames, List fieldTypes,
       Page page) {
     final String url = this.url + "/druid/v2/?pretty";
     final Map requestHeaders =
@@ -117,7 +117,7 @@ public void request(QueryType queryType, String data, Sink sink,
   /** Parses the output of a query, sending the results to a
    * {@link Sink}. */
   private static void parse(QueryType queryType, InputStream in, Sink sink,
-      List fieldNames, List fieldTypes, Page page) {
+      List fieldNames, List fieldTypes, Page page) {
     final JsonFactory factory = new JsonFactory();
     final Row.RowBuilder rowBuilder = Row.newBuilder(fieldNames.size());
 
@@ -296,19 +296,22 @@ private static void parse(QueryType queryType, InputStream in, Sink sink,
     }
   }
 
-  private static void parseFields(List fieldNames, List fieldTypes,
+  private static void parseFields(List fieldNames,
+      List fieldTypes,
       Row.RowBuilder rowBuilder, JsonParser parser) throws IOException {
     parseFields(fieldNames, fieldTypes, -1, rowBuilder, parser);
   }
 
-  private static void parseFields(List fieldNames, List fieldTypes,
+  private static void parseFields(List fieldNames,
+      List fieldTypes,
       int posTimestampField, Row.RowBuilder rowBuilder, JsonParser parser) throws IOException {
     while (parser.nextToken() == JsonToken.FIELD_NAME) {
       parseField(fieldNames, fieldTypes, posTimestampField, rowBuilder, parser);
     }
   }
 
-  private static void parseField(List fieldNames, List fieldTypes,
+  private static void parseField(List fieldNames,
+      List fieldTypes,
       int posTimestampField, Row.RowBuilder rowBuilder, JsonParser parser) throws IOException {
     final String fieldName = parser.currentName();
     parseFieldForName(fieldNames, fieldTypes, posTimestampField, rowBuilder, parser, fieldName);
@@ -316,7 +319,7 @@ private static void parseField(List fieldNames, List
 
   @SuppressWarnings("JavaUtilDate")
   private static void parseFieldForName(List fieldNames,
-      List fieldTypes,
+      List fieldTypes,
       int posTimestampField, Row.RowBuilder rowBuilder, JsonParser parser, String fieldName)
       throws IOException {
     // Move to next token, which is name's value
@@ -554,7 +557,7 @@ public Enumerable enumerable(final QueryType queryType,
           @Override public void run() {
             try {
               final Page page = new Page();
-              final List fieldTypes =
+              final List fieldTypes =
                   Collections.nCopies(fieldNames.size(), null);
               request(queryType, request, this, fieldNames, fieldTypes, page);
               enumerator.done.set(true);
@@ -572,7 +575,7 @@ public Enumerable enumerable(final QueryType queryType,
 
   /** Reads segment metadata, and populates a list of columns and metrics. */
   void metadata(String dataSourceName, String timestampColumnName,
-      List intervals,
+      @Nullable List intervals,
       Map fieldBuilder, Set metricNameBuilder,
       Map> complexMetrics) {
     final String url = this.url + "/druid/v2/?pretty";
@@ -680,9 +683,9 @@ private interface RunnableQueueSink extends Sink, Runnable {
   private static class BlockingQueueEnumerator implements Enumerator {
     final BlockingQueue queue = new ArrayBlockingQueue<>(1000);
     final AtomicBoolean done = new AtomicBoolean(false);
-    final Holder throwableHolder = Holder.empty();
+    final Holder<@Nullable Throwable> throwableHolder = Holder.empty();
 
-    E next;
+    @Nullable E next;
 
     @Override public E current() {
       if (next == null) {
@@ -727,30 +730,35 @@ static class Page {
   }
 
   /** Result of a "segmentMetadata" call, populated by Jackson. */
-  @SuppressWarnings({ "WeakerAccess", "unused" })
+  @SuppressWarnings({ "WeakerAccess", "unused", "NullAway.Init" })
   private static class JsonSegmentMetadata {
     public String id;
     public List intervals;
     public Map columns;
     public long size;
     public long numRows;
-    public Map aggregators;
+    /** Present only when the query asked for the "aggregators" analysis type
+     * and the segment carries aggregator metadata. */
+    public @Nullable Map aggregators;
   }
 
   /** Element of the "columns" collection in the result of a
    * "segmentMetadata" call, populated by Jackson. */
-  @SuppressWarnings({ "WeakerAccess", "unused" })
+  @SuppressWarnings({ "WeakerAccess", "unused", "NullAway.Init" })
   private static class JsonColumn {
     public String type;
     public boolean hasMultipleValues;
     public int size;
-    public Integer cardinality;
-    public String errorMessage;
+    /** Present only when the query asked for the "cardinality" analysis
+     * type. */
+    public @Nullable Integer cardinality;
+    /** Present only when Druid could not analyze the column. */
+    public @Nullable String errorMessage;
   }
 
   /** Element of the "aggregators" collection in the result of a
    * "segmentMetadata" call, populated by Jackson. */
-  @SuppressWarnings({ "WeakerAccess", "unused" })
+  @SuppressWarnings({ "WeakerAccess", "unused", "NullAway.Init" })
   private static class JsonAggregator {
     public String type;
     public String name;
diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidJsonFilter.java b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidJsonFilter.java
index e8ed7c33baae..7ff013681235 100644
--- a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidJsonFilter.java
+++ b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidJsonFilter.java
@@ -99,10 +99,10 @@ abstract class DruidJsonFilter implements DruidJson {
     }
     final boolean isNumeric = refNode.getType().getFamily() == SqlTypeFamily.NUMERIC
         || rexLiteral.getType().getFamily() == SqlTypeFamily.NUMERIC;
-    final Pair druidColumn =
+    final Pair<@Nullable String, @Nullable ExtractionFunction> druidColumn =
         DruidQuery.toDruidColumn(refNode, rowType, druidQuery);
-    final String columnName = druidColumn.left;
-    final ExtractionFunction extractionFunction = druidColumn.right;
+    final @Nullable String columnName = druidColumn.left;
+    final @Nullable ExtractionFunction extractionFunction = druidColumn.right;
     if (columnName == null) {
       // no column name better bail out.
       return null;
@@ -171,10 +171,10 @@ abstract class DruidJsonFilter implements DruidJson {
     }
     final boolean isNumeric = refNode.getType().getFamily() == SqlTypeFamily.NUMERIC
         || rexLiteral.getType().getFamily() == SqlTypeFamily.NUMERIC;
-    final Pair druidColumn =
+    final Pair<@Nullable String, @Nullable ExtractionFunction> druidColumn =
         DruidQuery.toDruidColumn(refNode, rowType, druidQuery);
-    final String columnName = druidColumn.left;
-    final ExtractionFunction extractionFunction = druidColumn.right;
+    final @Nullable String columnName = druidColumn.left;
+    final @Nullable ExtractionFunction extractionFunction = druidColumn.right;
     if (columnName == null) {
       // no column name better bail out.
       return null;
@@ -253,10 +253,10 @@ abstract class DruidJsonFilter implements DruidJson {
     }
     final RexCall rexCall = (RexCall) rexNode;
     final RexNode refNode = rexCall.getOperands().get(0);
-    Pair druidColumn = DruidQuery
+    Pair<@Nullable String, @Nullable ExtractionFunction> druidColumn = DruidQuery
         .toDruidColumn(refNode, rowType, druidQuery);
-    final String columnName = druidColumn.left;
-    final ExtractionFunction extractionFunction = druidColumn.right;
+    final @Nullable String columnName = druidColumn.left;
+    final @Nullable ExtractionFunction extractionFunction = druidColumn.right;
     if (columnName == null) {
       return null;
     }
@@ -287,11 +287,11 @@ abstract class DruidJsonFilter implements DruidJson {
         listBuilder.add(value);
       }
     }
-    Pair druidColumn = DruidQuery
+    Pair<@Nullable String, @Nullable ExtractionFunction> druidColumn = DruidQuery
         .toDruidColumn(((RexCall) e).getOperands().get(0),
         rowType, druidQuery);
-    final String columnName = druidColumn.left;
-    final ExtractionFunction extractionFunction = druidColumn.right;
+    final @Nullable String columnName = druidColumn.left;
+    final @Nullable ExtractionFunction extractionFunction = druidColumn.right;
     if (columnName == null) {
       return null;
     }
@@ -303,7 +303,8 @@ abstract class DruidJsonFilter implements DruidJson {
     }
   }
 
-  protected static @Nullable DruidJsonFilter toNotDruidFilter(DruidJsonFilter druidJsonFilter) {
+  protected static @Nullable DruidJsonFilter toNotDruidFilter(
+      @Nullable DruidJsonFilter druidJsonFilter) {
     if (druidJsonFilter == null) {
       return null;
     }
@@ -331,10 +332,10 @@ abstract class DruidJsonFilter implements DruidJson {
     }
     final boolean isNumeric = lhs.getType().getFamily() == SqlTypeFamily.NUMERIC
         || rhs.getType().getFamily() == SqlTypeFamily.NUMERIC;
-    final Pair druidColumn = DruidQuery
+    final Pair<@Nullable String, @Nullable ExtractionFunction> druidColumn = DruidQuery
         .toDruidColumn(refNode, rowType, query);
-    final String columnName = druidColumn.left;
-    final ExtractionFunction extractionFunction = druidColumn.right;
+    final @Nullable String columnName = druidColumn.left;
+    final @Nullable ExtractionFunction extractionFunction = druidColumn.right;
 
     if (columnName == null) {
       return null;
@@ -494,12 +495,12 @@ private static JsonExpressionFilter alwaysFalse() {
   private static class JsonSelector extends DruidJsonFilter {
     private final String dimension;
 
-    private final String value;
+    private final @Nullable String value;
 
-    private final ExtractionFunction extractionFunction;
+    private final @Nullable ExtractionFunction extractionFunction;
 
-    private JsonSelector(String dimension, String value,
-        ExtractionFunction extractionFunction) {
+    private JsonSelector(String dimension, @Nullable String value,
+        @Nullable ExtractionFunction extractionFunction) {
       super(Type.SELECTOR);
       this.dimension = dimension;
       this.value = value;
@@ -628,9 +629,8 @@ protected JsonInFilter(String dimension, List values,
     }
   }
 
-  public static DruidJsonFilter getSelectorFilter(String column, String value,
-      ExtractionFunction extractionFunction) {
-    requireNonNull(column, "column");
+  public static DruidJsonFilter getSelectorFilter(String column, @Nullable String value,
+      @Nullable ExtractionFunction extractionFunction) {
     return new JsonSelector(column, value, extractionFunction);
   }
 
diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidQuery.java b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidQuery.java
index 2d89420a13ee..29c44cdfc971 100644
--- a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidQuery.java
+++ b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidQuery.java
@@ -237,7 +237,8 @@ private static DruidQuery create(RelOptCluster cluster, RelTraitSet traitSet,
   /** Extends a DruidQuery. */
   public static DruidQuery extendQuery(DruidQuery query, RelNode r) {
     final ImmutableList.Builder builder = ImmutableList.builder();
-    return DruidQuery.create(query.getCluster(), r.getTraitSet().replace(query.getConvention()),
+    return DruidQuery.create(query.getCluster(),
+        r.getTraitSet().replace(requireNonNull(query.getConvention(), "convention")),
         query.getTable(), query.druidTable, query.intervals,
         builder.addAll(query.rels).add(r).build(), query.getOperatorConversionMap());
   }
@@ -268,10 +269,10 @@ private static boolean needUtcTimeExtract(RexNode rexNode) {
    * the input ref, or {@code Pair.of(null, null)} when cannot translate to a
    * valid Druid column
    */
-  protected static Pair toDruidColumn(RexNode rexNode,
-      RelDataType rowType, DruidQuery druidQuery) {
-    final String columnName;
-    final ExtractionFunction extractionFunction;
+  protected static Pair<@Nullable String, @Nullable ExtractionFunction> toDruidColumn(
+      RexNode rexNode, RelDataType rowType, DruidQuery druidQuery) {
+    final @Nullable String columnName;
+    final @Nullable ExtractionFunction extractionFunction;
     final Granularity granularity;
     switch (rexNode.getKind()) {
     case INPUT_REF:
@@ -377,20 +378,21 @@ private static boolean isValidLeafCast(RexNode rexNode) {
       return false;
     }
     final SqlTypeName toTypeName = rexNode.getType().getSqlTypeName();
-    if (toTypeName.getFamily() == SqlTypeFamily.CHARACTER) {
+    final SqlTypeFamily toFamily = toTypeName.getFamily();
+    if (toFamily == SqlTypeFamily.CHARACTER) {
       // CAST of input to character type
       return true;
     }
-    if (toTypeName.getFamily() == SqlTypeFamily.NUMERIC) {
+    if (toFamily == SqlTypeFamily.NUMERIC) {
       // CAST of input to numeric type, it is part of a bounded comparison
       return true;
     }
-    if (toTypeName.getFamily() == SqlTypeFamily.TIMESTAMP
-        || toTypeName.getFamily() == SqlTypeFamily.DATETIME) {
+    if (toFamily == SqlTypeFamily.TIMESTAMP
+        || toFamily == SqlTypeFamily.DATETIME) {
       // CAST of literal to timestamp type
       return true;
     }
-    if (toTypeName.getFamily().contains(input.getType())) {
+    if (toFamily != null && toFamily.contains(input.getType())) {
       // same type it is okay to push it
       return true;
     }
@@ -459,7 +461,7 @@ String signature() {
     return b.toString();
   }
 
-  @Override public boolean isValid(Litmus litmus, Context context) {
+  @Override public boolean isValid(Litmus litmus, @Nullable Context context) {
     if (!super.isValid(litmus, context)) {
       return false;
     }
@@ -594,7 +596,7 @@ public DruidTable getDruidTable() {
         // plan returning 2 columns.
         // A plan where all extra columns are pruned will be preferred.
         .multiplyBy(
-            RelMdUtil.linear(querySpec.fieldNames.size(), 2, 100, 1d, 2d))
+            RelMdUtil.linear(getQuerySpec().fieldNames.size(), 2, 100, 1d, 2d))
         .multiplyBy(getQueryTypeCostMultiplier())
         // A Scan leaf filter is better than having filter spec if possible.
         .multiplyBy(rels.size() > 1 && rels.get(1) instanceof Filter ? 0.5 : 1.0)
@@ -617,7 +619,7 @@ private double getIntervalCostMultiplier() {
 
   private double getQueryTypeCostMultiplier() {
     // Cost of Select > GroupBy > Timeseries > TopN
-    switch (querySpec.queryType) {
+    switch (getQuerySpec().queryType) {
     case SELECT:
       return .1;
     case GROUP_BY:
@@ -782,7 +784,7 @@ protected CalciteConnectionConfig getConnectionConfig() {
     final ImmutableList.Builder projectedColumnsBuilder = ImmutableList.builder();
     final List projects = projectRel.getProjects();
     for (RexNode project : projects) {
-      Pair druidColumn =
+      Pair<@Nullable String, @Nullable ExtractionFunction> druidColumn =
           toDruidColumn(project, inputRowType, druidQuery);
       boolean needExtractForOperand = project instanceof RexCall
           && ((RexCall) project).getOperands().stream().anyMatch(DruidQuery::needUtcTimeExtract);
@@ -854,7 +856,7 @@ protected CalciteConnectionConfig getConnectionConfig() {
         project = projectNode.getProjects().get(groupKey);
       }
 
-      Pair druidColumn =
+      Pair<@Nullable String, @Nullable ExtractionFunction> druidColumn =
           toDruidColumn(project, inputRowType, druidQuery);
       if (druidColumn.left != null && druidColumn.right == null) {
         // SIMPLE INPUT REF
@@ -870,13 +872,15 @@ protected CalciteConnectionConfig getConnectionConfig() {
         if (project.getKind() == SqlKind.EXTRACT) {
           columnPrefix =
               EXTRACT_COLUMN_NAME_PREFIX + "_" + requireNonNull(DruidDateTimeUtils
-                      .extractGranularity(project, druidQuery.getConnectionConfig().timeZone())
-                      .getType().lowerName);
+                      .extractGranularity(project, druidQuery.getConnectionConfig().timeZone()),
+                  "granularity")
+                  .getType().lowerName;
         } else if (project.getKind() == SqlKind.FLOOR) {
           columnPrefix =
               FLOOR_COLUMN_NAME_PREFIX + "_" + requireNonNull(DruidDateTimeUtils
-                      .extractGranularity(project, druidQuery.getConnectionConfig().timeZone())
-                      .getType().lowerName);
+                      .extractGranularity(project, druidQuery.getConnectionConfig().timeZone()),
+                  "granularity")
+                  .getType().lowerName;
         } else {
           columnPrefix = "extract";
         }
@@ -998,7 +1002,7 @@ protected CalciteConnectionConfig getConnectionConfig() {
     return aggregations;
   }
 
-  protected QuerySpec getQuery(RelDataType rowType, Filter filter,
+  protected QuerySpec getQuery(RelDataType rowType, @Nullable Filter filter,
       @Nullable Project project, @Nullable ImmutableBitSet groupSet,
       @Nullable List aggCalls, @Nullable List aggNames,
       @Nullable List collationIndexes,
@@ -1019,7 +1023,9 @@ protected QuerySpec getQuery(RelDataType rowType, Filter filter,
       if (project != null) {
         // project some fields only
         Pair, List> projectResult =
-            computeProjectAsScan(project, project.getInput().getRowType(), this);
+            requireNonNull(
+                computeProjectAsScan(project, project.getInput().getRowType(), this),
+                "computeProjectAsScan");
         scanColumnNames = projectResult.left;
         virtualColumnList.addAll(projectResult.right);
       } else {
@@ -1086,8 +1092,10 @@ protected QuerySpec getQuery(RelDataType rowType, Filter filter,
           .uniqueIndex(aggregateStageFieldNames, DruidExpressions::fromColumn);
       for (Pair pair : postProject.getNamedProjects()) {
         final RexNode postProjectRexNode = pair.left;
-        String expression = DruidExpressions
-              .toDruidExpression(postProjectRexNode, postAggInputRowType, this);
+        String expression =
+            requireNonNull(
+                DruidExpressions.toDruidExpression(postProjectRexNode, postAggInputRowType,
+                    this), "expression");
         final String existingFieldName = existingProjects.get(expression);
         if (existingFieldName != null) {
           // simple input ref or Druid runtime identity cast will skip it, since it is here already
@@ -1274,9 +1282,10 @@ private static JsonLimit computeSort(@Nullable Integer fetch,
   }
 
   private @Nullable String planAsTopN(List groupByKeyDims,
-      DruidJsonFilter jsonFilter,
+      @Nullable DruidJsonFilter jsonFilter,
       List virtualColumnList, List aggregations,
-      List postAggregations, JsonLimit limit, DruidJsonFilter havingFilter) {
+      List postAggregations, JsonLimit limit,
+      @Nullable DruidJsonFilter havingFilter) {
     if (havingFilter != null) {
       return null;
     }
@@ -1321,9 +1330,10 @@ private static JsonLimit computeSort(@Nullable Integer fetch,
   }
 
   private @Nullable String planAsGroupBy(List groupByKeyDims,
-      DruidJsonFilter jsonFilter,
+      @Nullable DruidJsonFilter jsonFilter,
       List virtualColumnList, List aggregations,
-      List postAggregations, JsonLimit limit, DruidJsonFilter havingFilter) {
+      List postAggregations, JsonLimit limit,
+      @Nullable DruidJsonFilter havingFilter) {
     final StringWriter sw = new StringWriter();
     final JsonFactory factory = new JsonFactory();
     try {
@@ -1434,7 +1444,9 @@ public String toQuery() {
     }
 
     // Convert from a complex metric
-    ComplexMetric complexMetric = druidQuery.druidTable.resolveComplexMetric(fieldName, aggCall);
+    ComplexMetric complexMetric =
+        fieldName == null ? null
+            : druidQuery.druidTable.resolveComplexMetric(fieldName, aggCall);
 
     switch (aggCall.getAggregation().getKind()) {
     case COUNT:
@@ -1443,7 +1455,7 @@ public String toQuery() {
           if (complexMetric == null) {
             aggregation =
                 new JsonCardinalityAggregation("cardinality", name,
-                    ImmutableList.of(fieldName));
+                    ImmutableList.of(requireNonNull(fieldName, "fieldName")));
           } else {
             aggregation =
                 new JsonAggregation(complexMetric.getMetricType(), name,
@@ -1464,7 +1476,8 @@ public String toQuery() {
           matchNulls = DruidJsonFilter.getSelectorFilter(fieldName, null, null);
         }
         aggregation =
-            new JsonFilteredAggregation(DruidJsonFilter.toNotDruidFilter(matchNulls),
+            new JsonFilteredAggregation(
+                requireNonNull(DruidJsonFilter.toNotDruidFilter(matchNulls)),
                 new JsonAggregation("count", name, fieldName, aggExpression));
       } else if (!aggCall.isDistinct()) {
         aggregation = new JsonAggregation("count", name, fieldName, aggExpression);
@@ -1555,7 +1568,7 @@ protected static void writeObject(JsonGenerator generator, Object o)
 
   /** Generates a JSON string to query metadata about a data source. */
   static String metadataQuery(String dataSourceName,
-      List intervals) {
+      @Nullable List intervals) {
     final StringWriter sw = new StringWriter();
     final JsonFactory factory = new JsonFactory();
     try {
@@ -1594,7 +1607,7 @@ public static class QuerySpec {
       return Objects.hash(queryType, queryString, fieldNames);
     }
 
-    @Override public boolean equals(Object obj) {
+    @Override public boolean equals(@Nullable Object obj) {
       return obj == this
           || obj instanceof QuerySpec
           && queryType == ((QuerySpec) obj).queryType
@@ -1634,7 +1647,7 @@ private static class DruidQueryNode implements Node {
     }
 
     @Override public void run() {
-      final List fieldTypes = new ArrayList<>();
+      final List fieldTypes = new ArrayList<>();
       for (RelDataTypeField field : query.getRowType().getFieldList()) {
         fieldTypes.add(getPrimitive(field));
       }
@@ -1658,7 +1671,7 @@ private static boolean containsLimit(QuerySpec querySpec) {
           + DRUID_QUERY_FETCH + "\":true");
     }
 
-    private static ColumnMetaData.Rep getPrimitive(RelDataTypeField field) {
+    private static ColumnMetaData.@Nullable Rep getPrimitive(RelDataTypeField field) {
       switch (field.getType().getSqlTypeName()) {
       case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
       case TIMESTAMP:
diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java
index 814b716c1cf7..05407be1c771 100644
--- a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java
+++ b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java
@@ -409,13 +409,9 @@ protected DruidProjectRule(DruidProjectRuleConfig config) {
           cluster.getTypeFactory().builder();
       final RelNode input = Util.last(query.rels);
       for (RexNode e : below) {
-        final String name;
-        if (e instanceof RexInputRef) {
-          name = input.getRowType().getFieldNames().get(((RexInputRef) e).getIndex());
-        } else {
-          name = null;
-        }
-        builder.add(name, e.getType());
+        // splitProjects puts nothing but input references below
+        final RexInputRef ref = (RexInputRef) e;
+        builder.add(input.getRowType().getFieldNames().get(ref.getIndex()), e.getType());
       }
       final RelNode newProject =
           project.copy(project.getTraitSet(), input, below, builder.build());
@@ -668,7 +664,6 @@ private static Set getUniqueFilterRefs(List calls) {
     private static DruidQuery optimizeFilteredAggregations(RelOptRuleCall call,
         DruidQuery query,
         Project project, Aggregate aggregate) {
-      Filter filter = null;
       final RexBuilder builder = query.getCluster().getRexBuilder();
       final RexExecutor executor =
           Util.first(query.getCluster().getPlanner().getExecutor(),
@@ -679,12 +674,11 @@ private static DruidQuery optimizeFilteredAggregations(RelOptRuleCall call,
       final RexSimplify simplify =
           new RexSimplify(builder, predicates, executor);
 
-      // if the druid query originally contained a filter
-      boolean containsFilter = false;
+      // the filter the druid query originally contained, if it contained one
+      Filter oldFilter = null;
       for (RelNode node : query.rels) {
         if (node instanceof Filter) {
-          filter = (Filter) node;
-          containsFilter = true;
+          oldFilter = (Filter) node;
           break;
         }
       }
@@ -720,9 +714,10 @@ private static DruidQuery optimizeFilteredAggregations(RelOptRuleCall call,
           aggregate.copy(aggregate.getTraitSet(), aggregate.getInput(),
               aggregate.getGroupSet(), aggregate.getGroupSets(), newCalls);
 
-      if (containsFilter) {
+      if (oldFilter != null) {
         // AND the current filterNode with the filter node inside filter
-        filterNode = builder.makeCall(SqlStdOperatorTable.AND, filterNode, filter.getCondition());
+        filterNode =
+            builder.makeCall(SqlStdOperatorTable.AND, filterNode, oldFilter.getCondition());
       }
 
       // Simplify the filter as much as possible
@@ -741,19 +736,20 @@ private static DruidQuery optimizeFilteredAggregations(RelOptRuleCall call,
         filterNode = tempFilterNode;
       }
 
-      filter = LogicalFilter.create(scan, filterNode);
+      final Filter filter = LogicalFilter.create(scan, filterNode);
 
       boolean addNewFilter = !filter.getCondition().isAlwaysTrue() && allHaveFilters;
       // Assumes that Filter nodes are always right after
       // TableScan nodes (which are always present)
-      int startIndex = containsFilter && addNewFilter ? 2 : 1;
+      int startIndex = oldFilter != null && addNewFilter ? 2 : 1;
 
       List newNodes =
           constructNewNodes(query.rels, addNewFilter, startIndex,
               filter, project, aggregate);
 
       return DruidQuery.create(query.getCluster(),
-             aggregate.getTraitSet().replace(query.getConvention()),
+             aggregate.getTraitSet()
+                 .replace(requireNonNull(query.getConvention(), "convention")),
              query.getTable(), query.druidTable, newNodes);
     }
 
diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidSqlCastConverter.java b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidSqlCastConverter.java
index 67f725f14d1b..fad069837b2e 100644
--- a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidSqlCastConverter.java
+++ b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidSqlCastConverter.java
@@ -53,7 +53,7 @@ public class DruidSqlCastConverter implements DruidSqlOperatorConverter {
     }
 
     final SqlTypeName fromType = operand.getType().getSqlTypeName();
-    String fromTypeString = dateTimeFormatString(fromType);
+    @Nullable String fromTypeString = dateTimeFormatString(fromType);
     final SqlTypeName toType = rexNode.getType().getSqlTypeName();
     final String timeZoneConf = druidQuery.getConnectionConfig().timeZone();
     final TimeZone timeZone = TimeZone.getTimeZone(timeZoneConf);
@@ -136,7 +136,7 @@ public class DruidSqlCastConverter implements DruidSqlOperatorConverter {
   private static String castCharToDateTime(
       TimeZone timeZone,
       String operand,
-      final SqlTypeName toType, String format) {
+      final SqlTypeName toType, @Nullable String format) {
     // Cast strings to date times by parsing them from SQL format.
     final String timestampExpression =
         DruidExpressions.functionCall("timestamp_parse",
diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidTable.java b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidTable.java
index 925cab5d7b85..d6183adc63ba 100644
--- a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidTable.java
+++ b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidTable.java
@@ -122,11 +122,9 @@ public DruidTable(DruidSchema schema, String dataSource,
    * @return A table
    */
   static Table create(DruidSchema druidSchema, String dataSourceName,
-      List intervals, Map fieldMap,
+      @Nullable List intervals, Map fieldMap,
       Set metricNameSet, String timestampColumnName,
       DruidConnectionImpl connection, Map> complexMetrics) {
-    requireNonNull(connection, "connection");
-
     connection.metadata(dataSourceName, timestampColumnName, intervals,
             fieldMap, metricNameSet, complexMetrics);
 
@@ -202,10 +200,11 @@ && isCountDistinct(call))
         && isValidParentKind(parent);
   }
 
-  private static boolean isValidParentKind(SqlNode node) {
-    return node.getKind() == SqlKind.SELECT
+  private static boolean isValidParentKind(@Nullable SqlNode node) {
+    return node != null
+            && (node.getKind() == SqlKind.SELECT
             || node.getKind() == SqlKind.FILTER
-            || isSupportedPostAggOperation(node.getKind());
+            || isSupportedPostAggOperation(node.getKind()));
   }
 
   private static boolean isCountDistinct(SqlCall call) {
@@ -239,7 +238,8 @@ public boolean isComplexMetric(String alias) {
   }
 
   @Override public RelDataType getRowType(RelDataTypeFactory typeFactory) {
-    final RelDataType rowType = protoRowType.apply(typeFactory);
+    final RelDataType rowType =
+        requireNonNull(protoRowType, "protoRowType").apply(typeFactory);
     final List fieldNames = rowType.getFieldNames();
     checkArgument(fieldNames.contains(timestampFieldName));
     checkArgument(fieldNames.containsAll(metricFieldNames));
diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidTableFactory.java b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidTableFactory.java
index b4174ab1e642..b60b607cca87 100644
--- a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidTableFactory.java
+++ b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidTableFactory.java
@@ -53,7 +53,8 @@ private DruidTableFactory() {}
   // name that is also the same name as a complex metric
   @Override public Table create(SchemaPlus schema, String name, Map operand,
       @Nullable RelDataType rowType) {
-    final DruidSchema druidSchema = schema.unwrap(DruidSchema.class);
+    final DruidSchema druidSchema =
+        requireNonNull(schema.unwrap(DruidSchema.class), "druidSchema");
     // If "dataSource" operand is present it overrides the table name.
     final String dataSource = (String) operand.get("dataSource");
     final Set metricNameBuilder = new LinkedHashSet<>();
@@ -121,7 +122,7 @@ private DruidTableFactory() {}
           }
           metricName = (String) map2.get("name");
 
-          final String type = (String) map2.get("type");
+          final String type = requireNonNull((String) map2.get("type"), "type");
           fieldName = (String) map2.get("fieldName");
 
           druidType = DruidType.getTypeFromMetric(type);
@@ -148,7 +149,7 @@ private DruidTableFactory() {}
       }
     }
     final Object interval = operand.get("interval");
-    final List intervals;
+    final @Nullable List intervals;
     if (interval instanceof String) {
       intervals =
           ImmutableList.of(
diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidType.java b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidType.java
index b9b5737396a3..d7f35e73998b 100644
--- a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidType.java
+++ b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidType.java
@@ -45,7 +45,6 @@ public boolean isComplex() {
 
   /** Returns a DruidType matching the given String type from a Druid metric. */
   static DruidType getTypeFromMetric(String type) {
-    requireNonNull(type, "type");
     if (type.equals("hyperUnique")) {
       return HYPER_UNIQUE;
     } else if (type.equals("thetaSketch")) {
diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/TimeExtractionFunction.java b/druid/src/main/java/org/apache/calcite/adapter/druid/TimeExtractionFunction.java
index 6d9e6e21a150..d4e3772071dd 100644
--- a/druid/src/main/java/org/apache/calcite/adapter/druid/TimeExtractionFunction.java
+++ b/druid/src/main/java/org/apache/calcite/adapter/druid/TimeExtractionFunction.java
@@ -68,12 +68,12 @@ public class TimeExtractionFunction implements ExtractionFunction {
           TimeUnitRange.SECOND);
 
   private final String format;
-  private final Granularity granularity;
+  private final @Nullable Granularity granularity;
   private final String timeZone;
-  private final String local;
+  private final @Nullable String local;
 
-  public TimeExtractionFunction(String format, Granularity granularity, String timeZone,
-      String local) {
+  public TimeExtractionFunction(String format, @Nullable Granularity granularity, String timeZone,
+      @Nullable String local) {
     this.format = format;
     this.granularity = granularity;
     this.timeZone = timeZone;
@@ -93,7 +93,7 @@ public TimeExtractionFunction(String format, Granularity granularity, String tim
   public String getFormat() {
     return format;
   }
-  public Granularity getGranularity() {
+  public @Nullable Granularity getGranularity() {
     return granularity;
   }
 
@@ -202,7 +202,7 @@ public static boolean isValidTimeFloor(RexNode rexNode) {
     final RexCall rexCall = (RexCall) rexNode;
     final String castFormat = DruidSqlCastConverter
         .dateTimeFormatString(rexCall.getType().getSqlTypeName());
-    final String timeZoneId = timeZone == null ? null : timeZone.getID();
+    final String timeZoneId = timeZone.getID();
     if (castFormat == null) {
       // unknown format
       return null;
diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/VirtualColumn.java b/druid/src/main/java/org/apache/calcite/adapter/druid/VirtualColumn.java
index 44ce00aecf12..8f4b8e6d40cf 100644
--- a/druid/src/main/java/org/apache/calcite/adapter/druid/VirtualColumn.java
+++ b/druid/src/main/java/org/apache/calcite/adapter/druid/VirtualColumn.java
@@ -18,6 +18,8 @@
 
 import com.fasterxml.jackson.core.JsonGenerator;
 
+import org.jspecify.annotations.Nullable;
+
 import java.io.IOException;
 import java.util.Locale;
 
@@ -36,9 +38,9 @@ public class VirtualColumn implements DruidJson {
 
   private final DruidType outputType;
 
-  public VirtualColumn(String name, String expression, DruidType outputType) {
-    this.name = requireNonNull(name, "name");
-    this.expression = requireNonNull(expression, "expression");
+  public VirtualColumn(String name, String expression, @Nullable DruidType outputType) {
+    this.name = name;
+    this.expression = expression;
     this.outputType = outputType == null ? DruidType.FLOAT : outputType;
   }
 
@@ -67,11 +69,11 @@ public DruidType getOutputType() {
    * Virtual Column builder.
    */
   public static class Builder {
-    private String name;
+    private @Nullable String name;
 
-    private String expression;
+    private @Nullable String expression;
 
-    private DruidType type;
+    private @Nullable DruidType type;
 
     public Builder withName(String name) {
       this.name = name;
@@ -83,13 +85,14 @@ public Builder withExpression(String expression) {
       return this;
     }
 
-    public Builder withType(DruidType type) {
+    public Builder withType(@Nullable DruidType type) {
       this.type = type;
       return this;
     }
 
     public VirtualColumn build() {
-      return new VirtualColumn(name, expression, type);
+      return new VirtualColumn(requireNonNull(name, "name"),
+          requireNonNull(expression, "expression"), type);
     }
   }
 
diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/package-info.java b/druid/src/main/java/org/apache/calcite/adapter/druid/package-info.java
index 3cdc5b256e4b..5802b194bc7f 100644
--- a/druid/src/main/java/org/apache/calcite/adapter/druid/package-info.java
+++ b/druid/src/main/java/org/apache/calcite/adapter/druid/package-info.java
@@ -18,4 +18,7 @@
 /**
  * Query provider based on a Druid database.
  */
+@NullMarked
 package org.apache.calcite.adapter.druid;
+
+import org.jspecify.annotations.NullMarked;

From 8ac874aba663de58eecd718f9fe4b701cb196fa7 Mon Sep 17 00:00:00 2001
From: Vladimir Sitnikov 
Date: Tue, 25 Aug 2026 20:23:58 +0300
Subject: [PATCH 541/562] [CALCITE-7736] Put :file under nullness verification

The file adapter now declares its package @NullMarked, and NullAway verifies
it. A CSV cell, a table name in a model, and a field configuration are all
absent-able, and the signatures now say so: field() reads a row whose cells may
be absent, the row converters carry a nullable element type, and FileSchema
falls back to the source path through Util.firstNonNull rather than through the
@Contract on Util.first, which NullAway cannot read.

Two lazily populated fields were the reason for the remaining reports.
FileReader.getTable wrote its result into tableElement and returned nothing, so
no caller could see that the field was populated; it is now readTable, which
returns the element it read. The bad-source-column check in FileRowConverter
looked the heading up twice, once to validate and once to take the index, and
now does both at once.

A model that names no file for a CSV table, or no url for an HTML table, was
already a NullPointerException deeper in; requireNonNull names the missing
operand instead.

Co-Authored-By: Claude Opus 5 
---
 build.gradle.kts                              |  2 +-
 .../calcite/adapter/file/CsvEnumerator.java   | 14 +++++------
 .../adapter/file/CsvProjectTableScanRule.java |  3 ++-
 .../calcite/adapter/file/CsvTableFactory.java |  4 +++-
 .../calcite/adapter/file/FileEnumerator.java  |  4 +++-
 .../calcite/adapter/file/FileFieldType.java   |  2 +-
 .../calcite/adapter/file/FileReader.java      | 23 +++++++++++--------
 .../adapter/file/FileRowConverter.java        | 20 ++++++++--------
 .../calcite/adapter/file/FileSchema.java      | 13 +++++++----
 .../adapter/file/FileSchemaFactory.java       |  8 ++-----
 .../calcite/adapter/file/FileTable.java       |  4 ++--
 .../calcite/adapter/file/JsonEnumerator.java  |  5 ++--
 .../calcite/adapter/file/package-info.java    |  3 +++
 13 files changed, 58 insertions(+), 47 deletions(-)

diff --git a/build.gradle.kts b/build.gradle.kts
index 9219111450fb..735f87b78ead 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -94,7 +94,7 @@ val werror by props(true) // treat javac warnings as errors
 
 // Projects whose main code NullAway verifies. The other projects are not annotated well enough
 // yet, so NullAway would only produce noise there.
-val nullawayProjects = listOf(":linq4j", ":core", ":server", ":druid")
+val nullawayProjects = listOf(":linq4j", ":core", ":server", ":druid", ":file")
 
 val hepLargePlanModeTestIncludes = mapOf(
     ":core" to listOf(
diff --git a/file/src/main/java/org/apache/calcite/adapter/file/CsvEnumerator.java b/file/src/main/java/org/apache/calcite/adapter/file/CsvEnumerator.java
index f54e4202da6a..f4444d4b45b3 100644
--- a/file/src/main/java/org/apache/calcite/adapter/file/CsvEnumerator.java
+++ b/file/src/main/java/org/apache/calcite/adapter/file/CsvEnumerator.java
@@ -139,8 +139,8 @@ public CsvEnumerator(Source source, AtomicBoolean cancelFlag, boolean stream,
     }
   }
 
-  static RowConverter converter(List fieldTypes,
-      List fields) {
+  static RowConverter converter(
+      List fieldTypes, List fields) {
     if (fields.size() == 1) {
       final int field = fields.get(0);
       return new SingleColumnRowConverter(fieldTypes.get(field), field);
@@ -179,8 +179,8 @@ public static RelDataType deduceRowType(JavaTypeFactory typeFactory,
           String typeString = string.substring(colon + 1);
           Matcher decimalMatcher = DECIMAL_TYPE_PATTERN.matcher(typeString);
           if (decimalMatcher.matches()) {
-            int precision = parseInt(decimalMatcher.group(1));
-            int scale = parseInt(decimalMatcher.group(2));
+            int precision = parseInt(requireNonNull(decimalMatcher.group(1), "precision"));
+            int scale = parseInt(requireNonNull(decimalMatcher.group(2), "scale"));
             fieldType = parseDecimalSqlType(typeFactory, precision, scale);
           } else {
             switch (typeString) {
@@ -370,14 +370,14 @@ private static RelDataType toNullableRelDataType(JavaTypeFactory typeFactory,
   }
 
   /** Returns a field from a CSV row, or null if the row is too short. */
-  private static @Nullable String field(String[] strings, int index) {
+  private static @Nullable String field(@Nullable String[] strings, int index) {
     return index < strings.length ? strings[index] : null;
   }
 
   /** Row converter.
    *
    * @param  element type */
-  abstract static class RowConverter {
+  abstract static class RowConverter {
     abstract E convertRow(@Nullable String[] rows);
 
     @Nullable RelDataType getFieldType(int index) {
@@ -550,7 +550,7 @@ static class ArrayRowConverter extends RowConverter<@Nullable Object[]> {
   }
 
   /** Single column row converter. */
-  private static class SingleColumnRowConverter extends RowConverter {
+  private static class SingleColumnRowConverter extends RowConverter<@Nullable Object> {
     private final RelDataType fieldType;
     private final int fieldIndex;
 
diff --git a/file/src/main/java/org/apache/calcite/adapter/file/CsvProjectTableScanRule.java b/file/src/main/java/org/apache/calcite/adapter/file/CsvProjectTableScanRule.java
index 79ba80722e40..c1a4b5390bdc 100644
--- a/file/src/main/java/org/apache/calcite/adapter/file/CsvProjectTableScanRule.java
+++ b/file/src/main/java/org/apache/calcite/adapter/file/CsvProjectTableScanRule.java
@@ -23,6 +23,7 @@
 import org.apache.calcite.rex.RexNode;
 
 import org.immutables.value.Value;
+import org.jspecify.annotations.Nullable;
 
 import java.util.List;
 
@@ -71,7 +72,7 @@ protected CsvProjectTableScanRule(Config config) {
             scan.condition));
   }
 
-  private static int[] getProjectFields(List exps) {
+  private static int @Nullable [] getProjectFields(List exps) {
     final int[] fields = new int[exps.size()];
     for (int i = 0; i < exps.size(); i++) {
       final RexNode exp = exps.get(i);
diff --git a/file/src/main/java/org/apache/calcite/adapter/file/CsvTableFactory.java b/file/src/main/java/org/apache/calcite/adapter/file/CsvTableFactory.java
index 78ada9185ab1..021a4424556c 100644
--- a/file/src/main/java/org/apache/calcite/adapter/file/CsvTableFactory.java
+++ b/file/src/main/java/org/apache/calcite/adapter/file/CsvTableFactory.java
@@ -32,6 +32,8 @@
 import java.io.File;
 import java.util.Map;
 
+import static java.util.Objects.requireNonNull;
+
 /**
  * Factory that creates a {@link CsvTranslatableTable}.
  *
@@ -46,7 +48,7 @@ public CsvTableFactory() {
 
   @Override public CsvTable create(SchemaPlus schema, String name,
       Map operand, @Nullable RelDataType rowType) {
-    String fileName = (String) operand.get("file");
+    String fileName = requireNonNull((String) operand.get("file"), "file");
     final File base =
         (File) operand.get(ModelHandler.ExtraOperand.BASE_DIRECTORY.camelName);
     final Source source = Sources.file(base, fileName);
diff --git a/file/src/main/java/org/apache/calcite/adapter/file/FileEnumerator.java b/file/src/main/java/org/apache/calcite/adapter/file/FileEnumerator.java
index 638393326cfa..dfd416b0fa72 100644
--- a/file/src/main/java/org/apache/calcite/adapter/file/FileEnumerator.java
+++ b/file/src/main/java/org/apache/calcite/adapter/file/FileEnumerator.java
@@ -23,6 +23,8 @@
 
 import java.util.Iterator;
 
+import static org.apache.calcite.linq4j.Nullness.castNonNull;
+
 /**
  * Wraps {@link FileReader} and {@link FileRowConverter}, enumerates tr DOM
  * elements as table rows.
@@ -48,7 +50,7 @@ class FileEnumerator implements Enumerator {
     if (current == null) {
       this.moveNext();
     }
-    return current;
+    return castNonNull(current);
   }
 
   @Override public boolean moveNext() {
diff --git a/file/src/main/java/org/apache/calcite/adapter/file/FileFieldType.java b/file/src/main/java/org/apache/calcite/adapter/file/FileFieldType.java
index 71caff6057a2..409a219785fc 100644
--- a/file/src/main/java/org/apache/calcite/adapter/file/FileFieldType.java
+++ b/file/src/main/java/org/apache/calcite/adapter/file/FileFieldType.java
@@ -78,7 +78,7 @@ public RelDataType toType(JavaTypeFactory typeFactory) {
     return typeFactory.createJavaType(clazz);
   }
 
-  public static FileFieldType of(String typeString) {
+  public static @Nullable FileFieldType of(String typeString) {
     return MAP.get(typeString);
   }
 }
diff --git a/file/src/main/java/org/apache/calcite/adapter/file/FileReader.java b/file/src/main/java/org/apache/calcite/adapter/file/FileReader.java
index 632dceac5e08..388d6b5e6ccc 100644
--- a/file/src/main/java/org/apache/calcite/adapter/file/FileReader.java
+++ b/file/src/main/java/org/apache/calcite/adapter/file/FileReader.java
@@ -60,7 +60,7 @@ public FileReader(Source source) {
     this(source, null, null);
   }
 
-  private void getTable() throws FileReaderException {
+  private Element readTable() throws FileReaderException {
     final Document doc;
     try {
       String proto = source.protocol();
@@ -77,7 +77,7 @@ private void getTable() throws FileReaderException {
       throw new FileReaderException("Cannot read " + source, e);
     }
 
-    this.tableElement = (this.selector != null && !this.selector.isEmpty())
+    return (this.selector != null && !this.selector.isEmpty())
         ? getSelectedTable(doc, this.selector) : getBestTable(doc);
   }
 
@@ -133,21 +133,24 @@ private static Element getBestTable(Document doc) throws FileReaderException {
 
   void refresh() throws FileReaderException {
     this.headings = null;
-    getTable();
+    this.tableElement = readTable();
   }
 
   Elements getHeadings() {
-    if (this.headings == null) {
+    Elements headings = this.headings;
+    if (headings == null) {
       this.iterator();
+      headings = requireNonNull(this.headings, "headings");
     }
 
-    return this.headings;
+    return headings;
   }
 
   @Override public FileReaderIterator iterator() {
-    if (this.tableElement == null) {
+    Element tableElement = this.tableElement;
+    if (tableElement == null) {
       try {
-        getTable();
+        tableElement = this.tableElement = readTable();
       } catch (RuntimeException | Error e) {
         throw e;
       } catch (Exception e) {
@@ -156,7 +159,7 @@ Elements getHeadings() {
     }
 
     FileReaderIterator iterator =
-        new FileReaderIterator(this.tableElement.select("tr"));
+        new FileReaderIterator(tableElement.select("tr"));
 
     // if we haven't cached the headings, get them
     // TODO: this needs to be reworked to properly cache the headings
@@ -166,7 +169,7 @@ Elements getHeadings() {
       // if not, generate some default column names
       if (headings.isEmpty()) {
         // rewind and peek at the first row of data
-        iterator = new FileReaderIterator(this.tableElement.select("tr"));
+        iterator = new FileReaderIterator(tableElement.select("tr"));
         Elements firstRow = iterator.next("td");
         int i = 0;
         headings = new Elements();
@@ -177,7 +180,7 @@ Elements getHeadings() {
           headings.add(th);
         }
         // rewind, so queries see the first row
-        iterator = new FileReaderIterator(this.tableElement.select("tr"));
+        iterator = new FileReaderIterator(tableElement.select("tr"));
       }
       this.headings = headings;
     }
diff --git a/file/src/main/java/org/apache/calcite/adapter/file/FileRowConverter.java b/file/src/main/java/org/apache/calcite/adapter/file/FileRowConverter.java
index 94bb6362f6a7..01fbac323114 100644
--- a/file/src/main/java/org/apache/calcite/adapter/file/FileRowConverter.java
+++ b/file/src/main/java/org/apache/calcite/adapter/file/FileRowConverter.java
@@ -42,6 +42,7 @@
 import java.util.regex.Pattern;
 
 import static org.apache.calcite.util.Util.first;
+import static org.apache.calcite.util.Util.firstNonNull;
 
 import static java.lang.Boolean.parseBoolean;
 import static java.lang.Byte.parseByte;
@@ -72,7 +73,7 @@ class FileRowConverter {
 
   /** Creates a FileRowConverter. */
   FileRowConverter(FileReader fileReader,
-      List> fieldConfigs) {
+      @Nullable List> fieldConfigs) {
     this.fileReader = fileReader;
     this.fieldConfigs = fieldConfigs;
   }
@@ -106,14 +107,14 @@ private void initialize() {
           for (Map fieldConfig : this.fieldConfigs) {
 
             String thName = (String) fieldConfig.get("th");
+            Integer sourceIx = thName == null ? null : headerMap.get(thName);
+            if (thName == null || sourceIx == null) {
+              throw new Exception("bad source column name: '" + thName + "'");
+            }
             String name = thName;
             String newName;
             FileFieldType type = null;
             boolean skip = false;
-
-            if (!headerMap.containsKey(thName)) {
-              throw new Exception("bad source column name: '" + thName + "'");
-            }
             if ((newName = (String) fieldConfig.get("name")) != null) {
               name = newName;
             }
@@ -131,7 +132,6 @@ private void initialize() {
               skip = parseBoolean(sSkip);
             }
 
-            Integer sourceIx = headerMap.get(thName);
             colNames.add(name);
             sources.add(thName);
             if (!skip) {
@@ -170,7 +170,7 @@ private void addFieldDef(String name, @Nullable FileFieldType type,
   /** Converts a row of JSoup Elements to an array of java objects. */
   Object toRow(Elements rowElements, int[] projection) {
     initialize();
-    final Object[] objects = new Object[projection.length];
+    final @Nullable Object[] objects = new @Nullable Object[projection.length];
 
     for (int i = 0; i < projection.length; i++) {
       int field = projection[i];
@@ -224,11 +224,11 @@ private static class CellReader {
 
     CellReader(Map config) {
       final @Nullable String unusedType = (String) config.get("type");
-      this.selector = first((String) config.get("selector"), "*");
+      this.selector = firstNonNull((String) config.get("selector"), "*");
       this.selectedElement = (Integer) config.get("selectedElement");
       @Nullable String replace = (String) config.get("replace");
       this.replacePattern = replace == null ? null : Pattern.compile(replace);
-      this.replaceWith = first((String) config.get("replaceWith"), "");
+      this.replaceWith = firstNonNull((String) config.get("replaceWith"), "");
       @Nullable String match = (String) config.get("match");
       this.matchPattern = match == null ? null : Pattern.compile(match);
       this.matchSeq = first((Integer) config.get("matchSeq"), 0);
@@ -262,7 +262,7 @@ private static class CellReader {
         List allMatches = new ArrayList<>();
         Matcher m = this.matchPattern.matcher(cellString);
         while (m.find()) {
-          allMatches.add(m.group());
+          allMatches.add(requireNonNull(m.group(), "m.group()"));
         }
         if (!allMatches.isEmpty()) {
           return allMatches.get(this.matchSeq);
diff --git a/file/src/main/java/org/apache/calcite/adapter/file/FileSchema.java b/file/src/main/java/org/apache/calcite/adapter/file/FileSchema.java
index ef8616bc1cbd..ff58d9818d2b 100644
--- a/file/src/main/java/org/apache/calcite/adapter/file/FileSchema.java
+++ b/file/src/main/java/org/apache/calcite/adapter/file/FileSchema.java
@@ -34,6 +34,8 @@
 import java.util.List;
 import java.util.Map;
 
+import static java.util.Objects.requireNonNull;
+
 /**
  * Schema mapped onto a set of URLs / HTML tables. Each table in the schema
  * is an HTML table on a URL.
@@ -122,7 +124,7 @@ private static String trim(String s, String suffix) {
   private boolean addTable(ImmutableMap.Builder builder,
       Map tableDef) {
     final String tableName = (String) tableDef.get("name");
-    final String url = (String) tableDef.get("url");
+    final String url = requireNonNull((String) tableDef.get("url"), "url");
     final Source source0 = Sources.url(url);
     final Source source;
     if (baseDirectory == null) {
@@ -134,26 +136,27 @@ private boolean addTable(ImmutableMap.Builder builder,
   }
 
   private static boolean addTable(ImmutableMap.Builder builder,
-      Source source, String tableName, @Nullable Map tableDef) {
+      Source source, @Nullable String tableName,
+      @Nullable Map tableDef) {
     final Source sourceSansGz = source.trim(".gz");
     final Source sourceSansJson = sourceSansGz.trimOrNull(".json");
     if (sourceSansJson != null) {
       final Table table = new JsonScannableTable(source);
-      builder.put(Util.first(tableName, sourceSansJson.path()), table);
+      builder.put(Util.firstNonNull(tableName, sourceSansJson.path()), table);
       return true;
     }
     final Source sourceSansCsv = sourceSansGz.trimOrNull(".csv");
     if (sourceSansCsv != null) {
       final Table table =
           new CsvTranslatableTable(source, null, CSVParser.DEFAULT_SEPARATOR);
-      builder.put(Util.first(tableName, sourceSansCsv.path()), table);
+      builder.put(Util.firstNonNull(tableName, sourceSansCsv.path()), table);
       return true;
     }
 
     if (tableDef != null) {
       try {
         FileTable table = FileTable.create(source, tableDef);
-        builder.put(Util.first(tableName, source.path()), table);
+        builder.put(Util.firstNonNull(tableName, source.path()), table);
         return true;
       } catch (Exception e) {
         throw new RuntimeException("Unable to instantiate table for: "
diff --git a/file/src/main/java/org/apache/calcite/adapter/file/FileSchemaFactory.java b/file/src/main/java/org/apache/calcite/adapter/file/FileSchemaFactory.java
index 43d952bbac51..3c1add1a16fb 100644
--- a/file/src/main/java/org/apache/calcite/adapter/file/FileSchemaFactory.java
+++ b/file/src/main/java/org/apache/calcite/adapter/file/FileSchemaFactory.java
@@ -50,14 +50,10 @@ private FileSchemaFactory() {
     final File baseDirectory =
         (File) operand.get(ModelHandler.ExtraOperand.BASE_DIRECTORY.camelName);
     final String directory = (String) operand.get("directory");
-    File directoryFile = null;
+    File directoryFile = baseDirectory;
     if (directory != null) {
       directoryFile = new File(directory);
-    }
-    if (baseDirectory != null) {
-      if (directoryFile == null) {
-        directoryFile = baseDirectory;
-      } else if (!directoryFile.isAbsolute()) {
+      if (baseDirectory != null && !directoryFile.isAbsolute()) {
         directoryFile = new File(baseDirectory, directory);
       }
     }
diff --git a/file/src/main/java/org/apache/calcite/adapter/file/FileTable.java b/file/src/main/java/org/apache/calcite/adapter/file/FileTable.java
index b3a2aba3acff..35cf32f83d61 100644
--- a/file/src/main/java/org/apache/calcite/adapter/file/FileTable.java
+++ b/file/src/main/java/org/apache/calcite/adapter/file/FileTable.java
@@ -53,9 +53,9 @@ class FileTable extends AbstractQueryableTable
   private final FileRowConverter converter;
 
   /** Creates a FileTable. */
-  private FileTable(Source source, String selector, Integer index,
+  private FileTable(Source source, @Nullable String selector, @Nullable Integer index,
       @Nullable RelProtoDataType protoRowType,
-      List> fieldConfigs) {
+      @Nullable List> fieldConfigs) {
     super(Object[].class);
 
     this.protoRowType = protoRowType;
diff --git a/file/src/main/java/org/apache/calcite/adapter/file/JsonEnumerator.java b/file/src/main/java/org/apache/calcite/adapter/file/JsonEnumerator.java
index c5e975f1e381..12dbc1f2e38b 100644
--- a/file/src/main/java/org/apache/calcite/adapter/file/JsonEnumerator.java
+++ b/file/src/main/java/org/apache/calcite/adapter/file/JsonEnumerator.java
@@ -82,7 +82,8 @@ static JsonDataConverter deduceRowType(RelDataTypeFactory typeFactory, Source so
       }
 
     } catch (MismatchedInputException e) {
-      if (!e.getMessage().contains("No content")) {
+      final String message = e.getMessage();
+      if (message == null || !message.contains("No content")) {
         throw new RuntimeException("Couldn't read " + source, e);
       }
     } catch (Exception e) {
@@ -121,7 +122,7 @@ static JsonDataConverter deduceRowType(RelDataTypeFactory typeFactory, Source so
     return new JsonDataConverter(relDataType, list);
   }
 
-  @Override public Object[] current() {
+  @Override public @Nullable Object[] current() {
     return enumerator.current();
   }
 
diff --git a/file/src/main/java/org/apache/calcite/adapter/file/package-info.java b/file/src/main/java/org/apache/calcite/adapter/file/package-info.java
index 2a726e9f4d73..205115b35b37 100644
--- a/file/src/main/java/org/apache/calcite/adapter/file/package-info.java
+++ b/file/src/main/java/org/apache/calcite/adapter/file/package-info.java
@@ -22,4 +22,7 @@
  * table appears as a table.  Full select SQL operations are available on those
  * tables.
  */
+@NullMarked
 package org.apache.calcite.adapter.file;
+
+import org.jspecify.annotations.NullMarked;

From 6e3153058089585cc5745de6fc450ef210079f4c Mon Sep 17 00:00:00 2001
From: Vladimir Sitnikov 
Date: Tue, 25 Aug 2026 20:30:57 +0300
Subject: [PATCH 542/562] [CALCITE-7736] Put :kafka under nullness verification

The model that the adapter's own test uses names neither bootstrap.servers nor
topic.name, because a table that injects its own consumer needs neither, so
both options are optional and KafkaTableOptions now says so. That reaches
KafkaRowConverter.rowDataType, whose topic name is absent for such a table;
neither implementation looks at it. The bootstrap servers are required on the
path that builds a consumer, and requireNonNull there names the missing operand
rather than letting the Kafka client report it.

Co-Authored-By: Claude Opus 5 
---
 build.gradle.kts                              |  2 +-
 .../adapter/kafka/KafkaRowConverter.java      |  7 +++--
 .../adapter/kafka/KafkaRowConverterImpl.java  |  4 ++-
 .../adapter/kafka/KafkaStreamTable.java       |  5 +++-
 .../adapter/kafka/KafkaTableFactory.java      |  4 +--
 .../adapter/kafka/KafkaTableOptions.java      | 30 +++++++++++--------
 .../calcite/adapter/kafka/package-info.java   |  3 ++
 7 files changed, 36 insertions(+), 19 deletions(-)

diff --git a/build.gradle.kts b/build.gradle.kts
index 735f87b78ead..e5b81f2e8a2d 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -94,7 +94,7 @@ val werror by props(true) // treat javac warnings as errors
 
 // Projects whose main code NullAway verifies. The other projects are not annotated well enough
 // yet, so NullAway would only produce noise there.
-val nullawayProjects = listOf(":linq4j", ":core", ":server", ":druid", ":file")
+val nullawayProjects = listOf(":linq4j", ":core", ":server", ":druid", ":file", ":kafka")
 
 val hepLargePlanModeTestIncludes = mapOf(
     ":core" to listOf(
diff --git a/kafka/src/main/java/org/apache/calcite/adapter/kafka/KafkaRowConverter.java b/kafka/src/main/java/org/apache/calcite/adapter/kafka/KafkaRowConverter.java
index e209536ae260..9b577ee807d7 100644
--- a/kafka/src/main/java/org/apache/calcite/adapter/kafka/KafkaRowConverter.java
+++ b/kafka/src/main/java/org/apache/calcite/adapter/kafka/KafkaRowConverter.java
@@ -21,6 +21,8 @@
 import org.apache.kafka.clients.consumer.ConsumerConfig;
 import org.apache.kafka.clients.consumer.ConsumerRecord;
 
+import org.jspecify.annotations.Nullable;
+
 /**
  * Interface to handle formatting between Kafka message and Calcite row.
  *
@@ -35,10 +37,11 @@ public interface KafkaRowConverter {
   /**
    * Generates the row type for a given Kafka topic.
    *
-   * @param topicName Kafka topic name
+   * @param topicName Kafka topic name, or null if the table reads from a
+   *                  consumer that subscribed on its own
    * @return row type
    */
-  RelDataType rowDataType(String topicName);
+  RelDataType rowDataType(@Nullable String topicName);
 
   /**
    * Parses and reformats a Kafka message from the consumer,
diff --git a/kafka/src/main/java/org/apache/calcite/adapter/kafka/KafkaRowConverterImpl.java b/kafka/src/main/java/org/apache/calcite/adapter/kafka/KafkaRowConverterImpl.java
index 36f673f3908d..b55b84e6dde1 100644
--- a/kafka/src/main/java/org/apache/calcite/adapter/kafka/KafkaRowConverterImpl.java
+++ b/kafka/src/main/java/org/apache/calcite/adapter/kafka/KafkaRowConverterImpl.java
@@ -24,6 +24,8 @@
 
 import org.apache.kafka.clients.consumer.ConsumerRecord;
 
+import org.jspecify.annotations.Nullable;
+
 /**
  * Default implementation of {@link KafkaRowConverter}, both key and value are byte[].
  */
@@ -34,7 +36,7 @@ public class KafkaRowConverterImpl implements KafkaRowConverter
    * @param topicName Kafka topic name
    * @return row type
    */
-  @Override public RelDataType rowDataType(final String topicName) {
+  @Override public RelDataType rowDataType(final @Nullable String topicName) {
     final RelDataTypeFactory typeFactory =
         new SqlTypeFactoryImpl(RelDataTypeSystem.DEFAULT);
     final RelDataTypeFactory.Builder fieldInfo = typeFactory.builder();
diff --git a/kafka/src/main/java/org/apache/calcite/adapter/kafka/KafkaStreamTable.java b/kafka/src/main/java/org/apache/calcite/adapter/kafka/KafkaStreamTable.java
index 8a3954b3a532..e5e4e434e566 100644
--- a/kafka/src/main/java/org/apache/calcite/adapter/kafka/KafkaStreamTable.java
+++ b/kafka/src/main/java/org/apache/calcite/adapter/kafka/KafkaStreamTable.java
@@ -45,6 +45,8 @@
 import java.util.Properties;
 import java.util.concurrent.atomic.AtomicBoolean;
 
+import static java.util.Objects.requireNonNull;
+
 /**
  * A table that maps to an Apache Kafka topic.
  *
@@ -79,7 +81,8 @@ private class KafkaStreamTableEnumerable extends AbstractEnumerable<@Nullable Ob
 
       Properties consumerConfig = new Properties();
       consumerConfig.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG,
-          tableOptions.getBootstrapServers());
+          requireNonNull(tableOptions.getBootstrapServers(),
+              KafkaTableConstants.SCHEMA_BOOTSTRAP_SERVERS));
       // by default it's 
       consumerConfig.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG,
           "org.apache.kafka.common.serialization.ByteArrayDeserializer");
diff --git a/kafka/src/main/java/org/apache/calcite/adapter/kafka/KafkaTableFactory.java b/kafka/src/main/java/org/apache/calcite/adapter/kafka/KafkaTableFactory.java
index efeb20aba260..825fe3dc54dd 100644
--- a/kafka/src/main/java/org/apache/calcite/adapter/kafka/KafkaTableFactory.java
+++ b/kafka/src/main/java/org/apache/calcite/adapter/kafka/KafkaTableFactory.java
@@ -44,9 +44,9 @@ public KafkaTableFactory() {
     final KafkaTableOptions tableOptionBuilder = new KafkaTableOptions();
 
     tableOptionBuilder.setBootstrapServers(
-        (String) operand.getOrDefault(KafkaTableConstants.SCHEMA_BOOTSTRAP_SERVERS, null));
+        (String) operand.get(KafkaTableConstants.SCHEMA_BOOTSTRAP_SERVERS));
     tableOptionBuilder.setTopicName(
-        (String) operand.getOrDefault(KafkaTableConstants.SCHEMA_TOPIC_NAME, null));
+        (String) operand.get(KafkaTableConstants.SCHEMA_TOPIC_NAME));
 
     final KafkaRowConverter rowConverter;
     if (operand.containsKey(KafkaTableConstants.SCHEMA_ROW_CONVERTER)) {
diff --git a/kafka/src/main/java/org/apache/calcite/adapter/kafka/KafkaTableOptions.java b/kafka/src/main/java/org/apache/calcite/adapter/kafka/KafkaTableOptions.java
index f54e7831509a..28de0f043a38 100644
--- a/kafka/src/main/java/org/apache/calcite/adapter/kafka/KafkaTableOptions.java
+++ b/kafka/src/main/java/org/apache/calcite/adapter/kafka/KafkaTableOptions.java
@@ -18,33 +18,38 @@
 
 import org.apache.kafka.clients.consumer.Consumer;
 
+import org.jspecify.annotations.Nullable;
+
 import java.util.Map;
 
 /**
  * Available options for {@link KafkaStreamTable}.
  */
 public final class KafkaTableOptions {
-  private String bootstrapServers;
-  private String topicName;
+  private @Nullable String bootstrapServers;
+  private @Nullable String topicName;
+  /** Set by {@link KafkaTableFactory} right after construction, which is why
+   * it is not initialized here. */
+  @SuppressWarnings("NullAway.Init")
   private KafkaRowConverter rowConverter;
-  private Map consumerParams;
+  private @Nullable Map consumerParams;
   // added to inject MockConsumer for testing.
-  private Consumer consumer;
+  private @Nullable Consumer consumer;
 
-  public String getBootstrapServers() {
+  public @Nullable String getBootstrapServers() {
     return bootstrapServers;
   }
 
-  public KafkaTableOptions setBootstrapServers(final String bootstrapServers) {
+  public KafkaTableOptions setBootstrapServers(final @Nullable String bootstrapServers) {
     this.bootstrapServers = bootstrapServers;
     return this;
   }
 
-  public String getTopicName() {
+  public @Nullable String getTopicName() {
     return topicName;
   }
 
-  public KafkaTableOptions setTopicName(final String topicName) {
+  public KafkaTableOptions setTopicName(final @Nullable String topicName) {
     this.topicName = topicName;
     return this;
   }
@@ -59,20 +64,21 @@ public KafkaTableOptions setRowConverter(
     return this;
   }
 
-  public Map getConsumerParams() {
+  public @Nullable Map getConsumerParams() {
     return consumerParams;
   }
 
-  public KafkaTableOptions setConsumerParams(final Map consumerParams) {
+  public KafkaTableOptions setConsumerParams(
+      final @Nullable Map consumerParams) {
     this.consumerParams = consumerParams;
     return this;
   }
 
-  public Consumer getConsumer() {
+  public @Nullable Consumer getConsumer() {
     return consumer;
   }
 
-  public KafkaTableOptions setConsumer(final Consumer consumer) {
+  public KafkaTableOptions setConsumer(final @Nullable Consumer consumer) {
     this.consumer = consumer;
     return this;
   }
diff --git a/kafka/src/main/java/org/apache/calcite/adapter/kafka/package-info.java b/kafka/src/main/java/org/apache/calcite/adapter/kafka/package-info.java
index 3524be916282..68681d2c8ebe 100644
--- a/kafka/src/main/java/org/apache/calcite/adapter/kafka/package-info.java
+++ b/kafka/src/main/java/org/apache/calcite/adapter/kafka/package-info.java
@@ -20,4 +20,7 @@
  *
  * 
      One Kafka topic is mapping to one STREAM table.
  */
+@NullMarked
 package org.apache.calcite.adapter.kafka;
+
+import org.jspecify.annotations.NullMarked;

From c9f57bcb620a6d3604e295d102131569565844b6 Mon Sep 17 00:00:00 2001
From: Vladimir Sitnikov 
Date: Tue, 25 Aug 2026 20:35:34 +0300
Subject: [PATCH 543/562] [CALCITE-7736] Put :spark under nullness verification

SparkValues read the rowType field, which AbstractRelNode keeps as a lazily
computed cache, where it meant the row type its constructor was given;
getRowType() is the accessor that always has one. EnumerableToSparkConverter
throws before it reaches its unfinished body, so the body is gone and the
comment that describes what it would generate stays.

RexToLixTranslator.translateCondition passes its correlates argument straight
to setCorrelates, which has always accepted null, so the parameter says so now.
That is what lets SparkCalc convert a program that has no correlates.

Co-Authored-By: Claude Opus 5 
---
 build.gradle.kts                              |  2 +-
 .../enumerable/RexToLixTranslator.java        |  2 +-
 .../spark/EnumerableToSparkConverter.java     | 24 +------------------
 .../calcite/adapter/spark/HttpServer.java     |  2 +-
 .../calcite/adapter/spark/SparkMethod.java    |  4 +++-
 .../calcite/adapter/spark/SparkRules.java     |  6 ++---
 .../spark/SparkToEnumerableConverter.java     |  4 +++-
 .../calcite/adapter/spark/package-info.java   |  3 +++
 8 files changed, 16 insertions(+), 31 deletions(-)

diff --git a/build.gradle.kts b/build.gradle.kts
index e5b81f2e8a2d..9d3457b7f8ae 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -94,7 +94,7 @@ val werror by props(true) // treat javac warnings as errors
 
 // Projects whose main code NullAway verifies. The other projects are not annotated well enough
 // yet, so NullAway would only produce noise there.
-val nullawayProjects = listOf(":linq4j", ":core", ":server", ":druid", ":file", ":kafka")
+val nullawayProjects = listOf(":linq4j", ":core", ":server", ":druid", ":file", ":kafka", ":spark")
 
 val hepLargePlanModeTestIncludes = mapOf(
     ":core" to listOf(
diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java
index 37c4978d94e3..614b6ae42239 100644
--- a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java
+++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java
@@ -1351,7 +1351,7 @@ public static Expression translateCondition(RexProgram program,
    */
   public static Expression translateCondition(RexProgram program,
       JavaTypeFactory typeFactory, BlockBuilder list, InputGetter inputGetter,
-      Function1 correlates, SqlConformance conformance,
+      @Nullable Function1 correlates, SqlConformance conformance,
       boolean nullable, RexImplementorTable implementorTable) {
     RexLocalRef condition = program.getCondition();
     if (condition == null) {
diff --git a/spark/src/main/java/org/apache/calcite/adapter/spark/EnumerableToSparkConverter.java b/spark/src/main/java/org/apache/calcite/adapter/spark/EnumerableToSparkConverter.java
index 45ce57f684bf..3255af6c9f2d 100644
--- a/spark/src/main/java/org/apache/calcite/adapter/spark/EnumerableToSparkConverter.java
+++ b/spark/src/main/java/org/apache/calcite/adapter/spark/EnumerableToSparkConverter.java
@@ -69,28 +69,6 @@ protected EnumerableToSparkConverter(RelOptCluster cluster,
     // Generate:
     //   Enumerable source = ...;
     //   return SparkRuntime.createRdd(sparkContext, source);
-    if (true) {
-      throw new RuntimeException("EnumerableToSparkConverter is not implemented");
-    }
-    final BlockBuilder list = new BlockBuilder();
-    final PhysType physType =
-        PhysTypeImpl.of(
-            implementor.getTypeFactory(), getRowType(),
-            JavaRowFormat.CUSTOM);
-    final Expression source = null; // TODO:
-    final Expression sparkContext =
-        Expressions.call(
-            SparkMethod.GET_SPARK_CONTEXT.method,
-            implementor.getRootExpression());
-    final Expression rdd =
-        list.append(
-            "rdd",
-            Expressions.call(
-                SparkMethod.CREATE_RDD.method,
-                sparkContext,
-                source));
-    list.add(
-        Expressions.return_(null, rdd));
-    return implementor.result(physType, list.toBlock());
+    throw new RuntimeException("EnumerableToSparkConverter is not implemented");
   }
 }
diff --git a/spark/src/main/java/org/apache/calcite/adapter/spark/HttpServer.java b/spark/src/main/java/org/apache/calcite/adapter/spark/HttpServer.java
index d8b3227e8379..1e8031d0ed64 100644
--- a/spark/src/main/java/org/apache/calcite/adapter/spark/HttpServer.java
+++ b/spark/src/main/java/org/apache/calcite/adapter/spark/HttpServer.java
@@ -40,7 +40,7 @@
  * 
      Based on Spark HttpServer, wraps a Jetty server.
  */
 class HttpServer {
-  private static String localIpAddress;
+  private static @Nullable String localIpAddress;
 
   private final File resourceBase;
 
diff --git a/spark/src/main/java/org/apache/calcite/adapter/spark/SparkMethod.java b/spark/src/main/java/org/apache/calcite/adapter/spark/SparkMethod.java
index b1448e43ef91..4e8c5238fcde 100644
--- a/spark/src/main/java/org/apache/calcite/adapter/spark/SparkMethod.java
+++ b/spark/src/main/java/org/apache/calcite/adapter/spark/SparkMethod.java
@@ -24,6 +24,8 @@
 import org.apache.spark.api.java.JavaSparkContext;
 import org.apache.spark.api.java.function.FlatMapFunction;
 
+
+import org.jspecify.annotations.Nullable;
 import java.lang.reflect.Method;
 import java.util.HashMap;
 
@@ -57,7 +59,7 @@ public enum SparkMethod {
     this.method = Types.lookupMethod(clazz, methodName, argumentTypes);
   }
 
-  public static SparkMethod lookup(Method method) {
+  public static @Nullable SparkMethod lookup(Method method) {
     return MAP.get(method);
   }
 }
diff --git a/spark/src/main/java/org/apache/calcite/adapter/spark/SparkRules.java b/spark/src/main/java/org/apache/calcite/adapter/spark/SparkRules.java
index 51bab088d1f2..fe899d83dbf1 100644
--- a/spark/src/main/java/org/apache/calcite/adapter/spark/SparkRules.java
+++ b/spark/src/main/java/org/apache/calcite/adapter/spark/SparkRules.java
@@ -194,7 +194,7 @@ public static class SparkValues extends Values implements SparkRel {
         RelTraitSet traitSet, List inputs) {
       assert inputs.isEmpty();
       return new SparkValues(
-          getCluster(), rowType, tuples, traitSet);
+          getCluster(), getRowType(), tuples, traitSet);
     }
 
     @Override public Result implementSpark(Implementor implementor) {
@@ -215,7 +215,7 @@ public static class SparkValues extends Values implements SparkRel {
       final Type rowClass = physType.getJavaRowType();
 
       final List expressions = new ArrayList<>();
-      final List fields = rowType.getFieldList();
+      final List fields = getRowType().getFieldList();
       for (List tuple : tuples) {
         final List literals = new ArrayList<>();
         for (Pair pair
@@ -257,7 +257,7 @@ private static class SparkCalcRule extends ConverterRule {
       super(config);
     }
 
-    @Override public RelNode convert(RelNode rel) {
+    @Override public @Nullable RelNode convert(RelNode rel) {
       final LogicalCalc calc = (LogicalCalc) rel;
 
       // If there's a multiset, let FarragoMultisetSplitter work on it
diff --git a/spark/src/main/java/org/apache/calcite/adapter/spark/SparkToEnumerableConverter.java b/spark/src/main/java/org/apache/calcite/adapter/spark/SparkToEnumerableConverter.java
index edc4b893dcf8..24fd5203aae3 100644
--- a/spark/src/main/java/org/apache/calcite/adapter/spark/SparkToEnumerableConverter.java
+++ b/spark/src/main/java/org/apache/calcite/adapter/spark/SparkToEnumerableConverter.java
@@ -40,6 +40,8 @@
 
 import java.util.List;
 
+import static java.util.Objects.requireNonNull;
+
 /**
  * Relational expression that converts input of
  * {@link org.apache.calcite.adapter.spark.SparkRel#CONVENTION Spark convention}
@@ -65,7 +67,7 @@ protected SparkToEnumerableConverter(RelOptCluster cluster,
 
   @Override public @Nullable RelOptCost computeSelfCost(RelOptPlanner planner,
       RelMetadataQuery mq) {
-    return super.computeSelfCost(planner, mq).multiplyBy(.01);
+    return requireNonNull(super.computeSelfCost(planner, mq)).multiplyBy(.01);
   }
 
   @Override public Result implement(EnumerableRelImplementor implementor, Prefer pref) {
diff --git a/spark/src/main/java/org/apache/calcite/adapter/spark/package-info.java b/spark/src/main/java/org/apache/calcite/adapter/spark/package-info.java
index 9180bbdd2b8e..8c1f48fe721a 100644
--- a/spark/src/main/java/org/apache/calcite/adapter/spark/package-info.java
+++ b/spark/src/main/java/org/apache/calcite/adapter/spark/package-info.java
@@ -18,4 +18,7 @@
 /**
  * Adapter based on the Apache Spark data management system.
  */
+@NullMarked
 package org.apache.calcite.adapter.spark;
+
+import org.jspecify.annotations.NullMarked;

From 7510726efa141f3c6a4c09c61e301feeb0bccc90 Mon Sep 17 00:00:00 2001
From: Vladimir Sitnikov 
Date: Tue, 25 Aug 2026 20:38:02 +0300
Subject: [PATCH 544/562] [CALCITE-7736] Put :babel under nullness verification

The call factory for Babel's CREATE TABLE takes the collection type out of a
symbol literal, and SqlLiteral.symbolValue returns null for a literal that
holds no symbol. The parser always writes one, so requireNonNull states that
rather than leaving the constructor to find out.

Co-Authored-By: Claude Opus 5 
---
 .../org/apache/calcite/sql/babel/SqlBabelCreateTable.java     | 4 +++-
 .../main/java/org/apache/calcite/sql/babel/package-info.java  | 3 +++
 .../org/apache/calcite/sql/babel/postgres/package-info.java   | 3 +++
 build.gradle.kts                                              | 2 +-
 4 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/babel/src/main/java/org/apache/calcite/sql/babel/SqlBabelCreateTable.java b/babel/src/main/java/org/apache/calcite/sql/babel/SqlBabelCreateTable.java
index 5a4653c2fd83..6a9d95f2968a 100644
--- a/babel/src/main/java/org/apache/calcite/sql/babel/SqlBabelCreateTable.java
+++ b/babel/src/main/java/org/apache/calcite/sql/babel/SqlBabelCreateTable.java
@@ -44,7 +44,9 @@ public class SqlBabelCreateTable extends SqlCreateTable {
           (operator, functionQualifier, pos, operands) ->
               new SqlBabelCreateTable(pos,
                   requireNonNull((SqlLiteral) operands[0]).booleanValue(),
-                  requireNonNull((SqlLiteral) operands[1]).symbolValue(TableCollectionType.class),
+                  requireNonNull(
+                      requireNonNull((SqlLiteral) operands[1])
+                          .symbolValue(TableCollectionType.class)),
                   requireNonNull((SqlLiteral) operands[2]).booleanValue(),
                   requireNonNull((SqlLiteral) operands[3]).booleanValue(),
                   (SqlIdentifier) requireNonNull(operands[4]),
diff --git a/babel/src/main/java/org/apache/calcite/sql/babel/package-info.java b/babel/src/main/java/org/apache/calcite/sql/babel/package-info.java
index 4e83feda71be..833af52c5cd5 100644
--- a/babel/src/main/java/org/apache/calcite/sql/babel/package-info.java
+++ b/babel/src/main/java/org/apache/calcite/sql/babel/package-info.java
@@ -18,4 +18,7 @@
 /**
  * Parse tree for SQL extensions used by the Babel parser.
  */
+@NullMarked
 package org.apache.calcite.sql.babel;
+
+import org.jspecify.annotations.NullMarked;
diff --git a/babel/src/main/java/org/apache/calcite/sql/babel/postgres/package-info.java b/babel/src/main/java/org/apache/calcite/sql/babel/postgres/package-info.java
index 5d09fea82b03..c50372c1ead0 100644
--- a/babel/src/main/java/org/apache/calcite/sql/babel/postgres/package-info.java
+++ b/babel/src/main/java/org/apache/calcite/sql/babel/postgres/package-info.java
@@ -18,4 +18,7 @@
 /**
  * Parse tree for PostgreSQL extensions used by the Babel parser.
  */
+@NullMarked
 package org.apache.calcite.sql.babel.postgres;
+
+import org.jspecify.annotations.NullMarked;
diff --git a/build.gradle.kts b/build.gradle.kts
index 9d3457b7f8ae..9f93be3bd4fc 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -94,7 +94,7 @@ val werror by props(true) // treat javac warnings as errors
 
 // Projects whose main code NullAway verifies. The other projects are not annotated well enough
 // yet, so NullAway would only produce noise there.
-val nullawayProjects = listOf(":linq4j", ":core", ":server", ":druid", ":file", ":kafka", ":spark")
+val nullawayProjects = listOf(":linq4j", ":core", ":server", ":druid", ":file", ":kafka", ":spark", ":babel")
 
 val hepLargePlanModeTestIncludes = mapOf(
     ":core" to listOf(

From 1b29b40e9a1e75692d7413b1864c335f0858a8c4 Mon Sep 17 00:00:00 2001
From: Vladimir Sitnikov 
Date: Tue, 25 Aug 2026 20:45:38 +0300
Subject: [PATCH 545/562] [CALCITE-7736] Put :redis under nullness verification

A Redis schema with no password is the ordinary case, and it reaches the pool
config through RedisConfig and RedisJedisManager, both of which say so now.
RedisSchema validated its operands through isEmptyObject, whose result NullAway
cannot connect back to the value, so the checks name the value they read and
read it once. RedisTable had a RedisEnumerator field that nothing ever read.

The anonymous Enumerable in RedisTable.scan is now a named inner class. NullAway
checks an anonymous class's overrides against the erased supertype, dropping the
@Nullable on its type argument, so it reported the enumerator() override as a
nullability mismatch; a named subclass with the same type argument is accepted.

Co-Authored-By: Claude Opus 5 
---
 build.gradle.kts                              |  2 +-
 .../calcite/adapter/redis/RedisConfig.java    |  8 +++---
 .../adapter/redis/RedisDataFormat.java        |  4 ++-
 .../adapter/redis/RedisDataProcess.java       |  2 +-
 .../calcite/adapter/redis/RedisDataType.java  |  4 ++-
 .../adapter/redis/RedisEnumerator.java        | 11 +++++---
 .../adapter/redis/RedisJedisManager.java      |  8 +++---
 .../calcite/adapter/redis/RedisSchema.java    | 26 +++++++++----------
 .../adapter/redis/RedisSchemaFactory.java     | 11 +++++---
 .../calcite/adapter/redis/RedisTable.java     | 22 +++++++++-------
 .../adapter/redis/RedisTableFactory.java      |  5 +++-
 .../adapter/redis/RedisTableFieldInfo.java    |  4 +++
 .../calcite/adapter/redis/package-info.java   |  3 +++
 .../redis/embedded/util/package-info.java     |  3 +++
 14 files changed, 71 insertions(+), 42 deletions(-)

diff --git a/build.gradle.kts b/build.gradle.kts
index 9f93be3bd4fc..15bf1ecb8945 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -94,7 +94,7 @@ val werror by props(true) // treat javac warnings as errors
 
 // Projects whose main code NullAway verifies. The other projects are not annotated well enough
 // yet, so NullAway would only produce noise there.
-val nullawayProjects = listOf(":linq4j", ":core", ":server", ":druid", ":file", ":kafka", ":spark", ":babel")
+val nullawayProjects = listOf(":linq4j", ":core", ":server", ":druid", ":file", ":kafka", ":spark", ":babel", ":redis")
 
 val hepLargePlanModeTestIncludes = mapOf(
     ":core" to listOf(
diff --git a/redis/src/main/java/org/apache/calcite/adapter/redis/RedisConfig.java b/redis/src/main/java/org/apache/calcite/adapter/redis/RedisConfig.java
index 7ca72a12eb0f..954132f891c0 100644
--- a/redis/src/main/java/org/apache/calcite/adapter/redis/RedisConfig.java
+++ b/redis/src/main/java/org/apache/calcite/adapter/redis/RedisConfig.java
@@ -16,6 +16,8 @@
  */
 package org.apache.calcite.adapter.redis;
 
+import org.jspecify.annotations.Nullable;
+
 /**
  * Set the redis config.
  */
@@ -23,9 +25,9 @@ public class RedisConfig {
   private final String host;
   private final int port;
   private final int database;
-  private final String password;
+  private final @Nullable String password;
 
-  public RedisConfig(String host, int port, int database, String password) {
+  public RedisConfig(String host, int port, int database, @Nullable String password) {
     this.host = host;
     this.port = port;
     this.database = database;
@@ -44,7 +46,7 @@ public int getDatabase() {
     return database;
   }
 
-  public String getPassword() {
+  public @Nullable String getPassword() {
     return password;
   }
 }
diff --git a/redis/src/main/java/org/apache/calcite/adapter/redis/RedisDataFormat.java b/redis/src/main/java/org/apache/calcite/adapter/redis/RedisDataFormat.java
index 2dc3a0ccd2d9..c3178d92be0b 100644
--- a/redis/src/main/java/org/apache/calcite/adapter/redis/RedisDataFormat.java
+++ b/redis/src/main/java/org/apache/calcite/adapter/redis/RedisDataFormat.java
@@ -16,6 +16,8 @@
  */
 package org.apache.calcite.adapter.redis;
 
+import org.jspecify.annotations.Nullable;
+
 /**
  * Define the data processing type of redis.
  */
@@ -43,7 +45,7 @@ public enum RedisDataFormat {
     this.typeName = typeName;
   }
 
-  public static RedisDataFormat fromTypeName(String typeName) {
+  public static @Nullable RedisDataFormat fromTypeName(String typeName) {
     for (RedisDataFormat type : RedisDataFormat.values()) {
       if (type.getTypeName().equals(typeName)) {
         return type;
diff --git a/redis/src/main/java/org/apache/calcite/adapter/redis/RedisDataProcess.java b/redis/src/main/java/org/apache/calcite/adapter/redis/RedisDataProcess.java
index 024a0e7df91f..398859b3adbd 100644
--- a/redis/src/main/java/org/apache/calcite/adapter/redis/RedisDataProcess.java
+++ b/redis/src/main/java/org/apache/calcite/adapter/redis/RedisDataProcess.java
@@ -85,7 +85,7 @@ private Object[] parseJson(String value) {
         if (obj == null) {
           arr[i] = "";
         } else {
-          arr[i] = jsonNode.findValue(fields.get(i).get("mapping").toString());
+          arr[i] = jsonNode.findValue(obj.toString());
         }
       }
     } catch (Exception e) {
diff --git a/redis/src/main/java/org/apache/calcite/adapter/redis/RedisDataType.java b/redis/src/main/java/org/apache/calcite/adapter/redis/RedisDataType.java
index 1e19d6b7f049..5a789ae64067 100644
--- a/redis/src/main/java/org/apache/calcite/adapter/redis/RedisDataType.java
+++ b/redis/src/main/java/org/apache/calcite/adapter/redis/RedisDataType.java
@@ -16,6 +16,8 @@
  */
 package org.apache.calcite.adapter.redis;
 
+import org.jspecify.annotations.Nullable;
+
 /**
  * All available data type for Redis.
  */
@@ -74,7 +76,7 @@ public enum RedisDataType {
     this.typeName = typeName;
   }
 
-  public static RedisDataType fromTypeName(String typeName) {
+  public static @Nullable RedisDataType fromTypeName(String typeName) {
     for (RedisDataType type : RedisDataType.values()) {
       if (type.getTypeName().equals(typeName)) {
         return type;
diff --git a/redis/src/main/java/org/apache/calcite/adapter/redis/RedisEnumerator.java b/redis/src/main/java/org/apache/calcite/adapter/redis/RedisEnumerator.java
index dede17cd48f0..7c9440f40f43 100644
--- a/redis/src/main/java/org/apache/calcite/adapter/redis/RedisEnumerator.java
+++ b/redis/src/main/java/org/apache/calcite/adapter/redis/RedisEnumerator.java
@@ -19,6 +19,8 @@
 import org.apache.calcite.linq4j.Enumerator;
 import org.apache.calcite.linq4j.Linq4j;
 
+
+import org.jspecify.annotations.Nullable;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
@@ -30,8 +32,8 @@
 /**
  * Implementation of {@link RedisEnumerator}.
  */
-class RedisEnumerator implements Enumerator {
-  private final Enumerator enumerator;
+class RedisEnumerator implements Enumerator<@Nullable Object[]> {
+  private final Enumerator<@Nullable Object[]> enumerator;
 
   RedisEnumerator(RedisConfig redisConfig, RedisSchema schema, String tableName) {
     RedisTableFieldInfo tableFieldInfo = schema.getTableFieldInfo(tableName);
@@ -60,13 +62,14 @@ static Map deduceRowType(RedisTableFieldInfo tableFieldInfo) {
       fieldBuilder.put("key", "key");
     } else {
       for (LinkedHashMap field : tableFieldInfo.getFields()) {
-        fieldBuilder.put(field.get("name").toString(), field.get("type").toString());
+        fieldBuilder.put(requireNonNull(field.get("name"), "name").toString(),
+            requireNonNull(field.get("type"), "type").toString());
       }
     }
     return fieldBuilder;
   }
 
-  @Override public Object[] current() {
+  @Override public @Nullable Object[] current() {
     return enumerator.current();
   }
 
diff --git a/redis/src/main/java/org/apache/calcite/adapter/redis/RedisJedisManager.java b/redis/src/main/java/org/apache/calcite/adapter/redis/RedisJedisManager.java
index 226af4b7b779..a1a55b7fdcbe 100644
--- a/redis/src/main/java/org/apache/calcite/adapter/redis/RedisJedisManager.java
+++ b/redis/src/main/java/org/apache/calcite/adapter/redis/RedisJedisManager.java
@@ -26,6 +26,7 @@
 import com.google.common.cache.RemovalListener;
 import com.google.common.cache.RemovalNotification;
 
+import org.jspecify.annotations.Nullable;
 import org.slf4j.Logger;
 
 import redis.clients.jedis.Jedis;
@@ -44,11 +45,12 @@ public class RedisJedisManager implements AutoCloseable {
   private final JedisPoolConfig jedisPoolConfig;
 
   private final String host;
-  private final String password;
+  private final @Nullable String password;
   private final int port;
   private final int database;
 
-  public RedisJedisManager(String host, int port, int database, String password) {
+  public RedisJedisManager(String host, int port, int database,
+      @Nullable String password) {
     JedisPoolConfig jedisPoolConfig = new JedisPoolConfig();
     jedisPoolConfig.setMaxTotal(GenericObjectPoolConfig.DEFAULT_MAX_TOTAL);
     jedisPoolConfig.setMaxIdle(GenericObjectPoolConfig.DEFAULT_MAX_IDLE);
@@ -73,7 +75,7 @@ public Jedis getResource() {
   }
 
   private JedisPool createConsumer() {
-    String pwd = password;
+    @Nullable String pwd = password;
     if (pwd == null || pwd.isEmpty()) {
       pwd = null;
     }
diff --git a/redis/src/main/java/org/apache/calcite/adapter/redis/RedisSchema.java b/redis/src/main/java/org/apache/calcite/adapter/redis/RedisSchema.java
index 41038af92f66..4ed12728d641 100644
--- a/redis/src/main/java/org/apache/calcite/adapter/redis/RedisSchema.java
+++ b/redis/src/main/java/org/apache/calcite/adapter/redis/RedisSchema.java
@@ -52,13 +52,13 @@ class RedisSchema extends AbstractSchema {
   public final String host;
   public final int port;
   public final int database;
-  public final String password;
+  public final @Nullable String password;
   public final List> tables;
 
   RedisSchema(String host,
       int port,
       int database,
-      String password,
+      @Nullable String password,
       List> tables) {
     this.host = host;
     this.port = port;
@@ -93,21 +93,21 @@ public RedisTableFieldInfo getTableFieldInfo(String tableName) {
       if (jsonCustomTable.name.equals(tableName)) {
         Map map =
             requireNonNull(jsonCustomTable.operand, OPERAND);
-        if (isEmptyObject(map.get(DATA_FORMAT))) {
+        final Object dataFormatValue = map.get(DATA_FORMAT);
+        if (dataFormatValue == null
+            || isEmptyObject(dataFormatValue)
+            || RedisDataFormat.fromTypeName(dataFormatValue.toString()) == null) {
           throw new RuntimeException("dataFormat is invalid, it must be raw, csv or json");
         }
-        RedisDataFormat dataFormatEnum =
-            RedisDataFormat.fromTypeName(map.get(DATA_FORMAT).toString());
-        if (dataFormatEnum == null) {
-          throw new RuntimeException("dataFormat is invalid, it must be raw, csv or json");
-        }
-        if (isEmptyObject(map.get(FIELDS))) {
+        final Object fieldsValue = map.get(FIELDS);
+        if (fieldsValue == null || isEmptyObject(fieldsValue)) {
           throw new RuntimeException("fields is null");
         }
-        dataFormat = map.get(DATA_FORMAT).toString();
-        fields = (List>) map.get(FIELDS);
-        if (map.get(KEY_DELIMITER) != null) {
-          keyDelimiter = map.get(KEY_DELIMITER).toString();
+        dataFormat = dataFormatValue.toString();
+        fields = (List>) fieldsValue;
+        final Object keyDelimiterValue = map.get(KEY_DELIMITER);
+        if (keyDelimiterValue != null) {
+          keyDelimiter = keyDelimiterValue.toString();
         }
         break;
       }
diff --git a/redis/src/main/java/org/apache/calcite/adapter/redis/RedisSchemaFactory.java b/redis/src/main/java/org/apache/calcite/adapter/redis/RedisSchemaFactory.java
index 90787b0e541f..c4b5b65542b6 100644
--- a/redis/src/main/java/org/apache/calcite/adapter/redis/RedisSchemaFactory.java
+++ b/redis/src/main/java/org/apache/calcite/adapter/redis/RedisSchemaFactory.java
@@ -20,12 +20,15 @@
 import org.apache.calcite.schema.SchemaFactory;
 import org.apache.calcite.schema.SchemaPlus;
 
+
+import org.jspecify.annotations.Nullable;
 import java.util.List;
 import java.util.Map;
 
 import static com.google.common.base.Preconditions.checkArgument;
 
 import static java.lang.Integer.parseInt;
+import static java.util.Objects.requireNonNull;
 
 /**
  * Factory that creates a {@link RedisSchema}.
@@ -52,11 +55,11 @@ public RedisSchemaFactory() {
 
     @SuppressWarnings("unchecked") List> tables =
         (List) operand.get("tables");
-    String host = operand.get("host").toString();
+    String host = requireNonNull(operand.get("host")).toString();
     int port = (int) operand.get("port");
-    int database = parseInt(operand.get("database").toString());
-    String password = operand.get("password") == null ? null
-        : operand.get("password").toString();
+    int database = parseInt(requireNonNull(operand.get("database")).toString());
+    final Object passwordValue = operand.get("password");
+    String password = passwordValue == null ? null : passwordValue.toString();
     return new RedisSchema(host, port, database, password, tables);
   }
 }
diff --git a/redis/src/main/java/org/apache/calcite/adapter/redis/RedisTable.java b/redis/src/main/java/org/apache/calcite/adapter/redis/RedisTable.java
index c5e562f2af5b..9c7d3288bad8 100644
--- a/redis/src/main/java/org/apache/calcite/adapter/redis/RedisTable.java
+++ b/redis/src/main/java/org/apache/calcite/adapter/redis/RedisTable.java
@@ -44,16 +44,15 @@ public class RedisTable extends AbstractTable
 
   final RedisSchema schema;
   final String tableName;
-  final RelProtoDataType protoRowType;
+  final @Nullable RelProtoDataType protoRowType;
   final ImmutableMap allFields;
   final String dataFormat;
   final RedisConfig redisConfig;
-  RedisEnumerator redisEnumerator;
 
   public RedisTable(
       RedisSchema schema,
       String tableName,
-      RelProtoDataType protoRowType,
+      @Nullable RelProtoDataType protoRowType,
       Map allFields,
       String dataFormat,
       RedisConfig redisConfig) {
@@ -86,7 +85,7 @@ static Table create(
       RedisSchema schema,
       String tableName,
       RedisConfig redisConfig,
-      RelProtoDataType protoRowType) {
+      @Nullable RelProtoDataType protoRowType) {
     RedisTableFieldInfo tableFieldInfo = schema.getTableFieldInfo(tableName);
     Map allFields = RedisEnumerator.deduceRowType(tableFieldInfo);
     return new RedisTable(schema, tableName, protoRowType,
@@ -97,7 +96,7 @@ static Table create(
       RedisSchema schema,
       String tableName,
       Map operand,
-      RelProtoDataType protoRowType) {
+      @Nullable RelProtoDataType protoRowType) {
     RedisConfig redisConfig =
         new RedisConfig(schema.host, schema.port,
             schema.database, schema.password);
@@ -105,10 +104,13 @@ static Table create(
   }
 
   @Override public Enumerable<@Nullable Object[]> scan(DataContext root) {
-    return new AbstractEnumerable() {
-      @Override public Enumerator enumerator() {
-        return new RedisEnumerator(redisConfig, schema, tableName);
-      }
-    };
+    return new RedisEnumerable();
+  }
+
+  /** Enumerable that reads the table's rows from Redis. */
+  private class RedisEnumerable extends AbstractEnumerable<@Nullable Object[]> {
+    @Override public Enumerator<@Nullable Object[]> enumerator() {
+      return new RedisEnumerator(redisConfig, schema, tableName);
+    }
   }
 }
diff --git a/redis/src/main/java/org/apache/calcite/adapter/redis/RedisTableFactory.java b/redis/src/main/java/org/apache/calcite/adapter/redis/RedisTableFactory.java
index 97e5d0938c9c..ad26a62fc2de 100644
--- a/redis/src/main/java/org/apache/calcite/adapter/redis/RedisTableFactory.java
+++ b/redis/src/main/java/org/apache/calcite/adapter/redis/RedisTableFactory.java
@@ -27,6 +27,8 @@
 
 import java.util.Map;
 
+import static java.util.Objects.requireNonNull;
+
 /**
  * Implementation of {@link TableFactory} for Redis.
  *
@@ -42,7 +44,8 @@ private RedisTableFactory() {
   // name that is also the same name as a complex metric
   @Override public Table create(SchemaPlus schema, String tableName, Map operand,
       @Nullable RelDataType rowType) {
-    final RedisSchema redisSchema = schema.unwrap(RedisSchema.class);
+    final RedisSchema redisSchema =
+        requireNonNull(schema.unwrap(RedisSchema.class), "redisSchema");
     final RelProtoDataType protoRowType =
         rowType != null ? RelDataTypeImpl.proto(rowType) : null;
     return RedisTable.create(redisSchema, tableName, operand, protoRowType);
diff --git a/redis/src/main/java/org/apache/calcite/adapter/redis/RedisTableFieldInfo.java b/redis/src/main/java/org/apache/calcite/adapter/redis/RedisTableFieldInfo.java
index 1a6d247f167b..3fa7cb967e97 100644
--- a/redis/src/main/java/org/apache/calcite/adapter/redis/RedisTableFieldInfo.java
+++ b/redis/src/main/java/org/apache/calcite/adapter/redis/RedisTableFieldInfo.java
@@ -21,7 +21,11 @@
 
 /**
  * get the redis table's field info.
+ *
+ * 
      {@link RedisSchema#getTableFieldInfo} sets every field right after
+ * construction, which is why they are not initialized here.
  */
+@SuppressWarnings("NullAway.Init")
 public class RedisTableFieldInfo {
   private String tableName;
   private String dataFormat;
diff --git a/redis/src/main/java/org/apache/calcite/adapter/redis/package-info.java b/redis/src/main/java/org/apache/calcite/adapter/redis/package-info.java
index 4fbe0b1c39af..04594e255ac0 100644
--- a/redis/src/main/java/org/apache/calcite/adapter/redis/package-info.java
+++ b/redis/src/main/java/org/apache/calcite/adapter/redis/package-info.java
@@ -18,4 +18,7 @@
 /**
  * Redis adapter.
  */
+@NullMarked
 package org.apache.calcite.adapter.redis;
+
+import org.jspecify.annotations.NullMarked;
diff --git a/redis/src/main/java/redis/embedded/util/package-info.java b/redis/src/main/java/redis/embedded/util/package-info.java
index 5e31ad3901b5..e2852387b66d 100644
--- a/redis/src/main/java/redis/embedded/util/package-info.java
+++ b/redis/src/main/java/redis/embedded/util/package-info.java
@@ -18,4 +18,7 @@
 /**
  * Utility classes for embedded Redis.
  */
+@NullMarked
 package redis.embedded.util;
+
+import org.jspecify.annotations.NullMarked;

From 800b91c3b31d9204200e3f484b83809e77f9db00 Mon Sep 17 00:00:00 2001
From: Vladimir Sitnikov 
Date: Tue, 25 Aug 2026 20:59:17 +0300
Subject: [PATCH 546/562] [CALCITE-7736] Put :splunk under nullness
 verification

The push-down rule builds a search string from whichever of the two projections
and the two row types the match happened to have, so those parameters are
optional and the signature says so. A literal that is neither numeric nor CHAR
yields no search text, which is how getFilter already reads the result.

SplunkResultEnumerator reads the CSV header in its constructor, and a header it
could not read leaves the field names absent; moveNext now stops instead of
dereferencing them. close() swallowed the NullPointerException it raised on a
null Closeable, and returns early instead.

Co-Authored-By: Claude Opus 5 
---
 build.gradle.kts                               |  2 +-
 .../adapter/splunk/SplunkPushDownRule.java     | 18 +++++++++++-------
 .../adapter/splunk/SplunkTableScan.java        | 17 ++++++++++-------
 .../calcite/adapter/splunk/package-info.java   |  3 +++
 .../splunk/search/SplunkConnectionImpl.java    | 15 ++++++++++-----
 .../adapter/splunk/search/package-info.java    |  3 +++
 .../adapter/splunk/util/package-info.java      |  3 +++
 7 files changed, 41 insertions(+), 20 deletions(-)

diff --git a/build.gradle.kts b/build.gradle.kts
index 15bf1ecb8945..f7fe1e850d07 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -94,7 +94,7 @@ val werror by props(true) // treat javac warnings as errors
 
 // Projects whose main code NullAway verifies. The other projects are not annotated well enough
 // yet, so NullAway would only produce noise there.
-val nullawayProjects = listOf(":linq4j", ":core", ":server", ":druid", ":file", ":kafka", ":spark", ":babel", ":redis")
+val nullawayProjects = listOf(":linq4j", ":core", ":server", ":druid", ":file", ":kafka", ":spark", ":babel", ":redis", ":splunk")
 
 val hepLargePlanModeTestIncludes = mapOf(
     ":core" to listOf(
diff --git a/splunk/src/main/java/org/apache/calcite/adapter/splunk/SplunkPushDownRule.java b/splunk/src/main/java/org/apache/calcite/adapter/splunk/SplunkPushDownRule.java
index 44400635ebb2..af514dfd1199 100644
--- a/splunk/src/main/java/org/apache/calcite/adapter/splunk/SplunkPushDownRule.java
+++ b/splunk/src/main/java/org/apache/calcite/adapter/splunk/SplunkPushDownRule.java
@@ -43,12 +43,15 @@
 import com.google.common.collect.ImmutableSet;
 
 import org.immutables.value.Value;
+import org.jspecify.annotations.Nullable;
 import org.slf4j.Logger;
 
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Set;
 
+import static java.util.Objects.requireNonNull;
+
 /**
  * Planner rule to push filters and projections to Splunk.
  */
@@ -210,10 +213,10 @@ protected SplunkPushDownRule(RelOptRuleOperand operand,
   protected RelNode appendSearchString(
       String toAppend,
       SplunkTableScan splunkRel,
-      LogicalProject topProj,
-      LogicalProject bottomProj,
-      RelDataType topRow,
-      RelDataType bottomRow) {
+      @Nullable LogicalProject topProj,
+      @Nullable LogicalProject bottomProj,
+      @Nullable RelDataType topRow,
+      @Nullable RelDataType bottomRow) {
     final RelOptCluster cluster = splunkRel.getCluster();
     StringBuilder updateSearchStr = new StringBuilder(splunkRel.search);
 
@@ -252,6 +255,7 @@ protected RelNode appendSearchString(
     // handle top projection (ie reordering and renaming)
     List newFields = bottomFields;
     if (topProj != null) {
+      requireNonNull(topFields, "topFields");
       LOGGER.debug("topProj: {}", topProj.getPermutation());
       newFields = new ArrayList<>();
       int i = 0;
@@ -415,13 +419,13 @@ public static String searchEscape(String str) {
     return str;
   }
 
-  private static String toString(boolean like, RexLiteral literal) {
+  private static @Nullable String toString(boolean like, RexLiteral literal) {
     String value = null;
     SqlTypeName litSqlType = literal.getTypeName();
     if (SqlTypeName.NUMERIC_TYPES.contains(litSqlType)) {
-      value = literal.getValue().toString();
+      value = requireNonNull(literal.getValue(), "literal value").toString();
     } else if (litSqlType == SqlTypeName.CHAR) {
-      value = ((NlsString) literal.getValue()).getValue();
+      value = ((NlsString) requireNonNull(literal.getValue(), "literal value")).getValue();
       if (like) {
         value = value.replace("%", "*");
       }
diff --git a/splunk/src/main/java/org/apache/calcite/adapter/splunk/SplunkTableScan.java b/splunk/src/main/java/org/apache/calcite/adapter/splunk/SplunkTableScan.java
index 996f4fe3c1f2..1cbef25c1620 100644
--- a/splunk/src/main/java/org/apache/calcite/adapter/splunk/SplunkTableScan.java
+++ b/splunk/src/main/java/org/apache/calcite/adapter/splunk/SplunkTableScan.java
@@ -39,6 +39,8 @@
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 
+
+import org.jspecify.annotations.Nullable;
 import java.lang.reflect.Method;
 import java.util.AbstractList;
 import java.util.Arrays;
@@ -61,8 +63,8 @@ public class SplunkTableScan
     implements EnumerableRel {
   final SplunkTable splunkTable;
   final String search;
-  final String earliest;
-  final String latest;
+  final @Nullable String earliest;
+  final @Nullable String latest;
   final List fieldList;
 
   protected SplunkTableScan(
@@ -70,8 +72,8 @@ protected SplunkTableScan(
       RelOptTable table,
       SplunkTable splunkTable,
       String search,
-      String earliest,
-      String latest,
+      @Nullable String earliest,
+      @Nullable String latest,
       List fieldList) {
     super(
         cluster,
@@ -105,7 +107,8 @@ protected SplunkTableScan(
         getCluster().getTypeFactory().builder();
     for (String field : fieldList) {
       // REVIEW: is case-sensitive match what we want here?
-      builder.add(table.getRowType().getField(field, true, false));
+      builder.add(
+          requireNonNull(table.getRowType().getField(field, true, false), field));
     }
     return builder.build();
   }
@@ -122,8 +125,8 @@ protected SplunkTableScan(
   @Override public Result implement(EnumerableRelImplementor implementor, Prefer pref) {
     Map map = ImmutableMap.builder()
         .put("search", search)
-        .put("earliest", Util.first(earliest, ""))
-        .put("latest", Util.first(latest, ""))
+        .put("earliest", Util.firstNonNull(earliest, ""))
+        .put("latest", Util.firstNonNull(latest, ""))
         .put("fieldList", fieldList)
         .build();
     if (CalciteSystemProperty.DEBUG.value()) {
diff --git a/splunk/src/main/java/org/apache/calcite/adapter/splunk/package-info.java b/splunk/src/main/java/org/apache/calcite/adapter/splunk/package-info.java
index cfead242ef1e..930f6f7992d2 100644
--- a/splunk/src/main/java/org/apache/calcite/adapter/splunk/package-info.java
+++ b/splunk/src/main/java/org/apache/calcite/adapter/splunk/package-info.java
@@ -22,4 +22,7 @@
  * "host", "index", "source", "sourcetype". It has a variable type, so other
  * fields are held in a map field called "_others".
  */
+@NullMarked
 package org.apache.calcite.adapter.splunk;
+
+import org.jspecify.annotations.NullMarked;
diff --git a/splunk/src/main/java/org/apache/calcite/adapter/splunk/search/SplunkConnectionImpl.java b/splunk/src/main/java/org/apache/calcite/adapter/splunk/search/SplunkConnectionImpl.java
index 1ff97d5a36e2..baddab789e91 100644
--- a/splunk/src/main/java/org/apache/calcite/adapter/splunk/search/SplunkConnectionImpl.java
+++ b/splunk/src/main/java/org/apache/calcite/adapter/splunk/search/SplunkConnectionImpl.java
@@ -68,7 +68,7 @@ public class SplunkConnectionImpl implements SplunkConnection {
   final URL url;
   final String username;
   final String password;
-  String sessionKey;
+  @Nullable String sessionKey;
   final Map requestHeaders = new HashMap<>();
 
   public SplunkConnectionImpl(String url, String username, String password)
@@ -83,7 +83,10 @@ public SplunkConnectionImpl(URL url, String username, String password) {
     connect();
   }
 
-  private static void close(Closeable c) {
+  private static void close(@Nullable Closeable c) {
+    if (c == null) {
+      return;
+    }
     try {
       c.close();
     } catch (Exception ignore) {
@@ -392,18 +395,20 @@ public SplunkResultEnumerator(InputStream in,
     }
 
     @Override public Object current() {
-      return current;
+      return requireNonNull(current, "current");
     }
 
     @Override public boolean moveNext() {
       try {
+        final String[] fieldNames = this.fieldNames;
         String[] line;
-        while ((line = csvReader.readNext()) != null) {
+        while (fieldNames != null && (line = csvReader.readNext()) != null) {
           if (line.length == fieldNames.length) {
             switch (source) {
             case -3:
               // Re-map using sources
-              String[] mapped = new String[sources.length];
+              final int[] sources = requireNonNull(this.sources, "sources");
+              final @Nullable String[] mapped = new String[sources.length];
               for (int i = 0; i < sources.length; i++) {
                 int source1 = sources[i];
                 mapped[i] = source1 < 0 ? null : line[source1];
diff --git a/splunk/src/main/java/org/apache/calcite/adapter/splunk/search/package-info.java b/splunk/src/main/java/org/apache/calcite/adapter/splunk/search/package-info.java
index c9c40b31a43c..7815db7a35eb 100644
--- a/splunk/src/main/java/org/apache/calcite/adapter/splunk/search/package-info.java
+++ b/splunk/src/main/java/org/apache/calcite/adapter/splunk/search/package-info.java
@@ -18,4 +18,7 @@
 /**
  * Executes queries via Splunk's REST API.
  */
+@NullMarked
 package org.apache.calcite.adapter.splunk.search;
+
+import org.jspecify.annotations.NullMarked;
diff --git a/splunk/src/main/java/org/apache/calcite/adapter/splunk/util/package-info.java b/splunk/src/main/java/org/apache/calcite/adapter/splunk/util/package-info.java
index cc1ddc1e5e71..731f16ac2148 100644
--- a/splunk/src/main/java/org/apache/calcite/adapter/splunk/util/package-info.java
+++ b/splunk/src/main/java/org/apache/calcite/adapter/splunk/util/package-info.java
@@ -18,4 +18,7 @@
 /**
  * Utilities for RPC to Splunk.
  */
+@NullMarked
 package org.apache.calcite.adapter.splunk.util;
+
+import org.jspecify.annotations.NullMarked;

From dd2041f0d9a755c13c312679378d0df7a0c9cbcf Mon Sep 17 00:00:00 2001
From: Vladimir Sitnikov 
Date: Tue, 25 Aug 2026 21:08:28 +0300
Subject: [PATCH 547/562] [CALCITE-7736] Put :mongodb under nullness
 verification

A MongoDB document has no value for a field it does not carry, so a projection
of one field enumerates nulls, and the getter, the enumerator and the enumerable
that carry it now say so. That needed the element type of AbstractEnumerable to
admit null, which Enumerable and Queryable have admitted all along; the four
interfaces between them said otherwise, and now agree.

The filter translator builds its documents with JsonBuilder, whose maps and
lists hold absent values, and it passes a null operator to mean equality, which
translateOp2 has always read that way.

The two anonymous Enumerables in MongoTable are named classes, to avoid
uber/NullAway#1746.

Co-Authored-By: Claude Opus 5 
---
 build.gradle.kts                              |  2 +-
 .../calcite/linq4j/AbstractEnumerable.java    |  5 +-
 .../calcite/linq4j/DefaultEnumerable.java     |  3 +-
 .../calcite/linq4j/ExtendedEnumerable.java    |  2 +-
 .../linq4j/ExtendedOrderedEnumerable.java     |  5 +-
 .../calcite/linq4j/OrderedEnumerable.java     |  4 +-
 .../apache/calcite/linq4j/RawEnumerable.java  |  4 +-
 .../adapter/mongodb/MongoEnumerator.java      | 25 ++---
 .../calcite/adapter/mongodb/MongoFilter.java  | 66 +++++++------
 .../calcite/adapter/mongodb/MongoProject.java |  4 +-
 .../calcite/adapter/mongodb/MongoRules.java   |  6 +-
 .../adapter/mongodb/MongoSchemaFactory.java   |  8 +-
 .../calcite/adapter/mongodb/MongoSort.java    | 10 +-
 .../calcite/adapter/mongodb/MongoTable.java   | 99 +++++++++++++------
 .../mongodb/MongoToEnumerableConverter.java   | 10 +-
 .../calcite/adapter/mongodb/package-info.java |  3 +
 16 files changed, 168 insertions(+), 88 deletions(-)

diff --git a/build.gradle.kts b/build.gradle.kts
index f7fe1e850d07..3213720c0afc 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -94,7 +94,7 @@ val werror by props(true) // treat javac warnings as errors
 
 // Projects whose main code NullAway verifies. The other projects are not annotated well enough
 // yet, so NullAway would only produce noise there.
-val nullawayProjects = listOf(":linq4j", ":core", ":server", ":druid", ":file", ":kafka", ":spark", ":babel", ":redis", ":splunk")
+val nullawayProjects = listOf(":linq4j", ":core", ":server", ":druid", ":file", ":kafka", ":spark", ":babel", ":redis", ":splunk", ":mongodb")
 
 val hepLargePlanModeTestIncludes = mapOf(
     ":core" to listOf(
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/AbstractEnumerable.java b/linq4j/src/main/java/org/apache/calcite/linq4j/AbstractEnumerable.java
index 1a798704d12d..ac9629b1ef25 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/AbstractEnumerable.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/AbstractEnumerable.java
@@ -16,6 +16,8 @@
  */
 package org.apache.calcite.linq4j;
 
+import org.jspecify.annotations.Nullable;
+
 import java.util.Iterator;
 
 /**
@@ -28,7 +30,8 @@
  *
  * @param  Element type
  */
-public abstract class AbstractEnumerable extends DefaultEnumerable {
+public abstract class AbstractEnumerable
+    extends DefaultEnumerable {
   @Override public Iterator iterator() {
     return Linq4j.enumeratorIterator(enumerator());
   }
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/DefaultEnumerable.java b/linq4j/src/main/java/org/apache/calcite/linq4j/DefaultEnumerable.java
index f2d2741ea132..e3b9e8f3637b 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/DefaultEnumerable.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/DefaultEnumerable.java
@@ -54,7 +54,8 @@
  *
  * @param  Element type
  */
-public abstract class DefaultEnumerable implements OrderedEnumerable {
+public abstract class DefaultEnumerable
+    implements OrderedEnumerable {
 
   /**
    * Derived classes might wish to override this method to return the "outer"
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedEnumerable.java b/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedEnumerable.java
index 171b3e7d9303..be0169e35571 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedEnumerable.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedEnumerable.java
@@ -48,7 +48,7 @@
  *
  * @param  Element type
  */
-public interface ExtendedEnumerable {
+public interface ExtendedEnumerable {
 
   /**
    * Performs an operation for each member of this enumeration.
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedOrderedEnumerable.java b/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedOrderedEnumerable.java
index 852b8c671c59..82e86f9ebe5d 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedOrderedEnumerable.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedOrderedEnumerable.java
@@ -16,6 +16,8 @@
  */
 package org.apache.calcite.linq4j;
 
+import org.jspecify.annotations.Nullable;
+
 import org.apache.calcite.linq4j.function.Function1;
 
 import java.util.Comparator;
@@ -25,7 +27,8 @@
  *
  * @param  Element type
  */
-public interface ExtendedOrderedEnumerable extends Enumerable {
+public interface ExtendedOrderedEnumerable
+    extends Enumerable {
   /**
    * Performs a subsequent ordering of the elements in an
    * {@link OrderedEnumerable} according to a key, using a specified
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/OrderedEnumerable.java b/linq4j/src/main/java/org/apache/calcite/linq4j/OrderedEnumerable.java
index 1634f23412d0..03cd06c7ef39 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/OrderedEnumerable.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/OrderedEnumerable.java
@@ -16,12 +16,14 @@
  */
 package org.apache.calcite.linq4j;
 
+import org.jspecify.annotations.Nullable;
+
 /**
  * Represents the result of applying a sorting operation to an
  * {@link org.apache.calcite.linq4j.Enumerable}.
  *
  * @param  element type
  */
-public interface OrderedEnumerable
+public interface OrderedEnumerable
     extends Enumerable, ExtendedOrderedEnumerable {
 }
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/RawEnumerable.java b/linq4j/src/main/java/org/apache/calcite/linq4j/RawEnumerable.java
index 18d6c701cd7b..9f83e9eab29f 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/RawEnumerable.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/RawEnumerable.java
@@ -16,6 +16,8 @@
  */
 package org.apache.calcite.linq4j;
 
+import org.jspecify.annotations.Nullable;
+
 /**
  * Exposes the enumerator, which supports a simple iteration over a collection,
  * without the extension methods.
@@ -29,7 +31,7 @@
  * @param  Element type
  * @see Enumerable
  */
-public interface RawEnumerable {
+public interface RawEnumerable {
   /**
    * Returns an enumerator that iterates through a collection.
    */
diff --git a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoEnumerator.java b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoEnumerator.java
index b8de7e0597b4..8eb7f85efed1 100644
--- a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoEnumerator.java
+++ b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoEnumerator.java
@@ -39,10 +39,12 @@
 
 import static java.lang.String.format;
 
+import static java.util.Objects.requireNonNull;
+
 /** Enumerator that reads from a MongoDB collection. */
-class MongoEnumerator implements Enumerator {
+class MongoEnumerator implements Enumerator<@Nullable Object> {
   private final Iterator cursor;
-  private final Function1 getter;
+  private final Function1 getter;
   private @Nullable Object current;
 
   /** Creates a MongoEnumerator.
@@ -51,7 +53,7 @@ class MongoEnumerator implements Enumerator {
    * @param getter Converts an object into a list of fields
    */
   MongoEnumerator(Iterator cursor,
-      Function1 getter) {
+      Function1 getter) {
     this.cursor = cursor;
     this.getter = getter;
   }
@@ -95,7 +97,7 @@ static Function1 mapGetter() {
   }
 
   /** Returns a function that projects a single field. */
-  static Function1 singletonGetter(final String fieldName,
+  static Function1 singletonGetter(final String fieldName,
       final Class fieldClass) {
     return a0 -> convert(fieldName, a0.get(fieldName), fieldClass);
   }
@@ -104,10 +106,10 @@ static Function1 singletonGetter(final String fieldName,
    *
    * @param fields List of fields to project; or null to return map
    */
-  static Function1 listGetter(
+  static Function1 listGetter(
       final List> fields) {
     return a0 -> {
-      Object[] objects = new Object[fields.size()];
+      final @Nullable Object[] objects = new Object[fields.size()];
       for (int i = 0; i < fields.size(); i++) {
         final Map.Entry field = fields.get(i);
         final String name = field.getKey();
@@ -117,8 +119,8 @@ static Function1 listGetter(
     };
   }
 
-  static Function1 getter(
-      List> fields) {
+  static Function1 getter(
+      @Nullable List> fields) {
     //noinspection unchecked
     return fields == null
         ? (Function1) mapGetter()
@@ -162,13 +164,14 @@ static Function1 getter(
    * 
    */
   @SuppressWarnings("JavaUtilDate")
-  private static Object convert(String fieldName, Object o, Class clazz) {
-    if (o == null) {
+  private static @Nullable Object convert(String fieldName, @Nullable Object o,
+      @Nullable Class clazz) {
+    if (o == null || clazz == null) {
       return null;
     }
     Primitive primitive = Primitive.of(clazz);
     if (primitive != null) {
-      clazz = primitive.boxClass;
+      clazz = requireNonNull(primitive.boxClass, "boxClass");
     } else {
       primitive = Primitive.ofBox(clazz);
     }
diff --git a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoFilter.java b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoFilter.java
index ad1330c02246..8e5154a6c326 100644
--- a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoFilter.java
+++ b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoFilter.java
@@ -48,6 +48,8 @@
 import java.util.List;
 import java.util.Map;
 
+import static java.util.Objects.requireNonNull;
+
 /**
  * Implementation of a {@link org.apache.calcite.rel.core.Filter}
  * relational expression in MongoDB.
@@ -65,7 +67,7 @@ public MongoFilter(
 
   @Override public @Nullable RelOptCost computeSelfCost(RelOptPlanner planner,
       RelMetadataQuery mq) {
-    return super.computeSelfCost(planner, mq).multiplyBy(0.1);
+    return requireNonNull(super.computeSelfCost(planner, mq)).multiplyBy(0.1);
   }
 
   @Override public MongoFilter copy(RelTraitSet traitSet, RelNode input,
@@ -94,16 +96,16 @@ static class Translator {
     }
 
     private String translateMatch(RexNode condition) {
-      Map map = builder.map();
+      Map map = builder.map();
       map.put("$match", translateOr(condition));
       return builder.toJsonString(map);
     }
 
-    private Map translateOr(RexNode condition) {
+    private Map translateOr(RexNode condition) {
       final RexNode condition2 =
           RexUtil.expandSearch(rexBuilder, null, condition);
 
-      List> list = new ArrayList<>();
+      List> list = new ArrayList<>();
       for (RexNode node : RelOptUtil.disjunctions(condition2)) {
         list.add(translateAnd(node));
       }
@@ -111,7 +113,7 @@ private Map translateOr(RexNode condition) {
       case 1:
         return list.get(0);
       default:
-        Map map = builder.map();
+        Map map = builder.map();
         map.put("$or", list);
         return map;
       }
@@ -119,34 +121,34 @@ private Map translateOr(RexNode condition) {
 
     /** Translates a condition that may be an AND of other conditions. Gathers
      * together conditions that apply to the same field. */
-    private Map translateAnd(RexNode node0) {
+    private Map translateAnd(RexNode node0) {
       final Multimap> multimap =
           HashMultimap.create();
       final Map eqMap =
           new LinkedHashMap<>();
-      final List> orMapList = new ArrayList<>();
+      final List> orMapList = new ArrayList<>();
       for (RexNode node : RelOptUtil.conjunctions(node0)) {
         translateMatch2(node, orMapList, multimap, eqMap);
       }
-      Map map = builder.map();
+      Map map = builder.map();
       for (Map.Entry entry : eqMap.entrySet()) {
         multimap.removeAll(entry.getKey());
         map.put(entry.getKey(), literalValue(entry.getValue()));
       }
       for (Map.Entry>> entry
           : multimap.asMap().entrySet()) {
-        Map map2 = builder.map();
+        Map map2 = builder.map();
         for (Pair s : entry.getValue()) {
           String op = s.left;
           if ("$ne".equals(op))  {
             if (map2.containsKey("$nin")) {
               map2.computeIfPresent("$nin", (k, v) -> {
-                ((List) v).add(literalValue(s.right));
+                ((List<@Nullable Object>) v).add(literalValue(s.right));
                 return v;
               });
             } else if (map2.containsKey(op)) {
               // if two $ne conditions, translate to $nin op
-              List ninList = builder.list();
+              List<@Nullable Object> ninList = builder.list();
               ninList.add(map2.remove(op));
               ninList.add(literalValue(s.right));
               map2.put("$nin", ninList);
@@ -161,7 +163,7 @@ private Map translateAnd(RexNode node0) {
         map.put(entry.getKey(), map2);
       }
       if (!orMapList.isEmpty()) {
-        Map andMap = builder.map();
+        Map andMap = builder.map();
         if (!map.isEmpty()) {
           orMapList.add(map);
         }
@@ -171,7 +173,8 @@ private Map translateAnd(RexNode node0) {
       return map;
     }
 
-    private static void addPredicate(Map map, String op, Object v) {
+    private static void addPredicate(Map map, String op,
+        @Nullable Object v) {
       if (map.containsKey(op) && stronger(op, map.get(op), v)) {
         return;
       }
@@ -184,7 +187,7 @@ private static void addPredicate(Map map, String op, Object v) {
      * 
      For example, {@code stronger("$lt", 100, 200)} returns true, because
      * "< 100" is a more powerful condition than "< 200".
      */
-    private static boolean stronger(String key, Object v0, Object v1) {
+    private static boolean stronger(String key, @Nullable Object v0, @Nullable Object v1) {
       if (key.equals("$lt") || key.equals("$lte")) {
         if (v0 instanceof Number && v1 instanceof Number) {
           return ((Number) v0).doubleValue() < ((Number) v1).doubleValue();
@@ -199,11 +202,11 @@ private static boolean stronger(String key, Object v0, Object v1) {
       return false;
     }
 
-    private static Object literalValue(RexLiteral literal) {
+    private static @Nullable Object literalValue(RexLiteral literal) {
       return literal.getValue2();
     }
 
-    private Void translateMatch2(RexNode node, List> orMapList,
+    private Void translateMatch2(RexNode node, List> orMapList,
         Multimap> multimap, Map eqMap) {
       switch (node.getKind()) {
       case EQUALS:
@@ -233,15 +236,15 @@ private Void translateMatch2(RexNode node, List> orMapList,
       }
     }
 
-    private Void translateOrAddToList(RexNode node, List> orMapList) {
-      Map or = translateOr(node);
+    private Void translateOrAddToList(RexNode node, List> orMapList) {
+      Map or = translateOr(node);
       orMapList.add(or);
       return null;
     }
 
     /** Translates a call to a binary operator, reversing arguments if
      * necessary. */
-    private Void translateBinary(String op, String rop, RexCall call,
+    private @Nullable Void translateBinary(@Nullable String op, @Nullable String rop, RexCall call,
         Multimap> multimap, Map eqMap) {
       final RexNode left = call.operands.get(0);
       final RexNode right = call.operands.get(1);
@@ -257,7 +260,7 @@ private Void translateBinary(String op, String rop, RexCall call,
     }
 
     /** Translates a call to a binary operator. Returns whether successful. */
-    private boolean translateBinary2(String op, RexNode left, RexNode right,
+    private boolean translateBinary2(@Nullable String op, RexNode left, RexNode right,
         Multimap> multimap, Map eqMap) {
       switch (right.getKind()) {
       case LITERAL:
@@ -286,7 +289,7 @@ private boolean translateBinary2(String op, RexNode left, RexNode right,
       }
     }
 
-    private static void translateOp2(String op, String name, RexLiteral right,
+    private static void translateOp2(@Nullable String op, String name, RexLiteral right,
         Multimap> multimap, Map eqMap) {
       if (op == null) {
         // E.g.: {deptno: 100}
@@ -320,7 +323,8 @@ private Void translateLike(RexCall call,
         throw new AssertionError("cannot translate LIKE with non-literal pattern: " + call);
       }
       final RexLiteral patternLiteral = (RexLiteral) right;
-      final String sqlPattern = patternLiteral.getValue2().toString();
+      final String sqlPattern =
+          requireNonNull(patternLiteral.getValue2(), "pattern").toString();
 
       final @Nullable String escapeStr = escapeStr(call);
       final String finalRegex = Like.sqlToRegexAnchored(sqlPattern, escapeStr);
@@ -344,7 +348,7 @@ private Void translateLike(RexCall call,
     }
 
     /** Translates NOT to a MongoDB $nor expression. */
-    private Void translateNot(RexCall call, List> orMapList) {
+    private Void translateNot(RexCall call, List> orMapList) {
       final RexNode operand = call.operands.get(0);
       switch (operand.getKind()) {
       case LIKE:
@@ -355,7 +359,7 @@ private Void translateNot(RexCall call, List> orMapList) {
     }
 
     /** Translates NOT LIKE to {$nor: [{field: {$regex: ...}}]}. */
-    private Void translateNotLike(RexCall call, List> orMapList) {
+    private Void translateNotLike(RexCall call, List> orMapList) {
       final RexNode left = stripCast(call.operands.get(0));
       final RexNode right = call.operands.get(1);
 
@@ -363,7 +367,8 @@ private Void translateNotLike(RexCall call, List> orMapList)
         throw new AssertionError("cannot translate NOT LIKE with non-literal pattern: " + call);
       }
       final RexLiteral patternLiteral = (RexLiteral) right;
-      final String sqlPattern = patternLiteral.getValue2().toString();
+      final String sqlPattern =
+          requireNonNull(patternLiteral.getValue2(), "pattern").toString();
 
       final @Nullable String escapeStr = escapeStr(call);
       final String finalRegex = Like.sqlToRegexAnchored(sqlPattern, escapeStr);
@@ -385,13 +390,13 @@ private Void translateNotLike(RexCall call, List> orMapList)
         throw new AssertionError("cannot translate NOT LIKE " + call);
       }
 
-      Map regexMap = builder.map();
-      Map regexOp = builder.map();
+      Map regexMap = builder.map();
+      Map regexOp = builder.map();
       regexOp.put("$regex", finalRegex);
       regexMap.put(name, regexOp);
-      List norList = builder.list();
+      List<@Nullable Object> norList = builder.list();
       norList.add(regexMap);
-      Map norMap = builder.map();
+      Map norMap = builder.map();
       norMap.put("$nor", norList);
       orMapList.add(norMap);
       return null;
@@ -414,7 +419,8 @@ private static RexNode stripCast(RexNode node) {
       if (escapeNode.getKind() != SqlKind.LITERAL) {
         throw new AssertionError("cannot translate LIKE with non-literal escape: " + call);
       }
-      final String escape = ((RexLiteral) escapeNode).getValue2().toString();
+      final String escape =
+          requireNonNull(((RexLiteral) escapeNode).getValue2(), "escape").toString();
       if (escape.length() != 1) {
         throw new AssertionError("cannot translate LIKE with multi-character escape: " + call);
       }
diff --git a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoProject.java b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoProject.java
index 79f060b006a1..bedf2f53aa7d 100644
--- a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoProject.java
+++ b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoProject.java
@@ -35,6 +35,8 @@
 import org.jspecify.annotations.Nullable;
 
 import java.util.ArrayList;
+
+import static java.util.Objects.requireNonNull;
 import java.util.List;
 
 /**
@@ -64,7 +66,7 @@ public MongoProject(RelOptCluster cluster, RelTraitSet traitSet,
 
   @Override public @Nullable RelOptCost computeSelfCost(RelOptPlanner planner,
       RelMetadataQuery mq) {
-    return super.computeSelfCost(planner, mq).multiplyBy(0.1);
+    return requireNonNull(super.computeSelfCost(planner, mq)).multiplyBy(0.1);
   }
 
   @Override public void implement(Implementor implementor) {
diff --git a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoRules.java b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoRules.java
index fe7c65db1059..a3120f15910c 100644
--- a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoRules.java
+++ b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoRules.java
@@ -48,6 +48,8 @@
 
 import org.slf4j.Logger;
 
+
+import org.jspecify.annotations.Nullable;
 import java.util.AbstractList;
 import java.util.HashMap;
 import java.util.List;
@@ -72,7 +74,7 @@ private MongoRules() {}
   };
 
   /** Returns 'string' if it is a call to item['string'], null otherwise. */
-  static String isItem(RexCall call) {
+  static @Nullable String isItem(RexCall call) {
     if (call.getOperator() != SqlStdOperatorTable.ITEM) {
       return null;
     }
@@ -514,7 +516,7 @@ private static class MongoAggregateRule extends MongoConverterRule {
       super(config);
     }
 
-    @Override public RelNode convert(RelNode rel) {
+    @Override public @Nullable RelNode convert(RelNode rel) {
       final LogicalAggregate agg = (LogicalAggregate) rel;
       final RelTraitSet traitSet =
           agg.getTraitSet().replace(out);
diff --git a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoSchemaFactory.java b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoSchemaFactory.java
index b581d1d396fa..2e633082f2ed 100644
--- a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoSchemaFactory.java
+++ b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoSchemaFactory.java
@@ -25,8 +25,12 @@
 import com.mongodb.MongoClientSettings;
 import com.mongodb.MongoCredential;
 
+
+import org.jspecify.annotations.Nullable;
 import java.util.Map;
 
+import static java.util.Objects.requireNonNull;
+
 /**
  * Factory that creates a {@link MongoSchema}.
  *
@@ -40,7 +44,7 @@ public MongoSchemaFactory() {
   @Override public Schema create(SchemaPlus parentSchema, String name,
       Map operand) {
     final String host = (String) operand.get("host");
-    final String database = (String) operand.get("database");
+    final String database = requireNonNull((String) operand.get("database"), "database");
     final String authMechanismName = (String) operand.get("authMechanism");
 
     final MongoClientSettings.Builder settings =
@@ -61,7 +65,7 @@ private static MongoCredential createCredential(Map map) {
         AuthenticationMechanism.fromMechanismName(authMechanismName);
     final String username = (String) map.get("username");
     final String authDatabase = (String) map.get("authDatabase");
-    final String password = (String) map.get("password");
+    final String password = requireNonNull((String) map.get("password"), "password");
 
     switch (authenticationMechanism) {
     case PLAIN:
diff --git a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoSort.java b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoSort.java
index 45d7d7e2cc03..b6d1c9de4b7e 100644
--- a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoSort.java
+++ b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoSort.java
@@ -33,6 +33,8 @@
 import org.jspecify.annotations.Nullable;
 
 import java.util.ArrayList;
+
+import static java.util.Objects.requireNonNull;
 import java.util.List;
 
 /**
@@ -41,7 +43,8 @@
 */
 public class MongoSort extends Sort implements MongoRel {
   public MongoSort(RelOptCluster cluster, RelTraitSet traitSet,
-      RelNode child, RelCollation collation, RexNode offset, RexNode fetch) {
+      RelNode child, RelCollation collation, @Nullable RexNode offset,
+      @Nullable RexNode fetch) {
     super(cluster, traitSet, child, collation, offset, fetch);
     assert getConvention() == MongoRel.CONVENTION;
     assert getConvention() == child.getConvention();
@@ -49,11 +52,12 @@ public MongoSort(RelOptCluster cluster, RelTraitSet traitSet,
 
   @Override public @Nullable RelOptCost computeSelfCost(RelOptPlanner planner,
       RelMetadataQuery mq) {
-    return super.computeSelfCost(planner, mq).multiplyBy(0.05);
+    return requireNonNull(super.computeSelfCost(planner, mq)).multiplyBy(0.05);
   }
 
   @Override public Sort copy(RelTraitSet traitSet, RelNode input,
-      RelCollation newCollation, RexNode offset, RexNode fetch) {
+      RelCollation newCollation, @Nullable RexNode offset,
+      @Nullable RexNode fetch) {
     return new MongoSort(getCluster(), traitSet, input, collation, offset,
         fetch);
   }
diff --git a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoTable.java b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoTable.java
index 4a5694279bc0..3852f34748b4 100644
--- a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoTable.java
+++ b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoTable.java
@@ -42,9 +42,13 @@
 import org.bson.Document;
 import org.bson.conversions.Bson;
 
+
+import org.jspecify.annotations.Nullable;
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
+
+import static java.util.Objects.requireNonNull;
 import java.util.Map;
 
 /**
@@ -97,22 +101,41 @@ public class MongoTable extends AbstractQueryableTable
    * @param fields List of fields to project; or null to return map
    * @return Enumerator of results
    */
-  private Enumerable find(MongoDatabase mongoDb, String filterJson,
-      String projectJson, List> fields) {
+  private Enumerable<@Nullable Object> find(MongoDatabase mongoDb,
+      @Nullable String filterJson, @Nullable String projectJson,
+      @Nullable List> fields) {
     final MongoCollection collection =
         mongoDb.getCollection(collectionName);
     final Bson filter =
         filterJson == null ? null : BsonDocument.parse(filterJson);
     final Bson project =
         projectJson == null ? null : BsonDocument.parse(projectJson);
-    final Function1 getter = MongoEnumerator.getter(fields);
-    return new AbstractEnumerable() {
-      @Override public Enumerator enumerator() {
-        @SuppressWarnings("unchecked") final FindIterable cursor =
-            collection.find(filter).projection(project);
-        return new MongoEnumerator(cursor.iterator(), getter);
-      }
-    };
+    final Function1 getter =
+        MongoEnumerator.getter(fields);
+    return new FindEnumerable(collection, filter, project, getter);
+  }
+
+  /** Enumerable over the results of a "find" operation. */
+  private static class FindEnumerable
+      extends AbstractEnumerable<@Nullable Object> {
+    private final MongoCollection collection;
+    private final @Nullable Bson filter;
+    private final @Nullable Bson project;
+    private final Function1 getter;
+
+    FindEnumerable(MongoCollection collection, @Nullable Bson filter,
+        @Nullable Bson project, Function1 getter) {
+      this.collection = collection;
+      this.filter = filter;
+      this.project = project;
+      this.getter = getter;
+    }
+
+    @Override public Enumerator<@Nullable Object> enumerator() {
+      @SuppressWarnings("unchecked") final FindIterable cursor =
+          collection.find(filter).projection(project);
+      return new MongoEnumerator(cursor.iterator(), getter);
+    }
   }
 
   /** Executes an "aggregate" operation on the underlying collection.
@@ -128,28 +151,44 @@ private Enumerable find(MongoDatabase mongoDb, String filterJson,
    * @param operations One or more JSON strings
    * @return Enumerator of results
    */
-  private Enumerable aggregate(final MongoDatabase mongoDb,
-      final List> fields,
+  private Enumerable<@Nullable Object> aggregate(final MongoDatabase mongoDb,
+      final @Nullable List> fields,
       final List operations) {
     final List list = new ArrayList<>();
     for (String operation : operations) {
       list.add(BsonDocument.parse(operation));
     }
-    final Function1 getter =
+    final Function1 getter =
         MongoEnumerator.getter(fields);
-    return new AbstractEnumerable() {
-      @Override public Enumerator enumerator() {
-        final Iterator resultIterator;
-        try {
-          resultIterator = mongoDb.getCollection(collectionName)
-              .aggregate(list).iterator();
-        } catch (Exception e) {
-          throw new RuntimeException("While running MongoDB query "
-              + Util.toString(operations, "[", ",\n", "]"), e);
-        }
-        return new MongoEnumerator(resultIterator, getter);
+    return new AggregateEnumerable(mongoDb, list, operations, getter);
+  }
+
+  /** Enumerable over the results of an "aggregate" operation. */
+  private class AggregateEnumerable extends AbstractEnumerable<@Nullable Object> {
+    private final MongoDatabase mongoDb;
+    private final List list;
+    private final List operations;
+    private final Function1 getter;
+
+    AggregateEnumerable(MongoDatabase mongoDb, List list,
+        List operations, Function1 getter) {
+      this.mongoDb = mongoDb;
+      this.list = list;
+      this.operations = operations;
+      this.getter = getter;
+    }
+
+    @Override public Enumerator<@Nullable Object> enumerator() {
+      final Iterator resultIterator;
+      try {
+        resultIterator = mongoDb.getCollection(collectionName)
+            .aggregate(list).iterator();
+      } catch (Exception e) {
+        throw new RuntimeException("While running MongoDB query "
+            + Util.toString(operations, "[", ",\n", "]"), e);
       }
-    };
+      return new MongoEnumerator(resultIterator, getter);
+    }
   }
 
   /** Implementation of {@link org.apache.calcite.linq4j.Queryable} based on
@@ -170,7 +209,7 @@ public static class MongoQueryable extends AbstractTableQueryable {
     }
 
     private MongoDatabase getMongoDb() {
-      return schema.unwrap(MongoSchema.class).mongoDb;
+      return requireNonNull(schema.unwrap(MongoSchema.class), "mongoSchema").mongoDb;
     }
 
     private MongoTable getTable() {
@@ -182,7 +221,8 @@ private MongoTable getTable() {
      * @see org.apache.calcite.adapter.mongodb.MongoMethod#MONGO_QUERYABLE_AGGREGATE
      */
     @SuppressWarnings("UnusedDeclaration")
-    public Enumerable aggregate(List> fields,
+    public Enumerable<@Nullable Object> aggregate(
+        @Nullable List> fields,
         List operations) {
       return getTable().aggregate(getMongoDb(), fields, operations);
     }
@@ -197,8 +237,9 @@ public Enumerable aggregate(List> fields,
      * @see org.apache.calcite.adapter.mongodb.MongoMethod#MONGO_QUERYABLE_FIND
      */
     @SuppressWarnings("UnusedDeclaration")
-    public Enumerable find(String filterJson,
-        String projectJson, List> fields) {
+    public Enumerable<@Nullable Object> find(@Nullable String filterJson,
+        @Nullable String projectJson,
+        @Nullable List> fields) {
       return getTable().find(getMongoDb(), filterJson, projectJson, fields);
     }
   }
diff --git a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoToEnumerableConverter.java b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoToEnumerableConverter.java
index daa0faf8fd03..dd3d5399e938 100644
--- a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoToEnumerableConverter.java
+++ b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoToEnumerableConverter.java
@@ -43,6 +43,8 @@
 import org.jspecify.annotations.Nullable;
 
 import java.util.AbstractList;
+
+import static java.util.Objects.requireNonNull;
 import java.util.List;
 
 /**
@@ -65,7 +67,7 @@ protected MongoToEnumerableConverter(
 
   @Override public @Nullable RelOptCost computeSelfCost(RelOptPlanner planner,
       RelMetadataQuery mq) {
-    return super.computeSelfCost(planner, mq).multiplyBy(.1);
+    return requireNonNull(super.computeSelfCost(planner, mq)).multiplyBy(.1);
   }
 
   @Override public Result implement(EnumerableRelImplementor implementor, Prefer pref) {
@@ -104,8 +106,10 @@ protected MongoToEnumerableConverter(
                 Pair.class));
     final Expression table =
         list.append("table",
-            mongoImplementor.table.getExpression(
-                MongoTable.MongoQueryable.class));
+            requireNonNull(
+                requireNonNull(mongoImplementor.table, "table")
+                    .getExpression(MongoTable.MongoQueryable.class),
+                "table expression"));
     List opList = mongoImplementor.list.rightList();
     final Expression ops =
         list.append("ops",
diff --git a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/package-info.java b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/package-info.java
index bb18a6935bed..28166576dc36 100644
--- a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/package-info.java
+++ b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/package-info.java
@@ -18,4 +18,7 @@
 /**
  * Query provider based on a MongoDB database.
  */
+@NullMarked
 package org.apache.calcite.adapter.mongodb;
+
+import org.jspecify.annotations.NullMarked;

From 147fb1bee7cf002c20fee85fd3eddfe17f6dc432 Mon Sep 17 00:00:00 2001
From: Vladimir Sitnikov 
Date: Tue, 25 Aug 2026 21:11:00 +0300
Subject: [PATCH 548/562] [CALCITE-7736] Put :cassandra under nullness
 verification

A Cassandra row has no value for a column it never set, so the enumerator and
the enumerable that carries it enumerate nulls. The tuple components a STRUCT
holds are already collected through requireNonNull, which is where the comment
saying null cannot appear inside a collection lives.

The enumerable is a named class rather than an anonymous one, to avoid
uber/NullAway#1746.

Co-Authored-By: Claude Opus 5 
---
 build.gradle.kts                              |  2 +-
 .../cassandra/CassandraEnumerator.java        |  8 +--
 .../cassandra/CassandraSchemaFactory.java     |  4 +-
 .../adapter/cassandra/CassandraTable.java     | 50 +++++++++++++------
 .../adapter/cassandra/package-info.java       |  3 ++
 5 files changed, 46 insertions(+), 21 deletions(-)

diff --git a/build.gradle.kts b/build.gradle.kts
index 3213720c0afc..538614209aec 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -94,7 +94,7 @@ val werror by props(true) // treat javac warnings as errors
 
 // Projects whose main code NullAway verifies. The other projects are not annotated well enough
 // yet, so NullAway would only produce noise there.
-val nullawayProjects = listOf(":linq4j", ":core", ":server", ":druid", ":file", ":kafka", ":spark", ":babel", ":redis", ":splunk", ":mongodb")
+val nullawayProjects = listOf(":linq4j", ":core", ":server", ":druid", ":file", ":kafka", ":spark", ":babel", ":redis", ":splunk", ":mongodb", ":cassandra")
 
 val hepLargePlanModeTestIncludes = mapOf(
     ":core" to listOf(
diff --git a/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraEnumerator.java b/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraEnumerator.java
index f24ccd973a3c..9e8a99779eb0 100644
--- a/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraEnumerator.java
+++ b/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraEnumerator.java
@@ -45,7 +45,7 @@
 import static java.util.Objects.requireNonNull;
 
 /** Enumerator that reads from a Cassandra column family. */
-class CassandraEnumerator implements Enumerator {
+class CassandraEnumerator implements Enumerator<@Nullable Object> {
   private final Iterator iterator;
   private final List fieldTypes;
   @Nullable private Row current;
@@ -68,13 +68,13 @@ class CassandraEnumerator implements Enumerator {
    *
    * @return A new row from the results
    */
-  @Override public Object current() {
+  @Override public @Nullable Object current() {
     if (fieldTypes.size() == 1) {
       // If we just have one field, produce it directly
       return currentRowField(0);
     } else {
       // Build an array with all fields in this row
-      Object[] row = new Object[fieldTypes.size()];
+      final @Nullable Object[] row = new Object[fieldTypes.size()];
       for (int i = 0; i < fieldTypes.size(); i++) {
         row[i] = currentRowField(i);
       }
@@ -126,7 +126,7 @@ class CassandraEnumerator implements Enumerator {
       final TupleValue tupleValue = (TupleValue) obj;
       int numComponents = tupleValue.getType().getComponentTypes().size();
       return IntStream.range(0, numComponents)
-          .mapToObj(i ->
+          .<@Nullable Object>mapToObj(i ->
               tupleValue.get(i,
                   CodecRegistry.DEFAULT.codecFor(
                       tupleValue.getType().getComponentTypes().get(i))))
diff --git a/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraSchemaFactory.java b/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraSchemaFactory.java
index 1ad7b119ffb4..8feb201b7156 100644
--- a/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraSchemaFactory.java
+++ b/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraSchemaFactory.java
@@ -36,6 +36,8 @@
 
 import static java.lang.Integer.parseInt;
 
+import static java.util.Objects.requireNonNull;
+
 /**
  * Factory that creates a {@link CassandraSchema}.
  */
@@ -59,7 +61,7 @@ public CassandraSchemaFactory() {
     final Map sessionMap = projectMapOverKeys(operand, SESSION_DEFINING_KEYS);
 
     INFO_TO_SESSION.computeIfAbsent(sessionMap, m -> {
-      String host = (String) m.get("host");
+      String host = requireNonNull((String) m.get("host"), "host");
       String username = (String) m.get("username");
       String password = (String) m.get("password");
       int port = getPort(m);
diff --git a/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraTable.java b/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraTable.java
index bba5bdad07fe..9ee4c533b881 100644
--- a/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraTable.java
+++ b/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraTable.java
@@ -117,7 +117,7 @@ public List getClusteringOrder() {
     return clusteringOrder;
   }
 
-  public Enumerable query(final CqlSession session) {
+  public Enumerable<@Nullable Object> query(final CqlSession session) {
     return query(session, ImmutableList.of(), ImmutableList.of(),
         ImmutableList.of(), ImmutableList.of(), 0, -1);
   }
@@ -129,7 +129,8 @@ public Enumerable query(final CqlSession session) {
    * @param predicates A list of predicates which should be used in the query
    * @return Enumerator of results
    */
-  public Enumerable query(final CqlSession session, List> fields,
+  public Enumerable<@Nullable Object> query(final CqlSession session,
+      List> fields,
         final List> selectFields, List predicates,
         List order, final Integer offset, final Integer fetch) {
     // Build the type of the resulting row based on the provided fields
@@ -138,7 +139,7 @@ public Enumerable query(final CqlSession session, List addField = fieldName -> {
+    Function1 addField = fieldName -> {
       RelDataType relDataType =
           requireNonNull(rowType.getField(fieldName, true, false)).getType();
       fieldInfo.add(fieldName, relDataType).nullable(true);
@@ -218,18 +219,37 @@ public Enumerable query(final CqlSession session, List() {
-      @Override public Enumerator enumerator() {
-        final ResultSet results = session.execute(queryBuilder.toString());
-        // Skip results until we get to the right offset
-        int skip = 0;
-        Enumerator enumerator = new CassandraEnumerator(results, resultRowType);
-        while (skip < offset && enumerator.moveNext()) {
-          skip++;
-        }
-        return enumerator;
+    return new CassandraEnumerable(session, queryBuilder.toString(), resultRowType,
+        offset);
+  }
+
+  /** Enumerable over the rows a CQL query returns. */
+  private static class CassandraEnumerable
+      extends AbstractEnumerable<@Nullable Object> {
+    private final CqlSession session;
+    private final String query;
+    private final RelProtoDataType resultRowType;
+    private final Integer offset;
+
+    CassandraEnumerable(CqlSession session, String query,
+        RelProtoDataType resultRowType, Integer offset) {
+      this.session = session;
+      this.query = query;
+      this.resultRowType = resultRowType;
+      this.offset = offset;
+    }
+
+    @Override public Enumerator<@Nullable Object> enumerator() {
+      final ResultSet results = session.execute(query);
+      // Skip results until we get to the right offset
+      int skip = 0;
+      Enumerator<@Nullable Object> enumerator =
+          new CassandraEnumerator(results, resultRowType);
+      while (skip < offset && enumerator.moveNext()) {
+        skip++;
       }
-    };
+      return enumerator;
+    }
   }
 
   @Override public  Queryable asQueryable(QueryProvider queryProvider,
@@ -275,7 +295,7 @@ private CqlSession getSession() {
      * @see org.apache.calcite.adapter.cassandra.CassandraMethod#CASSANDRA_QUERYABLE_QUERY
      */
     @SuppressWarnings("UnusedDeclaration")
-    public @Nullable Enumerable query(List> fields,
+    public @Nullable Enumerable<@Nullable Object> query(List> fields,
         List> selectFields, List predicates,
         List order, Integer offset, Integer fetch) {
       return getTable().query(getSession(), fields, selectFields, predicates,
diff --git a/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/package-info.java b/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/package-info.java
index c9bc16403e90..f5614f9955e9 100644
--- a/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/package-info.java
+++ b/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/package-info.java
@@ -20,4 +20,7 @@
  *
  * 
      There is one table for each Cassandra column family.
  */
+@NullMarked
 package org.apache.calcite.adapter.cassandra;
+
+import org.jspecify.annotations.NullMarked;

From 6d6fd0078f965f0960bc46c9eb3532e881317b2f Mon Sep 17 00:00:00 2001
From: Vladimir Sitnikov 
Date: Tue, 25 Aug 2026 21:12:36 +0300
Subject: [PATCH 549/562] [CALCITE-7736] Put :pig under nullness verification

The Pig rel nodes look down the tree for the table they act on, and a tree with
no table underneath returns none, which is what RelNode.getTable has always
said. PigToEnumerableConverter read the rowType field, AbstractRelNode's lazily
computed cache, where it meant this node's row type.

A model that names no file or no columns for a Pig table reached the File and
the array with a null; requireNonNull names the missing operand.

Co-Authored-By: Claude Opus 5 
---
 build.gradle.kts                                       |  2 +-
 .../org/apache/calcite/adapter/pig/PigAggregate.java   |  6 ++++--
 .../java/org/apache/calcite/adapter/pig/PigFilter.java |  4 +++-
 .../java/org/apache/calcite/adapter/pig/PigJoin.java   |  4 +++-
 .../org/apache/calcite/adapter/pig/PigProject.java     |  4 +++-
 .../java/org/apache/calcite/adapter/pig/PigRel.java    |  5 ++++-
 .../apache/calcite/adapter/pig/PigTableFactory.java    | 10 +++++++---
 .../calcite/adapter/pig/PigToEnumerableConverter.java  |  2 +-
 .../org/apache/calcite/adapter/pig/package-info.java   |  3 +++
 9 files changed, 29 insertions(+), 11 deletions(-)

diff --git a/build.gradle.kts b/build.gradle.kts
index 538614209aec..90df618526d5 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -94,7 +94,7 @@ val werror by props(true) // treat javac warnings as errors
 
 // Projects whose main code NullAway verifies. The other projects are not annotated well enough
 // yet, so NullAway would only produce noise there.
-val nullawayProjects = listOf(":linq4j", ":core", ":server", ":druid", ":file", ":kafka", ":spark", ":babel", ":redis", ":splunk", ":mongodb", ":cassandra")
+val nullawayProjects = listOf(":linq4j", ":core", ":server", ":druid", ":file", ":kafka", ":spark", ":babel", ":redis", ":splunk", ":mongodb", ":cassandra", ":pig")
 
 val hepLargePlanModeTestIncludes = mapOf(
     ":core" to listOf(
diff --git a/pig/src/main/java/org/apache/calcite/adapter/pig/PigAggregate.java b/pig/src/main/java/org/apache/calcite/adapter/pig/PigAggregate.java
index 742c82816b5f..39b839c9734a 100644
--- a/pig/src/main/java/org/apache/calcite/adapter/pig/PigAggregate.java
+++ b/pig/src/main/java/org/apache/calcite/adapter/pig/PigAggregate.java
@@ -29,6 +29,8 @@
 
 import com.google.common.collect.ImmutableList;
 
+
+import org.jspecify.annotations.Nullable;
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
@@ -57,7 +59,7 @@ public PigAggregate(RelOptCluster cluster, RelTraitSet traitSet,
   }
 
   @Override public Aggregate copy(RelTraitSet traitSet, RelNode input,
-      ImmutableBitSet groupSet, List groupSets,
+      ImmutableBitSet groupSet, @Nullable List groupSets,
       List aggCalls) {
     return new PigAggregate(input.getCluster(), traitSet, input, groupSet,
         groupSets, aggCalls);
@@ -86,7 +88,7 @@ private String getPigAggregateStatement(Implementor implementor) {
    * Override this method so it looks down the tree to find the table this node
    * is acting on.
    */
-  @Override public RelOptTable getTable() {
+  @Override public @Nullable RelOptTable getTable() {
     return getInput().getTable();
   }
 
diff --git a/pig/src/main/java/org/apache/calcite/adapter/pig/PigFilter.java b/pig/src/main/java/org/apache/calcite/adapter/pig/PigFilter.java
index cc6a8a8c0f07..31538d5ff65a 100644
--- a/pig/src/main/java/org/apache/calcite/adapter/pig/PigFilter.java
+++ b/pig/src/main/java/org/apache/calcite/adapter/pig/PigFilter.java
@@ -27,6 +27,8 @@
 import org.apache.calcite.rex.RexLiteral;
 import org.apache.calcite.rex.RexNode;
 
+
+import org.jspecify.annotations.Nullable;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -58,7 +60,7 @@ public PigFilter(RelOptCluster cluster, RelTraitSet traitSet, RelNode input, Rex
    * Override this method so it looks down the tree to find the table this node
    * is acting on.
    */
-  @Override public RelOptTable getTable() {
+  @Override public @Nullable RelOptTable getTable() {
     return getInput().getTable();
   }
 
diff --git a/pig/src/main/java/org/apache/calcite/adapter/pig/PigJoin.java b/pig/src/main/java/org/apache/calcite/adapter/pig/PigJoin.java
index 948ae3f3bb95..16ea0cb4366d 100644
--- a/pig/src/main/java/org/apache/calcite/adapter/pig/PigJoin.java
+++ b/pig/src/main/java/org/apache/calcite/adapter/pig/PigJoin.java
@@ -30,6 +30,8 @@
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 
+
+import org.jspecify.annotations.Nullable;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -60,7 +62,7 @@ public PigJoin(RelOptCluster cluster, RelTraitSet traitSet, RelNode left, RelNod
    * The Pig alias of the joined relation will have the same name as one from
    * the left side of the join.
    */
-  @Override public RelOptTable getTable() {
+  @Override public @Nullable RelOptTable getTable() {
     return getLeft().getTable();
   }
 
diff --git a/pig/src/main/java/org/apache/calcite/adapter/pig/PigProject.java b/pig/src/main/java/org/apache/calcite/adapter/pig/PigProject.java
index e62c9106a7cc..f98a21314179 100644
--- a/pig/src/main/java/org/apache/calcite/adapter/pig/PigProject.java
+++ b/pig/src/main/java/org/apache/calcite/adapter/pig/PigProject.java
@@ -27,6 +27,8 @@
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 
+
+import org.jspecify.annotations.Nullable;
 import java.util.List;
 
 /** Implementation of {@link org.apache.calcite.rel.core.Project} in
@@ -53,7 +55,7 @@ public PigProject(RelOptCluster cluster, RelTraitSet traitSet, RelNode input,
    * Override this method so it looks down the tree to find the table this node
    * is acting on.
    */
-  @Override public RelOptTable getTable() {
+  @Override public @Nullable RelOptTable getTable() {
     return getInput().getTable();
   }
 }
diff --git a/pig/src/main/java/org/apache/calcite/adapter/pig/PigRel.java b/pig/src/main/java/org/apache/calcite/adapter/pig/PigRel.java
index 6776b47a7040..cdd5101e0263 100644
--- a/pig/src/main/java/org/apache/calcite/adapter/pig/PigRel.java
+++ b/pig/src/main/java/org/apache/calcite/adapter/pig/PigRel.java
@@ -20,6 +20,8 @@
 import org.apache.calcite.rel.RelNode;
 
 import java.util.ArrayList;
+
+import static java.util.Objects.requireNonNull;
 import java.util.List;
 
 /**
@@ -55,7 +57,8 @@ class Implementor {
     private final List statements = new ArrayList<>();
 
     public String getTableName(RelNode input) {
-      final List qualifiedName = input.getTable().getQualifiedName();
+      final List qualifiedName =
+          requireNonNull(input.getTable(), "table").getQualifiedName();
       return qualifiedName.get(qualifiedName.size() - 1);
     }
 
diff --git a/pig/src/main/java/org/apache/calcite/adapter/pig/PigTableFactory.java b/pig/src/main/java/org/apache/calcite/adapter/pig/PigTableFactory.java
index 78544bbe49f4..0f3a2f24b762 100644
--- a/pig/src/main/java/org/apache/calcite/adapter/pig/PigTableFactory.java
+++ b/pig/src/main/java/org/apache/calcite/adapter/pig/PigTableFactory.java
@@ -27,6 +27,8 @@
 import java.util.List;
 import java.util.Map;
 
+import static java.util.Objects.requireNonNull;
+
 /**
  * Factory that creates a {@link PigTable}.
  *
@@ -40,16 +42,18 @@ public PigTableFactory() {
   @SuppressWarnings("unchecked")
   @Override public PigTable create(SchemaPlus schema, String name,
       Map operand, @Nullable RelDataType rowType) {
-    String fileName = (String) operand.get("file");
+    String fileName = requireNonNull((String) operand.get("file"), "file");
     File file = new File(fileName);
     final File base =
         (File) operand.get(ModelHandler.ExtraOperand.BASE_DIRECTORY.camelName);
     if (base != null && !file.isAbsolute()) {
       file = new File(base, fileName);
     }
-    final List fieldNames = (List) operand.get("columns");
+    final List fieldNames =
+        requireNonNull((List) operand.get("columns"), "columns");
     final PigTable result = new PigTable(file.getAbsolutePath(), fieldNames.toArray(new String[0]));
-    schema.unwrap(PigSchema.class).registerTable(name, result);
+    requireNonNull(schema.unwrap(PigSchema.class), "pigSchema")
+        .registerTable(name, result);
     return result;
   }
 }
diff --git a/pig/src/main/java/org/apache/calcite/adapter/pig/PigToEnumerableConverter.java b/pig/src/main/java/org/apache/calcite/adapter/pig/PigToEnumerableConverter.java
index 4a6fe1b74f07..b4f3eed3d397 100644
--- a/pig/src/main/java/org/apache/calcite/adapter/pig/PigToEnumerableConverter.java
+++ b/pig/src/main/java/org/apache/calcite/adapter/pig/PigToEnumerableConverter.java
@@ -68,7 +68,7 @@ protected PigToEnumerableConverter(
   @Override public Result implement(EnumerableRelImplementor implementor, Prefer pref) {
     final BlockBuilder list = new BlockBuilder();
     final PhysType physType =
-        PhysTypeImpl.of(implementor.getTypeFactory(), rowType,
+        PhysTypeImpl.of(implementor.getTypeFactory(), getRowType(),
             pref.prefer(JavaRowFormat.ARRAY));
     PigRel.Implementor impl = new PigRel.Implementor();
     impl.visitChild(0, getInput());
diff --git a/pig/src/main/java/org/apache/calcite/adapter/pig/package-info.java b/pig/src/main/java/org/apache/calcite/adapter/pig/package-info.java
index 89e965be72cb..5c30dfacf7eb 100644
--- a/pig/src/main/java/org/apache/calcite/adapter/pig/package-info.java
+++ b/pig/src/main/java/org/apache/calcite/adapter/pig/package-info.java
@@ -19,4 +19,7 @@
  * Pig query provider.
  *
  */
+@NullMarked
 package org.apache.calcite.adapter.pig;
+
+import org.jspecify.annotations.NullMarked;

From db3858dd1219de8e95df3409d6fe6aff2d460fc1 Mon Sep 17 00:00:00 2001
From: Vladimir Sitnikov 
Date: Tue, 25 Aug 2026 21:14:03 +0300
Subject: [PATCH 550/562] [CALCITE-7736] Put :arrow under nullness verification

An Arrow vector holds a null wherever the column has no value, which getValue
returns for a timestamp and the enumerator hands on, so the enumerator, the
enumerable and the query that builds it carry a nullable element type. The
precondition on query's field list is one the ImmutableIntList parameter
already makes.

Co-Authored-By: Claude Opus 5 
---
 .../calcite/adapter/arrow/AbstractArrowEnumerator.java | 10 ++++++----
 .../apache/calcite/adapter/arrow/ArrowEnumerable.java  |  6 ++++--
 .../org/apache/calcite/adapter/arrow/ArrowTable.java   |  7 +++----
 .../org/apache/calcite/adapter/arrow/package-info.java |  3 +++
 build.gradle.kts                                       |  2 +-
 5 files changed, 17 insertions(+), 11 deletions(-)

diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/AbstractArrowEnumerator.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/AbstractArrowEnumerator.java
index 486e3f60bd8d..226f8ff54b23 100644
--- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/AbstractArrowEnumerator.java
+++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/AbstractArrowEnumerator.java
@@ -28,6 +28,8 @@
 import org.apache.arrow.vector.types.TimeUnit;
 import org.apache.arrow.vector.types.pojo.ArrowType;
 
+
+import org.jspecify.annotations.Nullable;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
@@ -35,7 +37,7 @@
 /**
  * Enumerator that reads from a collection of Arrow value-vectors.
  */
-abstract class AbstractArrowEnumerator implements Enumerator {
+abstract class AbstractArrowEnumerator implements Enumerator<@Nullable Object> {
   protected final ArrowFileReader arrowFileReader;
   protected final List fields;
   protected final List valueVectors;
@@ -82,11 +84,11 @@ protected boolean loadNextNonEmptyArrowBatch() {
     }
   }
 
-  @Override public Object current() {
+  @Override public @Nullable Object current() {
     if (fields.size() == 1) {
       return getValue(this.valueVectors.get(0), currRowIndex);
     }
-    Object[] current = new Object[valueVectors.size()];
+    final @Nullable Object[] current = new Object[valueVectors.size()];
     for (int i = 0; i < valueVectors.size(); i++) {
       ValueVector vector = this.valueVectors.get(i);
       current[i] = getValue(vector, currRowIndex);
@@ -99,7 +101,7 @@ protected boolean loadNextNonEmptyArrowBatch() {
    * 
      For {@link TimeStampVector}, converts the raw value to
    * milliseconds since epoch, which is the representation used by
    * Calcite's Enumerable runtime for TIMESTAMP types. */
-  protected static Object getValue(ValueVector vector, int index) {
+  protected static @Nullable Object getValue(ValueVector vector, int index) {
     if (vector instanceof TimeStampVector) {
       if (vector.isNull(index)) {
         return null;
diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowEnumerable.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowEnumerable.java
index b9c0c4171e65..ef20e1ea2793 100644
--- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowEnumerable.java
+++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowEnumerable.java
@@ -24,12 +24,14 @@
 import org.apache.arrow.vector.ipc.ArrowFileReader;
 import org.apache.arrow.vector.types.pojo.Schema;
 
+
+import org.jspecify.annotations.Nullable;
 import java.util.List;
 
 /**
  * Enumerable that reads from Arrow value-vectors.
  */
-class ArrowEnumerable extends AbstractEnumerable {
+class ArrowEnumerable extends AbstractEnumerable<@Nullable Object> {
   private final ArrowFileReader arrowFileReader;
   private final ImmutableIntList fields;
   private final List>> conditions;
@@ -45,7 +47,7 @@ class ArrowEnumerable extends AbstractEnumerable {
     this.onClose = onClose;
   }
 
-  @Override public Enumerator enumerator() {
+  @Override public Enumerator<@Nullable Object> enumerator() {
     try {
       if (!conditions.isEmpty()) {
         return new ArrowFilterEnumerator(arrowFileReader, fields,
diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTable.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTable.java
index 95ee7867a5b4..0a2ee330d410 100644
--- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTable.java
+++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTable.java
@@ -100,9 +100,8 @@ public class ArrowTable extends AbstractTable
   /** Called via code generation; see uses of
    * {@link org.apache.calcite.adapter.arrow.ArrowMethod#ARROW_QUERY}. */
   @SuppressWarnings("unused")
-  public Enumerable query(DataContext root, ImmutableIntList fields,
-      List>> conditions) {
-    requireNonNull(fields, "fields");
+  public Enumerable<@Nullable Object> query(@Nullable DataContext root,
+      ImmutableIntList fields, List>> conditions) {
 
     FileInputStream fis = null;
     try {
@@ -188,7 +187,7 @@ private ArrowTable getTable() {
      * @return result as enumerable
      */
     @SuppressWarnings("UnusedDeclaration")
-    public Enumerable query(List fields,
+    public Enumerable<@Nullable Object> query(List fields,
         List>> conditions) {
       final ImmutableIntList fieldList = ImmutableIntList.copyOf(fields);
       return getTable().query(null, fieldList, conditions);
diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/package-info.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/package-info.java
index 51ced6b206b3..5d94b4ab9fb8 100644
--- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/package-info.java
+++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/package-info.java
@@ -18,4 +18,7 @@
 /**
  * Query provider that reads from Arrow files.
  */
+@NullMarked
 package org.apache.calcite.adapter.arrow;
+
+import org.jspecify.annotations.NullMarked;
diff --git a/build.gradle.kts b/build.gradle.kts
index 90df618526d5..8a954383611e 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -94,7 +94,7 @@ val werror by props(true) // treat javac warnings as errors
 
 // Projects whose main code NullAway verifies. The other projects are not annotated well enough
 // yet, so NullAway would only produce noise there.
-val nullawayProjects = listOf(":linq4j", ":core", ":server", ":druid", ":file", ":kafka", ":spark", ":babel", ":redis", ":splunk", ":mongodb", ":cassandra", ":pig")
+val nullawayProjects = listOf(":linq4j", ":core", ":server", ":druid", ":file", ":kafka", ":spark", ":babel", ":redis", ":splunk", ":mongodb", ":cassandra", ":pig", ":arrow")
 
 val hepLargePlanModeTestIncludes = mapOf(
     ":core" to listOf(

From 102ae21e6de82b5f8ca52b7a5f9f028207b0d4b9 Mon Sep 17 00:00:00 2001
From: Vladimir Sitnikov 
Date: Tue, 25 Aug 2026 21:15:27 +0300
Subject: [PATCH 551/562] [CALCITE-7736] Put :innodb under nullness
 verification

An InnoDB row has no value for a column that holds none, which the enumerator
returns and the row array carries. The implementor and the internal expression
node are filled in as the translation proceeds, so their fields are marked
NullAway.Init rather than pretending a half-built object never exists. A model
that names no sql file or no data file path reached the schema with a null;
requireNonNull names the missing operand.

Co-Authored-By: Claude Opus 5 
---
 build.gradle.kts                                         | 2 +-
 .../apache/calcite/adapter/innodb/InnodbEnumerator.java  | 6 +++---
 .../calcite/adapter/innodb/InnodbFilterTranslator.java   | 6 +++++-
 .../org/apache/calcite/adapter/innodb/InnodbRel.java     | 4 ++++
 .../calcite/adapter/innodb/InnodbSchemaFactory.java      | 9 +++++++--
 .../org/apache/calcite/adapter/innodb/InnodbSort.java    | 3 ++-
 .../org/apache/calcite/adapter/innodb/InnodbTable.java   | 4 +++-
 .../org/apache/calcite/adapter/innodb/package-info.java  | 3 +++
 8 files changed, 28 insertions(+), 9 deletions(-)

diff --git a/build.gradle.kts b/build.gradle.kts
index 8a954383611e..b512bc38a078 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -94,7 +94,7 @@ val werror by props(true) // treat javac warnings as errors
 
 // Projects whose main code NullAway verifies. The other projects are not annotated well enough
 // yet, so NullAway would only produce noise there.
-val nullawayProjects = listOf(":linq4j", ":core", ":server", ":druid", ":file", ":kafka", ":spark", ":babel", ":redis", ":splunk", ":mongodb", ":cassandra", ":pig", ":arrow")
+val nullawayProjects = listOf(":linq4j", ":core", ":server", ":druid", ":file", ":kafka", ":spark", ":babel", ":redis", ":splunk", ":mongodb", ":cassandra", ":pig", ":arrow", ":innodb")
 
 val hepLargePlanModeTestIncludes = mapOf(
     ":core" to listOf(
diff --git a/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbEnumerator.java b/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbEnumerator.java
index 2d27d9a474d5..528f3ae3a6c8 100644
--- a/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbEnumerator.java
+++ b/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbEnumerator.java
@@ -39,7 +39,7 @@
 /**
  * Enumerator that reads from InnoDB data file.
  */
-class InnodbEnumerator implements Enumerator {
+class InnodbEnumerator implements Enumerator<@Nullable Object> {
   private final Iterator iterator;
   private @Nullable GenericRecord current;
   private final List fieldTypes;
@@ -61,13 +61,13 @@ class InnodbEnumerator implements Enumerator {
    *
    * @return a new row from the results
    */
-  @Override public Object current() {
+  @Override public @Nullable Object current() {
     if (fieldTypes.size() == 1) {
       // If we just have one field, produce it directly
       return currentRowField(fieldTypes.get(0));
     } else {
       // Build an array with all fields in this row
-      Object[] row = new Object[fieldTypes.size()];
+      final @Nullable Object[] row = new Object[fieldTypes.size()];
       for (int i = 0; i < fieldTypes.size(); i++) {
         row[i] = currentRowField(fieldTypes.get(i));
       }
diff --git a/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbFilterTranslator.java b/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbFilterTranslator.java
index 59e5332dfaa0..16fe1b8c9f05 100644
--- a/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbFilterTranslator.java
+++ b/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbFilterTranslator.java
@@ -477,7 +477,11 @@ private boolean nonForceIndexOrMatchForceIndexName(IndexCondition indexCondition
         .map(indexCondition::nameMatch).orElse(true);
   }
 
-  /** Internal representation of a row expression. */
+  /** Internal representation of a row expression.
+   *
+   * 
      Every field is set as the translator walks the expression, which is why
+   * they are not initialized here. */
+  @SuppressWarnings("NullAway.Init")
   private static class InternalRexNode {
     /** Relation expression node. */
     RexNode node;
diff --git a/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbRel.java b/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbRel.java
index 4cf1ca292d33..91297c598f38 100644
--- a/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbRel.java
+++ b/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbRel.java
@@ -39,7 +39,11 @@ class Implementor {
     IndexCondition indexCondition = IndexCondition.EMPTY_CONDITION;
     boolean ascOrder = true;
 
+    /** Set by {@code InnodbTableScan.implement} before anything reads it. */
+    @SuppressWarnings("NullAway.Init")
     RelOptTable table;
+    /** Set by {@code InnodbTableScan.implement} before anything reads it. */
+    @SuppressWarnings("NullAway.Init")
     InnodbTable innodbTable;
 
     public void addSelectFields(Map fields) {
diff --git a/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbSchemaFactory.java b/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbSchemaFactory.java
index 00b0b287ba00..7bb14e96ae40 100644
--- a/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbSchemaFactory.java
+++ b/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbSchemaFactory.java
@@ -21,6 +21,8 @@
 import org.apache.calcite.schema.SchemaPlus;
 
 import java.util.List;
+
+import static java.util.Objects.requireNonNull;
 import java.util.Map;
 
 /**
@@ -32,8 +34,11 @@ public InnodbSchemaFactory() {
 
   @Override public Schema create(SchemaPlus parentSchema, String name,
       Map operand) {
-    final List sqlFilePathList = (List) operand.get("sqlFilePath");
-    final String ibdDataFileBasePath = (String) operand.get("ibdDataFileBasePath");
+    final List sqlFilePathList =
+        requireNonNull((List) operand.get("sqlFilePath"), "sqlFilePath");
+    final String ibdDataFileBasePath =
+        requireNonNull((String) operand.get("ibdDataFileBasePath"),
+            "ibdDataFileBasePath");
     final String timeZone = (String) operand.get("timeZone");
     if (timeZone != null && !timeZone.isEmpty()) {
       System.setProperty("innodb.java.reader.server.timezone", timeZone);
diff --git a/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbSort.java b/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbSort.java
index f3a01c2d1bdb..b95f8aa9e26a 100644
--- a/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbSort.java
+++ b/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbSort.java
@@ -59,7 +59,8 @@ public class InnodbSort extends Sort implements InnodbRel {
   }
 
   @Override public Sort copy(RelTraitSet traitSet, RelNode input,
-      RelCollation newCollation, RexNode offset, RexNode fetch) {
+      RelCollation newCollation, @Nullable RexNode offset,
+      @Nullable RexNode fetch) {
     return new InnodbSort(getCluster(), traitSet, input, collation);
   }
 
diff --git a/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbTable.java b/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbTable.java
index 06ca97d2dc4f..ebe5940df5db 100644
--- a/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbTable.java
+++ b/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbTable.java
@@ -52,6 +52,8 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+
+import org.jspecify.annotations.Nullable;
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
@@ -150,7 +152,7 @@ public Enumerable query(
     final RelDataTypeFactory.Builder fieldInfo = typeFactory.builder();
     final RelDataType rowType = getRowType(typeFactory);
 
-    Function1 addField = fieldName -> {
+    Function1 addField = fieldName -> {
       final RelDataTypeField field =
           requireNonNull(rowType.getField(fieldName, true, false));
       RelDataType relDataType = field.getType();
diff --git a/innodb/src/main/java/org/apache/calcite/adapter/innodb/package-info.java b/innodb/src/main/java/org/apache/calcite/adapter/innodb/package-info.java
index 298f95846df8..27cb2df5eb77 100644
--- a/innodb/src/main/java/org/apache/calcite/adapter/innodb/package-info.java
+++ b/innodb/src/main/java/org/apache/calcite/adapter/innodb/package-info.java
@@ -18,4 +18,7 @@
 /**
  * InnoDB query provider.
  */
+@NullMarked
 package org.apache.calcite.adapter.innodb;
+
+import org.jspecify.annotations.NullMarked;

From 21b1631fb86602eccb8219c3c356cfe787d50ce6 Mon Sep 17 00:00:00 2001
From: Vladimir Sitnikov 
Date: Tue, 25 Aug 2026 21:25:16 +0300
Subject: [PATCH 552/562] [CALCITE-7736] Put :piglet under nullness
 verification

Piglet keeps four maps from a relation to its alias and its Pig operator, and
looking a name up in them can miss, which the getters now say. Handler looked
its relations up the same way and pushed whatever came back, including nothing;
it now reports the unknown name instead of failing later in the builder.

PigTable.scan returned null rather than an enumerable, and nothing could have
used it; it throws. SqlUserDefinedFunction declared its operand type inference
non-null though SqlFunction below it has always accepted none, which is what
PigUserDefinedFunction passes.

Co-Authored-By: Claude Opus 5 
---
 build.gradle.kts                              |  2 +-
 .../sql/validate/SqlUserDefinedFunction.java  |  8 ++---
 .../piglet/DynamicTupleRecordType.java        |  7 ++--
 .../org/apache/calcite/piglet/Handler.java    | 32 +++++++++++++------
 .../apache/calcite/piglet/PigRelBuilder.java  | 32 ++++++++++++-------
 .../calcite/piglet/PigRelExVisitor.java       | 13 +++++---
 .../calcite/piglet/PigRelOpInnerVisitor.java  |  6 ++--
 .../calcite/piglet/PigRelOpVisitor.java       | 23 +++++++++----
 .../apache/calcite/piglet/PigRelOpWalker.java |  5 ++-
 .../apache/calcite/piglet/PigRelSqlUdfs.java  | 25 +++++++++++----
 .../calcite/piglet/PigRelUdfConverter.java    |  4 ++-
 .../org/apache/calcite/piglet/PigTable.java   |  2 +-
 .../calcite/piglet/PigToSqlAggregateRule.java | 14 +++++---
 .../apache/calcite/piglet/PigUdfFinder.java   |  6 ++--
 .../piglet/PigUserDefinedFunction.java        | 10 +++---
 .../apache/calcite/piglet/package-info.java   |  3 ++
 16 files changed, 130 insertions(+), 62 deletions(-)

diff --git a/build.gradle.kts b/build.gradle.kts
index b512bc38a078..58534824d6e0 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -94,7 +94,7 @@ val werror by props(true) // treat javac warnings as errors
 
 // Projects whose main code NullAway verifies. The other projects are not annotated well enough
 // yet, so NullAway would only produce noise there.
-val nullawayProjects = listOf(":linq4j", ":core", ":server", ":druid", ":file", ":kafka", ":spark", ":babel", ":redis", ":splunk", ":mongodb", ":cassandra", ":pig", ":arrow", ":innodb")
+val nullawayProjects = listOf(":linq4j", ":core", ":server", ":druid", ":file", ":kafka", ":spark", ":babel", ":redis", ":splunk", ":mongodb", ":cassandra", ":pig", ":arrow", ":innodb", ":piglet")
 
 val hepLargePlanModeTestIncludes = mapOf(
     ":core" to listOf(
diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlUserDefinedFunction.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlUserDefinedFunction.java
index 6d54c876b25f..236694a3df34 100644
--- a/core/src/main/java/org/apache/calcite/sql/validate/SqlUserDefinedFunction.java
+++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlUserDefinedFunction.java
@@ -48,7 +48,7 @@ public class SqlUserDefinedFunction extends SqlFunction {
   @Deprecated // to be removed before 2.0
   public SqlUserDefinedFunction(SqlIdentifier opName,
       SqlReturnTypeInference returnTypeInference,
-      SqlOperandTypeInference operandTypeInference,
+      @Nullable SqlOperandTypeInference operandTypeInference,
       @Nullable SqlOperandTypeChecker operandTypeChecker,
       List paramTypes,
       Function function) {
@@ -62,7 +62,7 @@ public SqlUserDefinedFunction(SqlIdentifier opName,
   /** Creates a {@link SqlUserDefinedFunction}. */
   public SqlUserDefinedFunction(SqlIdentifier opName, SqlKind kind,
       SqlReturnTypeInference returnTypeInference,
-      SqlOperandTypeInference operandTypeInference,
+      @Nullable SqlOperandTypeInference operandTypeInference,
       @Nullable SqlOperandMetadata operandMetadata,
       Function function) {
     this(opName, kind, returnTypeInference, operandTypeInference,
@@ -72,7 +72,7 @@ public SqlUserDefinedFunction(SqlIdentifier opName, SqlKind kind,
   /** Creates a {@link SqlUserDefinedFunction} with sql syntax. */
   public SqlUserDefinedFunction(SqlIdentifier opName, SqlKind kind,
       SqlReturnTypeInference returnTypeInference,
-      SqlOperandTypeInference operandTypeInference,
+      @Nullable SqlOperandTypeInference operandTypeInference,
       @Nullable SqlOperandMetadata operandMetadata,
       Function function, SqlSyntax syntax) {
     this(opName, kind, returnTypeInference, operandTypeInference,
@@ -82,7 +82,7 @@ public SqlUserDefinedFunction(SqlIdentifier opName, SqlKind kind,
   /** Constructor used internally and by derived classes. */
   protected SqlUserDefinedFunction(SqlIdentifier opName, SqlKind kind,
       SqlReturnTypeInference returnTypeInference,
-      SqlOperandTypeInference operandTypeInference,
+      @Nullable SqlOperandTypeInference operandTypeInference,
       @Nullable SqlOperandMetadata operandMetadata,
       Function function,
       SqlFunctionCategory category, SqlSyntax syntax) {
diff --git a/piglet/src/main/java/org/apache/calcite/piglet/DynamicTupleRecordType.java b/piglet/src/main/java/org/apache/calcite/piglet/DynamicTupleRecordType.java
index 86c370089b6e..4a34791d1679 100644
--- a/piglet/src/main/java/org/apache/calcite/piglet/DynamicTupleRecordType.java
+++ b/piglet/src/main/java/org/apache/calcite/piglet/DynamicTupleRecordType.java
@@ -20,10 +20,13 @@
 import org.apache.calcite.rel.type.RelDataTypeFactory;
 import org.apache.calcite.rel.type.RelDataTypeField;
 
+
+import org.jspecify.annotations.Nullable;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 import static java.lang.Integer.parseInt;
+import static java.util.Objects.requireNonNull;
 
 /**
  * Represents Pig Tuples with unknown fields. The tuple field
@@ -37,7 +40,7 @@ public class DynamicTupleRecordType extends DynamicRecordTypeImpl {
     super(typeFactory);
   }
 
-  @Override public RelDataTypeField getField(String fieldName,
+  @Override public @Nullable RelDataTypeField getField(String fieldName,
       boolean caseSensitive, boolean elideRecord) {
     final int index = nameToIndex(fieldName);
     if (index >= 0) {
@@ -70,7 +73,7 @@ void resize(int size) {
   private static int nameToIndex(String fieldName) {
     Matcher matcher = INDEX_PATTERN.matcher(fieldName);
     if (matcher.find()) {
-      return parseInt(matcher.group(1));
+      return parseInt(requireNonNull(matcher.group(1), "index"));
     }
     return -1;
   }
diff --git a/piglet/src/main/java/org/apache/calcite/piglet/Handler.java b/piglet/src/main/java/org/apache/calcite/piglet/Handler.java
index d8acf8432a79..c9ea5fa73a2d 100644
--- a/piglet/src/main/java/org/apache/calcite/piglet/Handler.java
+++ b/piglet/src/main/java/org/apache/calcite/piglet/Handler.java
@@ -35,6 +35,8 @@
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
+
+import static java.util.Objects.requireNonNull;
 import java.util.Map;
 
 /**
@@ -49,6 +51,17 @@ public Handler(PigRelBuilder builder) {
     this.builder = builder;
   }
 
+  /** Returns the relational expression registered under a name.
+   *
+   * @throws IllegalArgumentException if no relation has that name */
+  private RelNode get(String name) {
+    final RelNode relNode = map.get(name);
+    if (relNode == null) {
+      throw new IllegalArgumentException("unknown relation '" + name + "'");
+    }
+    return relNode;
+  }
+
   /** Creates relational expressions for a given AST node. */
   public Handler handle(Ast.Node node) {
     final RelNode input;
@@ -68,7 +81,7 @@ public Handler handle(Ast.Node node) {
     case FOREACH:
       final Ast.ForeachStmt foreach = (Ast.ForeachStmt) node;
       builder.clear();
-      input = map.get(foreach.source.value);
+      input = get(foreach.source.value);
       builder.push(input);
       rexNodes = new ArrayList<>();
       for (Ast.Node exp : foreach.expList) {
@@ -80,7 +93,7 @@ public Handler handle(Ast.Node node) {
     case FOREACH_NESTED:
       final Ast.ForeachNestedStmt foreachNested = (Ast.ForeachNestedStmt) node;
       builder.clear();
-      input = map.get(foreachNested.source.value);
+      input = get(foreachNested.source.value);
       builder.push(input);
       System.out.println(input.getRowType());
       for (RelDataTypeField field : input.getRowType().getFieldList()) {
@@ -105,7 +118,7 @@ public Handler handle(Ast.Node node) {
     case FILTER:
       final Ast.FilterStmt filter = (Ast.FilterStmt) node;
       builder.clear();
-      input = map.get(filter.source.value);
+      input = get(filter.source.value);
       builder.push(input);
       RexNode rexNode = toRex(filter.condition);
       if (rexNode.getType().getSqlTypeName() != SqlTypeName.BOOLEAN) {
@@ -118,7 +131,7 @@ public Handler handle(Ast.Node node) {
     case DISTINCT:
       final Ast.DistinctStmt distinct = (Ast.DistinctStmt) node;
       builder.clear();
-      input = map.get(distinct.source.value);
+      input = get(distinct.source.value);
       builder.push(input);
       builder.distinct(null, -1);
       register(distinct.target.value);
@@ -126,7 +139,7 @@ public Handler handle(Ast.Node node) {
     case ORDER:
       final Ast.OrderStmt order = (Ast.OrderStmt) node;
       builder.clear();
-      input = map.get(order.source.value);
+      input = get(order.source.value);
       builder.push(input);
       final List nodes = new ArrayList<>();
       for (Pair field : order.fields) {
@@ -138,7 +151,7 @@ public Handler handle(Ast.Node node) {
     case LIMIT:
       final Ast.LimitStmt limit = (Ast.LimitStmt) node;
       builder.clear();
-      input = map.get(limit.source.value);
+      input = get(limit.source.value);
       final int count = ((Number) limit.count.value).intValue();
       builder.push(input);
       builder.limit(0, count);
@@ -147,7 +160,7 @@ public Handler handle(Ast.Node node) {
     case GROUP:
       final Ast.GroupStmt group = (Ast.GroupStmt) node;
       builder.clear();
-      input = map.get(group.source.value);
+      input = get(group.source.value);
       builder.push(input).as(group.source.value);
       final List groupKeys = new ArrayList<>();
       final List keys = new ArrayList<>();
@@ -168,7 +181,7 @@ public Handler handle(Ast.Node node) {
       return this;
     case DUMP:
       final Ast.DumpStmt dump = (Ast.DumpStmt) node;
-      final RelNode relNode = map.get(dump.relation.value);
+      final RelNode relNode = get(dump.relation.value);
       dump(relNode);
       return this; // nothing to do; contains no algebra
     default:
@@ -213,7 +226,8 @@ private ImmutableList bag(List nodeList,
     final ImmutableList.Builder listBuilder =
         ImmutableList.builder();
     for (Ast.Node node : nodeList) {
-      listBuilder.add(item(node, type.getComponentType()));
+      listBuilder.add(
+          item(node, requireNonNull(type.getComponentType(), "componentType")));
     }
     return listBuilder.build();
   }
diff --git a/piglet/src/main/java/org/apache/calcite/piglet/PigRelBuilder.java b/piglet/src/main/java/org/apache/calcite/piglet/PigRelBuilder.java
index 87c467209b01..fe6a4bfa2a22 100644
--- a/piglet/src/main/java/org/apache/calcite/piglet/PigRelBuilder.java
+++ b/piglet/src/main/java/org/apache/calcite/piglet/PigRelBuilder.java
@@ -52,6 +52,8 @@
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 
+
+import org.jspecify.annotations.Nullable;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -88,7 +90,7 @@ public static PigRelBuilder create(FrameworkConfig config) {
     return new PigRelBuilder(
         transform(config.getContext(), c -> c.withBloat(-1)),
         relBuilder.getCluster(),
-        relBuilder.getRelOptSchema());
+        requireNonNull(relBuilder.getRelOptSchema(), "relOptSchema"));
   }
 
   private static Context transform(Context context,
@@ -98,19 +100,19 @@ private static Context transform(Context context,
     return Contexts.of(transform.apply(config), context);
   }
 
-  public RelNode getRel(String alias) {
+  public @Nullable RelNode getRel(String alias) {
     return aliasMap.get(alias);
   }
 
-  public RelNode getRel(Operator pig) {
+  public @Nullable RelNode getRel(Operator pig) {
     return pigRelMap.get(pig);
   }
 
-  Operator getPig(RelNode rel)  {
+  @Nullable Operator getPig(RelNode rel)  {
     return relPigMap.get(rel);
   }
 
-  String getAlias(RelNode rel)  {
+  @Nullable String getAlias(RelNode rel)  {
     return reverseAliasMap.get(rel);
   }
 
@@ -123,7 +125,7 @@ CorrelationId nextCorrelId() {
     return new CorrelationId(nextCorrelId++);
   }
 
-  public String getAlias() {
+  public @Nullable String getAlias() {
     final RelNode input = peek();
     if (reverseAliasMap.containsKey(input)) {
       return reverseAliasMap.get(input);
@@ -165,7 +167,8 @@ public boolean checkMap(LogicalRelationalOperator pigOp) {
    * @param alias the alias
    * @param updatePigRelMap whether to update the PigRelMap
    */
-  public void updateAlias(Operator pigOp, String alias, boolean updatePigRelMap) {
+  public void updateAlias(Operator pigOp, String alias,
+      boolean updatePigRelMap) {
     final RelNode rel = peek();
     if (updatePigRelMap) {
       pigRelMap.put(pigOp, rel);
@@ -233,7 +236,7 @@ void replaceTop(RelNode newRel) {
    * @param tableNames The names of the table to scan
    * @return This builder
    */
-  public RelBuilder scan(RelOptTable userSchema, String... tableNames) {
+  public RelBuilder scan(@Nullable RelOptTable userSchema, String... tableNames) {
     // First, look up the database schema to find the table schema with the given names
     final List names = ImmutableList.copyOf(tableNames);
     requireNonNull(relOptSchema, "relOptSchema");
@@ -294,7 +297,9 @@ public RelBuilder scan(RelDataType rowType, String... tableNames) {
    */
   public RelBuilder scan(RelDataType rowType, List tableNames) {
     final RelOptTable relOptTable =
-        PigTable.createRelOptTable(getRelOptSchema(), rowType, tableNames);
+        PigTable.createRelOptTable(
+            requireNonNull(getRelOptSchema(), "relOptSchema"), rowType,
+            tableNames);
     return scan(relOptTable);
   }
 
@@ -486,7 +491,9 @@ public RelBuilder multiSetFlatten(List flattenCols, List flatte
     for (int i = 0; i < colCount; i++) {
       if (flattenCols.indexOf(i) >= 0) {
         // The original multiset columns to be flattened, select new flattened columns instead
-        RelDataType componentType = inputFields.get(i).getType().getComponentType();
+        RelDataType componentType =
+            requireNonNull(inputFields.get(i).getType().getComponentType(),
+                "componentType");
         final int numSubFields = componentType.isStruct() ? componentType.getFieldCount() : 1;
         for (int j = 0; j < numSubFields; j++) {
           finnalCols.add(field(colCount + flattenCount));
@@ -554,7 +561,8 @@ public RelBuilder collect() {
     project(ImmutableList.of(literal("all"), row));
 
     // Update the alias map for the new projected rel.
-    updateAlias(getPig(inputRel), getAlias(inputRel), false);
+    updateAlias(requireNonNull(getPig(inputRel), "pigOp"),
+        requireNonNull(getAlias(inputRel), "alias"), false);
 
     // Build a single group for all rows
     cogroup(ImmutableList.of(groupKey(ImmutableList.of(field(0)))));
@@ -609,7 +617,7 @@ RelBuilder store(String storeAlias) {
    * Gets all relational plans corresponding to Pig Store operators.
    *
    */
-  public List getRelsForStores() {
+  public @Nullable List getRelsForStores() {
     if (storeMap.isEmpty()) {
       return null;
     }
diff --git a/piglet/src/main/java/org/apache/calcite/piglet/PigRelExVisitor.java b/piglet/src/main/java/org/apache/calcite/piglet/PigRelExVisitor.java
index 49a748e5b22d..8b5aaacf274d 100644
--- a/piglet/src/main/java/org/apache/calcite/piglet/PigRelExVisitor.java
+++ b/piglet/src/main/java/org/apache/calcite/piglet/PigRelExVisitor.java
@@ -236,7 +236,7 @@ private ImmutableList buildBinaryOperands() {
           pigRelOp.getPlan().getPredecessors(pigRelOp).get(op.getInputNum());
       if (builder.checkMap(childOp)) {
         // Inner plan that has been processed before (nested foreach or flatten)
-        builder.push(builder.getRel(childOp));
+        builder.push(requireNonNull(builder.getRel(childOp), "rel"));
         final List fields = builder.getFields(inputCount, inputOrdinal, op.getColNum());
 
         for (int i = fields.size() - 1; i >= 0; i--) {
@@ -364,7 +364,8 @@ private static RexNode replacePatternIfPossible(RexNode rexNode) {
             builder, op.getFuncSpec(), buildOperands(numAgrs), returnType));
 
     String className = op.getFuncSpec().getClassName();
-    SqlOperator sqlOp = ((RexCall) stack.peek()).getOperator();
+    SqlOperator sqlOp =
+        ((RexCall) requireNonNull(stack.peek(), "stack.peek()")).getOperator();
     if (sqlOp instanceof SqlUserDefinedFunction) {
       ScalarFunctionImpl sqlFunc =
           (ScalarFunctionImpl) ((SqlUserDefinedFunction) sqlOp).getFunction();
@@ -450,9 +451,11 @@ private static int optSize(List list) {
     final int index = ((BigDecimal) ((RexLiteral) operand2).getValue()).intValue();
 
     RelNode referencedRel =
-        builder.getRel(
-            ((LogicalRelationalOperator) op.getImplicitReferencedOperator())
-                .getAlias());
+        requireNonNull(
+            builder.getRel(
+                ((LogicalRelationalOperator) op.getImplicitReferencedOperator())
+                    .getAlias()),
+            "referencedRel");
     builder.push(referencedRel);
     List projectCol = Lists.newArrayList(builder.field(index));
     builder.project(projectCol);
diff --git a/piglet/src/main/java/org/apache/calcite/piglet/PigRelOpInnerVisitor.java b/piglet/src/main/java/org/apache/calcite/piglet/PigRelOpInnerVisitor.java
index f890283a1153..a8b0c861bf05 100644
--- a/piglet/src/main/java/org/apache/calcite/piglet/PigRelOpInnerVisitor.java
+++ b/piglet/src/main/java/org/apache/calcite/piglet/PigRelOpInnerVisitor.java
@@ -41,6 +41,8 @@
 
 import com.google.common.collect.ImmutableSet;
 
+
+import org.jspecify.annotations.Nullable;
 import java.util.ArrayDeque;
 import java.util.ArrayList;
 import java.util.Deque;
@@ -141,7 +143,7 @@ private void doGenerateWithoutMultisetFlatten(LOGenerate gen, List mult
       List flattenOutputAliases) throws FrontendException {
     final List pigProjections = gen.getOutputPlans();
     final List innerCols = new ArrayList<>(); // For projection expressions
-    final List fieldAlias = new ArrayList<>(); // For projection names/alias
+    final List<@Nullable String> fieldAlias = new ArrayList<>(); // For projection names/alias
 
     if (gen.getOutputPlanSchemas() == null) {
       throw new IllegalArgumentException(
@@ -189,7 +191,7 @@ private void doGenerateWithoutMultisetFlatten(LOGenerate gen, List mult
         }
       } else {
         innerCols.add(rexNode);
-        String alias = null;
+        @Nullable String alias = null;
         if (outputFieldSchema.size() == 1) {
           // If simple type, take user alias if available
           alias = outputFieldSchema.getField(0).alias;
diff --git a/piglet/src/main/java/org/apache/calcite/piglet/PigRelOpVisitor.java b/piglet/src/main/java/org/apache/calcite/piglet/PigRelOpVisitor.java
index e1c90e25cc2e..18e2bf554a91 100644
--- a/piglet/src/main/java/org/apache/calcite/piglet/PigRelOpVisitor.java
+++ b/piglet/src/main/java/org/apache/calcite/piglet/PigRelOpVisitor.java
@@ -69,6 +69,8 @@
 
 import com.google.common.collect.ImmutableList;
 
+
+import org.jspecify.annotations.Nullable;
 import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -76,6 +78,8 @@
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
+
+import static java.util.Objects.requireNonNull;
 import java.util.Set;
 
 /**
@@ -85,7 +89,7 @@
 class PigRelOpVisitor extends PigRelOpWalker.PlanPreVisitor {
   // The relational algebra builder customized for Pig
   protected final PigRelBuilder builder;
-  private Operator currentRoot;
+  private @Nullable Operator currentRoot;
 
   /** Type of Pig groups. */
   private enum GroupType {
@@ -112,7 +116,7 @@ private enum GroupType {
     this.currentRoot = null;
   }
 
-  Operator getCurrentRoot() {
+  @Nullable Operator getCurrentRoot() {
     return currentRoot;
   }
 
@@ -142,7 +146,9 @@ List translate() throws FrontendException {
     String fullName = load.getSchemaFile();
     if (fullName.contains("file://")) {
       // load from database catalog. Pig will see it as a file in the working directory
-      fullName = Paths.get(load.getSchemaFile()).getFileName().toString();
+      fullName =
+          requireNonNull(Paths.get(load.getSchemaFile()).getFileName(),
+              "fileName").toString();
     }
     String[] tableNames;
     if (fullName.startsWith("/")) {
@@ -154,7 +160,7 @@ List translate() throws FrontendException {
       tableNames = fullName.split("\\.");
     }
     final LogicalSchema pigSchema = load.getSchema();
-    final RelOptTable pigRelOptTable;
+    final @Nullable RelOptTable pigRelOptTable;
     if (pigSchema == null) {
       pigRelOptTable = null;
     } else {
@@ -162,8 +168,9 @@ List translate() throws FrontendException {
       // relational row type
       final RelDataType rowType = PigTypes.convertSchema(pigSchema);
       pigRelOptTable =
-          PigTable.createRelOptTable(builder.getRelOptSchema(), rowType,
-              Arrays.asList(tableNames));
+          PigTable.createRelOptTable(
+              requireNonNull(builder.getRelOptSchema(), "relOptSchema"),
+              rowType, Arrays.asList(tableNames));
     }
     builder.scan(pigRelOptTable, tableNames);
     builder.register(load);
@@ -367,7 +374,9 @@ private void preprocessCogroup(LOCogroup loCogroup, boolean isCubeRollup)
               getGroupRowOperands(fieldRels, isCubeRollup));
       fieldRels.add(row);
       builder.project(fieldRels);
-      builder.updateAlias(builder.getPig(originalRel), builder.getAlias(originalRel), false);
+      builder.updateAlias(
+          requireNonNull(builder.getPig(originalRel), "pigOp"),
+          requireNonNull(builder.getAlias(originalRel), "alias"), false);
     }
   }
 
diff --git a/piglet/src/main/java/org/apache/calcite/piglet/PigRelOpWalker.java b/piglet/src/main/java/org/apache/calcite/piglet/PigRelOpWalker.java
index 841024160a14..cc5df3e72ae3 100644
--- a/piglet/src/main/java/org/apache/calcite/piglet/PigRelOpWalker.java
+++ b/piglet/src/main/java/org/apache/calcite/piglet/PigRelOpWalker.java
@@ -25,6 +25,8 @@
 import org.apache.pig.newplan.logical.relational.LogicalRelationalNodesVisitor;
 import org.apache.pig.newplan.logical.relational.LogicalRelationalOperator;
 
+
+import org.jspecify.annotations.Nullable;
 import java.util.Collection;
 
 /**
@@ -69,7 +71,8 @@ abstract static class PlanPreVisitor extends LogicalRelationalNodesVisitor {
    * @param visitor The visitor of each Pig logical operator node
    * @throws FrontendException Exception during processing Pig operator
    */
-  private void postOrderWalk(Operator root, PlanPreVisitor visitor) throws FrontendException {
+  private void postOrderWalk(@Nullable Operator root, PlanPreVisitor visitor)
+      throws FrontendException {
     if (root == null || visitor.preVisit((LogicalRelationalOperator) root)) {
       return;
     }
diff --git a/piglet/src/main/java/org/apache/calcite/piglet/PigRelSqlUdfs.java b/piglet/src/main/java/org/apache/calcite/piglet/PigRelSqlUdfs.java
index 3018929d511a..1c75d19238cb 100644
--- a/piglet/src/main/java/org/apache/calcite/piglet/PigRelSqlUdfs.java
+++ b/piglet/src/main/java/org/apache/calcite/piglet/PigRelSqlUdfs.java
@@ -56,6 +56,8 @@
 
 import static org.apache.calcite.piglet.PigTypes.TYPE_FACTORY;
 
+import static java.util.Objects.requireNonNull;
+
 /**
  * User-defined functions ({@link SqlUserDefinedFunction UDFs})
  * needed for Pig-to-{@link RelNode} translation.
@@ -66,11 +68,17 @@ private PigRelSqlUdfs() {
 
   // Defines ScalarFunc from their implementations
   private static final ScalarFunction PIG_TUPLE_FUNC =
-      ScalarFunctionImpl.create(PigRelSqlUdfs.class, "buildTuple");
+      requireNonNull(
+          ScalarFunctionImpl.create(PigRelSqlUdfs.class, "buildTuple"),
+          "buildTuple");
   private static final ScalarFunction PIG_BAG_FUNC =
-      ScalarFunctionImpl.create(PigRelSqlUdfs.class, "buildBag");
+      requireNonNull(
+          ScalarFunctionImpl.create(PigRelSqlUdfs.class, "buildBag"),
+          "buildBag");
   private static final ScalarFunction MULTISET_PROJECTION_FUNC =
-      ScalarFunctionImpl.create(PigRelSqlUdfs.class, "projectMultiset");
+      requireNonNull(
+          ScalarFunctionImpl.create(PigRelSqlUdfs.class, "projectMultiset"),
+          "projectMultiset");
 
   /**
    * Multiset projection projects a subset of columns from the component type
@@ -149,7 +157,9 @@ private static SqlReturnTypeInference multisetProjectionInfer() {
       final List fields = source.getComponentType().getFieldList();
       // Project a multiset of single column
       if (opBinding.getOperandCount() == 2) {
-        final int fieldNo = opBinding.getOperandLiteralValue(1, Integer.class);
+        final int fieldNo =
+            requireNonNull(opBinding.getOperandLiteralValue(1, Integer.class),
+                "fieldNo");
         if (fields.size() == 1) {
           // Corner case: source with only single column, nothing to do.
           assert fieldNo == 0;
@@ -162,7 +172,9 @@ private static SqlReturnTypeInference multisetProjectionInfer() {
       final List destNames = new ArrayList<>();
       final List destTypes = new ArrayList<>();
       for (int i = 1; i < opBinding.getOperandCount(); i++) {
-        final int fieldNo = opBinding.getOperandLiteralValue(i, Integer.class);
+        final int fieldNo =
+            requireNonNull(opBinding.getOperandLiteralValue(i, Integer.class),
+                "fieldNo");
         destNames.add(fields.get(fieldNo).getName());
         destTypes.add(fields.get(fieldNo).getType());
       }
@@ -203,7 +215,8 @@ private static SqlOperandMetadata multisetProjectionCheck() {
             return false;
           }
           final int fieldNo =
-              callBinding.getOperandLiteralValue(i, Integer.class);
+              requireNonNull(callBinding.getOperandLiteralValue(i, Integer.class),
+                  "fieldNo");
           // Field number should between 0 and maxFieldNo
           if (fieldNo < 0 || fieldNo > maxFieldNo) {
             return false;
diff --git a/piglet/src/main/java/org/apache/calcite/piglet/PigRelUdfConverter.java b/piglet/src/main/java/org/apache/calcite/piglet/PigRelUdfConverter.java
index b378002e0aa0..ad2a909370bc 100644
--- a/piglet/src/main/java/org/apache/calcite/piglet/PigRelUdfConverter.java
+++ b/piglet/src/main/java/org/apache/calcite/piglet/PigRelUdfConverter.java
@@ -32,6 +32,8 @@
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 
+
+import org.jspecify.annotations.Nullable;
 import java.lang.reflect.Method;
 import java.util.Map;
 
@@ -169,7 +171,7 @@ static RexNode convertPigFunction(PigRelBuilder builder, FuncSpec pigFunc,
    *
    * @param call Pig aggregate UDF call
    */
-  static SqlAggFunction getSqlAggFuncForPigUdf(RexCall call) {
+  static @Nullable SqlAggFunction getSqlAggFuncForPigUdf(RexCall call) {
     if (!(call.getOperator() instanceof PigUserDefinedFunction)) {
       return null;
     }
diff --git a/piglet/src/main/java/org/apache/calcite/piglet/PigTable.java b/piglet/src/main/java/org/apache/calcite/piglet/PigTable.java
index d57fe4acf2a7..9c3648e67878 100644
--- a/piglet/src/main/java/org/apache/calcite/piglet/PigTable.java
+++ b/piglet/src/main/java/org/apache/calcite/piglet/PigTable.java
@@ -71,6 +71,6 @@ public static RelOptTable createRelOptTable(RelOptSchema schema,
   }
 
   @Override public Enumerable<@Nullable Object[]> scan(final DataContext root) {
-    return null;
+    throw new UnsupportedOperationException("PigTable cannot be scanned");
   }
 }
diff --git a/piglet/src/main/java/org/apache/calcite/piglet/PigToSqlAggregateRule.java b/piglet/src/main/java/org/apache/calcite/piglet/PigToSqlAggregateRule.java
index cf3d46b752de..0a46dd28ef33 100644
--- a/piglet/src/main/java/org/apache/calcite/piglet/PigToSqlAggregateRule.java
+++ b/piglet/src/main/java/org/apache/calcite/piglet/PigToSqlAggregateRule.java
@@ -45,6 +45,7 @@
 import java.util.Map;
 
 import static org.apache.calcite.piglet.PigTypes.TYPE_FACTORY;
+import static java.util.Objects.requireNonNull;
 
 /**
  * Planner rule that converts Pig aggregate UDF calls to built-in SQL
@@ -127,10 +128,10 @@ private static class RexCallReplacer extends RexShuttle {
     private final Map replacementMap;
     private final RexBuilder builder;
     private final int oldProjectCol;
-    private final RexNode newProjectCol;
+    private final @Nullable RexNode newProjectCol;
 
     RexCallReplacer(RexBuilder builder, Map replacementMap,
-        int oldProjectCol, RexNode newProjectCol) {
+        int oldProjectCol, @Nullable RexNode newProjectCol) {
       this.replacementMap = replacementMap;
       this.builder = builder;
       this.oldProjectCol = oldProjectCol;
@@ -269,7 +270,7 @@ private static class RexCallReplacer extends RexShuttle {
     }
     for (RexCall rexCall : pigAggUdfs) {
       final List aggOperands = new ArrayList<>();
-      for (int i : aggCallColumns.get(rexCall)) {
+      for (int i : requireNonNull(aggCallColumns.get(rexCall), "aggCallColumns")) {
         aggOperands.add(relBuilder.field(i));
       }
       if (isMultisetProjection(rexCall)) {
@@ -290,7 +291,8 @@ private static class RexCallReplacer extends RexShuttle {
         }
       } else {
         final SqlAggFunction udf =
-            PigRelUdfConverter.getSqlAggFuncForPigUdf(rexCall);
+            requireNonNull(PigRelUdfConverter.getSqlAggFuncForPigUdf(rexCall),
+                "udf");
         aggCalls.add(relBuilder.aggregateCall(udf, aggOperands));
       }
     }
@@ -403,7 +405,9 @@ private static List getColsFromMultisetProjection(RexCall multisetProje
     for (int i = 1; i < multisetProjection.getOperands().size(); i++) {
       final RexLiteral indexLiteral =
           (RexLiteral) multisetProjection.getOperands().get(i);
-      columns.add(((BigDecimal) indexLiteral.getValue()).intValue());
+      columns.add(
+          ((BigDecimal) requireNonNull(indexLiteral.getValue(), "index"))
+              .intValue());
     }
     return columns;
   }
diff --git a/piglet/src/main/java/org/apache/calcite/piglet/PigUdfFinder.java b/piglet/src/main/java/org/apache/calcite/piglet/PigUdfFinder.java
index 45f07c605afe..292dc4053796 100644
--- a/piglet/src/main/java/org/apache/calcite/piglet/PigUdfFinder.java
+++ b/piglet/src/main/java/org/apache/calcite/piglet/PigUdfFinder.java
@@ -18,6 +18,8 @@
 
 import com.google.common.collect.ImmutableMap;
 
+
+import org.jspecify.annotations.Nullable;
 import java.lang.reflect.Method;
 import java.util.HashMap;
 import java.util.Locale;
@@ -88,12 +90,12 @@ Method findPigUdfImplementationMethod(Class clazz) {
   /**
    * Finds "exec" method from a given array of methods.
    */
-  private static Method findExecMethod(Method[] methods) {
+  private static @Nullable Method findExecMethod(Method @Nullable [] methods) {
     if (methods == null) {
       return null;
     }
 
-    Method returnedMethod = null;
+    @Nullable Method returnedMethod = null;
     for (Method method : methods) {
       if (method.getName().equals("exec")) {
         // There may be two methods named "exec", one of them just returns a
diff --git a/piglet/src/main/java/org/apache/calcite/piglet/PigUserDefinedFunction.java b/piglet/src/main/java/org/apache/calcite/piglet/PigUserDefinedFunction.java
index 2a33b87b2019..97c731a70930 100644
--- a/piglet/src/main/java/org/apache/calcite/piglet/PigUserDefinedFunction.java
+++ b/piglet/src/main/java/org/apache/calcite/piglet/PigUserDefinedFunction.java
@@ -31,16 +31,18 @@
 
 import com.google.common.collect.ImmutableList;
 
+import org.jspecify.annotations.Nullable;
+
 /** Pig user-defined function. */
 public class PigUserDefinedFunction extends SqlUserDefinedFunction {
-  public final FuncSpec funcSpec;
+  public final @Nullable FuncSpec funcSpec;
 
   private PigUserDefinedFunction(SqlIdentifier opName,
       SqlReturnTypeInference returnTypeInference,
-      SqlOperandTypeInference operandTypeInference,
+      @Nullable SqlOperandTypeInference operandTypeInference,
       SqlOperandMetadata operandMetadata,
       Function function,
-      FuncSpec funcSpec) {
+      @Nullable FuncSpec funcSpec) {
     super(opName, SqlKind.OTHER_FUNCTION, returnTypeInference,
         operandTypeInference, operandMetadata, function,
         SqlFunctionCategory.USER_DEFINED_CONSTRUCTOR, SqlSyntax.FUNCTION);
@@ -50,7 +52,7 @@ private PigUserDefinedFunction(SqlIdentifier opName,
   public PigUserDefinedFunction(String name,
       SqlReturnTypeInference returnTypeInference,
       SqlOperandMetadata operandMetadata, Function function,
-      FuncSpec funcSpec) {
+      @Nullable FuncSpec funcSpec) {
     this(new SqlIdentifier(ImmutableList.of(name), SqlParserPos.ZERO),
         returnTypeInference, null, operandMetadata, function, funcSpec);
   }
diff --git a/piglet/src/main/java/org/apache/calcite/piglet/package-info.java b/piglet/src/main/java/org/apache/calcite/piglet/package-info.java
index e54e6a13dc8b..c23a16dc81ad 100644
--- a/piglet/src/main/java/org/apache/calcite/piglet/package-info.java
+++ b/piglet/src/main/java/org/apache/calcite/piglet/package-info.java
@@ -16,4 +16,7 @@
  */
 
 /** Piglet, a Pig-like language. */
+@NullMarked
 package org.apache.calcite.piglet;
+
+import org.jspecify.annotations.NullMarked;

From 1712a2ed483a2af70723ed43d4f9e4490a94681e Mon Sep 17 00:00:00 2001
From: Vladimir Sitnikov 
Date: Tue, 25 Aug 2026 21:28:33 +0300
Subject: [PATCH 553/562] [CALCITE-7736] Finish the :innodb enumerable the
 module's own commit left behind

The query enumerable still had the non-null element type, and its anonymous
form ran into uber/NullAway#1746 once the type argument admitted null. It is a
named class now, like the ones in :redis, :mongodb and :cassandra.

Co-Authored-By: Claude Opus 5 
---
 .../calcite/adapter/innodb/InnodbTable.java   | 146 +++++++++++-------
 1 file changed, 93 insertions(+), 53 deletions(-)

diff --git a/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbTable.java b/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbTable.java
index ebe5940df5db..c7331aec8685 100644
--- a/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbTable.java
+++ b/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbTable.java
@@ -117,7 +117,7 @@ public Set getIndexesNameSet() {
         .build();
   }
 
-  public Enumerable query(final TableReaderFactory tableReaderFactory) {
+  public Enumerable<@Nullable Object> query(final TableReaderFactory tableReaderFactory) {
     return query(tableReaderFactory, ImmutableList.of(), ImmutableList.of(),
         IndexCondition.EMPTY_CONDITION, true);
   }
@@ -132,7 +132,7 @@ public Enumerable query(final TableReaderFactory tableReaderFactory) {
    * @param ascOrder           if scan ordering is ascending
    * @return Enumerator of results
    */
-  public Enumerable query(
+  public Enumerable<@Nullable Object> query(
       final TableReaderFactory tableReaderFactory,
       final List> fields,
       final List> selectFields,
@@ -176,56 +176,10 @@ public Enumerable query(
 
     TableReader tableReader = tableReaderFactory.createTableReader(tableName);
     tableReader.open();
-    return new AbstractEnumerable() {
-      @Override public Enumerator enumerator() {
-        Iterator resultIterator;
-        LOGGER.debug("Create query iterator, queryType={}, indexName={}, "
-                + "pointQueryKey={}, projection={}, rangeQueryKey={}{} AND {}{}, "
-                + "ascOrder={}", queryType, indexName, pointQueryKey,
-            selectedColumnNames, rangeQueryLowerKey, rangeQueryLowerOp,
-            rangeQueryUpperKey, rangeQueryUpperOp, ascOrder);
-        switch (queryType) {
-          case PK_POINT_QUERY:
-            resultIterator =
-                RecordIterator.create(tableReader
-                    .queryByPrimaryKey(pointQueryKey, selectedColumnNames));
-            break;
-          case PK_RANGE_QUERY:
-            resultIterator =
-                tableReader.getRangeQueryIterator(rangeQueryLowerKey,
-                    rangeQueryLowerOp, rangeQueryUpperKey, rangeQueryUpperOp,
-                    selectedColumnNames, ascOrder);
-            break;
-          case SK_POINT_QUERY:
-            resultIterator =
-                tableReader.getRecordIteratorBySk(indexName, pointQueryKey,
-                    ComparisonOperator.GTE, pointQueryKey,
-                    ComparisonOperator.LTE, selectedColumnNames, ascOrder);
-            break;
-          case SK_RANGE_QUERY:
-          case SK_FULL_SCAN:
-            resultIterator =
-                tableReader.getRecordIteratorBySk(indexName, rangeQueryLowerKey,
-                    rangeQueryLowerOp, rangeQueryUpperKey, rangeQueryUpperOp,
-                    selectedColumnNames, ascOrder);
-            break;
-          case PK_FULL_SCAN:
-            resultIterator =
-                tableReader.getQueryAllIterator(selectedColumnNames, ascOrder);
-            break;
-          default:
-            throw new AssertionError("query type is invalid");
-        }
-
-        RelDataType rowType = resultRowType.apply(typeFactory);
-        return new InnodbEnumerator(resultIterator, rowType) {
-          @Override public void close() {
-            super.close();
-            tableReader.close();
-          }
-        };
-      }
-    };
+    return new InnodbEnumerable(tableReader, queryType, indexName, pointQueryKey,
+        selectedColumnNames, rangeQueryLowerKey, rangeQueryLowerOp,
+        rangeQueryUpperKey, rangeQueryUpperOp, ascOrder, resultRowType,
+        typeFactory);
   }
 
   @Override public  Queryable asQueryable(QueryProvider queryProvider,
@@ -275,11 +229,97 @@ private TableReaderFactory getTableReaderFactory() {
      * @see org.apache.calcite.adapter.innodb.InnodbMethod#INNODB_QUERYABLE_QUERY
      */
     @SuppressWarnings("UnusedDeclaration")
-    public Enumerable query(List> fields,
+    public Enumerable<@Nullable Object> query(List> fields,
         List> selectFields,
         IndexCondition condition, Boolean ascOrder) {
       return getTable().query(getTableReaderFactory(), fields, selectFields,
           condition, ascOrder);
     }
   }
+
+  /** Enumerable over the records an InnoDB query reads. */
+  private static class InnodbEnumerable extends AbstractEnumerable<@Nullable Object> {
+    private final TableReader tableReader;
+    private final QueryType queryType;
+    private final String indexName;
+    private final List pointQueryKey;
+    private final List selectedColumnNames;
+    private final List rangeQueryLowerKey;
+    private final ComparisonOperator rangeQueryLowerOp;
+    private final List rangeQueryUpperKey;
+    private final ComparisonOperator rangeQueryUpperOp;
+    private final Boolean ascOrder;
+    private final RelProtoDataType resultRowType;
+    private final RelDataTypeFactory typeFactory;
+
+    InnodbEnumerable(TableReader tableReader, QueryType queryType,
+        String indexName, List pointQueryKey,
+        List selectedColumnNames, List rangeQueryLowerKey,
+        ComparisonOperator rangeQueryLowerOp, List rangeQueryUpperKey,
+        ComparisonOperator rangeQueryUpperOp, Boolean ascOrder,
+        RelProtoDataType resultRowType, RelDataTypeFactory typeFactory) {
+      this.tableReader = tableReader;
+      this.queryType = queryType;
+      this.indexName = indexName;
+      this.pointQueryKey = pointQueryKey;
+      this.selectedColumnNames = selectedColumnNames;
+      this.rangeQueryLowerKey = rangeQueryLowerKey;
+      this.rangeQueryLowerOp = rangeQueryLowerOp;
+      this.rangeQueryUpperKey = rangeQueryUpperKey;
+      this.rangeQueryUpperOp = rangeQueryUpperOp;
+      this.ascOrder = ascOrder;
+      this.resultRowType = resultRowType;
+      this.typeFactory = typeFactory;
+    }
+
+    @Override public Enumerator<@Nullable Object> enumerator() {
+          Iterator resultIterator;
+          LOGGER.debug("Create query iterator, queryType={}, indexName={}, "
+                  + "pointQueryKey={}, projection={}, rangeQueryKey={}{} AND {}{}, "
+                  + "ascOrder={}", queryType, indexName, pointQueryKey,
+              selectedColumnNames, rangeQueryLowerKey, rangeQueryLowerOp,
+              rangeQueryUpperKey, rangeQueryUpperOp, ascOrder);
+          switch (queryType) {
+            case PK_POINT_QUERY:
+              resultIterator =
+                  RecordIterator.create(tableReader
+                      .queryByPrimaryKey(pointQueryKey, selectedColumnNames));
+              break;
+            case PK_RANGE_QUERY:
+              resultIterator =
+                  tableReader.getRangeQueryIterator(rangeQueryLowerKey,
+                      rangeQueryLowerOp, rangeQueryUpperKey, rangeQueryUpperOp,
+                      selectedColumnNames, ascOrder);
+              break;
+            case SK_POINT_QUERY:
+              resultIterator =
+                  tableReader.getRecordIteratorBySk(indexName, pointQueryKey,
+                      ComparisonOperator.GTE, pointQueryKey,
+                      ComparisonOperator.LTE, selectedColumnNames, ascOrder);
+              break;
+            case SK_RANGE_QUERY:
+            case SK_FULL_SCAN:
+              resultIterator =
+                  tableReader.getRecordIteratorBySk(indexName, rangeQueryLowerKey,
+                      rangeQueryLowerOp, rangeQueryUpperKey, rangeQueryUpperOp,
+                      selectedColumnNames, ascOrder);
+              break;
+            case PK_FULL_SCAN:
+              resultIterator =
+                  tableReader.getQueryAllIterator(selectedColumnNames, ascOrder);
+              break;
+            default:
+              throw new AssertionError("query type is invalid");
+          }
+
+          RelDataType rowType = resultRowType.apply(typeFactory);
+          return new InnodbEnumerator(resultIterator, rowType) {
+            @Override public void close() {
+              super.close();
+              tableReader.close();
+            }
+          };
+    }
+  }
+
 }

From 4ccd4070fcbe98592e4c09db7ae4ed2a54ad791b Mon Sep 17 00:00:00 2001
From: Vladimir Sitnikov 
Date: Tue, 25 Aug 2026 21:29:25 +0300
Subject: [PATCH 554/562] [CALCITE-7736] Let PigAggregate take the absent
 grouping sets its base class allows

Aggregate.copy is handed no grouping sets when the aggregate has a single
group, and PigAggregate.copy passes that straight to its own constructor,
which declared them required.

Co-Authored-By: Claude Opus 5 
---
 .../java/org/apache/calcite/adapter/pig/PigAggregate.java     | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pig/src/main/java/org/apache/calcite/adapter/pig/PigAggregate.java b/pig/src/main/java/org/apache/calcite/adapter/pig/PigAggregate.java
index 39b839c9734a..946b10fadd24 100644
--- a/pig/src/main/java/org/apache/calcite/adapter/pig/PigAggregate.java
+++ b/pig/src/main/java/org/apache/calcite/adapter/pig/PigAggregate.java
@@ -45,7 +45,7 @@ public class PigAggregate extends Aggregate implements PigRel {
   /** Creates a PigAggregate. */
   public PigAggregate(RelOptCluster cluster, RelTraitSet traitSet,
       RelNode input, ImmutableBitSet groupSet,
-      List groupSets, List aggCalls) {
+      @Nullable List groupSets, List aggCalls) {
     super(cluster, traitSet, ImmutableList.of(), input, groupSet, groupSets, aggCalls);
     assert getConvention() == PigRel.CONVENTION;
   }
@@ -53,7 +53,7 @@ public PigAggregate(RelOptCluster cluster, RelTraitSet traitSet,
   @Deprecated // to be removed before 2.0
   public PigAggregate(RelOptCluster cluster, RelTraitSet traitSet,
       RelNode input, boolean indicator, ImmutableBitSet groupSet,
-      List groupSets, List aggCalls) {
+      @Nullable List groupSets, List aggCalls) {
     this(cluster, traitSet, input, groupSet, groupSets, aggCalls);
     checkIndicator(indicator);
   }

From 1a27df0023ddade5cabda49f7cb819106de3a9e0 Mon Sep 17 00:00:00 2001
From: Vladimir Sitnikov 
Date: Tue, 25 Aug 2026 21:32:17 +0300
Subject: [PATCH 555/562] [CALCITE-7736] Put the imports the module commits
 added in the order autostyle wants

Lint:skip

Co-Authored-By: Claude Opus 5 
---
 .../arrow/AbstractArrowEnumerator.java        |  2 +-
 .../adapter/arrow/ArrowEnumerable.java        |  2 +-
 .../calcite/adapter/arrow/ArrowTable.java     |  2 -
 .../cassandra/CassandraSchemaFactory.java     |  1 -
 .../adapter/innodb/InnodbSchemaFactory.java   |  2 +-
 .../calcite/adapter/innodb/InnodbTable.java   | 95 +++++++++----------
 .../linq4j/ExtendedOrderedEnumerable.java     |  4 +-
 .../adapter/mongodb/MongoEnumerator.java      |  1 -
 .../calcite/adapter/mongodb/MongoProject.java |  2 +-
 .../calcite/adapter/mongodb/MongoRules.java   |  3 +-
 .../adapter/mongodb/MongoSchemaFactory.java   |  2 -
 .../calcite/adapter/mongodb/MongoSort.java    |  2 +-
 .../calcite/adapter/mongodb/MongoTable.java   |  5 +-
 .../mongodb/MongoToEnumerableConverter.java   |  2 +-
 .../calcite/adapter/pig/PigAggregate.java     |  2 +-
 .../apache/calcite/adapter/pig/PigFilter.java |  2 +-
 .../apache/calcite/adapter/pig/PigJoin.java   |  2 +-
 .../calcite/adapter/pig/PigProject.java       |  2 +-
 .../apache/calcite/adapter/pig/PigRel.java    |  2 +-
 .../piglet/DynamicTupleRecordType.java        |  2 +-
 .../org/apache/calcite/piglet/Handler.java    |  2 +-
 .../apache/calcite/piglet/PigRelBuilder.java  |  2 +-
 .../calcite/piglet/PigRelOpInnerVisitor.java  |  2 +-
 .../calcite/piglet/PigRelOpVisitor.java       |  4 +-
 .../apache/calcite/piglet/PigRelOpWalker.java |  2 +-
 .../calcite/piglet/PigRelUdfConverter.java    |  2 +-
 .../calcite/piglet/PigToSqlAggregateRule.java |  1 +
 .../apache/calcite/piglet/PigUdfFinder.java   |  2 +-
 .../adapter/redis/RedisEnumerator.java        |  2 +-
 .../adapter/redis/RedisSchemaFactory.java     |  2 -
 .../spark/EnumerableToSparkConverter.java     |  6 --
 .../calcite/adapter/spark/SparkMethod.java    |  2 +-
 .../adapter/splunk/SplunkTableScan.java       |  2 +-
 33 files changed, 76 insertions(+), 92 deletions(-)

diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/AbstractArrowEnumerator.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/AbstractArrowEnumerator.java
index 226f8ff54b23..2b9ede090d41 100644
--- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/AbstractArrowEnumerator.java
+++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/AbstractArrowEnumerator.java
@@ -28,8 +28,8 @@
 import org.apache.arrow.vector.types.TimeUnit;
 import org.apache.arrow.vector.types.pojo.ArrowType;
 
-
 import org.jspecify.annotations.Nullable;
+
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowEnumerable.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowEnumerable.java
index ef20e1ea2793..fce539797c6e 100644
--- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowEnumerable.java
+++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowEnumerable.java
@@ -24,8 +24,8 @@
 import org.apache.arrow.vector.ipc.ArrowFileReader;
 import org.apache.arrow.vector.types.pojo.Schema;
 
-
 import org.jspecify.annotations.Nullable;
+
 import java.util.List;
 
 /**
diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTable.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTable.java
index 0a2ee330d410..961becda7aab 100644
--- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTable.java
+++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTable.java
@@ -52,8 +52,6 @@
 import java.lang.reflect.Type;
 import java.util.List;
 
-import static java.util.Objects.requireNonNull;
-
 /**
  * Table backed by an Apache Arrow file.
  *
diff --git a/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraSchemaFactory.java b/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraSchemaFactory.java
index 8feb201b7156..6f84accd04df 100644
--- a/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraSchemaFactory.java
+++ b/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraSchemaFactory.java
@@ -35,7 +35,6 @@
 import java.util.stream.Collectors;
 
 import static java.lang.Integer.parseInt;
-
 import static java.util.Objects.requireNonNull;
 
 /**
diff --git a/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbSchemaFactory.java b/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbSchemaFactory.java
index 7bb14e96ae40..a00ece5e1933 100644
--- a/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbSchemaFactory.java
+++ b/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbSchemaFactory.java
@@ -21,9 +21,9 @@
 import org.apache.calcite.schema.SchemaPlus;
 
 import java.util.List;
+import java.util.Map;
 
 import static java.util.Objects.requireNonNull;
-import java.util.Map;
 
 /**
  * Factory that creates a {@link InnodbSchema}.
diff --git a/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbTable.java b/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbTable.java
index c7331aec8685..9a2c76450c15 100644
--- a/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbTable.java
+++ b/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbTable.java
@@ -49,11 +49,10 @@
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 
+import org.jspecify.annotations.Nullable;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-
-import org.jspecify.annotations.Nullable;
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
@@ -273,52 +272,52 @@ private static class InnodbEnumerable extends AbstractEnumerable<@Nullable Objec
     }
 
     @Override public Enumerator<@Nullable Object> enumerator() {
-          Iterator resultIterator;
-          LOGGER.debug("Create query iterator, queryType={}, indexName={}, "
-                  + "pointQueryKey={}, projection={}, rangeQueryKey={}{} AND {}{}, "
-                  + "ascOrder={}", queryType, indexName, pointQueryKey,
-              selectedColumnNames, rangeQueryLowerKey, rangeQueryLowerOp,
-              rangeQueryUpperKey, rangeQueryUpperOp, ascOrder);
-          switch (queryType) {
-            case PK_POINT_QUERY:
-              resultIterator =
-                  RecordIterator.create(tableReader
-                      .queryByPrimaryKey(pointQueryKey, selectedColumnNames));
-              break;
-            case PK_RANGE_QUERY:
-              resultIterator =
-                  tableReader.getRangeQueryIterator(rangeQueryLowerKey,
-                      rangeQueryLowerOp, rangeQueryUpperKey, rangeQueryUpperOp,
-                      selectedColumnNames, ascOrder);
-              break;
-            case SK_POINT_QUERY:
-              resultIterator =
-                  tableReader.getRecordIteratorBySk(indexName, pointQueryKey,
-                      ComparisonOperator.GTE, pointQueryKey,
-                      ComparisonOperator.LTE, selectedColumnNames, ascOrder);
-              break;
-            case SK_RANGE_QUERY:
-            case SK_FULL_SCAN:
-              resultIterator =
-                  tableReader.getRecordIteratorBySk(indexName, rangeQueryLowerKey,
-                      rangeQueryLowerOp, rangeQueryUpperKey, rangeQueryUpperOp,
-                      selectedColumnNames, ascOrder);
-              break;
-            case PK_FULL_SCAN:
-              resultIterator =
-                  tableReader.getQueryAllIterator(selectedColumnNames, ascOrder);
-              break;
-            default:
-              throw new AssertionError("query type is invalid");
-          }
-
-          RelDataType rowType = resultRowType.apply(typeFactory);
-          return new InnodbEnumerator(resultIterator, rowType) {
-            @Override public void close() {
-              super.close();
-              tableReader.close();
-            }
-          };
+      Iterator resultIterator;
+      LOGGER.debug("Create query iterator, queryType={}, indexName={}, "
+              + "pointQueryKey={}, projection={}, rangeQueryKey={}{} AND {}{}, "
+              + "ascOrder={}", queryType, indexName, pointQueryKey,
+          selectedColumnNames, rangeQueryLowerKey, rangeQueryLowerOp,
+          rangeQueryUpperKey, rangeQueryUpperOp, ascOrder);
+      switch (queryType) {
+      case PK_POINT_QUERY:
+        resultIterator =
+            RecordIterator.create(tableReader
+                .queryByPrimaryKey(pointQueryKey, selectedColumnNames));
+        break;
+      case PK_RANGE_QUERY:
+        resultIterator =
+            tableReader.getRangeQueryIterator(rangeQueryLowerKey,
+                rangeQueryLowerOp, rangeQueryUpperKey, rangeQueryUpperOp,
+                selectedColumnNames, ascOrder);
+        break;
+      case SK_POINT_QUERY:
+        resultIterator =
+            tableReader.getRecordIteratorBySk(indexName, pointQueryKey,
+                ComparisonOperator.GTE, pointQueryKey,
+                ComparisonOperator.LTE, selectedColumnNames, ascOrder);
+        break;
+      case SK_RANGE_QUERY:
+      case SK_FULL_SCAN:
+        resultIterator =
+            tableReader.getRecordIteratorBySk(indexName, rangeQueryLowerKey,
+                rangeQueryLowerOp, rangeQueryUpperKey, rangeQueryUpperOp,
+                selectedColumnNames, ascOrder);
+        break;
+      case PK_FULL_SCAN:
+        resultIterator =
+            tableReader.getQueryAllIterator(selectedColumnNames, ascOrder);
+        break;
+      default:
+        throw new AssertionError("query type is invalid");
+      }
+
+      RelDataType rowType = resultRowType.apply(typeFactory);
+      return new InnodbEnumerator(resultIterator, rowType) {
+        @Override public void close() {
+          super.close();
+          tableReader.close();
+        }
+      };
     }
   }
 
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedOrderedEnumerable.java b/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedOrderedEnumerable.java
index 82e86f9ebe5d..cb6ec685bf67 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedOrderedEnumerable.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedOrderedEnumerable.java
@@ -16,10 +16,10 @@
  */
 package org.apache.calcite.linq4j;
 
-import org.jspecify.annotations.Nullable;
-
 import org.apache.calcite.linq4j.function.Function1;
 
+import org.jspecify.annotations.Nullable;
+
 import java.util.Comparator;
 
 /**
diff --git a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoEnumerator.java b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoEnumerator.java
index 8eb7f85efed1..d86bdfcb168a 100644
--- a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoEnumerator.java
+++ b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoEnumerator.java
@@ -38,7 +38,6 @@
 import java.util.Map;
 
 import static java.lang.String.format;
-
 import static java.util.Objects.requireNonNull;
 
 /** Enumerator that reads from a MongoDB collection. */
diff --git a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoProject.java b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoProject.java
index bedf2f53aa7d..9917d4e7f823 100644
--- a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoProject.java
+++ b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoProject.java
@@ -35,9 +35,9 @@
 import org.jspecify.annotations.Nullable;
 
 import java.util.ArrayList;
+import java.util.List;
 
 import static java.util.Objects.requireNonNull;
-import java.util.List;
 
 /**
  * Implementation of {@link org.apache.calcite.rel.core.Project}
diff --git a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoRules.java b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoRules.java
index a3120f15910c..d43a7f67a025 100644
--- a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoRules.java
+++ b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoRules.java
@@ -46,10 +46,9 @@
 import org.apache.calcite.util.Util;
 import org.apache.calcite.util.trace.CalciteTrace;
 
+import org.jspecify.annotations.Nullable;
 import org.slf4j.Logger;
 
-
-import org.jspecify.annotations.Nullable;
 import java.util.AbstractList;
 import java.util.HashMap;
 import java.util.List;
diff --git a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoSchemaFactory.java b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoSchemaFactory.java
index 2e633082f2ed..8fc451b8edd8 100644
--- a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoSchemaFactory.java
+++ b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoSchemaFactory.java
@@ -25,8 +25,6 @@
 import com.mongodb.MongoClientSettings;
 import com.mongodb.MongoCredential;
 
-
-import org.jspecify.annotations.Nullable;
 import java.util.Map;
 
 import static java.util.Objects.requireNonNull;
diff --git a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoSort.java b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoSort.java
index b6d1c9de4b7e..c9c4d5a16e2a 100644
--- a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoSort.java
+++ b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoSort.java
@@ -33,9 +33,9 @@
 import org.jspecify.annotations.Nullable;
 
 import java.util.ArrayList;
+import java.util.List;
 
 import static java.util.Objects.requireNonNull;
-import java.util.List;
 
 /**
 * Implementation of {@link org.apache.calcite.rel.core.Sort}
diff --git a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoTable.java b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoTable.java
index 3852f34748b4..35462016cd8a 100644
--- a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoTable.java
+++ b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoTable.java
@@ -41,15 +41,14 @@
 import org.bson.BsonDocument;
 import org.bson.Document;
 import org.bson.conversions.Bson;
-
-
 import org.jspecify.annotations.Nullable;
+
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
 
 import static java.util.Objects.requireNonNull;
-import java.util.Map;
 
 /**
  * Table based on a MongoDB collection.
diff --git a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoToEnumerableConverter.java b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoToEnumerableConverter.java
index dd3d5399e938..577e1e58f2e7 100644
--- a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoToEnumerableConverter.java
+++ b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoToEnumerableConverter.java
@@ -43,9 +43,9 @@
 import org.jspecify.annotations.Nullable;
 
 import java.util.AbstractList;
+import java.util.List;
 
 import static java.util.Objects.requireNonNull;
-import java.util.List;
 
 /**
  * Relational expression representing a scan of a table in a Mongo data source.
diff --git a/pig/src/main/java/org/apache/calcite/adapter/pig/PigAggregate.java b/pig/src/main/java/org/apache/calcite/adapter/pig/PigAggregate.java
index 946b10fadd24..69a70da2ea6e 100644
--- a/pig/src/main/java/org/apache/calcite/adapter/pig/PigAggregate.java
+++ b/pig/src/main/java/org/apache/calcite/adapter/pig/PigAggregate.java
@@ -29,8 +29,8 @@
 
 import com.google.common.collect.ImmutableList;
 
-
 import org.jspecify.annotations.Nullable;
+
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
diff --git a/pig/src/main/java/org/apache/calcite/adapter/pig/PigFilter.java b/pig/src/main/java/org/apache/calcite/adapter/pig/PigFilter.java
index 31538d5ff65a..7c7d4a6eee77 100644
--- a/pig/src/main/java/org/apache/calcite/adapter/pig/PigFilter.java
+++ b/pig/src/main/java/org/apache/calcite/adapter/pig/PigFilter.java
@@ -27,8 +27,8 @@
 import org.apache.calcite.rex.RexLiteral;
 import org.apache.calcite.rex.RexNode;
 
-
 import org.jspecify.annotations.Nullable;
+
 import java.util.ArrayList;
 import java.util.List;
 
diff --git a/pig/src/main/java/org/apache/calcite/adapter/pig/PigJoin.java b/pig/src/main/java/org/apache/calcite/adapter/pig/PigJoin.java
index 16ea0cb4366d..997522ff9a69 100644
--- a/pig/src/main/java/org/apache/calcite/adapter/pig/PigJoin.java
+++ b/pig/src/main/java/org/apache/calcite/adapter/pig/PigJoin.java
@@ -30,8 +30,8 @@
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 
-
 import org.jspecify.annotations.Nullable;
+
 import java.util.ArrayList;
 import java.util.List;
 
diff --git a/pig/src/main/java/org/apache/calcite/adapter/pig/PigProject.java b/pig/src/main/java/org/apache/calcite/adapter/pig/PigProject.java
index f98a21314179..c43bfdad3985 100644
--- a/pig/src/main/java/org/apache/calcite/adapter/pig/PigProject.java
+++ b/pig/src/main/java/org/apache/calcite/adapter/pig/PigProject.java
@@ -27,8 +27,8 @@
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 
-
 import org.jspecify.annotations.Nullable;
+
 import java.util.List;
 
 /** Implementation of {@link org.apache.calcite.rel.core.Project} in
diff --git a/pig/src/main/java/org/apache/calcite/adapter/pig/PigRel.java b/pig/src/main/java/org/apache/calcite/adapter/pig/PigRel.java
index cdd5101e0263..530bb06c5ace 100644
--- a/pig/src/main/java/org/apache/calcite/adapter/pig/PigRel.java
+++ b/pig/src/main/java/org/apache/calcite/adapter/pig/PigRel.java
@@ -20,9 +20,9 @@
 import org.apache.calcite.rel.RelNode;
 
 import java.util.ArrayList;
+import java.util.List;
 
 import static java.util.Objects.requireNonNull;
-import java.util.List;
 
 /**
  * Relational expression that uses the Pig calling convention.
diff --git a/piglet/src/main/java/org/apache/calcite/piglet/DynamicTupleRecordType.java b/piglet/src/main/java/org/apache/calcite/piglet/DynamicTupleRecordType.java
index 4a34791d1679..8555d4281449 100644
--- a/piglet/src/main/java/org/apache/calcite/piglet/DynamicTupleRecordType.java
+++ b/piglet/src/main/java/org/apache/calcite/piglet/DynamicTupleRecordType.java
@@ -20,8 +20,8 @@
 import org.apache.calcite.rel.type.RelDataTypeFactory;
 import org.apache.calcite.rel.type.RelDataTypeField;
 
-
 import org.jspecify.annotations.Nullable;
+
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
diff --git a/piglet/src/main/java/org/apache/calcite/piglet/Handler.java b/piglet/src/main/java/org/apache/calcite/piglet/Handler.java
index c9ea5fa73a2d..6dd03aeba116 100644
--- a/piglet/src/main/java/org/apache/calcite/piglet/Handler.java
+++ b/piglet/src/main/java/org/apache/calcite/piglet/Handler.java
@@ -35,9 +35,9 @@
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 import static java.util.Objects.requireNonNull;
-import java.util.Map;
 
 /**
  * Walks over a Piglet AST and calls the corresponding methods in a
diff --git a/piglet/src/main/java/org/apache/calcite/piglet/PigRelBuilder.java b/piglet/src/main/java/org/apache/calcite/piglet/PigRelBuilder.java
index fe6a4bfa2a22..a6b2b5d099f5 100644
--- a/piglet/src/main/java/org/apache/calcite/piglet/PigRelBuilder.java
+++ b/piglet/src/main/java/org/apache/calcite/piglet/PigRelBuilder.java
@@ -52,8 +52,8 @@
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 
-
 import org.jspecify.annotations.Nullable;
+
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
diff --git a/piglet/src/main/java/org/apache/calcite/piglet/PigRelOpInnerVisitor.java b/piglet/src/main/java/org/apache/calcite/piglet/PigRelOpInnerVisitor.java
index a8b0c861bf05..d1a0d4fcd4fe 100644
--- a/piglet/src/main/java/org/apache/calcite/piglet/PigRelOpInnerVisitor.java
+++ b/piglet/src/main/java/org/apache/calcite/piglet/PigRelOpInnerVisitor.java
@@ -41,8 +41,8 @@
 
 import com.google.common.collect.ImmutableSet;
 
-
 import org.jspecify.annotations.Nullable;
+
 import java.util.ArrayDeque;
 import java.util.ArrayList;
 import java.util.Deque;
diff --git a/piglet/src/main/java/org/apache/calcite/piglet/PigRelOpVisitor.java b/piglet/src/main/java/org/apache/calcite/piglet/PigRelOpVisitor.java
index 18e2bf554a91..49e5c3534e9b 100644
--- a/piglet/src/main/java/org/apache/calcite/piglet/PigRelOpVisitor.java
+++ b/piglet/src/main/java/org/apache/calcite/piglet/PigRelOpVisitor.java
@@ -69,8 +69,8 @@
 
 import com.google.common.collect.ImmutableList;
 
-
 import org.jspecify.annotations.Nullable;
+
 import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -78,9 +78,9 @@
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 
 import static java.util.Objects.requireNonNull;
-import java.util.Set;
 
 /**
  * Visits Pig logical operators and converts them into corresponding relational
diff --git a/piglet/src/main/java/org/apache/calcite/piglet/PigRelOpWalker.java b/piglet/src/main/java/org/apache/calcite/piglet/PigRelOpWalker.java
index cc5df3e72ae3..5ca47d8f7eb0 100644
--- a/piglet/src/main/java/org/apache/calcite/piglet/PigRelOpWalker.java
+++ b/piglet/src/main/java/org/apache/calcite/piglet/PigRelOpWalker.java
@@ -25,8 +25,8 @@
 import org.apache.pig.newplan.logical.relational.LogicalRelationalNodesVisitor;
 import org.apache.pig.newplan.logical.relational.LogicalRelationalOperator;
 
-
 import org.jspecify.annotations.Nullable;
+
 import java.util.Collection;
 
 /**
diff --git a/piglet/src/main/java/org/apache/calcite/piglet/PigRelUdfConverter.java b/piglet/src/main/java/org/apache/calcite/piglet/PigRelUdfConverter.java
index ad2a909370bc..bba18c789d76 100644
--- a/piglet/src/main/java/org/apache/calcite/piglet/PigRelUdfConverter.java
+++ b/piglet/src/main/java/org/apache/calcite/piglet/PigRelUdfConverter.java
@@ -32,8 +32,8 @@
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 
-
 import org.jspecify.annotations.Nullable;
+
 import java.lang.reflect.Method;
 import java.util.Map;
 
diff --git a/piglet/src/main/java/org/apache/calcite/piglet/PigToSqlAggregateRule.java b/piglet/src/main/java/org/apache/calcite/piglet/PigToSqlAggregateRule.java
index 0a46dd28ef33..ff1787cdd2c8 100644
--- a/piglet/src/main/java/org/apache/calcite/piglet/PigToSqlAggregateRule.java
+++ b/piglet/src/main/java/org/apache/calcite/piglet/PigToSqlAggregateRule.java
@@ -45,6 +45,7 @@
 import java.util.Map;
 
 import static org.apache.calcite.piglet.PigTypes.TYPE_FACTORY;
+
 import static java.util.Objects.requireNonNull;
 
 /**
diff --git a/piglet/src/main/java/org/apache/calcite/piglet/PigUdfFinder.java b/piglet/src/main/java/org/apache/calcite/piglet/PigUdfFinder.java
index 292dc4053796..d345068db9b1 100644
--- a/piglet/src/main/java/org/apache/calcite/piglet/PigUdfFinder.java
+++ b/piglet/src/main/java/org/apache/calcite/piglet/PigUdfFinder.java
@@ -18,8 +18,8 @@
 
 import com.google.common.collect.ImmutableMap;
 
-
 import org.jspecify.annotations.Nullable;
+
 import java.lang.reflect.Method;
 import java.util.HashMap;
 import java.util.Locale;
diff --git a/redis/src/main/java/org/apache/calcite/adapter/redis/RedisEnumerator.java b/redis/src/main/java/org/apache/calcite/adapter/redis/RedisEnumerator.java
index 7c9440f40f43..9a6c874858da 100644
--- a/redis/src/main/java/org/apache/calcite/adapter/redis/RedisEnumerator.java
+++ b/redis/src/main/java/org/apache/calcite/adapter/redis/RedisEnumerator.java
@@ -19,8 +19,8 @@
 import org.apache.calcite.linq4j.Enumerator;
 import org.apache.calcite.linq4j.Linq4j;
 
-
 import org.jspecify.annotations.Nullable;
+
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
diff --git a/redis/src/main/java/org/apache/calcite/adapter/redis/RedisSchemaFactory.java b/redis/src/main/java/org/apache/calcite/adapter/redis/RedisSchemaFactory.java
index c4b5b65542b6..8c0962108aa6 100644
--- a/redis/src/main/java/org/apache/calcite/adapter/redis/RedisSchemaFactory.java
+++ b/redis/src/main/java/org/apache/calcite/adapter/redis/RedisSchemaFactory.java
@@ -20,8 +20,6 @@
 import org.apache.calcite.schema.SchemaFactory;
 import org.apache.calcite.schema.SchemaPlus;
 
-
-import org.jspecify.annotations.Nullable;
 import java.util.List;
 import java.util.Map;
 
diff --git a/spark/src/main/java/org/apache/calcite/adapter/spark/EnumerableToSparkConverter.java b/spark/src/main/java/org/apache/calcite/adapter/spark/EnumerableToSparkConverter.java
index 3255af6c9f2d..36952c2329ec 100644
--- a/spark/src/main/java/org/apache/calcite/adapter/spark/EnumerableToSparkConverter.java
+++ b/spark/src/main/java/org/apache/calcite/adapter/spark/EnumerableToSparkConverter.java
@@ -17,12 +17,6 @@
 package org.apache.calcite.adapter.spark;
 
 import org.apache.calcite.adapter.enumerable.EnumerableConvention;
-import org.apache.calcite.adapter.enumerable.JavaRowFormat;
-import org.apache.calcite.adapter.enumerable.PhysType;
-import org.apache.calcite.adapter.enumerable.PhysTypeImpl;
-import org.apache.calcite.linq4j.tree.BlockBuilder;
-import org.apache.calcite.linq4j.tree.Expression;
-import org.apache.calcite.linq4j.tree.Expressions;
 import org.apache.calcite.plan.ConventionTraitDef;
 import org.apache.calcite.plan.RelOptCluster;
 import org.apache.calcite.plan.RelOptCost;
diff --git a/spark/src/main/java/org/apache/calcite/adapter/spark/SparkMethod.java b/spark/src/main/java/org/apache/calcite/adapter/spark/SparkMethod.java
index 4e8c5238fcde..449290abaab4 100644
--- a/spark/src/main/java/org/apache/calcite/adapter/spark/SparkMethod.java
+++ b/spark/src/main/java/org/apache/calcite/adapter/spark/SparkMethod.java
@@ -24,8 +24,8 @@
 import org.apache.spark.api.java.JavaSparkContext;
 import org.apache.spark.api.java.function.FlatMapFunction;
 
-
 import org.jspecify.annotations.Nullable;
+
 import java.lang.reflect.Method;
 import java.util.HashMap;
 
diff --git a/splunk/src/main/java/org/apache/calcite/adapter/splunk/SplunkTableScan.java b/splunk/src/main/java/org/apache/calcite/adapter/splunk/SplunkTableScan.java
index 1cbef25c1620..43d96b5b34c1 100644
--- a/splunk/src/main/java/org/apache/calcite/adapter/splunk/SplunkTableScan.java
+++ b/splunk/src/main/java/org/apache/calcite/adapter/splunk/SplunkTableScan.java
@@ -39,8 +39,8 @@
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 
-
 import org.jspecify.annotations.Nullable;
+
 import java.lang.reflect.Method;
 import java.util.AbstractList;
 import java.util.Arrays;

From bddd775a7f539f13c4fd4fe3b982009c042159ae Mon Sep 17 00:00:00 2001
From: Vladimir Sitnikov 
Date: Tue, 25 Aug 2026 21:41:52 +0300
Subject: [PATCH 556/562] [CALCITE-7736] Put :geode under nullness verification

A Geode entry has no value for a field it does not carry, which the converters
return and the enumerator hands on. The two schema factories read four operands
out of the model and passed them on unchecked; requireNonNull names whichever
one is missing. The lazily built table maps and the limit an implement context
may not have say so.

Region's value type is a bytecode wildcard whose upper bound reads as nullable,
so the value constraint cannot be held in a Class; a raw Class avoids it.
See uber/NullAway#1732.

Co-Authored-By: Claude Opus 5 
---
 build.gradle.kts                              |  2 +-
 .../adapter/geode/rel/GeodeAggregate.java     |  9 ++--
 .../adapter/geode/rel/GeodeEnumerator.java    |  2 +-
 .../adapter/geode/rel/GeodeFilter.java        |  4 +-
 .../calcite/adapter/geode/rel/GeodeRel.java   |  8 +++-
 .../adapter/geode/rel/GeodeSchema.java        |  4 +-
 .../adapter/geode/rel/GeodeSchemaFactory.java | 17 +++++--
 .../calcite/adapter/geode/rel/GeodeSort.java  |  3 +-
 .../calcite/adapter/geode/rel/GeodeTable.java | 45 ++++++++++++-------
 .../adapter/geode/rel/package-info.java       |  3 ++
 .../geode/simple/GeodeSimpleEnumerator.java   |  9 ++--
 .../geode/simple/GeodeSimpleSchema.java       |  4 +-
 .../simple/GeodeSimpleSchemaFactory.java      | 17 +++++--
 .../adapter/geode/simple/package-info.java    |  3 ++
 .../adapter/geode/util/GeodeUtils.java        | 23 ++++++----
 .../adapter/geode/util/package-info.java      |  3 ++
 16 files changed, 109 insertions(+), 47 deletions(-)

diff --git a/build.gradle.kts b/build.gradle.kts
index 58534824d6e0..e92ee3ed909a 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -94,7 +94,7 @@ val werror by props(true) // treat javac warnings as errors
 
 // Projects whose main code NullAway verifies. The other projects are not annotated well enough
 // yet, so NullAway would only produce noise there.
-val nullawayProjects = listOf(":linq4j", ":core", ":server", ":druid", ":file", ":kafka", ":spark", ":babel", ":redis", ":splunk", ":mongodb", ":cassandra", ":pig", ":arrow", ":innodb", ":piglet")
+val nullawayProjects = listOf(":linq4j", ":core", ":server", ":druid", ":file", ":kafka", ":spark", ":babel", ":redis", ":splunk", ":mongodb", ":cassandra", ":pig", ":arrow", ":innodb", ":piglet", ":geode")
 
 val hepLargePlanModeTestIncludes = mapOf(
     ":core" to listOf(
diff --git a/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeAggregate.java b/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeAggregate.java
index 0b9041087618..80ba104512bc 100644
--- a/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeAggregate.java
+++ b/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeAggregate.java
@@ -51,7 +51,7 @@ public GeodeAggregate(RelOptCluster cluster,
       RelTraitSet traitSet,
       RelNode input,
       ImmutableBitSet groupSet,
-      List groupSets,
+      @Nullable List groupSets,
       List aggCalls) {
     super(cluster, traitSet, ImmutableList.of(), input, groupSet, groupSets, aggCalls);
 
@@ -73,14 +73,14 @@ public GeodeAggregate(RelOptCluster cluster,
       RelNode input,
       boolean indicator,
       ImmutableBitSet groupSet,
-      List groupSets,
+      @Nullable List groupSets,
       List aggCalls) {
     this(cluster, traitSet, input, groupSet, groupSets, aggCalls);
     checkIndicator(indicator);
   }
 
   @Override public Aggregate copy(RelTraitSet traitSet, RelNode input,
-      ImmutableBitSet groupSet, List groupSets,
+      ImmutableBitSet groupSet, @Nullable List groupSets,
       List aggCalls) {
     return new GeodeAggregate(getCluster(), traitSet, input, groupSet,
         groupSets, aggCalls);
@@ -125,7 +125,8 @@ public GeodeAggregate(RelOptCluster cluster,
       String oqlAggregateCall =
           Util.toString(aggCallFieldNames, functionName + "(", ", ", ")");
 
-      aggregateFunctionMap.put(aggCall.getName(), oqlAggregateCall);
+      aggregateFunctionMap.put(
+          requireNonNull(aggCall.getName(), "aggCall name"), oqlAggregateCall);
     }
 
     geodeImplementContext.addAggregateFunctions(aggregateFunctionMap.build());
diff --git a/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeEnumerator.java b/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeEnumerator.java
index 93e28a3013c6..de56b8ccd6b6 100644
--- a/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeEnumerator.java
+++ b/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeEnumerator.java
@@ -38,7 +38,7 @@
 /**
  * Enumerator that reads from a Geode Regions.
  */
-class GeodeEnumerator implements Enumerator {
+class GeodeEnumerator implements Enumerator<@Nullable Object> {
 
   protected static final Logger LOGGER = LoggerFactory.getLogger(GeodeEnumerator.class.getName());
 
diff --git a/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeFilter.java b/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeFilter.java
index b1cf8f5ac6b2..31e1a74b46db 100644
--- a/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeFilter.java
+++ b/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeFilter.java
@@ -261,8 +261,8 @@ private List getLeftNodeDisjunctions(RexNode node, List disjun
     private String translateOr(List disjunctions) {
       List predicates = new ArrayList<>();
 
-      List leftFieldNameList = new ArrayList<>();
-      List inSetLeftFieldNameList = new ArrayList<>();
+      List<@Nullable String> leftFieldNameList = new ArrayList<>();
+      List<@Nullable String> inSetLeftFieldNameList = new ArrayList<>();
 
       for (RexNode node : disjunctions) {
         final String leftNodeFieldName = getLeftNodeFieldNameForNode(node);
diff --git a/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeRel.java b/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeRel.java
index 01e18ca969f6..587e4bb07102 100644
--- a/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeRel.java
+++ b/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeRel.java
@@ -20,6 +20,8 @@
 import org.apache.calcite.plan.RelOptTable;
 import org.apache.calcite.rel.RelNode;
 
+import org.jspecify.annotations.Nullable;
+
 import java.util.ArrayList;
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -61,10 +63,14 @@ class GeodeImplementContext {
 
     final Map oqlAggregateFunctions = new LinkedHashMap<>();
 
-    Long limitValue;
+    @Nullable Long limitValue;
 
+    /** Set by {@code GeodeTableScan.implement} before anything reads it. */
+    @SuppressWarnings("NullAway.Init")
     RelOptTable table;
 
+    /** Set by {@code GeodeTableScan.implement} before anything reads it. */
+    @SuppressWarnings("NullAway.Init")
     GeodeTable geodeTable;
 
     /**
diff --git a/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeSchema.java b/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeSchema.java
index 49457d538e71..a01fabd42096 100644
--- a/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeSchema.java
+++ b/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeSchema.java
@@ -26,6 +26,8 @@
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 
+import org.jspecify.annotations.Nullable;
+
 import java.util.List;
 import java.util.Map;
 
@@ -38,7 +40,7 @@ public class GeodeSchema extends AbstractSchema {
 
   final GemFireCache cache;
   private final List regionNames;
-  private ImmutableMap tableMap;
+  private @Nullable ImmutableMap tableMap;
 
   public GeodeSchema(final GemFireCache gemFireCache, final Iterable regionNames) {
     super();
diff --git a/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeSchemaFactory.java b/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeSchemaFactory.java
index 5c1f5a679bd4..d5ff2de6b4f5 100644
--- a/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeSchemaFactory.java
+++ b/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeSchemaFactory.java
@@ -29,6 +29,9 @@
 
 import static org.apache.calcite.adapter.geode.util.GeodeUtils.createClientCache;
 
+import static java.lang.Integer.parseInt;
+import static java.util.Objects.requireNonNull;
+
 /**
  * Factory that creates a {@link GeodeSchema}.
  */
@@ -49,10 +52,16 @@ public GeodeSchemaFactory() {
   @Override public synchronized Schema create(SchemaPlus parentSchema, String name,
       Map operand) {
     Map map = (Map) operand;
-    String locatorHost = (String) map.get(LOCATOR_HOST);
-    int locatorPort = Integer.valueOf((String) map.get(LOCATOR_PORT));
-    String[] regionNames = ((String) map.get(REGIONS)).split(COMMA_DELIMITER);
-    String pbxSerializablePackagePath = (String) map.get(PDX_SERIALIZABLE_PACKAGE_PATH);
+    String locatorHost =
+        requireNonNull((String) map.get(LOCATOR_HOST), "locatorHost");
+    int locatorPort =
+        parseInt(
+            requireNonNull((String) map.get(LOCATOR_PORT), "locatorPort"));
+    String[] regionNames =
+        requireNonNull((String) map.get(REGIONS), "regions").split(COMMA_DELIMITER);
+    String pbxSerializablePackagePath =
+        requireNonNull((String) map.get(PDX_SERIALIZABLE_PACKAGE_PATH),
+            "pdxSerializablePackagePath");
 
     boolean allowSpatialFunctions = true;
     if (map.containsKey(ALLOW_SPATIAL_FUNCTIONS)) {
diff --git a/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeSort.java b/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeSort.java
index fa150e978a26..8420f2f33d02 100644
--- a/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeSort.java
+++ b/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeSort.java
@@ -65,7 +65,8 @@ public class GeodeSort extends Sort implements GeodeRel {
   }
 
   @Override public Sort copy(RelTraitSet traitSet, RelNode input,
-      RelCollation newCollation, RexNode offset, RexNode fetch) {
+      RelCollation newCollation, @Nullable RexNode offset,
+      @Nullable RexNode fetch) {
     return new GeodeSort(getCluster(), traitSet, input, collation, fetch);
   }
 
diff --git a/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeTable.java b/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeTable.java
index 83075a7db007..62a1c127e5b2 100644
--- a/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeTable.java
+++ b/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeTable.java
@@ -87,7 +87,7 @@ public class GeodeTable extends AbstractQueryableTable implements TranslatableTa
    * @param predicates  A list of predicates which should be used in the query
    * @return Enumerator of results
    */
-  public Enumerable query(final GemFireCache clientCache,
+  public Enumerable<@Nullable Object> query(final GemFireCache clientCache,
       final List> fields,
       final List> selectFields,
       final List> aggregateFunctions,
@@ -197,20 +197,35 @@ public Enumerable query(final GemFireCache clientCache,
     Hook.QUERY_PLAN.run(oqlQuery);
     LOGGER.info("OQL: " + oqlQuery);
 
-    return new AbstractEnumerable() {
-      @Override public Enumerator enumerator() {
-        final QueryService queryService = clientCache.getQueryService();
-        try {
-          SelectResults results = (SelectResults) queryService.newQuery(oqlQuery).execute();
-          return new GeodeEnumerator(results, resultRowType);
-        } catch (Exception e) {
-          String message =
-              String.format(Locale.ROOT, "Failed to execute query [%s] on %s",
-                  oqlQuery, clientCache.getName());
-          throw new RuntimeException(message, e);
-        }
+    return new GeodeQueryEnumerable(clientCache, oqlQuery, resultRowType);
+  }
+
+  /** Enumerable over the results of an OQL query. */
+  private static class GeodeQueryEnumerable
+      extends AbstractEnumerable<@Nullable Object> {
+    private final GemFireCache clientCache;
+    private final String oqlQuery;
+    private final RelProtoDataType resultRowType;
+
+    GeodeQueryEnumerable(GemFireCache clientCache, String oqlQuery,
+        RelProtoDataType resultRowType) {
+      this.clientCache = clientCache;
+      this.oqlQuery = oqlQuery;
+      this.resultRowType = resultRowType;
+    }
+
+    @Override public Enumerator<@Nullable Object> enumerator() {
+      final QueryService queryService = clientCache.getQueryService();
+      try {
+        SelectResults results = (SelectResults) queryService.newQuery(oqlQuery).execute();
+        return new GeodeEnumerator(results, resultRowType);
+      } catch (Exception e) {
+        String message =
+            String.format(Locale.ROOT, "Failed to execute query [%s] on %s",
+                oqlQuery, clientCache.getName());
+        throw new RuntimeException(message, e);
       }
-    };
+    }
   }
 
   @Override public  Queryable asQueryable(QueryProvider queryProvider,
@@ -262,7 +277,7 @@ private GemFireCache getClientCache() {
      * Called via code-generation.
      */
     @SuppressWarnings("UnusedDeclaration")
-    public Enumerable query(
+    public Enumerable<@Nullable Object> query(
         List> fields,
         List> selectFields,
         List> aggregateFunctions,
diff --git a/geode/src/main/java/org/apache/calcite/adapter/geode/rel/package-info.java b/geode/src/main/java/org/apache/calcite/adapter/geode/rel/package-info.java
index c58923468d6d..d6b5fa6c318e 100644
--- a/geode/src/main/java/org/apache/calcite/adapter/geode/rel/package-info.java
+++ b/geode/src/main/java/org/apache/calcite/adapter/geode/rel/package-info.java
@@ -18,4 +18,7 @@
 /**
  * Query provider based on Apache Geode (Gemfire) in-memory data grid.
  */
+@NullMarked
 package org.apache.calcite.adapter.geode.rel;
+
+import org.jspecify.annotations.NullMarked;
diff --git a/geode/src/main/java/org/apache/calcite/adapter/geode/simple/GeodeSimpleEnumerator.java b/geode/src/main/java/org/apache/calcite/adapter/geode/simple/GeodeSimpleEnumerator.java
index 341edf72b72e..b3b20ee8b830 100644
--- a/geode/src/main/java/org/apache/calcite/adapter/geode/simple/GeodeSimpleEnumerator.java
+++ b/geode/src/main/java/org/apache/calcite/adapter/geode/simple/GeodeSimpleEnumerator.java
@@ -26,16 +26,19 @@
 
 import java.util.Iterator;
 
+import static org.apache.calcite.linq4j.Nullness.castNonNull;
+
 /**
  * Geode Simple Enumerator.
  *
  * @param  Element type
  */
-public abstract class GeodeSimpleEnumerator implements Enumerator {
+public abstract class GeodeSimpleEnumerator
+    implements Enumerator {
 
   private @Nullable Iterator results;
 
-  private E current;
+  private @Nullable E current;
 
   protected GeodeSimpleEnumerator(ClientCache clientCache, String regionName) {
     QueryService queryService = clientCache.getQueryService();
@@ -49,7 +52,7 @@ protected GeodeSimpleEnumerator(ClientCache clientCache, String regionName) {
   }
 
   @Override public E current() {
-    return current;
+    return castNonNull(current);
   }
 
   @Override public boolean moveNext() {
diff --git a/geode/src/main/java/org/apache/calcite/adapter/geode/simple/GeodeSimpleSchema.java b/geode/src/main/java/org/apache/calcite/adapter/geode/simple/GeodeSimpleSchema.java
index 2c8c374ae8dc..d051612b1182 100644
--- a/geode/src/main/java/org/apache/calcite/adapter/geode/simple/GeodeSimpleSchema.java
+++ b/geode/src/main/java/org/apache/calcite/adapter/geode/simple/GeodeSimpleSchema.java
@@ -25,6 +25,8 @@
 
 import com.google.common.collect.ImmutableMap;
 
+import org.jspecify.annotations.Nullable;
+
 import java.util.Map;
 
 import static org.apache.calcite.adapter.geode.util.GeodeUtils.autodetectRelTypeFromRegion;
@@ -38,7 +40,7 @@ public class GeodeSimpleSchema extends AbstractSchema {
   private final String[] regionNames;
   @SuppressWarnings("unused")
   private final ClientCache clientCache;
-  private ImmutableMap tableMap;
+  private @Nullable ImmutableMap tableMap;
 
   public GeodeSimpleSchema(String locatorHost, int locatorPort,
       String[] regionNames, String pdxAutoSerializerPackageExp) {
diff --git a/geode/src/main/java/org/apache/calcite/adapter/geode/simple/GeodeSimpleSchemaFactory.java b/geode/src/main/java/org/apache/calcite/adapter/geode/simple/GeodeSimpleSchemaFactory.java
index 1a867d33a33c..5390e96879fc 100644
--- a/geode/src/main/java/org/apache/calcite/adapter/geode/simple/GeodeSimpleSchemaFactory.java
+++ b/geode/src/main/java/org/apache/calcite/adapter/geode/simple/GeodeSimpleSchemaFactory.java
@@ -22,6 +22,9 @@
 
 import java.util.Map;
 
+import static java.lang.Integer.parseInt;
+import static java.util.Objects.requireNonNull;
+
 /**
  * Geode Simple Table Schema Factory.
  */
@@ -41,10 +44,16 @@ public GeodeSimpleSchemaFactory() {
       String name, Map operand) {
     Map map = (Map) operand;
 
-    String locatorHost = (String) map.get(LOCATOR_HOST);
-    int locatorPort = Integer.valueOf((String) map.get(LOCATOR_PORT));
-    String[] regionNames = ((String) map.get(REGIONS)).split(COMMA_DELIMITER);
-    String pdxSerializablePackagePath = (String) map.get(PDX_SERIALIZABLE_PACKAGE_PATH);
+    String locatorHost =
+        requireNonNull((String) map.get(LOCATOR_HOST), "locatorHost");
+    int locatorPort =
+        parseInt(
+            requireNonNull((String) map.get(LOCATOR_PORT), "locatorPort"));
+    String[] regionNames =
+        requireNonNull((String) map.get(REGIONS), "regions").split(COMMA_DELIMITER);
+    String pdxSerializablePackagePath =
+        requireNonNull((String) map.get(PDX_SERIALIZABLE_PACKAGE_PATH),
+            "pdxSerializablePackagePath");
 
     return new GeodeSimpleSchema(locatorHost, locatorPort, regionNames, pdxSerializablePackagePath);
   }
diff --git a/geode/src/main/java/org/apache/calcite/adapter/geode/simple/package-info.java b/geode/src/main/java/org/apache/calcite/adapter/geode/simple/package-info.java
index 48295dc3e30a..c3ba3ffc2a6c 100644
--- a/geode/src/main/java/org/apache/calcite/adapter/geode/simple/package-info.java
+++ b/geode/src/main/java/org/apache/calcite/adapter/geode/simple/package-info.java
@@ -18,4 +18,7 @@
 /**
  * Query evaluation runtime for Apache Geode adapter.
  */
+@NullMarked
 package org.apache.calcite.adapter.geode.simple;
+
+import org.jspecify.annotations.NullMarked;
diff --git a/geode/src/main/java/org/apache/calcite/adapter/geode/util/GeodeUtils.java b/geode/src/main/java/org/apache/calcite/adapter/geode/util/GeodeUtils.java
index b9066de66b7a..5a5d80e50a99 100644
--- a/geode/src/main/java/org/apache/calcite/adapter/geode/util/GeodeUtils.java
+++ b/geode/src/main/java/org/apache/calcite/adapter/geode/util/GeodeUtils.java
@@ -158,7 +158,7 @@ public static synchronized Region createRegion(GemFireCache cache, String region
   public static @Nullable Object convertToRowValues(
       List relDataTypeFields, Object geodeResultObject) {
 
-    Object values;
+    @Nullable Object values;
 
     if (geodeResultObject instanceof Struct) {
       values = handleStructEntry(relDataTypeFields, geodeResultObject);
@@ -171,12 +171,12 @@ public static synchronized Region createRegion(GemFireCache cache, String region
     return values;
   }
 
-  private static Object handleStructEntry(
+  private static @Nullable Object handleStructEntry(
       List relDataTypeFields, Object obj) {
 
     Struct struct = (Struct) obj;
 
-    Object[] values = new Object[relDataTypeFields.size()];
+    final @Nullable Object[] values = new Object[relDataTypeFields.size()];
 
     int index = 0;
     for (RelDataTypeField relDataTypeField : relDataTypeFields) {
@@ -199,12 +199,12 @@ private static Object handleStructEntry(
     return values;
   }
 
-  private static Object handlePdxInstanceEntry(
+  private static @Nullable Object handlePdxInstanceEntry(
       List relDataTypeFields, Object obj) {
 
     PdxInstance pdxEntry = (PdxInstance) obj;
 
-    Object[] values = new Object[relDataTypeFields.size()];
+    final @Nullable Object[] values = new Object[relDataTypeFields.size()];
 
     int index = 0;
     for (RelDataTypeField relDataTypeField : relDataTypeFields) {
@@ -236,7 +236,7 @@ private static Object handlePdxInstanceEntry(
       return null;
     }
 
-    Object[] values = new Object[relDataTypeFields.size()];
+    final @Nullable Object[] values = new Object[relDataTypeFields.size()];
 
     int index = 0;
     for (RelDataTypeField relDataTypeField : relDataTypeFields) {
@@ -252,7 +252,7 @@ private static Object handlePdxInstanceEntry(
   }
 
   @SuppressWarnings("JavaUtilDate")
-  private static Object convert(Object o, Class clazz) {
+  private static @Nullable Object convert(@Nullable Object o, Class clazz) {
     if (o == null) {
       return null;
     }
@@ -293,7 +293,11 @@ public static RelDataType autodetectRelTypeFromRegion(Region region) {
     requireNonNull(region, "region");
 
     // try to detect type using value constraints (if they exists)
-    final Class constraint = region.getAttributes().getValueConstraint();
+    // The raw Class avoids the bytecode wildcard of Region's value type, whose
+    // upper bound reads as nullable and so is rejected where Class is
+    // expected. See https://github.com/uber/NullAway/issues/1732
+    @SuppressWarnings("rawtypes") final @Nullable Class constraint =
+        region.getAttributes().getValueConstraint();
     if (constraint != null && !PdxInstance.class.isAssignableFrom(constraint)) {
       return new JavaTypeFactoryExtImpl().createStructType(constraint);
     }
@@ -313,7 +317,8 @@ public static RelDataType autodetectRelTypeFromRegion(Region region) {
       throw new IllegalStateException(message);
     }
 
-    final Object entry = region.get(iter.next());
+    final Object entry =
+        requireNonNull(region.get(iter.next()), "region entry");
     return createRelDataType(entry);
   }
 
diff --git a/geode/src/main/java/org/apache/calcite/adapter/geode/util/package-info.java b/geode/src/main/java/org/apache/calcite/adapter/geode/util/package-info.java
index a7a33b16bd4a..a2cca0871628 100644
--- a/geode/src/main/java/org/apache/calcite/adapter/geode/util/package-info.java
+++ b/geode/src/main/java/org/apache/calcite/adapter/geode/util/package-info.java
@@ -18,4 +18,7 @@
 /**
  * Utilities for Apache Geode adapter.
  */
+@NullMarked
 package org.apache.calcite.adapter.geode.util;
+
+import org.jspecify.annotations.NullMarked;

From 873a81d425a6ef1fba0f69b6a883ef53fdef8e6f Mon Sep 17 00:00:00 2001
From: Vladimir Sitnikov 
Date: Tue, 25 Aug 2026 21:53:37 +0300
Subject: [PATCH 557/562] [CALCITE-7736] Put :elasticsearch under nullness
 verification

An Elasticsearch hit carries either _source or fields and never both, so each
of the two is absent half the time, and a document has no value for a field it
does not carry. That runs through the getters, the row converters and the
aggregation buckets, whose key is absent for a missing bucket.

The predicate analyzer reads a literal that may hold no value: a range bound
needs one and says so, while a term query writes whatever it got. A LIKE with
no escape, a projection that is not an item reference, and an expression the
analyzer cannot convert are all absent results the callers already handled.

The schema factory takes the credentials and the path prefix from the model,
where they are optional.

Co-Authored-By: Claude Opus 5 
---
 build.gradle.kts                              |  2 +-
 .../elasticsearch/ElasticsearchAggregate.java |  9 +--
 .../ElasticsearchEnumerators.java             | 20 +++---
 .../elasticsearch/ElasticsearchFilter.java    |  8 ++-
 .../elasticsearch/ElasticsearchJson.java      | 65 +++++++++----------
 .../elasticsearch/ElasticsearchMapping.java   |  2 +-
 .../elasticsearch/ElasticsearchRel.java       | 11 +++-
 .../ElasticsearchSchemaFactory.java           |  9 ++-
 .../ElasticsearchSearchResult.java            |  4 +-
 .../elasticsearch/ElasticsearchTable.java     | 12 ++--
 .../MapProjectionFieldVisitor.java            |  6 +-
 .../elasticsearch/PredicateAnalyzer.java      | 54 ++++++++-------
 .../adapter/elasticsearch/QueryBuilders.java  | 34 +++++-----
 .../adapter/elasticsearch/Scrolling.java      |  6 +-
 .../adapter/elasticsearch/package-info.java   |  3 +
 15 files changed, 141 insertions(+), 104 deletions(-)

diff --git a/build.gradle.kts b/build.gradle.kts
index e92ee3ed909a..30e13b4db992 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -94,7 +94,7 @@ val werror by props(true) // treat javac warnings as errors
 
 // Projects whose main code NullAway verifies. The other projects are not annotated well enough
 // yet, so NullAway would only produce noise there.
-val nullawayProjects = listOf(":linq4j", ":core", ":server", ":druid", ":file", ":kafka", ":spark", ":babel", ":redis", ":splunk", ":mongodb", ":cassandra", ":pig", ":arrow", ":innodb", ":piglet", ":geode")
+val nullawayProjects = listOf(":linq4j", ":core", ":server", ":druid", ":file", ":kafka", ":spark", ":babel", ":redis", ":splunk", ":mongodb", ":cassandra", ":pig", ":arrow", ":innodb", ":piglet", ":geode", ":elasticsearch")
 
 val hepLargePlanModeTestIncludes = mapOf(
     ":core" to listOf(
diff --git a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchAggregate.java b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchAggregate.java
index 28d2783735ac..1c72946558de 100644
--- a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchAggregate.java
+++ b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchAggregate.java
@@ -61,7 +61,7 @@ public class ElasticsearchAggregate extends Aggregate implements ElasticsearchRe
       RelTraitSet traitSet,
       RelNode input,
       ImmutableBitSet groupSet,
-      List groupSets,
+      @Nullable List groupSets,
       List aggCalls) throws InvalidRelException  {
     super(cluster, traitSet, ImmutableList.of(), input, groupSet, groupSets, aggCalls);
 
@@ -107,14 +107,14 @@ public class ElasticsearchAggregate extends Aggregate implements ElasticsearchRe
       RelNode input,
       boolean indicator,
       ImmutableBitSet groupSet,
-      List groupSets,
+      @Nullable List groupSets,
       List aggCalls) throws InvalidRelException {
     this(cluster, traitSet, input, groupSet, groupSets, aggCalls);
     checkIndicator(indicator);
   }
 
   @Override public Aggregate copy(RelTraitSet traitSet, RelNode input,
-      ImmutableBitSet groupSet, List groupSets,
+      ImmutableBitSet groupSet, @Nullable List groupSets,
       List aggCalls) {
     try {
       return new ElasticsearchAggregate(getCluster(), traitSet, input,
@@ -155,7 +155,8 @@ public class ElasticsearchAggregate extends Aggregate implements ElasticsearchRe
         field.put("size", 1);
       }
 
-      implementor.addAggregation(aggCall.getName(), aggregation.toString());
+      implementor.addAggregation(
+          requireNonNull(aggCall.getName(), "aggCall name"), aggregation.toString());
     }
   }
 
diff --git a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchEnumerators.java b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchEnumerators.java
index 4adad00a6710..a258d8a4e75f 100644
--- a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchEnumerators.java
+++ b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchEnumerators.java
@@ -20,6 +20,8 @@
 import org.apache.calcite.linq4j.function.Function1;
 import org.apache.calcite.linq4j.tree.Primitive;
 
+import org.jspecify.annotations.Nullable;
+
 import java.util.Date;
 import java.util.List;
 import java.util.Map;
@@ -40,7 +42,7 @@ private static Function1 mapGetter() {
     return ElasticsearchJson.SearchHit::sourceOrFields;
   }
 
-  private static Function1 singletonGetter(
+  private static Function1 singletonGetter(
       final String fieldName,
       final Class fieldClass,
       final Map mapping) {
@@ -52,7 +54,7 @@ private static Function1 singletonGetter(
         key = mapping.getOrDefault(fieldName, fieldName);
       }
 
-      final Object value;
+      final @Nullable Object value;
       if (ElasticsearchConstants.ID.equals(key)
           || ElasticsearchConstants.ID.equals(mapping.getOrDefault(fieldName, fieldName))) {
         // is the original projection on _id field?
@@ -72,10 +74,10 @@ private static Function1 singletonGetter(
    *
    * @return function that converts the search result into a generic array
    */
-  private static Function1 listGetter(
+  private static Function1 listGetter(
       final List> fields, Map mapping) {
     return hit -> {
-      Object[] objects = new Object[fields.size()];
+      final @Nullable Object[] objects = new Object[fields.size()];
       for (int i = 0; i < fields.size(); i++) {
         final Map.Entry field = fields.get(i);
         final String key;
@@ -85,7 +87,7 @@ private static Function1 listGetter(
           key = mapping.getOrDefault(field.getKey(), field.getKey());
         }
 
-        final Object value;
+        final @Nullable Object value;
         if (ElasticsearchConstants.ID.equals(key)
             || ElasticsearchConstants.ID.equals(mapping.get(field.getKey()))
             || ElasticsearchConstants.ID.equals(field.getKey())) {
@@ -102,7 +104,7 @@ private static Function1 listGetter(
     };
   }
 
-  static Function1 getter(
+  static Function1 getter(
       List> fields, Map mapping) {
     requireNonNull(fields, "fields");
     //noinspection unchecked
@@ -120,13 +122,13 @@ static Function1 getter(
   }
 
   @SuppressWarnings("JavaUtilDate")
-  private static Object convert(Object o, Class clazz) {
-    if (o == null) {
+  private static @Nullable Object convert(@Nullable Object o, @Nullable Class clazz) {
+    if (o == null || clazz == null) {
       return null;
     }
     Primitive primitive = Primitive.of(clazz);
     if (primitive != null) {
-      clazz = primitive.boxClass;
+      clazz = requireNonNull(primitive.boxClass, "boxClass");
     } else {
       primitive = Primitive.ofBox(clazz);
     }
diff --git a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchFilter.java b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchFilter.java
index 123024980279..0fa2dd721838 100644
--- a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchFilter.java
+++ b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchFilter.java
@@ -99,9 +99,13 @@ String translateMatch(RexNode condition) throws IOException,
         }
       }
       if (disMax) {
-        QueryBuilders.disMaxQueryBuilder(PredicateAnalyzer.analyze(condition)).writeJson(generator);
+        QueryBuilders.disMaxQueryBuilder(
+            requireNonNull(PredicateAnalyzer.analyze(condition), "query"))
+            .writeJson(generator);
       } else {
-        QueryBuilders.constantScoreQuery(PredicateAnalyzer.analyze(condition)).writeJson(generator);
+        QueryBuilders.constantScoreQuery(
+            requireNonNull(PredicateAnalyzer.analyze(condition), "query"))
+            .writeJson(generator);
       }
       generator.flush();
       generator.close();
diff --git a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchJson.java b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchJson.java
index be04172422c7..68c61bc15e78 100644
--- a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchJson.java
+++ b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchJson.java
@@ -49,7 +49,6 @@
 import java.util.function.BiConsumer;
 import java.util.function.Consumer;
 import java.util.function.Predicate;
-import java.util.stream.StreamSupport;
 
 import static java.util.Collections.unmodifiableMap;
 import static java.util.Objects.requireNonNull;
@@ -69,7 +68,7 @@ private ElasticsearchJson() {}
    * Visits leaves of the aggregation where all values are stored.
    */
   static void visitValueNodes(Aggregations aggregations,
-      Consumer> consumer) {
+      Consumer> consumer) {
     requireNonNull(aggregations, "aggregations");
     requireNonNull(consumer, "consumer");
 
@@ -81,12 +80,12 @@ static void visitValueNodes(Aggregations aggregations,
     rows.forEach((k, v) -> {
       if (v.stream().allMatch(val -> val instanceof GroupValue)) {
         v.forEach(tuple -> {
-          Map groupRow = new LinkedHashMap<>(k.keys);
+          Map groupRow = new LinkedHashMap<>(k.keys);
           groupRow.put(tuple.getName(), tuple.value());
           consumer.accept(groupRow);
         });
       } else {
-        Map row = new LinkedHashMap<>(k.keys);
+        Map row = new LinkedHashMap<>(k.keys);
         v.forEach(val -> row.put(val.getName(), val.value()));
         consumer.accept(row);
       }
@@ -172,10 +171,10 @@ private static void visitMappingProperties(Deque path,
    * Identifies a Calcite row (as in relational algebra).
    */
   private static class RowKey {
-    private final Map keys;
+    private final Map keys;
     private final int hashCode;
 
-    private RowKey(final Map keys) {
+    private RowKey(final Map keys) {
       this.keys = requireNonNull(keys, "keys");
       this.hashCode = Objects.hashCode(keys);
     }
@@ -184,14 +183,13 @@ private RowKey(List buckets) {
       this(toMap(buckets));
     }
 
-    private static Map toMap(Iterable buckets) {
-      return StreamSupport.stream(buckets.spliterator(), false)
-          .collect(LinkedHashMap::new,
-              (m, v) -> m.put(v.getName(), v.key()),
-              LinkedHashMap::putAll);
+    private static Map toMap(Iterable buckets) {
+      final Map map = new LinkedHashMap<>();
+      buckets.forEach(b -> map.put(b.getName(), b.key()));
+      return map;
     }
 
-    @Override public boolean equals(final Object o) {
+    @Override public boolean equals(final @Nullable Object o) {
       if (this == o) {
         return true;
       }
@@ -368,13 +366,13 @@ static class SearchHit {
      * ID of the document (not available in aggregations).
      */
     private final String id;
-    private final Map source;
-    private final Map fields;
+    private final @Nullable Map source;
+    private final @Nullable Map fields;
 
     @JsonCreator
     SearchHit(@JsonProperty(ElasticsearchConstants.ID) final String id,
-                      @JsonProperty("_source") final Map source,
-                      @JsonProperty("fields") final Map fields) {
+                      @JsonProperty("_source") final @Nullable Map source,
+                      @JsonProperty("fields") final @Nullable Map fields) {
       this.id = requireNonNull(id, "id");
 
       // both can't be null
@@ -406,7 +404,7 @@ public String id() {
       return id;
     }
 
-    Object valueOrNull(String name) {
+    @Nullable Object valueOrNull(String name) {
       requireNonNull(name, "name");
 
       // for "select *" return whole document
@@ -462,16 +460,16 @@ Object valueOrNull(String name) {
       return null;
     }
 
-    Map source() {
+    @Nullable Map source() {
       return source;
     }
 
-    Map fields() {
+    @Nullable Map fields() {
       return fields;
     }
 
     Map sourceOrFields() {
-      return source != null ? source : fields;
+      return source != null ? source : requireNonNull(fields, "fields");
     }
   }
 
@@ -483,7 +481,7 @@ Map sourceOrFields() {
   static class Aggregations implements Iterable {
 
     private final List aggregations;
-    private Map aggregationsAsMap;
+    private @Nullable Map aggregationsAsMap;
 
     Aggregations(List aggregations) {
       this.aggregations = requireNonNull(aggregations, "aggregations");
@@ -521,11 +519,11 @@ final Map asMap() {
      * Returns the aggregation that is associated with the specified name.
      */
     @SuppressWarnings("unchecked")
-    public final  A get(String name) {
+    public final  @Nullable A get(String name) {
       return (A) asMap().get(name);
     }
 
-    @Override public final boolean equals(Object obj) {
+    @Override public final boolean equals(@Nullable Object obj) {
       if (obj == null || getClass() != obj.getClass()) {
         return false;
       }
@@ -589,11 +587,11 @@ List buckets() {
    * by a key, and can potentially hold sub-aggregations computed over all documents in it.
    */
   static class Bucket implements HasAggregations, Aggregation {
-    private final Object key;
+    private final @Nullable Object key;
     private final String name;
     private final Aggregations aggregations;
 
-    Bucket(final Object key,
+    Bucket(final @Nullable Object key,
         final String name,
         final Aggregations aggregations) {
       this.key = key; // key can be set after construction
@@ -604,7 +602,7 @@ static class Bucket implements HasAggregations, Aggregation {
     /**
      * Returns the key associated with the bucket.
      */
-    Object key() {
+    @Nullable Object key() {
       return key;
     }
 
@@ -640,9 +638,9 @@ boolean hasNoAggregations() {
    */
   static class MultiValue implements Aggregation {
     private final String name;
-    private final Map values;
+    private final Map values;
 
-    MultiValue(final String name, final Map values) {
+    MultiValue(final String name, final Map values) {
       this.name = requireNonNull(name, "name");
       this.values = requireNonNull(values, "values");
     }
@@ -651,7 +649,7 @@ static class MultiValue implements Aggregation {
       return name;
     }
 
-    Map values() {
+    Map values() {
       return values;
     }
 
@@ -678,15 +676,16 @@ Object value() {
    * In order that rows which have the same key can be put into result map.
    */
   static class GroupValue extends MultiValue {
-    GroupValue(String name, Map values) {
+    GroupValue(String name, Map values) {
       super(name, values);
     }
 
     /**
      * Constructs a {@link GroupValue} instance with a single value.
      */
-    static GroupValue of(String name, Object value) {
-      return new GroupValue(name, Collections.singletonMap("value", value));
+    static GroupValue of(String name, @Nullable Object value) {
+      return new GroupValue(name,
+          Collections.singletonMap("value", value));
     }
   }
 
@@ -779,7 +778,7 @@ private static Bucket parseBucket(JsonParser parser, String name, ObjectNode nod
       }
 
       final JsonNode keyNode = node.get("key");
-      final Object key;
+      final @Nullable Object key;
       if (isMissingBucket(keyNode) || keyNode.isNull()) {
         key = null;
       } else if (keyNode.isTextual()) {
diff --git a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchMapping.java b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchMapping.java
index 9a0baac8c5ec..d5c3c3eca312 100644
--- a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchMapping.java
+++ b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchMapping.java
@@ -112,7 +112,7 @@ static class Datatype {
             .collect(Collectors.toSet());
 
     private final String name;
-    private final JsonNode missingValue;
+    private final @Nullable JsonNode missingValue;
 
     private Datatype(final String name) {
       this.name = requireNonNull(name, "name");
diff --git a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchRel.java b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchRel.java
index 8cc4f524975b..6b12c03d1ba4 100644
--- a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchRel.java
+++ b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchRel.java
@@ -23,6 +23,8 @@
 import org.apache.calcite.sql.fun.SqlStdOperatorTable;
 import org.apache.calcite.util.Pair;
 
+import org.jspecify.annotations.Nullable;
+
 import java.util.ArrayList;
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -91,16 +93,21 @@ class Implementor {
      *
      * @see From/Size
      */
-    Long offset;
+    @Nullable Long offset;
 
     /**
      * Number of records to return. Equivalent to {@code size} in ES query.
      *
      * @see From/Size
      */
-    Long fetch;
+    @Nullable Long fetch;
 
+    /** Set by {@code ElasticsearchTableScan.implement} before anything reads it. */
+    @SuppressWarnings("NullAway.Init")
     RelOptTable table;
+
+    /** Set by {@code ElasticsearchTableScan.implement} before anything reads it. */
+    @SuppressWarnings("NullAway.Init")
     ElasticsearchTable elasticsearchTable;
 
     void add(String findOp) {
diff --git a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchSchemaFactory.java b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchSchemaFactory.java
index 84030378b8d6..f1154340e822 100644
--- a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchSchemaFactory.java
+++ b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchSchemaFactory.java
@@ -39,6 +39,7 @@
 
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.client.RestClientBuilder;
+import org.jspecify.annotations.Nullable;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -92,7 +93,7 @@ public class ElasticsearchSchemaFactory implements SchemaFactory {
 
           try {
             // Free resources allocated by this RestClient
-            notice.getValue().close();
+            requireNonNull(notice.getValue(), "client").close();
           } catch (IOException ex) {
             LOGGER.warn("Could not close RestClient {}", notice.getValue(), ex);
           }
@@ -205,8 +206,10 @@ protected static List getSortedHost(List hosts) {
    * @return new or cached low-level rest http client for ES
    */
   @SuppressWarnings({"java:S4830", "java:S5527"})
-  private static RestClient connect(List hosts, String pathPrefix,
-                                    String username, String password,
+  private static RestClient connect(List hosts,
+                                    @Nullable String pathPrefix,
+                                    @Nullable String username,
+                                    @Nullable String password,
                                     boolean disableSSLVerification) {
 
     requireNonNull(hosts, "hosts or coordinates");
diff --git a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchSearchResult.java b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchSearchResult.java
index 6611ba94f401..cae63898bf59 100644
--- a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchSearchResult.java
+++ b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchSearchResult.java
@@ -20,6 +20,8 @@
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.fasterxml.jackson.annotation.JsonProperty;
 
+import org.jspecify.annotations.Nullable;
+
 import java.time.Duration;
 import java.util.Iterator;
 import java.util.List;
@@ -135,7 +137,7 @@ public String id() {
      * @param name attribute name
      * @return value from result (_source or fields)
      */
-    Object value(String name) {
+    @Nullable Object value(String name) {
       requireNonNull(name, "name");
 
       if (!sourceOrFields().containsKey(name)) {
diff --git a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchTable.java b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchTable.java
index 29ad974fd572..c64bf1011111 100644
--- a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchTable.java
+++ b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchTable.java
@@ -40,6 +40,8 @@
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.google.common.collect.ImmutableMap;
 
+import org.jspecify.annotations.Nullable;
+
 import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.util.ArrayList;
@@ -108,7 +110,7 @@ String scriptedFieldPrefix() {
    * @param aggregations aggregation functions
    * @return Enumerator of results
    */
-  private Enumerable find(List ops,
+  private Enumerable<@Nullable Object> find(List ops,
       List> fields,
       List> sort,
       List> nullsSort,
@@ -149,7 +151,7 @@ private Enumerable find(List ops,
       query.put("size", fetch);
     }
 
-    final Function1 getter =
+    final Function1 getter =
         ElasticsearchEnumerators.getter(fields, ImmutableMap.copyOf(mappings));
 
     Iterable iter;
@@ -164,7 +166,7 @@ private Enumerable find(List ops,
     return Linq4j.asEnumerable(iter).select(getter);
   }
 
-  private Enumerable aggregate(List ops,
+  private Enumerable<@Nullable Object> aggregate(List ops,
       List> fields,
       List> sort,
       List groupBy,
@@ -305,7 +307,7 @@ private Enumerable aggregate(List ops,
       }
     }
 
-    final Function1 getter =
+    final Function1 getter =
         ElasticsearchEnumerators.getter(fields, ImmutableMap.copyOf(mapping));
 
     ElasticsearchJson.SearchHits hits =
@@ -369,7 +371,7 @@ private ElasticsearchTable getTable() {
      * @return result as enumerable
      */
     @SuppressWarnings("UnusedDeclaration")
-    public Enumerable find(List ops,
+    public Enumerable<@Nullable Object> find(List ops,
          List> fields,
          List> sort,
          List> nullsSort,
diff --git a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/MapProjectionFieldVisitor.java b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/MapProjectionFieldVisitor.java
index 455bb3c8cabb..3db21db886a0 100644
--- a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/MapProjectionFieldVisitor.java
+++ b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/MapProjectionFieldVisitor.java
@@ -21,10 +21,12 @@
 import org.apache.calcite.rex.RexVisitorImpl;
 import org.apache.calcite.sql.fun.SqlStdOperatorTable;
 
+import org.jspecify.annotations.Nullable;
+
 /**
  * Visitor that extracts the actual field name from an item expression.
  */
-class MapProjectionFieldVisitor extends RexVisitorImpl {
+class MapProjectionFieldVisitor extends RexVisitorImpl<@Nullable String> {
 
   static final MapProjectionFieldVisitor INSTANCE = new MapProjectionFieldVisitor();
 
@@ -32,7 +34,7 @@ private MapProjectionFieldVisitor() {
     super(true);
   }
 
-  @Override public String visitCall(RexCall call) {
+  @Override public @Nullable String visitCall(RexCall call) {
     if (call.op == SqlStdOperatorTable.ITEM) {
       return ((RexLiteral) call.getOperands().get(1)).getValueAs(String.class);
     }
diff --git a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/PredicateAnalyzer.java b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/PredicateAnalyzer.java
index de51a90013e1..a641c8e02ee0 100644
--- a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/PredicateAnalyzer.java
+++ b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/PredicateAnalyzer.java
@@ -36,6 +36,8 @@
 import com.google.common.collect.BoundType;
 import com.google.common.collect.Range;
 
+import org.jspecify.annotations.Nullable;
+
 import java.util.ArrayList;
 import java.util.GregorianCalendar;
 import java.util.LinkedHashMap;
@@ -107,11 +109,12 @@ private PredicateAnalyzer() {}
    * @return search query which can be used to query ES cluster
    * @throws ExpressionNotAnalyzableException when expression can't processed by this analyzer
    */
-  static QueryBuilder analyze(RexNode expression) throws ExpressionNotAnalyzableException {
+  static @Nullable QueryBuilder analyze(RexNode expression)
+      throws ExpressionNotAnalyzableException {
     requireNonNull(expression, "expression");
     try {
       // visits expression tree
-      QueryExpression e = (QueryExpression) expression.accept(new Visitor());
+      @Nullable QueryExpression e = (QueryExpression) expression.accept(new Visitor());
 
       if (e != null && e.isPartial()) {
         throw new UnsupportedOperationException("Can't handle partial QueryExpression: " + e);
@@ -383,7 +386,8 @@ private QueryExpression binary(RexCall call) {
       case LIKE:
         if (call.getOperands().size() == 3) {
           final Expression e = call.getOperands().get(2).accept(this);
-          LiteralExpression escape = expressAsLiteral(e);
+          LiteralExpression escape =
+              requireNonNull(expressAsLiteral(e), "escape");
           return QueryExpression.create(pair.getKey()).like(pair.getValue(), escape);
         }
         return QueryExpression.create(pair.getKey()).like(pair.getValue());
@@ -545,7 +549,7 @@ private static NamedFieldExpression toNamedField(RexLiteral literal) {
     /**
      * Try to convert a generic expression into a literal expression.
      */
-    private static LiteralExpression expressAsLiteral(Expression exp) {
+    private static @Nullable LiteralExpression expressAsLiteral(Expression exp) {
 
       if (exp instanceof LiteralExpression) {
         return (LiteralExpression) exp;
@@ -785,7 +789,7 @@ private CompoundQueryExpression(boolean partial, BoolQueryBuilder builder) {
   static class SimpleQueryExpression extends QueryExpression {
 
     private final NamedFieldExpression rel;
-    private QueryBuilder builder;
+    private @Nullable QueryBuilder builder;
 
     private String getFieldReference() {
       return rel.getReference();
@@ -843,7 +847,7 @@ private SimpleQueryExpression(NamedFieldExpression rel) {
     }
 
     @Override public QueryExpression equals(LiteralExpression literal) {
-      Object value = literal.value();
+      @Nullable Object value = literal.value();
       if (value instanceof GregorianCalendar) {
         builder = boolQuery()
                 .must(addFormatIfNecessary(literal, rangeQuery(getFieldReference()).gte(value)))
@@ -855,7 +859,7 @@ private SimpleQueryExpression(NamedFieldExpression rel) {
     }
 
     @Override public QueryExpression notEquals(LiteralExpression literal) {
-      Object value = literal.value();
+      @Nullable Object value = literal.value();
       if (value instanceof GregorianCalendar) {
         builder = boolQuery()
                 .should(addFormatIfNecessary(literal, rangeQuery(getFieldReference()).gt(value)))
@@ -870,7 +874,7 @@ private SimpleQueryExpression(NamedFieldExpression rel) {
     }
 
     @Override public QueryExpression gt(LiteralExpression literal) {
-      Object value = literal.value();
+      Object value = requireNonNull(literal.value(), "value");
       builder =
           addFormatIfNecessary(literal,
               rangeQuery(getFieldReference()).gt(value));
@@ -878,19 +882,19 @@ private SimpleQueryExpression(NamedFieldExpression rel) {
     }
 
     @Override public QueryExpression gte(LiteralExpression literal) {
-      Object value = literal.value();
+      Object value = requireNonNull(literal.value(), "value");
       builder = addFormatIfNecessary(literal, rangeQuery(getFieldReference()).gte(value));
       return this;
     }
 
     @Override public QueryExpression lt(LiteralExpression literal) {
-      Object value = literal.value();
+      Object value = requireNonNull(literal.value(), "value");
       builder = addFormatIfNecessary(literal, rangeQuery(getFieldReference()).lt(value));
       return this;
     }
 
     @Override public QueryExpression lte(LiteralExpression literal) {
-      Object value = literal.value();
+      Object value = requireNonNull(literal.value(), "value");
       builder = addFormatIfNecessary(literal, rangeQuery(getFieldReference()).lte(value));
       return this;
     }
@@ -905,13 +909,15 @@ private SimpleQueryExpression(NamedFieldExpression rel) {
     }
 
     @Override public QueryExpression in(LiteralExpression literal) {
-      Iterable iterable = (Iterable) literal.value();
+      Iterable iterable =
+          (Iterable) requireNonNull(literal.value(), "value");
       builder = termsQuery(getFieldReference(), iterable);
       return this;
     }
 
     @Override public QueryExpression notIn(LiteralExpression literal) {
-      Iterable iterable = (Iterable) literal.value();
+      Iterable iterable =
+          (Iterable) requireNonNull(literal.value(), "value");
       builder = boolQuery().mustNot(termsQuery(getFieldReference(), iterable));
       return this;
     }
@@ -1036,21 +1042,21 @@ static boolean isCastExpression(Expression exp) {
    */
   static final class NamedFieldExpression implements TerminalExpression {
 
-    private final String name;
+    private final @Nullable String name;
 
     private NamedFieldExpression() {
       this.name = null;
     }
 
-    private NamedFieldExpression(RexInputRef schemaField) {
+    private NamedFieldExpression(@Nullable RexInputRef schemaField) {
       this.name = schemaField == null ? null : schemaField.getName();
     }
 
-    private NamedFieldExpression(RexLiteral literal) {
+    private NamedFieldExpression(@Nullable RexLiteral literal) {
       this.name = literal == null ? null : RexLiteral.stringValue(literal);
     }
 
-    String getRootName() {
+    @Nullable String getRootName() {
       return name;
     }
 
@@ -1059,7 +1065,7 @@ boolean isMetaField() {
     }
 
     String getReference() {
-      return getRootName();
+      return requireNonNull(getRootName(), "name");
     }
   }
 
@@ -1074,7 +1080,7 @@ static final class LiteralExpression implements TerminalExpression {
       this.literal = literal;
     }
 
-    Object value() {
+    @Nullable Object value() {
 
       if (isSarg()) {
         return sargValue();
@@ -1085,7 +1091,7 @@ Object value() {
       } else if (isBoolean()) {
         return booleanValue();
       } else if (isString()) {
-        return RexLiteral.stringValue(literal);
+        return stringValue();
       } else {
         return rawValue();
       }
@@ -1112,11 +1118,11 @@ public boolean isSarg() {
     }
 
     long longValue() {
-      return ((Number) literal.getValue()).longValue();
+      return ((Number) requireNonNull(literal.getValue(), "value")).longValue();
     }
 
     double doubleValue() {
-      return ((Number) literal.getValue()).doubleValue();
+      return ((Number) requireNonNull(literal.getValue(), "value")).doubleValue();
     }
 
     boolean booleanValue() {
@@ -1124,7 +1130,7 @@ boolean booleanValue() {
     }
 
     String stringValue() {
-      return RexLiteral.stringValue(literal);
+      return requireNonNull(RexLiteral.stringValue(literal), "stringValue");
     }
 
     List sargValue() {
@@ -1154,7 +1160,7 @@ Object sargPointValue(Object point, SqlTypeName sqlTypeName) {
       }
     }
 
-    Object rawValue() {
+    @Nullable Object rawValue() {
       return literal.getValue();
     }
   }
diff --git a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/QueryBuilders.java b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/QueryBuilders.java
index 17d289499ba6..5c5f67ced8b0 100644
--- a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/QueryBuilders.java
+++ b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/QueryBuilders.java
@@ -18,6 +18,8 @@
 
 import com.fasterxml.jackson.core.JsonGenerator;
 
+import org.jspecify.annotations.Nullable;
+
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -118,7 +120,7 @@ static TermQueryBuilder termQuery(String name, boolean value) {
    * @param name  The name of the field
    * @param value The value of the term
    */
-  static TermQueryBuilder termQuery(String name, Object value) {
+  static TermQueryBuilder termQuery(String name, @Nullable Object value) {
     return new TermQueryBuilder(name, value);
   }
 
@@ -138,7 +140,7 @@ static MatchesQueryBuilder matchesQuery(String name, Iterable values) {
    * @param name  The name of the field
    * @param value The value of the term
    */
-  static MatchQueryBuilder matchQuery(String name, Object value) {
+  static MatchQueryBuilder matchQuery(String name, @Nullable Object value) {
     return new MatchQueryBuilder(name, value);
   }
 
@@ -309,9 +311,9 @@ private static void writeJsonArray(String field, List clauses, Jso
    */
   static class TermQueryBuilder extends QueryBuilder {
     private final String fieldName;
-    private final Object value;
+    private final @Nullable Object value;
 
-    private TermQueryBuilder(final String fieldName, final Object value) {
+    private TermQueryBuilder(final String fieldName, final @Nullable Object value) {
       this.fieldName = requireNonNull(fieldName, "fieldName");
       this.value = requireNonNull(value, "value");
     }
@@ -361,9 +363,9 @@ private TermsQueryBuilder(final String fieldName, final Iterable values) {
    */
   static class MatchQueryBuilder extends QueryBuilder {
     private final String fieldName;
-    private final Object value;
+    private final @Nullable Object value;
 
-    private MatchQueryBuilder(final String fieldName, final Object value) {
+    private MatchQueryBuilder(final String fieldName, final @Nullable Object value) {
       this.fieldName = requireNonNull(fieldName, "fieldName");
       this.value = requireNonNull(value, "value");
     }
@@ -415,7 +417,8 @@ private MatchesQueryBuilder(final String fieldName, final Iterable values) {
    * @param value JSON value to write
    * @throws IOException if can't write to output
    */
-  private static void writeObject(JsonGenerator generator, Object value) throws IOException {
+  private static void writeObject(JsonGenerator generator, @Nullable Object value)
+      throws IOException {
     generator.writeObject(value);
   }
 
@@ -425,12 +428,12 @@ private static void writeObject(JsonGenerator generator, Object value) throws IO
   static class RangeQueryBuilder extends QueryBuilder {
     private final String fieldName;
 
-    private Object lt;
+    private @Nullable Object lt;
     private boolean lte;
-    private Object gt;
+    private @Nullable Object gt;
     private boolean gte;
 
-    private String format;
+    private @Nullable String format;
 
     private RangeQueryBuilder(final String fieldName) {
       this.fieldName = requireNonNull(fieldName, "fieldName");
@@ -537,8 +540,8 @@ public static String replaceWildcard(String value, Map kv, Strin
         String current = value.substring(index, index + 1);
         if (index == 0) {
           if (!current.equals(escape)) {
-            current = kv.keySet().contains(current) ? kv.get(current) : current;
-            ret.add(current);
+            final String replacement = kv.get(current);
+            ret.add(replacement != null ? replacement : current);
           } else {
             escapeCount++;
           }
@@ -560,12 +563,13 @@ public static String replaceWildcard(String value, Map kv, Strin
         }
 
         String last = value.substring(index - 1, index);
-        if (kv.keySet().contains(current)) {
+        final String replacement = kv.get(current);
+        if (replacement != null) {
           if (!last.equals(escape)) {
-            ret.add(kv.get(current));
+            ret.add(replacement);
           } else {
             if (escapeCount % 2 == 0) {
-              ret.add(kv.get(current));
+              ret.add(replacement);
             } else {
               ret.add(current);
             }
diff --git a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/Scrolling.java b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/Scrolling.java
index 64b1d34681df..d6c440d8da50 100644
--- a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/Scrolling.java
+++ b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/Scrolling.java
@@ -20,6 +20,8 @@
 import com.google.common.collect.AbstractSequentialIterator;
 import com.google.common.collect.Iterators;
 
+import org.jspecify.annotations.Nullable;
+
 import java.util.Collections;
 import java.util.Iterator;
 import java.util.function.Consumer;
@@ -103,7 +105,7 @@ private static class AutoClosingIterator implements Iterator delegate,
@@ -155,7 +157,7 @@ private SequentialIterator(final ElasticsearchJson.Result first,
       this.limit = limit;
     }
 
-    @Override protected ElasticsearchJson.Result computeNext(
+    @Override protected ElasticsearchJson.@Nullable Result computeNext(
         final ElasticsearchJson.Result previous) {
       final int hits = previous.searchHits().hits().size();
       if (hits == 0 || count >= limit) {
diff --git a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/package-info.java b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/package-info.java
index 485c65e5909a..5ddb06863569 100644
--- a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/package-info.java
+++ b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/package-info.java
@@ -18,4 +18,7 @@
 /**
  * Query provider based on an Elasticsearch2 DB.
  */
+@NullMarked
 package org.apache.calcite.adapter.elasticsearch;
+
+import org.jspecify.annotations.NullMarked;

From 069cce7fb8277122495aba56662abd4ec87dc43c Mon Sep 17 00:00:00 2001
From: Vladimir Sitnikov 
Date: Tue, 25 Aug 2026 22:00:30 +0300
Subject: [PATCH 558/562] [CALCITE-7736] Put :plus under nullness verification

An os table function returns a row whose columns are absent wherever the
command printed nothing, which the enumerators and the line parsers now carry.
Ten of them built the same anonymous enumerable over an osquery table; they
share OsQueryEnumerable instead, which also avoids uber/NullAway#1746, as do
the named enumerator in the stdin function and the named line parser in vmstat.

os.name is absent on a JVM that does not publish it, and the table functions
switch on it. SqlShell prints a column that has no value, and looks a column
label up in a map that may not hold it. The Avatica server for Chinook holds
its server and its meta instance from the point it starts them.

Co-Authored-By: Claude Opus 5 
---
 build.gradle.kts                              |  2 +-
 .../adapter/os/CpuInfoTableFunction.java      |  8 +-
 .../adapter/os/CpuTimeTableFunction.java      |  8 +-
 .../adapter/os/FilesTableFunction.java        |  5 +-
 .../os/InterfaceAddressesTableFunction.java   |  8 +-
 .../os/InterfaceDetailsTableFunction.java     |  8 +-
 .../adapter/os/JavaInfoTableFunction.java     |  8 +-
 .../adapter/os/MemoryInfoTableFunction.java   |  8 +-
 .../adapter/os/MountsTableFunction.java       |  8 +-
 .../apache/calcite/adapter/os/OsQuery.java    |  9 +-
 .../calcite/adapter/os/OsQueryEnumerable.java | 37 +++++++
 .../adapter/os/OsVersionTableFunction.java    |  8 +-
 .../apache/calcite/adapter/os/Processes.java  | 10 +-
 .../calcite/adapter/os/PsTableFunction.java   | 21 ++--
 .../apache/calcite/adapter/os/SqlShell.java   | 11 ++-
 .../adapter/os/StdinTableFunction.java        | 97 +++++++++++--------
 .../adapter/os/SystemInfoTableFunction.java   |  8 +-
 .../adapter/os/VmstatTableFunction.java       | 71 ++++++++------
 .../calcite/adapter/os/package-info.java      |  3 +
 .../calcite/adapter/tpcds/TpcdsSchema.java    |  6 +-
 .../calcite/adapter/tpcds/package-info.java   |  3 +
 .../calcite/adapter/tpch/package-info.java    |  3 +
 .../calcite/adapter/utils/package-info.java   |  3 +
 .../calcite/chinook/ChinookAvaticaServer.java |  8 +-
 .../apache/calcite/chinook/package-info.java  |  3 +
 25 files changed, 200 insertions(+), 164 deletions(-)
 create mode 100644 plus/src/main/java/org/apache/calcite/adapter/os/OsQueryEnumerable.java

diff --git a/build.gradle.kts b/build.gradle.kts
index 30e13b4db992..d7d4f9fe5173 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -94,7 +94,7 @@ val werror by props(true) // treat javac warnings as errors
 
 // Projects whose main code NullAway verifies. The other projects are not annotated well enough
 // yet, so NullAway would only produce noise there.
-val nullawayProjects = listOf(":linq4j", ":core", ":server", ":druid", ":file", ":kafka", ":spark", ":babel", ":redis", ":splunk", ":mongodb", ":cassandra", ":pig", ":arrow", ":innodb", ":piglet", ":geode", ":elasticsearch")
+val nullawayProjects = listOf(":linq4j", ":core", ":server", ":druid", ":file", ":kafka", ":spark", ":babel", ":redis", ":splunk", ":mongodb", ":cassandra", ":pig", ":arrow", ":innodb", ":piglet", ":geode", ":elasticsearch", ":plus")
 
 val hepLargePlanModeTestIncludes = mapOf(
     ":core" to listOf(
diff --git a/plus/src/main/java/org/apache/calcite/adapter/os/CpuInfoTableFunction.java b/plus/src/main/java/org/apache/calcite/adapter/os/CpuInfoTableFunction.java
index e92c3feb51b5..8238c203fdbc 100644
--- a/plus/src/main/java/org/apache/calcite/adapter/os/CpuInfoTableFunction.java
+++ b/plus/src/main/java/org/apache/calcite/adapter/os/CpuInfoTableFunction.java
@@ -17,9 +17,7 @@
 package org.apache.calcite.adapter.os;
 
 import org.apache.calcite.DataContext;
-import org.apache.calcite.linq4j.AbstractEnumerable;
 import org.apache.calcite.linq4j.Enumerable;
-import org.apache.calcite.linq4j.Enumerator;
 import org.apache.calcite.rel.type.RelDataType;
 import org.apache.calcite.rel.type.RelDataTypeFactory;
 import org.apache.calcite.schema.ScannableTable;
@@ -37,11 +35,7 @@ private CpuInfoTableFunction() {
   public static ScannableTable eval(boolean b) {
     return new AbstractBaseScannableTable() {
       @Override public Enumerable<@Nullable Object[]> scan(DataContext root) {
-        return new AbstractEnumerable() {
-          @Override public Enumerator enumerator() {
-            return new OsQuery("cpu_info");
-          }
-        };
+        return new OsQueryEnumerable("cpu_info");
       }
 
       @Override public RelDataType getRowType(RelDataTypeFactory typeFactory) {
diff --git a/plus/src/main/java/org/apache/calcite/adapter/os/CpuTimeTableFunction.java b/plus/src/main/java/org/apache/calcite/adapter/os/CpuTimeTableFunction.java
index a303143ce367..e33af84a8393 100644
--- a/plus/src/main/java/org/apache/calcite/adapter/os/CpuTimeTableFunction.java
+++ b/plus/src/main/java/org/apache/calcite/adapter/os/CpuTimeTableFunction.java
@@ -17,9 +17,7 @@
 package org.apache.calcite.adapter.os;
 
 import org.apache.calcite.DataContext;
-import org.apache.calcite.linq4j.AbstractEnumerable;
 import org.apache.calcite.linq4j.Enumerable;
-import org.apache.calcite.linq4j.Enumerator;
 import org.apache.calcite.rel.type.RelDataType;
 import org.apache.calcite.rel.type.RelDataTypeFactory;
 import org.apache.calcite.schema.ScannableTable;
@@ -37,11 +35,7 @@ private CpuTimeTableFunction() {
   public static ScannableTable eval(boolean b) {
     return new AbstractBaseScannableTable() {
       @Override public Enumerable<@Nullable Object[]> scan(DataContext root) {
-        return new AbstractEnumerable() {
-          @Override public Enumerator enumerator() {
-            return new OsQuery("cpu_time");
-          }
-        };
+        return new OsQueryEnumerable("cpu_time");
       }
 
       @Override public RelDataType getRowType(RelDataTypeFactory typeFactory) {
diff --git a/plus/src/main/java/org/apache/calcite/adapter/os/FilesTableFunction.java b/plus/src/main/java/org/apache/calcite/adapter/os/FilesTableFunction.java
index 6e8162f730ee..ecd31bdf7845 100644
--- a/plus/src/main/java/org/apache/calcite/adapter/os/FilesTableFunction.java
+++ b/plus/src/main/java/org/apache/calcite/adapter/os/FilesTableFunction.java
@@ -205,7 +205,8 @@ private boolean isGnuStat() {
         final RelDataType rowType = getRowType(typeFactory);
         final List fieldNames =
             ImmutableList.copyOf(rowType.getFieldNames());
-        final String osName = System.getProperty("os.name");
+        final String osName =
+            requireNonNull(System.getProperty("os.name"), "os.name");
         final String osVersion = System.getProperty("os.version");
         Util.discard(osVersion);
         final Enumerable enumerable;
@@ -255,7 +256,7 @@ private static class FilesTableFunctionEnumerator implements Enumerator<@Nullabl
       this.osName = osName;
     }
 
-    @Override public Object[] current() {
+    @Override public @Nullable Object[] current() {
       return requireNonNull(current, "current");
     }
 
diff --git a/plus/src/main/java/org/apache/calcite/adapter/os/InterfaceAddressesTableFunction.java b/plus/src/main/java/org/apache/calcite/adapter/os/InterfaceAddressesTableFunction.java
index 30cb54c66f84..31c922b0f7c1 100644
--- a/plus/src/main/java/org/apache/calcite/adapter/os/InterfaceAddressesTableFunction.java
+++ b/plus/src/main/java/org/apache/calcite/adapter/os/InterfaceAddressesTableFunction.java
@@ -17,9 +17,7 @@
 package org.apache.calcite.adapter.os;
 
 import org.apache.calcite.DataContext;
-import org.apache.calcite.linq4j.AbstractEnumerable;
 import org.apache.calcite.linq4j.Enumerable;
-import org.apache.calcite.linq4j.Enumerator;
 import org.apache.calcite.rel.type.RelDataType;
 import org.apache.calcite.rel.type.RelDataTypeFactory;
 import org.apache.calcite.schema.ScannableTable;
@@ -37,11 +35,7 @@ private InterfaceAddressesTableFunction() {
   public static ScannableTable eval(boolean b) {
     return new AbstractBaseScannableTable() {
       @Override public Enumerable<@Nullable Object[]> scan(DataContext root) {
-        return new AbstractEnumerable() {
-          @Override public Enumerator enumerator() {
-            return new OsQuery("interface_addresses");
-          }
-        };
+        return new OsQueryEnumerable("interface_addresses");
       }
 
       @Override public RelDataType getRowType(RelDataTypeFactory typeFactory) {
diff --git a/plus/src/main/java/org/apache/calcite/adapter/os/InterfaceDetailsTableFunction.java b/plus/src/main/java/org/apache/calcite/adapter/os/InterfaceDetailsTableFunction.java
index 9b2964bc6471..d193e7b567f5 100644
--- a/plus/src/main/java/org/apache/calcite/adapter/os/InterfaceDetailsTableFunction.java
+++ b/plus/src/main/java/org/apache/calcite/adapter/os/InterfaceDetailsTableFunction.java
@@ -17,9 +17,7 @@
 package org.apache.calcite.adapter.os;
 
 import org.apache.calcite.DataContext;
-import org.apache.calcite.linq4j.AbstractEnumerable;
 import org.apache.calcite.linq4j.Enumerable;
-import org.apache.calcite.linq4j.Enumerator;
 import org.apache.calcite.rel.type.RelDataType;
 import org.apache.calcite.rel.type.RelDataTypeFactory;
 import org.apache.calcite.schema.ScannableTable;
@@ -37,11 +35,7 @@ private InterfaceDetailsTableFunction() {
   public static ScannableTable eval(boolean b) {
     return new AbstractBaseScannableTable() {
       @Override public Enumerable<@Nullable Object[]> scan(DataContext root) {
-        return new AbstractEnumerable() {
-          @Override public Enumerator enumerator() {
-            return new OsQuery("interface_details");
-          }
-        };
+        return new OsQueryEnumerable("interface_details");
       }
 
       @Override public RelDataType getRowType(RelDataTypeFactory typeFactory) {
diff --git a/plus/src/main/java/org/apache/calcite/adapter/os/JavaInfoTableFunction.java b/plus/src/main/java/org/apache/calcite/adapter/os/JavaInfoTableFunction.java
index 75f52a77aa65..86a893f06642 100644
--- a/plus/src/main/java/org/apache/calcite/adapter/os/JavaInfoTableFunction.java
+++ b/plus/src/main/java/org/apache/calcite/adapter/os/JavaInfoTableFunction.java
@@ -17,9 +17,7 @@
 package org.apache.calcite.adapter.os;
 
 import org.apache.calcite.DataContext;
-import org.apache.calcite.linq4j.AbstractEnumerable;
 import org.apache.calcite.linq4j.Enumerable;
-import org.apache.calcite.linq4j.Enumerator;
 import org.apache.calcite.rel.type.RelDataType;
 import org.apache.calcite.rel.type.RelDataTypeFactory;
 import org.apache.calcite.schema.ScannableTable;
@@ -37,11 +35,7 @@ private JavaInfoTableFunction() {
   public static ScannableTable eval(boolean b) {
     return new AbstractBaseScannableTable() {
       @Override public Enumerable<@Nullable Object[]> scan(DataContext root) {
-        return new AbstractEnumerable() {
-          @Override public Enumerator enumerator() {
-            return new OsQuery("java_info");
-          }
-        };
+        return new OsQueryEnumerable("java_info");
       }
 
       @Override public RelDataType getRowType(RelDataTypeFactory typeFactory) {
diff --git a/plus/src/main/java/org/apache/calcite/adapter/os/MemoryInfoTableFunction.java b/plus/src/main/java/org/apache/calcite/adapter/os/MemoryInfoTableFunction.java
index d1c4d66b5348..f13bdd5dff89 100644
--- a/plus/src/main/java/org/apache/calcite/adapter/os/MemoryInfoTableFunction.java
+++ b/plus/src/main/java/org/apache/calcite/adapter/os/MemoryInfoTableFunction.java
@@ -17,9 +17,7 @@
 package org.apache.calcite.adapter.os;
 
 import org.apache.calcite.DataContext;
-import org.apache.calcite.linq4j.AbstractEnumerable;
 import org.apache.calcite.linq4j.Enumerable;
-import org.apache.calcite.linq4j.Enumerator;
 import org.apache.calcite.rel.type.RelDataType;
 import org.apache.calcite.rel.type.RelDataTypeFactory;
 import org.apache.calcite.schema.ScannableTable;
@@ -37,11 +35,7 @@ private MemoryInfoTableFunction() {
   public static ScannableTable eval(boolean b) {
     return new AbstractBaseScannableTable() {
       @Override public Enumerable<@Nullable Object[]> scan(DataContext root) {
-        return new AbstractEnumerable() {
-          @Override public Enumerator enumerator() {
-            return new OsQuery("memory_info");
-          }
-        };
+        return new OsQueryEnumerable("memory_info");
       }
 
       @Override public RelDataType getRowType(RelDataTypeFactory typeFactory) {
diff --git a/plus/src/main/java/org/apache/calcite/adapter/os/MountsTableFunction.java b/plus/src/main/java/org/apache/calcite/adapter/os/MountsTableFunction.java
index 3f488f623441..7467ef8eaefa 100644
--- a/plus/src/main/java/org/apache/calcite/adapter/os/MountsTableFunction.java
+++ b/plus/src/main/java/org/apache/calcite/adapter/os/MountsTableFunction.java
@@ -17,9 +17,7 @@
 package org.apache.calcite.adapter.os;
 
 import org.apache.calcite.DataContext;
-import org.apache.calcite.linq4j.AbstractEnumerable;
 import org.apache.calcite.linq4j.Enumerable;
-import org.apache.calcite.linq4j.Enumerator;
 import org.apache.calcite.rel.type.RelDataType;
 import org.apache.calcite.rel.type.RelDataTypeFactory;
 import org.apache.calcite.schema.ScannableTable;
@@ -37,11 +35,7 @@ private MountsTableFunction() {
   public static ScannableTable eval(boolean b) {
     return new AbstractBaseScannableTable() {
       @Override public Enumerable<@Nullable Object[]> scan(DataContext root) {
-        return new AbstractEnumerable() {
-          @Override public Enumerator enumerator() {
-            return new OsQuery("mounts");
-          }
-        };
+        return new OsQueryEnumerable("mounts");
       }
 
       @Override public RelDataType getRowType(RelDataTypeFactory typeFactory) {
diff --git a/plus/src/main/java/org/apache/calcite/adapter/os/OsQuery.java b/plus/src/main/java/org/apache/calcite/adapter/os/OsQuery.java
index 83c01811df68..78dc9d22a0b1 100644
--- a/plus/src/main/java/org/apache/calcite/adapter/os/OsQuery.java
+++ b/plus/src/main/java/org/apache/calcite/adapter/os/OsQuery.java
@@ -20,6 +20,7 @@
 import org.apache.calcite.linq4j.Linq4j;
 import org.apache.calcite.util.trace.CalciteTrace;
 
+import org.jspecify.annotations.Nullable;
 import org.slf4j.Logger;
 
 import java.util.ArrayList;
@@ -29,20 +30,20 @@
 /**
  * Enumerator that reads from OS's System.
  */
-public class OsQuery implements Enumerator {
+public class OsQuery implements Enumerator<@Nullable Object[]> {
   private static final Logger LOGGER = CalciteTrace.getParserTracer();
 
-  private final Enumerator enumerator;
+  private final Enumerator<@Nullable Object[]> enumerator;
 
   public OsQuery(String type) {
     this.enumerator = Linq4j.enumerator(eval(type));
   }
 
-  public Enumerator getEnumerator() {
+  public Enumerator<@Nullable Object[]> getEnumerator() {
     return enumerator;
   }
 
-  @Override public Object[] current() {
+  @Override public @Nullable Object[] current() {
     return enumerator.current();
   }
 
diff --git a/plus/src/main/java/org/apache/calcite/adapter/os/OsQueryEnumerable.java b/plus/src/main/java/org/apache/calcite/adapter/os/OsQueryEnumerable.java
new file mode 100644
index 000000000000..141dab2ce56a
--- /dev/null
+++ b/plus/src/main/java/org/apache/calcite/adapter/os/OsQueryEnumerable.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.adapter.os;
+
+import org.apache.calcite.linq4j.AbstractEnumerable;
+import org.apache.calcite.linq4j.Enumerator;
+
+import org.jspecify.annotations.Nullable;
+
+/**
+ * Enumerable over the rows that an {@code osquery} query returns.
+ */
+class OsQueryEnumerable extends AbstractEnumerable<@Nullable Object[]> {
+  private final String tableName;
+
+  OsQueryEnumerable(String tableName) {
+    this.tableName = tableName;
+  }
+
+  @Override public Enumerator<@Nullable Object[]> enumerator() {
+    return new OsQuery(tableName);
+  }
+}
diff --git a/plus/src/main/java/org/apache/calcite/adapter/os/OsVersionTableFunction.java b/plus/src/main/java/org/apache/calcite/adapter/os/OsVersionTableFunction.java
index 0c78bbb341d6..6a57cbfcd16f 100644
--- a/plus/src/main/java/org/apache/calcite/adapter/os/OsVersionTableFunction.java
+++ b/plus/src/main/java/org/apache/calcite/adapter/os/OsVersionTableFunction.java
@@ -17,9 +17,7 @@
 package org.apache.calcite.adapter.os;
 
 import org.apache.calcite.DataContext;
-import org.apache.calcite.linq4j.AbstractEnumerable;
 import org.apache.calcite.linq4j.Enumerable;
-import org.apache.calcite.linq4j.Enumerator;
 import org.apache.calcite.rel.type.RelDataType;
 import org.apache.calcite.rel.type.RelDataTypeFactory;
 import org.apache.calcite.schema.ScannableTable;
@@ -37,11 +35,7 @@ private OsVersionTableFunction() {
   public static ScannableTable eval(boolean b) {
     return new AbstractBaseScannableTable() {
       @Override public Enumerable<@Nullable Object[]> scan(DataContext root) {
-        return new AbstractEnumerable() {
-          @Override public Enumerator enumerator() {
-            return new OsQuery("os_version");
-          }
-        };
+        return new OsQueryEnumerable("os_version");
       }
 
       @Override public RelDataType getRowType(RelDataTypeFactory typeFactory) {
diff --git a/plus/src/main/java/org/apache/calcite/adapter/os/Processes.java b/plus/src/main/java/org/apache/calcite/adapter/os/Processes.java
index bb38ebba3f04..d9c1f8e361df 100644
--- a/plus/src/main/java/org/apache/calcite/adapter/os/Processes.java
+++ b/plus/src/main/java/org/apache/calcite/adapter/os/Processes.java
@@ -20,6 +20,8 @@
 import org.apache.calcite.linq4j.Enumerable;
 import org.apache.calcite.linq4j.Enumerator;
 
+import org.jspecify.annotations.Nullable;
+
 import java.io.BufferedInputStream;
 import java.io.BufferedReader;
 import java.io.IOException;
@@ -88,10 +90,10 @@ private static class ProcessLinesEnumerator
           new InputStreamReader(bis, StandardCharsets.UTF_8);
       final BufferedReader br = new BufferedReader(isr);
       return new Enumerator() {
-        private String line;
+        private @Nullable String line;
 
         @Override public String current() {
-          return line;
+          return requireNonNull(line, "line");
         }
 
         @Override public boolean moveNext() {
@@ -140,10 +142,10 @@ private static class SeparatedLinesEnumerable
       final BufferedReader br = new BufferedReader(isr);
       return new Enumerator() {
         private final StringBuilder b = new StringBuilder();
-        private String line;
+        private @Nullable String line;
 
         @Override public String current() {
-          return line;
+          return requireNonNull(line, "line");
         }
 
         @Override public boolean moveNext() {
diff --git a/plus/src/main/java/org/apache/calcite/adapter/os/PsTableFunction.java b/plus/src/main/java/org/apache/calcite/adapter/os/PsTableFunction.java
index 2337bbf27c2e..f6c41747b8a9 100644
--- a/plus/src/main/java/org/apache/calcite/adapter/os/PsTableFunction.java
+++ b/plus/src/main/java/org/apache/calcite/adapter/os/PsTableFunction.java
@@ -39,6 +39,7 @@
 
 import static java.lang.Float.parseFloat;
 import static java.lang.Long.parseLong;
+import static java.util.Objects.requireNonNull;
 
 /**
  * Table function that executes the OS "ps" command
@@ -87,9 +88,9 @@ private PsTableFunction() {
    * predefined list of parameters.
    */
   @VisibleForTesting
-  protected static class LineParser implements Function1 {
+  protected static class LineParser implements Function1 {
 
-    @Override public Object[] apply(String line) {
+    @Override public @Nullable Object[] apply(String line) {
       final String[] tokens = line.trim().split(" +");
       final Object[] values = new Object[PS_FIELD_NAMES.size()];
 
@@ -156,17 +157,18 @@ private static Object field(String field, String value) {
         final Matcher m1 =
             MINUTE_SECOND_MILLIS_PATTERN.matcher(value);
         if (m1.matches()) {
-          final long h = parseLong(m1.group(1));
-          final long m = parseLong(m1.group(2));
-          final long s = parseLong(m1.group(3));
+          final long h = parseLong(requireNonNull(m1.group(1), "group"));
+          final long m = parseLong(requireNonNull(m1.group(2), "group"));
+          final long s = parseLong(requireNonNull(m1.group(3), "group"));
           return h * 3600000L + m * 60000L + s * 1000L;
         }
         final Matcher m2 =
             HOUR_MINUTE_SECOND_PATTERN.matcher(value);
         if (m2.matches()) {
-          final long m = parseLong(m2.group(1));
-          final long s = parseLong(m2.group(2));
-          StringBuilder g3 = new StringBuilder(m2.group(3));
+          final long m = parseLong(requireNonNull(m2.group(1), "group"));
+          final long s = parseLong(requireNonNull(m2.group(2), "group"));
+          StringBuilder g3 =
+              new StringBuilder(requireNonNull(m2.group(3), "group"));
           while (g3.length() < 3) {
             g3.append("0");
           }
@@ -190,7 +192,8 @@ public static ScannableTable eval(boolean b) {
         final RelDataType rowType = getRowType(root.getTypeFactory());
         final List fieldNames = ImmutableList.copyOf(rowType.getFieldNames());
         final String[] args;
-        final String osName = System.getProperty("os.name");
+        final String osName =
+            requireNonNull(System.getProperty("os.name"), "os.name");
         final String osVersion = System.getProperty("os.version");
         Util.discard(osVersion);
         switch (osName) {
diff --git a/plus/src/main/java/org/apache/calcite/adapter/os/SqlShell.java b/plus/src/main/java/org/apache/calcite/adapter/os/SqlShell.java
index 20c3fac5a9ca..cd451a319939 100644
--- a/plus/src/main/java/org/apache/calcite/adapter/os/SqlShell.java
+++ b/plus/src/main/java/org/apache/calcite/adapter/os/SqlShell.java
@@ -23,6 +23,8 @@
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Maps;
 
+import org.jspecify.annotations.Nullable;
+
 import java.io.InputStreamReader;
 import java.io.OutputStreamWriter;
 import java.io.PrintWriter;
@@ -274,7 +276,7 @@ enum Format {
         }
       }
 
-      private void value(StringBuilder b, String s) {
+      private void value(StringBuilder b, @Nullable String s) {
         if (s == null) {
           // do nothing - unfortunately same as empty string
         } else if (s.contains("\"")) {
@@ -312,7 +314,8 @@ private void value(StringBuilder b, String s) {
           json.append(b, 0,
               Maps.asMap(fields, columnLabel -> {
                 try {
-                  final int i1 = fieldOrdinals.get(columnLabel);
+                  final int i1 =
+                      requireNonNull(fieldOrdinals.get(columnLabel), columnLabel);
                   switch (m.getColumnType(i1)) {
                   case Types.BOOLEAN:
                     final boolean b1 = r.getBoolean(i1);
@@ -358,7 +361,7 @@ private void value(StringBuilder b, String s) {
 
         final ResultSetMetaData m = r.getMetaData();
         final int n = m.getColumnCount();
-        final List values = new ArrayList<>();
+        final List<@Nullable String> values = new ArrayList<>();
         final int[] lengths = new int[n];
         final boolean[] rights = new boolean[n];
         for (int i = 0; i < n; i++) {
@@ -435,7 +438,7 @@ private void value(StringBuilder b, String s) {
         out.println();
       }
 
-      private void value(StringBuilder b, String value, int length,
+      private void value(StringBuilder b, @Nullable String value, int length,
           boolean right) {
         if (value == null) {
           pad(b, length, ' ');
diff --git a/plus/src/main/java/org/apache/calcite/adapter/os/StdinTableFunction.java b/plus/src/main/java/org/apache/calcite/adapter/os/StdinTableFunction.java
index 53e7424ecf1b..64fbe23cef93 100644
--- a/plus/src/main/java/org/apache/calcite/adapter/os/StdinTableFunction.java
+++ b/plus/src/main/java/org/apache/calcite/adapter/os/StdinTableFunction.java
@@ -46,47 +46,7 @@ public static ScannableTable eval(boolean b) {
     return new AbstractBaseScannableTable() {
       @Override public Enumerable<@Nullable Object[]> scan(DataContext root) {
         final InputStream is = DataContext.Variable.STDIN.get(root);
-        return new AbstractEnumerable() {
-          final InputStreamReader in =
-              new InputStreamReader(is, StandardCharsets.UTF_8);
-          final BufferedReader br = new BufferedReader(in);
-
-          @Override public Enumerator enumerator() {
-            return new Enumerator() {
-              @Nullable String line;
-              int i;
-
-              @Override public Object[] current() {
-                if (line == null) {
-                  throw new NoSuchElementException();
-                }
-                return new Object[] {i, line};
-              }
-
-              @Override public boolean moveNext() {
-                try {
-                  line = br.readLine();
-                  ++i;
-                  return line != null;
-                } catch (IOException e) {
-                  throw new RuntimeException(e);
-                }
-              }
-
-              @Override public void reset() {
-                throw new UnsupportedOperationException();
-              }
-
-              @Override public void close() {
-                try {
-                  br.close();
-                } catch (IOException e) {
-                  throw new RuntimeException(e);
-                }
-              }
-            };
-          }
-        };
+        return new StdinEnumerable(is);
       }
 
       @Override public RelDataType getRowType(RelDataTypeFactory typeFactory) {
@@ -97,4 +57,59 @@ public static ScannableTable eval(boolean b) {
       }
     };
   }
+
+  /** Enumerable over the lines of standard input. */
+  private static class StdinEnumerable
+      extends AbstractEnumerable<@Nullable Object[]> {
+    private final BufferedReader br;
+
+    StdinEnumerable(InputStream is) {
+      this.br =
+          new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8));
+    }
+
+    @Override public Enumerator<@Nullable Object[]> enumerator() {
+      return new StdinEnumerator(br);
+    }
+
+    /** Enumerator over the lines of standard input. */
+    private static class StdinEnumerator implements Enumerator<@Nullable Object[]> {
+      private final BufferedReader br;
+      private @Nullable String line;
+      private int i;
+
+      StdinEnumerator(BufferedReader br) {
+        this.br = br;
+      }
+
+      @Override public @Nullable Object[] current() {
+        if (line == null) {
+          throw new NoSuchElementException();
+        }
+        return new Object[] {i, line};
+      }
+
+      @Override public boolean moveNext() {
+        try {
+          line = br.readLine();
+          ++i;
+          return line != null;
+        } catch (IOException e) {
+          throw new RuntimeException(e);
+        }
+      }
+
+      @Override public void reset() {
+        throw new UnsupportedOperationException();
+      }
+
+      @Override public void close() {
+        try {
+          br.close();
+        } catch (IOException e) {
+          throw new RuntimeException(e);
+        }
+      }
+    }
+  }
 }
diff --git a/plus/src/main/java/org/apache/calcite/adapter/os/SystemInfoTableFunction.java b/plus/src/main/java/org/apache/calcite/adapter/os/SystemInfoTableFunction.java
index f01aedb127d0..dd75b53bca60 100644
--- a/plus/src/main/java/org/apache/calcite/adapter/os/SystemInfoTableFunction.java
+++ b/plus/src/main/java/org/apache/calcite/adapter/os/SystemInfoTableFunction.java
@@ -17,9 +17,7 @@
 package org.apache.calcite.adapter.os;
 
 import org.apache.calcite.DataContext;
-import org.apache.calcite.linq4j.AbstractEnumerable;
 import org.apache.calcite.linq4j.Enumerable;
-import org.apache.calcite.linq4j.Enumerator;
 import org.apache.calcite.rel.type.RelDataType;
 import org.apache.calcite.rel.type.RelDataTypeFactory;
 import org.apache.calcite.schema.ScannableTable;
@@ -37,11 +35,7 @@ private SystemInfoTableFunction() {
   public static ScannableTable eval(boolean b) {
     return new AbstractBaseScannableTable() {
       @Override public Enumerable<@Nullable Object[]> scan(DataContext root) {
-        return new AbstractEnumerable() {
-          @Override public Enumerator enumerator() {
-            return new OsQuery("system_info");
-          }
-        };
+        return new OsQueryEnumerable("system_info");
       }
 
       @Override public RelDataType getRowType(RelDataTypeFactory typeFactory) {
diff --git a/plus/src/main/java/org/apache/calcite/adapter/os/VmstatTableFunction.java b/plus/src/main/java/org/apache/calcite/adapter/os/VmstatTableFunction.java
index 87eb474df478..0f043d6a87f5 100644
--- a/plus/src/main/java/org/apache/calcite/adapter/os/VmstatTableFunction.java
+++ b/plus/src/main/java/org/apache/calcite/adapter/os/VmstatTableFunction.java
@@ -32,6 +32,7 @@
 import java.util.List;
 
 import static java.lang.Long.parseLong;
+import static java.util.Objects.requireNonNull;
 
 /**
  * Table function that executes the OS "vmstat" command
@@ -48,7 +49,8 @@ public static ScannableTable eval(boolean b) {
         final List fieldNames =
             ImmutableList.copyOf(rowType.getFieldNames());
         final String[] args;
-        final String osName = System.getProperty("os.name");
+        final String osName =
+            requireNonNull(System.getProperty("os.name"), "os.name");
         final String osVersion = System.getProperty("os.version");
         Util.discard(osVersion);
         // Fork out to a shell so that we can get normal text-munging support.
@@ -63,39 +65,12 @@ public static ScannableTable eval(boolean b) {
         default:
           args = new String[]{"/bin/sh", "-c", "vmstat -n | tail -n +3"};
         }
-        return Processes.processLines(args)
-            .select(
-                new Function1() {
-                  @Override public Object[] apply(String line) {
-                    final String[] fields = line.trim().split("\\s+");
-                    final Object[] values = new Object[fieldNames.size()];
-                    for (int i = 0; i < values.length; i++) {
-                      try {
-                        values[i] = field(fieldNames.get(i), fields[i]);
-                      } catch (RuntimeException e) {
-                        e.printStackTrace(System.out);
-                        throw new RuntimeException("while parsing value ["
-                            + fields[i] + "] of field [" + fieldNames.get(i)
-                            + "] in line [" + line + "]");
-                      }
-                    }
-                    return values;
-                  }
-
-                  private Object field(@SuppressWarnings("unused") String field, String value) {
-                    if (value.isEmpty()) {
-                      return 0;
-                    }
-                    if (value.endsWith(".")) {
-                      return parseLong(value);
-                    }
-                    return parseLong(value);
-                  }
-                });
+        return Processes.processLines(args).select(new LineParser(fieldNames));
       }
 
       @Override public RelDataType getRowType(RelDataTypeFactory typeFactory) {
-        final String osName = System.getProperty("os.name");
+        final String osName =
+            requireNonNull(System.getProperty("os.name"), "os.name");
         final RelDataTypeFactory.Builder builder = typeFactory.builder();
         switch (osName) {
         case "Mac OS X":
@@ -147,4 +122,38 @@ private Object field(@SuppressWarnings("unused") String field, String value) {
       }
     };
   }
+
+  /** Parses one line of vmstat output into a row. */
+  private static class LineParser implements Function1 {
+    private final List fieldNames;
+
+    LineParser(List fieldNames) {
+      this.fieldNames = fieldNames;
+    }
+
+    @Override public @Nullable Object[] apply(String line) {
+      final String[] fields = line.trim().split("\\s+");
+      final @Nullable Object[] values = new Object[fieldNames.size()];
+      for (int i = 0; i < values.length; i++) {
+        try {
+          values[i] = field(fieldNames.get(i), fields[i]);
+        } catch (RuntimeException e) {
+          e.printStackTrace(System.out);
+          throw new RuntimeException("while parsing value ["
+              + fields[i] + "] of field [" + fieldNames.get(i)
+              + "] in line [" + line + "]");
+        }
+      }
+      return values;
+    }
+
+    private static Object field(@SuppressWarnings("unused") String field,
+        String value) {
+      if (value.isEmpty()) {
+        return 0;
+      }
+      return parseLong(value);
+    }
+  }
+
 }
diff --git a/plus/src/main/java/org/apache/calcite/adapter/os/package-info.java b/plus/src/main/java/org/apache/calcite/adapter/os/package-info.java
index 9ec1e4679a64..d3073f4a1c23 100644
--- a/plus/src/main/java/org/apache/calcite/adapter/os/package-info.java
+++ b/plus/src/main/java/org/apache/calcite/adapter/os/package-info.java
@@ -19,4 +19,7 @@
  * The OS adapter contains various table functions that let you query data
  * sources in your operating system and environment.
  */
+@NullMarked
 package org.apache.calcite.adapter.os;
+
+import org.jspecify.annotations.NullMarked;
diff --git a/plus/src/main/java/org/apache/calcite/adapter/tpcds/TpcdsSchema.java b/plus/src/main/java/org/apache/calcite/adapter/tpcds/TpcdsSchema.java
index 153e8da71124..acd2426179de 100644
--- a/plus/src/main/java/org/apache/calcite/adapter/tpcds/TpcdsSchema.java
+++ b/plus/src/main/java/org/apache/calcite/adapter/tpcds/TpcdsSchema.java
@@ -182,13 +182,13 @@ private class TpcdsSchemaQueryable extends AbstractTableQueryable<@Nullable Obje
 
     /** Selector for {@link TpcdsSchema}. */
     private class TpcdsSchemaSelector
-        implements Function1>, Enumerable<@Nullable Object[]>> {
+        implements Function1>, Enumerable<@Nullable Object[]>> {
       final Column[] columns = tpcdsTable.getColumns();
 
       @Override public Enumerable<@Nullable Object[]> apply(
-          List> inRows) {
+          List> inRows) {
         final List<@Nullable Object[]> rows = new ArrayList<>();
-        for (List<@Nullable String> strings : inRows) {
+        for (List strings : inRows) {
           final @Nullable Object[] values = new Object[columns.length];
           for (int i = 0; i < strings.size(); i++) {
             values[i] = convert(strings.get(i), columns[i]);
diff --git a/plus/src/main/java/org/apache/calcite/adapter/tpcds/package-info.java b/plus/src/main/java/org/apache/calcite/adapter/tpcds/package-info.java
index 0bfb52c6adc4..4749dcc6d0d4 100644
--- a/plus/src/main/java/org/apache/calcite/adapter/tpcds/package-info.java
+++ b/plus/src/main/java/org/apache/calcite/adapter/tpcds/package-info.java
@@ -18,4 +18,7 @@
 /**
  * TPC-DS schema.
  */
+@NullMarked
 package org.apache.calcite.adapter.tpcds;
+
+import org.jspecify.annotations.NullMarked;
diff --git a/plus/src/main/java/org/apache/calcite/adapter/tpch/package-info.java b/plus/src/main/java/org/apache/calcite/adapter/tpch/package-info.java
index c76dfe13fb83..3f92405009cd 100644
--- a/plus/src/main/java/org/apache/calcite/adapter/tpch/package-info.java
+++ b/plus/src/main/java/org/apache/calcite/adapter/tpch/package-info.java
@@ -18,4 +18,7 @@
 /**
  * TPC-H schema.
  */
+@NullMarked
 package org.apache.calcite.adapter.tpch;
+
+import org.jspecify.annotations.NullMarked;
diff --git a/plus/src/main/java/org/apache/calcite/adapter/utils/package-info.java b/plus/src/main/java/org/apache/calcite/adapter/utils/package-info.java
index 619fdc2c2d9a..4d162f69334c 100644
--- a/plus/src/main/java/org/apache/calcite/adapter/utils/package-info.java
+++ b/plus/src/main/java/org/apache/calcite/adapter/utils/package-info.java
@@ -18,4 +18,7 @@
 /**
  * Used to put OS adapter related util classes.
  */
+@NullMarked
 package org.apache.calcite.adapter.utils;
+
+import org.jspecify.annotations.NullMarked;
diff --git a/plus/src/main/java/org/apache/calcite/chinook/ChinookAvaticaServer.java b/plus/src/main/java/org/apache/calcite/chinook/ChinookAvaticaServer.java
index 4bcb00540d40..df71789eab35 100644
--- a/plus/src/main/java/org/apache/calcite/chinook/ChinookAvaticaServer.java
+++ b/plus/src/main/java/org/apache/calcite/chinook/ChinookAvaticaServer.java
@@ -25,6 +25,8 @@
 
 import net.hydromatic.chinook.data.hsqldb.ChinookHsqldb;
 
+import org.jspecify.annotations.Nullable;
+
 import java.io.IOException;
 import java.sql.SQLException;
 import java.util.List;
@@ -34,6 +36,8 @@
  * between Avatica JDBC transport and Calcite.
  */
 public class ChinookAvaticaServer {
+  /** Set by {@link #startWithCalcite} or {@link #startWithRaw}. */
+  @SuppressWarnings("NullAway.Init")
   private HttpServer server;
 
   public void startWithCalcite() throws Exception {
@@ -62,7 +66,7 @@ public static class CalciteChinookMetaFactory implements Meta.Factory {
     private static final CalciteConnectionProvider CONNECTION_PROVIDER =
         new CalciteConnectionProvider();
 
-    private static volatile JdbcMeta instance = null;
+    private static volatile @Nullable JdbcMeta instance = null;
 
     private static JdbcMeta getInstance() {
       if (instance == null) {
@@ -88,7 +92,7 @@ private static JdbcMeta getInstance() {
    * Factory for Chinook Calcite database wrapped in meta for Avatica.
    */
   public static class RawChinookMetaFactory implements Meta.Factory {
-    private static volatile JdbcMeta instance = null;
+    private static volatile @Nullable JdbcMeta instance = null;
 
     private static JdbcMeta getInstance() {
       if (instance == null) {
diff --git a/plus/src/main/java/org/apache/calcite/chinook/package-info.java b/plus/src/main/java/org/apache/calcite/chinook/package-info.java
index 3ce1f9feb3cd..6bcc23f81f9e 100644
--- a/plus/src/main/java/org/apache/calcite/chinook/package-info.java
+++ b/plus/src/main/java/org/apache/calcite/chinook/package-info.java
@@ -18,4 +18,7 @@
 /**
  * End to end tests.
  */
+@NullMarked
 package org.apache.calcite.chinook;
+
+import org.jspecify.annotations.NullMarked;

From 80761ff4389d826cdbf26f85586e671fe2676255 Mon Sep 17 00:00:00 2001
From: Vladimir Sitnikov 
Date: Tue, 25 Aug 2026 22:10:27 +0300
Subject: [PATCH 559/562] [CALCITE-7736] Put :example:csv and :example:function
 under nullness verification

A model that names no directory for a CSV schema, and no file for a CSV table,
reached the File with a null; requireNonNull names the missing operand. The
filterable table pushed down a literal that may hold no value. The maze
enumerates without a solution set when the table is asked for the maze alone,
which is what the null argument means.

Co-Authored-By: Claude Opus 5 
---
 build.gradle.kts                                            | 2 +-
 .../org/apache/calcite/adapter/csv/CsvFilterableTable.java  | 6 +++++-
 .../org/apache/calcite/adapter/csv/CsvSchemaFactory.java    | 5 ++++-
 .../apache/calcite/adapter/csv/CsvStreamTableFactory.java   | 4 +++-
 .../org/apache/calcite/adapter/csv/CsvTableFactory.java     | 4 +++-
 .../java/org/apache/calcite/adapter/csv/package-info.java   | 3 +++
 .../src/main/java/org/apache/calcite/example/maze/Maze.java | 4 +++-
 .../java/org/apache/calcite/example/maze/MazeTable.java     | 2 +-
 .../java/org/apache/calcite/example/maze/package-info.java  | 3 +++
 9 files changed, 26 insertions(+), 7 deletions(-)

diff --git a/build.gradle.kts b/build.gradle.kts
index d7d4f9fe5173..be463f4dab77 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -94,7 +94,7 @@ val werror by props(true) // treat javac warnings as errors
 
 // Projects whose main code NullAway verifies. The other projects are not annotated well enough
 // yet, so NullAway would only produce noise there.
-val nullawayProjects = listOf(":linq4j", ":core", ":server", ":druid", ":file", ":kafka", ":spark", ":babel", ":redis", ":splunk", ":mongodb", ":cassandra", ":pig", ":arrow", ":innodb", ":piglet", ":geode", ":elasticsearch", ":plus")
+val nullawayProjects = listOf(":linq4j", ":core", ":server", ":druid", ":file", ":kafka", ":spark", ":babel", ":redis", ":splunk", ":mongodb", ":cassandra", ":pig", ":arrow", ":innodb", ":piglet", ":geode", ":elasticsearch", ":plus", ":example:csv", ":example:function")
 
 val hepLargePlanModeTestIncludes = mapOf(
     ":core" to listOf(
diff --git a/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvFilterableTable.java b/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvFilterableTable.java
index e8be1b736538..16d6307f1df9 100644
--- a/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvFilterableTable.java
+++ b/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvFilterableTable.java
@@ -38,6 +38,8 @@
 import java.util.List;
 import java.util.concurrent.atomic.AtomicBoolean;
 
+import static java.util.Objects.requireNonNull;
+
 /**
  * Table based on a CSV file that can implement simple filtering.
  *
@@ -103,7 +105,9 @@ private static boolean addFilter(RexNode filter, @Nullable Object[] filterValues
           && right instanceof RexLiteral) {
         final int index = ((RexInputRef) left).getIndex();
         if (filterValues[index] == null) {
-          filterValues[index] = ((RexLiteral) right).getValue2().toString();
+          filterValues[index] =
+              requireNonNull(((RexLiteral) right).getValue2(), "value")
+                  .toString();
           return true;
         }
       }
diff --git a/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvSchemaFactory.java b/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvSchemaFactory.java
index 79b96af9cc9b..a7f0bf940765 100644
--- a/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvSchemaFactory.java
+++ b/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvSchemaFactory.java
@@ -25,6 +25,8 @@
 import java.util.Locale;
 import java.util.Map;
 
+import static java.util.Objects.requireNonNull;
+
 /**
  * Factory that creates a {@link CsvSchema}.
  *
@@ -41,7 +43,8 @@ private CsvSchemaFactory() {
 
   @Override public Schema create(SchemaPlus parentSchema, String name,
       Map operand) {
-    final String directory = (String) operand.get("directory");
+    final String directory =
+        requireNonNull((String) operand.get("directory"), "directory");
     final File base =
         (File) operand.get(ModelHandler.ExtraOperand.BASE_DIRECTORY.camelName);
     File directoryFile = new File(directory);
diff --git a/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvStreamTableFactory.java b/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvStreamTableFactory.java
index 615f91f20c3f..1f5715ea324d 100644
--- a/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvStreamTableFactory.java
+++ b/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvStreamTableFactory.java
@@ -30,6 +30,8 @@
 import java.io.File;
 import java.util.Map;
 
+import static java.util.Objects.requireNonNull;
+
 /**
  * Factory that creates a {@link CsvTranslatableTable}.
  *
@@ -44,7 +46,7 @@ public CsvStreamTableFactory() {
 
   @Override public CsvTable create(SchemaPlus schema, String name,
       Map operand, @Nullable RelDataType rowType) {
-    String fileName = (String) operand.get("file");
+    String fileName = requireNonNull((String) operand.get("file"), "file");
     File file = new File(fileName);
     final File base =
         (File) operand.get(ModelHandler.ExtraOperand.BASE_DIRECTORY.camelName);
diff --git a/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvTableFactory.java b/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvTableFactory.java
index d7efbe97e392..668829589e11 100644
--- a/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvTableFactory.java
+++ b/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvTableFactory.java
@@ -30,6 +30,8 @@
 import java.io.File;
 import java.util.Map;
 
+import static java.util.Objects.requireNonNull;
+
 /**
  * Factory that creates a {@link CsvTranslatableTable}.
  *
@@ -44,7 +46,7 @@ public CsvTableFactory() {
 
   @Override public CsvTable create(SchemaPlus schema, String name,
       Map operand, @Nullable RelDataType rowType) {
-    String fileName = (String) operand.get("file");
+    String fileName = requireNonNull((String) operand.get("file"), "file");
     final File base =
         (File) operand.get(ModelHandler.ExtraOperand.BASE_DIRECTORY.camelName);
     final Source source = Sources.file(base, fileName);
diff --git a/example/csv/src/main/java/org/apache/calcite/adapter/csv/package-info.java b/example/csv/src/main/java/org/apache/calcite/adapter/csv/package-info.java
index 5cb4d328b692..183c00978e91 100644
--- a/example/csv/src/main/java/org/apache/calcite/adapter/csv/package-info.java
+++ b/example/csv/src/main/java/org/apache/calcite/adapter/csv/package-info.java
@@ -22,4 +22,7 @@
  * directory appears as a table.  Full SQL operations are available on
  * those tables.
  */
+@NullMarked
 package org.apache.calcite.adapter.csv;
+
+import org.jspecify.annotations.NullMarked;
diff --git a/example/function/src/main/java/org/apache/calcite/example/maze/Maze.java b/example/function/src/main/java/org/apache/calcite/example/maze/Maze.java
index 08dcff8311a2..f37060c6771e 100644
--- a/example/function/src/main/java/org/apache/calcite/example/maze/Maze.java
+++ b/example/function/src/main/java/org/apache/calcite/example/maze/Maze.java
@@ -18,6 +18,8 @@
 
 import org.apache.calcite.linq4j.Enumerator;
 
+import org.jspecify.annotations.Nullable;
+
 import java.io.PrintWriter;
 import java.util.ArrayDeque;
 import java.util.ArrayList;
@@ -102,7 +104,7 @@ public void print(PrintWriter pw, boolean space) {
   }
 
   /** Generates a list of lines representing the maze in text form. */
-  public Enumerator enumerator(final Set solutionSet) {
+  public Enumerator enumerator(final @Nullable Set solutionSet) {
     final CellContent cellContent;
     if (solutionSet == null) {
       cellContent = CellContent.SPACE;
diff --git a/example/function/src/main/java/org/apache/calcite/example/maze/MazeTable.java b/example/function/src/main/java/org/apache/calcite/example/maze/MazeTable.java
index d715666ea88d..8cee3529584e 100644
--- a/example/function/src/main/java/org/apache/calcite/example/maze/MazeTable.java
+++ b/example/function/src/main/java/org/apache/calcite/example/maze/MazeTable.java
@@ -107,7 +107,7 @@ private class MazeTableEnumerable extends AbstractEnumerable<@Nullable Object[]>
     }
 
     @Override public Enumerator<@Nullable Object[]> enumerator() {
-      final Set solutionSet;
+      final @Nullable Set solutionSet;
       if (solution) {
         solutionSet = maze.solve(0, 0);
       } else {
diff --git a/example/function/src/main/java/org/apache/calcite/example/maze/package-info.java b/example/function/src/main/java/org/apache/calcite/example/maze/package-info.java
index baebf64a9c83..67d48ddfe10b 100644
--- a/example/function/src/main/java/org/apache/calcite/example/maze/package-info.java
+++ b/example/function/src/main/java/org/apache/calcite/example/maze/package-info.java
@@ -18,4 +18,7 @@
 /**
  * User-defined table function that generates a maze.
  */
+@NullMarked
 package org.apache.calcite.example.maze;
+
+import org.jspecify.annotations.NullMarked;

From 370f7baa95e7846e22493cd256530213f3513eab Mon Sep 17 00:00:00 2001
From: Vladimir Sitnikov 
Date: Wed, 26 Aug 2026 08:44:27 +0300
Subject: [PATCH 560/562] [CALCITE-7736] Put :testkit under nullness
 verification

The test fixtures carry absent values everywhere a SQL value can be null: the
expected result of checkScalar and checkString, the column a result set reads,
the origins and unique keys the metadata query may not know, and the alias a
Pig relation may not have. The mock catalog and the mock planner fill their
fields in as registration proceeds, so those are marked NullAway.Init.

Twelve classes took the @Nullable argument that Object.equals has always
allowed. DiffRepository reads a DOM, where a node list yields no node past its
length and an attribute may carry no value; the reads that cannot miss say so
by name. The eight schemata packages that had no package-info.java now have
one.

BaseQueryable in :linq4j required a provider, though Smalls builds one that
overrides enumerator() and never asks a provider to execute it; getProvider
still refuses to return null. Four anonymous enumerables became named classes,
to avoid uber/NullAway#1746.

Co-Authored-By: Claude Opus 5 
---
 build.gradle.kts                              |   2 +-
 .../apache/calcite/linq4j/BaseQueryable.java  |  12 +-
 .../calcite/sql/parser/SqlParserTest.java     |  29 ++-
 .../calcite/sql/parser/package-info.java      |   3 +
 .../calcite/sql/test/AbstractSqlTester.java   |   3 +-
 .../calcite/sql/test/ResultCheckers.java      |  16 +-
 .../calcite/sql/test/SqlOperatorFixture.java  |   7 +-
 .../org/apache/calcite/sql/test/SqlTests.java |  27 ++-
 .../apache/calcite/sql/test/package-info.java |   3 +
 .../apache/calcite/test/CalciteAssert.java    |  45 ++--
 .../calcite/test/ConnectionFactories.java     |   7 +-
 .../apache/calcite/test/DiffRepository.java   |  43 ++--
 .../org/apache/calcite/test/Matchers.java     |   5 +-
 .../calcite/test/MockRelOptPlanner.java       |  10 +-
 .../calcite/test/MockSqlOperatorTable.java    |  11 +-
 .../org/apache/calcite/test/QuidemTest.java   |  45 ++--
 .../calcite/test/RelMetadataFixture.java      |  19 +-
 .../apache/calcite/test/RelOptFixture.java    |   5 +-
 .../org/apache/calcite/test/RelSupplier.java  |   6 +-
 .../calcite/test/SqlOperatorFixtureImpl.java  |   2 +-
 .../apache/calcite/test/SqlOperatorTest.java  |  14 +-
 .../apache/calcite/test/SqlToRelTestBase.java |   5 +-
 .../calcite/test/SqlValidatorFixture.java     |   8 +-
 .../test/catalog/MockCatalogReader.java       |  48 ++--
 .../test/catalog/MockCatalogReaderSimple.java |   3 +-
 .../calcite/test/catalog/package-info.java    |   3 +
 .../org/apache/calcite/test/package-info.java |   3 +
 .../test/schemata/bookstore/package-info.java |  24 ++
 .../schemata/catchall/CatchallSchema.java     |   6 +-
 .../test/schemata/catchall/package-info.java  |  24 ++
 .../test/schemata/countries/package-info.java |  24 ++
 .../schemata/foodmart/FoodmartSchema.java     |   2 +-
 .../test/schemata/foodmart/package-info.java  |  24 ++
 .../calcite/test/schemata/hr/Department.java  |   2 +-
 .../calcite/test/schemata/hr/Dependent.java   |   4 +-
 .../calcite/test/schemata/hr/Employee.java    |   2 +-
 .../calcite/test/schemata/hr/Event.java       |   2 +-
 .../test/schemata/hr/HierarchySchema.java     |   4 +-
 .../calcite/test/schemata/hr/Location.java    |   4 +-
 .../test/schemata/hr/NullableTest.java        |   2 +-
 .../test/schemata/hr/package-info.java        |  24 ++
 .../test/schemata/lingual/LingualEmp.java     |   4 +-
 .../test/schemata/lingual/package-info.java   |  24 ++
 .../orderstream/InfiniteOrdersTable.java      |  40 ++--
 .../orderstream/OrdersHistoryTable.java       |   6 +-
 .../orderstream/OrdersStreamTableFactory.java |   8 +-
 .../schemata/orderstream/OrdersTable.java     |   6 +-
 .../schemata/orderstream/ProductsTable.java   |   6 +-
 .../schemata/orderstream/package-info.java    |  24 ++
 .../test/schemata/tpch/package-info.java      |  24 ++
 .../java/org/apache/calcite/util/Smalls.java  | 210 ++++++++++--------
 .../org/apache/calcite/util/TestUtil.java     |  30 +--
 .../org/apache/calcite/util/package-info.java |   3 +
 53 files changed, 624 insertions(+), 293 deletions(-)
 create mode 100644 testkit/src/main/java/org/apache/calcite/test/schemata/bookstore/package-info.java
 create mode 100644 testkit/src/main/java/org/apache/calcite/test/schemata/catchall/package-info.java
 create mode 100644 testkit/src/main/java/org/apache/calcite/test/schemata/countries/package-info.java
 create mode 100644 testkit/src/main/java/org/apache/calcite/test/schemata/foodmart/package-info.java
 create mode 100644 testkit/src/main/java/org/apache/calcite/test/schemata/hr/package-info.java
 create mode 100644 testkit/src/main/java/org/apache/calcite/test/schemata/lingual/package-info.java
 create mode 100644 testkit/src/main/java/org/apache/calcite/test/schemata/orderstream/package-info.java
 create mode 100644 testkit/src/main/java/org/apache/calcite/test/schemata/tpch/package-info.java

diff --git a/build.gradle.kts b/build.gradle.kts
index be463f4dab77..f01ff0934eda 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -94,7 +94,7 @@ val werror by props(true) // treat javac warnings as errors
 
 // Projects whose main code NullAway verifies. The other projects are not annotated well enough
 // yet, so NullAway would only produce noise there.
-val nullawayProjects = listOf(":linq4j", ":core", ":server", ":druid", ":file", ":kafka", ":spark", ":babel", ":redis", ":splunk", ":mongodb", ":cassandra", ":pig", ":arrow", ":innodb", ":piglet", ":geode", ":elasticsearch", ":plus", ":example:csv", ":example:function")
+val nullawayProjects = listOf(":linq4j", ":core", ":server", ":druid", ":file", ":kafka", ":spark", ":babel", ":redis", ":splunk", ":mongodb", ":cassandra", ":pig", ":arrow", ":innodb", ":piglet", ":geode", ":elasticsearch", ":plus", ":example:csv", ":example:function", ":testkit")
 
 val hepLargePlanModeTestIncludes = mapOf(
     ":core" to listOf(
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/BaseQueryable.java b/linq4j/src/main/java/org/apache/calcite/linq4j/BaseQueryable.java
index 37b5cbdca775..c067ae989d21 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/BaseQueryable.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/BaseQueryable.java
@@ -23,6 +23,8 @@
 import java.lang.reflect.Type;
 import java.util.Iterator;
 
+import static java.util.Objects.requireNonNull;
+
 /**
  * Skeleton implementation of {@link Queryable}.
  *
@@ -34,11 +36,13 @@
  */
 public abstract class BaseQueryable
     extends AbstractQueryable {
-  protected final QueryProvider provider;
+  /** Provider, or null for a queryable that overrides {@link #enumerator()}
+   * and so never asks the provider to execute it. */
+  protected final @Nullable QueryProvider provider;
   protected final Type elementType;
   protected final @Nullable Expression expression;
 
-  protected BaseQueryable(QueryProvider provider, Type elementType,
+  protected BaseQueryable(@Nullable QueryProvider provider, Type elementType,
       @Nullable Expression expression) {
     this.provider = provider;
     this.elementType = elementType;
@@ -46,7 +50,7 @@ protected BaseQueryable(QueryProvider provider, Type elementType,
   }
 
   @Override public QueryProvider getProvider() {
-    return provider;
+    return requireNonNull(provider, "provider");
   }
 
   @Override public Type getElementType() {
@@ -62,6 +66,6 @@ protected BaseQueryable(QueryProvider provider, Type elementType,
   }
 
   @Override public Enumerator enumerator() {
-    return provider.executeQuery(this);
+    return getProvider().executeQuery(this);
   }
 }
diff --git a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java
index 4d5c9e806d49..8f683c3021d8 100644
--- a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java
+++ b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java
@@ -6908,7 +6908,7 @@ private static Matcher isCharLiteral(String s) {
       }
     });
     assertNotSame(sqlNodeVisited, sqlNode);
-    assertThat(sqlNodeVisited.getKind(), is(SqlKind.INSERT));
+    assertThat(requireNonNull(sqlNodeVisited, "sqlNodeVisited").getKind(), is(SqlKind.INSERT));
   }
 
   @Test void testSqlInsertSqlBasicCallToString() {
@@ -6923,7 +6923,8 @@ private static Matcher isCharLiteral(String s) {
     final String str0 = "INSERT INTO `EMPS`\n"
         + "SELECT *\n"
         + "FROM `EMPS`";
-    assertThat(str0, is(toLinux(sqlNodeVisited0.toString())));
+    assertThat(str0,
+        is(toLinux(requireNonNull(sqlNodeVisited0, "sqlNodeVisited0").toString())));
 
     final String sql1 = "insert into emps select empno from emps";
     final SqlNode sqlNode1 = sql(sql1).node();
@@ -6936,7 +6937,8 @@ private static Matcher isCharLiteral(String s) {
     final String str1 = "INSERT INTO `EMPS`\n"
         + "SELECT `EMPNO`\n"
         + "FROM `EMPS`";
-    assertThat(str1, is(toLinux(sqlNodeVisited1.toString())));
+    assertThat(str1,
+        is(toLinux(requireNonNull(sqlNodeVisited1, "sqlNodeVisited1").toString())));
   }
 
   @Test void testVisitSqlUpdateWithSqlShuttle() {
@@ -6949,7 +6951,7 @@ private static Matcher isCharLiteral(String s) {
       }
     });
     assertNotSame(sqlNodeVisited, sqlNode);
-    assertThat(sqlNodeVisited.getKind(), is(SqlKind.UPDATE));
+    assertThat(requireNonNull(sqlNodeVisited, "sqlNodeVisited").getKind(), is(SqlKind.UPDATE));
     final String str1 = "UPDATE `EMPS` AS `E` SET `E`.`SAL` = 0\n"
         + "WHERE `E`.`SAL` < 0";
     assertThat(str1, is(toLinux(sqlNodeVisited.toString())));
@@ -8298,7 +8300,8 @@ private static Consumer> checkWarnings(
     SqlPrettyWriter writer = new SqlPrettyWriter();
     assertThat(writer.format(opt.name()), equalTo("\"SCHEMA\""));
     writer = new SqlPrettyWriter();
-    assertThat(writer.format(opt.getValue()), equalTo("TRUE"));
+    assertThat(writer.format(requireNonNull(opt.getValue(), "value")),
+        equalTo("TRUE"));
     writer = new SqlPrettyWriter();
     assertThat(writer.format(opt),
         equalTo("ALTER SYSTEM SET \"SCHEMA\" = TRUE"));
@@ -8509,7 +8512,8 @@ private static Consumer> checkWarnings(
         return argHandler.result();
       }
     });
-    assertThat(toLinux(shuttled.toString()), is(expected));
+    assertThat(toLinux(requireNonNull(shuttled, "shuttled").toString()),
+        is(expected));
   }
 
   @Test void testMatchRecognize1() {
@@ -9929,7 +9933,8 @@ private static Consumer> checkWarnings(
     final String expected = "SELECT *\n"
         + "FROM `EMP`\n"
         + "/*+ `OPTIONS`('key1' = 'val1') */";
-    assertThat(toLinux(shuttled.toString()), is(expected));
+    assertThat(toLinux(requireNonNull(shuttled, "shuttled").toString()),
+        is(expected));
   }
 
   @Test void testInvalidHintFormat() {
@@ -10311,7 +10316,8 @@ static SqlWriterConfig.LineFolding nextLineFolding(Random random) {
     }
 
     static > E nextEnum(Random random, Class enumClass) {
-      final E[] constants = enumClass.getEnumConstants();
+      final E[] constants =
+          requireNonNull(enumClass.getEnumConstants(), "constants");
       return constants[random.nextInt(constants.length)];
     }
 
@@ -10329,7 +10335,7 @@ static void checkList(SqlNodeList sqlNodeList,
       }
     }
 
-    static SqlNode deepCopy(SqlNode sqlNode) {
+    static @Nullable SqlNode deepCopy(SqlNode sqlNode) {
       return sqlNode.accept(new SqlShuttle() {
         @Override public @Nullable SqlNode visit(final SqlCall call) {
           // Handler always creates a new copy of 'call'
@@ -10371,7 +10377,8 @@ static SqlNode deepCopy(SqlNode sqlNode) {
       assertThat(sql3, notNullValue());
 
       // Make a deep copy of the SqlNodeList, unparse it.
-      final SqlNodeList sqlNodeList3 = (SqlNodeList) deepCopy(sqlNodeList);
+      final SqlNodeList sqlNodeList3 =
+          (SqlNodeList) requireNonNull(deepCopy(sqlNodeList), "copy");
       final String sql4 = toSqlString(sqlNodeList3, simple());
       // Should be the same as we started with.
       assertThat(sql4, is(sql1));
@@ -10421,7 +10428,7 @@ static SqlNode deepCopy(SqlNode sqlNode) {
       assertThat(sql4, is(sql1));
 
       // Make a deep copy of the original SqlNode, unparse it.
-      final SqlNode sqlNode5 = deepCopy(sqlNode);
+      final SqlNode sqlNode5 = requireNonNull(deepCopy(sqlNode), "copy");
       final String actual5 = sqlNode5.toSqlString(writerTransform).getSql();
       assertThat(converter.apply(actual5), is(expected));
     }
diff --git a/testkit/src/main/java/org/apache/calcite/sql/parser/package-info.java b/testkit/src/main/java/org/apache/calcite/sql/parser/package-info.java
index 717d8d6860ea..b57b590f68ea 100644
--- a/testkit/src/main/java/org/apache/calcite/sql/parser/package-info.java
+++ b/testkit/src/main/java/org/apache/calcite/sql/parser/package-info.java
@@ -18,4 +18,7 @@
 /**
  * Classes for testing SQL Parser.
  */
+@NullMarked
 package org.apache.calcite.sql.parser;
+
+import org.jspecify.annotations.NullMarked;
diff --git a/testkit/src/main/java/org/apache/calcite/sql/test/AbstractSqlTester.java b/testkit/src/main/java/org/apache/calcite/sql/test/AbstractSqlTester.java
index 39b0b1e3fec3..6aaf44ffa235 100644
--- a/testkit/src/main/java/org/apache/calcite/sql/test/AbstractSqlTester.java
+++ b/testkit/src/main/java/org/apache/calcite/sql/test/AbstractSqlTester.java
@@ -76,7 +76,8 @@
  * {@link SqlValidator}.
  */
 public abstract class AbstractSqlTester implements SqlTester, AutoCloseable {
-  private  static final String NL = System.getProperty("line.separator");
+  private  static final String NL =
+      requireNonNull(System.getProperty("line.separator"), "line.separator");
 
   public AbstractSqlTester() {
   }
diff --git a/testkit/src/main/java/org/apache/calcite/sql/test/ResultCheckers.java b/testkit/src/main/java/org/apache/calcite/sql/test/ResultCheckers.java
index 1fba06067f8e..2c1bfd4d15ea 100644
--- a/testkit/src/main/java/org/apache/calcite/sql/test/ResultCheckers.java
+++ b/testkit/src/main/java/org/apache/calcite/sql/test/ResultCheckers.java
@@ -23,6 +23,7 @@
 import com.google.common.collect.ImmutableSet;
 
 import org.hamcrest.Matcher;
+import org.jspecify.annotations.Nullable;
 
 import java.math.BigDecimal;
 import java.sql.ResultSet;
@@ -36,6 +37,8 @@
 import java.util.Set;
 import java.util.regex.Pattern;
 
+import static org.apache.calcite.linq4j.Nullness.castNonNull;
+
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.MatcherAssert.assertThat;
@@ -82,8 +85,8 @@ public static SqlTester.ResultChecker isSingle(double delta, String value) {
     return isSingle(value);
   }
 
-  public static SqlTester.ResultChecker isSingle(String value) {
-    return new MatcherResultChecker<>(is(value),
+  public static SqlTester.ResultChecker isSingle(@Nullable String value) {
+    return new MatcherResultChecker<@Nullable String>(is(value),
         JdbcType.STRING_NULLABLE);
   }
 
@@ -107,7 +110,7 @@ public static SqlTester.ResultChecker isSet(String... values) {
   }
 
   public static SqlTester.ResultChecker isNullValue() {
-    return new RefSetResultChecker(Collections.singleton(null));
+    return new RefSetResultChecker(Collections.singleton(castNonNull(null)));
   }
 
   /**
@@ -129,7 +132,7 @@ static void compareResultSet(String sql, ResultSet resultSet,
       final String s = resultSet.getString(1);
       final String s0 = s == null ? "0" : s;
       final boolean wasNull0 = resultSet.wasNull();
-      actualSet.add(s);
+      actualSet.add(castNonNull(s));
       switch (rep) {
       case BOOLEAN:
       case PRIMITIVE_BOOLEAN:
@@ -221,7 +224,7 @@ static void compareResultSetWithPattern(String sql, ResultSet resultSet,
     if (resultSet.next()) {
       fail("Query \"" + sql + "\"returned 2 or more rows, expected 1");
     }
-    if (!pattern.matcher(actual).matches()) {
+    if (!pattern.matcher(requireNonNull(actual, "actual")).matches()) {
       fail("Query \"" + sql + "\"returned '"
               + actual
               + "', expected '"
@@ -308,7 +311,8 @@ static class PatternResultChecker implements SqlTester.ResultChecker {
    *
    * @param  Result type
    */
-  static class MatcherResultChecker implements SqlTester.ResultChecker {
+  static class MatcherResultChecker
+      implements SqlTester.ResultChecker {
     private final Matcher matcher;
     private final JdbcType jdbcType;
 
diff --git a/testkit/src/main/java/org/apache/calcite/sql/test/SqlOperatorFixture.java b/testkit/src/main/java/org/apache/calcite/sql/test/SqlOperatorFixture.java
index e2a0fb30ec5c..bdcc09fab6c9 100644
--- a/testkit/src/main/java/org/apache/calcite/sql/test/SqlOperatorFixture.java
+++ b/testkit/src/main/java/org/apache/calcite/sql/test/SqlOperatorFixture.java
@@ -44,6 +44,7 @@
 import java.util.function.Consumer;
 import java.util.function.UnaryOperator;
 
+import static org.apache.calcite.linq4j.Nullness.castNonNull;
 import static org.apache.calcite.rel.type.RelDataTypeImpl.NON_NULLABLE_SUFFIX;
 import static org.apache.calcite.sql.test.ResultCheckers.isSingle;
 
@@ -226,11 +227,11 @@ void checkScalar(String expression,
    */
   default void checkScalar(
       String expression,
-      Object result,
+      @Nullable Object result,
       String resultType) {
     checkType(expression, resultType);
     checkScalar(expression, SqlTests.ANY_TYPE_CHECKER,
-        ResultCheckers.createChecker(result));
+        ResultCheckers.createChecker(castNonNull(result)));
   }
 
   /**
@@ -330,7 +331,7 @@ void checkBoolean(
    */
   void checkString(
       String expression,
-      String result,
+      @Nullable String result,
       String resultType);
 
   /**
diff --git a/testkit/src/main/java/org/apache/calcite/sql/test/SqlTests.java b/testkit/src/main/java/org/apache/calcite/sql/test/SqlTests.java
index 021321c27350..62c9b8b859b6 100644
--- a/testkit/src/main/java/org/apache/calcite/sql/test/SqlTests.java
+++ b/testkit/src/main/java/org/apache/calcite/sql/test/SqlTests.java
@@ -46,6 +46,7 @@
 import static org.junit.jupiter.api.Assertions.fail;
 
 import static java.lang.Integer.parseInt;
+import static java.util.Objects.requireNonNull;
 
 /**
  * Utility methods.
@@ -261,7 +262,8 @@ public static void checkEx(@Nullable Throwable ex,
     int actualEndColumn = 99;
 
     if (ex instanceof ExceptionInInitializerError) {
-      ex = ((ExceptionInInitializerError) ex).getException();
+      final ExceptionInInitializerError error = (ExceptionInInitializerError) ex;
+      ex = requireNonNull(error.getException(), "exception in initializer");
     }
 
     // Search for an CalciteContextException somewhere in the stack.
@@ -338,16 +340,22 @@ public static void checkEx(@Nullable Throwable ex,
         java.util.regex.Matcher matcher =
             LINE_COL_TWICE_PATTERN.matcher(actualMessage);
         if (matcher.matches()) {
-          actualLine = parseInt(matcher.group(1));
-          actualColumn = parseInt(matcher.group(2));
-          actualEndLine = parseInt(matcher.group(3));
-          actualEndColumn = parseInt(matcher.group(4));
+          actualLine =
+              parseInt(requireNonNull(matcher.group(1), "group"));
+          actualColumn =
+              parseInt(requireNonNull(matcher.group(2), "group"));
+          actualEndLine =
+              parseInt(requireNonNull(matcher.group(3), "group"));
+          actualEndColumn =
+              parseInt(requireNonNull(matcher.group(4), "group"));
           actualMessage = matcher.group(5);
         } else {
           matcher = LINE_COL_PATTERN.matcher(actualMessage);
           if (matcher.matches()) {
-            actualLine = parseInt(matcher.group(1));
-            actualColumn = parseInt(matcher.group(2));
+            actualLine =
+              parseInt(requireNonNull(matcher.group(1), "group"));
+            actualColumn =
+              parseInt(requireNonNull(matcher.group(2), "group"));
           } else {
             if (expectedMsgPattern != null
                 && actualMessage.matches(expectedMsgPattern)) {
@@ -403,6 +411,11 @@ public static void checkEx(@Nullable Throwable ex,
       actualMessage = Util.toLinux(actualMessage);
     }
 
+    if (expectedMsgPattern == null) {
+      actualException.printStackTrace();
+      throw new AssertionError("Expected query not to throw exception, "
+          + "but it threw; query [" + sap.sql + "]", actualException);
+    }
     if (actualMessage == null
         || !actualMessage.matches(expectedMsgPattern)) {
       actualException.printStackTrace();
diff --git a/testkit/src/main/java/org/apache/calcite/sql/test/package-info.java b/testkit/src/main/java/org/apache/calcite/sql/test/package-info.java
index 3babd19e16b8..2047a71eb111 100644
--- a/testkit/src/main/java/org/apache/calcite/sql/test/package-info.java
+++ b/testkit/src/main/java/org/apache/calcite/sql/test/package-info.java
@@ -18,4 +18,7 @@
 /**
  * Classes for testing SQL.
  */
+@NullMarked
 package org.apache.calcite.sql.test;
+
+import org.jspecify.annotations.NullMarked;
diff --git a/testkit/src/main/java/org/apache/calcite/test/CalciteAssert.java b/testkit/src/main/java/org/apache/calcite/test/CalciteAssert.java
index 6a551a5957d0..2d591e91e4a2 100644
--- a/testkit/src/main/java/org/apache/calcite/test/CalciteAssert.java
+++ b/testkit/src/main/java/org/apache/calcite/test/CalciteAssert.java
@@ -133,6 +133,7 @@
 import java.util.stream.Collectors;
 import javax.sql.DataSource;
 
+import static org.apache.calcite.linq4j.Nullness.castNonNull;
 import static org.apache.calcite.test.Matchers.compose;
 import static org.apache.calcite.test.Matchers.containsStringLinux;
 import static org.apache.calcite.test.Matchers.isLinux;
@@ -197,7 +198,7 @@ private CalciteAssert() {}
           return this;
         }
 
-        @Override public  AssertThat doWithConnection(
+        @Override public  AssertThat doWithConnection(
             Function fn) {
           return this;
         }
@@ -374,7 +375,7 @@ public static Consumer checkUpdateCount(final int expected) {
   static Consumer consistentResult(final boolean ordered) {
     return new Consumer() {
       int executeCount = 0;
-      Collection expected;
+      @Nullable Collection expected;
 
       @Override public void accept(ResultSet resultSet) {
         ++executeCount;
@@ -386,7 +387,8 @@ static Consumer consistentResult(final boolean ordered) {
             expected = result;
           } else {
             @SuppressWarnings("UndefinedEquals")
-            boolean matches = expected.equals(result);
+            boolean matches =
+                requireNonNull(expected, "expected").equals(result);
             if (!matches) {
               // compare strings to get better error message
               assertThat(newlineList(result), equalTo(newlineList(expected)));
@@ -602,7 +604,7 @@ static void assertQuery(
           updateCount = statement.executeUpdate(sql);
         }
         if (exceptionChecker != null) {
-          exceptionChecker.accept(null);
+          exceptionChecker.accept(castNonNull(null));
           return;
         }
       } catch (Exception | Error e) {
@@ -613,10 +615,10 @@ static void assertQuery(
         throw e;
       }
       if (resultChecker != null) {
-        resultChecker.accept(resultSet);
+        resultChecker.accept(requireNonNull(resultSet, "resultSet"));
       }
       if (updateChecker != null) {
-        updateChecker.accept(updateCount);
+        updateChecker.accept(requireNonNull(updateCount, "updateCount"));
       }
       if (resultSet != null) {
         resultSet.close();
@@ -678,7 +680,7 @@ private static void assertPrepare(
           updateCount = statement.executeUpdate(sql);
         }
         if (exceptionChecker != null) {
-          exceptionChecker.accept(null);
+          exceptionChecker.accept(castNonNull(null));
           return;
         }
       } catch (Exception | Error e) {
@@ -689,10 +691,10 @@ private static void assertPrepare(
         throw e;
       }
       if (resultChecker != null) {
-        resultChecker.accept(resultSet);
+        resultChecker.accept(requireNonNull(resultSet, "resultSet"));
       }
       if (updateChecker != null) {
-        updateChecker.accept(updateCount);
+        updateChecker.accept(requireNonNull(updateCount, "updateCount"));
       }
       if (resultSet != null) {
         resultSet.close();
@@ -785,7 +787,7 @@ static ImmutableMultiset toSet(ResultSet resultSet)
 
   /** Calls a non-static method via reflection. Useful for testing methods that
    * don't exist in certain versions of the JDK. */
-  static Object call(Object o, String methodName, Object... args)
+  static @Nullable Object call(Object o, String methodName, Object... args)
       throws InvocationTargetException, IllegalAccessException {
     return method(o, methodName, args).invoke(o, args);
   }
@@ -1062,7 +1064,7 @@ static SchemaPlus addSchema_(SchemaPlus rootSchema, SchemaSpec schema) {
               .build();
         }
 
-        @Override public  C unwrap(Class aClass) {
+        @Override public  @Nullable C unwrap(Class aClass) {
           if (aClass.isAssignableFrom(SqlDialect.class)
               || aClass.isAssignableFrom(DataSource.class)) {
             return salesTable.unwrap(aClass);
@@ -1078,7 +1080,7 @@ static SchemaPlus addSchema_(SchemaPlus rootSchema, SchemaSpec schema) {
               .build();
         }
 
-        @Override public  C unwrap(Class aClass) {
+        @Override public  @Nullable C unwrap(Class aClass) {
           if (aClass.isAssignableFrom(SqlDialect.class)
               || aClass.isAssignableFrom(DataSource.class)) {
             return salesTable.unwrap(aClass);
@@ -1146,11 +1148,11 @@ private static SchemaPlus addSchemaIfNotExists(SchemaPlus rootSchema,
    * @param actual actual value
    */
   public static void assertArrayEqual(
-      String message, Object[] expected, Object[] actual) {
+      String message, Object @Nullable [] expected, Object @Nullable [] actual) {
     assertThat(message, str(actual), is(str(expected)));
   }
 
-  private static String str(Object[] objects) {
+  private static @Nullable String str(Object @Nullable [] objects) {
     return objects == null
           ? null
           : Arrays.stream(objects).map(Object::toString)
@@ -1297,10 +1299,10 @@ public final AssertThat withMaterializations(String model, final boolean existin
         final String... materializations) {
       return withMaterializations(model, builder -> {
         assert materializations.length % 2 == 0;
-        final List list = builder.list();
+        final List<@Nullable Object> list = builder.list();
         for (int i = 0; i < materializations.length; i++) {
           String table = materializations[i++];
-          final Map map = builder.map();
+          final Map map = builder.map();
           map.put("table", table);
           if (!existing) {
             map.put("view", table + "v");
@@ -1400,14 +1402,14 @@ public AssertThat connectThrows(Consumer exceptionChecker) {
       } catch (Throwable e) {
         throwable = e;
       }
-      exceptionChecker.accept(throwable);
+      exceptionChecker.accept(castNonNull(throwable));
       return this;
     }
 
     /** Creates a {@link org.apache.calcite.jdbc.CalciteConnection}
      * and executes a callback. */
-    public  AssertThat doWithConnection(Function fn)
-        throws Exception {
+    public  AssertThat doWithConnection(
+        Function fn) throws Exception {
       try (Connection connection = connectionFactory.createConnection()) {
         T t = fn.apply((CalciteConnection) connection);
         Util.discard(t);
@@ -1522,7 +1524,8 @@ public AssertQuery returns2(final String expected) {
       return returns(
           checkResult(expected,
               new ResultSetFormatter() {
-                @Override protected String adjustValue(String s) {
+                @Override protected @Nullable String adjustValue(
+                    @Nullable String s) {
                   if (s != null) {
                     if (s.contains(".")) {
                       while (s.endsWith("0")) {
@@ -2151,7 +2154,7 @@ ResultSetFormatter rowToString(ResultSet resultSet,
       return this;
     }
 
-    protected String adjustValue(String string) {
+    protected @Nullable String adjustValue(@Nullable String string) {
       if (string != null) {
         string = TestUtil.correctRoundedFloat(string);
       }
diff --git a/testkit/src/main/java/org/apache/calcite/test/ConnectionFactories.java b/testkit/src/main/java/org/apache/calcite/test/ConnectionFactories.java
index 2a09b9812600..d1d2016387c7 100644
--- a/testkit/src/main/java/org/apache/calcite/test/ConnectionFactories.java
+++ b/testkit/src/main/java/org/apache/calcite/test/ConnectionFactories.java
@@ -31,6 +31,8 @@
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 
+import org.jspecify.annotations.Nullable;
+
 import java.sql.Connection;
 import java.sql.DriverManager;
 import java.sql.SQLException;
@@ -102,9 +104,10 @@ private static class MapConnectionFactory implements ConnectionFactory {
       this.postProcessors = requireNonNull(postProcessors, "postProcessors");
     }
 
-    @Override public boolean equals(Object obj) {
+    @Override public boolean equals(@Nullable Object obj) {
       return this == obj
-          || obj.getClass() == MapConnectionFactory.class
+          || obj != null
+          && obj.getClass() == MapConnectionFactory.class
           && ((MapConnectionFactory) obj).map.equals(map)
           && ((MapConnectionFactory) obj).postProcessors.equals(postProcessors);
     }
diff --git a/testkit/src/main/java/org/apache/calcite/test/DiffRepository.java b/testkit/src/main/java/org/apache/calcite/test/DiffRepository.java
index 7ca26a1df08a..f0bf802254f5 100644
--- a/testkit/src/main/java/org/apache/calcite/test/DiffRepository.java
+++ b/testkit/src/main/java/org/apache/calcite/test/DiffRepository.java
@@ -17,7 +17,6 @@
 package org.apache.calcite.test;
 
 import org.apache.calcite.avatica.util.Spaces;
-import org.apache.calcite.linq4j.Nullness;
 import org.apache.calcite.util.Pair;
 import org.apache.calcite.util.Sources;
 import org.apache.calcite.util.Util;
@@ -298,7 +297,7 @@ private static URL findFile(Class clazz, final String suffix) {
   public static DiffRepository castNonNull(
       @Nullable DiffRepository diffRepos) {
     if (diffRepos != null) {
-      return Nullness.castNonNull(diffRepos);
+      return diffRepos;
     }
     throw new IllegalArgumentException("diffRepos is null; if you require a "
         + "DiffRepository, set it in your test's fixture() method");
@@ -395,9 +394,9 @@ private static String getText(Element element) {
     // all other child elements.
     final NodeList childNodes = element.getChildNodes();
     for (int i = 0; i < childNodes.getLength(); i++) {
-      Node node = childNodes.item(i);
+      Node node = requireNonNull(childNodes.item(i), "child");
       if (node instanceof CDATASection) {
-        return node.getNodeValue();
+        return requireNonNull(node.getNodeValue(), "CDATA text");
       }
     }
 
@@ -428,7 +427,7 @@ private static String getText(Element element) {
       @Nullable List> elements) {
     final NodeList childNodes = root.getChildNodes();
     for (int i = 0; i < childNodes.getLength(); i++) {
-      Node child = childNodes.item(i);
+      Node child = requireNonNull(childNodes.item(i), "child");
       if (child.getNodeName().equals(TEST_CASE_TAG)) {
         Element testCase = (Element) child;
         final String name = testCase.getAttribute(TEST_CASE_NAME_ATTR);
@@ -502,7 +501,7 @@ private static String getCurrentTestCaseName() {
   }
 
   public void assertEquals(String tag, String expected, String actual) {
-    final String testCaseName = getCurrentTestCaseName(true);
+    final String testCaseName = getCurrentTestCaseName();
     String expected2 = expand(tag, expected);
     if (expected2 == null) {
       update(testCaseName, expected, actual);
@@ -619,7 +618,8 @@ private synchronized void flushDoc() {
       return;
     }
     try {
-      boolean b = logFile.getParentFile().mkdirs();
+      boolean b =
+          requireNonNull(logFile.getParentFile(), "log file directory").mkdirs();
       Util.discard(b);
       try (Writer w = Util.printWriter(logFile)) {
         write(doc, w, indent);
@@ -645,7 +645,7 @@ private static SortedMap analyze(Element root) {
     final SortedMap testCases = new TreeMap<>();
     final NodeList childNodes = root.getChildNodes();
     for (int i = 0; i < childNodes.getLength(); i++) {
-      Node child = childNodes.item(i);
+      Node child = requireNonNull(childNodes.item(i), "child");
       if (child.getNodeName().equals(TEST_CASE_TAG)) {
         Element testCase = (Element) child;
         final String name = testCase.getAttribute(TEST_CASE_NAME_ATTR);
@@ -671,7 +671,7 @@ private static Boolean checkExists(Element root, Set javaTestMethods,
     }
     if (!existsOnlyInXml.isEmpty()) {
       for (String value : existsOnlyInXml) {
-        root.removeChild(testCases.get(value));
+        root.removeChild(requireNonNull(testCases.get(value), value));
       }
     }
 
@@ -689,7 +689,7 @@ private static ImmutableSortedSet validateOrder(Element root,
     final NodeList childNodes = root.getChildNodes();
 
     for (int i = 0; i < childNodes.getLength(); i++) {
-      Node child = childNodes.item(i);
+      Node child = requireNonNull(childNodes.item(i), "child");
       if (child.getNodeName().equals(TEST_CASE_TAG)) {
         Element testCase = (Element) child;
         String name = testCase.getAttribute(TEST_CASE_NAME_ATTR);
@@ -742,7 +742,7 @@ private static ImmutableSortedSet validateOrder(Element root,
     Element found = null;
     final List kills = new ArrayList<>();
     for (int i = 0; i < childNodes.getLength(); i++) {
-      Node child = childNodes.item(i);
+      Node child = requireNonNull(childNodes.item(i), "child");
       if (child.getNodeName().equals(RESOURCE_TAG)
           && resourceName.equals(
               ((Element) child).getAttribute(RESOURCE_NAME_ATTR))) {
@@ -762,7 +762,7 @@ private static ImmutableSortedSet validateOrder(Element root,
   private static void removeAllChildren(Element element) {
     final NodeList childNodes = element.getChildNodes();
     while (childNodes.getLength() > 0) {
-      element.removeChild(childNodes.item(0));
+      element.removeChild(requireNonNull(childNodes.item(0), "child"));
     }
   }
 
@@ -814,7 +814,7 @@ private static void writeNode(Node node, XmlOutput out) {
       out.print("\n");
       childNodes = node.getChildNodes();
       for (int i = 0; i < childNodes.getLength(); i++) {
-        Node child = childNodes.item(i);
+        Node child = requireNonNull(childNodes.item(i), "child");
         writeNode(child, out);
       }
       break;
@@ -825,19 +825,20 @@ private static void writeNode(Node node, XmlOutput out) {
       out.beginBeginTag(tagName);
 
       // Attributes.
-      final NamedNodeMap attributeMap = element.getAttributes();
+      final NamedNodeMap attributeMap =
+          requireNonNull(element.getAttributes(), "attributes");
       for (int i = 0; i < attributeMap.getLength(); i++) {
-        final Node att = attributeMap.item(i);
+        final Node att = requireNonNull(attributeMap.item(i), "attribute");
         out.attribute(
             att.getNodeName(),
-            att.getNodeValue());
+            requireNonNull(att.getNodeValue(), "attribute value"));
       }
       out.endBeginTag(tagName);
 
       // Write child nodes, ignoring attributes but including text.
       childNodes = node.getChildNodes();
       for (int i = 0; i < childNodes.getLength(); i++) {
-        Node child = childNodes.item(i);
+        Node child = requireNonNull(childNodes.item(i), "child");
         if (child.getNodeType() == Node.ATTRIBUTE_NODE) {
           continue;
         }
@@ -849,7 +850,7 @@ private static void writeNode(Node node, XmlOutput out) {
     case Node.ATTRIBUTE_NODE:
       out.attribute(
           node.getNodeName(),
-          node.getNodeValue());
+          requireNonNull(node.getNodeValue(), "attribute value"));
       break;
 
     case Node.CDATA_SECTION_NODE:
@@ -861,7 +862,7 @@ private static void writeNode(Node node, XmlOutput out) {
 
     case Node.TEXT_NODE:
       Text text = (Text) node;
-      final String wholeText = text.getNodeValue();
+      final String wholeText = requireNonNull(text.getNodeValue(), "text");
       if (!isWhitespace(wholeText)) {
         out.cdata(wholeText, false);
       }
@@ -979,7 +980,7 @@ private static class Key {
       return Objects.hash(clazz, baseRepository, filter);
     }
 
-    @Override public boolean equals(Object obj) {
+    @Override public boolean equals(@Nullable Object obj) {
       return this == obj
           || obj instanceof Key
           && clazz.equals(((Key) obj).clazz)
@@ -1006,7 +1007,7 @@ DiffRepository toRepo() {
   private static Iterable iterate(NodeList nodeList) {
     return new AbstractList() {
       @Override public Node get(int index) {
-        return nodeList.item(index);
+        return requireNonNull(nodeList.item(index), "node");
       }
 
       @Override public int size() {
diff --git a/testkit/src/main/java/org/apache/calcite/test/Matchers.java b/testkit/src/main/java/org/apache/calcite/test/Matchers.java
index 0ae23f31f1f0..cd1602fa7918 100644
--- a/testkit/src/main/java/org/apache/calcite/test/Matchers.java
+++ b/testkit/src/main/java/org/apache/calcite/test/Matchers.java
@@ -55,6 +55,8 @@
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.collect.ImmutableList.toImmutableList;
 
+import static org.apache.calcite.linq4j.Nullness.castNonNull;
+
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.Matchers.closeTo;
@@ -208,7 +210,8 @@ public static  Matcher compose(Matcher matcher,
    * @see Util#toLinux(String)
    */
   public static Matcher isLinux(final String value) {
-    return compose(Is.is(value), input -> input == null ? null : Util.toLinux(input));
+    return compose(Is.is(value),
+        input -> input == null ? castNonNull(null) : Util.toLinux(input));
   }
 
   /** Matcher that matches a {@link RelNode} if the {@code RelNode} is valid
diff --git a/testkit/src/main/java/org/apache/calcite/test/MockRelOptPlanner.java b/testkit/src/main/java/org/apache/calcite/test/MockRelOptPlanner.java
index 40549361543c..6dcf9bb8fbc1 100644
--- a/testkit/src/main/java/org/apache/calcite/test/MockRelOptPlanner.java
+++ b/testkit/src/main/java/org/apache/calcite/test/MockRelOptPlanner.java
@@ -39,6 +39,8 @@
 import java.util.List;
 import java.util.Map;
 
+import static java.util.Objects.requireNonNull;
+
 /**
  * MockRelOptPlanner is a mock implementation of the {@link RelOptPlanner}
  * interface.
@@ -46,10 +48,14 @@
 public class MockRelOptPlanner extends AbstractRelOptPlanner {
   //~ Instance fields --------------------------------------------------------
 
+  /** Set by {@link #setRoot}, which every caller does first. */
+  @SuppressWarnings("NullAway.Init")
   private RelNode root;
 
   private @Nullable RelOptRule rule;
 
+  /** Set while a rule fires. */
+  @SuppressWarnings("NullAway.Init")
   private RelNode transformationResult;
 
   private long metadataTimestamp = 0L;
@@ -118,7 +124,7 @@ private boolean matchRecursive(
       int ordinalInParent) {
     List bindings = new ArrayList();
     if (match(
-        rule.getOperand(),
+        requireNonNull(rule, "rule").getOperand(),
         rel,
         bindings)) {
       MockRuleCall call =
@@ -187,7 +193,7 @@ private static boolean match(RelOptRuleOperand operand, RelNode rel,
     return rel;
   }
 
-  @Override public RelNode ensureRegistered(RelNode rel, RelNode equivRel) {
+  @Override public RelNode ensureRegistered(RelNode rel, @Nullable RelNode equivRel) {
     return rel;
   }
 
diff --git a/testkit/src/main/java/org/apache/calcite/test/MockSqlOperatorTable.java b/testkit/src/main/java/org/apache/calcite/test/MockSqlOperatorTable.java
index 7f73e74b9821..0028f03d7610 100644
--- a/testkit/src/main/java/org/apache/calcite/test/MockSqlOperatorTable.java
+++ b/testkit/src/main/java/org/apache/calcite/test/MockSqlOperatorTable.java
@@ -216,6 +216,9 @@ public DedupFunction() {
   /** "TFRT" user-defined table function. */
   public static class TableFunctionReturnTableFunction extends SqlFunction
       implements SqlTableFunction {
+    /** Set by the constructor after the super call, which needs the operator
+     * to exist first. */
+    @SuppressWarnings("NullAway.Init")
     TableFunctionReturnTypeInference inference;
 
     public TableFunctionReturnTableFunction() {
@@ -283,7 +286,7 @@ private static RelDataType inferRowType(SqlOperatorBinding opBinding) {
       return ScoreTableFunction::inferRowType;
     }
 
-    @Override public TableCharacteristic tableCharacteristic(int ordinal) {
+    @Override public @Nullable TableCharacteristic tableCharacteristic(int ordinal) {
       return tableParams.get(ordinal);
     }
 
@@ -356,7 +359,7 @@ private static RelDataType inferRowType(SqlOperatorBinding opBinding) {
       return TopNTableFunction::inferRowType;
     }
 
-    @Override public TableCharacteristic tableCharacteristic(int ordinal) {
+    @Override public @Nullable TableCharacteristic tableCharacteristic(int ordinal) {
       return tableParams.get(ordinal);
     }
 
@@ -435,7 +438,7 @@ public SimilarlityTableFunction() {
           .build();
     }
 
-    @Override public TableCharacteristic tableCharacteristic(int ordinal) {
+    @Override public @Nullable TableCharacteristic tableCharacteristic(int ordinal) {
       return tableParams.get(ordinal);
     }
 
@@ -505,7 +508,7 @@ public InvalidTableFunction() {
           .build();
     }
 
-    @Override public TableCharacteristic tableCharacteristic(int ordinal) {
+    @Override public @Nullable TableCharacteristic tableCharacteristic(int ordinal) {
       return tableParams.get(ordinal);
     }
 
diff --git a/testkit/src/main/java/org/apache/calcite/test/QuidemTest.java b/testkit/src/main/java/org/apache/calcite/test/QuidemTest.java
index ac21cf102490..d92d81794c9e 100644
--- a/testkit/src/main/java/org/apache/calcite/test/QuidemTest.java
+++ b/testkit/src/main/java/org/apache/calcite/test/QuidemTest.java
@@ -85,6 +85,7 @@
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
+import static org.apache.calcite.linq4j.Nullness.castNonNull;
 import static org.apache.calcite.runtime.SqlFunctions.resetThreadSequences;
 import static org.apache.calcite.sql2rel.SqlToRelConverter.DEFAULT_IN_SUB_QUERY_THRESHOLD;
 
@@ -162,10 +163,11 @@ protected boolean useTopDownGeneralDecorrelator() {
     case "use_new_decorr":
       return useTopDownGeneralDecorrelator();
     case "jdk18":
-      return System.getProperty("java.version").startsWith("1.8");
+      return requireNonNull(System.getProperty("java.version"), "java.version")
+          .startsWith("1.8");
     case "fixed":
       // Quidem requires a Java 8 function
-      return (Function) v -> {
+      return (Function) v -> {
         switch (v) {
         case "calcite1045":
           return Bug.CALCITE_1045_FIXED;
@@ -175,7 +177,7 @@ protected boolean useTopDownGeneralDecorrelator() {
         return null;
       };
     case "not":
-      return (Function) v -> {
+      return (Function) v -> {
         final Object o = getEnv(v);
         if (o instanceof Function) {
           @SuppressWarnings("unchecked") final Function f =
@@ -208,7 +210,8 @@ protected static Collection data(String first) {
     final URL inUrl = QuidemTest.class.getResource("/" + n2u(first));
     final File firstFile = Sources.of(requireNonNull(inUrl, "inUrl")).file();
     final int commonPrefixLength = firstFile.getAbsolutePath().length() - first.length();
-    final File dir = firstFile.getParentFile();
+    final File dir =
+        requireNonNull(firstFile.getParentFile(), "parent of " + firstFile);
     final List paths = new ArrayList<>();
     final FilenameFilter filter = new PatternFilenameFilter(".*\\.iq$");
     for (File f : Util.first(dir.listFiles(filter), new File[0])) {
@@ -257,7 +260,8 @@ protected void checkRun(String path) throws Exception {
       outFile = replaceDir(inFile, "resources", "quidem/"
           + getClass().getSimpleName());
     }
-    Util.discard(outFile.getParentFile().mkdirs());
+    Util.discard(
+        requireNonNull(outFile.getParentFile(), "parent of " + outFile).mkdirs());
     try (Reader reader = Util.reader(inFile);
          Writer writer = Util.printWriter(outFile);
          Closer closer = new Closer()) {
@@ -317,7 +321,7 @@ protected void checkRun(String path) throws Exception {
               if (value.equals("original")) {
                 closer.add(
                     Hook.PROGRAM.addThread((Consumer>)
-                        holder -> holder.set(null)));
+                        holder -> holder.set(castNonNull(null))));
               } else {
                 closer.add(
                     Hook.PROGRAM.addThread((Consumer>)
@@ -384,15 +388,18 @@ private static void parseRules(String value, List rulesAdd,
     Matcher matcher = pattern.matcher(value);
 
     while (matcher.find()) {
-      char operation = matcher.group(1).charAt(0);
+      char operation =
+          requireNonNull(matcher.group(1), "operation").charAt(0);
       String ruleSource = matcher.group(3);
-      String ruleName = matcher.group(4);
+      String ruleName = requireNonNull(matcher.group(4), "ruleName");
 
       try {
         if (ruleSource == null || ruleSource.equals("CoreRules")) {
           setRules(operation, getCoreRule(ruleName), rulesAdd, rulesRemove);
         } else if (ruleSource.equals("EnumerableRules")) {
-          Object rule = EnumerableRules.class.getField(ruleName).get(null);
+          Object rule =
+              requireNonNull(EnumerableRules.class.getField(ruleName).get(null),
+                  ruleName);
           setRules(operation, (RelOptRule) rule, rulesAdd, rulesRemove);
         } else {
           throw new RuntimeException("Unknown rule: " + ruleName);
@@ -413,9 +420,10 @@ private static void applyRulesInOrder(String value,
     Matcher matcher = pattern.matcher(value);
 
     while (matcher.find()) {
-      char operation = matcher.group(1).charAt(0);
+      char operation =
+          requireNonNull(matcher.group(1), "operation").charAt(0);
       String ruleSource = matcher.group(3);
-      String ruleName = matcher.group(4);
+      String ruleName = requireNonNull(matcher.group(4), "ruleName");
 
       try {
         RelOptRule rule;
@@ -423,7 +431,9 @@ private static void applyRulesInOrder(String value,
         if (ruleSource == null || ruleSource.equals("CoreRules")) {
           rule = getCoreRule(ruleName);
         } else if (ruleSource.equals("EnumerableRules")) {
-          Object ruleObj = EnumerableRules.class.getField(ruleName).get(null);
+          Object ruleObj =
+              requireNonNull(EnumerableRules.class.getField(ruleName).get(null),
+                  ruleName);
           rule = (RelOptRule) ruleObj;
           targetVolcano = true;
         } else {
@@ -550,12 +560,12 @@ public void test(String path) throws Exception {
   /** Quidem connection factory for Calcite's built-in test schemas. */
   protected class QuidemConnectionFactory
       implements Quidem.ConnectionFactory {
-    public Connection connect(String name) throws Exception {
+    public @Nullable Connection connect(String name) throws Exception {
       return connect(name, false);
     }
 
-    @Override public Connection connect(String name, boolean reference)
-        throws Exception {
+    @Override public @Nullable Connection connect(String name,
+        boolean reference) throws Exception {
       if (reference) {
         if (name.equals("foodmart")) {
           final ConnectionSpec db =
@@ -654,8 +664,9 @@ public Connection connect(String name) throws Exception {
         final Connection connection = customize(CalciteAssert.that()
             .withSchema("s", new AbstractSchema()))
             .connect();
-        connection.unwrap(CalciteConnection.class).getRootSchema()
-            .subSchemas().get("s")
+        requireNonNull(
+            connection.unwrap(CalciteConnection.class).getRootSchema()
+                .subSchemas().get("s"), "schema s")
             .add("my_seq",
                 new AbstractTable() {
                   @Override public RelDataType getRowType(
diff --git a/testkit/src/main/java/org/apache/calcite/test/RelMetadataFixture.java b/testkit/src/main/java/org/apache/calcite/test/RelMetadataFixture.java
index 4214587ddc58..cf264898889e 100644
--- a/testkit/src/main/java/org/apache/calcite/test/RelMetadataFixture.java
+++ b/testkit/src/main/java/org/apache/calcite/test/RelMetadataFixture.java
@@ -47,6 +47,7 @@
 import com.google.common.collect.Multimap;
 
 import org.hamcrest.Matcher;
+import org.jspecify.annotations.Nullable;
 
 import java.util.Collection;
 import java.util.HashMap;
@@ -69,6 +70,8 @@
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
+import static java.util.Objects.requireNonNull;
+
 /**
  * Parameters for a Metadata test.
  */
@@ -228,7 +231,7 @@ public RelNode toRel() {
   public RelMetadataFixture assertCpuCost(Matcher matcher,
       String reason) {
     RelNode rel = toRel();
-    RelOptCost cost = computeRelSelfCost(rel);
+    RelOptCost cost = requireNonNull(computeRelSelfCost(rel), "cost");
     assertThat(reason + "\n"
             + "sql:" + relSupplier + "\n"
             + "plan:" + RelOptUtil.toString(rel, SqlExplainLevel.ALL_ATTRIBUTES),
@@ -236,7 +239,7 @@ public RelMetadataFixture assertCpuCost(Matcher matcher,
     return this;
   }
 
-  private static RelOptCost computeRelSelfCost(RelNode rel) {
+  private static @Nullable RelOptCost computeRelSelfCost(RelNode rel) {
     final RelMetadataQuery mq = rel.getCluster().getMetadataQuery();
     RelOptPlanner planner = new VolcanoPlanner();
     return rel.computeSelfCost(planner, mq);
@@ -279,7 +282,8 @@ private RelMetadataFixture checkColumnOrigin(
       Consumer> action) {
     RelNode rel = toRel();
     final RelMetadataQuery mq = rel.getCluster().getMetadataQuery();
-    final Set columnOrigins = mq.getColumnOrigins(rel, 0);
+    final Set columnOrigins =
+        requireNonNull(mq.getColumnOrigins(rel, 0), "columnOrigins");
     action.accept(columnOrigins);
     return this;
   }
@@ -365,7 +369,8 @@ public RelMetadataFixture assertThatUniqueKeysAre(boolean ignoreNulls,
       ImmutableBitSet... expectedUniqueKeys) {
     RelNode rel = toRel();
     final RelMetadataQuery mq = rel.getCluster().getMetadataQuery();
-    Set result = mq.getUniqueKeys(rel, ignoreNulls);
+    Set result =
+        requireNonNull(mq.getUniqueKeys(rel, ignoreNulls), "uniqueKeys");
     assertThat(result, notNullValue());
     assertThat("unique keys, sql: " + relSupplier
             + ", rel: " + RelOptUtil.toString(rel),
@@ -382,7 +387,8 @@ public RelMetadataFixture assertThatUniqueKeysAre(boolean ignoreNulls,
    */
   private static void checkUniqueConsistent(RelNode rel, boolean ignoreNulls) {
     final RelMetadataQuery mq = rel.getCluster().getMetadataQuery();
-    final Set uniqueKeys = mq.getUniqueKeys(rel, ignoreNulls);
+    final Set uniqueKeys =
+        requireNonNull(mq.getUniqueKeys(rel, ignoreNulls), "uniqueKeys");
     assertThat(uniqueKeys, notNullValue());
     for (ImmutableBitSet key : uniqueKeys) {
       Boolean result2 = mq.areColumnsUnique(rel, key, ignoreNulls);
@@ -489,7 +495,8 @@ public RelMetadataFixture assertThatNodeTypeCount(
       Matcher, Integer>> matcher) {
     final RelNode rel = toRel();
     final RelMetadataQuery mq = rel.getCluster().getMetadataQuery();
-    final Multimap, RelNode> result = mq.getNodeTypes(rel);
+    final Multimap, RelNode> result =
+        requireNonNull(mq.getNodeTypes(rel), "nodeTypes");
     assertThat(result, notNullValue());
     final Map, Integer> resultCount = new HashMap<>();
     for (Map.Entry, Collection> e : result.asMap().entrySet()) {
diff --git a/testkit/src/main/java/org/apache/calcite/test/RelOptFixture.java b/testkit/src/main/java/org/apache/calcite/test/RelOptFixture.java
index d65d3f2fb42f..4afc67f94d8a 100644
--- a/testkit/src/main/java/org/apache/calcite/test/RelOptFixture.java
+++ b/testkit/src/main/java/org/apache/calcite/test/RelOptFixture.java
@@ -97,7 +97,7 @@ public class RelOptFixture {
   final SqlTestFactory factory;
   final @Nullable DiffRepository diffRepos;
   final @Nullable HepProgram preProgram;
-  final RelOptPlanner planner;
+  final @Nullable RelOptPlanner planner;
   final ImmutableMap> hooks;
   final BiFunction before;
   final BiFunction after;
@@ -107,7 +107,7 @@ public class RelOptFixture {
 
   RelOptFixture(SqlTester tester, SqlTestFactory factory,
       @Nullable DiffRepository diffRepos, RelSupplier relSupplier,
-      @Nullable HepProgram preProgram, RelOptPlanner planner,
+      @Nullable HepProgram preProgram, @Nullable RelOptPlanner planner,
       ImmutableMap> hooks,
       BiFunction before,
       BiFunction after,
@@ -391,6 +391,7 @@ private void checkPlanning(boolean unchanged) {
     } else {
       r2 = relBefore;
     }
+    final RelOptPlanner planner = requireNonNull(this.planner, "planner");
     planner.setRoot(r2);
     final RelNode r3 = planner.findBestExp();
 
diff --git a/testkit/src/main/java/org/apache/calcite/test/RelSupplier.java b/testkit/src/main/java/org/apache/calcite/test/RelSupplier.java
index ffcc7bd45f4d..b89a6dc7adeb 100644
--- a/testkit/src/main/java/org/apache/calcite/test/RelSupplier.java
+++ b/testkit/src/main/java/org/apache/calcite/test/RelSupplier.java
@@ -26,6 +26,8 @@
 import org.apache.calcite.tools.Programs;
 import org.apache.calcite.tools.RelBuilder;
 
+import org.jspecify.annotations.Nullable;
+
 import java.util.List;
 import java.util.function.Function;
 
@@ -84,7 +86,7 @@ private SqlRelSupplier(String sql) {
       return sql;
     }
 
-    @Override public boolean equals(Object o) {
+    @Override public boolean equals(@Nullable Object o) {
       return o == this
           || o instanceof SqlRelSupplier
           && ((SqlRelSupplier) o).sql.equals(this.sql);
@@ -123,7 +125,7 @@ private FnRelSupplier(Function relFn) {
       return relFn.hashCode();
     }
 
-    @Override public boolean equals(Object o) {
+    @Override public boolean equals(@Nullable Object o) {
       return o == this
           || o instanceof FnRelSupplier
           && ((FnRelSupplier) o).relFn == relFn;
diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorFixtureImpl.java b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorFixtureImpl.java
index b491d6f7290f..5bb6694b5ec4 100644
--- a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorFixtureImpl.java
+++ b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorFixtureImpl.java
@@ -276,7 +276,7 @@ private void checkAgg(String expr, String[] inputValues,
 
   @Override public void checkString(
       String expression,
-      String result,
+      @Nullable String result,
       String expectedType) {
     SqlTester.TypeChecker typeChecker =
         new SqlTests.StringTypeChecker(expectedType);
diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java
index fe5c32b7adaa..ca4ba61a9e1a 100644
--- a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java
+++ b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java
@@ -16721,13 +16721,13 @@ void testTimestampDiff(boolean coercionEnabled) {
     final SqlOperatorFixture f = fixture();
     QUANTIFY_OPERATORS.forEach(operator -> f.setFor(operator, SqlOperatorFixture.VmName.EXPAND));
 
-    Function2 checkBoolean = (sql, result) -> {
+    Function2 checkBoolean = (sql, result) -> {
       f.checkBoolean(sql.replace("COLLECTION", "ARRAY"), result);
       f.checkBoolean(sql.replace("COLLECTION", "MULTISET"), result);
       return null;
     };
 
-    Function1 checkNull = sql -> {
+    Function1 checkNull = sql -> {
       f.checkNull(sql.replace("COLLECTION", "ARRAY"));
       f.checkNull(sql.replace("COLLECTION", "MULTISET"));
       return null;
@@ -18543,11 +18543,11 @@ private List getValues(BasicSqlType type, boolean inBound) {
    */
   private static class ValueOrExceptionResultChecker
       implements SqlTester.ResultChecker {
-    private final Object expected;
+    private final @Nullable Object expected;
     private final Pattern[] patterns;
 
     ValueOrExceptionResultChecker(
-        Object expected, Pattern... patterns) {
+        @Nullable Object expected, Pattern... patterns) {
       this.expected = expected;
       this.patterns = patterns;
     }
@@ -18624,16 +18624,16 @@ public TesterImpl() {
   /** A type, a value, and its {@link SqlNode} representation. */
   static class ValueType {
     final RelDataType type;
-    final Object value;
+    final @Nullable Object value;
     final SqlNode node;
 
-    ValueType(RelDataType type, Object value) {
+    ValueType(RelDataType type, @Nullable Object value) {
       this.type = type;
       this.value = value;
       this.node = literal(type, value);
     }
 
-    private SqlNode literal(RelDataType type, Object value) {
+    private SqlNode literal(RelDataType type, @Nullable Object value) {
       if (value == null) {
         return SqlStdOperatorTable.CAST.createCall(
             SqlParserPos.ZERO,
diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlToRelTestBase.java b/testkit/src/main/java/org/apache/calcite/test/SqlToRelTestBase.java
index 05d5b4efdd8e..41e5c99bf5c9 100644
--- a/testkit/src/main/java/org/apache/calcite/test/SqlToRelTestBase.java
+++ b/testkit/src/main/java/org/apache/calcite/test/SqlToRelTestBase.java
@@ -29,6 +29,8 @@
 
 import java.util.List;
 
+import static java.util.Objects.requireNonNull;
+
 /**
  * SqlToRelTestBase is an abstract base for tests which involve conversion from
  * SQL to relational algebra.
@@ -42,7 +44,8 @@
 public abstract class SqlToRelTestBase {
   //~ Static fields/initializers ---------------------------------------------
 
-  protected static final String NL = System.getProperty("line.separator");
+  protected static final String NL =
+      requireNonNull(System.getProperty("line.separator"), "line.separator");
 
   //~ Instance fields --------------------------------------------------------
 
diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlValidatorFixture.java b/testkit/src/main/java/org/apache/calcite/test/SqlValidatorFixture.java
index c96c98f725f8..6222e37346c1 100644
--- a/testkit/src/main/java/org/apache/calcite/test/SqlValidatorFixture.java
+++ b/testkit/src/main/java/org/apache/calcite/test/SqlValidatorFixture.java
@@ -48,6 +48,7 @@
 import org.apache.calcite.util.Util;
 
 import org.hamcrest.Matcher;
+import org.jspecify.annotations.Nullable;
 
 import java.nio.charset.Charset;
 import java.util.List;
@@ -288,7 +289,7 @@ public SqlValidatorFixture assertMonotonicity(
         (sap, validator, n) -> {
           final RelDataType rowType = validator.getValidatedNodeType(n);
           final SqlValidatorNamespace selectNamespace =
-              validator.getNamespace(n);
+              requireNonNull(validator.getNamespace(n), "namespace");
           final String field0 = rowType.getFieldList().get(0).getName();
           final SqlMonotonicity monotonicity =
               selectNamespace.getMonotonicity(field0);
@@ -332,6 +333,7 @@ public void assertCollation(Matcher collationMatcher,
               RelDataType actualType = fields.get(0).getType();
               SqlCollation collation = actualType.getCollation();
               assertThat(collation, notNullValue());
+              requireNonNull(collation, "collation");
               assertThat(collation.getCollationName(), collationMatcher);
               assertThat(collation.getCoercibility(), coercibilityMatcher);
             }));
@@ -435,10 +437,10 @@ public SqlValidatorFixture isAggregate(Matcher matcher) {
    */
   public SqlValidatorFixture assertFieldOrigin(Matcher matcher) {
     tester.validateAndThen(factory, toSql(false), (sap, validator, n) -> {
-      final List> list = validator.getFieldOrigins(n);
+      final List<@Nullable List> list = validator.getFieldOrigins(n);
       final StringBuilder buf = new StringBuilder("{");
       int i = 0;
-      for (List strings : list) {
+      for (@Nullable List strings : list) {
         if (i++ > 0) {
           buf.append(", ");
         }
diff --git a/testkit/src/main/java/org/apache/calcite/test/catalog/MockCatalogReader.java b/testkit/src/main/java/org/apache/calcite/test/catalog/MockCatalogReader.java
index e0e84ae923ee..9e7266c79ede 100644
--- a/testkit/src/main/java/org/apache/calcite/test/catalog/MockCatalogReader.java
+++ b/testkit/src/main/java/org/apache/calcite/test/catalog/MockCatalogReader.java
@@ -107,6 +107,8 @@
 import java.util.Map;
 import java.util.Set;
 
+import static org.apache.calcite.linq4j.Nullness.castNonNull;
+
 import static java.util.Objects.requireNonNull;
 
 /**
@@ -319,7 +321,11 @@ public static class MockTable extends Prepare.AbstractPreparingTable
     protected final List keyList = new ArrayList<>();
     protected final List referentialConstraints =
         new ArrayList<>();
+    /** Set by {@code onRegister}, before anything asks for the row type. */
+    @SuppressWarnings("NullAway.Init")
     protected RelDataType rowType;
+    /** Set by {@code onRegister}, before anything asks for the collations. */
+    @SuppressWarnings("NullAway.Init")
     protected List collationList;
     protected final List names;
     protected final Double maxRowCount;
@@ -445,7 +451,7 @@ protected ModifiableTable(String tableName) {
         throw new UnsupportedOperationException();
       }
 
-      @Override public  C unwrap(Class aClass) {
+      @Override public  @Nullable C unwrap(Class aClass) {
         if (aClass.isInstance(initializerFactory)) {
           return aClass.cast(initializerFactory);
         } else if (aClass.isInstance(MockTable.this)) {
@@ -550,7 +556,7 @@ public static MockTable create(MockCatalogReader catalogReader,
       return table;
     }
 
-    @Override public  T unwrap(Class clazz) {
+    @Override public  @Nullable T unwrap(Class clazz) {
       if (clazz.isInstance(this)) {
         return clazz.cast(this);
       }
@@ -762,8 +768,9 @@ public static MockModifiableViewRelOptTable create(MockModifiableViewTable modif
               : NullInitializerExpressionFactory.INSTANCE;
       return new MockModifiableViewRelOptTable(modifiableViewTable,
           catalogReader, catalogName, schemaName, name, stream, rowCount,
-          resolver, Util.first(initializerExpressionFactory,
-          NullInitializerExpressionFactory.INSTANCE));
+          castNonNull(resolver),
+          Util.firstNonNull(initializerExpressionFactory,
+              NullInitializerExpressionFactory.INSTANCE));
     }
 
     public static MockViewTableMacro viewMacro(CalciteSchema schema, String viewSql,
@@ -781,7 +788,7 @@ public static MockViewTableMacro viewMacro(CalciteSchema schema, String viewSql,
           monotonicColumnSet, kind, resolver, initializerFactory);
     }
 
-    @Override public  T unwrap(Class clazz) {
+    @Override public  @Nullable T unwrap(Class clazz) {
       if (clazz.isInstance(modifiableViewTable)) {
         return clazz.cast(modifiableViewTable);
       }
@@ -799,13 +806,17 @@ public static class MockViewTableMacro extends ViewTableMacro {
 
       @Override protected ModifiableViewTable modifiableViewTable(
           CalcitePrepare.AnalyzeViewResult parsed, String viewSql,
-          List schemaPath, List viewPath, CalciteSchema schema) {
+          List schemaPath, @Nullable List viewPath,
+          CalciteSchema schema) {
         final JavaTypeFactory typeFactory = (JavaTypeFactory) parsed.typeFactory;
         final Type elementType = typeFactory.getJavaClass(parsed.rowType);
         return new MockModifiableViewTable(elementType,
             RelDataTypeImpl.proto(parsed.rowType), viewSql, schemaPath, viewPath,
-            parsed.table, Schemas.path(schema.root(), parsed.tablePath),
-            parsed.constraint, parsed.columnMapping);
+            requireNonNull(parsed.table, "table"),
+            Schemas.path(schema.root(),
+                requireNonNull(parsed.tablePath, "tablePath")),
+            requireNonNull(parsed.constraint, "constraint"),
+            requireNonNull(parsed.columnMapping, "columnMapping"));
       }
     }
 
@@ -816,7 +827,7 @@ public static class MockModifiableViewTable extends ModifiableViewTable {
       private final RexNode constraint;
 
       MockModifiableViewTable(Type elementType, RelProtoDataType rowType,
-          String viewSql, List schemaPath, List viewPath,
+          String viewSql, List schemaPath, @Nullable List viewPath,
           Table table, Path tablePath, RexNode constraint,
           ImmutableIntList columnMapping) {
         super(elementType, rowType, viewSql, schemaPath, viewPath, table,
@@ -862,8 +873,9 @@ public static MockRelViewTable create(ViewTable viewTable,
               : NullInitializerExpressionFactory.INSTANCE;
       return new MockRelViewTable(viewTable,
           catalogReader, catalogName, schemaName, name, stream, rowCount,
-          resolver, Util.first(initializerExpressionFactory,
-          NullInitializerExpressionFactory.INSTANCE));
+          castNonNull(resolver),
+          Util.firstNonNull(initializerExpressionFactory,
+              NullInitializerExpressionFactory.INSTANCE));
     }
 
     @Override public RelDataType getRowType() {
@@ -874,7 +886,7 @@ public static MockRelViewTable create(ViewTable viewTable,
       return viewTable.toRel(context, this);
     }
 
-    @Override public  T unwrap(Class clazz) {
+    @Override public  @Nullable T unwrap(Class clazz) {
       if (clazz.isInstance(viewTable)) {
         return clazz.cast(viewTable);
       }
@@ -899,7 +911,7 @@ public abstract static class MockViewTable extends MockTable {
       super(catalogReader, catalogName, schemaName, name, stream, false,
           rowCount, resolver, initializerFactory);
       this.fromTable = fromTable;
-      this.table = fromTable.unwrap(Table.class);
+      this.table = fromTable.unwrapOrThrow(Table.class);
       this.mapping = mapping;
     }
 
@@ -907,7 +919,7 @@ public abstract static class MockViewTable extends MockTable {
     private class ModifiableView extends AbstractModifiableView
         implements Wrapper {
       @Override public Table getTable() {
-        return fromTable.unwrap(Table.class);
+        return fromTable.unwrapOrThrow(Table.class);
       }
 
       @Override public Path getTablePath() {
@@ -941,7 +953,7 @@ private class ModifiableView extends AbstractModifiableView
             });
       }
 
-      @Override public  C unwrap(Class aClass) {
+      @Override public  @Nullable C unwrap(Class aClass) {
         if (table instanceof Wrapper) {
           final C c = ((Wrapper) table).unwrap(aClass);
           if (c != null) {
@@ -968,7 +980,7 @@ private class ModifiableViewWithCustomColumnResolving
         return resolver.resolveColumn(rowType, typeFactory, names);
       }
 
-      @Override public  C unwrap(Class aClass) {
+      @Override public  @Nullable C unwrap(Class aClass) {
         if (table instanceof Wrapper) {
           final C c = ((Wrapper) table).unwrap(aClass);
           if (c != null) {
@@ -1007,7 +1019,7 @@ protected abstract RexNode getConstraint(RexBuilder rexBuilder,
           ImmutableSet.of());
     }
 
-    @Override public  T unwrap(Class clazz) {
+    @Override public  @Nullable T unwrap(Class clazz) {
       if (clazz.isAssignableFrom(ModifiableView.class)) {
         ModifiableView view = resolver == null
             ? new ModifiableView()
@@ -1106,7 +1118,7 @@ private static class WrapperTable implements Table, Wrapper {
       this.table = table;
     }
 
-    @Override public  C unwrap(Class aClass) {
+    @Override public  @Nullable C unwrap(Class aClass) {
       return aClass.isInstance(this) ? aClass.cast(this)
           : aClass.isInstance(table) ? aClass.cast(table)
           : null;
diff --git a/testkit/src/main/java/org/apache/calcite/test/catalog/MockCatalogReaderSimple.java b/testkit/src/main/java/org/apache/calcite/test/catalog/MockCatalogReaderSimple.java
index e464c3855c5c..1ccdf5be5c25 100644
--- a/testkit/src/main/java/org/apache/calcite/test/catalog/MockCatalogReaderSimple.java
+++ b/testkit/src/main/java/org/apache/calcite/test/catalog/MockCatalogReaderSimple.java
@@ -36,6 +36,7 @@
 import com.google.common.collect.ImmutableList;
 
 import org.jspecify.annotations.NonNull;
+import org.jspecify.annotations.Nullable;
 
 import java.math.BigDecimal;
 import java.util.Arrays;
@@ -69,7 +70,7 @@ protected MockCatalogReaderSimple(RelDataTypeFactory typeFactory,
     return new MockCatalogReaderSimple(typeFactory, caseSensitive).init();
   }
 
-  @Override public RelDataType getNamedType(SqlIdentifier typeName) {
+  @Override public @Nullable RelDataType getNamedType(SqlIdentifier typeName) {
     if (typeName.equalsDeep(addressType.getSqlIdentifier(), Litmus.IGNORE)) {
       return addressType;
     } else {
diff --git a/testkit/src/main/java/org/apache/calcite/test/catalog/package-info.java b/testkit/src/main/java/org/apache/calcite/test/catalog/package-info.java
index d8131acceb2f..63cc4b0fb9e7 100644
--- a/testkit/src/main/java/org/apache/calcite/test/catalog/package-info.java
+++ b/testkit/src/main/java/org/apache/calcite/test/catalog/package-info.java
@@ -18,4 +18,7 @@
 /**
  * Classes for testing Catalog.
  */
+@NullMarked
 package org.apache.calcite.test.catalog;
+
+import org.jspecify.annotations.NullMarked;
diff --git a/testkit/src/main/java/org/apache/calcite/test/package-info.java b/testkit/src/main/java/org/apache/calcite/test/package-info.java
index 2c6b382c358b..0485523a9767 100644
--- a/testkit/src/main/java/org/apache/calcite/test/package-info.java
+++ b/testkit/src/main/java/org/apache/calcite/test/package-info.java
@@ -18,4 +18,7 @@
 /**
  * Classes for testing Calcite.
  */
+@NullMarked
 package org.apache.calcite.test;
+
+import org.jspecify.annotations.NullMarked;
diff --git a/testkit/src/main/java/org/apache/calcite/test/schemata/bookstore/package-info.java b/testkit/src/main/java/org/apache/calcite/test/schemata/bookstore/package-info.java
new file mode 100644
index 000000000000..b16f422752a8
--- /dev/null
+++ b/testkit/src/main/java/org/apache/calcite/test/schemata/bookstore/package-info.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Bookstore schema, used by tests that query nested collections.
+ */
+@NullMarked
+package org.apache.calcite.test.schemata.bookstore;
+
+import org.jspecify.annotations.NullMarked;
diff --git a/testkit/src/main/java/org/apache/calcite/test/schemata/catchall/CatchallSchema.java b/testkit/src/main/java/org/apache/calcite/test/schemata/catchall/CatchallSchema.java
index 2aad0b3c4b8c..a8e98d194816 100644
--- a/testkit/src/main/java/org/apache/calcite/test/schemata/catchall/CatchallSchema.java
+++ b/testkit/src/main/java/org/apache/calcite/test/schemata/catchall/CatchallSchema.java
@@ -35,6 +35,8 @@
 import java.util.Date;
 import java.util.List;
 
+import static java.util.Objects.requireNonNull;
+
 /**
  * Object whose fields are relations. Called "catch-all" because it's OK
  * if tests add new fields.
@@ -97,9 +99,9 @@ public class CatchallSchema {
   private static boolean isNumeric(Class type) {
     switch (Primitive.flavor(type)) {
     case BOX:
-      return Primitive.ofBox(type).isNumeric();
+      return requireNonNull(Primitive.ofBox(type), "boxed primitive").isNumeric();
     case PRIMITIVE:
-      return Primitive.of(type).isNumeric();
+      return requireNonNull(Primitive.of(type), "primitive").isNumeric();
     default:
       return Number.class.isAssignableFrom(type); // e.g. BigDecimal
     }
diff --git a/testkit/src/main/java/org/apache/calcite/test/schemata/catchall/package-info.java b/testkit/src/main/java/org/apache/calcite/test/schemata/catchall/package-info.java
new file mode 100644
index 000000000000..5902505804cb
--- /dev/null
+++ b/testkit/src/main/java/org/apache/calcite/test/schemata/catchall/package-info.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Schema of Java objects that covers every type the reflective schema maps.
+ */
+@NullMarked
+package org.apache.calcite.test.schemata.catchall;
+
+import org.jspecify.annotations.NullMarked;
diff --git a/testkit/src/main/java/org/apache/calcite/test/schemata/countries/package-info.java b/testkit/src/main/java/org/apache/calcite/test/schemata/countries/package-info.java
new file mode 100644
index 000000000000..d7447d9a46be
--- /dev/null
+++ b/testkit/src/main/java/org/apache/calcite/test/schemata/countries/package-info.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Table functions over a fixed list of countries and states.
+ */
+@NullMarked
+package org.apache.calcite.test.schemata.countries;
+
+import org.jspecify.annotations.NullMarked;
diff --git a/testkit/src/main/java/org/apache/calcite/test/schemata/foodmart/FoodmartSchema.java b/testkit/src/main/java/org/apache/calcite/test/schemata/foodmart/FoodmartSchema.java
index 4ff4ef972a7f..138b81380dea 100644
--- a/testkit/src/main/java/org/apache/calcite/test/schemata/foodmart/FoodmartSchema.java
+++ b/testkit/src/main/java/org/apache/calcite/test/schemata/foodmart/FoodmartSchema.java
@@ -65,7 +65,7 @@ public SalesFact(int cust_id, int prod_id) {
       this.prod_id = prod_id;
     }
 
-    @Override public boolean equals(Object obj) {
+    @Override public boolean equals(@Nullable Object obj) {
       return obj == this
           || obj instanceof SalesFact
           && cust_id == ((SalesFact) obj).cust_id
diff --git a/testkit/src/main/java/org/apache/calcite/test/schemata/foodmart/package-info.java b/testkit/src/main/java/org/apache/calcite/test/schemata/foodmart/package-info.java
new file mode 100644
index 000000000000..ee9f3c195336
--- /dev/null
+++ b/testkit/src/main/java/org/apache/calcite/test/schemata/foodmart/package-info.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Foodmart schema, a JDBC schema that many tests query.
+ */
+@NullMarked
+package org.apache.calcite.test.schemata.foodmart;
+
+import org.jspecify.annotations.NullMarked;
diff --git a/testkit/src/main/java/org/apache/calcite/test/schemata/hr/Department.java b/testkit/src/main/java/org/apache/calcite/test/schemata/hr/Department.java
index f1a7cf47d2ef..44e11fddce4d 100644
--- a/testkit/src/main/java/org/apache/calcite/test/schemata/hr/Department.java
+++ b/testkit/src/main/java/org/apache/calcite/test/schemata/hr/Department.java
@@ -45,7 +45,7 @@ public Department(int deptno, String name, List employees,
         + ", employees: " + employees + ", location: " + location + "]";
   }
 
-  @Override public boolean equals(Object obj) {
+  @Override public boolean equals(@Nullable Object obj) {
     return obj == this
         || obj instanceof Department
         && deptno == ((Department) obj).deptno;
diff --git a/testkit/src/main/java/org/apache/calcite/test/schemata/hr/Dependent.java b/testkit/src/main/java/org/apache/calcite/test/schemata/hr/Dependent.java
index d8a04fdb786e..421e0d93a948 100644
--- a/testkit/src/main/java/org/apache/calcite/test/schemata/hr/Dependent.java
+++ b/testkit/src/main/java/org/apache/calcite/test/schemata/hr/Dependent.java
@@ -16,6 +16,8 @@
  */
 package org.apache.calcite.test.schemata.hr;
 
+import org.jspecify.annotations.Nullable;
+
 import java.util.Objects;
 
 /**
@@ -34,7 +36,7 @@ public Dependent(int empid, String name) {
     return "Dependent [empid: " + empid + ", name: " + name + "]";
   }
 
-  @Override public boolean equals(Object obj) {
+  @Override public boolean equals(@Nullable Object obj) {
     return obj == this
         || obj instanceof Dependent
         && empid == ((Dependent) obj).empid
diff --git a/testkit/src/main/java/org/apache/calcite/test/schemata/hr/Employee.java b/testkit/src/main/java/org/apache/calcite/test/schemata/hr/Employee.java
index c3e045df4fc9..0d49383f5e28 100644
--- a/testkit/src/main/java/org/apache/calcite/test/schemata/hr/Employee.java
+++ b/testkit/src/main/java/org/apache/calcite/test/schemata/hr/Employee.java
@@ -44,7 +44,7 @@ public Employee(int empid, int deptno, String name, float salary,
         + ", name: " + name + "]";
   }
 
-  @Override public boolean equals(Object obj) {
+  @Override public boolean equals(@Nullable Object obj) {
     return obj == this
         || obj instanceof Employee
         && empid == ((Employee) obj).empid;
diff --git a/testkit/src/main/java/org/apache/calcite/test/schemata/hr/Event.java b/testkit/src/main/java/org/apache/calcite/test/schemata/hr/Event.java
index 269125a83a7f..498ad1b08bba 100644
--- a/testkit/src/main/java/org/apache/calcite/test/schemata/hr/Event.java
+++ b/testkit/src/main/java/org/apache/calcite/test/schemata/hr/Event.java
@@ -37,7 +37,7 @@ public Event(int eventid, @Nullable Timestamp ts) {
     return "Event [eventid: " + eventid + ", ts: " + ts + "]";
   }
 
-  @Override public boolean equals(Object obj) {
+  @Override public boolean equals(@Nullable Object obj) {
     return obj == this
         || obj instanceof Event
         && eventid == ((Event) obj).eventid;
diff --git a/testkit/src/main/java/org/apache/calcite/test/schemata/hr/HierarchySchema.java b/testkit/src/main/java/org/apache/calcite/test/schemata/hr/HierarchySchema.java
index f6c3eeb66174..dbfd53eec6af 100644
--- a/testkit/src/main/java/org/apache/calcite/test/schemata/hr/HierarchySchema.java
+++ b/testkit/src/main/java/org/apache/calcite/test/schemata/hr/HierarchySchema.java
@@ -16,6 +16,8 @@
  */
 package org.apache.calcite.test.schemata.hr;
 
+import org.jspecify.annotations.Nullable;
+
 import java.util.Arrays;
 import java.util.Objects;
 
@@ -73,7 +75,7 @@ public Hierarchy(int managerid, int subordinateid) {
       return "Hierarchy [managerid: " + managerid + ", subordinateid: " + subordinateid + "]";
     }
 
-    @Override public boolean equals(Object obj) {
+    @Override public boolean equals(@Nullable Object obj) {
       return obj == this
           || obj instanceof Hierarchy
           && managerid == ((Hierarchy) obj).managerid
diff --git a/testkit/src/main/java/org/apache/calcite/test/schemata/hr/Location.java b/testkit/src/main/java/org/apache/calcite/test/schemata/hr/Location.java
index 769c61914e9f..d42f657f663c 100644
--- a/testkit/src/main/java/org/apache/calcite/test/schemata/hr/Location.java
+++ b/testkit/src/main/java/org/apache/calcite/test/schemata/hr/Location.java
@@ -16,6 +16,8 @@
  */
 package org.apache.calcite.test.schemata.hr;
 
+import org.jspecify.annotations.Nullable;
+
 import java.util.Objects;
 
 /**
@@ -34,7 +36,7 @@ public Location(int x, int y) {
     return "Location [x: " + x + ", y: " + y + "]";
   }
 
-  @Override public boolean equals(Object obj) {
+  @Override public boolean equals(@Nullable Object obj) {
     return obj == this
         || obj instanceof Location
         && x == ((Location) obj).x
diff --git a/testkit/src/main/java/org/apache/calcite/test/schemata/hr/NullableTest.java b/testkit/src/main/java/org/apache/calcite/test/schemata/hr/NullableTest.java
index d6b87b97f63e..93830513043e 100644
--- a/testkit/src/main/java/org/apache/calcite/test/schemata/hr/NullableTest.java
+++ b/testkit/src/main/java/org/apache/calcite/test/schemata/hr/NullableTest.java
@@ -39,7 +39,7 @@ public NullableTest(@Nullable Integer col1, @Nullable Integer col2,
     return "DependentNullable [col1: " + col1 + ", col2: " + col2 + ", col3: " + col3 + "]";
   }
 
-  @Override public boolean equals(Object obj) {
+  @Override public boolean equals(@Nullable Object obj) {
     return obj == this
         || obj instanceof NullableTest
         && Objects.equals(col1, ((NullableTest) obj).col1)
diff --git a/testkit/src/main/java/org/apache/calcite/test/schemata/hr/package-info.java b/testkit/src/main/java/org/apache/calcite/test/schemata/hr/package-info.java
new file mode 100644
index 000000000000..fff89959ca7d
--- /dev/null
+++ b/testkit/src/main/java/org/apache/calcite/test/schemata/hr/package-info.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * HR schema, the small employee-and-department schema most tests query.
+ */
+@NullMarked
+package org.apache.calcite.test.schemata.hr;
+
+import org.jspecify.annotations.NullMarked;
diff --git a/testkit/src/main/java/org/apache/calcite/test/schemata/lingual/LingualEmp.java b/testkit/src/main/java/org/apache/calcite/test/schemata/lingual/LingualEmp.java
index 32509b30c7fb..458661700acf 100644
--- a/testkit/src/main/java/org/apache/calcite/test/schemata/lingual/LingualEmp.java
+++ b/testkit/src/main/java/org/apache/calcite/test/schemata/lingual/LingualEmp.java
@@ -16,6 +16,8 @@
  */
 package org.apache.calcite.test.schemata.lingual;
 
+import org.jspecify.annotations.Nullable;
+
 import java.util.Objects;
 
 /**
@@ -30,7 +32,7 @@ public LingualEmp(int EMPNO, int DEPTNO) {
     this.DEPTNO = DEPTNO;
   }
 
-  @Override public boolean equals(Object obj) {
+  @Override public boolean equals(@Nullable Object obj) {
     return obj == this
         || obj instanceof LingualEmp
         && EMPNO == ((LingualEmp) obj).EMPNO;
diff --git a/testkit/src/main/java/org/apache/calcite/test/schemata/lingual/package-info.java b/testkit/src/main/java/org/apache/calcite/test/schemata/lingual/package-info.java
new file mode 100644
index 000000000000..f3c5887d5458
--- /dev/null
+++ b/testkit/src/main/java/org/apache/calcite/test/schemata/lingual/package-info.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Lingual schema, a variant of the HR schema with nullable columns.
+ */
+@NullMarked
+package org.apache.calcite.test.schemata.lingual;
+
+import org.jspecify.annotations.NullMarked;
diff --git a/testkit/src/main/java/org/apache/calcite/test/schemata/orderstream/InfiniteOrdersTable.java b/testkit/src/main/java/org/apache/calcite/test/schemata/orderstream/InfiniteOrdersTable.java
index 0c01004a2d48..c053ca6beaf1 100644
--- a/testkit/src/main/java/org/apache/calcite/test/schemata/orderstream/InfiniteOrdersTable.java
+++ b/testkit/src/main/java/org/apache/calcite/test/schemata/orderstream/InfiniteOrdersTable.java
@@ -32,24 +32,28 @@
 public class InfiniteOrdersTable extends BaseOrderStreamTable
     implements StreamableTable {
   @Override public Enumerable<@Nullable Object[]> scan(DataContext root) {
-    return Linq4j.asEnumerable(() -> new Iterator() {
-      private final String[] items = {"paint", "paper", "brush"};
-      private int counter = 0;
-
-      @Override public boolean hasNext() {
-        return true;
-      }
-
-      @Override public Object[] next() {
-        final int index = counter++;
-        return new Object[]{
-            System.currentTimeMillis(), index, items[index % items.length], 10};
-      }
-
-      @Override public void remove() {
-        throw new UnsupportedOperationException();
-      }
-    });
+    return Linq4j.asEnumerable(InfiniteOrdersIterator::new);
+  }
+
+  /** Iterator that returns an unbounded stream of orders. */
+  private static class InfiniteOrdersIterator
+      implements Iterator<@Nullable Object[]> {
+    private final String[] items = {"paint", "paper", "brush"};
+    private int counter = 0;
+
+    @Override public boolean hasNext() {
+      return true;
+    }
+
+    @Override public @Nullable Object[] next() {
+      final int index = counter++;
+      return new Object[]{
+          System.currentTimeMillis(), index, items[index % items.length], 10};
+    }
+
+    @Override public void remove() {
+      throw new UnsupportedOperationException();
+    }
   }
 
   @Override public Table stream() {
diff --git a/testkit/src/main/java/org/apache/calcite/test/schemata/orderstream/OrdersHistoryTable.java b/testkit/src/main/java/org/apache/calcite/test/schemata/orderstream/OrdersHistoryTable.java
index dca1a92a5825..81fab52183b4 100644
--- a/testkit/src/main/java/org/apache/calcite/test/schemata/orderstream/OrdersHistoryTable.java
+++ b/testkit/src/main/java/org/apache/calcite/test/schemata/orderstream/OrdersHistoryTable.java
@@ -26,13 +26,13 @@
 
 /** Table representing the history of the ORDERS stream. */
 public class OrdersHistoryTable extends BaseOrderStreamTable {
-  private final ImmutableList rows;
+  private final ImmutableList<@Nullable Object[]> rows;
 
-  public OrdersHistoryTable(ImmutableList rows) {
+  public OrdersHistoryTable(ImmutableList<@Nullable Object[]> rows) {
     this.rows = rows;
   }
 
   @Override public Enumerable<@Nullable Object[]> scan(DataContext root) {
-    return Linq4j.asEnumerable(rows);
+    return Linq4j.<@Nullable Object[]>asEnumerable(rows);
   }
 }
diff --git a/testkit/src/main/java/org/apache/calcite/test/schemata/orderstream/OrdersStreamTableFactory.java b/testkit/src/main/java/org/apache/calcite/test/schemata/orderstream/OrdersStreamTableFactory.java
index e305ef0a88d8..ca9991c37422 100644
--- a/testkit/src/main/java/org/apache/calcite/test/schemata/orderstream/OrdersStreamTableFactory.java
+++ b/testkit/src/main/java/org/apache/calcite/test/schemata/orderstream/OrdersStreamTableFactory.java
@@ -40,7 +40,7 @@ public OrdersStreamTableFactory() {
     return new OrdersTable(getRowList());
   }
 
-  public static ImmutableList getRowList() {
+  public static ImmutableList<@Nullable Object[]> getRowList() {
     final Object[][] rows = {
         {ts(10, 15, 0), 1, "paint", 10},
         {ts(10, 24, 15), 2, "paper", 5},
@@ -48,7 +48,11 @@ public static ImmutableList getRowList() {
         {ts(10, 58, 0), 4, "paint", 3},
         {ts(11, 10, 0), 5, "paint", 3}
     };
-    return ImmutableList.copyOf(rows);
+    final ImmutableList.Builder<@Nullable Object[]> list = ImmutableList.builder();
+    for (Object[] row : rows) {
+      list.add(row);
+    }
+    return list.build();
   }
 
   private static Object ts(int h, int m, int s) {
diff --git a/testkit/src/main/java/org/apache/calcite/test/schemata/orderstream/OrdersTable.java b/testkit/src/main/java/org/apache/calcite/test/schemata/orderstream/OrdersTable.java
index 36bafa2b990d..022571929f49 100644
--- a/testkit/src/main/java/org/apache/calcite/test/schemata/orderstream/OrdersTable.java
+++ b/testkit/src/main/java/org/apache/calcite/test/schemata/orderstream/OrdersTable.java
@@ -34,14 +34,14 @@
  */
 public class OrdersTable extends BaseOrderStreamTable
     implements StreamableTable {
-  private final ImmutableList rows;
+  private final ImmutableList<@Nullable Object[]> rows;
 
-  public OrdersTable(ImmutableList rows) {
+  public OrdersTable(ImmutableList<@Nullable Object[]> rows) {
     this.rows = rows;
   }
 
   @Override public Enumerable<@Nullable Object[]> scan(DataContext root) {
-    return Linq4j.asEnumerable(rows);
+    return Linq4j.<@Nullable Object[]>asEnumerable(rows);
   }
 
   @Override public Table stream() {
diff --git a/testkit/src/main/java/org/apache/calcite/test/schemata/orderstream/ProductsTable.java b/testkit/src/main/java/org/apache/calcite/test/schemata/orderstream/ProductsTable.java
index b2ef4d259709..286a5d74b215 100644
--- a/testkit/src/main/java/org/apache/calcite/test/schemata/orderstream/ProductsTable.java
+++ b/testkit/src/main/java/org/apache/calcite/test/schemata/orderstream/ProductsTable.java
@@ -39,9 +39,9 @@
  * Table representing the PRODUCTS relation.
  */
 public class ProductsTable implements ScannableTable {
-  private final ImmutableList rows;
+  private final ImmutableList<@Nullable Object[]> rows;
 
-  public ProductsTable(ImmutableList rows) {
+  public ProductsTable(ImmutableList<@Nullable Object[]> rows) {
     this.rows = rows;
   }
 
@@ -51,7 +51,7 @@ public ProductsTable(ImmutableList rows) {
       .build();
 
   @Override public Enumerable<@Nullable Object[]> scan(DataContext root) {
-    return Linq4j.asEnumerable(rows);
+    return Linq4j.<@Nullable Object[]>asEnumerable(rows);
   }
 
   @Override public RelDataType getRowType(RelDataTypeFactory typeFactory) {
diff --git a/testkit/src/main/java/org/apache/calcite/test/schemata/orderstream/package-info.java b/testkit/src/main/java/org/apache/calcite/test/schemata/orderstream/package-info.java
new file mode 100644
index 000000000000..feae065c4458
--- /dev/null
+++ b/testkit/src/main/java/org/apache/calcite/test/schemata/orderstream/package-info.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Streaming order tables, used by tests that query a stream.
+ */
+@NullMarked
+package org.apache.calcite.test.schemata.orderstream;
+
+import org.jspecify.annotations.NullMarked;
diff --git a/testkit/src/main/java/org/apache/calcite/test/schemata/tpch/package-info.java b/testkit/src/main/java/org/apache/calcite/test/schemata/tpch/package-info.java
new file mode 100644
index 000000000000..42822f8dc86b
--- /dev/null
+++ b/testkit/src/main/java/org/apache/calcite/test/schemata/tpch/package-info.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * TPC-H schema, used by tests that plan the TPC-H queries.
+ */
+@NullMarked
+package org.apache.calcite.test.schemata.tpch;
+
+import org.jspecify.annotations.NullMarked;
diff --git a/testkit/src/main/java/org/apache/calcite/util/Smalls.java b/testkit/src/main/java/org/apache/calcite/util/Smalls.java
index 20e210c7bb22..e4ff258edeae 100644
--- a/testkit/src/main/java/org/apache/calcite/util/Smalls.java
+++ b/testkit/src/main/java/org/apache/calcite/util/Smalls.java
@@ -213,10 +213,10 @@ public static QueryableTable generateStrings(final Integer count) {
 
                   int i = 0;
                   int curI;
-                  String curS;
+                  @Nullable String curS;
 
                   @Override public IntString current() {
-                    return new IntString(curI, curS);
+                    return new IntString(curI, requireNonNull(curS, "curS"));
                   }
 
                   @Override public boolean moveNext() {
@@ -371,25 +371,7 @@ public static ScannableTable dummyTableFuncWithTwoParams(final long param1, fina
       }
 
       @Override public Enumerable<@Nullable Object[]> scan(DataContext root) {
-        return new AbstractEnumerable() {
-          @Override public Enumerator enumerator() {
-            return new Enumerator() {
-              @Override public Object[] current() {
-                return new Object[] {};
-              }
-
-              @Override public boolean moveNext() {
-                return false;
-              }
-
-              @Override public void reset() {
-              }
-
-              @Override public void close() {
-              }
-            };
-          }
-        };
+        return Linq4j.emptyEnumerable();
       }
 
       @Override public Statistic getStatistic() {
@@ -420,36 +402,7 @@ public static ScannableTable fibonacciTableWithLimit(final long limit) {
       }
 
       @Override public Enumerable<@Nullable Object[]> scan(DataContext root) {
-        return new AbstractEnumerable() {
-          @Override public Enumerator enumerator() {
-            return new Enumerator() {
-              private long prev = 1;
-              private long current = 0;
-
-              @Override public Object[] current() {
-                return new Object[] {current};
-              }
-
-              @Override public boolean moveNext() {
-                final long next = current + prev;
-                if (limit >= 0 && next > limit) {
-                  return false;
-                }
-                prev = current;
-                current = next;
-                return true;
-              }
-
-              @Override public void reset() {
-                prev = 0;
-                current = 1;
-              }
-
-              @Override public void close() {
-              }
-            };
-          }
-        };
+        return new FibonacciEnumerable(limit);
       }
 
       @Override public Statistic getStatistic() {
@@ -622,7 +575,7 @@ public MyPlusInitFunction(FunctionContext fx) {
       }
       THREAD_DIGEST.set(b.toString());
       this.initY = fx.isArgumentConstant(1)
-          ? fx.getArgumentValueAs(1, Integer.class)
+          ? requireNonNull(fx.getArgumentValueAs(1, Integer.class), "y")
           : 100;
     }
 
@@ -641,8 +594,8 @@ public static class MyDeterministicPlusFunction {
       INSTANCE_COUNT.get().incrementAndGet();
     }
 
-    public Integer eval(@Parameter(name = "x") Integer x,
-        @Parameter(name = "y") Integer y) {
+    public @Nullable Integer eval(@Parameter(name = "x") @Nullable Integer x,
+        @Parameter(name = "y") @Nullable Integer y) {
       if (x == null || y == null) {
         return null;
       }
@@ -683,7 +636,8 @@ public static String eval(@Parameter(name = "o") Object o) {
   /** Example of a semi-strict UDF.
    * (Returns null if its parameter is null or if its length is 4.) */
   public static class Null4Function {
-    @SemiStrict public static String eval(@Parameter(name = "s") String s) {
+    @SemiStrict public static @Nullable String eval(
+        @Parameter(name = "s") @Nullable String s) {
       if (s == null || s.length() == 4) {
         return null;
       }
@@ -695,7 +649,8 @@ public static class Null4Function {
    * Throws {@link NullPointerException} if argument is null.
    * Returns null if its argument's length is 8. */
   public static class Null8Function {
-    @SemiStrict public static String eval(@Parameter(name = "s") String s) {
+    @SemiStrict public static @Nullable String eval(
+        @Parameter(name = "s") String s) {
       if (s.length() == 8) {
         return null;
       }
@@ -857,29 +812,30 @@ public static java.sql.Date toDateFun(int v) {
       return SqlFunctions.internalToDate(v);
     }
 
-    public static java.sql.Date toDateFun(Long v) {
+    public static java.sql.@Nullable Date toDateFun(@Nullable Long v) {
       return v == null ? null : SqlFunctions.internalToDate(v.intValue());
     }
     public static java.sql.Timestamp toTimestampFun(Long v) {
       return SqlFunctions.internalToTimestamp(v);
     }
-    public static java.sql.Time toTimeFun(Long v) {
+    public static java.sql.@Nullable Time toTimeFun(@Nullable Long v) {
       return v == null ? null : SqlFunctions.internalToTime(v.intValue());
     }
 
     /** For overloaded user-defined functions that have {@code double} and
      * {@code BigDecimal} arguments will go wrong. */
-    public static double toDouble(BigDecimal var) {
+    public static double toDouble(@Nullable BigDecimal var) {
       return var == null ? 0.0d : var.doubleValue();
     }
-    public static double toDouble(Double var) {
+    public static double toDouble(@Nullable Double var) {
       return var == null ? 0.0d : var;
     }
-    public static double toDouble(Float var) {
+    public static double toDouble(@Nullable Float var) {
       return var == null ? 0.0d : Double.valueOf(var.toString());
     }
 
-    public static List arrayAppendFun(List v, Integer i) {
+    public static @Nullable List arrayAppendFun(@Nullable List v,
+        @Nullable Integer i) {
       if (v == null || i == null) {
         return null;
       } else {
@@ -1473,7 +1429,7 @@ public static class SimpleTable extends AbstractQueryableTable
       implements TranslatableTable {
     private final String[] columnNames = { "A", "B" };
     private final Class[] columnTypes = { String.class, Integer.class };
-    private final Object[][] rows = new Object[3][];
+    private final @Nullable Object[][] rows = new @Nullable Object[3][];
 
     public SimpleTable() {
       super(Object[].class);
@@ -1495,11 +1451,11 @@ public SimpleTable() {
       return typeFactory.createStructType(columnDesc);
     }
 
-    public Iterator iterator() {
+    public Iterator<@Nullable Object[]> iterator() {
       return Linq4j.enumeratorIterator(enumerator());
     }
 
-    public Enumerator enumerator() {
+    public Enumerator<@Nullable Object[]> enumerator() {
       return enumeratorImpl(null);
     }
 
@@ -1514,43 +1470,54 @@ public Enumerator enumerator() {
       };
     }
 
-    private Enumerator enumeratorImpl(final int[] fields) {
-      return new Enumerator() {
-        private Object[] current;
-        private final Iterator iterator = Arrays.asList(rows)
-            .iterator();
+    private Enumerator<@Nullable Object[]> enumeratorImpl(
+        final int @Nullable [] fields) {
+      return new SimpleEnumerator(rows, fields);
+    }
 
-        @Override public Object[] current() {
-          return current;
-        }
+    /** Enumerator over the rows of a {@link SimpleTable}. */
+    private static class SimpleEnumerator
+        implements Enumerator<@Nullable Object[]> {
+      private final int @Nullable [] fields;
+      private final Iterator<@Nullable Object[]> iterator;
+      private @Nullable Object @Nullable [] current;
 
-        @Override public boolean moveNext() {
-          if (iterator.hasNext()) {
-            Object[] full = iterator.next();
-            current = fields != null ? convertRow(full) : full;
-            return true;
-          } else {
-            current = null;
-            return false;
-          }
-        }
+      SimpleEnumerator(@Nullable Object[][] rows, int @Nullable [] fields) {
+        this.fields = fields;
+        this.iterator = Arrays.asList(rows).iterator();
+      }
 
-        @Override public void reset() {
-          throw new UnsupportedOperationException();
-        }
+      @Override public @Nullable Object[] current() {
+        return requireNonNull(current, "current");
+      }
 
-        @Override public void close() {
-          // noop
+      @Override public boolean moveNext() {
+        if (iterator.hasNext()) {
+          @Nullable Object[] full = iterator.next();
+          current = fields != null ? convertRow(full, fields) : full;
+          return true;
+        } else {
+          current = null;
+          return false;
         }
+      }
 
-        private Object[] convertRow(Object[] full) {
-          final Object[] objects = new Object[fields.length];
-          for (int i = 0; i < fields.length; i++) {
-            objects[i] = full[fields[i]];
-          }
-          return objects;
+      @Override public void reset() {
+        throw new UnsupportedOperationException();
+      }
+
+      @Override public void close() {
+        // noop
+      }
+
+      private static @Nullable Object[] convertRow(@Nullable Object[] full,
+          int[] fields) {
+        final @Nullable Object[] objects = new Object[fields.length];
+        for (int i = 0; i < fields.length; i++) {
+          objects[i] = full[fields[i]];
         }
-      };
+        return objects;
+      }
     }
 
     @Override public RelNode toRel(
@@ -1573,7 +1540,7 @@ private Object[] convertRow(Object[] full) {
 
   /** User-defined function that decodes a Base64 string to bytes. */
   public static class MyUnbase64Function {
-    public static ByteString eval(String s) {
+    public static @Nullable ByteString eval(@Nullable String s) {
       if (s == null) {
         return null;
       }
@@ -1603,7 +1570,7 @@ public static int eval(byte[] bytes) {
 
   /** User-defined function with return type Character[]. */
   public static class CharacterArrayFunction {
-    public static Character[] eval(String s) {
+    public static Character @Nullable [] eval(@Nullable String s) {
       if (s == null) {
         return null;
       }
@@ -1614,4 +1581,53 @@ public static Character[] eval(String s) {
       return characters;
     }
   }
+
+  /** Enumerable over the Fibonacci numbers up to a limit. */
+  private static class FibonacciEnumerable
+      extends AbstractEnumerable<@Nullable Object[]> {
+    private final long limit;
+
+    FibonacciEnumerable(long limit) {
+      this.limit = limit;
+    }
+
+    @Override public Enumerator<@Nullable Object[]> enumerator() {
+      return new FibonacciEnumerator(limit);
+    }
+  }
+
+  /** Enumerator over the Fibonacci numbers up to a limit. */
+  private static class FibonacciEnumerator
+      implements Enumerator<@Nullable Object[]> {
+    private final long limit;
+    private long prev = 1;
+    private long current = 0;
+
+    FibonacciEnumerator(long limit) {
+      this.limit = limit;
+    }
+
+    @Override public @Nullable Object[] current() {
+      return new Object[] {current};
+    }
+
+    @Override public boolean moveNext() {
+      final long next = current + prev;
+      if (limit >= 0 && next > limit) {
+        return false;
+      }
+      prev = current;
+      current = next;
+      return true;
+    }
+
+    @Override public void reset() {
+      prev = 0;
+      current = 1;
+    }
+
+    @Override public void close() {
+    }
+  }
+
 }
diff --git a/testkit/src/main/java/org/apache/calcite/util/TestUtil.java b/testkit/src/main/java/org/apache/calcite/util/TestUtil.java
index f8e4e60f2c95..929dba353135 100644
--- a/testkit/src/main/java/org/apache/calcite/util/TestUtil.java
+++ b/testkit/src/main/java/org/apache/calcite/util/TestUtil.java
@@ -20,6 +20,7 @@
 import com.google.common.base.Suppliers;
 import com.google.common.collect.ImmutableSortedSet;
 
+import org.jspecify.annotations.Nullable;
 import org.junit.jupiter.api.Assertions;
 
 import java.io.File;
@@ -35,7 +36,7 @@
 
 import static com.google.common.base.Preconditions.checkArgument;
 
-import static org.apache.calcite.util.Util.first;
+import static org.apache.calcite.util.Util.firstNonNull;
 
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.junit.jupiter.api.Assertions.fail;
@@ -59,10 +60,12 @@ public abstract class TestUtil {
       "\\\\n\"" + Util.LINE_SEPARATOR + " + \"";
 
   private static final String JAVA_VERSION =
-      System.getProperties().getProperty("java.version");
+      requireNonNull(System.getProperties().getProperty("java.version"),
+          "java.version");
 
   public static final Version AVATICA_VERSION =
-      Version.of(first(System.getProperty("calcite.avatica.version"), "0"));
+      Version.of(
+          firstNonNull(System.getProperty("calcite.avatica.version"), "0"));
 
   private static final Supplier GUAVA_MAJOR_VERSION =
       Suppliers.memoize(TestUtil::computeGuavaMajorVersion);
@@ -245,11 +248,11 @@ public static String correctRoundedFloat(String s) {
     }
     final Matcher m = TRAILING_ZERO_PATTERN.matcher(s);
     if (m.matches()) {
-      s = s.substring(0, s.length() - m.group(2).length());
+      s = s.substring(0, s.length() - requireNonNull(m.group(2), "group").length());
     }
     final Matcher m2 = TRAILING_NINE_PATTERN.matcher(s);
     if (m2.matches()) {
-      s = s.substring(0, s.length() - m2.group(2).length());
+      s = s.substring(0, s.length() - requireNonNull(m2.group(2), "group").length());
       if (s.length() > 0) {
         final char c = s.charAt(s.length() - 1);
         switch (c) {
@@ -304,7 +307,7 @@ static int majorVersionFromString(String version) {
       throw new IllegalArgumentException("Can't parse (detect) JDK version from " + version);
     }
 
-    return parseInt(matcher.group());
+    return parseInt(requireNonNull(matcher.group(), "version"));
   }
 
   /** Returns the Guava major version. */
@@ -328,7 +331,7 @@ private static int computeGuavaMajorVersion() {
   }
 
   /** Returns the JVM vendor. */
-  public static String getJavaVirtualMachineVendor() {
+  public static @Nullable String getJavaVirtualMachineVendor() {
     return System.getProperty("java.vm.vendor");
   }
 
@@ -343,19 +346,18 @@ public static File getBaseDir(Class klass) {
         Sources.of(requireNonNull(resource, "resource")).file();
 
     File file = classFile.getAbsoluteFile();
-    for (int i = 0; i < 42; i++) {
-      if (isProjectDir(file)) {
-        // Ok, file == BASE/testkit/
-        break;
-      }
-      file = file.getParentFile();
+    for (int i = 0; i < 42 && !isProjectDir(file); i++) {
+      file =
+          requireNonNull(file.getParentFile(),
+              () -> "no project directory above "
+                  + classFile.getAbsolutePath());
     }
     if (!isProjectDir(file)) {
       fail("Could not find pom.xml, build.gradle.kts or gradle.properties. "
           + "Started with " + classFile.getAbsolutePath()
           + ", the current path is " + file.getAbsolutePath());
     }
-    return file.getParentFile();
+    return requireNonNull(file.getParentFile(), "parent of project directory");
   }
 
   private static boolean isProjectDir(File dir) {
diff --git a/testkit/src/main/java/org/apache/calcite/util/package-info.java b/testkit/src/main/java/org/apache/calcite/util/package-info.java
index d1fa6d454499..61c0b0329929 100644
--- a/testkit/src/main/java/org/apache/calcite/util/package-info.java
+++ b/testkit/src/main/java/org/apache/calcite/util/package-info.java
@@ -18,4 +18,7 @@
 /**
  * Classes for testing Calcite.
  */
+@NullMarked
 package org.apache.calcite.util;
+
+import org.jspecify.annotations.NullMarked;

From 8b9df291719955532ed0729b46f0473b103d6a97 Mon Sep 17 00:00:00 2001
From: Vladimir Sitnikov 
Date: Wed, 26 Aug 2026 08:52:48 +0300
Subject: [PATCH 561/562] [CALCITE-7736] Put :ubenchmark under nullness
 verification

The benchmarks keep their sources in the jmh source set rather than a main one,
so the verification follows them there, and the errorprone CI job builds
jmhClasses alongside classes. A JMH state class is filled in by the @Setup
methods and the @Param values, which is what NullAway.Init says; the rest is the
usual: a statistics map that a phase may not have produced, an employee with no
commission, and an edge the graph may not hold.

Two Error Prone warnings that the jmh source set had never been built against
are fixed rather than suppressed: a helper that reads no instance state is
static, and a parse failure during setup is thrown rather than printed.

Co-Authored-By: Claude Opus 5 
---
 .github/workflows/main.yml                    |  2 +-
 build.gradle.kts                              |  6 ++-
 .../enumerable/CodeGenerationBenchmark.java   | 15 ++++++-
 .../adapter/enumerable/package-info.java      |  3 ++
 ...bstractRelNodeGetRelTypeNameBenchmark.java |  8 +++-
 .../DefaultDirectedGraphBenchmark.java        |  8 ++--
 .../benchmarks/LargePlanBenchmark.java        | 40 ++++++++++++-------
 .../calcite/benchmarks/MetadataBenchmark.java |  1 +
 .../calcite/benchmarks/ParserBenchmark.java   |  1 +
 .../ParserInstantiationBenchmark.java         |  1 +
 .../calcite/benchmarks/PreconditionTest.java  |  1 +
 .../RelNodeConversionBenchmark.java           |  5 ++-
 .../calcite/benchmarks/StatementTest.java     | 16 +++++---
 .../benchmarks/StringConstructBenchmark.java  |  2 +
 .../benchmarks/TypeDigestBenchmark.java       |  3 +-
 .../calcite/benchmarks/package-info.java      |  3 ++
 16 files changed, 84 insertions(+), 31 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index de8ef5f4b8db..035b8724b868 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -363,7 +363,7 @@ jobs:
           job-id: errprone
           remote-build-cache-proxy-enabled: false
           # ErrorProne checks for Beta APIs, so use the latest supported Guava version
-          arguments: --scan --no-parallel --no-daemon -Pguava.version=${{ env.GUAVA_MAX }} -PenableErrorprone classes
+          arguments: --scan --no-parallel --no-daemon -Pguava.version=${{ env.GUAVA_MAX }} -PenableErrorprone classes jmhClasses
 
   linux-slow:
     # Run slow tests when the commit is on main or it is requested explicitly by adding an
diff --git a/build.gradle.kts b/build.gradle.kts
index f01ff0934eda..290857ddfda5 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -94,7 +94,7 @@ val werror by props(true) // treat javac warnings as errors
 
 // Projects whose main code NullAway verifies. The other projects are not annotated well enough
 // yet, so NullAway would only produce noise there.
-val nullawayProjects = listOf(":linq4j", ":core", ":server", ":druid", ":file", ":kafka", ":spark", ":babel", ":redis", ":splunk", ":mongodb", ":cassandra", ":pig", ":arrow", ":innodb", ":piglet", ":geode", ":elasticsearch", ":plus", ":example:csv", ":example:function", ":testkit")
+val nullawayProjects = listOf(":linq4j", ":core", ":server", ":druid", ":file", ":kafka", ":spark", ":babel", ":redis", ":splunk", ":mongodb", ":cassandra", ":pig", ":arrow", ":innodb", ":piglet", ":geode", ":elasticsearch", ":plus", ":example:csv", ":example:function", ":testkit", ":ubenchmark")
 
 val hepLargePlanModeTestIncludes = mapOf(
     ":core" to listOf(
@@ -784,7 +784,9 @@ allprojects {
             }
             val nullawayEnabled = project.path in nullawayProjects
             tasks.withType().configureEach {
-                val mainCode = name == "compileJava"
+                // :ubenchmark keeps its sources in the jmh source set, and they are
+                // the module's own code just as much as a main source set is
+                val mainCode = name == "compileJava" || name == "compileJmhJava"
                 // NullAway reports every error it finds, and javac hides all but the first 100
                 options.compilerArgs.addAll(listOf("-Xmaxerrs", "10000"))
                 options.errorprone {
diff --git a/ubenchmark/src/jmh/java/org/apache/calcite/adapter/enumerable/CodeGenerationBenchmark.java b/ubenchmark/src/jmh/java/org/apache/calcite/adapter/enumerable/CodeGenerationBenchmark.java
index 22c745dd8f86..7c3f899e8a91 100644
--- a/ubenchmark/src/jmh/java/org/apache/calcite/adapter/enumerable/CodeGenerationBenchmark.java
+++ b/ubenchmark/src/jmh/java/org/apache/calcite/adapter/enumerable/CodeGenerationBenchmark.java
@@ -89,6 +89,7 @@ public class CodeGenerationBenchmark {
    * exploited by the embedded compiler in order to dynamically build a Java class.
    */
   @State(Scope.Thread)
+  @SuppressWarnings("NullAway.Init") // JMH sets the fields via @Setup and @Param
   public static class QueryState {
     /**
      * The number of distinct queries to be generated.
@@ -111,8 +112,12 @@ public static class QueryState {
     /**
      * The necessary plan information for every generated query.
      */
+    /** Set by the {@code @Setup} method that JMH calls before the trial. */
+    @SuppressWarnings("NullAway.Init")
     PlanInfo[] planInfos;
 
+    /** Set by the {@code @Setup} method that JMH calls before the trial. */
+    @SuppressWarnings("NullAway.Init")
     ICompilerFactory compilerFactory;
 
     private int currentPlan = 0;
@@ -222,7 +227,11 @@ int nextPlan() {
     }
   }
 
-  /** Plan information. */
+  /** Plan information.
+   *
+   * 
      Every field is filled in as the plan is built, which is why they are
+   * not initialized here. */
+  @SuppressWarnings("NullAway.Init")
   private static class PlanInfo {
     ClassDeclaration classExpr;
     EnumerableRel plan;
@@ -234,10 +243,14 @@ private static class PlanInfo {
    * once at the beginning of each iteration.
    */
   @State(Scope.Thread)
+  @SuppressWarnings("NullAway.Init") // JMH sets the fields via @Setup and @Param
   public static class CacheState {
     @Param({"10", "100", "1000"})
     int cacheSize;
 
+    /** Set by the {@code @Setup} method that JMH calls before each
+     * iteration. */
+    @SuppressWarnings("NullAway.Init")
     Cache cache;
 
     @Setup(Level.Iteration)
diff --git a/ubenchmark/src/jmh/java/org/apache/calcite/adapter/enumerable/package-info.java b/ubenchmark/src/jmh/java/org/apache/calcite/adapter/enumerable/package-info.java
index 9bfc9122dee4..4e22e37c8cac 100644
--- a/ubenchmark/src/jmh/java/org/apache/calcite/adapter/enumerable/package-info.java
+++ b/ubenchmark/src/jmh/java/org/apache/calcite/adapter/enumerable/package-info.java
@@ -18,4 +18,7 @@
 /**
  * Benchmarks for Enumerable adapter.
  */
+@NullMarked
 package org.apache.calcite.adapter.enumerable;
+
+import org.jspecify.annotations.NullMarked;
diff --git a/ubenchmark/src/jmh/java/org/apache/calcite/benchmarks/AbstractRelNodeGetRelTypeNameBenchmark.java b/ubenchmark/src/jmh/java/org/apache/calcite/benchmarks/AbstractRelNodeGetRelTypeNameBenchmark.java
index 5ff669c2c265..93f3ae83fbed 100644
--- a/ubenchmark/src/jmh/java/org/apache/calcite/benchmarks/AbstractRelNodeGetRelTypeNameBenchmark.java
+++ b/ubenchmark/src/jmh/java/org/apache/calcite/benchmarks/AbstractRelNodeGetRelTypeNameBenchmark.java
@@ -18,6 +18,7 @@
 
 import org.apache.calcite.rel.AbstractRelNode;
 
+import org.jspecify.annotations.Nullable;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.Fork;
@@ -39,6 +40,8 @@
 import java.util.Random;
 import java.util.concurrent.TimeUnit;
 
+import static java.util.Objects.requireNonNull;
+
 /**
  * A benchmark of alternative implementations for {@link AbstractRelNode#getRelTypeName()}
  * method.
@@ -56,6 +59,7 @@ public class AbstractRelNodeGetRelTypeNameBenchmark {
    * {@link org.apache.calcite.rel.RelNode} interface.
    */
   @State(Scope.Thread)
+  @SuppressWarnings("NullAway.Init") // JMH sets the fields via @Setup and @Param
   public static class ClassNameState {
 
     private final String[] fullNames = new String[]{
@@ -185,7 +189,7 @@ public static class ClassNameState {
     @Param({"11", "31", "63"})
     private long seed;
 
-    private Random r = null;
+    private @Nullable Random r = null;
 
     /**
      * Sets up the random number generator at the beginning of each iteration.
@@ -203,7 +207,7 @@ public void setupRandom() {
      * interface.
      */
     public String nextName() {
-      return fullNames[r.nextInt(fullNames.length)];
+      return fullNames[requireNonNull(r, "r").nextInt(fullNames.length)];
     }
   }
 
diff --git a/ubenchmark/src/jmh/java/org/apache/calcite/benchmarks/DefaultDirectedGraphBenchmark.java b/ubenchmark/src/jmh/java/org/apache/calcite/benchmarks/DefaultDirectedGraphBenchmark.java
index 85b9f494b573..3446ffd1bb50 100644
--- a/ubenchmark/src/jmh/java/org/apache/calcite/benchmarks/DefaultDirectedGraphBenchmark.java
+++ b/ubenchmark/src/jmh/java/org/apache/calcite/benchmarks/DefaultDirectedGraphBenchmark.java
@@ -22,6 +22,7 @@
 
 import com.google.common.collect.Lists;
 
+import org.jspecify.annotations.Nullable;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.Level;
@@ -53,7 +54,7 @@ private Node(int id) {
       this.id = id;
     }
 
-    @Override public boolean equals(Object o) {
+    @Override public boolean equals(@Nullable Object o) {
       return o == this
           || o instanceof Node
           && ((Node) o).id == id;
@@ -68,6 +69,7 @@ private Node(int id) {
    * State object for the benchmarks.
    */
   @State(Scope.Benchmark)
+  @SuppressWarnings("NullAway.Init") // JMH sets the fields via @Setup and @Param
   public static class GraphState {
 
     static final int NUM_LAYERS = 8;
@@ -182,14 +184,14 @@ public boolean addVertexBenchmark(GraphState state) {
   @Benchmark
   @BenchmarkMode(Mode.AverageTime)
   @OutputTimeUnit(TimeUnit.MICROSECONDS)
-  public DefaultEdge addEdgeBenchmark(GraphState state) {
+  public @Nullable DefaultEdge addEdgeBenchmark(GraphState state) {
     return state.graph.addEdge(state.nodes.get(0), state.nodes.get(5));
   }
 
   @Benchmark
   @BenchmarkMode(Mode.AverageTime)
   @OutputTimeUnit(TimeUnit.MICROSECONDS)
-  public DefaultEdge getEdgeBenchmark(GraphState state) {
+  public @Nullable DefaultEdge getEdgeBenchmark(GraphState state) {
     return state.graph.getEdge(state.nodes.get(0), state.nodes.get(1));
   }
 
diff --git a/ubenchmark/src/jmh/java/org/apache/calcite/benchmarks/LargePlanBenchmark.java b/ubenchmark/src/jmh/java/org/apache/calcite/benchmarks/LargePlanBenchmark.java
index 91455da17263..bc17a091d1a3 100644
--- a/ubenchmark/src/jmh/java/org/apache/calcite/benchmarks/LargePlanBenchmark.java
+++ b/ubenchmark/src/jmh/java/org/apache/calcite/benchmarks/LargePlanBenchmark.java
@@ -33,6 +33,7 @@
 
 import com.google.common.collect.ImmutableList;
 
+import org.jspecify.annotations.Nullable;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.Fork;
@@ -55,6 +56,8 @@
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
 
+import static java.util.Objects.requireNonNull;
+
 /**
  * Benchmark that constructs a synthetic query plan consisting of a large plan.
  *
@@ -75,6 +78,7 @@
 @OutputTimeUnit(TimeUnit.MILLISECONDS)
 @State(Scope.Thread)
 @Threads(1)
+@SuppressWarnings("NullAway.Init") // JMH sets the fields via @Setup and @Param
 public class LargePlanBenchmark {
 
   @Param({"100", "1000", "10000", "100000"})
@@ -82,6 +86,7 @@ public class LargePlanBenchmark {
 
   // For large plans, "DEPTH_FIRST", "BOTTOM_UP", and "TOP_DOWN" are slower than ARBITRARY
   @Param({"ARBITRARY"})
+  @SuppressWarnings("NullAway.Init") // set by JMH from the @Param values
   String matchOrder;
 
   // Enable validation mode to verify rule application counts across different orders.
@@ -89,6 +94,7 @@ public class LargePlanBenchmark {
 
   boolean isLargePlanMode = true; // false is very slow in 10000 unions
   boolean isEnableFiredRulesCache = true;
+  @SuppressWarnings("NullAway.Init") // set by the @Setup method
   private static RelBuilder builder;
 
   // All available match orders for validation
@@ -227,22 +233,24 @@ public Map>> testLargeUnionPlan(
       planner.setRoot(root);
 
       // Phase 1: Execute FILTER_REDUCE_EXPRESSIONS
-      Map> beforeFilter =
+      @Nullable Map> beforeFilter =
           collectStats ? snapshotRuleAttempts(planner) : null;
       planner.executeProgram(filterReduce);
       if (collectStats) {
         stats.put("FILTER",
-            subtractRuleAttempts(snapshotRuleAttempts(planner), beforeFilter));
+            subtractRuleAttempts(snapshotRuleAttempts(planner),
+                requireNonNull(beforeFilter, "beforeFilter")));
       }
       planner.clearRules();
 
       // Phase 2: Execute PROJECT_REDUCE_EXPRESSIONS
-      Map> beforeProject =
+      @Nullable Map> beforeProject =
           collectStats ? snapshotRuleAttempts(planner) : null;
       planner.executeProgram(projectReduce);
       if (collectStats) {
         stats.put("PROJECT",
-            subtractRuleAttempts(snapshotRuleAttempts(planner), beforeProject));
+            subtractRuleAttempts(snapshotRuleAttempts(planner),
+                requireNonNull(beforeProject, "beforeProject")));
       }
       planner.clearRules();
 
@@ -299,7 +307,7 @@ public void runValidation() {
 
     boolean allPassed = true;
     Map>>> baselineStats =
-        allStats.get("ARBITRARY");
+        requireNonNull(allStats.get("ARBITRARY"), "ARBITRARY");
 
     for (String order : ALL_MATCH_ORDERS) {
       if (order.equals("ARBITRARY")) {
@@ -308,11 +316,13 @@ public void runValidation() {
 
       System.out.println("Comparing " + order + " against ARBITRARY:");
       Map>>> orderStats =
-          allStats.get(order);
+          requireNonNull(allStats.get(order), order);
 
       for (int size : VALIDATION_SIZES) {
         boolean sizePassed =
-            validateSizeStats(order, size, baselineStats.get(size), orderStats.get(size));
+            validateSizeStats(order, size,
+                requireNonNull(baselineStats.get(size), "baseline"),
+                requireNonNull(orderStats.get(size), "order"));
         if (!sizePassed) {
           allPassed = false;
         }
@@ -344,14 +354,16 @@ private boolean validateSizeStats(String order, int size,
     StringBuilder sb = new StringBuilder();
     sb.append(String.format(Locale.ROOT, "  Size %4d: ", size));
 
-    Map> baselineFilter = baseline.get("FILTER");
-    Map> testFilter = test.get("FILTER");
+    @Nullable Map> baselineFilter =
+        baseline.get("FILTER");
+    @Nullable Map> testFilter = test.get("FILTER");
     if (!comparePhaseStats("FILTER", baselineFilter, testFilter, sb)) {
       passed = false;
     }
 
-    Map> baselineProject = baseline.get("PROJECT");
-    Map> testProject = test.get("PROJECT");
+    @Nullable Map> baselineProject =
+        baseline.get("PROJECT");
+    @Nullable Map> testProject = test.get("PROJECT");
     if (!comparePhaseStats("PROJECT", baselineProject, testProject, sb)) {
       passed = false;
     }
@@ -366,8 +378,8 @@ private boolean validateSizeStats(String order, int size,
   }
 
   private boolean comparePhaseStats(String phase,
-      Map> baseline,
-      Map> test,
+      @Nullable Map> baseline,
+      @Nullable Map> test,
       StringBuilder sb) {
 
     if (baseline == null && test == null) {
@@ -450,7 +462,7 @@ public void runBenchmark() {
 
     for (String order : ALL_MATCH_ORDERS) {
       System.out.println("Testing match order: " + order);
-      int[] sizes = orderSizes.get(order);
+      int[] sizes = requireNonNull(orderSizes.get(order), order);
 
       for (int size : sizes) {
         int nodeCount = 4 * size + 3;
diff --git a/ubenchmark/src/jmh/java/org/apache/calcite/benchmarks/MetadataBenchmark.java b/ubenchmark/src/jmh/java/org/apache/calcite/benchmarks/MetadataBenchmark.java
index dece4c9aff93..2f357c3c1578 100644
--- a/ubenchmark/src/jmh/java/org/apache/calcite/benchmarks/MetadataBenchmark.java
+++ b/ubenchmark/src/jmh/java/org/apache/calcite/benchmarks/MetadataBenchmark.java
@@ -55,6 +55,7 @@
 @Threads(1)
 @OutputTimeUnit(TimeUnit.MILLISECONDS)
 @BenchmarkMode(Mode.AverageTime)
+@SuppressWarnings("NullAway.Init") // JMH sets the fields via @Setup and @Param
 public class MetadataBenchmark {
 
   @Setup
diff --git a/ubenchmark/src/jmh/java/org/apache/calcite/benchmarks/ParserBenchmark.java b/ubenchmark/src/jmh/java/org/apache/calcite/benchmarks/ParserBenchmark.java
index b39905e25dac..77cd8b929fab 100644
--- a/ubenchmark/src/jmh/java/org/apache/calcite/benchmarks/ParserBenchmark.java
+++ b/ubenchmark/src/jmh/java/org/apache/calcite/benchmarks/ParserBenchmark.java
@@ -51,6 +51,7 @@
 @Threads(1)
 @BenchmarkMode(Mode.AverageTime)
 @OutputTimeUnit(TimeUnit.MICROSECONDS)
+@SuppressWarnings("NullAway.Init") // JMH sets the fields via @Setup and @Param
 public class ParserBenchmark {
 
   @Param({ "1000" })
diff --git a/ubenchmark/src/jmh/java/org/apache/calcite/benchmarks/ParserInstantiationBenchmark.java b/ubenchmark/src/jmh/java/org/apache/calcite/benchmarks/ParserInstantiationBenchmark.java
index bb804bbf9f31..264c36ef114e 100644
--- a/ubenchmark/src/jmh/java/org/apache/calcite/benchmarks/ParserInstantiationBenchmark.java
+++ b/ubenchmark/src/jmh/java/org/apache/calcite/benchmarks/ParserInstantiationBenchmark.java
@@ -50,6 +50,7 @@
 @Threads(1)
 @BenchmarkMode(Mode.AverageTime)
 @OutputTimeUnit(TimeUnit.MICROSECONDS)
+@SuppressWarnings("NullAway.Init") // JMH sets the fields via @Setup and @Param
 public class ParserInstantiationBenchmark {
 
   @Param({"0", "100"})
diff --git a/ubenchmark/src/jmh/java/org/apache/calcite/benchmarks/PreconditionTest.java b/ubenchmark/src/jmh/java/org/apache/calcite/benchmarks/PreconditionTest.java
index e4975dd2c8e2..c005590f7f1b 100644
--- a/ubenchmark/src/jmh/java/org/apache/calcite/benchmarks/PreconditionTest.java
+++ b/ubenchmark/src/jmh/java/org/apache/calcite/benchmarks/PreconditionTest.java
@@ -34,6 +34,7 @@
  */
 @BenchmarkMode(Mode.AverageTime)
 @State(Scope.Benchmark)
+@SuppressWarnings("NullAway.Init") // JMH sets the fields via @Setup and @Param
 public class PreconditionTest {
   boolean fire = true;
   String param = "world";
diff --git a/ubenchmark/src/jmh/java/org/apache/calcite/benchmarks/RelNodeConversionBenchmark.java b/ubenchmark/src/jmh/java/org/apache/calcite/benchmarks/RelNodeConversionBenchmark.java
index cbabec73c1fb..17fb5d4db09f 100644
--- a/ubenchmark/src/jmh/java/org/apache/calcite/benchmarks/RelNodeConversionBenchmark.java
+++ b/ubenchmark/src/jmh/java/org/apache/calcite/benchmarks/RelNodeConversionBenchmark.java
@@ -71,6 +71,7 @@
 @OutputTimeUnit(TimeUnit.MILLISECONDS)
 @State(Scope.Benchmark)
 @Threads(1)
+@SuppressWarnings("NullAway.Init") // JMH sets the fields via @Setup and @Param
 public class RelNodeConversionBenchmark {
 
   /**
@@ -134,6 +135,7 @@ public void setup(int length, int columnLength) {
    * A state holding information needed to parse.
    */
   @State(Scope.Thread)
+  @SuppressWarnings("NullAway.Init") // JMH sets the fields via @Setup and @Param
   public static class SqlToRelNodeBenchmarkState extends RelNodeConversionBenchmarkState {
     @Param({"10000"})
     int length;
@@ -165,6 +167,7 @@ public RelNode parse(SqlToRelNodeBenchmarkState state) throws Exception {
    * A state holding information needed to convert To Rel.
    */
   @State(Scope.Thread)
+  @SuppressWarnings("NullAway.Init") // JMH sets the fields via @Setup and @Param
   public static class SqlNodeToRelNodeBenchmarkState extends RelNodeConversionBenchmarkState {
     @Param({"10000"})
     int length;
@@ -179,7 +182,7 @@ public void setUp() {
       try {
         sqlNode = p.validate(p.parse(sql));
       } catch (Exception e) {
-        e.printStackTrace();
+        throw new RuntimeException("while parsing " + sql, e);
       }
     }
 
diff --git a/ubenchmark/src/jmh/java/org/apache/calcite/benchmarks/StatementTest.java b/ubenchmark/src/jmh/java/org/apache/calcite/benchmarks/StatementTest.java
index c93486080eb9..9f5fbc2b11d0 100644
--- a/ubenchmark/src/jmh/java/org/apache/calcite/benchmarks/StatementTest.java
+++ b/ubenchmark/src/jmh/java/org/apache/calcite/benchmarks/StatementTest.java
@@ -20,6 +20,7 @@
 import org.apache.calcite.jdbc.CalciteConnection;
 import org.apache.calcite.schema.SchemaPlus;
 
+import org.jspecify.annotations.Nullable;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.Level;
@@ -40,6 +41,8 @@
 import java.util.Properties;
 import java.util.Random;
 
+import static org.apache.calcite.linq4j.Nullness.castNonNull;
+
 /**
  * Compares {@link java.sql.Statement} vs {@link java.sql.PreparedStatement}.
  *
@@ -66,6 +69,7 @@ public class StatementTest {
    */
   @State(Scope.Thread)
   @BenchmarkMode(Mode.AverageTime)
+  @SuppressWarnings("NullAway.Init") // JMH sets the fields via @Setup and @Param
   public static class HrConnection {
     final Connection con;
     int id;
@@ -124,7 +128,7 @@ public static class HrPreparedStatement extends HrConnection {
   }
 
   @Benchmark
-  public String prepareBindExecute(HrConnection state) throws SQLException {
+  public @Nullable String prepareBindExecute(HrConnection state) throws SQLException {
     Connection con = state.con;
     Statement st = null;
     ResultSet rs = null;
@@ -144,7 +148,7 @@ public String prepareBindExecute(HrConnection state) throws SQLException {
   }
 
   @Benchmark
-  public String bindExecute(HrPreparedStatement state)
+  public @Nullable String bindExecute(HrPreparedStatement state)
       throws SQLException {
     PreparedStatement st = state.ps;
     ResultSet rs = null;
@@ -161,7 +165,7 @@ public String bindExecute(HrPreparedStatement state)
   }
 
   @Benchmark
-  public String executeQuery(HrConnection state) throws SQLException {
+  public @Nullable String executeQuery(HrConnection state) throws SQLException {
     Connection con = state.con;
     Statement st = null;
     ResultSet rs = null;
@@ -178,7 +182,7 @@ public String executeQuery(HrConnection state) throws SQLException {
   }
 
   @Benchmark
-  public String forEach(HrConnection state) {
+  public @Nullable String forEach(HrConnection state) {
     final Employee[] emps = state.hr.emps;
     for (Employee emp : emps) {
       if (emp.empid == state.id) {
@@ -188,7 +192,7 @@ public String forEach(HrConnection state) {
     return null;
   }
 
-  private static void close(ResultSet rs, Statement st) {
+  private static void close(@Nullable ResultSet rs, @Nullable Statement st) {
     if (rs != null) {
       try {
         rs.close();
@@ -214,7 +218,7 @@ public static class HrSchema {
     public final Employee[] emps = {
         new Employee(100, 10, "Bill", 10000, 1000),
         new Employee(200, 20, "Eric", 8000, 500),
-        new Employee(150, 10, "Sebastian", 7000, null),
+        new Employee(150, 10, "Sebastian", 7000, castNonNull(null)),
         new Employee(110, 10, "Theodore", 11500, 250),
     };
     public final Department[] depts = {
diff --git a/ubenchmark/src/jmh/java/org/apache/calcite/benchmarks/StringConstructBenchmark.java b/ubenchmark/src/jmh/java/org/apache/calcite/benchmarks/StringConstructBenchmark.java
index 53ca532f1cd8..f9669986de44 100644
--- a/ubenchmark/src/jmh/java/org/apache/calcite/benchmarks/StringConstructBenchmark.java
+++ b/ubenchmark/src/jmh/java/org/apache/calcite/benchmarks/StringConstructBenchmark.java
@@ -61,6 +61,7 @@ public class StringConstructBenchmark {
    * benchmark.
    */
   @State(Scope.Thread)
+  @SuppressWarnings("NullAway.Init") // JMH sets the fields via @Setup and @Param
   public static class WriterState {
     public Writer writer;
 
@@ -75,6 +76,7 @@ public void setup() {
    * operations.
    */
   @State(Scope.Thread)
+  @SuppressWarnings("NullAway.Init") // JMH sets the fields via @Setup and @Param
   public static class AppenderState {
     /**
      * The type of the appender to be initialised.
diff --git a/ubenchmark/src/jmh/java/org/apache/calcite/benchmarks/TypeDigestBenchmark.java b/ubenchmark/src/jmh/java/org/apache/calcite/benchmarks/TypeDigestBenchmark.java
index a02a6e836a0b..ac516e6166c9 100644
--- a/ubenchmark/src/jmh/java/org/apache/calcite/benchmarks/TypeDigestBenchmark.java
+++ b/ubenchmark/src/jmh/java/org/apache/calcite/benchmarks/TypeDigestBenchmark.java
@@ -54,6 +54,7 @@
 @OutputTimeUnit(TimeUnit.MICROSECONDS)
 @State(Scope.Thread)
 @Threads(1)
+@SuppressWarnings("NullAway.Init") // JMH sets the fields via @Setup and @Param
 public class TypeDigestBenchmark {
 
   @Param({"1", "50", "500", "5000", "50000"})
@@ -68,7 +69,7 @@ public void setup() {
     type2 = createType(topN);
   }
 
-  private RelDataType createType(int n) {
+  private static RelDataType createType(int n) {
     RelBuilder builder =
         RelBuilder.create(Frameworks.newConfigBuilder()
             .defaultSchema(Frameworks.createRootSchema(true))
diff --git a/ubenchmark/src/jmh/java/org/apache/calcite/benchmarks/package-info.java b/ubenchmark/src/jmh/java/org/apache/calcite/benchmarks/package-info.java
index 37d29d443ab0..c11a890b1b7b 100644
--- a/ubenchmark/src/jmh/java/org/apache/calcite/benchmarks/package-info.java
+++ b/ubenchmark/src/jmh/java/org/apache/calcite/benchmarks/package-info.java
@@ -18,4 +18,7 @@
 /**
  * JMH benchmarks for Calcite.
  */
+@NullMarked
 package org.apache.calcite.benchmarks;
+
+import org.jspecify.annotations.NullMarked;

From 5df3a4dc3d60dba54b98bdbf68bac9bba2d6f80c Mon Sep 17 00:00:00 2001
From: Vladimir Sitnikov 
Date: Wed, 26 Aug 2026 08:52:48 +0300
Subject: [PATCH 562/562] [CALCITE-7736] Add JSpecify Java 8 transformation
 workaround

Co-Authored-By: Codex GPT 5.6-Terra 
---
 build.gradle.kts                              | 89 +++++++++++++++++++
 .../adapter/cassandra/CassandraTable.java     |  4 +-
 .../calcite/adapter/clone/ArrayTable.java     | 18 ++--
 .../calcite/adapter/clone/ColumnLoader.java   |  2 +-
 .../calcite/adapter/enumerable/EnumUtils.java | 16 ++--
 .../enumerable/EnumerableBindable.java        |  4 +-
 .../enumerable/EnumerableInterpretable.java   |  4 +-
 .../enumerable/EnumerableTableModify.java     | 10 +--
 .../adapter/enumerable/EnumerableWindow.java  |  2 +-
 .../enumerable/RexToLixTranslator.java        | 12 +--
 .../calcite/adapter/jdbc/JdbcSchema.java      |  2 +-
 .../calcite/adapter/jdbc/JdbcUtils.java       |  2 +-
 .../calcite/interpreter/CollectNode.java      |  2 +-
 .../interpreter/TableFunctionScanNode.java    |  2 +-
 .../calcite/interpreter/TableScanNode.java    |  4 +-
 .../java/org/apache/calcite/jdbc/Driver.java  |  8 +-
 .../apache/calcite/materialize/Lattice.java   |  4 +-
 .../calcite/materialize/LatticeSuggester.java |  6 +-
 .../apache/calcite/model/ModelHandler.java    |  6 +-
 .../org/apache/calcite/plan/RelOptUtil.java   |  6 +-
 .../calcite/plan/RexImplicationChecker.java   | 30 +++----
 .../calcite/plan/SubstitutionVisitor.java     |  2 +-
 .../calcite/plan/VisitorDataContext.java      |  2 +-
 .../calcite/prepare/CalcitePrepareImpl.java   |  4 +-
 .../org/apache/calcite/prepare/Prepare.java   |  2 +-
 .../org/apache/calcite/profile/Profiler.java  | 12 +--
 .../apache/calcite/profile/ProfilerImpl.java  |  2 +-
 .../apache/calcite/rel/AbstractRelNode.java   | 10 +--
 .../calcite/rel/externalize/RelDotWriter.java |  2 +-
 .../calcite/rel/externalize/RelJson.java      | 38 ++++----
 .../rel/externalize/RelJsonWriter.java        | 12 +--
 .../rel/externalize/RelWriterImpl.java        |  6 +-
 .../calcite/rel/externalize/RelXmlWriter.java |  4 +-
 .../rel/metadata/MetadataFactoryImpl.java     |  2 +-
 .../rel/metadata/RelMdColumnOrigins.java      |  2 +-
 .../calcite/rel/metadata/RelMdSize.java       | 32 +++----
 .../rel/metadata/RelMetadataQuery.java        |  2 +-
 ...AggregateExpandDistinctAggregatesRule.java |  2 +-
 .../rel/rules/FilterMultiJoinMergeRule.java   |  2 +-
 .../rel/rules/JoinToMultiJoinRule.java        | 16 ++--
 .../calcite/rel/rules/LoptMultiJoin.java      |  4 +-
 .../apache/calcite/rel/rules/MultiJoin.java   |  2 +-
 .../rel/rules/ProjectTableScanRule.java       |  2 +-
 .../calcite/rel/rules/SetOpToFilterRule.java  | 12 +--
 .../materialize/MaterializedViewRule.java     |  2 +-
 .../org/apache/calcite/rex/RexExecutable.java |  6 +-
 .../apache/calcite/rex/RexProgramBuilder.java |  2 +-
 .../org/apache/calcite/rex/RexSimplify.java   |  4 +-
 .../rex/RexSqlStandardConvertletTable.java    | 10 +--
 .../java/org/apache/calcite/rex/RexUtil.java  | 10 +--
 .../apache/calcite/runtime/JsonFunctions.java |  4 +-
 .../org/apache/calcite/runtime/PairList.java  |  6 +-
 .../calcite/runtime/ResultSetEnumerable.java  |  4 +-
 .../apache/calcite/runtime/SqlFunctions.java  | 18 ++--
 .../apache/calcite/runtime/XmlFunctions.java  |  2 +-
 .../runtime/variant/VariantNonNull.java       |  4 +-
 .../apache/calcite/sql/SqlCallBinding.java    |  2 +-
 .../org/apache/calcite/sql/SqlNodeList.java   |  6 +-
 .../java/org/apache/calcite/sql/SqlPivot.java |  2 +-
 .../org/apache/calcite/sql/SqlSetOption.java  |  2 +-
 .../org/apache/calcite/sql/SqlUnpivot.java    |  2 +-
 .../java/org/apache/calcite/sql/SqlUtil.java  |  8 +-
 .../calcite/sql/fun/SqlInternalOperators.java |  2 +-
 .../calcite/sql/fun/SqlJsonValueFunction.java |  2 +-
 .../calcite/sql/fun/SqlLibraryOperators.java  |  2 +-
 .../sql/fun/SqlMapValueConstructor.java       |  4 +-
 .../calcite/sql/parser/SqlParserUtil.java     |  2 +-
 .../apache/calcite/sql/type/OperandTypes.java |  2 +-
 .../apache/calcite/sql/util/SqlShuttle.java   |  4 +-
 .../validate/SqlUserDefinedTableFunction.java |  4 +-
 .../validate/SqlUserDefinedTableMacro.java    |  4 +-
 .../sql/validate/SqlValidatorImpl.java        | 16 ++--
 .../calcite/sql2rel/SqlToRelConverter.java    | 12 +--
 .../org/apache/calcite/tools/RelBuilder.java  |  8 +-
 .../org/apache/calcite/util/JsonBuilder.java  |  2 +-
 .../org/apache/calcite/util/ReflectUtil.java  |  2 +-
 .../org/apache/calcite/util/XmlOutput.java    |  2 +-
 .../apache/calcite/plan/RelWriterTest.java    |  2 +-
 .../rel/logical/ToLogicalConverterTest.java   |  2 +-
 .../CorrelateProjectExtractorTest.java        |  8 +-
 .../calcite/sql2rel/RelDecorrelatorTest.java  |  2 +-
 .../calcite/sql2rel/RelFieldTrimmerTest.java  | 10 +--
 .../apache/calcite/test/InterpreterTest.java  |  2 +-
 .../apache/calcite/test/RelBuilderTest.java   | 18 ++--
 .../apache/calcite/test/RelMetadataTest.java  |  4 +-
 .../apache/calcite/test/RelOptRulesTest.java  |  2 +-
 .../org/apache/calcite/util/UtilTest.java     |  8 +-
 .../adapter/druid/DruidConnectionImpl.java    |  2 +-
 .../adapter/druid/DruidJsonFilter.java        | 10 +--
 .../calcite/adapter/druid/DruidQuery.java     |  6 +-
 .../elasticsearch/ElasticsearchJson.java      |  6 +-
 .../elasticsearch/ElasticsearchTable.java     |  4 +-
 .../adapter/csv/CsvProjectTableScanRule.java  |  2 +-
 .../calcite/adapter/file/JsonEnumerator.java  |  2 +-
 .../adapter/geode/rel/GeodeFilter.java        |  4 +-
 .../calcite/adapter/innodb/InnodbTable.java   |  2 +-
 .../calcite/linq4j/EnumerableDefaults.java    | 14 +--
 .../calcite/adapter/mongodb/MongoFilter.java  | 26 +++---
 .../calcite/adapter/mongodb/MongoTable.java   |  4 +-
 .../calcite/piglet/PigRelOpInnerVisitor.java  |  2 +-
 .../apache/calcite/adapter/os/SqlShell.java   |  2 +-
 .../calcite/adapter/tpcds/TpcdsSchema.java    |  2 +-
 .../calcite/adapter/tpcds/TpcdsTest.java      |  2 +-
 .../calcite/server/ServerDdlExecutor.java     | 24 ++---
 .../apache/calcite/test/CalciteAssert.java    |  4 +-
 .../apache/calcite/test/SqlOperatorTest.java  |  4 +-
 .../calcite/test/SqlValidatorFixture.java     |  2 +-
 .../orderstream/OrdersStreamTableFactory.java |  2 +-
 108 files changed, 421 insertions(+), 332 deletions(-)

diff --git a/build.gradle.kts b/build.gradle.kts
index 290857ddfda5..989c6ba941cb 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -25,9 +25,24 @@ import com.github.vlsi.gradle.properties.dsl.props
 import com.github.vlsi.gradle.release.RepositoryType
 import de.thetaphi.forbiddenapis.gradle.CheckForbiddenApis
 import de.thetaphi.forbiddenapis.gradle.CheckForbiddenApisExtension
+import java.util.zip.ZipEntry
+import java.util.zip.ZipFile
+import java.util.zip.ZipInputStream
+import java.util.zip.ZipOutputStream
 import net.ltgt.gradle.errorprone.errorprone
 import org.apache.calcite.buildtools.asmchecker.AsmCheckerTask
 import org.apache.calcite.buildtools.buildext.dsl.ParenthesisBalancer
+import org.gradle.api.artifacts.transform.CacheableTransform
+import org.gradle.api.artifacts.transform.InputArtifact
+import org.gradle.api.artifacts.transform.TransformAction
+import org.gradle.api.artifacts.transform.TransformOutputs
+import org.gradle.api.artifacts.transform.TransformParameters
+import org.gradle.api.artifacts.type.ArtifactTypeDefinition
+import org.gradle.api.attributes.Attribute
+import org.gradle.api.file.FileSystemLocation
+import org.gradle.api.provider.Provider
+import org.gradle.api.tasks.PathSensitive
+import org.gradle.api.tasks.PathSensitivity
 import org.gradle.api.tasks.testing.logging.TestExceptionFormat
 
 plugins {
@@ -73,6 +88,62 @@ tasks.wrapper {
     distributionType = Wrapper.DistributionType.BIN
 }
 
+val jspecifyJava8Compatible =
+    Attribute.of("org.apache.calcite.jspecify-java8-compatible", Boolean::class.javaObjectType)
+
+/** Removes the Java 9-only MODULE target from JSpecify's @NullMarked annotation. Workaround for https://github.com/jspecify/jspecify/issues/795 */
+@CacheableTransform
+abstract class JSpecifyJava8Transform : TransformAction {
+    @get:InputArtifact
+    @get:PathSensitive(PathSensitivity.NAME_ONLY)
+    abstract val inputArtifact: Provider
+
+    override fun transform(outputs: TransformOutputs) {
+        val input = inputArtifact.get().asFile
+        if (input.length() > 64_000L ||
+            !ZipFile(input).use { it.getEntry("org/jspecify/annotations/NullMarked.class") != null }) {
+            outputs.file(input)
+            return
+        }
+        val output = outputs.file(input.name)
+        ZipInputStream(input.inputStream().buffered()).use { source ->
+            ZipOutputStream(output.outputStream().buffered()).use { destination ->
+                while (true) {
+                    val entry = source.nextEntry ?: break
+                    destination.putNextEntry(ZipEntry(entry.name))
+                    if (!entry.isDirectory) {
+                        val bytes = source.readBytes()
+                        destination.write(
+                            if (entry.name == "org/jspecify/annotations/NullMarked.class") {
+                                bytes.replaceModuleTarget()
+                            } else bytes
+                        )
+                    }
+                    destination.closeEntry()
+                    source.closeEntry()
+                }
+            }
+        }
+    }
+
+    private fun ByteArray.replaceModuleTarget(): ByteArray {
+        fun utf8Entry(value: String): ByteArray {
+            val bytes = value.toByteArray(Charsets.US_ASCII)
+            return byteArrayOf(1, 0, bytes.size.toByte()) + bytes
+        }
+
+        val module = utf8Entry("MODULE")
+        val method = "METHOD".toByteArray(Charsets.US_ASCII)
+        val signature = utf8Entry("Ljava/lang/annotation/ElementType;") + module
+        val offset = (0..(size - signature.size)).singleOrNull { index ->
+            signature.indices.all { offset -> this[index + offset] == signature[offset] }
+        } ?: return this
+        return copyOf().also {
+            method.copyInto(it, offset + signature.size - method.size)
+        }
+    }
+}
+
 fun reportsForHumans() = !(System.getenv()["CI"]?.toBoolean() ?: false)
 
 val lastEditYear by extra(lastEditYear())
@@ -584,6 +655,24 @@ allprojects {
     }
 
     plugins.withType {
+        configurations.configureEach {
+            if (isCanBeResolved && !isCanBeConsumed) {
+                attributes.attribute(jspecifyJava8Compatible, true)
+            }
+        }
+        dependencies {
+            attributesSchema {
+                attribute(jspecifyJava8Compatible)
+            }
+            artifactTypes.getByName(ArtifactTypeDefinition.JAR_TYPE)
+                .attributes.attribute(jspecifyJava8Compatible, false)
+            registerTransform(JSpecifyJava8Transform::class) {
+                from.attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.JAR_TYPE)
+                    .attribute(jspecifyJava8Compatible, false)
+                to.attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.JAR_TYPE)
+                    .attribute(jspecifyJava8Compatible, true)
+            }
+        }
         configure {
             sourceCompatibility = JavaVersion.VERSION_1_8
             targetCompatibility = JavaVersion.VERSION_1_8
diff --git a/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraTable.java b/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraTable.java
index 9ee4c533b881..ee5268d55aac 100644
--- a/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraTable.java
+++ b/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraTable.java
@@ -139,7 +139,7 @@ public List getClusteringOrder() {
     final RelDataTypeFactory.Builder fieldInfo = typeFactory.builder();
     final RelDataType rowType = getRowType(typeFactory);
 
-    Function1 addField = fieldName -> {
+    Function1 addField = fieldName -> {
       RelDataType relDataType =
           requireNonNull(rowType.getField(fieldName, true, false)).getType();
       fieldInfo.add(fieldName, relDataType).nullable(true);
@@ -243,7 +243,7 @@ private static class CassandraEnumerable
       final ResultSet results = session.execute(query);
       // Skip results until we get to the right offset
       int skip = 0;
-      Enumerator<@Nullable Object> enumerator =
+      Enumerator enumerator =
           new CassandraEnumerator(results, resultRowType);
       while (skip < offset && enumerator.moveNext()) {
         skip++;
diff --git a/core/src/main/java/org/apache/calcite/adapter/clone/ArrayTable.java b/core/src/main/java/org/apache/calcite/adapter/clone/ArrayTable.java
index 712f3231edb0..97a60e8828fd 100644
--- a/core/src/main/java/org/apache/calcite/adapter/clone/ArrayTable.java
+++ b/core/src/main/java/org/apache/calcite/adapter/clone/ArrayTable.java
@@ -280,7 +280,7 @@ public static class ObjectArray implements Representation {
 
     @Override public Object freeze(ColumnLoader.ValueSet valueSet, int @Nullable [] sources) {
       // We assume the values have been canonized.
-      final List<@Nullable Comparable> list = permuteList(valueSet.values, sources);
+      final List list = permuteList(valueSet.values, sources);
       return list.toArray(new @Nullable Comparable[0]);
     }
 
@@ -442,7 +442,7 @@ public static class ObjectDictionary implements Representation {
       Arrays.sort(nonNullCodeValues, 0, n);
       ColumnLoader.ValueSet codeValueSet =
           new ColumnLoader.ValueSet(int.class);
-      final List<@Nullable Comparable> list = permuteList(valueSet.values, sources);
+      final List list = permuteList(valueSet.values, sources);
       for (Comparable value : list) {
         int code;
         if (value == null) {
@@ -462,14 +462,14 @@ public static class ObjectDictionary implements Representation {
     }
 
     @Override public Object permute(Object dataSet, int[] sources) {
-      final Pair pair = unfreeze(dataSet);
+      final Pair pair = unfreeze(dataSet);
       Object codes = pair.left;
       @Nullable Comparable[] codeValues = pair.right;
       return Pair.of(representation.permute(codes, sources), codeValues);
     }
 
     @Override public @Nullable Object getObject(Object dataSet, int ordinal) {
-      final Pair pair = unfreeze(dataSet);
+      final Pair pair = unfreeze(dataSet);
       int code = representation.getInt(pair.left, ordinal);
       return pair.right[code];
     }
@@ -480,7 +480,7 @@ public static class ObjectDictionary implements Representation {
     }
 
     @Override public int size(Object dataSet) {
-      final Pair pair = unfreeze(dataSet);
+      final Pair pair = unfreeze(dataSet);
       return representation.size(pair.left);
     }
 
@@ -595,7 +595,7 @@ public static class Constant implements Representation {
     }
 
     @Override public @Nullable Object getObject(Object dataSet, int ordinal) {
-      Pair<@Nullable Object, Integer> pair = unfreeze(dataSet);
+      Pair pair = unfreeze(dataSet);
       return pair.left;
     }
 
@@ -605,12 +605,12 @@ public static class Constant implements Representation {
     }
 
     @Override public int size(Object dataSet) {
-      Pair<@Nullable Object, Integer> pair = unfreeze(dataSet);
+      Pair pair = unfreeze(dataSet);
       return pair.right;
     }
 
     @Override public String toString(Object dataSet) {
-      Pair<@Nullable Object, Integer> pair = unfreeze(dataSet);
+      Pair pair = unfreeze(dataSet);
       return Collections.nCopies(pair.right, pair.left).toString();
     }
   }
@@ -648,7 +648,7 @@ public static class BitSlicedPrimitiveArray implements Representation {
 
     @Override public Object freeze(ColumnLoader.ValueSet valueSet, int @Nullable [] sources) {
       final int chunksPerWord = 64 / bitCount;
-      final List<@Nullable Comparable> valueList =
+      final List valueList =
           permuteList(valueSet.values, sources);
       final int valueCount = valueList.size();
       final int wordCount =
diff --git a/core/src/main/java/org/apache/calcite/adapter/clone/ColumnLoader.java b/core/src/main/java/org/apache/calcite/adapter/clone/ColumnLoader.java
index 181b78d6990a..d106dcf17644 100644
--- a/core/src/main/java/org/apache/calcite/adapter/clone/ColumnLoader.java
+++ b/core/src/main/java/org/apache/calcite/adapter/clone/ColumnLoader.java
@@ -249,7 +249,7 @@ private void load(final RelDataType elementType,
       switch (rep) {
       case OBJECT:
       case JAVA_SQL_TIMESTAMP:
-        final List<@Nullable Long> longs =
+        final List longs =
             Util.transform((List<@Nullable Timestamp>) list,
                 t -> t == null ? null : t.getTime());
         return longs;
diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java
index 0e94d780dd1c..ebca70ba62d0 100644
--- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java
+++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java
@@ -1202,7 +1202,7 @@ private static class SessionizationEnumerator implements Enumerator<@Nullable Ob
     }
 
     private void initialize() {
-      List<@Nullable Object[]> elements = new ArrayList<>();
+      List elements = new ArrayList<>();
       // initialize() will be called when inputEnumerator.moveNext() is true,
       // thus firstly should take the current element.
       elements.add(inputEnumerator.current());
@@ -1213,7 +1213,7 @@ private void initialize() {
 
       // The windows of each key are kept sorted by start time; the merge
       // below only compares a window with the one that precedes it.
-      Map<@Nullable Object, NavigableMap, List<@Nullable Object[]>>>
+      Map, List>>
           sessionKeyMap = new HashMap<>();
       for (@Nullable Object[] element : elements) {
         // A key column index of -1 means that there is no key; every element
@@ -1222,7 +1222,7 @@ private void initialize() {
         Object watermark =
             requireNonNull(element[indexOfWatermarkedColumn],
                 "element[indexOfWatermarkedColumn]");
-        NavigableMap, List<@Nullable Object[]>> session =
+        NavigableMap, List> session =
             sessionKeyMap.computeIfAbsent(key, k -> new TreeMap<>());
         Pair initWindow =
             computeInitWindow(SqlFunctions.toLong(watermark), gap);
@@ -1230,12 +1230,12 @@ private void initialize() {
       }
 
       // merge per key session windows if there is any overlap between windows.
-      for (Map.Entry<@Nullable Object, NavigableMap, List<@Nullable Object[]>>>
+      for (Map.Entry, List>>
           perKeyEntry : sessionKeyMap.entrySet()) {
-        Map, List<@Nullable Object[]>> finalWindowElementsMap = new HashMap<>();
+        Map, List> finalWindowElementsMap = new HashMap<>();
         Pair currentWindow = null;
-        List<@Nullable Object[]> tempElementList = new ArrayList<>();
-        for (Map.Entry, List<@Nullable Object[]>> sessionEntry
+        List tempElementList = new ArrayList<>();
+        for (Map.Entry, List> sessionEntry
             : perKeyEntry.getValue().entrySet()) {
           // check the next window can be merged.
           if (currentWindow == null || !isOverlapped(currentWindow, sessionEntry.getKey())) {
@@ -1261,7 +1261,7 @@ private void initialize() {
         }
 
         // construct final results from finalWindowElementsMap.
-        for (Map.Entry, List<@Nullable Object[]>> finalWindowElementsEntry
+        for (Map.Entry, List> finalWindowElementsEntry
             : finalWindowElementsMap.entrySet()) {
           for (@Nullable Object[] element : finalWindowElementsEntry.getValue()) {
             @Nullable Object[] curWithWindow = new Object[element.length + 2];
diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableBindable.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableBindable.java
index c2b4a28fb824..79cf6424bb73 100644
--- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableBindable.java
+++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableBindable.java
@@ -77,8 +77,8 @@ protected EnumerableBindable(RelOptCluster cluster, RelNode input) {
       final Sink sink =
           requireNonNull(implementor.relSinks.get(EnumerableBindable.this),
               () -> "relSinks.get is null for " + EnumerableBindable.this).get(0);
-      final Enumerable<@Nullable Object[]> enumerable = bind(implementor.dataContext);
-      final Enumerator<@Nullable Object[]> enumerator = enumerable.enumerator();
+      final Enumerable enumerable = bind(implementor.dataContext);
+      final Enumerator enumerator = enumerable.enumerator();
       while (enumerator.moveNext()) {
         sink.send(Row.asCopy(enumerator.current()));
       }
diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableInterpretable.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableInterpretable.java
index df9bd78b2ea5..16d58f8dc7f7 100644
--- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableInterpretable.java
+++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableInterpretable.java
@@ -85,7 +85,7 @@ protected EnumerableInterpretable(RelOptCluster cluster, RelNode input) {
         toBindable(implementor.internalParameters, implementor.spark,
             (EnumerableRel) getInput(), EnumerableRel.Prefer.ARRAY);
     final ArrayBindable arrayBindable = box(bindable);
-    final Enumerable<@Nullable Object[]> enumerable =
+    final Enumerable enumerable =
         arrayBindable.bind(implementor.dataContext);
     return new EnumerableNode(enumerable, implementor.compiler, this);
   }
@@ -263,7 +263,7 @@ private static class EnumerableNode implements Node {
     }
 
     @Override public void run() throws InterruptedException {
-      final Enumerator<@Nullable Object[]> enumerator = enumerable.enumerator();
+      final Enumerator enumerator = enumerable.enumerator();
       while (enumerator.moveNext()) {
         @Nullable Object[] values = enumerator.current();
         sink.send(Row.of(values));
diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableTableModify.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableTableModify.java
index 44756f77f150..4a78a5107563 100644
--- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableTableModify.java
+++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableTableModify.java
@@ -223,11 +223,11 @@ private Result implementUpdate(
    */
   public static long applyUpdateOneToOne(Enumerable source, List sink,
       int tableFieldCount, int[] updateColumnIndices) {
-    final Map, Deque> updatesByKey = new HashMap<>();
+    final Map, Deque> updatesByKey = new HashMap<>();
     try (Enumerator e = source.enumerator()) {
       while (e.moveNext()) {
         final Object[] sourceRow = e.current();
-        final List<@Nullable Object> key =
+        final List key =
             Arrays.asList(Arrays.copyOf(sourceRow, tableFieldCount));
         final Object[] newRow = applyUpdate(sourceRow, tableFieldCount, updateColumnIndices);
         updatesByKey.computeIfAbsent(key, k -> new ArrayDeque<>()).addLast(newRow);
@@ -484,17 +484,17 @@ public static void applyDeleteRowsByKey(Enumerable sourceKeys,
       Collection sinkRows, Function1 sinkKeySelector) {
 
     // Build a map of source keys to the number of sink rows that must be removed for each.
-    final Map, Integer> pendingByKey = new HashMap<>();
+    final Map, Integer> pendingByKey = new HashMap<>();
     try (Enumerator e = sourceKeys.enumerator()) {
       while (e.moveNext()) {
-        final List<@Nullable Object> key = keyOf(e.current());
+        final List key = keyOf(e.current());
         pendingByKey.put(key, pendingByKey.getOrDefault(key, 0) + 1);
       }
     }
 
     // Iterate over sink rows and remove matching rows based on key.
     for (java.util.Iterator it = sinkRows.iterator(); it.hasNext();) {
-      final List<@Nullable Object> key = keyOf(sinkKeySelector.apply(it.next()));
+      final List key = keyOf(sinkKeySelector.apply(it.next()));
       final Integer pending = pendingByKey.get(key);
       if (pending == null || pending == 0) {
         continue;
diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java
index 818f70a6bf86..c7991aa5b863 100644
--- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java
+++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java
@@ -262,7 +262,7 @@ private static void sampleOfTheGeneratedWindowedAggregate() {
                       BuiltInMethod.COLLECTION_SIZE.method)),
               false);
 
-      Pair<@Nullable Expression, @Nullable Expression> collationKey =
+      Pair collationKey =
           getRowCollationKey(builder, inputPhysType, group, windowIdx);
       Expression keySelector = collationKey.left;
       Expression keyComparator = collationKey.right;
diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java
index 614b6ae42239..94aff86d750a 100644
--- a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java
+++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java
@@ -1246,7 +1246,7 @@ public List translateList(
       RexImpTable.NullAs nullAs,
       List storageTypes) {
     final List list = new ArrayList<>();
-    for (Pair e : Pair.zip(operandList, storageTypes)) {
+    for (Pair e : Pair.zip(operandList, storageTypes)) {
       list.add(translate(e.left, nullAs, e.right));
     }
     return list;
@@ -1451,7 +1451,7 @@ private static Expression scaleValue(
    * }
    */
   @Override public Result visitInputRef(RexInputRef inputRef) {
-    final Pair key = Pair.of(inputRef, currentStorageType);
+    final Pair key = Pair.of(inputRef, currentStorageType);
     // If the RexInputRef has been visited under current storage type already,
     // it is not necessary to visit it again, just return the result.
     if (rexWithStorageTypeResultMap.containsKey(key)) {
@@ -1632,7 +1632,7 @@ private ConstantExpression getTypedNullLiteral(RexLiteral literal) {
       throw new RuntimeException("cannot translate call " + call);
     }
     final List operandList = call.getOperands();
-    final List<@Nullable Type> storageTypes = EnumUtils.internalTypes(operandList);
+    final List storageTypes = EnumUtils.internalTypes(operandList);
     final List operandResults = new ArrayList<>();
     for (int i = 0; i < operandList.size(); i++) {
       final Result operandResult =
@@ -1739,7 +1739,7 @@ private static void implementRecursively(RexToLixTranslator currentTranslator,
       List operandList, ParameterExpression valueVariable, int pos) {
     final BlockBuilder currentBlockBuilder =
         currentTranslator.getBlockBuilder();
-    final List<@Nullable Type> storageTypes =
+    final List storageTypes =
         EnumUtils.internalTypes(operandList);
     // [ELSE] clause
     if (pos == operandList.size() - 1) {
@@ -1810,7 +1810,7 @@ private Result toInnerStorageType(Result result, Type storageType) {
   }
 
   @Override public Result visitDynamicParam(RexDynamicParam dynamicParam) {
-    final Pair key =
+    final Pair key =
         Pair.of(dynamicParam, currentStorageType);
     if (rexWithStorageTypeResultMap.containsKey(key)) {
       return rexWithStorageTypeResultMap.get(key);
@@ -1855,7 +1855,7 @@ private Result toInnerStorageType(Result result, Type storageType) {
   }
 
   @Override public Result visitFieldAccess(RexFieldAccess fieldAccess) {
-    final Pair key =
+    final Pair key =
         Pair.of(fieldAccess, currentStorageType);
     if (rexWithStorageTypeResultMap.containsKey(key)) {
       return rexWithStorageTypeResultMap.get(key);
diff --git a/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcSchema.java b/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcSchema.java
index f88d5e1da465..19f007e5eeed 100644
--- a/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcSchema.java
+++ b/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcSchema.java
@@ -253,7 +253,7 @@ public DataSource getDataSource() {
   }
 
   private Stream getMetaTableStream(String tableNamePattern) {
-    final Pair<@Nullable String, @Nullable String> catalogSchema = getCatalogSchema();
+    final Pair catalogSchema = getCatalogSchema();
     final Stream tableDefs;
     Connection connection = null;
     ResultSet resultSet = null;
diff --git a/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcUtils.java b/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcUtils.java
index 0d4ba2a09207..bf11879289e7 100644
--- a/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcUtils.java
+++ b/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcUtils.java
@@ -273,7 +273,7 @@ public DataSource get(String url, @Nullable String driverClassName,
         @Nullable String username, @Nullable String password) {
       // Get data source objects from a cache, so that we don't have to sniff
       // out what kind of database they are quite as often.
-      final List<@Nullable String> key =
+      final List key =
           ImmutableNullableList.<@Nullable String>of(url, username, password,
               driverClassName);
       return cache.getUnchecked(key);
diff --git a/core/src/main/java/org/apache/calcite/interpreter/CollectNode.java b/core/src/main/java/org/apache/calcite/interpreter/CollectNode.java
index d57346f656d1..7a9621bcf39f 100644
--- a/core/src/main/java/org/apache/calcite/interpreter/CollectNode.java
+++ b/core/src/main/java/org/apache/calcite/interpreter/CollectNode.java
@@ -37,7 +37,7 @@ public CollectNode(Compiler compiler, Collect rel) {
 
   @Override public void run() throws InterruptedException {
     Row row;
-    List<@Nullable Object[]> values = new ArrayList<>();
+    List values = new ArrayList<>();
     while ((row = source.receive()) != null) {
       values.add(row.getValues());
     }
diff --git a/core/src/main/java/org/apache/calcite/interpreter/TableFunctionScanNode.java b/core/src/main/java/org/apache/calcite/interpreter/TableFunctionScanNode.java
index 295fb39bd83f..23916068fd0a 100644
--- a/core/src/main/java/org/apache/calcite/interpreter/TableFunctionScanNode.java
+++ b/core/src/main/java/org/apache/calcite/interpreter/TableFunctionScanNode.java
@@ -61,7 +61,7 @@ private TableFunctionScanNode(Compiler compiler, TableFunctionScan rel) {
   @Override public void run() throws InterruptedException {
     final Object o = scalar.execute(context);
     if (o instanceof Enumerable) {
-      @SuppressWarnings("unchecked") final Enumerable<@Nullable Object> enumerable =
+      @SuppressWarnings("unchecked") final Enumerable enumerable =
           (Enumerable<@Nullable Object>) o;
       for (final Enumerator enumerator =
                enumerable.select(mapFn).enumerator();
diff --git a/core/src/main/java/org/apache/calcite/interpreter/TableScanNode.java b/core/src/main/java/org/apache/calcite/interpreter/TableScanNode.java
index b2fe5e487cdc..350562c07a2d 100644
--- a/core/src/main/java/org/apache/calcite/interpreter/TableScanNode.java
+++ b/core/src/main/java/org/apache/calcite/interpreter/TableScanNode.java
@@ -172,7 +172,7 @@ private static TableScanNode createFilterable(Compiler compiler,
       FilterableTable filterableTable) {
     final DataContext root = compiler.getDataContext();
     final List mutableFilters = Lists.newArrayList(filters);
-    final Enumerable<@Nullable Object[]> enumerable =
+    final Enumerable enumerable =
         filterableTable.scan(root, mutableFilters);
     for (RexNode filter : mutableFilters) {
       if (!filters.contains(filter)) {
@@ -223,7 +223,7 @@ private static TableScanNode createProjectableFilterable(Compiler compiler,
           continue;
         }
       }
-      final Enumerable<@Nullable Object[]> enumerable1 =
+      final Enumerable enumerable1 =
           pfTable.scan(root, mutableFilters, projectInts);
       final Enumerable rowEnumerable = Enumerables.toRow(enumerable1);
       final ImmutableIntList rejectedProjects;
diff --git a/core/src/main/java/org/apache/calcite/jdbc/Driver.java b/core/src/main/java/org/apache/calcite/jdbc/Driver.java
index ce0a7b8434b1..f7d89f19d5b3 100644
--- a/core/src/main/java/org/apache/calcite/jdbc/Driver.java
+++ b/core/src/main/java/org/apache/calcite/jdbc/Driver.java
@@ -182,17 +182,17 @@ protected Function0 createPrepareFactory() {
         }
         if (schemaFactory != null) {
           final JsonBuilder json = new JsonBuilder();
-          final Map root = json.map();
+          final Map root = json.map();
           root.put("version", "1.0");
           root.put("defaultSchema", schemaName);
-          final List<@Nullable Object> schemaList = json.list();
+          final List schemaList = json.list();
           root.put("schemas", schemaList);
-          final Map schema = json.map();
+          final Map schema = json.map();
           schemaList.add(schema);
           schema.put("type", "custom");
           schema.put("name", schemaName);
           schema.put("factory", schemaFactory.getClass().getName());
-          final Map operandMap = json.map();
+          final Map operandMap = json.map();
           schema.put("operand", operandMap);
           for (Map.Entry entry : Util.toMap(info).entrySet()) {
             if (entry.getKey().startsWith("schema.")) {
diff --git a/core/src/main/java/org/apache/calcite/materialize/Lattice.java b/core/src/main/java/org/apache/calcite/materialize/Lattice.java
index 904412e6e3b2..4fd23a892ddb 100644
--- a/core/src/main/java/org/apache/calcite/materialize/Lattice.java
+++ b/core/src/main/java/org/apache/calcite/materialize/Lattice.java
@@ -832,7 +832,7 @@ public Builder(LatticeSpace space, CalciteSchema schema, String sql) {
       populate(relNodes, tempLinks, parsed.root.rel);
 
       // Get aliases.
-      List<@Nullable String> aliases = new ArrayList<>();
+      List aliases = new ArrayList<>();
       SqlNode from = requireNonNull(((SqlSelect) parsed.sqlNode).getFrom());
       populateAliases(from, aliases, null);
 
@@ -840,7 +840,7 @@ public Builder(LatticeSpace space, CalciteSchema schema, String sql) {
       final DirectedGraph graph =
           DefaultDirectedGraph.create(Edge.FACTORY);
       final List vertices = new ArrayList<>();
-      for (Pair p : Pair.zip(relNodes, aliases)) {
+      for (Pair p : Pair.zip(relNodes, aliases)) {
         final LatticeTable table = space.register(p.left.getTable());
         final Vertex vertex = new Vertex(table, p.right);
         graph.addVertex(vertex);
diff --git a/core/src/main/java/org/apache/calcite/materialize/LatticeSuggester.java b/core/src/main/java/org/apache/calcite/materialize/LatticeSuggester.java
index db07bbb85004..d164b77a6dcc 100644
--- a/core/src/main/java/org/apache/calcite/materialize/LatticeSuggester.java
+++ b/core/src/main/java/org/apache/calcite/materialize/LatticeSuggester.java
@@ -177,7 +177,7 @@ private void addFrame(Query q, Frame frame, List lattices) {
     }
 
     // Translate the query graph to mutable nodes
-    final IdentityHashMap nodes = new IdentityHashMap<>();
+    final IdentityHashMap nodes = new IdentityHashMap<>();
     final Map nodesByParent = new HashMap<>();
     final List rootNodes = new ArrayList<>();
     for (TableRef tableRef : TopologicalOrderIterator.of(g)) {
@@ -191,7 +191,7 @@ private void addFrame(Query q, Frame frame, List lattices) {
       case 1:
         final StepRef edge = edges.get(0);
         final MutableNode parent = nodes.get(edge.source());
-        final List<@Nullable Object> key =
+        final List key =
             FlatLists.of(parent, tableRef.table, edge.step.keys);
         final MutableNode existingNode = nodesByParent.get(key);
         if (existingNode == null) {
@@ -464,7 +464,7 @@ private static void frames(List frames, final Query q, RelNode r) {
         final List<@Nullable ColRef> columns;
 
         {
-          final ImmutableNullableList.Builder<@Nullable ColRef> columnBuilder =
+          final ImmutableNullableList.Builder columnBuilder =
               ImmutableNullableList.builder();
           for (Pair p : project.getNamedProjects()) {
             @SuppressWarnings("NullAway")
diff --git a/core/src/main/java/org/apache/calcite/model/ModelHandler.java b/core/src/main/java/org/apache/calcite/model/ModelHandler.java
index 0f433161017d..b6a7623e522c 100644
--- a/core/src/main/java/org/apache/calcite/model/ModelHandler.java
+++ b/core/src/main/java/org/apache/calcite/model/ModelHandler.java
@@ -235,7 +235,7 @@ public static void addFunctions(ClassNameFilter filter, SchemaPlus schema,
   }
 
   public void visit(JsonRoot jsonRoot) {
-    final Pair<@Nullable String, SchemaPlus> pair =
+    final Pair pair =
         Pair.of(null, rootSchema);
     schemaStack.push(pair);
     for (JsonType rootType : jsonRoot.types) {
@@ -244,7 +244,7 @@ public void visit(JsonRoot jsonRoot) {
     for (JsonSchema schema : jsonRoot.schemas) {
       schema.accept(this);
     }
-    final Pair p = schemaStack.pop();
+    final Pair p = schemaStack.pop();
     assert p == pair;
   }
 
@@ -296,7 +296,7 @@ private void populateSchema(JsonSchema jsonSchema, SchemaPlus schema) {
     final Pair pair = Pair.of(jsonSchema.name, schema);
     schemaStack.push(pair);
     jsonSchema.visitChildren(this);
-    final Pair p = schemaStack.pop();
+    final Pair p = schemaStack.pop();
     assert p == pair;
   }
 
diff --git a/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java b/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java
index baec2a0c88c3..3983b734bf58 100644
--- a/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java
+++ b/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java
@@ -1922,10 +1922,10 @@ public static void projectJoinInputs(
     int origRightInputSize = rightRel.getRowType().getFieldCount();
 
     final List newLeftFields = new ArrayList<>();
-    final List<@Nullable String> newLeftFieldNames = new ArrayList<>();
+    final List newLeftFieldNames = new ArrayList<>();
 
     final List newRightFields = new ArrayList<>();
-    final List<@Nullable String> newRightFieldNames = new ArrayList<>();
+    final List newRightFieldNames = new ArrayList<>();
     int leftKeyCount = leftJoinKeys.size();
     int rightKeyCount = rightJoinKeys.size();
     int i;
@@ -3956,7 +3956,7 @@ public static RelNode pushDownJoinConditions(Join originalJoin,
               extraLeftExprs, extraRightExprs, relBuilder.getRexBuilder());
     }
 
-    final PairList pairs = PairList.of();
+    final PairList pairs = PairList.of();
     relBuilder.push(originalJoin.getLeft());
     if (!extraLeftExprs.isEmpty()) {
       final List fields =
diff --git a/core/src/main/java/org/apache/calcite/plan/RexImplicationChecker.java b/core/src/main/java/org/apache/calcite/plan/RexImplicationChecker.java
index 08ca75dfe503..f27e018bace9 100644
--- a/core/src/main/java/org/apache/calcite/plan/RexImplicationChecker.java
+++ b/core/src/main/java/org/apache/calcite/plan/RexImplicationChecker.java
@@ -213,11 +213,11 @@ private boolean implies2(RexNode first, RexNode second) {
       return false;
     }
 
-    ImmutableList.Builder>> usagesBuilder =
+    ImmutableList.Builder>> usagesBuilder =
         ImmutableList.builder();
-    for (Map.Entry> entry
+    for (Map.Entry> entry
         : firstUsageFinder.usageMap.entrySet()) {
-      ImmutableSet.Builder> usageBuilder =
+      ImmutableSet.Builder> usageBuilder =
           ImmutableSet.builder();
       if (!entry.getValue().usageList.isEmpty()) {
         entry.getValue().usageList.rightList().forEach(v ->
@@ -226,10 +226,10 @@ private boolean implies2(RexNode first, RexNode second) {
       }
     }
 
-    final Set>> usages =
+    final Set>> usages =
         Sets.cartesianProduct(usagesBuilder.build());
 
-    for (List> usageList : usages) {
+    for (List> usageList : usages) {
       // Get the literals from first conjunction and executes second conjunction
       // using them.
       //
@@ -306,16 +306,16 @@ private boolean isSatisfiable(RexNode second, @Nullable DataContext dataValues)
    */
   private static boolean checkSupport(InputUsageFinder firstUsageFinder,
       InputUsageFinder secondUsageFinder) {
-    final Map> firstUsageMap =
+    final Map> firstUsageMap =
         firstUsageFinder.usageMap;
-    final Map> secondUsageMap =
+    final Map> secondUsageMap =
         secondUsageFinder.usageMap;
 
-    for (Map.Entry> entry
+    for (Map.Entry> entry
         : secondUsageMap.entrySet()) {
-      final InputRefUsage secondUsage =
+      final InputRefUsage secondUsage =
           entry.getValue();
-      final PairList secondUsageList =
+      final PairList secondUsageList =
           secondUsage.usageList;
       final int secondLen = secondUsageList.size();
 
@@ -323,7 +323,7 @@ private static boolean checkSupport(InputUsageFinder firstUsageFinder,
         return false;
       }
 
-      final InputRefUsage firstUsage =
+      final InputRefUsage firstUsage =
           firstUsageMap.get(entry.getKey());
 
       if (firstUsage == null
@@ -332,7 +332,7 @@ private static boolean checkSupport(InputUsageFinder firstUsageFinder,
         return false;
       }
 
-      final PairList firstUsageList =
+      final PairList firstUsageList =
           firstUsage.usageList;
       final int firstLen = firstUsageList.size();
 
@@ -450,7 +450,7 @@ private static class InputUsageFinder extends RexVisitorImpl<@Nullable Void> {
     }
 
     @Override public Void visitInputRef(RexInputRef inputRef) {
-      InputRefUsage inputRefUse = getUsageMap(inputRef);
+      InputRefUsage inputRefUse = getUsageMap(inputRef);
       inputRefUse.usageCount++;
       return null;
     }
@@ -502,13 +502,13 @@ private void updateBinaryOpUsage(RexCall call) {
 
     private void updateUsage(SqlOperator op, RexInputRef inputRef,
         @Nullable RexNode literal) {
-      final InputRefUsage inputRefUse =
+      final InputRefUsage inputRefUse =
           getUsageMap(inputRef);
       inputRefUse.usageList.add(op, literal);
     }
 
     private InputRefUsage getUsageMap(RexInputRef rex) {
-      InputRefUsage inputRefUse = usageMap.get(rex);
+      InputRefUsage inputRefUse = usageMap.get(rex);
       if (inputRefUse == null) {
         inputRefUse = new InputRefUsage<>();
         usageMap.put(rex, inputRefUse);
diff --git a/core/src/main/java/org/apache/calcite/plan/SubstitutionVisitor.java b/core/src/main/java/org/apache/calcite/plan/SubstitutionVisitor.java
index b82f73b12f5b..d98fa36c0f30 100644
--- a/core/src/main/java/org/apache/calcite/plan/SubstitutionVisitor.java
+++ b/core/src/main/java/org/apache/calcite/plan/SubstitutionVisitor.java
@@ -224,7 +224,7 @@ public SubstitutionVisitor(RelNode target_, RelNode query_,
     this.query = Holder.of(MutableRels.toMutable(query_));
     this.target = MutableRels.toMutable(target_);
     this.relBuilder = relBuilderFactory.create(cluster, null);
-    final Set<@Nullable MutableRel> parents = Sets.newIdentityHashSet();
+    final Set parents = Sets.newIdentityHashSet();
     final List allNodes = new ArrayList<>();
     final MutableRelVisitor visitor =
         new MutableRelVisitor() {
diff --git a/core/src/main/java/org/apache/calcite/plan/VisitorDataContext.java b/core/src/main/java/org/apache/calcite/plan/VisitorDataContext.java
index cb04e1519e4c..75f8792f4db0 100644
--- a/core/src/main/java/org/apache/calcite/plan/VisitorDataContext.java
+++ b/core/src/main/java/org/apache/calcite/plan/VisitorDataContext.java
@@ -99,7 +99,7 @@ public VisitorDataContext(@Nullable Object[] values) {
       List> usageList) {
     final int size = rowType.getFieldList().size();
     final @Nullable Object[] values = new Object[size];
-    for (Pair elem : usageList) {
+    for (Pair elem : usageList) {
       Pair value = getValue(elem.getKey(), elem.getValue());
       if (value == null) {
         LOGGER.warn("{} is not handled for {} for checking implication",
diff --git a/core/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java b/core/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java
index 8df45081ab5d..ea2cd495d196 100644
--- a/core/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java
+++ b/core/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java
@@ -570,7 +570,7 @@ private static  CalciteSignature simplePrepare(Context context, String sql
     @SuppressWarnings("unchecked")
     final List list = (List) ImmutableList.of(1);
     final List origin = null;
-    final List<@Nullable List> origins =
+    final List> origins =
         Collections.nCopies(x.getFieldCount(), origin);
     final List columns =
         getColumnMetaDataList(typeFactory, x, x, origins);
@@ -715,7 +715,7 @@  CalciteSignature prepare2_(
     }
 
     RelDataType jdbcType = makeStruct(typeFactory, x);
-    final List> originList = preparedResult.getFieldOrigins();
+    final List> originList = preparedResult.getFieldOrigins();
     final List columns =
         getColumnMetaDataList(typeFactory, x, jdbcType, originList);
     Class resultClazz = null;
diff --git a/core/src/main/java/org/apache/calcite/prepare/Prepare.java b/core/src/main/java/org/apache/calcite/prepare/Prepare.java
index 0e6be4cfe01f..315aab38ce4c 100644
--- a/core/src/main/java/org/apache/calcite/prepare/Prepare.java
+++ b/core/src/main/java/org/apache/calcite/prepare/Prepare.java
@@ -187,7 +187,7 @@ protected RelRoot optimize(RelRoot root,
 
   protected Program getProgram() {
     // Allow a test to override the default program.
-    final Holder<@Nullable Program> holder = Holder.empty();
+    final Holder holder = Holder.empty();
     Hook.PROGRAM.run(holder);
     @Nullable Program holderValue = holder.get();
     if (holderValue != null) {
diff --git a/core/src/main/java/org/apache/calcite/profile/Profiler.java b/core/src/main/java/org/apache/calcite/profile/Profiler.java
index 26d8e98f2f1a..472dc8df1a4e 100644
--- a/core/src/main/java/org/apache/calcite/profile/Profiler.java
+++ b/core/src/main/java/org/apache/calcite/profile/Profiler.java
@@ -113,7 +113,7 @@ public RowCount(int rowCount) {
     }
 
     @Override public Object toMap(JsonBuilder jsonBuilder) {
-      final Map map = jsonBuilder.map();
+      final Map map = jsonBuilder.map();
       map.put("type", "rowCount");
       map.put("rowCount", rowCount);
       return map;
@@ -129,7 +129,7 @@ public Unique(SortedSet columns) {
     }
 
     @Override public Object toMap(JsonBuilder jsonBuilder) {
-      final Map map = jsonBuilder.map();
+      final Map map = jsonBuilder.map();
       map.put("type", "unique");
       map.put("columns", FunctionalDependency.getObjects(jsonBuilder, columns));
       return map;
@@ -147,7 +147,7 @@ class FunctionalDependency implements Statistic {
     }
 
     @Override public Object toMap(JsonBuilder jsonBuilder) {
-      final Map map = jsonBuilder.map();
+      final Map map = jsonBuilder.map();
       map.put("type", "fd");
       map.put("columns", getObjects(jsonBuilder, columns));
       map.put("dependentColumn", dependentColumn.name);
@@ -156,7 +156,7 @@ class FunctionalDependency implements Statistic {
 
     private static List<@Nullable Object> getObjects(JsonBuilder jsonBuilder,
         NavigableSet columns) {
-      final List<@Nullable Object> list = jsonBuilder.list();
+      final List list = jsonBuilder.list();
       for (Column column : columns) {
         list.add(column.name);
       }
@@ -203,11 +203,11 @@ public Distribution(SortedSet columns, @Nullable SortedSet v
     }
 
     @Override public Object toMap(JsonBuilder jsonBuilder) {
-      final Map map = jsonBuilder.map();
+      final Map map = jsonBuilder.map();
       map.put("type", "distribution");
       map.put("columns", FunctionalDependency.getObjects(jsonBuilder, columns));
       if (values != null) {
-        List<@Nullable Object> list = jsonBuilder.list();
+        List list = jsonBuilder.list();
         for (Comparable value : values) {
           if (value instanceof java.sql.Date) {
             value = value.toString();
diff --git a/core/src/main/java/org/apache/calcite/profile/ProfilerImpl.java b/core/src/main/java/org/apache/calcite/profile/ProfilerImpl.java
index dd137cbd09a4..7c29c720a995 100644
--- a/core/src/main/java/org/apache/calcite/profile/ProfilerImpl.java
+++ b/core/src/main/java/org/apache/calcite/profile/ProfilerImpl.java
@@ -628,7 +628,7 @@ static class CompositeCollector extends Collector {
         // Too many values. Switch to a sketch collector.
         final HllCompositeCollector collector =
             new HllCompositeCollector(space, columnOrdinals);
-        final List<@Nullable Comparable> list =
+        final List list =
             new ArrayList<>(
                 Collections.<@Nullable Comparable>nCopies(
                     columnOrdinals[columnOrdinals.length - 1] + 1, null));
diff --git a/core/src/main/java/org/apache/calcite/rel/AbstractRelNode.java b/core/src/main/java/org/apache/calcite/rel/AbstractRelNode.java
index d248e01e1f9d..25b67b564cd6 100644
--- a/core/src/main/java/org/apache/calcite/rel/AbstractRelNode.java
+++ b/core/src/main/java/org/apache/calcite/rel/AbstractRelNode.java
@@ -365,14 +365,14 @@ public RelWriter explainTerms(RelWriter pw) {
     if (!result) {
       return false;
     }
-    PairList items1 = this.getDigestItems();
-    PairList items2 = that.getDigestItems();
+    PairList items1 = this.getDigestItems();
+    PairList items2 = that.getDigestItems();
     if (items1.size() != items2.size()) {
       return false;
     }
     for (int i = 0; result && i < items1.size(); i++) {
-      Map.Entry attr1 = items1.get(i);
-      Map.Entry attr2 = items2.get(i);
+      Map.Entry attr1 = items1.get(i);
+      Map.Entry attr2 = items2.get(i);
       if (attr1.getValue() instanceof RelNode) {
         result = ((RelNode) attr1.getValue()).deepEquals(attr2.getValue());
       } else {
@@ -390,7 +390,7 @@ public RelWriter explainTerms(RelWriter pw) {
   @API(since = "1.25", status = API.Status.MAINTAINED)
   @Override public int deepHashCode() {
     int result = 31 + getTraitSet().hashCode();
-    PairList items = this.getDigestItems();
+    PairList items = this.getDigestItems();
     for (@Nullable Object value : items.rightList()) {
       final int h;
       if (value == null) {
diff --git a/core/src/main/java/org/apache/calcite/rel/externalize/RelDotWriter.java b/core/src/main/java/org/apache/calcite/rel/externalize/RelDotWriter.java
index bf0ef3c4a5cf..39b4ce52842e 100644
--- a/core/src/main/java/org/apache/calcite/rel/externalize/RelDotWriter.java
+++ b/core/src/main/java/org/apache/calcite/rel/externalize/RelDotWriter.java
@@ -105,7 +105,7 @@ protected String getRelNodeLabel(
     sb.setLength(0);
 
     if (detailLevel != SqlExplainLevel.NO_ATTRIBUTES) {
-      for (Pair value : values) {
+      for (Pair value : values) {
         if (value.right instanceof RelNode) {
           continue;
         }
diff --git a/core/src/main/java/org/apache/calcite/rel/externalize/RelJson.java b/core/src/main/java/org/apache/calcite/rel/externalize/RelJson.java
index 1e6aae9c43bb..dc341161d2de 100644
--- a/core/src/main/java/org/apache/calcite/rel/externalize/RelJson.java
+++ b/core/src/main/java/org/apache/calcite/rel/externalize/RelJson.java
@@ -300,7 +300,7 @@ private static RexNode translateInput(RelJson relJson, int input,
   }
 
   public Object toJson(SqlParserPos pos) {
-    final Map map = jsonBuilder().map();
+    final Map map = jsonBuilder().map();
     map.put("line", pos.getLineNum());
     map.put("column", pos.getColumnNum());
     map.put("end_line", pos.getEndLineNum());
@@ -311,7 +311,7 @@ public Object toJson(SqlParserPos pos) {
   public Object toJson(RelCollationImpl node) {
     final List list = new ArrayList<>();
     for (RelFieldCollation fieldCollation : node.getFieldCollations()) {
-      final Map map = jsonBuilder().map();
+      final Map map = jsonBuilder().map();
       map.put("field", fieldCollation.getFieldIndex());
       map.put("direction", fieldCollation.getDirection().name());
       map.put("nulls", fieldCollation.nullDirection.name());
@@ -354,7 +354,7 @@ public RelDistribution toDistribution(Map map) {
   }
 
   private Object toJson(RelDistribution relDistribution) {
-    final Map map = jsonBuilder().map();
+    final Map map = jsonBuilder().map();
     map.put("type", relDistribution.getType().name());
     if (!relDistribution.getKeys().isEmpty()) {
       map.put("keys", relDistribution.getKeys());
@@ -446,8 +446,8 @@ private RelDataType getRelDataType(RelDataTypeFactory typeFactory, Map map = jsonBuilder().map();
-    final Map aggMap = toJson(node.getAggregation());
+    final Map map = jsonBuilder().map();
+    final Map aggMap = toJson(node.getAggregation());
     if (node.getAggregation().getFunctionType().isUserDefined()) {
       aggMap.put("class", node.getAggregation().getClass().getName());
     }
@@ -478,13 +478,13 @@ public Object toJson(AggregateCall node) {
     } else if (value instanceof CorrelationId) {
       return toJson((CorrelationId) value);
     } else if (value instanceof List || value instanceof Set) {
-      final List<@Nullable Object> list = jsonBuilder().list();
+      final List list = jsonBuilder().list();
       for (Object o : (Collection) value) {
         list.add(toJson(o));
       }
       return list;
     } else if (value instanceof ImmutableBitSet) {
-      final List<@Nullable Object> list = jsonBuilder().list();
+      final List list = jsonBuilder().list();
       for (Integer integer : (ImmutableBitSet) value) {
         list.add(toJson(integer));
       }
@@ -519,7 +519,7 @@ public Object toJson(AggregateCall node) {
   }
 
   public > Object toJson(Sarg node) {
-    final Map map = jsonBuilder().map();
+    final Map map = jsonBuilder().map();
     map.put("rangeSet", toJson(node.rangeSet));
     map.put("nullAs", RelEnumTypes.fromEnum(node.nullAs));
     return map;
@@ -544,9 +544,9 @@ public > List toJson(Range range) {
   }
 
   private Object toJson(RelDataType node) {
-    final Map map = jsonBuilder().map();
+    final Map map = jsonBuilder().map();
     if (node.isStruct()) {
-      final List<@Nullable Object> list = jsonBuilder().list();
+      final List list = jsonBuilder().list();
       for (RelDataTypeField field : node.getFieldList()) {
         list.add(toJson(field));
       }
@@ -587,7 +587,7 @@ private static Object toJson(CorrelationId node) {
   }
 
   public Object toJson(RexNode node) {
-    final Map map;
+    final Map map;
     switch (node.getKind()) {
     case DYNAMIC_PARAM:
       map = jsonBuilder().map();
@@ -640,7 +640,7 @@ public Object toJson(RexNode node) {
     case LAMBDA: {
       RexLambda lambda = (RexLambda) node;
       map = jsonBuilder().map();
-      final List<@Nullable Object> parameters = jsonBuilder().list();
+      final List parameters = jsonBuilder().list();
       for (RexLambdaRef param : lambda.getParameters()) {
         parameters.add(toJson(param));
       }
@@ -657,7 +657,7 @@ public Object toJson(RexNode node) {
           map.put("pos", toJson(call.getParserPosition()));
         }
         map.put("op", toJson(call.getOperator()));
-        final List<@Nullable Object> list = jsonBuilder().list();
+        final List list = jsonBuilder().list();
         for (RexNode operand : call.getOperands()) {
           list.add(toJson(operand));
         }
@@ -693,7 +693,7 @@ public Object toJson(RexNode node) {
   }
 
   private Object toJson(RexWindow window) {
-    final Map map = jsonBuilder().map();
+    final Map map = jsonBuilder().map();
     if (!window.partitionKeys.isEmpty()) {
       map.put("partition", toJson(window.partitionKeys));
     }
@@ -721,7 +721,7 @@ private Object toJson(RexWindow window) {
   }
 
   private Object toJson(RexFieldCollation collation) {
-    final Map map = jsonBuilder().map();
+    final Map map = jsonBuilder().map();
     map.put("expr", toJson(collation.left));
     map.put("direction", collation.getDirection().name());
     map.put("null-direction", collation.getNullDirection().name());
@@ -729,7 +729,7 @@ private Object toJson(RexFieldCollation collation) {
   }
 
   private Object toJson(RexWindowBound windowBound) {
-    final Map map = jsonBuilder().map();
+    final Map map = jsonBuilder().map();
     if (windowBound.isCurrentRow()) {
       map.put("type", "CURRENT_ROW");
     } else if (windowBound.isUnbounded()) {
@@ -765,10 +765,10 @@ public RexNode toRex(RelOptCluster cluster, Object o) {
       return null;
     // Support JSON deserializing of non-default Map classes such as gson LinkedHashMap
     } else if (Map.class.isAssignableFrom(o.getClass())) {
-      final Map map = (Map) o;
+      final Map map = (Map) o;
       final RelDataTypeFactory typeFactory = cluster.getTypeFactory();
       if (map.containsKey("op")) {
-        final Map opMap = get(map, "op");
+        final Map opMap = get(map, "op");
         if (map.containsKey("class")) {
           opMap.put("class", get(map, "class"));
         }
@@ -1199,7 +1199,7 @@ private List toRexList(RelInput relInput, List operands) {
 
   private Map toJson(SqlOperator operator) {
     // User-defined operators are not yet handled.
-    Map map = jsonBuilder().map();
+    Map map = jsonBuilder().map();
     map.put("name", operator.getName());
     map.put("kind", operator.kind.toString());
     map.put("syntax", operator.getSyntax().toString());
diff --git a/core/src/main/java/org/apache/calcite/rel/externalize/RelJsonWriter.java b/core/src/main/java/org/apache/calcite/rel/externalize/RelJsonWriter.java
index 26f1db978973..803c9f92bd5c 100644
--- a/core/src/main/java/org/apache/calcite/rel/externalize/RelJsonWriter.java
+++ b/core/src/main/java/org/apache/calcite/rel/externalize/RelJsonWriter.java
@@ -74,18 +74,18 @@ public RelJsonWriter(JsonBuilder jsonBuilder,
   //~ Methods ------------------------------------------------------------------
 
   protected void explain_(RelNode rel, List> values) {
-    final Map map = jsonBuilder.map();
+    final Map map = jsonBuilder.map();
 
     map.put("id", null); // ensure that id is the first attribute
     map.put("relOp", relJson.classToTypeName(rel.getClass()));
-    for (Pair value : values) {
+    for (Pair value : values) {
       if (value.right instanceof RelNode) {
         continue;
       }
       put(map, value.left, value.right);
     }
     // omit 'inputs: ["3"]' if "3" is the preceding rel
-    final List<@Nullable Object> list = explainInputs(rel.getInputs());
+    final List list = explainInputs(rel.getInputs());
     if (list.size() != 1 || !Objects.equals(list.get(0), previousId)) {
       map.put("inputs", list);
     }
@@ -103,7 +103,7 @@ private void put(Map map, String name, @Nullable Objec
   }
 
   private List<@Nullable Object> explainInputs(List inputs) {
-    final List<@Nullable Object> list = jsonBuilder.list();
+    final List list = jsonBuilder.list();
     for (RelNode input : inputs) {
       String id = relIdMap.get(input);
       if (id == null) {
@@ -129,7 +129,7 @@ private void put(Map map, String name, @Nullable Objec
   }
 
   @Override public RelWriter done(RelNode node) {
-    final List> valuesCopy =
+    final List> valuesCopy =
         ImmutableList.copyOf(values);
     values.clear();
     explain_(node, valuesCopy);
@@ -145,7 +145,7 @@ private void put(Map map, String name, @Nullable Objec
    * explained.
    */
   public String asString() {
-    final Map map = jsonBuilder.map();
+    final Map map = jsonBuilder.map();
     map.put("rels", relList);
     return jsonBuilder.toJsonString(map);
   }
diff --git a/core/src/main/java/org/apache/calcite/rel/externalize/RelWriterImpl.java b/core/src/main/java/org/apache/calcite/rel/externalize/RelWriterImpl.java
index f8cd466292b7..3e420dbea8f7 100644
--- a/core/src/main/java/org/apache/calcite/rel/externalize/RelWriterImpl.java
+++ b/core/src/main/java/org/apache/calcite/rel/externalize/RelWriterImpl.java
@@ -87,7 +87,7 @@ protected void explain_(RelNode rel,
     s.append(rel.getRelTypeName());
     if (detailLevel != SqlExplainLevel.NO_ATTRIBUTES) {
       int j = 0;
-      for (Pair value : values) {
+      for (Pair value : values) {
         if (value.right instanceof RelNode) {
           continue;
         }
@@ -160,7 +160,7 @@ private void explainInputs(List inputs) {
 
   @Override public RelWriter done(RelNode node) {
     assert checkInputsPresentInExplain(node);
-    final List> valuesCopy =
+    final List> valuesCopy =
         ImmutableList.copyOf(values);
     values.clear();
     explain_(node, valuesCopy);
@@ -186,7 +186,7 @@ private boolean checkInputsPresentInExplain(RelNode node) {
    */
   public String simple() {
     final StringBuilder buf = new StringBuilder("(");
-    for (Ord> ord : Ord.zip(values)) {
+    for (Ord> ord : Ord.zip(values)) {
       if (ord.i > 0) {
         buf.append(", ");
       }
diff --git a/core/src/main/java/org/apache/calcite/rel/externalize/RelXmlWriter.java b/core/src/main/java/org/apache/calcite/rel/externalize/RelXmlWriter.java
index fef5fbf0eaa5..59abe61def48 100644
--- a/core/src/main/java/org/apache/calcite/rel/externalize/RelXmlWriter.java
+++ b/core/src/main/java/org/apache/calcite/rel/externalize/RelXmlWriter.java
@@ -99,7 +99,7 @@ private void explainGeneric(
     xmlOutput.endBeginTag("RelNode");
 
     final List inputs = new ArrayList<>();
-    for (Pair pair : values) {
+    for (Pair pair : values) {
       if (pair.right instanceof RelNode) {
         inputs.add((RelNode) pair.right);
         continue;
@@ -144,7 +144,7 @@ private void explainSpecific(
     xmlOutput.beginBeginTag(tagName);
     xmlOutput.attribute("id", rel.getId() + "");
 
-    for (Pair value : values) {
+    for (Pair value : values) {
       if (value.right instanceof RelNode) {
         continue;
       }
diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/MetadataFactoryImpl.java b/core/src/main/java/org/apache/calcite/rel/metadata/MetadataFactoryImpl.java
index e74d71f9e438..782b2a02aa9d 100644
--- a/core/src/main/java/org/apache/calcite/rel/metadata/MetadataFactoryImpl.java
+++ b/core/src/main/java/org/apache/calcite/rel/metadata/MetadataFactoryImpl.java
@@ -59,7 +59,7 @@ public MetadataFactoryImpl(RelMetadataProvider provider) {
     //noinspection RedundantTypeArguments
     return CacheLoader., Class>,
         UnboundMetadata<@Nullable Metadata>>from(key -> {
-          final UnboundMetadata<@Nullable Metadata> function =
+          final UnboundMetadata function =
               provider.apply(key.left, key.right);
           // Return DUMMY, not null, so the cache knows to not ask again.
           return function != null ? function : DUMMY;
diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdColumnOrigins.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdColumnOrigins.java
index 6eda2bf7f742..c07cd5afaf04 100644
--- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdColumnOrigins.java
+++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdColumnOrigins.java
@@ -298,7 +298,7 @@ private RelMdColumnOrigins() {}
   private static @Nullable Set getMultipleColumns(RexNode rexNode, RelNode input,
       final RelMetadataQuery mq) {
     final Set set = new HashSet<>();
-    final RexVisitor<@Nullable Void> visitor =
+    final RexVisitor visitor =
         new RexVisitorImpl<@Nullable Void>(true) {
           @Override public Void visitInputRef(RexInputRef inputRef) {
             Set inputSet =
diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdSize.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdSize.java
index 33a540ba736c..e3b123e8dbee 100644
--- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdSize.java
+++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdSize.java
@@ -91,13 +91,13 @@ protected RelMdSize() {}
    * @see org.apache.calcite.rel.metadata.RelMetadataQuery#getAverageRowSize
    */
   public @Nullable Double averageRowSize(RelNode rel, RelMetadataQuery mq) {
-    final List<@Nullable Double> averageColumnSizes = mq.getAverageColumnSizes(rel);
+    final List averageColumnSizes = mq.getAverageColumnSizes(rel);
     if (averageColumnSizes == null) {
       return null;
     }
     double d = 0d;
     final List fields = rel.getRowType().getFieldList();
-    for (Pair<@Nullable Double, RelDataTypeField> p
+    for (Pair p
         : Pair.zip(averageColumnSizes, fields)) {
       if (p.left == null) {
         Double fieldValueSize = averageFieldValueSize(p.right);
@@ -139,9 +139,9 @@ protected RelMdSize() {}
   }
 
   public @Nullable List<@Nullable Double> averageColumnSizes(Project rel, RelMetadataQuery mq) {
-    final List<@Nullable Double> inputColumnSizes =
+    final List inputColumnSizes =
         mq.getAverageColumnSizesNotNull(rel.getInput());
-    final ImmutableNullableList.Builder<@Nullable Double> sizes = ImmutableNullableList.builder();
+    final ImmutableNullableList.Builder sizes = ImmutableNullableList.builder();
     for (RexNode project : rel.getProjects()) {
       sizes.add(averageRexSize(project, inputColumnSizes));
     }
@@ -149,9 +149,9 @@ protected RelMdSize() {}
   }
 
   public @Nullable List<@Nullable Double> averageColumnSizes(Calc rel, RelMetadataQuery mq) {
-    final List<@Nullable Double> inputColumnSizes =
+    final List inputColumnSizes =
         mq.getAverageColumnSizesNotNull(rel.getInput());
-    final ImmutableNullableList.Builder<@Nullable Double> sizes = ImmutableNullableList.builder();
+    final ImmutableNullableList.Builder sizes = ImmutableNullableList.builder();
     rel.getProgram().split().left.forEach(
         exp -> sizes.add(averageRexSize(exp, inputColumnSizes)));
     return sizes.build();
@@ -159,7 +159,7 @@ protected RelMdSize() {}
 
   public @Nullable List<@Nullable Double> averageColumnSizes(Values rel, RelMetadataQuery mq) {
     final List fields = rel.getRowType().getFieldList();
-    final ImmutableNullableList.Builder<@Nullable Double> list = ImmutableNullableList.builder();
+    final ImmutableNullableList.Builder list = ImmutableNullableList.builder();
     for (int i = 0; i < fields.size(); i++) {
       RelDataTypeField field = fields.get(i);
       if (rel.getTuples().isEmpty()) {
@@ -184,7 +184,7 @@ protected RelMdSize() {}
       return handler.averageColumnSizes(rel, mq);
     }
     final List fields = rel.getRowType().getFieldList();
-    final ImmutableNullableList.Builder<@Nullable Double> list = ImmutableNullableList.builder();
+    final ImmutableNullableList.Builder list = ImmutableNullableList.builder();
     for (RelDataTypeField field : fields) {
       list.add(averageTypeValueSize(field.getType()));
     }
@@ -192,9 +192,9 @@ protected RelMdSize() {}
   }
 
   public List<@Nullable Double> averageColumnSizes(Aggregate rel, RelMetadataQuery mq) {
-    final List<@Nullable Double> inputColumnSizes =
+    final List inputColumnSizes =
         mq.getAverageColumnSizesNotNull(rel.getInput());
-    final ImmutableNullableList.Builder<@Nullable Double> list = ImmutableNullableList.builder();
+    final ImmutableNullableList.Builder list = ImmutableNullableList.builder();
     for (int key : rel.getGroupSet()) {
       list.add(inputColumnSizes.get(key));
     }
@@ -213,8 +213,8 @@ protected RelMdSize() {}
     boolean semiOrAntijoin = !rel.getJoinType().projectsRight();
     final RelNode left = rel.getLeft();
     final RelNode right = rel.getRight();
-    final @Nullable List<@Nullable Double> lefts = mq.getAverageColumnSizes(left);
-    final @Nullable List<@Nullable Double> rights;
+    final @Nullable List lefts = mq.getAverageColumnSizes(left);
+    final @Nullable List rights;
     if (semiOrAntijoin) {
       if (rel.getJoinType() == JoinRelType.LEFT_MARK) {
         RelDataTypeField markColType =
@@ -253,9 +253,9 @@ protected RelMdSize() {}
 
   public @Nullable List<@Nullable Double> averageColumnSizes(Union rel, RelMetadataQuery mq) {
     final int fieldCount = rel.getRowType().getFieldCount();
-    List> inputColumnSizeList = new ArrayList<>();
+    List> inputColumnSizeList = new ArrayList<>();
     for (RelNode input : rel.getInputs()) {
-      final List<@Nullable Double> inputSizes = mq.getAverageColumnSizes(input);
+      final List inputSizes = mq.getAverageColumnSizes(input);
       if (inputSizes != null) {
         inputColumnSizeList.add(inputSizes);
       }
@@ -268,13 +268,13 @@ protected RelMdSize() {}
     default:
       break;
     }
-    final ImmutableNullableList.Builder<@Nullable Double> sizes =
+    final ImmutableNullableList.Builder sizes =
         ImmutableNullableList.builder();
     int nn = 0;
     for (int i = 0; i < fieldCount; i++) {
       double d = 0d;
       int n = 0;
-      for (List<@Nullable Double> inputColumnSizes : inputColumnSizeList) {
+      for (List inputColumnSizes : inputColumnSizeList) {
         Double d2 = inputColumnSizes.get(i);
         if (d2 != null) {
           d += d2;
diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMetadataQuery.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMetadataQuery.java
index 0f2adec9e408..56a02006cf99 100644
--- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMetadataQuery.java
+++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMetadataQuery.java
@@ -733,7 +733,7 @@ public RelDistribution distribution(RelNode rel) {
   /** As {@link #getAverageColumnSizes(org.apache.calcite.rel.RelNode)} but
    * never returns a null list, only ever a list of nulls. */
   public List<@Nullable Double> getAverageColumnSizesNotNull(RelNode rel) {
-    final @Nullable List<@Nullable Double> averageColumnSizes = getAverageColumnSizes(rel);
+    final @Nullable List averageColumnSizes = getAverageColumnSizes(rel);
     return averageColumnSizes == null
         ? Collections.nCopies(rel.getRowType().getFieldCount(), null)
         : averageColumnSizes;
diff --git a/core/src/main/java/org/apache/calcite/rel/rules/AggregateExpandDistinctAggregatesRule.java b/core/src/main/java/org/apache/calcite/rel/rules/AggregateExpandDistinctAggregatesRule.java
index 6fc0915bdb29..7f11e208fd1f 100644
--- a/core/src/main/java/org/apache/calcite/rel/rules/AggregateExpandDistinctAggregatesRule.java
+++ b/core/src/main/java/org/apache/calcite/rel/rules/AggregateExpandDistinctAggregatesRule.java
@@ -236,7 +236,7 @@ public AggregateExpandDistinctAggregatesRule(
     // Initially, the expressions point to the input field.
     final List aggFields =
         aggregate.getRowType().getFieldList();
-    final List<@Nullable RexInputRef> refs = new ArrayList<>();
+    final List refs = new ArrayList<>();
     final List fieldNames = aggregate.getRowType().getFieldNames();
     final ImmutableBitSet groupSet = aggregate.getGroupSet();
     final int groupCount = aggregate.getGroupCount();
diff --git a/core/src/main/java/org/apache/calcite/rel/rules/FilterMultiJoinMergeRule.java b/core/src/main/java/org/apache/calcite/rel/rules/FilterMultiJoinMergeRule.java
index 62d00f854cb6..34154b2a8fe2 100644
--- a/core/src/main/java/org/apache/calcite/rel/rules/FilterMultiJoinMergeRule.java
+++ b/core/src/main/java/org/apache/calcite/rel/rules/FilterMultiJoinMergeRule.java
@@ -70,7 +70,7 @@ public FilterMultiJoinMergeRule(Class filterClass,
 
     // Create a new post-join filter condition
     // Conditions are nullable, so ImmutableList can't be used here
-    List<@Nullable RexNode> filters =
+    List filters =
         Arrays.asList(filter.getCondition(), multiJoin.getPostJoinFilter());
 
     final RexBuilder rexBuilder = multiJoin.getCluster().getRexBuilder();
diff --git a/core/src/main/java/org/apache/calcite/rel/rules/JoinToMultiJoinRule.java b/core/src/main/java/org/apache/calcite/rel/rules/JoinToMultiJoinRule.java
index 826a924207d9..0b08d6e84bf0 100644
--- a/core/src/main/java/org/apache/calcite/rel/rules/JoinToMultiJoinRule.java
+++ b/core/src/main/java/org/apache/calcite/rel/rules/JoinToMultiJoinRule.java
@@ -145,7 +145,7 @@ public JoinToMultiJoinRule(Class joinClass,
 
     // combine the children MultiJoin inputs into an array of inputs
     // for the new MultiJoin
-    final List<@Nullable ImmutableBitSet> projFieldsList = new ArrayList<>();
+    final List projFieldsList = new ArrayList<>();
     final List joinFieldRefCountsList = new ArrayList<>();
     final List newInputs =
         combineInputs(
@@ -158,7 +158,7 @@ public JoinToMultiJoinRule(Class joinClass,
     // combine the outer join information from the left and right
     // inputs, and include the outer join information from the current
     // join, if it's a left/right outer join
-    final List> joinSpecs = new ArrayList<>();
+    final List> joinSpecs = new ArrayList<>();
     combineOuterJoins(
         origJoin,
         newInputs,
@@ -169,7 +169,7 @@ public JoinToMultiJoinRule(Class joinClass,
     // pull up the join filters from the children MultiJoinRels and
     // combine them with the join filter associated with this LogicalJoin to
     // form the join filter for the new MultiJoin
-    List<@Nullable RexNode> newJoinFilters = combineJoinFilters(origJoin, left, right);
+    List newJoinFilters = combineJoinFilters(origJoin, left, right);
 
     // add on the join field reference counts for the join condition
     // associated with this LogicalJoin
@@ -179,7 +179,7 @@ public JoinToMultiJoinRule(Class joinClass,
             origJoin.getCondition(),
             joinFieldRefCountsList);
 
-    List<@Nullable RexNode> newPostJoinFilters =
+    List newPostJoinFilters =
         combinePostJoinFilters(origJoin, left, right);
 
     final RexBuilder rexBuilder = origJoin.getCluster().getRexBuilder();
@@ -351,7 +351,7 @@ private static void copyOuterJoinInfo(
       int adjustmentAmount,
       @Nullable List srcFields,
       @Nullable List destFields) {
-    final List> srcJoinSpecs =
+    final List> srcJoinSpecs =
         Pair.zip(
             multiJoin.getJoinTypes(),
             multiJoin.getOuterJoinConditions());
@@ -363,7 +363,7 @@ private static void copyOuterJoinInfo(
       requireNonNull(destFields, "destFields");
       int[] adjustments = new int[srcFields.size()];
       Arrays.fill(adjustments, adjustmentAmount);
-      for (Pair src : srcJoinSpecs) {
+      for (Pair src : srcJoinSpecs) {
         destJoinSpecs.add(
             Pair.of(
                 src.left,
@@ -397,7 +397,7 @@ private static void copyOuterJoinInfo(
     // AND the join condition if this isn't a left or right outer join;
     // in those cases, the outer join condition is already tracked
     // separately
-    final List<@Nullable RexNode> filters = new ArrayList<>();
+    final List filters = new ArrayList<>();
     if ((joinType != JoinRelType.LEFT) && (joinType != JoinRelType.RIGHT)) {
       filters.add(join.getCondition());
     }
@@ -540,7 +540,7 @@ private static ImmutableMap addOnJoinFieldRefCounts(
       Join joinRel,
       RelNode left,
       RelNode right) {
-    final List<@Nullable RexNode> filters = new ArrayList<>();
+    final List filters = new ArrayList<>();
     if (right instanceof MultiJoin) {
       final MultiJoin multiRight = (MultiJoin) right;
       filters.add(
diff --git a/core/src/main/java/org/apache/calcite/rel/rules/LoptMultiJoin.java b/core/src/main/java/org/apache/calcite/rel/rules/LoptMultiJoin.java
index a526d19f2588..65f676a941dc 100644
--- a/core/src/main/java/org/apache/calcite/rel/rules/LoptMultiJoin.java
+++ b/core/src/main/java/org/apache/calcite/rel/rules/LoptMultiJoin.java
@@ -190,7 +190,7 @@ public LoptMultiJoin(MultiJoin multiJoin) {
         Lists.newArrayList(RelOptUtil.conjunctions(multiJoin.getJoinFilter()));
 
     allJoinFilters = new ArrayList<>(joinFilters);
-    List<@Nullable RexNode> outerJoinFilters = multiJoin.getOuterJoinConditions();
+    List outerJoinFilters = multiJoin.getOuterJoinConditions();
     for (int i = 0; i < nJoinFactors; i++) {
       allJoinFilters.addAll(RelOptUtil.conjunctions(outerJoinFilters.get(i)));
     }
@@ -210,7 +210,7 @@ public LoptMultiJoin(MultiJoin multiJoin) {
     // of outer join and the factors that a null-generating factor is dependent
     // upon.
     joinTypes = ImmutableList.copyOf(multiJoin.getJoinTypes());
-    List<@Nullable RexNode> outerJoinConds = this.multiJoin.getOuterJoinConditions();
+    List outerJoinConds = this.multiJoin.getOuterJoinConditions();
     outerJoinFactors = new ImmutableBitSet[nJoinFactors];
     for (int i = 0; i < nJoinFactors; i++) {
       RexNode outerJoinCond = outerJoinConds.get(i);
diff --git a/core/src/main/java/org/apache/calcite/rel/rules/MultiJoin.java b/core/src/main/java/org/apache/calcite/rel/rules/MultiJoin.java
index 1a74c810bcd2..0f4815f3fa2c 100644
--- a/core/src/main/java/org/apache/calcite/rel/rules/MultiJoin.java
+++ b/core/src/main/java/org/apache/calcite/rel/rules/MultiJoin.java
@@ -189,7 +189,7 @@ private Map cloneJoinFieldRefCountsMap() {
 
   @Override public RelNode accept(RexShuttle shuttle) {
     RexNode joinFilter = shuttle.apply(this.joinFilter);
-    List<@Nullable RexNode> outerJoinConditions = shuttle.apply(this.outerJoinConditions);
+    List outerJoinConditions = shuttle.apply(this.outerJoinConditions);
     RexNode postJoinFilter = shuttle.apply(this.postJoinFilter);
 
     if (joinFilter == this.joinFilter
diff --git a/core/src/main/java/org/apache/calcite/rel/rules/ProjectTableScanRule.java b/core/src/main/java/org/apache/calcite/rel/rules/ProjectTableScanRule.java
index 0f7d5063de0d..40c28129d7b9 100644
--- a/core/src/main/java/org/apache/calcite/rel/rules/ProjectTableScanRule.java
+++ b/core/src/main/java/org/apache/calcite/rel/rules/ProjectTableScanRule.java
@@ -108,7 +108,7 @@ protected void apply(RelOptRuleCall call, Project project, TableScan scan) {
     requireNonNull(table.unwrap(ProjectableFilterableTable.class));
 
     final List selectedColumns = new ArrayList<>();
-    final RexVisitorImpl<@Nullable Void> visitor = new RexVisitorImpl<@Nullable Void>(true) {
+    final RexVisitorImpl visitor = new RexVisitorImpl<@Nullable Void>(true) {
       @Override public Void visitInputRef(RexInputRef inputRef) {
         if (!selectedColumns.contains(inputRef.getIndex())) {
           selectedColumns.add(inputRef.getIndex());
diff --git a/core/src/main/java/org/apache/calcite/rel/rules/SetOpToFilterRule.java b/core/src/main/java/org/apache/calcite/rel/rules/SetOpToFilterRule.java
index 7f4ca4e25165..def80484ea70 100644
--- a/core/src/main/java/org/apache/calcite/rel/rules/SetOpToFilterRule.java
+++ b/core/src/main/java/org/apache/calcite/rel/rules/SetOpToFilterRule.java
@@ -124,7 +124,7 @@ private static void match(RelOptRuleCall call) {
     }
 
     final RelBuilder builder = call.builder();
-    Pair first = extractSourceAndCond(inputs.get(0).stripped());
+    Pair first = extractSourceAndCond(inputs.get(0).stripped());
 
     // Groups conditions by their source relational node and input position.
     // - Key: Pair of (sourceRelNode, inputPosition)
@@ -135,7 +135,7 @@ private static void match(RelOptRuleCall call) {
     // For invalid conditions (non-deterministic expressions or containing subqueries),
     // positions are tagged with their input indices to skip unmergeable inputs
     // during map-based grouping. Other positions are set to null.
-    Map, List<@Nullable RexNode>> sourceToConds =
+    Map, List> sourceToConds =
         new LinkedHashMap<>();
 
     RelNode firstSource = first.left;
@@ -144,7 +144,7 @@ private static void match(RelOptRuleCall call) {
 
     for (int i = 1; i < inputs.size(); i++) {
       final RelNode input = inputs.get(i).stripped();
-      final Pair pair = extractSourceAndCond(input);
+      final Pair pair = extractSourceAndCond(input);
       sourceToConds.computeIfAbsent(Pair.of(pair.left, pair.right != null ? null : i),
           k -> new ArrayList<>()).add(pair.right);
     }
@@ -154,10 +154,10 @@ private static void match(RelOptRuleCall call) {
     }
 
     int branchCount = 0;
-    for (Map.Entry, List<@Nullable RexNode>> entry
+    for (Map.Entry, List> entry
         : sourceToConds.entrySet()) {
-      Pair left = entry.getKey();
-      List<@Nullable RexNode> conds = entry.getValue();
+      Pair left = entry.getKey();
+      List conds = entry.getValue();
       // Single null condition indicates pass-through branch,
       // directly add its corresponding input to the new inputs list.
       if (conds.size() == 1 && conds.get(0) == null) {
diff --git a/core/src/main/java/org/apache/calcite/rel/rules/materialize/MaterializedViewRule.java b/core/src/main/java/org/apache/calcite/rel/rules/materialize/MaterializedViewRule.java
index 28da23359439..e9d58051c9ab 100644
--- a/core/src/main/java/org/apache/calcite/rel/rules/materialize/MaterializedViewRule.java
+++ b/core/src/main/java/org/apache/calcite/rel/rules/materialize/MaterializedViewRule.java
@@ -478,7 +478,7 @@ protected void perform(RelOptRuleCall call, @Nullable Project topProject, RelNod
               // We add (and push) the filter to the view plan before triggering the rewriting.
               // This is useful in case some of the columns can be folded to same value after
               // filter is added.
-              Pair<@Nullable RelNode, RelNode> pushedNodes =
+              Pair pushedNodes =
                   pushFilterToOriginalViewPlan(builder, topViewProject, viewNode, newPred);
               topViewProject = (Project) pushedNodes.left;
               viewNode = pushedNodes.right;
diff --git a/core/src/main/java/org/apache/calcite/rex/RexExecutable.java b/core/src/main/java/org/apache/calcite/rex/RexExecutable.java
index ae2a80521450..ed77840d48b2 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexExecutable.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexExecutable.java
@@ -64,7 +64,7 @@ public RexExecutable(String code, Object reason) {
       cbe.cook(new Scanner(null, new StringReader(code)));
       Class c = cbe.getClazz();
       //noinspection unchecked
-      final Constructor> constructor =
+      final Constructor> constructor =
           c.getConstructor();
       return constructor.newInstance();
     } catch (CompileException | IOException | InstantiationException
@@ -89,8 +89,8 @@ public void reduce(RexBuilder rexBuilder, List constExps,
       } else {
         assert values.length == constExps.size();
         final List successfullyReduced = new ArrayList<>(constExps.size());
-        final List<@Nullable Object> valueList = Arrays.asList(values);
-        for (Pair value : Pair.zip(constExps, valueList)) {
+        final List valueList = Arrays.asList(values);
+        for (Pair value : Pair.zip(constExps, valueList)) {
           successfullyReduced.add(
               rexBuilder.makeLiteral(value.right, value.left.getType(), true));
         }
diff --git a/core/src/main/java/org/apache/calcite/rex/RexProgramBuilder.java b/core/src/main/java/org/apache/calcite/rex/RexProgramBuilder.java
index 98ce0d41cbfd..77370779cc37 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexProgramBuilder.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexProgramBuilder.java
@@ -166,7 +166,7 @@ private static boolean assertionsAreEnabled() {
   }
 
   private void validate(final RexNode expr, final int fieldOrdinal) {
-    final RexVisitor<@Nullable Void> validator =
+    final RexVisitor validator =
         new RexVisitorImpl<@Nullable Void>(true) {
           @Override public Void visitInputRef(RexInputRef input) {
             final int index = input.getIndex();
diff --git a/core/src/main/java/org/apache/calcite/rex/RexSimplify.java b/core/src/main/java/org/apache/calcite/rex/RexSimplify.java
index a0902d1577cf..b1d04abe7564 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexSimplify.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexSimplify.java
@@ -2475,9 +2475,9 @@ private void verify(RexNode before, RexNode simplified, RexUnknownAs unknownAs)
           continue assignment_loop;
         }
       }
-      Pair<@Nullable Comparable, @Nullable RuntimeException> p0 =
+      Pair p0 =
           evaluate(foo0.e, map);
-      Pair<@Nullable Comparable, @Nullable RuntimeException> p1 =
+      Pair p1 =
           evaluate(foo1.e, map);
       if (p0.right != null || p1.right != null) {
         if (p0.right == null || p1.right == null) {
diff --git a/core/src/main/java/org/apache/calcite/rex/RexSqlStandardConvertletTable.java b/core/src/main/java/org/apache/calcite/rex/RexSqlStandardConvertletTable.java
index b6fa31adcde1..89f3bbc75cac 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexSqlStandardConvertletTable.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexSqlStandardConvertletTable.java
@@ -152,7 +152,7 @@ public RexSqlStandardConvertletTable() {
     final SqlOperator op = call.getOperator();
     final List operands = call.getOperands();
 
-    final @Nullable List<@Nullable SqlNode> exprs =
+    final @Nullable List exprs =
         convertExpressionList(converter, operands);
     if (exprs == null) {
       return null;
@@ -166,7 +166,7 @@ public RexSqlStandardConvertletTable() {
   private static @Nullable List<@Nullable SqlNode> convertExpressionList(
       RexToSqlNodeConverter converter,
       List nodes) {
-    final List<@Nullable SqlNode> exprs = new ArrayList<>();
+    final List exprs = new ArrayList<>();
     for (RexNode node : nodes) {
       @Nullable SqlNode converted = converter.convertNode(node);
       if (converted == null) {
@@ -197,7 +197,7 @@ protected void registerEquivOp(SqlOperator op) {
   private void registerTypeAppendOp(final SqlOperator op) {
     registerOp(
         op, (converter, call) -> {
-          @Nullable List<@Nullable SqlNode> operandList =
+          @Nullable List operandList =
               convertExpressionList(converter, call.operands);
           if (operandList == null) {
             return null;
@@ -219,7 +219,7 @@ private void registerCaseOp(final SqlOperator op) {
     registerOp(
         op, (converter, call) -> {
           assert op instanceof SqlCaseOperator;
-          @Nullable List<@Nullable SqlNode> operands =
+          @Nullable List operands =
               convertExpressionList(converter, call.operands);
           if (operands == null) {
             return null;
@@ -248,7 +248,7 @@ private static class EquivConvertlet implements RexSqlConvertlet {
     }
 
     @Override public @Nullable SqlNode convertCall(RexToSqlNodeConverter converter, RexCall call) {
-      @Nullable List<@Nullable SqlNode> operands =
+      @Nullable List operands =
           convertExpressionList(converter, call.operands);
       if (operands == null) {
         return null;
diff --git a/core/src/main/java/org/apache/calcite/rex/RexUtil.java b/core/src/main/java/org/apache/calcite/rex/RexUtil.java
index ed6ba5ef9c5f..87f644c7bbdd 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexUtil.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexUtil.java
@@ -828,7 +828,7 @@ public static boolean isConstant(RexNode node) {
    */
   public static boolean isDeterministic(RexNode e) {
     try {
-      RexVisitor<@Nullable Void> visitor =
+      RexVisitor visitor =
           new RexVisitorImpl<@Nullable Void>(true) {
             @Override public Void visitCall(RexCall call) {
               if (!call.getOperator().isDeterministic()) {
@@ -989,7 +989,7 @@ public static List retainDeterministic(List list) {
       final SqlOperator operator,
       RexNode node) {
     try {
-      RexVisitor<@Nullable Void> visitor =
+      RexVisitor visitor =
           new RexVisitorImpl<@Nullable Void>(true) {
             @Override public Void visitCall(RexCall call) {
               if (call.getOperator().equals(operator)) {
@@ -1014,7 +1014,7 @@ public static List retainDeterministic(List list) {
   public static boolean containsInputRef(
       RexNode node) {
     try {
-      RexVisitor<@Nullable Void> visitor =
+      RexVisitor visitor =
           new RexVisitorImpl<@Nullable Void>(true) {
             @Override public Void visitInputRef(RexInputRef inputRef) {
               throw new Util.FoundOne(inputRef);
@@ -1036,7 +1036,7 @@ public static boolean containsInputRef(
    */
   public static boolean containsFieldAccess(RexNode node) {
     try {
-      RexVisitor<@Nullable Void> visitor =
+      RexVisitor visitor =
           new RexVisitorImpl<@Nullable Void>(true) {
             @Override public Void visitFieldAccess(RexFieldAccess fieldAccess) {
               throw new Util.FoundOne(fieldAccess);
@@ -1286,7 +1286,7 @@ public static boolean containsTableInputRef(List nodes) {
    */
   public static @Nullable RexTableInputRef containsTableInputRef(RexNode node) {
     try {
-      RexVisitor<@Nullable Void> visitor =
+      RexVisitor visitor =
           new RexVisitorImpl<@Nullable Void>(true) {
             @Override public Void visitTableInputRef(RexTableInputRef inputRef) {
               throw new Util.FoundOne(inputRef);
diff --git a/core/src/main/java/org/apache/calcite/runtime/JsonFunctions.java b/core/src/main/java/org/apache/calcite/runtime/JsonFunctions.java
index a3e6b0c273dd..949a258d58e4 100644
--- a/core/src/main/java/org/apache/calcite/runtime/JsonFunctions.java
+++ b/core/src/main/java/org/apache/calcite/runtime/JsonFunctions.java
@@ -429,7 +429,7 @@ private static String jsonQueryEmptyObject(boolean jsonize) {
   public static String jsonObject(SqlJsonConstructorNullClause nullClause,
       @Nullable Object... kvs) {
     assert kvs.length % 2 == 0;
-    Map map = new HashMap<>();
+    Map map = new HashMap<>();
     for (int i = 0; i < kvs.length; i += 2) {
       String k = (String) kvs[i];
       Object v = kvs[i + 1];
@@ -464,7 +464,7 @@ public static void jsonObjectAggAdd(Map map, String k,
 
   public static String jsonArray(SqlJsonConstructorNullClause nullClause,
       @Nullable Object... elements) {
-    List<@Nullable Object> list = new ArrayList<>();
+    List list = new ArrayList<>();
     for (Object element : elements) {
       if (element == null) {
         if (nullClause == SqlJsonConstructorNullClause.NULL_ON_NULL) {
diff --git a/core/src/main/java/org/apache/calcite/runtime/PairList.java b/core/src/main/java/org/apache/calcite/runtime/PairList.java
index 667d7cb76839..59e15cdbf7f2 100644
--- a/core/src/main/java/org/apache/calcite/runtime/PairList.java
+++ b/core/src/main/java/org/apache/calcite/runtime/PairList.java
@@ -49,7 +49,7 @@ public interface PairList PairList of(T t, U u) {
-    final List<@Nullable Object> list = new ArrayList<>();
+    final List list = new ArrayList<>();
     list.add((Object) t);
     list.add((Object) u);
     return new PairLists.MutablePairList<>(list);
@@ -59,7 +59,7 @@ public interface PairList
       PairList copyOf(T t, U u, Object... rest) {
     checkArgument(rest.length % 2 == 0, "even number");
-    final List<@Nullable Object> list = Lists.asList(t, u, rest);
+    final List list = Lists.asList(t, u, rest);
     return new PairLists.MutablePairList<>(new ArrayList<>(list));
   }
 
@@ -81,7 +81,7 @@ PairList backedBy(List<@Nullable Object> list) {
   /** Creates a PairList from a Map. */
   @SuppressWarnings("RedundantCast")
   static  PairList of(Map map) {
-    final List<@Nullable Object> list = new ArrayList<>(map.size() * 2);
+    final List list = new ArrayList<>(map.size() * 2);
     map.forEach((t, u) -> {
       list.add((Object) t);
       list.add((Object) u);
diff --git a/core/src/main/java/org/apache/calcite/runtime/ResultSetEnumerable.java b/core/src/main/java/org/apache/calcite/runtime/ResultSetEnumerable.java
index c9370153a833..ec368b8f477f 100644
--- a/core/src/main/java/org/apache/calcite/runtime/ResultSetEnumerable.java
+++ b/core/src/main/java/org/apache/calcite/runtime/ResultSetEnumerable.java
@@ -99,7 +99,7 @@ public class ResultSetEnumerable extends AbstractEnu
 
   private static @Nullable Object[] convertColumns(ResultSet resultSet, ResultSetMetaData metaData,
       int columnCount) {
-    final List<@Nullable Object> list = new ArrayList<>(columnCount);
+    final List list = new ArrayList<>(columnCount);
     try {
       for (int i = 0; i < columnCount; i++) {
         if (metaData.getColumnType(i + 1) == Types.TIMESTAMP) {
@@ -441,7 +441,7 @@ private ResultSet resultSet() {
 
   private static @Nullable Object[] convertPrimitiveColumns(Primitive[] primitives,
       ResultSet resultSet, int columnCount) {
-    final List<@Nullable Object> list = new ArrayList<>(columnCount);
+    final List list = new ArrayList<>(columnCount);
     try {
       for (int i = 0; i < columnCount; i++) {
         list.add(primitives[i].jdbcGet(resultSet, i + 1));
diff --git a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java
index 4c1e47c9832e..71275e1bb4be 100644
--- a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java
+++ b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java
@@ -255,11 +255,11 @@ public class SqlFunctions {
    * See CALCITE-6393.
    */
   private static Enumerable<@Nullable Object[]> arrayCartesianProduct(Object[] lists) {
-    final List> enumerators = new ArrayList<>();
+    final List> enumerators = new ArrayList<>();
     for (Object list : lists) {
       enumerators.add(Linq4j.enumerator((List) list));
     }
-    final Enumerator> product = Linq4j.product(enumerators);
+    final Enumerator> product = Linq4j.product(enumerators);
     return new ArrayCartesianProductEnumerable(product);
   }
 
@@ -7503,7 +7503,7 @@ public static Map mapFromArrays(List keysArray, List valuesArray) {
     if (keysArray.size() != valuesArray.size()) {
       throw RESOURCE.illegalArgumentsInMapFromArraysFunc(keysArray.size(), valuesArray.size()).ex();
     }
-    final Map<@Nullable Object, @Nullable Object> map = new LinkedHashMap<>();
+    final Map map = new LinkedHashMap<>();
     for (int i = 0; i < keysArray.size(); i++) {
       map.put(keysArray.get(i), valuesArray.get(i));
     }
@@ -7512,7 +7512,7 @@ public static Map mapFromArrays(List keysArray, List valuesArray) {
 
   /** Support the MAP_FROM_ENTRIES function. */
   public static @Nullable Map mapFromEntries(List entries) {
-    final Map<@Nullable Object, @Nullable Object> map = new LinkedHashMap<>();
+    final Map map = new LinkedHashMap<>();
     for (Object entry : entries) {
       if (entry == null) {
         return null;
@@ -7527,7 +7527,7 @@ public static Map mapFromArrays(List keysArray, List valuesArray) {
    * 
      odd-indexed elements are keys and even-indexed elements are values.
    */
   public static Map map(Object... args) {
-    final Map<@Nullable Object, @Nullable Object> map = new LinkedHashMap<>();
+    final Map map = new LinkedHashMap<>();
     for (int i = 0; i < args.length; i += 2) {
       Object key = args[i];
       Object value = args[i + 1];
@@ -7556,7 +7556,7 @@ public static Map map(Object... args) {
     }
 
     // Build the result rows
-    List<@Nullable Object[]> result = new ArrayList<>(maxRows);
+    List result = new ArrayList<>(maxRows);
     for (int rowIdx = 0; rowIdx < maxRows; rowIdx++) {
       @Nullable Object[] row = new Object[queryLists.length];
       for (int queryIdx = 0; queryIdx < queryLists.length; queryIdx++) {
@@ -7575,7 +7575,7 @@ public static Map map(Object... args) {
 
   /** Support the STR_TO_MAP function. */
   public static Map strToMap(String string, String stringDelimiter, String keyValueDelimiter) {
-    final Map map = new LinkedHashMap<>();
+    final Map map = new LinkedHashMap<>();
     final String[] keyValues = string.split(stringDelimiter, -1);
     for (String s : keyValues) {
       String[] keyValueArray = s.split(keyValueDelimiter, 2);
@@ -7639,7 +7639,7 @@ private static int rfind(String string, String delim, int start) {
 
   /** Support the SLICE function. */
   public static List slice(List list) {
-    List<@Nullable Object> result = new ArrayList<>(list.size());
+    List result = new ArrayList<>(list.size());
     for (Object e : list) {
       result.add(structAccess(e, 0, null));
     }
@@ -7834,7 +7834,7 @@ public static String arrayToString(List list, String delimiter, @Nullable String
     } else {
       parts = string.split(delimiter);
     }
-    List<@Nullable String> result = new ArrayList<>(parts.length);
+    List result = new ArrayList<>(parts.length);
     for (String part : parts) {
       if (nullString != null && nullString.equals(part)) {
         result.add(null);
diff --git a/core/src/main/java/org/apache/calcite/runtime/XmlFunctions.java b/core/src/main/java/org/apache/calcite/runtime/XmlFunctions.java
index c3682b58245c..e0f993e91f01 100644
--- a/core/src/main/java/org/apache/calcite/runtime/XmlFunctions.java
+++ b/core/src/main/java/org/apache/calcite/runtime/XmlFunctions.java
@@ -124,7 +124,7 @@ private XmlFunctions() {
       try {
         NodeList nodes = (NodeList) xpathExpression
             .evaluate(documentNode, XPathConstants.NODESET);
-        List<@Nullable String> result = new ArrayList<>();
+        List result = new ArrayList<>();
         for (int i = 0; i < nodes.getLength(); i++) {
           Node item = castNonNull(nodes.item(i));
           Node firstChild =
diff --git a/core/src/main/java/org/apache/calcite/runtime/variant/VariantNonNull.java b/core/src/main/java/org/apache/calcite/runtime/variant/VariantNonNull.java
index d022cad04563..896d6ea41f91 100644
--- a/core/src/main/java/org/apache/calcite/runtime/variant/VariantNonNull.java
+++ b/core/src/main/java/org/apache/calcite/runtime/variant/VariantNonNull.java
@@ -497,7 +497,7 @@ public class VariantNonNull extends VariantSqlValue {
           RuntimeTypeInformation elementType = type.asGeneric().getTypeArgument(0);
           assert value instanceof List;
           List list = (List) value;
-          List<@Nullable Object> result = new ArrayList<>(list.size());
+          List result = new ArrayList<>(list.size());
           for (VariantSqlValue o : list) {
             @Nullable Object converted = o.cast(elementType);
             result.add(converted);
@@ -512,7 +512,7 @@ public class VariantNonNull extends VariantSqlValue {
           // Convert map to map: cast keys and values recursively
           RuntimeTypeInformation keyType = type.asGeneric().getTypeArgument(0);
           RuntimeTypeInformation valueType = type.asGeneric().getTypeArgument(0);
-          LinkedHashMap<@Nullable Object, @Nullable Object> result =
+          LinkedHashMap result =
               new LinkedHashMap<>(map.size());
           for (Map.Entry e : map.entrySet()) {
             @Nullable Object key = e.getKey().cast(keyType);
diff --git a/core/src/main/java/org/apache/calcite/sql/SqlCallBinding.java b/core/src/main/java/org/apache/calcite/sql/SqlCallBinding.java
index 1346ba0e46bb..3bdc99727fc6 100644
--- a/core/src/main/java/org/apache/calcite/sql/SqlCallBinding.java
+++ b/core/src/main/java/org/apache/calcite/sql/SqlCallBinding.java
@@ -282,7 +282,7 @@ public SqlCall permutedCall() {
     final SqlLiteral literal;
     switch (node.getKind()) {
     case ARRAY_VALUE_CONSTRUCTOR:
-      final List<@Nullable Object> list = new ArrayList<>();
+      final List list = new ArrayList<>();
       for (SqlNode o : ((SqlCall) node).getOperandList()) {
         list.add(valueAs(o, Object.class));
       }
diff --git a/core/src/main/java/org/apache/calcite/sql/SqlNodeList.java b/core/src/main/java/org/apache/calcite/sql/SqlNodeList.java
index bacd3b08ff55..f4f0d4f002b7 100644
--- a/core/src/main/java/org/apache/calcite/sql/SqlNodeList.java
+++ b/core/src/main/java/org/apache/calcite/sql/SqlNodeList.java
@@ -300,20 +300,20 @@ public static boolean isEmptyList(final SqlNode node) {
   }
 
   public static SqlNodeList of(SqlNode node1) {
-    final List<@Nullable SqlNode> list = new ArrayList<>(1);
+    final List list = new ArrayList<>(1);
     list.add(node1);
     return new SqlNodeList(SqlParserPos.ZERO, list);
   }
 
   public static SqlNodeList of(SqlNode node1, SqlNode node2) {
-    final List<@Nullable SqlNode> list = new ArrayList<>(2);
+    final List list = new ArrayList<>(2);
     list.add(node1);
     list.add(node2);
     return new SqlNodeList(SqlParserPos.ZERO, list);
   }
 
   public static SqlNodeList of(SqlNode node1, SqlNode node2, @Nullable SqlNode... nodes) {
-    final List<@Nullable SqlNode> list = new ArrayList<>(nodes.length + 2);
+    final List list = new ArrayList<>(nodes.length + 2);
     list.add(node1);
     list.add(node2);
     Collections.addAll(list, nodes);
diff --git a/core/src/main/java/org/apache/calcite/sql/SqlPivot.java b/core/src/main/java/org/apache/calcite/sql/SqlPivot.java
index d22de8549de3..d0b38c85e63e 100644
--- a/core/src/main/java/org/apache/calcite/sql/SqlPivot.java
+++ b/core/src/main/java/org/apache/calcite/sql/SqlPivot.java
@@ -175,7 +175,7 @@ static SqlNodeList toNodes(SqlNode node) {
    * that are not used will become "GROUP BY" columns. */
   public Set usedColumnNames() {
     final Set columnNames = new HashSet<>();
-    final SqlVisitor<@Nullable Void> nameCollector = new SqlBasicVisitor<@Nullable Void>() {
+    final SqlVisitor nameCollector = new SqlBasicVisitor<@Nullable Void>() {
       @Override public @Nullable Void visit(SqlIdentifier id) {
         columnNames.add(Util.last(id.names));
         return super.visit(id);
diff --git a/core/src/main/java/org/apache/calcite/sql/SqlSetOption.java b/core/src/main/java/org/apache/calcite/sql/SqlSetOption.java
index 9af4a9fb897c..edf215363ea6 100644
--- a/core/src/main/java/org/apache/calcite/sql/SqlSetOption.java
+++ b/core/src/main/java/org/apache/calcite/sql/SqlSetOption.java
@@ -137,7 +137,7 @@ public SqlSetOption(SqlParserPos pos, @Nullable String scope, SqlIdentifier name
 
   @SuppressWarnings("NullAway")
   @Override public List getOperandList() {
-    final List<@Nullable SqlNode> operandList = new ArrayList<>();
+    final List operandList = new ArrayList<>();
     if (scope == null) {
       operandList.add(null);
     } else {
diff --git a/core/src/main/java/org/apache/calcite/sql/SqlUnpivot.java b/core/src/main/java/org/apache/calcite/sql/SqlUnpivot.java
index bd46e038e62a..6c628acb5e49 100644
--- a/core/src/main/java/org/apache/calcite/sql/SqlUnpivot.java
+++ b/core/src/main/java/org/apache/calcite/sql/SqlUnpivot.java
@@ -144,7 +144,7 @@ public void forEachNameValues(
    * clause. All columns that are not used will be part of the returned row. */
   public Set usedColumnNames() {
     final Set columnNames = new HashSet<>();
-    final SqlVisitor<@Nullable Void> nameCollector = new SqlBasicVisitor<@Nullable Void>() {
+    final SqlVisitor nameCollector = new SqlBasicVisitor<@Nullable Void>() {
       @Override public @Nullable Void visit(SqlIdentifier id) {
         columnNames.add(Util.last(id.names));
         return super.visit(id);
diff --git a/core/src/main/java/org/apache/calcite/sql/SqlUtil.java b/core/src/main/java/org/apache/calcite/sql/SqlUtil.java
index 6dcb1f20713f..8f9bcf2ce706 100644
--- a/core/src/main/java/org/apache/calcite/sql/SqlUtil.java
+++ b/core/src/main/java/org/apache/calcite/sql/SqlUtil.java
@@ -716,9 +716,9 @@ private static Iterator filterRoutinesByParameterTypeAndName(
           }
           final SqlOperandMetadata operandMetadata = (SqlOperandMetadata) operandTypeChecker;
           @SuppressWarnings("NullAway")
-          final List<@Nullable RelDataType> paramTypes =
+          final List paramTypes =
               operandMetadata.paramTypes(typeFactory);
-          final List<@Nullable RelDataType> permutedArgTypes;
+          final List permutedArgTypes;
           if (argNames != null) {
             final List paramNames = operandMetadata.paramNames();
             permutedArgTypes = permuteArgTypes(paramNames, argNames, argTypes);
@@ -731,7 +731,7 @@ private static Iterator filterRoutinesByParameterTypeAndName(
               paramTypes.add(null);
             }
           }
-          for (Pair<@Nullable RelDataType, @Nullable RelDataType> p
+          for (Pair p
               : Pair.zip(paramTypes, permutedArgTypes)) {
             final RelDataType argType = p.right;
             final RelDataType paramType = p.left;
@@ -1334,7 +1334,7 @@ public static boolean containsAgg(SqlNode node) {
   public static boolean containsCall(SqlNode node,
       Predicate callPredicate) {
     try {
-      SqlVisitor<@Nullable Void> visitor =
+      SqlVisitor visitor =
           new SqlBasicVisitor<@Nullable Void>() {
             @Override public @Nullable Void visit(SqlCall call) {
               if (callPredicate.test(call)) {
diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlInternalOperators.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlInternalOperators.java
index e86830b156fd..1d0c54ec9234 100644
--- a/core/src/main/java/org/apache/calcite/sql/fun/SqlInternalOperators.java
+++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlInternalOperators.java
@@ -58,7 +58,7 @@ private SqlInternalOperators() {
         @Override public void unparse(SqlWriter writer, SqlCall call,
             int leftPrec, int rightPrec) {
           @SuppressWarnings("NullAway")
-          List<@Nullable SqlNode> operandList = call.getOperandList();
+          List operandList = call.getOperandList();
           writer.list(SqlWriter.FrameTypeEnum.PARENTHESES, SqlWriter.COMMA,
               SqlNodeList.of(call.getParserPosition(), operandList));
         }
diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlJsonValueFunction.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlJsonValueFunction.java
index cf4e53713f5c..b063993e47b9 100644
--- a/core/src/main/java/org/apache/calcite/sql/fun/SqlJsonValueFunction.java
+++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlJsonValueFunction.java
@@ -88,7 +88,7 @@ private static RelDataType getDefaultType(SqlOperatorBinding opBinding) {
    * Returns new operand list with type specification removed.
    */
   public static List removeTypeSpecOperands(SqlCall call) {
-    List<@Nullable SqlNode> operands = new ArrayList<>(call.getOperandList());
+    List operands = new ArrayList<>(call.getOperandList());
     if (hasExplicitTypeSpec(call.getOperandList())) {
       operands.set(2, null);
       operands.set(3, null);
diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java
index d58dc0ba751c..a799778d3103 100644
--- a/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java
+++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java
@@ -1404,7 +1404,7 @@ private static RelDataType arrayReturnType(SqlOperatorBinding opBinding) {
           SqlFunctionCategory.SYSTEM);
 
   private static RelDataType mapReturnType(SqlOperatorBinding opBinding) {
-    Pair<@Nullable RelDataType, @Nullable RelDataType> type =
+    Pair type =
         getComponentTypes(
             opBinding.getTypeFactory(), opBinding.collectOperandTypes());
     return SqlTypeUtil.createMapType(
diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlMapValueConstructor.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlMapValueConstructor.java
index 52d56ce6f900..64f969d60780 100644
--- a/core/src/main/java/org/apache/calcite/sql/fun/SqlMapValueConstructor.java
+++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlMapValueConstructor.java
@@ -48,7 +48,7 @@ public SqlMapValueConstructor() {
 
   @SuppressWarnings("NullAway")
   @Override public RelDataType inferReturnType(SqlOperatorBinding opBinding) {
-    Pair<@Nullable RelDataType, @Nullable RelDataType> type =
+    Pair type =
         getComponentTypes(
             opBinding.getTypeFactory(), opBinding.collectOperandTypes());
 
@@ -72,7 +72,7 @@ public SqlMapValueConstructor() {
     if (argTypes.size() % 2 > 0) {
       throw callBinding.newValidationError(RESOURCE.mapRequiresEvenArgCount());
     }
-    final Pair<@Nullable RelDataType, @Nullable RelDataType> componentType =
+    final Pair componentType =
         getComponentTypes(
             callBinding.getTypeFactory(), argTypes);
     if (null == componentType.left || null == componentType.right) {
diff --git a/core/src/main/java/org/apache/calcite/sql/parser/SqlParserUtil.java b/core/src/main/java/org/apache/calcite/sql/parser/SqlParserUtil.java
index 486c586ed51d..e340ffbc41f8 100644
--- a/core/src/main/java/org/apache/calcite/sql/parser/SqlParserUtil.java
+++ b/core/src/main/java/org/apache/calcite/sql/parser/SqlParserUtil.java
@@ -978,7 +978,7 @@ private static SqlNode convert(PrecedenceClimbingParser.Token token) {
     case CALL:
       final PrecedenceClimbingParser.Call call =
           (PrecedenceClimbingParser.Call) token;
-      final List<@Nullable SqlNode> list = new ArrayList<>();
+      final List list = new ArrayList<>();
       for (PrecedenceClimbingParser.Token arg : call.args) {
         list.add(convert(arg));
       }
diff --git a/core/src/main/java/org/apache/calcite/sql/type/OperandTypes.java b/core/src/main/java/org/apache/calcite/sql/type/OperandTypes.java
index 7bdeb0142545..f61a4666f2ce 100644
--- a/core/src/main/java/org/apache/calcite/sql/type/OperandTypes.java
+++ b/core/src/main/java/org/apache/calcite/sql/type/OperandTypes.java
@@ -1659,7 +1659,7 @@ private static class MapFunctionOperandTypeChecker
       if (argTypes.size() % 2 != 0) {
         throw callBinding.newValidationError(RESOURCE.mapRequiresEvenArgCount());
       }
-      final Pair<@Nullable RelDataType, @Nullable RelDataType> componentType =
+      final Pair componentType =
           getComponentTypes(
               callBinding.getTypeFactory(), argTypes);
       // check key type & value type
diff --git a/core/src/main/java/org/apache/calcite/sql/util/SqlShuttle.java b/core/src/main/java/org/apache/calcite/sql/util/SqlShuttle.java
index 5e948f6943dc..ca372a9ab034 100644
--- a/core/src/main/java/org/apache/calcite/sql/util/SqlShuttle.java
+++ b/core/src/main/java/org/apache/calcite/sql/util/SqlShuttle.java
@@ -71,7 +71,7 @@ public class SqlShuttle extends SqlBasicVisitor<@Nullable SqlNode> {
 
   @Override public @Nullable SqlNode visit(SqlNodeList nodeList) {
     boolean update = false;
-    final List<@Nullable SqlNode> newList = new ArrayList<>(nodeList.size());
+    final List newList = new ArrayList<>(nodeList.size());
     for (SqlNode operand : nodeList) {
       SqlNode clonedOperand;
       if (operand == null) {
@@ -107,7 +107,7 @@ protected class CallCopyingArgHandler implements ArgHandler<@Nullable SqlNode> {
     public CallCopyingArgHandler(SqlCall call, boolean alwaysCopy) {
       this.call = call;
       this.update = false;
-      final List<@Nullable SqlNode> operands = (List<@Nullable SqlNode>) call.getOperandList();
+      final List operands = (List<@Nullable SqlNode>) call.getOperandList();
       this.clonedOperands = operands.toArray(new SqlNode[0]);
       this.alwaysCopy = alwaysCopy;
     }
diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlUserDefinedTableFunction.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlUserDefinedTableFunction.java
index 1a3880d552be..01bd17f11427 100644
--- a/core/src/main/java/org/apache/calcite/sql/validate/SqlUserDefinedTableFunction.java
+++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlUserDefinedTableFunction.java
@@ -82,7 +82,7 @@ public SqlUserDefinedTableFunction(SqlIdentifier opName, SqlKind kind,
   }
 
   private RelDataType inferRowType(SqlOperatorBinding callBinding) {
-    List<@Nullable Object> arguments =
+    List arguments =
         SqlUserDefinedTableMacro.convertArguments(callBinding, function,
             getNameAsId(), false);
     return getFunction().getRowType(callBinding.getTypeFactory(), arguments);
@@ -97,7 +97,7 @@ private RelDataType inferRowType(SqlOperatorBinding callBinding) {
    * @return element type of the table (e.g. {@code Object[].class})
    */
   public Type getElementType(SqlOperatorBinding callBinding) {
-    List<@Nullable Object> arguments =
+    List arguments =
         SqlUserDefinedTableMacro.convertArguments(callBinding, function,
             getNameAsId(), false);
     return getFunction().getElementType(arguments);
diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlUserDefinedTableMacro.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlUserDefinedTableMacro.java
index 0ff9d1faac52..3f24abe0989c 100644
--- a/core/src/main/java/org/apache/calcite/sql/validate/SqlUserDefinedTableMacro.java
+++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlUserDefinedTableMacro.java
@@ -86,7 +86,7 @@ public SqlUserDefinedTableMacro(SqlIdentifier opName, SqlKind kind,
 
   /** Returns the table in this UDF, or null if there is no table. */
   public TranslatableTable getTable(SqlOperatorBinding callBinding) {
-    List<@Nullable Object> arguments =
+    List arguments =
         convertArguments(callBinding, tableMacro, getNameAsId(), true);
     return tableMacro.apply(arguments);
   }
@@ -104,7 +104,7 @@ public TranslatableTable getTable(SqlOperatorBinding callBinding) {
   static List<@Nullable Object> convertArguments(SqlOperatorBinding callBinding,
       Function function, SqlIdentifier opName, boolean failOnNonLiteral) {
     RelDataTypeFactory typeFactory = callBinding.getTypeFactory();
-    List<@Nullable Object> arguments = new ArrayList<>(callBinding.getOperandCount());
+    List arguments = new ArrayList<>(callBinding.getOperandCount());
     Ord.forEach(function.getParameters(), (parameter, i) -> {
       final RelDataType type = parameter.getType(typeFactory);
       final Object value;
diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java
index fc7238394450..5cd9d81deb64 100644
--- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java
+++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java
@@ -4509,7 +4509,7 @@ protected void validateSelect(
 
     // Make sure that items in FROM clause have distinct aliases.
     final SelectScope fromScope = (SelectScope) getFromScope(select);
-    List<@Nullable String> names = fromScope.getChildNames();
+    List names = fromScope.getChildNames();
     if (!catalogReader.nameMatcher().isCaseSensitive()) {
       //noinspection RedundantTypeArguments
       names = names.stream()
@@ -4715,7 +4715,7 @@ private void checkRollUp(@Nullable SqlNode grandParent, @Nullable SqlNode parent
       } else if (stripDot.getKind() == SqlKind.LAMBDA) {
         // do not need to check lambda
       } else {
-        List children =
+        List children =
             ((SqlCall) stripDot).getOperandList();
         for (SqlNode child : children) {
           checkRollUp(parent, current, child, scope, contextClause);
@@ -7391,7 +7391,7 @@ public void validatePivot(SqlPivot pivot) {
     // an aggregate or as an axis.
 
     // Aggregates, e.g. "PIVOT (sum(x) AS sum_x, count(*) AS c)"
-    final PairList<@Nullable String, RelDataType> aggNames = PairList.of();
+    final PairList aggNames = PairList.of();
     pivot.forEachAgg((alias, call) -> {
       call.validate(this, scope);
       final RelDataType type = deriveType(scope, call);
@@ -7822,7 +7822,7 @@ public SqlNode extendedExpandGroupBy(SqlNode expr,
     if (!sqlQuery.isA(SqlKind.QUERY)) {
       return Collections.nCopies(fieldCount, null);
     }
-    final List<@Nullable List> list = new ArrayList<>();
+    final List> list = new ArrayList<>();
     for (int i = 0; i < fieldCount; i++) {
       list.add(getFieldOrigin(sqlQuery, i));
     }
@@ -8325,11 +8325,11 @@ private SqlNode expandStarInRow(SqlNode node) {
       }
       final SelectScope selectScope = (SelectScope) scope;
       final List expandedOperands = new ArrayList<>();
-      final List<@Nullable String> expandedNames = new ArrayList<>();
+      final List expandedNames = new ArrayList<>();
       boolean expanded = false;
 
       // Retrieve field names stored in the operator (from ROW(v AS name, ...) syntax).
-      final @Nullable List<@Nullable String> origFieldNames =
+      final @Nullable List origFieldNames =
           call.getOperator() instanceof SqlRowOperator
               ? ((SqlRowOperator) call.getOperator()).getFieldNames()
               : null;
@@ -8937,7 +8937,7 @@ private void addOrdinal2ExpandSet(
      */
     private boolean containsIdentifier(SqlNode sqlNode, SqlIdentifier target) {
       try {
-        SqlVisitor<@Nullable Void> visitor =
+        SqlVisitor visitor =
             new SqlBasicVisitor<@Nullable Void>() {
               @Override public @Nullable Void visit(SqlIdentifier identifier) {
                 if (identifier.equalsDeep(target, Litmus.IGNORE)) {
@@ -9030,7 +9030,7 @@ private static class NavigationExpander extends NavigationModifier {
     @Override public @Nullable SqlNode visit(SqlCall call) {
       SqlKind kind = call.getKind();
       List operands = call.getOperandList();
-      List<@Nullable SqlNode> newOperands = new ArrayList<>();
+      List newOperands = new ArrayList<>();
 
       if (call.getFunctionQuantifier() != null
           && call.getFunctionQuantifier().getValue() == SqlSelectKeyword.DISTINCT) {
diff --git a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
index e19b638ddc7b..736afabbfe22 100644
--- a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
+++ b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
@@ -2979,7 +2979,7 @@ protected void convertMatchRecognize(Blackboard bb,
     // convert pattern
     final Set patternVarsSet = new HashSet<>();
     SqlNode pattern = matchRecognize.getPattern();
-    final SqlBasicVisitor<@Nullable RexNode> patternVarVisitor =
+    final SqlBasicVisitor patternVarVisitor =
         new PatternVarVisitor(patternVarsSet);
     final RexNode patternNode = pattern.accept(patternVarVisitor);
     if (patternNode == null) {
@@ -3095,7 +3095,7 @@ protected void convertPivot(Blackboard bb, SqlPivot pivot) {
 
     // 3. Gather columns used as arguments to aggregate functions.
     pivotBb.agg = aggConverter;
-    final List<@Nullable String> aggAliasList = new ArrayList<>();
+    final List aggAliasList = new ArrayList<>();
     assert aggConverter.aggCalls.isEmpty();
     pivot.forEachAgg((alias, call) -> {
       call.accept(aggConverter);
@@ -3967,7 +3967,7 @@ private void createAggImpl(Blackboard bb,
       }
 
       // compute inputs to the aggregator
-      final PairList preExprs;
+      final PairList preExprs;
       if (aggConverter.convertedInputExprs.isEmpty()) {
         // Special case for COUNT(*), where we can end up with no inputs
         // at all.  The rest of the system doesn't like 0-tuples, so we
@@ -4592,10 +4592,10 @@ protected RelNode convertColumnList(final SqlInsert call, RelNode source) {
     final RelOptTable targetTable = getTargetTable(call);
     final RelDataType targetRowType = RelOptTableImpl.realRowType(targetTable);
     final List targetFields = targetRowType.getFieldList();
-    final List<@Nullable RexNode> sourceExps =
+    final List sourceExps =
         new ArrayList<>(
             Collections.nCopies(targetFields.size(), null));
-    final List<@Nullable String> fieldNames =
+    final List fieldNames =
         new ArrayList<>(
             Collections.nCopies(targetFields.size(), null));
 
@@ -4948,7 +4948,7 @@ private RexNode convertIdentifier(
     }
 
     final SqlQualified qualified = bb.scope.fullyQualify(identifier);
-    final Pair> e0 =
+    final Pair> e0 =
         bb.lookupExp(qualified);
     RexNode e = e0.left;
     for (String name : qualified.suffix()) {
diff --git a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java
index 517d8684f808..22717b22df44 100644
--- a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java
+++ b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java
@@ -2128,7 +2128,7 @@ private RelBuilder project_(
       return this;
     }
 
-    final List<@Nullable String> fieldNameList = Lists.newArrayList(fieldNames);
+    final List fieldNameList = Lists.newArrayList(fieldNames);
     while (fieldNameList.size() < nodeList.size()) {
       fieldNameList.add(null);
     }
@@ -2339,7 +2339,7 @@ public RelBuilder projectNamed(Iterable nodes,
     @SuppressWarnings({"unchecked", "rawtypes"})
     final List nodeList =
         nodes instanceof List ? (List) nodes : ImmutableList.copyOf(nodes);
-    final List<@Nullable String> fieldNameList =
+    final List fieldNameList =
         fieldNames == null ? null
           : fieldNames instanceof List ? (List<@Nullable String>) fieldNames
           : ImmutableNullableList.copyOf(fieldNames);
@@ -2696,7 +2696,7 @@ private RelBuilder pruneAggregateInputFieldsAndDeduplicateAggCalls(
     // There are duplicate aggregate calls. Rebuild the list to eliminate
     // duplicates, then add a Project.
     final Set callSet = new HashSet<>();
-    final PairList projects = PairList.of();
+    final PairList projects = PairList.of();
     Util.range(groupSetAfterPruning.cardinality())
         .forEach(i -> projects.add(i, null));
     final List distinctAggregateCalls = new ArrayList<>();
@@ -3581,7 +3581,7 @@ public RelBuilder values(@Nullable String[] fieldNames, @Nullable Object... valu
           "Value count must be a positive multiple of field count");
     }
     final int rowCount = values.length / fieldNames.length;
-    for (Ord<@Nullable String> fieldName : Ord.zip(fieldNames)) {
+    for (Ord fieldName : Ord.zip(fieldNames)) {
       if (allNull(values, fieldName.i, fieldNames.length)) {
         throw new IllegalArgumentException("All values of field '" + fieldName.e
             + "' (field index " + fieldName.i + ")"
diff --git a/core/src/main/java/org/apache/calcite/util/JsonBuilder.java b/core/src/main/java/org/apache/calcite/util/JsonBuilder.java
index f814c3a49cb8..d13603509703 100644
--- a/core/src/main/java/org/apache/calcite/util/JsonBuilder.java
+++ b/core/src/main/java/org/apache/calcite/util/JsonBuilder.java
@@ -162,7 +162,7 @@ private void appendMap(
     buf.append("{");
     newline(buf, indent + 1);
     int n = 0;
-    for (Map.Entry entry : map.entrySet()) {
+    for (Map.Entry entry : map.entrySet()) {
       if (n++ > 0) {
         buf.append(",");
         newline(buf, indent + 1);
diff --git a/core/src/main/java/org/apache/calcite/util/ReflectUtil.java b/core/src/main/java/org/apache/calcite/util/ReflectUtil.java
index 7bb3476137dc..a8369828faf0 100644
--- a/core/src/main/java/org/apache/calcite/util/ReflectUtil.java
+++ b/core/src/main/java/org/apache/calcite/util/ReflectUtil.java
@@ -325,7 +325,7 @@ private static boolean invokeVisitorInternal(
     // the original visiteeClass has a diamond-shaped interface inheritance
     // graph. (This is common, for example, in JMI.) The idea is to avoid
     // iterating over a single interface's method more than once in a call.
-    Map, @Nullable Method> cache = new HashMap<>();
+    Map, Method> cache = new HashMap<>();
 
     return lookupVisitMethod(
         visitorClass,
diff --git a/core/src/main/java/org/apache/calcite/util/XmlOutput.java b/core/src/main/java/org/apache/calcite/util/XmlOutput.java
index f7d08ca4f6b8..a6046a8a6f4f 100644
--- a/core/src/main/java/org/apache/calcite/util/XmlOutput.java
+++ b/core/src/main/java/org/apache/calcite/util/XmlOutput.java
@@ -567,7 +567,7 @@ static class StringEscaper implements Cloneable {
      */
     public void defineEscape(char from, String to) {
       int i = (int) from;
-      List<@Nullable String> translationVector =
+      List translationVector =
           requireNonNull(this.translationVector, "translationVector");
       if (i >= translationVector.size()) {
         // Extend list by adding the requisite number of nulls.
diff --git a/core/src/test/java/org/apache/calcite/plan/RelWriterTest.java b/core/src/test/java/org/apache/calcite/plan/RelWriterTest.java
index 29147b9df6c4..db6225a08bd0 100644
--- a/core/src/test/java/org/apache/calcite/plan/RelWriterTest.java
+++ b/core/src/test/java/org/apache/calcite/plan/RelWriterTest.java
@@ -1549,7 +1549,7 @@ void testUDAF(SqlExplainFormat format) {
   /** Returns the schema of a {@link org.apache.calcite.rel.core.TableScan}
    * in this plan, or null if there are no scans. */
   private static RelOptSchema getSchema(RelNode rel) {
-    final Holder<@Nullable RelOptSchema> schemaHolder = Holder.empty();
+    final Holder schemaHolder = Holder.empty();
     rel.accept(
         new RelShuttleImpl() {
           @Override public RelNode visit(TableScan scan) {
diff --git a/core/src/test/java/org/apache/calcite/rel/logical/ToLogicalConverterTest.java b/core/src/test/java/org/apache/calcite/rel/logical/ToLogicalConverterTest.java
index 7dc5275f840f..80d3b1b55349 100644
--- a/core/src/test/java/org/apache/calcite/rel/logical/ToLogicalConverterTest.java
+++ b/core/src/test/java/org/apache/calcite/rel/logical/ToLogicalConverterTest.java
@@ -345,7 +345,7 @@ private void verify(RelNode rel, String expectedPhysical, String expectedLogical
 
   @Test void testCorrelation() {
     final RelBuilder builder = builder();
-    final Holder<@Nullable RexCorrelVariable> v = Holder.empty();
+    final Holder v = Holder.empty();
     final RelNode rel = builder.scan("EMP")
         .variable(v::set)
         .scan("DEPT")
diff --git a/core/src/test/java/org/apache/calcite/sql2rel/CorrelateProjectExtractorTest.java b/core/src/test/java/org/apache/calcite/sql2rel/CorrelateProjectExtractorTest.java
index 0bee7d0ef1bd..3e1c7f28fed0 100644
--- a/core/src/test/java/org/apache/calcite/sql2rel/CorrelateProjectExtractorTest.java
+++ b/core/src/test/java/org/apache/calcite/sql2rel/CorrelateProjectExtractorTest.java
@@ -53,7 +53,7 @@ public static Frameworks.ConfigBuilder config() {
 
   @Test void testSingleCorrelationCallOverVariableInFilter() {
     final RelBuilder builder = RelBuilder.create(config().build());
-    final Holder<@Nullable RexCorrelVariable> v = Holder.empty();
+    final Holder v = Holder.empty();
     RelNode before = builder.scan("EMP")
         .variable(v::set)
         .scan("DEPT")
@@ -88,7 +88,7 @@ public static Frameworks.ConfigBuilder config() {
    * CorrelateProjectExtractor does not handle nested field accesses cor0.field0.field1. */
   @Test void testNestedCorrelationFieldAccessInFilter() {
     final RelBuilder builder = RelBuilder.create(config().build());
-    final Holder<@Nullable RexCorrelVariable> v = Holder.empty();
+    final Holder v = Holder.empty();
     RelNode before = builder.scan("EMP")
         .project(
             builder.alias(
@@ -125,7 +125,7 @@ public static Frameworks.ConfigBuilder config() {
    * not prevent extracting the enclosing correlated call. */
   @Test void testCorrelationCallWithConstantCallOperandInFilter() {
     final RelBuilder builder = RelBuilder.create(config().build());
-    final Holder<@Nullable RexCorrelVariable> v = Holder.empty();
+    final Holder v = Holder.empty();
     RelNode before = builder.scan("EMP")
         .variable(v::set)
         .scan("DEPT")
@@ -158,7 +158,7 @@ public static Frameworks.ConfigBuilder config() {
 
   @Test void testDoubleCorrelationCallOverVariableInFilters() {
     final RelBuilder builder = RelBuilder.create(config().build());
-    final Holder<@Nullable RexCorrelVariable> v = Holder.empty();
+    final Holder v = Holder.empty();
     RelNode before = builder
         .scan("EMP")
         .variable(v::set)
diff --git a/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java b/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java
index 8c2d424133f3..468ad940a984 100644
--- a/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java
+++ b/core/src/test/java/org/apache/calcite/sql2rel/RelDecorrelatorTest.java
@@ -173,7 +173,7 @@ public static Frameworks.ConfigBuilder config() {
 
   @Test void testGroupKeyNotInFrontWhenDecorrelate() {
     final RelBuilder builder = RelBuilder.create(config().build());
-    final Holder<@Nullable RexCorrelVariable> v = Holder.empty();
+    final Holder v = Holder.empty();
     RelNode before = builder.scan("EMP")
         .variable(v::set)
         .scan("DEPT")
diff --git a/core/src/test/java/org/apache/calcite/sql2rel/RelFieldTrimmerTest.java b/core/src/test/java/org/apache/calcite/sql2rel/RelFieldTrimmerTest.java
index c1a23161e154..6208ecb9891f 100644
--- a/core/src/test/java/org/apache/calcite/sql2rel/RelFieldTrimmerTest.java
+++ b/core/src/test/java/org/apache/calcite/sql2rel/RelFieldTrimmerTest.java
@@ -589,7 +589,7 @@ public static Frameworks.ConfigBuilder config() {
    */
   @Test void testLogicalCorrelateFieldTrimmer() {
     final RelBuilder builder = RelBuilder.create(config().build());
-    final Holder<@Nullable RexCorrelVariable> v = Holder.empty();
+    final Holder v = Holder.empty();
     RelNode root = builder.scan("EMP")
         .projectPlus(builder.call(SqlStdOperatorTable.PLUS, builder.field(0), builder.field(0)))
         .variable(v::set)
@@ -633,7 +633,7 @@ public static Frameworks.ConfigBuilder config() {
    */
   @Test void testLogicalCorrelateFieldTrimmer2() {
     final RelBuilder builder = RelBuilder.create(config().build());
-    final Holder<@Nullable RexCorrelVariable> v = Holder.empty();
+    final Holder v = Holder.empty();
     RelNode root = builder.scan("EMP")
         .projectPlus(builder.call(SqlStdOperatorTable.PLUS, builder.field(0), builder.field(0)))
         .variable(v::set)
@@ -682,7 +682,7 @@ public static Frameworks.ConfigBuilder config() {
    */
   @Test void testTrimCorrelatedSubquery() {
     final RelBuilder builder = RelBuilder.create(config().build());
-    final Holder<@Nullable RexCorrelVariable> v = Holder.empty();
+    final Holder v = Holder.empty();
     builder.scan("EMP")
         .variable(v::set)
         .filter(
@@ -734,7 +734,7 @@ public static Frameworks.ConfigBuilder config() {
    */
   @Test void testTrimCorrelatedSubqueryInFilterCondition() {
     final RelBuilder builder = RelBuilder.create(config().build());
-    final Holder<@Nullable RexCorrelVariable> v = Holder.empty();
+    final Holder v = Holder.empty();
     RelNode original = builder.scan("EMP")
         .variable(v::set)
         .filter(ImmutableList.of(v.get().id),
@@ -775,7 +775,7 @@ public static Frameworks.ConfigBuilder config() {
 
   @Test void testTrimCorrelatedSubqueryInJoinCondition() {
     final RelBuilder builder = RelBuilder.create(config().build());
-    final Holder<@Nullable RexCorrelVariable> v = Holder.empty();
+    final Holder v = Holder.empty();
     final RelNode original =
         builder.scan("EMP")
             .variable(v::set)
diff --git a/core/src/test/java/org/apache/calcite/test/InterpreterTest.java b/core/src/test/java/org/apache/calcite/test/InterpreterTest.java
index e552642afdb8..63082c39900f 100644
--- a/core/src/test/java/org/apache/calcite/test/InterpreterTest.java
+++ b/core/src/test/java/org/apache/calcite/test/InterpreterTest.java
@@ -262,7 +262,7 @@ private static void assertInterpret(RelNode rel, DataContext dataContext,
       final List fieldTypes =
           Util.transform(rel.getRowType().getFieldList(),
               RelDataTypeField::getType);
-      final Function<@Nullable Object[], List<@Nullable Object>> converter =
+      final Function> converter =
           EnumUtils.toExternal(fieldTypes, DateTimeUtils.DEFAULT_ZONE);
       assertRows(interpreter, converter, unordered, rows);
     }
diff --git a/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java b/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java
index c3e1290edec5..d2d79c88ee48 100644
--- a/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java
+++ b/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java
@@ -3279,7 +3279,7 @@ private static RelBuilder assertSize(RelBuilder b,
 
   @Test void testCorrelationFails() {
     final RelBuilder builder = RelBuilder.create(config().build());
-    final Holder<@Nullable RexCorrelVariable> v = Holder.empty();
+    final Holder v = Holder.empty();
     try {
       builder.scan("EMP")
           .variable(v::set)
@@ -3296,7 +3296,7 @@ private static RelBuilder assertSize(RelBuilder b,
 
   @Test void testCorrelationWithCondition() {
     final RelBuilder builder = RelBuilder.create(config().build());
-    final Holder<@Nullable RexCorrelVariable> v = Holder.empty();
+    final Holder v = Holder.empty();
     RelNode root = builder.scan("EMP")
         .variable(v::set)
         .scan("DEPT")
@@ -3321,7 +3321,7 @@ private static RelBuilder assertSize(RelBuilder b,
 
   @Test void testTrivialCorrelation() {
     final RelBuilder builder = RelBuilder.create(config().build());
-    final Holder<@Nullable RexCorrelVariable> v = Holder.empty();
+    final Holder v = Holder.empty();
     RelNode root = builder.scan("EMP")
         .variable(v::set)
         .scan("DEPT")
@@ -3424,7 +3424,7 @@ private static RelBuilder assertSize(RelBuilder b,
     //     FROM dept
     //     WHERE deptno = emp.deptno)
     final Function f = b -> {
-      final Holder<@Nullable RexCorrelVariable> v = Holder.empty();
+      final Holder v = Holder.empty();
       return b.scan("EMP")
           .variable(v::set)
           .filter(ImmutableList.of(v.get().id),
@@ -5249,7 +5249,7 @@ private static RelBuilder assertSize(RelBuilder b,
   /** Tests filter builder with correlation variables. */
   @Test void testFilterWithCorrelationVariables() {
     final RelBuilder builder = RelBuilder.create(config().build());
-    final Holder<@Nullable RexCorrelVariable> v = Holder.empty();
+    final Holder v = Holder.empty();
     RelNode root = builder.scan("EMP")
         .variable(v::set)
         .scan("DEPT")
@@ -5434,7 +5434,7 @@ private void checkExpandTable(RelBuilder builder, Matcher matcher) {
 
   @Test void testCorrelate() {
     final RelBuilder builder = RelBuilder.create(config().build());
-    final Holder<@Nullable RexCorrelVariable> v = Holder.empty();
+    final Holder v = Holder.empty();
     RelNode root = builder.scan("EMP")
         .variable(v::set)
         .scan("DEPT")
@@ -5658,7 +5658,7 @@ private static RelNode buildSimpleCorrelateWithJoin(JoinRelType type) {
   }
 
   private static RelNode buildSimpleCorrelateWithJoin(JoinRelType type, RelBuilder builder) {
-    final Holder<@Nullable RexCorrelVariable> v = Holder.empty();
+    final Holder v = Holder.empty();
     return builder
         .scan("EMP")
         .variable(v::set)
@@ -5676,7 +5676,7 @@ private static RelNode buildCorrelateWithJoin(JoinRelType type) {
 
   private static RelNode buildCorrelateWithJoin(JoinRelType type, RelBuilder builder) {
     final RexBuilder rexBuilder = builder.getRexBuilder();
-    final Holder<@Nullable RexCorrelVariable> v = Holder.empty();
+    final Holder v = Holder.empty();
     return builder
         .scan("EMP")
         .variable(v::set)
@@ -5694,7 +5694,7 @@ private static RelNode buildCorrelateWithJoin(JoinRelType type, RelBuilder build
 
   @Test void testCorrelateWithComplexFields() {
     final RelBuilder builder = RelBuilder.create(config().build());
-    final Holder<@Nullable RexCorrelVariable> v = Holder.empty();
+    final Holder v = Holder.empty();
     RelNode root = builder.scan("EMP")
         .variable(v::set)
         .scan("DEPT")
diff --git a/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java b/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java
index efccc070148d..8acaf2c88571 100644
--- a/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java
+++ b/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java
@@ -2269,7 +2269,7 @@ private void checkColumnUniquenessForJoin(String sql) {
               .distinct()
               .filter(b.equals(b.field("SAL"), b.literal(1)))
               .build();
-          final Holder<@Nullable RexCorrelVariable> v = Holder.empty();
+          final Holder v = Holder.empty();
           final RelNode rel1 = b.scan("EMP")
               .variable(v::set)
               .project(b.field("DEPTNO"), b.field("SAL"))
@@ -3631,7 +3631,7 @@ private void checkAverageRowSize(RelOptCluster cluster, RelOptTable empTable,
   private void checkSize(String query, double expected) {
     final RelNode rel = sql(query).toRel();
     final RelMetadataQuery mq = rel.getCluster().getMetadataQuery();
-    final List<@Nullable Double> averageColumnSizes = mq.getAverageColumnSizes(rel);
+    final List averageColumnSizes = mq.getAverageColumnSizes(rel);
     assertNotNull(averageColumnSizes);
     assertThat(averageColumnSizes, hasSize(1));
     assertThat(averageColumnSizes.get(0), is(expected));
diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java
index 385fe5950c31..0bf0dd8a7ac8 100644
--- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java
+++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java
@@ -9906,7 +9906,7 @@ private void checkSemiJoinRuleOnAntiJoin(RelOptRule rule) {
     //               from emp
     //               where dept.deptno = emp.deptno
     //                 and emp.sal > 100)
-    final Holder<@Nullable RexCorrelVariable> v = Holder.empty();
+    final Holder v = Holder.empty();
     final Function relFn = b -> b
         .scan("DEPT")
         .variable(v::set)
diff --git a/core/src/test/java/org/apache/calcite/util/UtilTest.java b/core/src/test/java/org/apache/calcite/util/UtilTest.java
index 130a1949e0bf..0020f641b5ee 100644
--- a/core/src/test/java/org/apache/calcite/util/UtilTest.java
+++ b/core/src/test/java/org/apache/calcite/util/UtilTest.java
@@ -249,7 +249,7 @@ class UtilTest {
   }
 
   @Test void testJoinNullable() {
-    final List<@Nullable Object> parts = Arrays.asList("a", null, "b");
+    final List parts = Arrays.asList("a", null, "b");
     assertThat(Util.joinNullable(parts, ":"), is("a::b"));
     assertThat(Util.joinNullable(Collections.emptyList(), ","), is(""));
     assertThat(Util.joinNullable(Arrays.asList(null, null), ":"), is(":"));
@@ -2518,7 +2518,7 @@ private void checkListToString(String... strings) {
     assertThat(local1.get(), is("foo"));
     local1.set(null); // null values are allowed
 
-    final TryThreadLocal<@Nullable String> local2 =
+    final TryThreadLocal local2 =
         TryThreadLocal.of(null);
     assertThat(local2.get(), nullValue());
     TryThreadLocal.Memo memo2 = local2.push("a");
@@ -2539,7 +2539,7 @@ private void checkListToString(String... strings) {
     }
     assertThat(local2.get(), is("x"));
 
-    final Supplier<@NonNull String> stringSupplier =
+    final Supplier stringSupplier =
         new Supplier() {
       final Random random = new Random();
 
@@ -2564,7 +2564,7 @@ private void checkListToString(String... strings) {
     }
 
     @SuppressWarnings("DataFlowIssue")
-    final Supplier<@NonNull String> nullSupplier = () -> null;
+    final Supplier nullSupplier = () -> null;
     final TryThreadLocal local4 =
         TryThreadLocal.withInitial(nullSupplier);
     local4.set("abc");
diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidConnectionImpl.java b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidConnectionImpl.java
index 8f3f287cc4b5..0b0ee0051cf9 100644
--- a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidConnectionImpl.java
+++ b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidConnectionImpl.java
@@ -557,7 +557,7 @@ public Enumerable enumerable(final QueryType queryType,
           @Override public void run() {
             try {
               final Page page = new Page();
-              final List fieldTypes =
+              final List fieldTypes =
                   Collections.nCopies(fieldNames.size(), null);
               request(queryType, request, this, fieldNames, fieldTypes, page);
               enumerator.done.set(true);
diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidJsonFilter.java b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidJsonFilter.java
index 7ff013681235..2f4cfaef02b8 100644
--- a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidJsonFilter.java
+++ b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidJsonFilter.java
@@ -99,7 +99,7 @@ abstract class DruidJsonFilter implements DruidJson {
     }
     final boolean isNumeric = refNode.getType().getFamily() == SqlTypeFamily.NUMERIC
         || rexLiteral.getType().getFamily() == SqlTypeFamily.NUMERIC;
-    final Pair<@Nullable String, @Nullable ExtractionFunction> druidColumn =
+    final Pair druidColumn =
         DruidQuery.toDruidColumn(refNode, rowType, druidQuery);
     final @Nullable String columnName = druidColumn.left;
     final @Nullable ExtractionFunction extractionFunction = druidColumn.right;
@@ -171,7 +171,7 @@ abstract class DruidJsonFilter implements DruidJson {
     }
     final boolean isNumeric = refNode.getType().getFamily() == SqlTypeFamily.NUMERIC
         || rexLiteral.getType().getFamily() == SqlTypeFamily.NUMERIC;
-    final Pair<@Nullable String, @Nullable ExtractionFunction> druidColumn =
+    final Pair druidColumn =
         DruidQuery.toDruidColumn(refNode, rowType, druidQuery);
     final @Nullable String columnName = druidColumn.left;
     final @Nullable ExtractionFunction extractionFunction = druidColumn.right;
@@ -253,7 +253,7 @@ abstract class DruidJsonFilter implements DruidJson {
     }
     final RexCall rexCall = (RexCall) rexNode;
     final RexNode refNode = rexCall.getOperands().get(0);
-    Pair<@Nullable String, @Nullable ExtractionFunction> druidColumn = DruidQuery
+    Pair druidColumn = DruidQuery
         .toDruidColumn(refNode, rowType, druidQuery);
     final @Nullable String columnName = druidColumn.left;
     final @Nullable ExtractionFunction extractionFunction = druidColumn.right;
@@ -287,7 +287,7 @@ abstract class DruidJsonFilter implements DruidJson {
         listBuilder.add(value);
       }
     }
-    Pair<@Nullable String, @Nullable ExtractionFunction> druidColumn = DruidQuery
+    Pair druidColumn = DruidQuery
         .toDruidColumn(((RexCall) e).getOperands().get(0),
         rowType, druidQuery);
     final @Nullable String columnName = druidColumn.left;
@@ -332,7 +332,7 @@ abstract class DruidJsonFilter implements DruidJson {
     }
     final boolean isNumeric = lhs.getType().getFamily() == SqlTypeFamily.NUMERIC
         || rhs.getType().getFamily() == SqlTypeFamily.NUMERIC;
-    final Pair<@Nullable String, @Nullable ExtractionFunction> druidColumn = DruidQuery
+    final Pair druidColumn = DruidQuery
         .toDruidColumn(refNode, rowType, query);
     final @Nullable String columnName = druidColumn.left;
     final @Nullable ExtractionFunction extractionFunction = druidColumn.right;
diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidQuery.java b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidQuery.java
index 29c44cdfc971..e512c67b330e 100644
--- a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidQuery.java
+++ b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidQuery.java
@@ -784,7 +784,7 @@ protected CalciteConnectionConfig getConnectionConfig() {
     final ImmutableList.Builder projectedColumnsBuilder = ImmutableList.builder();
     final List projects = projectRel.getProjects();
     for (RexNode project : projects) {
-      Pair<@Nullable String, @Nullable ExtractionFunction> druidColumn =
+      Pair druidColumn =
           toDruidColumn(project, inputRowType, druidQuery);
       boolean needExtractForOperand = project instanceof RexCall
           && ((RexCall) project).getOperands().stream().anyMatch(DruidQuery::needUtcTimeExtract);
@@ -856,7 +856,7 @@ protected CalciteConnectionConfig getConnectionConfig() {
         project = projectNode.getProjects().get(groupKey);
       }
 
-      Pair<@Nullable String, @Nullable ExtractionFunction> druidColumn =
+      Pair druidColumn =
           toDruidColumn(project, inputRowType, druidQuery);
       if (druidColumn.left != null && druidColumn.right == null) {
         // SIMPLE INPUT REF
@@ -1647,7 +1647,7 @@ private static class DruidQueryNode implements Node {
     }
 
     @Override public void run() {
-      final List fieldTypes = new ArrayList<>();
+      final List fieldTypes = new ArrayList<>();
       for (RelDataTypeField field : query.getRowType().getFieldList()) {
         fieldTypes.add(getPrimitive(field));
       }
diff --git a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchJson.java b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchJson.java
index 68c61bc15e78..20f2cf0856f4 100644
--- a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchJson.java
+++ b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchJson.java
@@ -80,12 +80,12 @@ static void visitValueNodes(Aggregations aggregations,
     rows.forEach((k, v) -> {
       if (v.stream().allMatch(val -> val instanceof GroupValue)) {
         v.forEach(tuple -> {
-          Map groupRow = new LinkedHashMap<>(k.keys);
+          Map groupRow = new LinkedHashMap<>(k.keys);
           groupRow.put(tuple.getName(), tuple.value());
           consumer.accept(groupRow);
         });
       } else {
-        Map row = new LinkedHashMap<>(k.keys);
+        Map row = new LinkedHashMap<>(k.keys);
         v.forEach(val -> row.put(val.getName(), val.value()));
         consumer.accept(row);
       }
@@ -184,7 +184,7 @@ private RowKey(List buckets) {
     }
 
     private static Map toMap(Iterable buckets) {
-      final Map map = new LinkedHashMap<>();
+      final Map map = new LinkedHashMap<>();
       buckets.forEach(b -> map.put(b.getName(), b.key()));
       return map;
     }
diff --git a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchTable.java b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchTable.java
index c64bf1011111..54ad89cb9a74 100644
--- a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchTable.java
+++ b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchTable.java
@@ -151,7 +151,7 @@ String scriptedFieldPrefix() {
       query.put("size", fetch);
     }
 
-    final Function1 getter =
+    final Function1 getter =
         ElasticsearchEnumerators.getter(fields, ImmutableMap.copyOf(mappings));
 
     Iterable iter;
@@ -307,7 +307,7 @@ String scriptedFieldPrefix() {
       }
     }
 
-    final Function1 getter =
+    final Function1 getter =
         ElasticsearchEnumerators.getter(fields, ImmutableMap.copyOf(mapping));
 
     ElasticsearchJson.SearchHits hits =
diff --git a/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvProjectTableScanRule.java b/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvProjectTableScanRule.java
index a3fa3932966a..ce91fed60dd3 100644
--- a/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvProjectTableScanRule.java
+++ b/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvProjectTableScanRule.java
@@ -46,7 +46,7 @@ protected CsvProjectTableScanRule(Config config) {
   @Override public void onMatch(RelOptRuleCall call) {
     final LogicalProject project = call.rel(0);
     final CsvTableScan scan = call.rel(1);
-    int @Nullable [] fields = getProjectFields(project.getProjects());
+    int [] fields = getProjectFields(project.getProjects());
     if (fields == null) {
       // Project contains expressions more complex than just field references.
       return;
diff --git a/file/src/main/java/org/apache/calcite/adapter/file/JsonEnumerator.java b/file/src/main/java/org/apache/calcite/adapter/file/JsonEnumerator.java
index 12dbc1f2e38b..eb8cce0f5b7d 100644
--- a/file/src/main/java/org/apache/calcite/adapter/file/JsonEnumerator.java
+++ b/file/src/main/java/org/apache/calcite/adapter/file/JsonEnumerator.java
@@ -44,7 +44,7 @@ public class JsonEnumerator implements Enumerator<@Nullable Object[]> {
   private final Enumerator<@Nullable Object[]> enumerator;
 
   public JsonEnumerator(List list) {
-    List<@Nullable Object[]> objs = new ArrayList<>();
+    List objs = new ArrayList<>();
     for (Object obj : list) {
       if (obj instanceof Collection) {
         //noinspection unchecked
diff --git a/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeFilter.java b/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeFilter.java
index 31e1a74b46db..b1cf8f5ac6b2 100644
--- a/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeFilter.java
+++ b/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeFilter.java
@@ -261,8 +261,8 @@ private List getLeftNodeDisjunctions(RexNode node, List disjun
     private String translateOr(List disjunctions) {
       List predicates = new ArrayList<>();
 
-      List<@Nullable String> leftFieldNameList = new ArrayList<>();
-      List<@Nullable String> inSetLeftFieldNameList = new ArrayList<>();
+      List leftFieldNameList = new ArrayList<>();
+      List inSetLeftFieldNameList = new ArrayList<>();
 
       for (RexNode node : disjunctions) {
         final String leftNodeFieldName = getLeftNodeFieldNameForNode(node);
diff --git a/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbTable.java b/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbTable.java
index 9a2c76450c15..df1964d8a1da 100644
--- a/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbTable.java
+++ b/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbTable.java
@@ -151,7 +151,7 @@ public Set getIndexesNameSet() {
     final RelDataTypeFactory.Builder fieldInfo = typeFactory.builder();
     final RelDataType rowType = getRowType(typeFactory);
 
-    Function1 addField = fieldName -> {
+    Function1 addField = fieldName -> {
       final RelDataTypeField field =
           requireNonNull(rowType.getField(fieldName, true, false));
       RelDataType relDataType = field.getType();
diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java b/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java
index b42286a2d569..a06965020c3e 100644
--- a/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java
+++ b/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java
@@ -903,7 +903,7 @@ public static  Enumerable asofJoin(
     // - emit all items in the index
     Map> leftIndex = new HashMap<>();
     // For each left element the corresponding best right element
-    Map> rightIndex = new HashMap<>();
+    Map> rightIndex = new HashMap<>();
     // Outer elements that have null keys.  Will remain empty if !emitNullsOnRight.
     List outerWithNullKeys = new ArrayList<>();
     try (Enumerator os = outer.enumerator()) {
@@ -917,7 +917,7 @@ public static  Enumerable asofJoin(
           }
         } else {
           List left;
-          List<@Nullable TInner> right;
+          List right;
           if (!leftIndex.containsKey(key)) {
             left = new ArrayList<>();
             right = new ArrayList<>();
@@ -946,7 +946,7 @@ public static  Enumerable asofJoin(
           continue;
         }
         assert !left.isEmpty();
-        List<@Nullable TInner> best = requireNonNull(rightIndex.get(key));
+        List best = requireNonNull(rightIndex.get(key));
         assert left.size() == best.size();
         for (int i = 0; i < left.size(); i++) {
           TSource leftElement = left.get(i);
@@ -1030,7 +1030,7 @@ public static  Enumerable asofJoin(
                   TKey key = current.getKey();
                   List value = current.getValue();
                   left = new Linq4j.IterableEnumerator<>(value);
-                  List<@Nullable TInner> rightList =
+                  List rightList =
                       requireNonNull(rightIndex.get(key));
                   right = new Linq4j.IterableEnumerator<@Nullable TInner>(rightList);
                 } else {
@@ -3090,7 +3090,7 @@ public static  float max(Enumerable source,
    */
   public static  @Nullable BigDecimal min(Enumerable source,
       BigDecimalFunction1 selector) {
-    Function2<@Nullable BigDecimal, BigDecimal, BigDecimal> min = minFunction();
+    Function2 min = minFunction();
     return aggregate(source.select(selector), null, min);
   }
 
@@ -3189,7 +3189,7 @@ public static  float min(Enumerable source,
    */
   public static > @Nullable TResult min(
       Enumerable source, Function1 selector) {
-    Function2<@Nullable TResult, TResult, TResult> min = minFunction();
+    Function2 min = minFunction();
     return aggregate(source.select(selector), null, min);
   }
 
@@ -5223,7 +5223,7 @@ private boolean advance() {
           // accept a null left row; merge join rejects those join types up front, so
           // widening this one is sound
           @SuppressWarnings("unchecked") final
-              Function2<@Nullable TSource, @Nullable TInner, TResult> nullTolerant =
+              Function2 nullTolerant =
               (Function2<@Nullable TSource, @Nullable TInner, TResult>) resultSelector;
           results =
               EnumerableDefaults.nestedLoopJoin(
diff --git a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoFilter.java b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoFilter.java
index 8e5154a6c326..2834dafade0f 100644
--- a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoFilter.java
+++ b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoFilter.java
@@ -96,7 +96,7 @@ static class Translator {
     }
 
     private String translateMatch(RexNode condition) {
-      Map map = builder.map();
+      Map map = builder.map();
       map.put("$match", translateOr(condition));
       return builder.toJsonString(map);
     }
@@ -105,7 +105,7 @@ private String translateMatch(RexNode condition) {
       final RexNode condition2 =
           RexUtil.expandSearch(rexBuilder, null, condition);
 
-      List> list = new ArrayList<>();
+      List> list = new ArrayList<>();
       for (RexNode node : RelOptUtil.disjunctions(condition2)) {
         list.add(translateAnd(node));
       }
@@ -113,7 +113,7 @@ private String translateMatch(RexNode condition) {
       case 1:
         return list.get(0);
       default:
-        Map map = builder.map();
+        Map map = builder.map();
         map.put("$or", list);
         return map;
       }
@@ -126,18 +126,18 @@ private String translateMatch(RexNode condition) {
           HashMultimap.create();
       final Map eqMap =
           new LinkedHashMap<>();
-      final List> orMapList = new ArrayList<>();
+      final List> orMapList = new ArrayList<>();
       for (RexNode node : RelOptUtil.conjunctions(node0)) {
         translateMatch2(node, orMapList, multimap, eqMap);
       }
-      Map map = builder.map();
+      Map map = builder.map();
       for (Map.Entry entry : eqMap.entrySet()) {
         multimap.removeAll(entry.getKey());
         map.put(entry.getKey(), literalValue(entry.getValue()));
       }
       for (Map.Entry>> entry
           : multimap.asMap().entrySet()) {
-        Map map2 = builder.map();
+        Map map2 = builder.map();
         for (Pair s : entry.getValue()) {
           String op = s.left;
           if ("$ne".equals(op))  {
@@ -148,7 +148,7 @@ private String translateMatch(RexNode condition) {
               });
             } else if (map2.containsKey(op)) {
               // if two $ne conditions, translate to $nin op
-              List<@Nullable Object> ninList = builder.list();
+              List ninList = builder.list();
               ninList.add(map2.remove(op));
               ninList.add(literalValue(s.right));
               map2.put("$nin", ninList);
@@ -163,7 +163,7 @@ private String translateMatch(RexNode condition) {
         map.put(entry.getKey(), map2);
       }
       if (!orMapList.isEmpty()) {
-        Map andMap = builder.map();
+        Map andMap = builder.map();
         if (!map.isEmpty()) {
           orMapList.add(map);
         }
@@ -237,7 +237,7 @@ private Void translateMatch2(RexNode node, List> o
     }
 
     private Void translateOrAddToList(RexNode node, List> orMapList) {
-      Map or = translateOr(node);
+      Map or = translateOr(node);
       orMapList.add(or);
       return null;
     }
@@ -390,13 +390,13 @@ private Void translateNotLike(RexCall call, List>
         throw new AssertionError("cannot translate NOT LIKE " + call);
       }
 
-      Map regexMap = builder.map();
-      Map regexOp = builder.map();
+      Map regexMap = builder.map();
+      Map regexOp = builder.map();
       regexOp.put("$regex", finalRegex);
       regexMap.put(name, regexOp);
-      List<@Nullable Object> norList = builder.list();
+      List norList = builder.list();
       norList.add(regexMap);
-      Map norMap = builder.map();
+      Map norMap = builder.map();
       norMap.put("$nor", norList);
       orMapList.add(norMap);
       return null;
diff --git a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoTable.java b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoTable.java
index 35462016cd8a..5b3c6ce6faae 100644
--- a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoTable.java
+++ b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoTable.java
@@ -109,7 +109,7 @@ public class MongoTable extends AbstractQueryableTable
         filterJson == null ? null : BsonDocument.parse(filterJson);
     final Bson project =
         projectJson == null ? null : BsonDocument.parse(projectJson);
-    final Function1 getter =
+    final Function1 getter =
         MongoEnumerator.getter(fields);
     return new FindEnumerable(collection, filter, project, getter);
   }
@@ -157,7 +157,7 @@ private static class FindEnumerable
     for (String operation : operations) {
       list.add(BsonDocument.parse(operation));
     }
-    final Function1 getter =
+    final Function1 getter =
         MongoEnumerator.getter(fields);
     return new AggregateEnumerable(mongoDb, list, operations, getter);
   }
diff --git a/piglet/src/main/java/org/apache/calcite/piglet/PigRelOpInnerVisitor.java b/piglet/src/main/java/org/apache/calcite/piglet/PigRelOpInnerVisitor.java
index d1a0d4fcd4fe..79746967358e 100644
--- a/piglet/src/main/java/org/apache/calcite/piglet/PigRelOpInnerVisitor.java
+++ b/piglet/src/main/java/org/apache/calcite/piglet/PigRelOpInnerVisitor.java
@@ -143,7 +143,7 @@ private void doGenerateWithoutMultisetFlatten(LOGenerate gen, List mult
       List flattenOutputAliases) throws FrontendException {
     final List pigProjections = gen.getOutputPlans();
     final List innerCols = new ArrayList<>(); // For projection expressions
-    final List<@Nullable String> fieldAlias = new ArrayList<>(); // For projection names/alias
+    final List fieldAlias = new ArrayList<>(); // For projection names/alias
 
     if (gen.getOutputPlanSchemas() == null) {
       throw new IllegalArgumentException(
diff --git a/plus/src/main/java/org/apache/calcite/adapter/os/SqlShell.java b/plus/src/main/java/org/apache/calcite/adapter/os/SqlShell.java
index cd451a319939..aaef37af4b74 100644
--- a/plus/src/main/java/org/apache/calcite/adapter/os/SqlShell.java
+++ b/plus/src/main/java/org/apache/calcite/adapter/os/SqlShell.java
@@ -361,7 +361,7 @@ private void value(StringBuilder b, @Nullable String s) {
 
         final ResultSetMetaData m = r.getMetaData();
         final int n = m.getColumnCount();
-        final List<@Nullable String> values = new ArrayList<>();
+        final List values = new ArrayList<>();
         final int[] lengths = new int[n];
         final boolean[] rights = new boolean[n];
         for (int i = 0; i < n; i++) {
diff --git a/plus/src/main/java/org/apache/calcite/adapter/tpcds/TpcdsSchema.java b/plus/src/main/java/org/apache/calcite/adapter/tpcds/TpcdsSchema.java
index acd2426179de..8dac931a40d5 100644
--- a/plus/src/main/java/org/apache/calcite/adapter/tpcds/TpcdsSchema.java
+++ b/plus/src/main/java/org/apache/calcite/adapter/tpcds/TpcdsSchema.java
@@ -187,7 +187,7 @@ private class TpcdsSchemaSelector
 
       @Override public Enumerable<@Nullable Object[]> apply(
           List> inRows) {
-        final List<@Nullable Object[]> rows = new ArrayList<>();
+        final List rows = new ArrayList<>();
         for (List strings : inRows) {
           final @Nullable Object[] values = new Object[columns.length];
           for (int i = 0; i < strings.size(); i++) {
diff --git a/plus/src/test/java/org/apache/calcite/adapter/tpcds/TpcdsTest.java b/plus/src/test/java/org/apache/calcite/adapter/tpcds/TpcdsTest.java
index 54654907811e..b0324bc5d28e 100644
--- a/plus/src/test/java/org/apache/calcite/adapter/tpcds/TpcdsTest.java
+++ b/plus/src/test/java/org/apache/calcite/adapter/tpcds/TpcdsTest.java
@@ -303,7 +303,7 @@ private CalciteAssert.AssertQuery checkQuery(int i) {
   }
 
   public Frameworks.ConfigBuilder config() throws Exception {
-    final Holder<@Nullable SchemaPlus> root = Holder.empty();
+    final Holder root = Holder.empty();
     CalciteAssert.model(TPCDS_MODEL)
         .doWithConnection(connection -> {
           root.set(connection.getRootSchema().subSchemas().get("TPCDS"));
diff --git a/server/src/main/java/org/apache/calcite/server/ServerDdlExecutor.java b/server/src/main/java/org/apache/calcite/server/ServerDdlExecutor.java
index 0235f0e4a699..267a3127cb2a 100644
--- a/server/src/main/java/org/apache/calcite/server/ServerDdlExecutor.java
+++ b/server/src/main/java/org/apache/calcite/server/ServerDdlExecutor.java
@@ -196,7 +196,7 @@ static SqlNode renameColumns(@Nullable SqlNodeList columnList,
   /** Erase the table date that calcite-sever created. */
   static void erase(SqlIdentifier name, CalcitePrepare.Context context) {
     // Directly clearing data is more efficient than executing SQL
-    final Pair<@Nullable CalciteSchema, String> pair =
+    final Pair pair =
         schema(context, true, name);
     final CalciteSchema calciteSchema = requireNonNull(pair.left);
     final String tblName = pair.right;
@@ -257,7 +257,7 @@ static void populate(SqlIdentifier name, SqlNode query,
   /** Executes a {@code CREATE FOREIGN SCHEMA} command. */
   public void execute(SqlCreateForeignSchema create,
       CalcitePrepare.Context context) {
-    final Pair<@Nullable CalciteSchema, String> pair =
+    final Pair pair =
         schema(context, true, create.name);
     requireNonNull(pair.left); // TODO: should not assume parent schema exists
     if (pair.left.plus().subSchemas().get(pair.right) != null) {
@@ -317,7 +317,7 @@ public void execute(SqlCreateFunction create,
    * {@code DROP VIEW} commands. */
   public void execute(SqlDropObject drop,
       CalcitePrepare.Context context) {
-    final Pair<@Nullable CalciteSchema, String> pair =
+    final Pair pair =
         schema(context, false, drop.name);
     final @Nullable CalciteSchema schema =
         pair.left; // null if schema does not exist
@@ -379,7 +379,7 @@ public void execute(SqlDropObject drop,
    */
   public void execute(SqlTruncateTable truncate,
       CalcitePrepare.Context context) {
-    final Pair<@Nullable CalciteSchema, String> pair =
+    final Pair pair =
         schema(context, true, truncate.name);
     if (pair.left == null
         || pair.left.plus().tables().get(pair.right) == null) {
@@ -398,7 +398,7 @@ public void execute(SqlTruncateTable truncate,
   /** Executes a {@code CREATE MATERIALIZED VIEW} command. */
   public void execute(SqlCreateMaterializedView create,
       CalcitePrepare.Context context) {
-    final Pair<@Nullable CalciteSchema, String> pair =
+    final Pair pair =
         schema(context, true, create.name);
     if (pair.left != null
         && pair.left.plus().tables().get(pair.right) != null) {
@@ -433,7 +433,7 @@ public void execute(SqlCreateMaterializedView create,
   /** Executes a {@code CREATE SCHEMA} command. */
   public void execute(SqlCreateSchema create,
       CalcitePrepare.Context context) {
-    final Pair<@Nullable CalciteSchema, String> pair =
+    final Pair pair =
         schema(context, true, create.name);
     requireNonNull(pair.left); // TODO: should not assume parent schema exists
     if (pair.left.plus().subSchemas().get(pair.right) != null) {
@@ -452,7 +452,7 @@ public void execute(SqlCreateSchema create,
   /** Executes a {@code DROP SCHEMA} command. */
   public void execute(SqlDropSchema drop,
       CalcitePrepare.Context context) {
-    final Pair<@Nullable CalciteSchema, String> pair =
+    final Pair pair =
         schema(context, false, drop.name);
     final String name = pair.right;
     final boolean existed = pair.left != null
@@ -466,7 +466,7 @@ public void execute(SqlDropSchema drop,
   /** Executes a {@code CREATE TABLE} command. */
   public void execute(SqlCreateTable create,
       CalcitePrepare.Context context) {
-    final Pair<@Nullable CalciteSchema, String> pair =
+    final Pair pair =
         schema(context, true, create.name);
     requireNonNull(pair.left); // TODO: should not assume parent schema exists
     final JavaTypeFactory typeFactory = context.getTypeFactory();
@@ -590,7 +590,7 @@ public void execute(SqlCreateTable create,
   /** Executes a {@code CREATE TABLE LIKE} command. */
   public void execute(SqlCreateTableLike create,
       CalcitePrepare.Context context) {
-    final Pair<@Nullable CalciteSchema, String> pair =
+    final Pair pair =
         schema(context, true, create.name);
     requireNonNull(pair.left); // TODO: should not assume parent schema exists
     if (pair.left.plus().tables().get(pair.right) != null) {
@@ -605,7 +605,7 @@ public void execute(SqlCreateTableLike create,
       }
     }
 
-    final Pair<@Nullable CalciteSchema, String> sourceTablePair =
+    final Pair sourceTablePair =
         schema(context, true, create.sourceTable);
     final CalciteSchema schema =
         // TODO: should not assume parent schema exists
@@ -648,7 +648,7 @@ public void execute(SqlCreateTableLike create,
   /** Executes a {@code CREATE TYPE} command. */
   public void execute(SqlCreateType create,
       CalcitePrepare.Context context) {
-    final Pair<@Nullable CalciteSchema, String> pair =
+    final Pair pair =
         schema(context, true, create.name);
     requireNonNull(pair.left); // TODO: should not assume parent schema exists
     final SqlValidator validator = validator(context, false);
@@ -674,7 +674,7 @@ public void execute(SqlCreateType create,
   /** Executes a {@code CREATE VIEW} command. */
   public void execute(SqlCreateView create,
       CalcitePrepare.Context context) {
-    final Pair<@Nullable CalciteSchema, String> pair =
+    final Pair pair =
         schema(context, true, create.name);
     requireNonNull(pair.left); // TODO: should not assume parent schema exists
     final SchemaPlus schemaPlus = pair.left.plus();
diff --git a/testkit/src/main/java/org/apache/calcite/test/CalciteAssert.java b/testkit/src/main/java/org/apache/calcite/test/CalciteAssert.java
index 2d591e91e4a2..2f4c51f634d7 100644
--- a/testkit/src/main/java/org/apache/calcite/test/CalciteAssert.java
+++ b/testkit/src/main/java/org/apache/calcite/test/CalciteAssert.java
@@ -1299,10 +1299,10 @@ public final AssertThat withMaterializations(String model, final boolean existin
         final String... materializations) {
       return withMaterializations(model, builder -> {
         assert materializations.length % 2 == 0;
-        final List<@Nullable Object> list = builder.list();
+        final List list = builder.list();
         for (int i = 0; i < materializations.length; i++) {
           String table = materializations[i++];
-          final Map map = builder.map();
+          final Map map = builder.map();
           map.put("table", table);
           if (!existing) {
             map.put("view", table + "v");
diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java
index ca4ba61a9e1a..9cbaf290fe5b 100644
--- a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java
+++ b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java
@@ -16721,13 +16721,13 @@ void testTimestampDiff(boolean coercionEnabled) {
     final SqlOperatorFixture f = fixture();
     QUANTIFY_OPERATORS.forEach(operator -> f.setFor(operator, SqlOperatorFixture.VmName.EXPAND));
 
-    Function2 checkBoolean = (sql, result) -> {
+    Function2 checkBoolean = (sql, result) -> {
       f.checkBoolean(sql.replace("COLLECTION", "ARRAY"), result);
       f.checkBoolean(sql.replace("COLLECTION", "MULTISET"), result);
       return null;
     };
 
-    Function1 checkNull = sql -> {
+    Function1 checkNull = sql -> {
       f.checkNull(sql.replace("COLLECTION", "ARRAY"));
       f.checkNull(sql.replace("COLLECTION", "MULTISET"));
       return null;
diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlValidatorFixture.java b/testkit/src/main/java/org/apache/calcite/test/SqlValidatorFixture.java
index 6222e37346c1..fdc924ed7df1 100644
--- a/testkit/src/main/java/org/apache/calcite/test/SqlValidatorFixture.java
+++ b/testkit/src/main/java/org/apache/calcite/test/SqlValidatorFixture.java
@@ -437,7 +437,7 @@ public SqlValidatorFixture isAggregate(Matcher matcher) {
    */
   public SqlValidatorFixture assertFieldOrigin(Matcher matcher) {
     tester.validateAndThen(factory, toSql(false), (sap, validator, n) -> {
-      final List<@Nullable List> list = validator.getFieldOrigins(n);
+      final List> list = validator.getFieldOrigins(n);
       final StringBuilder buf = new StringBuilder("{");
       int i = 0;
       for (@Nullable List strings : list) {
diff --git a/testkit/src/main/java/org/apache/calcite/test/schemata/orderstream/OrdersStreamTableFactory.java b/testkit/src/main/java/org/apache/calcite/test/schemata/orderstream/OrdersStreamTableFactory.java
index ca9991c37422..3e890229f3a6 100644
--- a/testkit/src/main/java/org/apache/calcite/test/schemata/orderstream/OrdersStreamTableFactory.java
+++ b/testkit/src/main/java/org/apache/calcite/test/schemata/orderstream/OrdersStreamTableFactory.java
@@ -48,7 +48,7 @@ public OrdersStreamTableFactory() {
         {ts(10, 58, 0), 4, "paint", 3},
         {ts(11, 10, 0), 5, "paint", 3}
     };
-    final ImmutableList.Builder<@Nullable Object[]> list = ImmutableList.builder();
+    final ImmutableList.Builder list = ImmutableList.builder();
     for (Object[] row : rows) {
       list.add(row);
     }